def test_rs_initialize2():
sp = Searchspace(argument_param=("DISCRETE", [1, 5]))
rs = RandomSearch(5, sp, [])
with pytest.raises(NotImplementedError) as excinfo:
rs.initialize()
assert "Searchspace needs at least one continuous parameter" in str(excinfo.value)
def test_randomsearch_initialize():
sp = Searchspace(argument_param=("DOUBLE", [1, 5]), param2=("integer", [3, 4]))
rs = RandomSearch(5, sp, [])
rs.initialize()
assert len(rs.trial_buffer) == 5
def test_random_search(sc):
sp = Searchspace(argument_param=("DOUBLE", [1, 5]))
rs = RandomSearch()
rs.searchspace = sp
rs.num_trials = 5
exp_result = {"argument_param": "DOUBLE"}
assert sp.names() == exp_result
assert rs.num_trials == 5
assert rs.searchspace == sp
def test_randomsearch_init():
sp = Searchspace(argument_param=("DOUBLE", [1, 5]), param2=("integer", [3, 4]))
rs = RandomSearch(5, sp, [])
assert rs.num_trials == 5
assert rs.searchspace == sp
def __init__(self, experiment_type, **kwargs):
global driver_secret
# COMMON EXPERIMENT SETUP
self._final_store = []
self._trial_store = {}
self.num_executors = kwargs.get("num_executors")
self._message_q = queue.Queue()
self.name = kwargs.get("name")
self.experiment_done = False
self.worker_done = False
self.hb_interval = kwargs.get("hb_interval")
self.description = kwargs.get("description")
self.experiment_type = experiment_type
self.es_interval = kwargs.get("es_interval")
self.es_min = kwargs.get("es_min")
# TYPE-SPECIFIC EXPERIMENT SETUP
if self.experiment_type == "optimization":
# set up an optimization experiment
self.num_trials = kwargs.get("num_trials", 1)
searchspace = kwargs.get("searchspace")
if isinstance(searchspace, Searchspace):
self.searchspace = searchspace
elif searchspace is None:
self.searchspace = Searchspace()
else:
raise Exception(
"The experiment's search space should be an instance of maggy.Searchspace, "
"but it is {0} (of type '{1}').".format(
str(searchspace),
type(searchspace).__name__))
optimizer = kwargs.get("optimizer")
if optimizer is None:
if len(self.searchspace.names()) == 0:
self.optimizer = SingleRun()
else:
raise Exception(
"Searchspace has to be empty or None to use without optimizer"
)
elif isinstance(optimizer, str):
if optimizer.lower() == "randomsearch":
self.optimizer = RandomSearch()
elif optimizer.lower() == "asha":
self.optimizer = Asha()
elif optimizer.lower() == "none":
if len(self.searchspace.names()) == 0:
self.optimizer = SingleRun()
else:
raise Exception(
"Searchspace has to be empty or None to use without Optimizer."
)
else:
raise Exception(
"Unknown Optimizer. Can't initialize experiment driver."
)
elif isinstance(optimizer, AbstractOptimizer):
self.optimizer = optimizer
print("Custom Optimizer initialized.")
else:
raise Exception(
"The experiment's optimizer should either be an string indicating the name "
"of an implemented optimizer (such as 'randomsearch') or an instance of "
"maggy.optimizer.AbstractOptimizer, "
"but it is {0} (of type '{1}').".format(
str(optimizer),
type(optimizer).__name__))
direction = kwargs.get("direction", "max")
if isinstance(direction,
str) and direction.lower() in ["min", "max"]:
self.direction = direction.lower()
else:
raise Exception(
"The experiment's direction should be an string (either 'min' or 'max') "
"but it is {0} (of type '{1}').".format(
str(direction),
type(direction).__name__))
es_policy = kwargs.get("es_policy")
if isinstance(es_policy, str):
if es_policy.lower() == "median":
self.earlystop_check = MedianStoppingRule.earlystop_check
elif es_policy.lower() == "none":
self.earlystop_check = NoStoppingRule.earlystop_check
else:
raise Exception(
"The experiment's early stopping policy should either be a string ('median' or 'none') "
"or a custom policy that is an instance of maggy.earlystop.AbstractEarlyStop, "
"but it is {0} (of type '{1}').".format(
str(es_policy),
type(es_policy).__name__))
elif isinstance(es_policy, AbstractEarlyStop):
self.earlystop_check = es_policy.earlystop_check
print("Custom Early Stopping policy initialized.")
else:
raise Exception(
"The experiment's early stopping policy should either be a string ('median' or 'none') "
"or a custom policy that is an instance of maggy.earlystop.AbstractEarlyStop, "
"but it is {0} (of type '{1}').".format(
str(es_policy),
type(es_policy).__name__))
self.es_interval = kwargs.get("es_interval")
self.es_min = kwargs.get("es_min")
self.result = {
"best_val": "n.a.",
"num_trials": 0,
"early_stopped": 0
}
elif self.experiment_type == "ablation":
# set up an ablation study experiment
self.earlystop_check = NoStoppingRule.earlystop_check
ablation_study = kwargs.get("ablation_study")
if isinstance(ablation_study, AblationStudy):
self.ablation_study = ablation_study
else:
raise Exception(
"The experiment's ablation study configuration should be an instance of "
"maggy.ablation.AblationStudy, "
"but it is {0} (of type '{1}').".format(
str(ablation_study),
type(ablation_study).__name__))
searchspace = kwargs.get("searchspace")
if not searchspace:
self.searchspace = Searchspace()
else:
raise Exception(
"The experiment's search space should be None for ablation experiments, "
"but it is {0} (of type '{1}').".format(
str(searchspace),
type(searchspace).__name__))
ablator = kwargs.get("ablator")
if isinstance(ablator, str):
if ablator.lower() == "loco":
self.ablator = LOCO(ablation_study, self._final_store)
self.num_trials = self.ablator.get_number_of_trials()
if self.num_executors > self.num_trials:
self.num_executors = self.num_trials
else:
raise Exception(
"The experiment's ablation study policy should either be a string ('loco') "
"or a custom policy that is an instance of maggy.ablation.ablation.AbstractAblator, "
"but it is {0} (of type '{1}').".format(
str(ablator),
type(ablator).__name__))
elif isinstance(ablator, AbstractAblator):
self.ablator = ablator
print("Custom Ablator initialized. \n")
else:
raise Exception(
"The experiment's ablation study policy should either be a string ('loco') "
"or a custom policy that is an instance of maggy.ablation.ablation.AbstractAblator, "
"but it is {0} (of type '{1}').".format(
str(ablator),
type(ablator).__name__))
self.result = {
"best_val": "n.a.",
"num_trials": 0,
"early_stopped": "n.a"
}
else:
raise Exception(
"Unknown experiment type. experiment_type should be either 'optimization' or 'ablation', "
"but it is {0}.".format(str(self.experiment_type)))
# FINALIZE EXPERIMENT SETUP
self.server = rpc.Server(self.num_executors)
if not driver_secret:
driver_secret = self._generate_secret(
ExperimentDriver.SECRET_BYTES)
self._secret = driver_secret
self.job_start = datetime.now()
self.executor_logs = ""
self.maggy_log = ""
self.log_lock = threading.RLock()
self.log_file = kwargs.get("log_dir") + "/maggy.log"
self.log_dir = kwargs.get("log_dir")
self.exception = None
# Open File desc for HDFS to log
if not hopshdfs.exists(self.log_file):
hopshdfs.dump("", self.log_file)
self.fd = hopshdfs.open_file(self.log_file, flags="w")