def train_model(self, params):
# do we have enough time to train?
# if not, skip
if not self._time_ctrl.enough_time(
params["learner"]["model_type"], self._fit_level
):
logger.info(f"Cannot train {params['name']} because of the time constraint")
return False
# let's create directory to log all training artifacts
model_path = os.path.join(self._results_path, params["name"])
self.create_dir(model_path)
# prepare callbacks
early_stop = EarlyStopping(
{"metric": {"name": self._optimize_metric}, "log_to_dir": model_path}
)
learner_time_constraint = LearnerTimeConstraint(
{
"learner_time_limit": self._time_ctrl.learner_time_limit(
params["learner"]["model_type"],
self._fit_level,
self._validation.get("k_folds", 1.0),
),
"min_steps": params["additional"].get("min_steps"),
}
)
total_time_constraint = TotalTimeConstraint(
{
"total_time_limit": self._total_time_limit
if self._model_time_limit is None
else None,
"total_time_start": self._start_time,
}
)
# create model framework
mf = ModelFramework(
params,
callbacks=[early_stop, learner_time_constraint, total_time_constraint],
)
# start training
logger.info(
f"Train model #{len(self._models)+1} / Model name: {params['name']}"
)
mf.train(model_path)
# save the model
mf.save(model_path)
# and keep info about the model
self.keep_model(mf, model_path)
return True
def train_model(self, params):
model_path = os.path.join(self._results_path, params["name"])
early_stop = EarlyStopping({
"metric": {
"name": self._optimize_metric
},
"log_to_dir": model_path
})
learner_time_constraint = LearnerTimeConstraint({
"learner_time_limit":
self._get_learner_time_limit(
params["learner"]["model_type"]), # self._time_limit,
"min_steps":
params["additional"].get("min_steps"),
})
total_time_constraint = TotalTimeConstraint({
"total_time_limit":
self._total_time_limit if self._model_time_limit is None else None,
"total_time_start":
self._start_time,
})
mf = ModelFramework(
params,
callbacks=[
early_stop, learner_time_constraint, total_time_constraint
],
)
if self._enough_time_to_train(mf.get_type()):
# self.verbose_print(params["name"] + " training start ...")
logger.info(
f"Train model #{len(self._models)+1} / Model name: {params['name']}"
)
try:
os.mkdir(model_path)
except Exception as e:
raise AutoMLException(f"Cannot create directory {model_path}")
mf.train(model_path)
mf.save(model_path)
self._model_paths += [model_path]
self.keep_model(mf)
# save the best one in the case the training will be interrupted
self.select_and_save_best()
else:
logger.info(
f"Cannot train {mf.get_type()} because of time constraint")
def test_dont_stop(self):
params = {
"total_time_limit": 100,
"total_time_start": time.time(),
"expected_learners_cnt": 10,
}
callback = TotalTimeConstraint(params)
for i in range(10):
callback.add_and_set_learner(learner={})
callback.on_learner_train_start(logs=None)
callback.on_learner_train_end(logs=None)
def test_stop_on_first_learner(self):
params = {
"total_time_limit": 100,
"total_time_start": time.time(),
"expected_learners_cnt": 1001,
}
callback = TotalTimeConstraint(params)
callback.add_and_set_learner(learner={})
callback.on_learner_train_start(logs=None)
time.sleep(0.1)
with self.assertRaises(AutoMLException) as context:
callback.on_learner_train_end(logs=None)
self.assertTrue("Stop training after the first fold" in str(context.exception))
def test_stop_on_not_first_learner(self):
params = {
"total_time_limit": 100,
"total_time_start": time.time(),
"expected_learners_cnt": 10,
}
callback = TotalTimeConstraint(params)
callback.add_and_set_learner(learner={})
callback.on_learner_train_start(logs=None)
callback.on_learner_train_end(logs=None)
with self.assertRaises(AutoMLException) as context:
#
# hardcoded change just for tests!
callback.total_time_start = time.time() - 600 - 100 - 1
#
callback.add_and_set_learner(learner={})
callback.on_learner_train_start(logs=None)
callback.on_learner_train_end(logs=None)
self.assertTrue("Force to stop" in str(context.exception))