def _execute_trial(self, trial_number: int, repo_trial: Trial,
context: ExecutionContext,
validation_splits: List[Tuple[DataContainer,
DataContainer]]):
for training_data_container, validation_data_container in validation_splits:
p = copy.deepcopy(self.pipeline)
p.update_hyperparams(repo_trial.hyperparams)
repo_trial.set_hyperparams(p.get_hyperparams())
with repo_trial.new_validation_split(p) as repo_trial_split:
trial_split_description = self._get_trial_split_description(
repo_trial=repo_trial,
repo_trial_split=repo_trial_split,
validation_splits=validation_splits,
trial_number=trial_number)
self.print_func(
'fitting trial {}'.format(trial_split_description))
repo_trial_split = self.trainer.fit_trial_split(
trial_split=repo_trial_split,
train_data_container=training_data_container,
validation_data_container=validation_data_container,
context=context)
repo_trial_split.set_success()
self.print_func('success trial {} score: {}'.format(
trial_split_description,
repo_trial_split.get_validation_score()))
return repo_trial_split
def test_failure_trial_to_json():
hp = HyperparameterSamples({'a': 2})
repo = InMemoryHyperparamsRepository()
trial = Trial(save_trial_function=repo.save_trial,
hyperparams=hp,
main_metric_name='mse')
with trial:
trial_split = given_failed_trial_split(trial)
trial_json = trial.to_json()
assert trial_json['status'] == TRIAL_STATUS.FAILED.value
assert trial_json['error'] == str(trial_split.error)
assert trial_json['error_traceback'] == EXPECTED_ERROR_TRACEBACK
assert trial_json['main_metric_name'] == trial.main_metric_name
assert then_failed_validation_split_json_is_valid(
trial_json['validation_splits'][0], trial_split=trial_split)
start_time = datetime.datetime.strptime(trial_json['start_time'],
TRIAL_DATETIME_STR_FORMAT)
end_time = datetime.datetime.strptime(
trial_json['end_time'],
TRIAL_DATETIME_STR_FORMAT) + datetime.timedelta(hours=1)
assert start_time < end_time
def test_success_trial_to_json():
hp = HyperparameterSamples({'a': 2})
repo = InMemoryHyperparamsRepository()
trial = Trial(save_trial_function=repo.save_trial,
hyperparams=hp,
main_metric_name='mse')
with trial:
given_success_trial_validation_split(trial)
trial_json = trial.to_json()
assert trial_json['status'] == TRIAL_STATUS.SUCCESS.value
assert trial_json['error'] is None
assert trial_json['error_traceback'] is None
assert trial_json['main_metric_name'] == trial.main_metric_name
assert then_success_trial_split_json_is_valid(
trial_json['validation_splits'][0])
start_time = datetime.datetime.strptime(trial_json['start_time'],
TRIAL_DATETIME_STR_FORMAT)
end_time = datetime.datetime.strptime(
trial_json['end_time'],
TRIAL_DATETIME_STR_FORMAT) + datetime.timedelta(hours=1)
assert start_time < end_time
def setup(self):
self.hp = HyperparameterSamples({'a': 2})
self.repo = InMemoryHyperparamsRepository()
self.trial = Trial(trial_number=0,
save_trial_function=self.repo.save_trial,
hyperparams=self.hp,
main_metric_name=MAIN_METRIC_NAME)
def test_trial_split_is_new_best_score_should_return_true_with_a_new_best_score_after_multiple_scores(
):
hp = HyperparameterSamples({'a': 2})
repo = InMemoryHyperparamsRepository()
trial = Trial(save_trial_function=repo.save_trial,
hyperparams=hp,
main_metric_name=MAIN_METRIC_NAME)
with trial.new_validation_split(Identity()) as trial_split:
trial_split.add_metric_results_train(name=MAIN_METRIC_NAME,
score=0.5,
higher_score_is_better=False)
trial_split.add_metric_results_validation(name=MAIN_METRIC_NAME,
score=0.5,
higher_score_is_better=False)
trial_split.add_metric_results_train(name=MAIN_METRIC_NAME,
score=0.7,
higher_score_is_better=False)
trial_split.add_metric_results_validation(name=MAIN_METRIC_NAME,
score=0.7,
higher_score_is_better=False)
trial_split.add_metric_results_train(name=MAIN_METRIC_NAME,
score=0.4,
higher_score_is_better=False)
trial_split.add_metric_results_validation(name=MAIN_METRIC_NAME,
score=0.4,
higher_score_is_better=False)
assert trial_split.is_new_best_score()
def execute_trial(self,
pipeline: BaseStep,
trial_number: int,
repo_trial: Trial,
context: ExecutionContext,
validation_splits: List[Tuple[DataContainer,
DataContainer]],
n_trial: int,
delete_pipeline_on_completion: bool = True):
"""
Train pipeline using the validation splitter.
Track training, and validation metrics for each epoch.
:param pipeline: pipeline to train on
:param trial_number: trial number
:param repo_trial: repo trial
:param validation_splits: validation splits
:param context: execution context
:param n_trial: total number of trials that will be executed
:param delete_pipeline_on_completion: bool to delete pipeline on completion or not
:return: executed trial split
"""
for training_data_container, validation_data_container in validation_splits:
p = copy.deepcopy(pipeline)
p.update_hyperparams(repo_trial.hyperparams)
repo_trial.set_hyperparams(p.get_hyperparams())
repo_trial_split: TrialSplit = repo_trial.new_validation_split(
pipeline=p,
delete_pipeline_on_completion=delete_pipeline_on_completion)
with repo_trial_split:
trial_split_description = _get_trial_split_description(
repo_trial=repo_trial,
repo_trial_split_number=repo_trial_split.split_number,
validation_splits=validation_splits,
trial_number=trial_number,
n_trial=n_trial)
context.logger.info(
'fitting trial {}'.format(trial_split_description))
repo_trial_split = self.fit_trial_split(
trial_split=repo_trial_split,
train_data_container=training_data_container,
validation_data_container=validation_data_container,
context=context)
repo_trial_split.set_success()
context.logger.info(
'success trial {}\nbest score: {} at epoch {}'.format(
trial_split_description,
repo_trial_split.get_best_validation_score(),
repo_trial_split.get_n_epochs_to_best_validation_score(
)))
return repo_trial_split
def test_success_trial_get_validation_score():
hp = HyperparameterSamples({'a': 2})
trial = Trial(hyperparams=hp, main_metric_name='mse')
with trial:
given_success_trial_validation_split(trial, best_score=0.3)
validation_score = trial.get_validation_score()
assert validation_score == 0.3
def test_trial_should_create_new_split():
hp = HyperparameterSamples({'a': 2})
trial = Trial(hyperparams=hp, main_metric_name=MAIN_METRIC_NAME)
with trial.new_validation_split(Identity()) as trial_split:
trial_split.set_success()
assert isinstance(trial_split.start_time, datetime.datetime)
assert isinstance(trial_split.end_time, datetime.datetime)
assert trial_split.start_time < trial_split.end_time
assert trial.validation_splits[0] == trial_split
def test_success_trial_multiple_splits_should_average_the_scores():
hp = HyperparameterSamples({'a': 2})
trial = Trial(hyperparams=hp, main_metric_name='mse')
with trial:
given_success_trial_validation_split(trial, best_score=0.3)
given_success_trial_validation_split(trial, best_score=0.1)
validation_score = trial.get_validation_score()
assert validation_score == 0.2
def test_trial_with_failed_split_should_only_average_successful_splits():
hp = HyperparameterSamples({'a': 2})
trial = Trial(hyperparams=hp, main_metric_name='mse')
with trial:
given_success_trial_validation_split(trial, best_score=0.3)
given_success_trial_validation_split(trial, best_score=0.1)
given_failed_trial_split(trial)
validation_score = trial.get_validation_score()
assert validation_score == 0.2
def test_success_trial_get_validation_score():
hp = HyperparameterSamples({'a': 2})
repo = InMemoryHyperparamsRepository()
trial = Trial(save_trial_function=repo.save_trial,
hyperparams=hp,
main_metric_name='mse')
with trial:
given_success_trial_validation_split(trial, best_score=0.3)
validation_score = trial.get_validation_score()
assert validation_score == 0.3
def test_trial_split_is_new_best_score_should_return_true_with_one_score():
hp = HyperparameterSamples({'a': 2})
trial = Trial(hyperparams=hp, main_metric_name=MAIN_METRIC_NAME)
with trial.new_validation_split(Identity()) as trial_split:
trial_split.add_metric_results_train(name=MAIN_METRIC_NAME,
score=0.5,
higher_score_is_better=False)
trial_split.add_metric_results_validation(name=MAIN_METRIC_NAME,
score=0.5,
higher_score_is_better=False)
assert trial_split.is_new_best_score()
def load_all_trials(self, status: 'TRIAL_STATUS' = None) -> 'Trials':
"""
Load all hyperparameter trials with their corresponding score.
Reads all the saved trial json files, sorted by creation date.
:return: (hyperparams, scores)
"""
trials = Trials()
files = glob.glob(os.path.join(self.cache_folder, '*.json'))
# sort by created date:
def getmtimens(filename):
return os.stat(filename).st_mtime_ns
files.sort(key=getmtimens)
for base_path in files:
with open(base_path) as f:
try:
trial_json = json.load(f)
except Exception as err:
print('invalid trial json file'.format(base_path))
print(traceback.format_exc())
continue
if status is None or trial_json['status'] == status.value:
trials.append(Trial.from_json(
update_trial_function=self.save_trial,
trial_json=trial_json
))
return trials
def load_all_trials(self, status: 'TRIAL_STATUS' = None) -> 'Trials':
"""
Load all hyperparameter trials with their corresponding score.
Reads all the saved trial json files, sorted by creation date.
:return: (hyperparams, scores)
"""
trials = Trials()
files = glob.glob(os.path.join(self.cache_folder, '*.json'))
# sort by created date:
def getmtimens(filename):
return os.stat(filename).st_mtime_ns
files.sort(key=getmtimens)
for base_path in files:
with open(base_path) as f:
trial_json = json.load(f)
if status is None or trial_json['status'] == status.value:
trials.append(Trial.from_json(trial_json))
return trials
def save_score_for_success_trial(self, hyperparams: HyperparameterSamples, score: float):
self.trials.append(Trial(hyperparams, score, TRIAL_STATUS.SUCCESS))
if self.print_success_trial:
self.print_func('score: {}'.format(score))
self.print_func(
'hyperparams:\n{}'.format(json.dumps(hyperparams.to_nested_dict(), sort_keys=True, indent=4)))
def train(self, pipeline: BaseStep, data_inputs, expected_outputs=None) -> Trial:
"""
Train pipeline using the validation splitter.
Track training, and validation metrics for each epoch.
Note: the present method is just a shortcut to using the `execute_trial` method with less boilerplate code needed.
Refer to `execute_trial` for full flexibility
:param pipeline: pipeline to train on
:param data_inputs: data inputs
:param expected_outputs: expected ouptuts to fit on
:return: executed trial
"""
validation_splits: List[Tuple[DataContainer, DataContainer]] = self.validation_split_function.split_data_container(
DataContainer(data_inputs=data_inputs, expected_outputs=expected_outputs)
)
repo_trial: Trial = Trial(
pipeline=pipeline,
hyperparams=pipeline.get_hyperparams(),
main_metric_name=self.get_main_metric_name()
)
self.execute_trial(
pipeline=pipeline,
trial_number=1,
repo_trial=repo_trial,
context=ExecutionContext(),
validation_splits=validation_splits,
n_trial=1,
delete_pipeline_on_completion=False
)
return repo_trial
def test_success_trial_split_to_json():
hp = HyperparameterSamples({'a': 2})
trial = Trial(hyperparams=hp, main_metric_name=MAIN_METRIC_NAME)
with trial:
trial_split = given_success_trial_validation_split(trial)
trial_json = trial_split.to_json()
then_success_trial_split_json_is_valid(trial_json)
def test_failure_trial_split_to_json():
hp = HyperparameterSamples({'a': 2})
trial = Trial(hyperparams=hp, main_metric_name='mse')
with trial:
trial_split = given_failed_trial_split(trial)
trial_json = trial_split.to_json()
then_failed_validation_split_json_is_valid(trial_json, trial_split)
def test_trials_get_best_hyperparams_should_return_hyperparams_of_best_trial():
# Given
hp_trial_1 = HyperparameterSamples({'a': 2})
trial_1 = Trial(hyperparams=hp_trial_1, main_metric_name=MAIN_METRIC_NAME)
with trial_1:
given_success_trial_validation_split(trial_1, best_score=0.2)
hp_trial_2 = HyperparameterSamples({'b': 3})
trial_2 = Trial(hyperparams=hp_trial_2, main_metric_name=MAIN_METRIC_NAME)
with trial_2:
given_success_trial_validation_split(trial_2, best_score=0.1)
trials = Trials(trials=[trial_1, trial_2])
# When
best_hyperparams = trials.get_best_hyperparams()
# Then
assert best_hyperparams == hp_trial_2
def new_trial(self, auto_ml_container: 'AutoMLContainer') -> 'Trial':
"""
Create a new trial with the best next hyperparams.
:param auto_ml_container: auto ml data container
:return: trial
"""
hyperparams = self.hyperparameter_selection_strategy.find_next_best_hyperparams(auto_ml_container)
self.print_func('new trial:\n{}'.format(json.dumps(hyperparams.to_nested_dict(), sort_keys=True, indent=4)))
return Trial(hyperparams=hyperparams, main_metric_name=auto_ml_container.main_scoring_metric_name)
def test_failure_trial_split_to_json():
hp = HyperparameterSamples({'a': 2})
repo = InMemoryHyperparamsRepository()
trial = Trial(save_trial_function=repo.save_trial,
hyperparams=hp,
main_metric_name='mse')
with trial:
trial_split = given_failed_trial_split(trial)
trial_json = trial_split.to_json()
then_failed_validation_split_json_is_valid(trial_json, trial_split)
def test_success_trial_split_to_json():
hp = HyperparameterSamples({'a': 2})
repo = InMemoryHyperparamsRepository()
trial = Trial(save_trial_function=repo.save_trial,
hyperparams=hp,
main_metric_name=MAIN_METRIC_NAME)
with trial:
trial_split = given_success_trial_validation_split(trial)
trial_json = trial_split.to_json()
then_success_trial_split_json_is_valid(trial_json)
def new_trial(self, auto_ml_container: 'AutoMLContainer'):
"""
Create new hyperperams trial json file.
:param auto_ml_container: auto ml container
:return:
"""
hyperparams = self.hyperparameter_selection_strategy.find_next_best_hyperparams(auto_ml_container)
trial = Trial(hyperparams, cache_folder=self.cache_folder,
main_metric_name=auto_ml_container.main_scoring_metric_name)
self._create_trial_json(trial=trial)
return trial
def new_trial(self, auto_ml_container: 'AutoMLContainer') -> 'Trial':
hyperparams = self.hyperparameter_selection_strategy.find_next_best_hyperparams(
auto_ml_container)
logger = self._create_logger_for_trial(auto_ml_container.trial_number)
logger.info('\nnew trial: {}'.format(
json.dumps(hyperparams.to_nested_dict(), sort_keys=True,
indent=4)))
return Trial(
cache_folder=self.cache_folder,
save_trial_function=self.save_trial,
logger=logger,
hyperparams=hyperparams,
main_metric_name=auto_ml_container.main_scoring_metric_name)
def new_trial(self, auto_ml_container: 'AutoMLContainer') -> Trial:
"""
Create a new trial with the best next hyperparams.
:param context:
:param auto_ml_container: auto ml data container
:return: trial
"""
hyperparams = self.hyperparameter_selection_strategy.find_next_best_hyperparams(
auto_ml_container)
trial = Trial(
trial_number=auto_ml_container.trial_number,
hyperparams=hyperparams,
save_trial_function=self.save_trial,
cache_folder=self.cache_folder,
main_metric_name=auto_ml_container.main_scoring_metric_name)
return trial
def train(self,
pipeline: BaseStep,
data_inputs,
expected_outputs=None,
context: ExecutionContext = None,
trial_number=0) -> Trial:
"""
Train pipeline using the validation splitter.
Track training, and validation metrics for each epoch.
Note: the present method is just a shortcut to using the `execute_trial` method with less boilerplate code needed. Refer to `execute_trial` for full flexibility
:param pipeline: pipeline to train on
:param data_inputs: data inputs
:param expected_outputs: expected ouptuts to fit on
:return: executed trial
"""
assert not (
context is None
) # TODO: change order of arguments so that context isn't an optional argument
validation_splits: List[Tuple[
DataContainer,
DataContainer]] = self.validation_split_function.split_data_container(
DataContainer(data_inputs=data_inputs,
expected_outputs=expected_outputs),
context=context)
repo_trial: Trial = Trial(
pipeline=pipeline,
logger=context.logger,
hyperparams=pipeline.get_hyperparams(),
main_metric_name=self.get_main_metric_name(),
save_trial_function=self.hyperparams_repository.save_trial,
trial_number=trial_number)
self.execute_trial(pipeline=pipeline,
repo_trial=repo_trial,
context=context,
validation_splits=validation_splits,
n_trial=1,
delete_pipeline_on_completion=False)
return repo_trial
def test_trials_get_best_hyperparams_should_return_hyperparams_of_best_trial(
self):
# Given
trial_1 = self.trial
with trial_1:
self._given_success_trial_validation_split(trial_1, best_score=0.2)
hp_trial_2 = HyperparameterSamples({'b': 3})
trial_2 = Trial(trial_number=1,
save_trial_function=self.repo.save_trial,
hyperparams=hp_trial_2,
main_metric_name=MAIN_METRIC_NAME)
with trial_2:
self._given_success_trial_validation_split(trial_2, best_score=0.1)
trials = Trials(trials=[trial_1, trial_2])
# When
best_hyperparams = trials.get_best_hyperparams()
# Then
assert best_hyperparams == hp_trial_2
class TestTrials:
def setup(self):
self.hp = HyperparameterSamples({'a': 2})
self.repo = InMemoryHyperparamsRepository()
self.trial = Trial(trial_number=0,
save_trial_function=self.repo.save_trial,
hyperparams=self.hp,
main_metric_name=MAIN_METRIC_NAME)
def test_trial_should_have_end_time_later_than_start_time(self):
with self.trial.new_validation_split(Identity()) as trial_split:
time.sleep(0.001) # TODO: maybe remove sleep?
trial_split.set_success()
assert isinstance(trial_split.start_time, datetime.datetime)
assert isinstance(trial_split.end_time, datetime.datetime)
assert trial_split.start_time < trial_split.end_time
def test_trial_should_create_new_split(self):
with self.trial.new_validation_split(Identity()) as trial_split:
trial_split.set_success()
assert self.trial.validation_splits[0] == trial_split
def test_trial_split_is_new_best_score_should_return_true_with_one_score(
self):
with self.trial.new_validation_split(Identity()) as trial_split:
trial_split.add_metric_results_train(name=MAIN_METRIC_NAME,
score=0.5,
higher_score_is_better=False)
trial_split.add_metric_results_validation(
name=MAIN_METRIC_NAME, score=0.5, higher_score_is_better=False)
assert trial_split.is_new_best_score()
def test_trial_split_is_new_best_score_should_return_false_with_not_a_new_best_score(
self):
with self.trial.new_validation_split(Identity()) as trial_split:
trial_split.add_metric_results_train(name=MAIN_METRIC_NAME,
score=0.5,
higher_score_is_better=False)
trial_split.add_metric_results_validation(
name=MAIN_METRIC_NAME, score=0.5, higher_score_is_better=False)
trial_split.add_metric_results_train(name=MAIN_METRIC_NAME,
score=0.7,
higher_score_is_better=False)
trial_split.add_metric_results_validation(
name=MAIN_METRIC_NAME, score=0.7, higher_score_is_better=False)
assert not trial_split.is_new_best_score()
def test_trial_split_is_new_best_score_should_return_true_with_a_new_best_score_after_multiple_scores(
self):
with self.trial.new_validation_split(Identity()) as trial_split:
trial_split.add_metric_results_train(name=MAIN_METRIC_NAME,
score=0.5,
higher_score_is_better=False)
trial_split.add_metric_results_validation(
name=MAIN_METRIC_NAME, score=0.5, higher_score_is_better=False)
trial_split.add_metric_results_train(name=MAIN_METRIC_NAME,
score=0.7,
higher_score_is_better=False)
trial_split.add_metric_results_validation(
name=MAIN_METRIC_NAME, score=0.7, higher_score_is_better=False)
trial_split.add_metric_results_train(name=MAIN_METRIC_NAME,
score=0.4,
higher_score_is_better=False)
trial_split.add_metric_results_validation(
name=MAIN_METRIC_NAME, score=0.4, higher_score_is_better=False)
assert trial_split.is_new_best_score()
def test_success_trial_split_to_json(self):
with self.trial:
trial_split = self._given_success_trial_validation_split(
self.trial)
trial_json = trial_split.to_json()
self._then_success_trial_split_json_is_valid(trial_json)
def _then_success_trial_split_json_is_valid(self, trial_json):
assert trial_json['status'] == TRIAL_STATUS.SUCCESS.value
assert trial_json['error'] is None
assert trial_json['error_traceback'] is None
assert trial_json['metric_results'] == EXPECTED_METRIC_RESULTS
assert trial_json['main_metric_name'] == MAIN_METRIC_NAME
start_time = datetime.datetime.strptime(trial_json['start_time'],
TRIAL_DATETIME_STR_FORMAT)
end_time = datetime.datetime.strptime(
trial_json['end_time'],
TRIAL_DATETIME_STR_FORMAT) + datetime.timedelta(hours=1)
assert start_time < end_time
return True
def test_success_trial_to_json(self):
with self.trial:
self._given_success_trial_validation_split(self.trial)
trial_json = self.trial.to_json()
assert trial_json['status'] == TRIAL_STATUS.SUCCESS.value
assert trial_json['error'] is None
assert trial_json['error_traceback'] is None
assert trial_json['main_metric_name'] == self.trial.main_metric_name
assert self._then_success_trial_split_json_is_valid(
trial_json['validation_splits'][0])
start_time = datetime.datetime.strptime(trial_json['start_time'],
TRIAL_DATETIME_STR_FORMAT)
end_time = datetime.datetime.strptime(
trial_json['end_time'],
TRIAL_DATETIME_STR_FORMAT) + datetime.timedelta(hours=1)
assert start_time < end_time
def test_success_trial_get_validation_score(self):
with self.trial:
self._given_success_trial_validation_split(self.trial,
best_score=0.3)
validation_score = self.trial.get_validation_score()
assert validation_score == 0.3
def test_success_trial_multiple_splits_should_average_the_scores(self):
with self.trial:
self._given_success_trial_validation_split(self.trial,
best_score=0.3)
self._given_success_trial_validation_split(self.trial,
best_score=0.1)
validation_score = self.trial.get_validation_score()
assert validation_score == 0.2
def test_trial_with_failed_split_should_only_average_successful_splits(
self):
with self.trial:
self._given_success_trial_validation_split(self.trial,
best_score=0.3)
self._given_success_trial_validation_split(self.trial,
best_score=0.1)
self._given_failed_trial_split(self.trial)
validation_score = self.trial.get_validation_score()
assert validation_score == 0.2
def _given_success_trial_validation_split(self, trial, best_score=0.4):
with trial.new_validation_split(Identity()) as trial_split:
trial_split.add_metric_results_train(name=MAIN_METRIC_NAME,
score=0.5,
higher_score_is_better=False)
trial_split.add_metric_results_validation(
name=MAIN_METRIC_NAME, score=0.5, higher_score_is_better=False)
trial_split.add_metric_results_train(name=MAIN_METRIC_NAME,
score=0.7,
higher_score_is_better=False)
trial_split.add_metric_results_validation(
name=MAIN_METRIC_NAME, score=0.7, higher_score_is_better=False)
trial_split.add_metric_results_train(name=MAIN_METRIC_NAME,
score=best_score,
higher_score_is_better=False)
trial_split.add_metric_results_validation(
name=MAIN_METRIC_NAME,
score=best_score,
higher_score_is_better=False)
trial_split.set_success()
trial.set_success()
return trial_split
def test_failure_trial_split_to_json(self):
with self.trial:
trial_split = self._given_failed_trial_split(self.trial)
trial_json = trial_split.to_json()
self._then_failed_validation_split_json_is_valid(
trial_json, trial_split)
def _then_failed_validation_split_json_is_valid(self, trial_json,
trial_split):
assert trial_json['status'] == TRIAL_STATUS.FAILED.value
assert trial_json['error'] == str(trial_split.error)
assert trial_json['error_traceback'] == EXPECTED_ERROR_TRACEBACK
assert trial_json['metric_results'] == EXPECTED_METRIC_RESULTS
assert trial_json['main_metric_name'] == trial_split.main_metric_name
start_time = datetime.datetime.strptime(trial_json['start_time'],
TRIAL_DATETIME_STR_FORMAT)
end_time = datetime.datetime.strptime(
trial_json['end_time'],
TRIAL_DATETIME_STR_FORMAT) + datetime.timedelta(hours=1)
assert start_time < end_time
return True
def test_failure_trial_to_json(self):
with self.trial:
trial_split = self._given_failed_trial_split(self.trial)
trial_json = self.trial.to_json()
assert trial_json['status'] == TRIAL_STATUS.FAILED.value
assert trial_json['error'] == str(trial_split.error)
assert trial_json['error_traceback'] == EXPECTED_ERROR_TRACEBACK
assert trial_json['main_metric_name'] == self.trial.main_metric_name
assert self._then_failed_validation_split_json_is_valid(
trial_json['validation_splits'][0], trial_split=trial_split)
start_time = datetime.datetime.strptime(trial_json['start_time'],
TRIAL_DATETIME_STR_FORMAT)
end_time = datetime.datetime.strptime(
trial_json['end_time'],
TRIAL_DATETIME_STR_FORMAT) + datetime.timedelta(hours=1)
assert start_time < end_time
def _given_failed_trial_split(self, trial):
with trial.new_validation_split(Identity()) as trial_split:
trial_split.add_metric_results_train(name=MAIN_METRIC_NAME,
score=0.5,
higher_score_is_better=False)
trial_split.add_metric_results_validation(
name=MAIN_METRIC_NAME, score=0.5, higher_score_is_better=False)
trial_split.add_metric_results_train(name=MAIN_METRIC_NAME,
score=0.7,
higher_score_is_better=False)
trial_split.add_metric_results_validation(
name=MAIN_METRIC_NAME, score=0.7, higher_score_is_better=False)
trial_split.add_metric_results_train(name=MAIN_METRIC_NAME,
score=0.4,
higher_score_is_better=False)
trial_split.add_metric_results_validation(
name=MAIN_METRIC_NAME, score=0.4, higher_score_is_better=False)
error = IndexError('index error')
trial_split.set_failed(error)
trial.set_failed(error)
return trial_split
def test_trials_get_best_hyperparams_should_return_hyperparams_of_best_trial(
self):
# Given
trial_1 = self.trial
with trial_1:
self._given_success_trial_validation_split(trial_1, best_score=0.2)
hp_trial_2 = HyperparameterSamples({'b': 3})
trial_2 = Trial(trial_number=1,
save_trial_function=self.repo.save_trial,
hyperparams=hp_trial_2,
main_metric_name=MAIN_METRIC_NAME)
with trial_2:
self._given_success_trial_validation_split(trial_2, best_score=0.1)
trials = Trials(trials=[trial_1, trial_2])
# When
best_hyperparams = trials.get_best_hyperparams()
# Then
assert best_hyperparams == hp_trial_2
def from_json(trial_json) -> 'Trial':
return Trial(
hyperparams=trial_json['hyperparams'],
score=trial_json['score'],
status=trial_json['status']
)
