def load_all_trials(self, status: 'TRIAL_STATUS' = None) -> 'Trials':
"""
Load all hyperparameter trials with their corresponding score.
Reads all the saved trial json files, sorted by creation date.
:return: (hyperparams, scores)
"""
trials = Trials()
files = glob.glob(os.path.join(self.cache_folder, '*.json'))
# sort by created date:
def getmtimens(filename):
return os.stat(filename).st_mtime_ns
files.sort(key=getmtimens)
for base_path in files:
with open(base_path) as f:
try:
trial_json = json.load(f)
except Exception as err:
print('invalid trial json file'.format(base_path))
print(traceback.format_exc())
continue
if status is None or trial_json['status'] == status.value:
trials.append(Trial.from_json(
update_trial_function=self.save_trial,
trial_json=trial_json
))
return trials
class InMemoryHyperparamsRepository(HyperparamsRepository):
"""
In memory hyperparams repository that can print information about trials.
Useful for debugging.
Example usage :
.. code-block:: python
InMemoryHyperparamsRepository(
hyperparameter_selection_strategy=RandomSearchHyperparameterSelectionStrategy(),
print_func=print,
cache_folder='cache',
best_retrained_model_folder='best'
)
.. seealso::
:class:`AutoML`,
:class:`Trainer`,
:class:`~neuraxle.metaopt.trial.Trial`,
:class:`HyperparamsJSONRepository`,
:class:`BaseHyperparameterSelectionStrategy`,
:class:`RandomSearchHyperparameterSelectionStrategy`,
:class:`~neuraxle.hyperparams.space.HyperparameterSamples`
"""
def __init__(self,
hyperparameter_selection_strategy=None,
cache_folder: str = None,
best_retrained_model_folder=None):
HyperparamsRepository.__init__(
self,
hyperparameter_selection_strategy=hyperparameter_selection_strategy,
cache_folder=cache_folder,
best_retrained_model_folder=best_retrained_model_folder)
self.cache_folder = cache_folder
self.trials = Trials()
def load_all_trials(self, status: 'TRIAL_STATUS' = None) -> 'Trials':
"""
Load all trials with the given status.
:param status: trial status
:return: list of trials
"""
return self.trials.filter(status)
def _save_trial(self, trial: 'Trial'):
"""
Save trial.
:param trial: trial to save
:return:
"""
self.trials.append(trial)
def __init__(self,
hyperparameter_selection_strategy=None,
cache_folder: str = None,
best_retrained_model_folder=None):
HyperparamsRepository.__init__(
self,
hyperparameter_selection_strategy=hyperparameter_selection_strategy,
cache_folder=cache_folder,
best_retrained_model_folder=best_retrained_model_folder)
self.cache_folder = cache_folder
self.trials = Trials()
def __init__(self, hyperparameter_selection_strategy=None, print_func: Callable = None, cache_folder: str = None,
best_retrained_model_folder=None):
HyperparamsRepository.__init__(
self,
hyperparameter_selection_strategy=hyperparameter_selection_strategy,
cache_folder=cache_folder,
best_retrained_model_folder=best_retrained_model_folder
)
if print_func is None:
print_func = print
self.print_func = print_func
self.cache_folder = cache_folder
self.trials = Trials()
def test_trials_get_best_hyperparams_should_return_hyperparams_of_best_trial():
# Given
hp_trial_1 = HyperparameterSamples({'a': 2})
trial_1 = Trial(hyperparams=hp_trial_1, main_metric_name=MAIN_METRIC_NAME)
with trial_1:
given_success_trial_validation_split(trial_1, best_score=0.2)
hp_trial_2 = HyperparameterSamples({'b': 3})
trial_2 = Trial(hyperparams=hp_trial_2, main_metric_name=MAIN_METRIC_NAME)
with trial_2:
given_success_trial_validation_split(trial_2, best_score=0.1)
trials = Trials(trials=[trial_1, trial_2])
# When
best_hyperparams = trials.get_best_hyperparams()
# Then
assert best_hyperparams == hp_trial_2
def test_trials_get_best_hyperparams_should_return_hyperparams_of_best_trial(
self):
# Given
trial_1 = self.trial
with trial_1:
self._given_success_trial_validation_split(trial_1, best_score=0.2)
hp_trial_2 = HyperparameterSamples({'b': 3})
trial_2 = Trial(trial_number=1,
save_trial_function=self.repo.save_trial,
hyperparams=hp_trial_2,
main_metric_name=MAIN_METRIC_NAME)
with trial_2:
self._given_success_trial_validation_split(trial_2, best_score=0.1)
trials = Trials(trials=[trial_1, trial_2])
# When
best_hyperparams = trials.get_best_hyperparams()
# Then
assert best_hyperparams == hp_trial_2
class InMemoryHyperparamsRepository(HyperparamsRepository):
"""
In memory hyperparams repository that can print information about trials.
Useful for debugging.
Example usage :
.. code-block:: python
InMemoryHyperparamsRepository(
hyperparameter_selection_strategy=RandomSearchHyperparameterSelectionStrategy(),
print_func=print,
cache_folder='cache',
best_retrained_model_folder='best'
)
.. seealso::
:class:`AutoML`,
:class:`Trainer`,
:class:`~neuraxle.metaopt.trial.Trial`,
:class:`HyperparamsJSONRepository`,
:class:`BaseHyperparameterSelectionStrategy`,
:class:`RandomSearchHyperparameterSelectionStrategy`,
:class:`~neuraxle.hyperparams.space.HyperparameterSamples`
"""
def __init__(self, hyperparameter_selection_strategy=None, print_func: Callable = None, cache_folder: str = None,
best_retrained_model_folder=None):
HyperparamsRepository.__init__(
self,
hyperparameter_selection_strategy=hyperparameter_selection_strategy,
cache_folder=cache_folder,
best_retrained_model_folder=best_retrained_model_folder
)
if print_func is None:
print_func = print
self.print_func = print_func
self.cache_folder = cache_folder
self.trials = Trials()
def load_all_trials(self, status: 'TRIAL_STATUS' = None) -> 'Trials':
"""
Load all trials with the given status.
:param status: trial status
:return: list of trials
"""
return self.trials.filter(status)
def _save_trial(self, trial: 'Trial'):
"""
Save trial.
:param trial: trial to save
:return:
"""
self.print_func(trial)
self.trials.append(trial)
def new_trial(self, auto_ml_container: 'AutoMLContainer') -> 'Trial':
"""
Create a new trial with the best next hyperparams.
:param auto_ml_container: auto ml data container
:return: trial
"""
hyperparams = self.hyperparameter_selection_strategy.find_next_best_hyperparams(auto_ml_container)
self.print_func('new trial:\n{}'.format(json.dumps(hyperparams.to_nested_dict(), sort_keys=True, indent=4)))
return Trial(
save_trial_function=self.save_trial,
hyperparams=hyperparams,
main_metric_name=auto_ml_container.main_scoring_metric_name
)
