def load_all_trials(self, status: 'TRIAL_STATUS' = None) -> 'Trials': """ Load all hyperparameter trials with their corresponding score. Reads all the saved trial json files, sorted by creation date. :return: (hyperparams, scores) """ trials = Trials() files = glob.glob(os.path.join(self.cache_folder, '*.json')) # sort by created date: def getmtimens(filename): return os.stat(filename).st_mtime_ns files.sort(key=getmtimens) for base_path in files: with open(base_path) as f: try: trial_json = json.load(f) except Exception as err: print('invalid trial json file'.format(base_path)) print(traceback.format_exc()) continue if status is None or trial_json['status'] == status.value: trials.append(Trial.from_json( update_trial_function=self.save_trial, trial_json=trial_json )) return trials
def __init__(self, hyperparameter_selection_strategy=None, cache_folder: str = None, best_retrained_model_folder=None): HyperparamsRepository.__init__( self, hyperparameter_selection_strategy=hyperparameter_selection_strategy, cache_folder=cache_folder, best_retrained_model_folder=best_retrained_model_folder) self.cache_folder = cache_folder self.trials = Trials()
def __init__(self, hyperparameter_selection_strategy=None, print_func: Callable = None, cache_folder: str = None, best_retrained_model_folder=None): HyperparamsRepository.__init__( self, hyperparameter_selection_strategy=hyperparameter_selection_strategy, cache_folder=cache_folder, best_retrained_model_folder=best_retrained_model_folder ) if print_func is None: print_func = print self.print_func = print_func self.cache_folder = cache_folder self.trials = Trials()
def test_trials_get_best_hyperparams_should_return_hyperparams_of_best_trial(): # Given hp_trial_1 = HyperparameterSamples({'a': 2}) trial_1 = Trial(hyperparams=hp_trial_1, main_metric_name=MAIN_METRIC_NAME) with trial_1: given_success_trial_validation_split(trial_1, best_score=0.2) hp_trial_2 = HyperparameterSamples({'b': 3}) trial_2 = Trial(hyperparams=hp_trial_2, main_metric_name=MAIN_METRIC_NAME) with trial_2: given_success_trial_validation_split(trial_2, best_score=0.1) trials = Trials(trials=[trial_1, trial_2]) # When best_hyperparams = trials.get_best_hyperparams() # Then assert best_hyperparams == hp_trial_2
def test_trials_get_best_hyperparams_should_return_hyperparams_of_best_trial( self): # Given trial_1 = self.trial with trial_1: self._given_success_trial_validation_split(trial_1, best_score=0.2) hp_trial_2 = HyperparameterSamples({'b': 3}) trial_2 = Trial(trial_number=1, save_trial_function=self.repo.save_trial, hyperparams=hp_trial_2, main_metric_name=MAIN_METRIC_NAME) with trial_2: self._given_success_trial_validation_split(trial_2, best_score=0.1) trials = Trials(trials=[trial_1, trial_2]) # When best_hyperparams = trials.get_best_hyperparams() # Then assert best_hyperparams == hp_trial_2