def create_trials(self, complete, losses): if len(complete) > 0: trials = Trials() hist = self.create_history(complete) index = 0 for c in complete: if c >= len(losses): error( "Index {} is larger than the size of losses {}".format( c, len(losses))) loss = losses[c] rval_specs = [None] new_id = index rval_results = [] rval_results.append(create_ok_result(loss, c)) rval_miscs = [] rval_miscs.append(self.create_misc(index, hist)) hyperopt_trial = trials.new_trial_docs([new_id], rval_specs, rval_results, rval_miscs)[0] index += 1 if self.response_shaping is True: # transform log applied loss for enhancing optimization performance #debug("before scaling: {}".format(loss)) if self.shaping_func == "log_err": loss = apply_log_err(loss) elif self.shaping_func == "hybrid_log": loss = apply_hybrid_log(loss) else: debug("Invalid shaping function: {}".format( self.shaping_func)) hyperopt_trial['result'] = { 'loss': float(loss), 'status': STATUS_OK } hyperopt_trial['state'] = JOB_STATE_DONE #debug("History appended: {}-{}".format(c, loss)) trials.insert_trial_doc(hyperopt_trial) trials.refresh() return trials else: return Trials()
def create_trials(self, completed, losses): if len(completed) > 0: trials = Trials() hist = self.create_history(completed) #index = 0 #for c in completed: for index in range(len(completed)): c = completed[index] loss = losses[index] rval_specs = [None] new_id = index rval_results = [ ] rval_results.append(create_ok_result(loss, c)) rval_miscs = [ ] rval_miscs.append(self.create_misc(index, hist)) hopt_trial = trials.new_trial_docs([new_id], rval_specs, rval_results, rval_miscs)[0] if self.response_shaping is True: # transform log applied loss for enhancing optimization performance #debug("before scaling: {}".format(loss)) if self.shaping_func == "log_err": loss = apply_log_err(loss) elif self.shaping_func == "hybrid_log": loss = apply_hybrid_log(loss) else: debug("Invalid shaping function: {}".format(self.shaping_func)) if loss != None: hopt_trial['result'] = {'loss': float(loss), 'status': STATUS_OK} hopt_trial['state'] = JOB_STATE_DONE #debug("History appended: {}-{}".format(c, loss)) trials.insert_trial_doc(hopt_trial) trials.refresh() return trials else: return Trials()
def suggest(self, history, searchspace): """ Suggest params to maximize an objective function based on the function evaluation history using a tree of Parzen estimators (TPE), as implemented in the hyperopt package. Use of this function requires that hyperopt be installed. """ # This function is very odd, because as far as I can tell there's # no real documented API for any of the internals of hyperopt. Its # execution model is that hyperopt calls your objective function # (instead of merely providing you with suggested points, and then # you calling the function yourself), and its very tricky (for me) # to use the internal hyperopt data structures to get these predictions # out directly. # so they path we take in this function is to construct a synthetic # hyperopt.Trials database which from the `history`, and then call # hyoperopt.fmin with a dummy objective function that logs the value # used, and then return that value to our client. # The form of the hyperopt.Trials database isn't really documented in # the code -- most of this comes from reverse engineering it, by # running fmin() on a simple function and then inspecting the form of # the resulting trials object. if 'hyperopt' not in sys.modules: raise ImportError('No module named hyperopt') random = check_random_state(self.seed) hp_searchspace = searchspace.to_hyperopt() trials = Trials() for i, (params, scores, status) in enumerate(history): if status == 'SUCCEEDED': # we're doing maximization, hyperopt.fmin() does minimization, # so we need to swap the sign result = {'loss': -np.mean(scores), 'status': STATUS_OK} elif status == 'PENDING': result = {'status': STATUS_RUNNING} elif status == 'FAILED': result = {'status': STATUS_FAIL} else: raise RuntimeError('unrecognized status: %s' % status) # the vals key in the trials dict is basically just the params # dict, but enum variables (hyperopt hp.choice() nodes) are # different, because the index of the parameter is specified # in vals, not the parameter itself. vals = {} for var in searchspace: if isinstance(var, EnumVariable): # get the index in the choices of the parameter, and use # that. matches = [i for i, c in enumerate(var.choices) if c == params[var.name]] assert len(matches) == 1 vals[var.name] = matches else: # the other big difference is that all of the param values # are wrapped in length-1 lists. vals[var.name] = [params[var.name]] trials.insert_trial_doc({ 'misc': { 'cmd': ('domain_attachment', 'FMinIter_Domain'), 'idxs': dict((k, [i]) for k in hp_searchspace.keys()), 'tid': i, 'vals': vals, 'workdir': None}, 'result': result, 'tid': i, # bunch of fixed fields that hyperopt seems to require 'owner': None, 'spec': None, 'state': 2, 'book_time': None, 'exp_key': None, 'refresh_time': None, 'version': 0 }) trials.refresh() chosen_params_container = [] def mock_fn(x): # http://stackoverflow.com/a/3190783/1079728 # to get around no nonlocal keywork in python2 chosen_params_container.append(x) return 0 fmin(fn=mock_fn, algo=tpe.suggest, space=hp_searchspace, trials=trials, max_evals=len(trials.trials)+1, **self._hyperopt_fmin_random_kwarg(random)) chosen_params = chosen_params_container[0] return chosen_params
def suggest(self, history, searchspace): """ Suggest params to maximize an objective function based on the function evaluation history using a tree of Parzen estimators (TPE), as implemented in the hyperopt package. Use of this function requires that hyperopt be installed. """ # This function is very odd, because as far as I can tell there's # no real documented API for any of the internals of hyperopt. Its # execution model is that hyperopt calls your objective function # (instead of merely providing you with suggested points, and then # you calling the function yourself), and its very tricky (for me) # to use the internal hyperopt data structures to get these predictions # out directly. # so they path we take in this function is to construct a synthetic # hyperopt.Trials database which from the `history`, and then call # hyoperopt.fmin with a dummy objective function that logs the value # used, and then return that value to our client. # The form of the hyperopt.Trials database isn't really documented in # the code -- most of this comes from reverse engineering it, by # running fmin() on a simple function and then inspecting the form of # the resulting trials object. if 'hyperopt' not in sys.modules: raise ImportError('No module named hyperopt') random = check_random_state(self.seed) hp_searchspace = searchspace.to_hyperopt() trials = Trials() for i, (params, scores, status) in enumerate(history): if status == 'SUCCEEDED': # we're doing maximization, hyperopt.fmin() does minimization, # so we need to swap the sign result = {'loss': -np.mean(scores), 'status': STATUS_OK} elif status == 'PENDING': result = {'status': STATUS_RUNNING} elif status == 'FAILED': result = {'status': STATUS_FAIL} else: raise RuntimeError('unrecognized status: %s' % status) # the vals key in the trials dict is basically just the params # dict, but enum variables (hyperopt hp.choice() nodes) are # different, because the index of the parameter is specified # in vals, not the parameter itself. vals = {} for var in searchspace: if isinstance(var, EnumVariable): # get the index in the choices of the parameter, and use # that. matches = [ i for i, c in enumerate(var.choices) if c == params[var.name] ] assert len(matches) == 1 vals[var.name] = matches else: # the other big difference is that all of the param values # are wrapped in length-1 lists. vals[var.name] = [params[var.name]] trials.insert_trial_doc({ 'misc': { 'cmd': ('domain_attachment', 'FMinIter_Domain'), 'idxs': dict((k, [i]) for k in hp_searchspace.keys()), 'tid': i, 'vals': vals, 'workdir': None }, 'result': result, 'tid': i, # bunch of fixed fields that hyperopt seems to require 'owner': None, 'spec': None, 'state': 2, 'book_time': None, 'exp_key': None, 'refresh_time': None, 'version': 0 }) trials.refresh() chosen_params_container = [] def suggest(*args, **kwargs): return tpe.suggest(*args, **kwargs, gamma=self.gamma, n_startup_jobs=self.seeds) def mock_fn(x): # http://stackoverflow.com/a/3190783/1079728 # to get around no nonlocal keywork in python2 chosen_params_container.append(x) return 0 fmin(fn=mock_fn, algo=tpe.suggest, space=hp_searchspace, trials=trials, max_evals=len(trials.trials) + 1, **self._hyperopt_fmin_random_kwarg(random)) chosen_params = chosen_params_container[0] return chosen_params
def read_or_recreate_trials( hyperparameter_search_dir, tuning_dataset=None, test_dataset=None, tqdm=None, overwrite=False, do_print=True, trials_out_dict=None, results_out_dict=None, args_out_dict=None, params_out_dict=None, ): config = read_config(hyperparameter_search_dir)[0] hyperparameter_search_args = HyperparameterSearchArgs.from_json_file( os.path.join(hyperparameter_search_dir, HYPERPARAMETER_SEARCH_ARGS_FILENAME)) filepath = os.path.join(hyperparameter_search_dir, HYP_CONFIG_FILENAME) with open(filepath, mode='r') as f: raw_config = json.loads(f.read()) rotations = set( x for x in os.listdir(hyperparameter_search_dir)).intersection( set(str(i) for i in range(10))) if do_print: print("Observe runs for rotations: %s" % ', '.join(rotations)) if trials_out_dict is None: trials_out_dict = {} if results_out_dict is None: results_out_dict = {} if args_out_dict is None: args_out_dict = {} if params_out_dict is None: params_out_dict = {} if len(rotations) < 4 or tqdm is None: rotations_rng = rotations else: rotations_rng = tqdm(rotations, desc="Reading Rotations") for rotation in rotations_rng: for d in (results_out_dict, args_out_dict, params_out_dict): if rotation not in d: d[rotation] = {} rotation_results = results_out_dict[rotation] rotation_args = args_out_dict[rotation] rotation_params = params_out_dict[rotation] rotation_dir = os.path.join(hyperparameter_search_dir, rotation) run_names = [r for r in os.listdir(rotation_dir) if r != 'trials.pkl'] run_names_rng = run_names if tqdm is None else tqdm( run_names, desc="Reading Runs") for run_name in run_names_rng: run_dir = os.path.join(rotation_dir, run_name) if not os.path.isdir(run_dir): print(f"Found file; expecting directory! {run_dir})") continue elif os.path.isfile(os.path.join(run_dir, 'error.pkl')): continue # Lastly, if we don't need results, continue if run_name in rotation_results and rotation_results[ run_name] is not None: continue args_filepath = os.path.join(run_dir, ARGS_FILENAME) if not os.path.isfile(args_filepath): continue args = Args.from_json_file(args_filepath) num_epochs = args.epochs completed_training = os.path.isfile( os.path.join(run_dir, 'model.epoch-%d' % (num_epochs - 1))) if not completed_training: print( f"Run {run_dir} still training (or errored and didn't report)" ) continue tuning_result_filepath = os.path.join(run_dir, 'tuning_perf.pkl') if os.path.isfile(tuning_result_filepath): with open(tuning_result_filepath, mode='rb') as f: tuning = pickle.load(f) elif os.path.isfile( os.path.join(run_dir, 'tuning_perf_metrics.pkl')): with open(os.path.join(run_dir, 'tuning_perf_metrics.pkl'), mode='rb') as f: tuning = pickle.load(f) else: print(f'Missing tuning for {run_dir}') if tuning_dataset is not None: # set the necessary features in tuning dataset if args.do_masked_imputation: tuning_dataset.imputation_mask_rate = args.imputation_mask_rate _, _, tuning = evaluator.evaluate_multi( tuning_dataset, model_rundir=run_dir, num_random_endpoints=10, batch_size=1024, num_workers=27, evaluate_on_25=True, get_all_reprs=False, tqdm=tqdm) with open(tuning_result_filepath, mode='wb') as f: pickle.dump(tuning, f) else: print("Wasn't given a tuning dataset!") continue test_result_filepath = os.path.join(run_dir, 'test_perf.pkl') if os.path.isfile(test_result_filepath): with open(test_result_filepath, mode='rb') as f: test = pickle.load(f) elif os.path.isfile(os.path.join(run_dir, 'test_perf_metrics.pkl')): with open(os.path.join(run_dir, 'test_perf_metrics.pkl'), mode='rb') as f: test = pickle.load(f) else: print(f"Have tuning but missing test for {run_dir}/{run_name}") test = None if test_dataset is not None: if args.do_masked_imputation: test_dataset.imputation_mask_rate = args.imputation_mask_rate _, _, test = evaluator.evaluate_multi( test_dataset, model_rundir=run_dir, num_random_endpoints=10, batch_size=1024, num_workers=27, evaluate_on_25=True, get_all_reprs=False, tqdm=tqdm) with open(test_result_filepath, mode='wb') as f: pickle.dump(test, f) else: "Wasn't given a test dataset!" rotation_results[run_name] = (tuning, test) if run_name not in rotation_args or rotation_args[run_name] is None: rotation_args[run_name] = args if run_name not in rotation_params or rotation_params[ run_name] is None: params_filepath = os.path.join(run_dir, PARAMS_FILENAME) if os.path.isfile(params_filepath): with open(params_filepath, mode='rb') as f: rotation_params[run_name] = pickle.load(f) else: rotation_params[run_name] = args_to_params( rotation_args[run_name], raw_config) if rotation in trials_out_dict and trials_out_dict[ rotation] is not None: continue trials_filepath = os.path.join(rotation_dir, 'trials.pkl') if os.path.exists(trials_filepath) and not overwrite: with open(trials_filepath, mode='rb') as f: trials_out_dict[rotation] = pickle.load(f) continue # Rebuild Trials # TODO(mmd): Something wrong in misc.idxs... trials = Trials(exp_key='exp') #hyperparameter_search_dir for run_name in rotation_results: args = rotation_args[run_name] params = rotation_params[run_name] perf_metrics, test_perf_metrics = rotation_results[run_name] try: loss = ObjectiveFntr.perf_metrics_to_trial_result( perf_metrics, args, single_task=hyperparameter_search_args.single_task_search) except Exception as e: print( f"Errored computing tuning results for {hyperparameter_search_dir} on rotation " f"{rotation}, {run_name}: {e}") traceback.print_exc() continue try: if test_perf_metrics is not None: test_loss = ObjectiveFntr.perf_metrics_to_trial_result( test_perf_metrics, args, single_task=hyperparameter_search_args. single_task_search) else: test_loss = np.NaN except TypeError as e: test_loss = np.NaN except Exception as e: if "'NoneType' object is not subscriptable" in str(e): test_loss = np.NaN else: print( f"Errored computing test results for {hyperparameter_search_dir} on rotation {rotation}, " f"{run_name}: {e}") traceback.print_exc() continue loss_variance, test_loss_variance = np.NaN, np.NaN result = { 'status': STATUS_OK, 'loss': loss, 'loss_variance': loss_variance, 'test_loss': test_loss, 'test_loss_variance': test_loss_variance, } spec = params trials.insert_trial_doc({ 'tid': run_name, 'spec': spec, 'result': result, 'misc': { 'tid': run_name, 'cmd': '', 'idxs': [], 'vals': {k: [v] for k, v in spec.items()}, }, 'state': '', 'owner': '', 'book_time': 0, 'refresh_time': 0, 'exp_key': 'exp', # hyperparameter_search_dir, }) trials.refresh() trials_out_dict[rotation] = trials return config, results_out_dict, args_out_dict, params_out_dict, trials_out_dict
trials=cache_trials, max_evals=1, return_argmin=False, show_progressbar=False) trial = cache_trials.trials[-1] return trial, trial_parameters for i in range(5): t, tp = get_next_params(global_trials) mse = run_model(tp) t['result']['loss'] = mse t['refresh_time'] = datetime.datetime.now() global_trials.insert_trial_doc(t) global_trials.refresh() print(global_trials.best_trial) """ trials = Trials() print("First recommendation") fmin(param_extractor, space, algo=tpe.suggest, trials=trials, max_evals=1, return_argmin=False) new_trial = trials.trials[-1] new_trial['result']['loss'] = 0.4 print(new_trial) global_trials.insert_trial_doc(new_trial)
def read_or_recreate_trials(hyperparameter_search_dir, tuning_dataset=None, test_dataset=None, tqdm=None, overwrite=False): config = read_config(hyperparameter_search_dir)[0] filepath = os.path.join(hyperparameter_search_dir, HYP_CONFIG_FILENAME) with open(filepath, mode='r') as f: raw_config = json.loads(f.read()) rotations = set( x for x in os.listdir(hyperparameter_search_dir)).intersection( set(str(i) for i in range(10))) print("Observe runs for rotations: %s" % ', '.join(rotations)) all_trials = {} all_results = {} all_args = {} all_params = {} rotations_rng = rotations if len(rotations) < 4 or tqdm is None else tqdm( rotations) for rotation in rotations_rng: rotation_results = {} rotation_args = {} rotation_params = {} rotation_dir = os.path.join(hyperparameter_search_dir, rotation) run_names = [r for r in os.listdir(rotation_dir) if r != 'trials.pkl'] run_names_rng = run_names if tqdm is None else tqdm(run_names) for run_name in run_names: run_dir = os.path.join(rotation_dir, run_name) if not os.path.isdir(run_dir): print(run_dir) continue if os.path.isfile(os.path.join(run_dir, 'error.pkl')): continue args_filepath = os.path.join(run_dir, ARGS_FILENAME) if not os.path.isfile(args_filepath): continue args = Args.from_json_file(args_filepath) rotation_args[run_name] = args params_filepath = os.path.join(run_dir, PARAMS_FILENAME) if os.path.isfile(params_filepath): with open(params_filepath, mode='rb') as f: rotation_params[run_name] = pickle.load(f) else: rotation_params[run_name] = args_to_params( rotation_args[run_name], raw_config) num_epochs = args.epochs completed_training = os.path.isfile( os.path.join(run_dir, 'model.epoch-%d' % (num_epochs - 1))) if not completed_training: print("Run %s Still training (or errored and didn't report)" % run_name) continue tuning_result_filepath = os.path.join(run_dir, 'tuning_perf_metrics.pkl') if os.path.isfile(tuning_result_filepath): with open(tuning_result_filepath, mode='rb') as f: tuning = pickle.load(f) else: print('Missing tuning for %s' % run_name) if tuning_dataset is not None: _, _, tuning = evaluate_multi(tuning_dataset, model_rundir=run_dir, num_random_endpoints=10, batch_size=1024, num_workers=27, evaluate_on_25=True, get_all_reprs=False, tqdm=tqdm) with open(tuning_result_filepath, mode='wb') as f: pickle.dump(tuning, f) else: print("Wasn't given a tuning dataset!") continue test_result_filepath = os.path.join(run_dir, 'test_perf_metrics.pkl') if os.path.isfile(test_result_filepath): with open(test_result_filepath, mode='rb') as f: test = pickle.load(f) else: print('Have tuning but missing test for %s' % run_name) if test_dataset is not None: _, _, test = evaluate_multi(test_dataset, model_rundir=run_dir, num_random_endpoints=10, batch_size=1024, num_workers=27, evaluate_on_25=True, get_all_reprs=False, tqdm=tqdm) with open(test_result_filepath, mode='wb') as f: pickle.dump(test, f) else: "Wasn't given a test dataset!" rotation_results[run_name] = (tuning, test) all_results[rotation] = rotation_results all_args[rotation] = rotation_args all_params[rotation] = rotation_params trials_filepath = os.path.join(rotation_dir, 'trials.pkl') if os.path.exists(trials_filepath) and not overwrite: with open(trials_filepath, mode='rb') as f: all_trials[rotation] = pickle.load(f) continue # Rebuild Trials # TODO(mmd): Something wrong in misc.idxs... trials = Trials(exp_key='exp') #hyperparameter_search_dir for run_name in rotation_results: args = rotation_args[run_name] params = rotation_params[run_name] perf_metrics, test_perf_metrics = rotation_results[run_name] tuning_scores = -pd.Series( ObjectiveFntr.perf_metrics_to_trial_result(perf_metrics)) test_scores = -pd.Series( ObjectiveFntr.perf_metrics_to_trial_result(test_perf_metrics)) loss = tuning_scores.mean() loss_variance = tuning_scores.std()**2 test_loss = test_scores.mean() test_loss_variance = test_scores.std()**2 result = { 'status': STATUS_OK, 'loss': loss, 'loss_variance': loss_variance, 'test_loss': test_loss, 'test_loss_variance': test_loss_variance, } spec = params trials.insert_trial_doc({ 'tid': run_name, 'spec': spec, 'result': result, 'misc': { 'tid': run_name, 'cmd': '', 'idxs': [], 'vals': {k: [v] for k, v in spec.items()}, }, 'state': '', 'owner': '', 'book_time': 0, 'refresh_time': 0, 'exp_key': 'exp', # hyperparameter_search_dir, }) trials.refresh() all_trials[rotation] = trials return config, all_results, all_args, all_params, all_trials
class _HyperOpt(base.Optimizer): # pylint: disable=too-many-instance-attributes def __init__( self, parametrization: IntOrParameter, budget: tp.Optional[int] = None, num_workers: int = 1, *, prior_weight: float = 1.0, n_startup_jobs: int = 20, n_EI_candidates: int = 24, gamma: float = 0.25, verbose: bool = False, ) -> None: super().__init__(parametrization, budget=budget, num_workers=num_workers) try: # try to convert parametrization to hyperopt search space if not isinstance(self.parametrization, p.Instrumentation): raise NotImplementedError self.space = _get_search_space(self.parametrization.name, self.parametrization) self._transform = None except NotImplementedError: self._transform = transforms.ArctanBound(0, 1) self.space = { f"x_{i}": hp.uniform(f"x_{i}", 0, 1) for i in range(self.dimension) } self.trials = Trials() self.domain = Domain(fn=None, expr=self.space, pass_expr_memo_ctrl=False) self.tpe_args = { "prior_weight": prior_weight, "n_startup_jobs": n_startup_jobs, "n_EI_candidates": n_EI_candidates, "gamma": gamma, "verbose": verbose, } def _internal_ask_candidate(self) -> p.Parameter: # Inspired from FMinIter class (hyperopt) next_id = self.trials.new_trial_ids(1) new_trial = tpe.suggest(next_id, self.domain, self.trials, self._rng.randint(2**31 - 1), **self.tpe_args)[0] self.trials.insert_trial_doc(new_trial) self.trials.refresh() candidate = self.parametrization.spawn_child() if self._transform: data = np.array([ new_trial["misc"]["vals"][f"x_{i}"][0] for i in range(self.dimension) ]) candidate = candidate.set_standardized_data( self._transform.backward(data)) # For consistency, we need to update hyperopt history # when standardized data is changed if any(data != self._transform.forward( candidate.get_standardized_data( reference=self.parametrization))): for it, val in enumerate( self._transform.forward( candidate.get_standardized_data( reference=self.parametrization))): self.trials._dynamic_trials[ next_id[0]]["misc"]["vals"][f"x_{it}"][0] = val else: spec = hyperopt.base.spec_from_misc(new_trial["misc"]) config = hyperopt.space_eval(self.space, spec) candidate.value = _hp_dict_to_parametrization(config) candidate._meta["trial_id"] = new_trial["tid"] return candidate def _internal_tell_candidate(self, candidate: p.Parameter, loss: float) -> None: result = {"loss": loss, "status": "ok"} assert "trial_id" in candidate._meta tid = candidate._meta["trial_id"] assert self.trials._dynamic_trials[tid][ "state"] == hyperopt.JOB_STATE_NEW now = hyperopt.utils.coarse_utcnow() self.trials._dynamic_trials[tid]["book_time"] = now self.trials._dynamic_trials[tid]["refresh_time"] = now self.trials._dynamic_trials[tid]["state"] = hyperopt.JOB_STATE_DONE self.trials._dynamic_trials[tid]["result"] = result self.trials._dynamic_trials[tid][ "refresh_time"] = hyperopt.utils.coarse_utcnow() self.trials.refresh() def _internal_tell_not_asked(self, candidate: p.Parameter, loss: float) -> None: next_id = self.trials.new_trial_ids(1) new_trial = hyperopt.rand.suggest(next_id, self.domain, self.trials, self._rng.randint(2**31 - 1)) self.trials.insert_trial_docs(new_trial) self.trials.refresh() tid = next_id[0] if self._transform: data = candidate.get_standardized_data( reference=self.parametrization) data = self._transform.forward(data) self.trials._dynamic_trials[tid]["misc"]["vals"] = { f"x_{i}": [data[i]] for i in range(len(data)) } else: null_config: dict = { k: [] for k in self.trials._dynamic_trials[tid]["misc"] ["vals"].keys() } new_vals: dict = _hp_parametrization_to_dict(candidate, default=null_config) self.trials._dynamic_trials[tid]["misc"]["vals"] = new_vals self.trials.refresh() candidate._meta["trial_id"] = tid self._internal_tell_candidate(candidate, loss)
def read_or_recreate_trials(hyperparameter_search_dir, tqdm=None, overwrite=False): config = read_config(hyperparameter_search_dir)[0] filepath = os.path.join(hyperparameter_search_dir, HYP_CONFIG_FILENAME) with open(filepath, mode='r') as f: raw_config = json.load(f) all_params, all_results, all_configs = {}, {}, {} run_names = [r for r in os.listdir(hyperparameter_search_dir) if r != 'trials.pkl'] run_names_rng = run_names if tqdm is None else tqdm(run_names) for run_name in run_names: run_dir = os.path.join(hyperparameter_search_dir, run_name) if not os.path.isdir(run_dir): print(run_dir) continue if os.path.isfile(os.path.join(run_dir, 'error.pkl')): continue config_filepath = os.path.join(run_dir, CONFIG_FILENAME) if not os.path.isfile(config_filepath): continue with open(config_filepath, mode='r') as f: config = json.load(f) all_configs[run_name] = config params_filepath = os.path.join(run_dir, PARAMS_FILENAME) if os.path.isfile(params_filepath): with open(params_filepath, mode='rb') as f: constant, variable = pickle.load(f) all_params[run_name] = constant all_params[run_name].update(variable) else: raise NotImplementedError num_epochs = config['trainer']['epochs'] completed_training = trained_until(run_dir, run_name, num_epochs) if not completed_training: print("Run %s still training (or errored and didn't report)" % run_name) print(run_name, num_epochs) print(os.listdir(run_dir)) continue tuning_results_filename = os.path.join(run_dir, 'ir_metrics_%d.json' % num_epochs) assert os.path.isfile(tuning_results_filename), "Missing tuning results for %s" % run_dir with open(tuning_results_filename, mode='r') as f: all_results[run_name] = json.load(f) trials_filepath = os.path.join(hyperparameter_search_dir, 'trials.pkl') if os.path.exists(trials_filepath) and not overwrite: print("Reloading trials!") with open(trials_filepath, mode='rb') as f: trials = pickle.load(f) return config, all_results, all_configs, all_params, trials # Rebuild Trials # TODO(mmd): Something wrong in misc.idxs... trials = Trials(exp_key = 'exp') #hyperparameter_search_dir for run_name in all_results: configs = all_configs[run_name] params = all_params[run_name] loss = all_results[run_name]['Val (Pert):']['median_rank'] loss_variance, test_loss, test_loss_variance = np.NaN, np.NaN, np.NaN result = { 'status': STATUS_OK, 'loss': loss, 'loss_variance': loss_variance, 'test_loss': test_loss, 'test_loss_variance': test_loss_variance, } spec = params a = trials.insert_trial_doc({ 'tid': run_name, 'spec': spec, 'result': result, 'misc': { 'tid': run_name, 'cmd': '', 'idxs': [], 'vals': {k: [v] for k, v in spec.items()}, }, 'state': JOB_STATE_DONE, 'owner': '', 'book_time': 0, 'refresh_time': 0, 'exp_key': 'exp',# hyperparameter_search_dir, }) trials.refresh() return config, all_results, all_configs, all_params, trials