def protocol_eval(dataset, dataset_path, hp_fname, host, port, learning_algo, int_samples): data_obj = dataset(dataset_path) # -- load hp file hp_space, trials, _ = load_hp(hp_fname, host, port) # -- best trial try: best_trial = trials.best_trial except Exception, e: raise ValueError('problem retrieving best trial: %s' % (e))
def check_hp(hp_fname, host, port, hp_agg_list): hp_space, trials, n_startup_trials = load_hp(hp_fname, host, port) def nested_aggregation(agg_dict, param_nest, trial, nest_idx=0): if nest_idx == len(param_nest): return else: agg_key = param_nest[nest_idx] try: trial_key = trial['hps'][agg_key] except KeyError: return trial_loss = trial['result']['loss'] try: agg_dict[agg_key][trial_key]['count'] += 1 if trial_loss < agg_dict[agg_key][trial_key]['best']: agg_dict[agg_key][trial_key]['best'] = trial_loss except KeyError: if not agg_dict.has_key(agg_key): # -- new aggregation level agg_dict[agg_key] = {} agg_dict[agg_key][trial_key] = {'count': 1, 'best': trial_loss} nested_aggregation(agg_dict[agg_key][trial_key], param_nest, trial, nest_idx + 1) # -- accumulators rdict = { "hp_space": hp_space, "n_startup_trials": n_startup_trials, "ok": { "count": 0, "best": None, "agg": {}, }, "random_trials": { "count": 0, "ok": {}, "fail": {}, "best": None, }, "tpe_trials": { "count": 0, "best": None, "ok": {}, "fail": {}, }, } best_rnd = 1.0 best_tpe = 1.0 # -- transform readable_trials, best_hps = readable_hps(trials, hp_space) keys = np.array(readable_trials.keys()).astype('int') keys.sort() for i_trial, k in enumerate(keys): trial = readable_trials[k] status = trial['result']['status'] if status in ('ok', 'fail'): loss = trial['result']['loss'] if i_trial < n_startup_trials: rnd = rdict['random_trials'] rnd['count'] += 1 nested_aggregation(rnd[status], hp_agg_list, trial) if loss < best_rnd: rnd['best'] = trial best_rnd = loss else: tpe = rdict['tpe_trials'] tpe['count'] += 1 nested_aggregation(tpe[status], hp_agg_list, trial) if loss < best_tpe: tpe['best'] = trial best_tpe = loss if status == 'ok': ok = rdict['ok'] ok['count'] += 1 nested_aggregation(ok['agg'], hp_agg_list, trial) rdict['ok']['best'] = best_hps print json.dumps(rdict, indent=4) print 'done!'
def check_hp(hp_fname, host, port, hp_agg_list): hp_space, trials, n_startup_trials = load_hp(hp_fname, host, port) def nested_aggregation(agg_dict, param_nest, trial, nest_idx=0): if nest_idx == len(param_nest): return else: agg_key = param_nest[nest_idx] try: trial_key = trial['hps'][agg_key] except KeyError: return trial_loss = trial['result']['loss'] try: agg_dict[agg_key][trial_key]['count'] += 1 if trial_loss < agg_dict[agg_key][trial_key]['best']: agg_dict[agg_key][trial_key]['best'] = trial_loss except KeyError: if not agg_dict.has_key(agg_key): # -- new aggregation level agg_dict[agg_key] = {} agg_dict[agg_key][trial_key] = {'count': 1, 'best': trial_loss} nested_aggregation(agg_dict[agg_key][trial_key], param_nest, trial, nest_idx + 1) # -- accumulators rdict = {"hp_space": hp_space, "n_startup_trials": n_startup_trials, "ok": { "count": 0, "best": None, "agg": {}, }, "random_trials": { "count": 0, "ok": {}, "fail": {}, "best": None, }, "tpe_trials": { "count": 0, "best": None, "ok": {}, "fail": {}, }, } best_rnd = 1.0 best_tpe = 1.0 # -- transform readable_trials, best_hps = readable_hps(trials, hp_space) keys = np.array(readable_trials.keys()).astype('int') keys.sort() for i_trial, k in enumerate(keys): trial = readable_trials[k] status = trial['result']['status'] if status in ('ok', 'fail'): loss = trial['result']['loss'] if i_trial < n_startup_trials: rnd = rdict['random_trials'] rnd['count'] += 1 nested_aggregation(rnd[status], hp_agg_list, trial) if loss < best_rnd: rnd['best'] = trial best_rnd = loss else: tpe = rdict['tpe_trials'] tpe['count'] += 1 nested_aggregation(tpe[status], hp_agg_list, trial) if loss < best_tpe: tpe['best'] = trial best_tpe = loss if status == 'ok': ok = rdict['ok'] ok['count'] += 1 nested_aggregation(ok['agg'], hp_agg_list, trial) rdict['ok']['best'] = best_hps print json.dumps(rdict, indent=4) print 'done!'