def protocol_eval(dataset, dataset_path, hp_fname, host, port,
learning_algo, int_samples):
data_obj = dataset(dataset_path)
# -- load hp file
hp_space, trials, _ = load_hp(hp_fname, host, port)
# -- best trial
try:
best_trial = trials.best_trial
except Exception, e:
raise ValueError('problem retrieving best trial: %s' % (e))
def check_hp(hp_fname, host, port, hp_agg_list):
hp_space, trials, n_startup_trials = load_hp(hp_fname, host, port)
def nested_aggregation(agg_dict, param_nest, trial, nest_idx=0):
if nest_idx == len(param_nest):
return
else:
agg_key = param_nest[nest_idx]
try:
trial_key = trial['hps'][agg_key]
except KeyError:
return
trial_loss = trial['result']['loss']
try:
agg_dict[agg_key][trial_key]['count'] += 1
if trial_loss < agg_dict[agg_key][trial_key]['best']:
agg_dict[agg_key][trial_key]['best'] = trial_loss
except KeyError:
if not agg_dict.has_key(agg_key):
# -- new aggregation level
agg_dict[agg_key] = {}
agg_dict[agg_key][trial_key] = {'count': 1, 'best': trial_loss}
nested_aggregation(agg_dict[agg_key][trial_key], param_nest, trial,
nest_idx + 1)
# -- accumulators
rdict = {
"hp_space": hp_space,
"n_startup_trials": n_startup_trials,
"ok": {
"count": 0,
"best": None,
"agg": {},
},
"random_trials": {
"count": 0,
"ok": {},
"fail": {},
"best": None,
},
"tpe_trials": {
"count": 0,
"best": None,
"ok": {},
"fail": {},
},
}
best_rnd = 1.0
best_tpe = 1.0
# -- transform
readable_trials, best_hps = readable_hps(trials, hp_space)
keys = np.array(readable_trials.keys()).astype('int')
keys.sort()
for i_trial, k in enumerate(keys):
trial = readable_trials[k]
status = trial['result']['status']
if status in ('ok', 'fail'):
loss = trial['result']['loss']
if i_trial < n_startup_trials:
rnd = rdict['random_trials']
rnd['count'] += 1
nested_aggregation(rnd[status], hp_agg_list, trial)
if loss < best_rnd:
rnd['best'] = trial
best_rnd = loss
else:
tpe = rdict['tpe_trials']
tpe['count'] += 1
nested_aggregation(tpe[status], hp_agg_list, trial)
if loss < best_tpe:
tpe['best'] = trial
best_tpe = loss
if status == 'ok':
ok = rdict['ok']
ok['count'] += 1
nested_aggregation(ok['agg'], hp_agg_list, trial)
rdict['ok']['best'] = best_hps
print json.dumps(rdict, indent=4)
print 'done!'
def check_hp(hp_fname, host, port, hp_agg_list):
hp_space, trials, n_startup_trials = load_hp(hp_fname, host, port)
def nested_aggregation(agg_dict, param_nest, trial, nest_idx=0):
if nest_idx == len(param_nest):
return
else:
agg_key = param_nest[nest_idx]
try:
trial_key = trial['hps'][agg_key]
except KeyError:
return
trial_loss = trial['result']['loss']
try:
agg_dict[agg_key][trial_key]['count'] += 1
if trial_loss < agg_dict[agg_key][trial_key]['best']:
agg_dict[agg_key][trial_key]['best'] = trial_loss
except KeyError:
if not agg_dict.has_key(agg_key):
# -- new aggregation level
agg_dict[agg_key] = {}
agg_dict[agg_key][trial_key] = {'count': 1,
'best': trial_loss}
nested_aggregation(agg_dict[agg_key][trial_key], param_nest,
trial, nest_idx + 1)
# -- accumulators
rdict = {"hp_space": hp_space,
"n_startup_trials": n_startup_trials,
"ok": {
"count": 0,
"best": None,
"agg": {},
},
"random_trials": {
"count": 0,
"ok": {},
"fail": {},
"best": None,
},
"tpe_trials": {
"count": 0,
"best": None,
"ok": {},
"fail": {},
},
}
best_rnd = 1.0
best_tpe = 1.0
# -- transform
readable_trials, best_hps = readable_hps(trials, hp_space)
keys = np.array(readable_trials.keys()).astype('int')
keys.sort()
for i_trial, k in enumerate(keys):
trial = readable_trials[k]
status = trial['result']['status']
if status in ('ok', 'fail'):
loss = trial['result']['loss']
if i_trial < n_startup_trials:
rnd = rdict['random_trials']
rnd['count'] += 1
nested_aggregation(rnd[status], hp_agg_list, trial)
if loss < best_rnd:
rnd['best'] = trial
best_rnd = loss
else:
tpe = rdict['tpe_trials']
tpe['count'] += 1
nested_aggregation(tpe[status], hp_agg_list, trial)
if loss < best_tpe:
tpe['best'] = trial
best_tpe = loss
if status == 'ok':
ok = rdict['ok']
ok['count'] += 1
nested_aggregation(ok['agg'], hp_agg_list, trial)
rdict['ok']['best'] = best_hps
print json.dumps(rdict, indent=4)
print 'done!'
