def get_last_eval_vals(opt, vis_opt, eval_key, algo, env_name, num_trials,
trial_offset, bin_size):
# For each trial
eval_vals = []
for trial in range(trial_offset, trial_offset + num_trials):
# Get the logpath
logpath = os.path.join(opt['logs']['log_base'], opt['model']['mode'],
opt['logs']['exp_name'], algo, env_name,
'trial%d' % trial)
if not os.path.isdir(logpath):
return None
# Create the dashboard object
opt['env']['env-name'] = env_name
opt['alg'] = opt['alg_%s' % algo]
opt['optim'] = opt['optim_%s' % algo]
opt['alg']['algo'] = algo
opt['trial'] = trial
dash = Dashboard(opt, vis_opt, logpath, vis=False)
# Get data
try:
dash.preload_data()
raw_x, raw_y = dash.load_data('episode_monitor', 'scalar',
eval_key)
except Exception:
return None
# Get data from last bin
if not (len(raw_y) > bin_size):
return None
raw_vals = raw_y[-bin_size:]
assert (len(raw_vals) == bin_size)
raw_vals = [float(v) for v in raw_vals]
raw_val = np.mean(raw_vals)
eval_vals.append(raw_val)
# Return
return eval_vals
def get_eval_vals(opt,
vis_opt,
eval_key,
algo,
env_name,
num_trials,
trial_offset,
bin_size,
smooth,
mode='minmax'):
# For each trial
x_curves = []
y_curves = []
for trial in range(trial_offset, trial_offset + num_trials):
# Get the logpath
logpath = os.path.join(opt['logs']['log_base'], opt['model']['mode'],
opt['logs']['exp_name'], algo, env_name,
'trial%d' % trial)
print(logpath)
assert (os.path.isdir(logpath))
# Create the dashboard object
opt['env']['env-name'] = env_name
opt['alg'] = opt['alg_%s' % algo]
opt['optim'] = opt['optim_%s' % algo]
opt['alg']['algo'] = algo
opt['trial'] = trial
dash = Dashboard(opt, vis_opt, logpath, vis=True)
# Get data
dash.preload_data()
x, y = dash.load_data('episode_monitor', 'scalar', eval_key)
x = [float(i) for i in x]
y = [float(i.replace('\x00', '')) for i in y]
# Smooth and bin
if smooth == 1:
x, y = dash.smooth_curve(x, y)
elif smooth == 2:
y = medfilt(y, kernel_size=9)
x, y = dash.fix_point(x, y, bin_size)
# Append
x_curves.append(x)
y_curves.append(y)
# Interpolate the curves
# Get the combined list of all x values
union = set([])
for x_curve in x_curves:
union = union | set(x_curve)
all_x = sorted(list(union))
# Get interpolated y values of each list
interp_y_curves = []
for x_curve, y_curve in zip(x_curves, y_curves):
interp_y = np.interp(all_x, x_curve, y_curve)
interp_y_curves.append(interp_y)
# Get mean and variance curves
mean = np.mean(interp_y_curves, axis=0)
y_middle = mean
if mode == 'all':
y_top = interp_y_curves
y_bottom = None
elif mode == 'minmax':
y_bottom = np.min(interp_y_curves, axis=0)
y_top = np.max(interp_y_curves, axis=0)
elif mode == 'variance':
var = np.var(interp_y_curves, axis=0)
y_bottom = mean - var
y_top = mean + var
elif mode == 'std':
std = np.std(interp_y_curves, axis=0)
y_bottom = mean - std
y_top = mean + std
# Return
return np.array(all_x), y_middle, y_top, y_bottom