def generatePlotTTA(ax, exp_paths, bounds):
for exp_path in exp_paths:
exp = loadExperiment(exp_path)
results = loadResults(exp, errorfile)
const, unconst = tee(results)
color = colors[exp.agent]
label = exp.agent
if error == 'rmsve':
rmspbe = loadResults(exp, 'rmspbe_summary.npy')
rmspbe_unconst, rmspbe_const = tee(rmspbe)
rmspbe_const = whereParameterGreaterEq(rmspbe_const, 'ratio', 1)
best_rmspbe_unconst = getBest(rmspbe_unconst)
best_rmspbe_const = getBest(rmspbe_const)
best_unconst = find(unconst, best_rmspbe_unconst)
best_const = find(const, best_rmspbe_const)
elif error == 'rmspbe':
const = whereParameterGreaterEq(const, 'ratio', 1)
best_unconst = getBest(unconst)
best_const = getBest(const)
b = plotBest(best_unconst, ax, label=label + '_unc', color=color, dashed=True)
bounds.append(b)
b = plotBest(best_const, ax, label=label, color=color, dashed=False)
bounds.append(b)
def generatePlot(exp_paths):
ax = plt.gca()
# ax.semilogx()
for exp_path in exp_paths:
exp = loadExperiment(exp_path)
rmsve = loadResults(exp, 'errors_summary.npy')
rmspbe = loadResults(exp, 'rmspbe_summary.npy')
# if exp.agent == 'TDadagrad':
# continue
# best PBE using AUC
best = getBest(rmspbe)
best_rmsve = find(rmsve, best)
use_ideal_h = exp._d['metaParameters'].get('use_ideal_h', False)
dashed = use_ideal_h
color = colors[exp.agent]
label = exp.agent.replace('adagrad', '')
if use_ideal_h:
label += '-h*'
plotBest(best_rmsve, ax, label=label, color=color, dashed=dashed)
# plt.show()
save(exp, f'rmsve_over_rmspbe', type='svg')
plt.clf()
def generatePlot(exp_paths):
ax = plt.gca()
# ax.semilogx()
for exp_path in exp_paths:
exp = loadExperiment(exp_path)
if exp.agent == 'TDadagrad':
continue
use_ideal_h = exp._d['metaParameters'].get('use_ideal_h', False)
dashed = use_ideal_h
color = colors[exp.agent]
# load the errors and hnorm files
errors = loadResults(exp, 'errors_summary.npy')
results = loadResults(exp, 'ndh_summary.npy')
# choose the best parameters from the _errors_
best = getBestEnd(errors)
best_ndh = find(results, best)
label = exp.agent.replace('adagrad', '')
if use_ideal_h:
label += '-h*'
plotBest(best_ndh, ax, label=label, color=color, dashed=dashed)
# plt.show()
save(exp, f'norm_delta-hat')
plt.clf()
def generatePlotTTA(ax, exp_path, bounds):
exp = loadExperiment(exp_path)
results = loadResults(exp, errorfile)
const, unconst = tee(results)
color = colors[exp.agent]
label = exp.agent
const = whereParameterGreaterEq(const, 'ratio', 1)
if 'ReghTDC' in label:
const = whereParameterEquals(const, 'reg_h', 0.8)
best_const = getBest(const, bestBy=bestBy)
best_unconst = getBest(unconst, bestBy=bestBy)
if show_unconst and best_const != best_unconst:
b = plotBest(best_unconst,
ax,
window=window,
smoothing=smoothing,
label=label + '_unc',
color=color,
alpha=0.2,
dashed=True)
bounds.append(b)
b = plotBest(best_const,
ax,
window=window,
smoothing=smoothing,
label=label,
color=color,
alpha=0.2,
dashed=False)
bounds.append(b)
def generatePlot(ax, exp_paths, bounds):
for exp_path in exp_paths:
exp = ExperimentModel.load(exp_path)
raise Exception(
'Set the name of the results file saved from these experiments')
results = loadResults(exp, 'results.npy')
results = whereParameterEquals(results, 'initial_value', 0)
best = getBest(results)
b = plotBest(best, ax, label='TD', color='yellow', dashed=False)
bounds.append(b)
def generatePlot(exp_paths):
ax = plt.gca()
# ax.semilogx()
for exp_path in exp_paths:
exp = loadExperiment(exp_path)
# load the errors and hnorm files
errors = loadResults(exp, 'errors_summary.npy')
results = loadResults(exp, 'hnorm_summary.npy')
# choose the best parameters from the _errors_
best = getBestEnd(errors)
best_hnorm = find(results, best)
label = fileName(exp_path).replace('.json', '')
plotBest(best_hnorm, ax, label=label)
plt.show()
# save(exp, f'learning-curve')
plt.clf()
def generatePlot(ax, exp_paths, bounds):
for exp_path in exp_paths:
exp = loadExperiment(exp_path)
results = loadResults(exp, errorfile)
color = colors[exp.agent]
label = exp.agent
results = whereParameterEquals(results, 'batch_size', 4)
results = whereParameterLesserEq(results, 'ratio', 8)
results = whereParameterLesserEq(results, 'alpha', 0.5)
results = where(results, lambda r: r.params.get('ratio', 1) * r.params['alpha'] <= 1)
if 'ReghTDC' in label:
results = whereParameterEquals(results, 'reg_h', 1)
results = whereParameterEquals(results, 'ratio', 1)
elif 'TDRCC' in label:
results = whereParameterEquals(results, 'reg_h', 0.8)
results = whereParameterEquals(results, 'ratio', 1)
elif 'TDC' in label:
results = whereParameterGreaterEq(results, 'ratio', 1)
left, right = tee(results)
best_line = getBest(right).mean()
best = np.mean(best_line)
for result in left:
# print(label, result.params)
shade = 0.12
line = result.mean()
if np.mean(line) == best:
shade = 1
plotBest(result, ax, label=label, color=color, alphaMain=shade, dashed=False)
bounds.append(best_line[0])
def generatePlot(exp_path):
ax = plt.gca()
# ax.semilogx()
exp = loadExperiment(exp_path)
# load the errors and hnorm files
errors = loadResults(exp, 'errors_summary.npy')
results = loadResults(exp, 'stepsize_summary.npy')
# choose the best parameters from the _errors_
best = getBestEnd(errors)
best_ss = find(results, best)
alg = exp.agent.replace('adagrad', '')
plotBest(best_ss, ax, label=['w', 'h'])
ax.set_ylim([0, 4])
print(alg)
# plt.show()
save(exp, f'stepsizes-{alg}')
plt.clf()
def generatePlot(ax, exp_paths, bounds):
for exp_path in exp_paths:
exp = ExperimentModel.load(exp_path)
results = loadResults(exp, 'return_summary.npy')
# optionally force epsilon to be 0.15
# results = whereParameterEquals(results, 'epsilon', 0.15)
best = getBest(results)
print('best parameters:', exp_path)
print(best.params)
alg = exp.agent
b = plotBest(best, ax, label=alg, color=colors[alg], dashed=False)
bounds.append(b)
def generatePlot(ax, exp_paths, bounds):
for exp_path in exp_paths:
exp = loadExperiment(exp_path)
variance = loadResults(exp, 'expupd_summary.npy')
agent = exp.agent
color = colors[agent]
label = agent
best = getBest(variance)
if 'GTD2' in agent or 'TDC' in agent:
best.reducer(lambda m: m[:, 1])
else:
best.reducer(lambda m: m[:, 1])
b = plotBest(best, ax, color=color, label=label)
bounds.append(b)
def generatePlot(ax, exp_paths, bounds):
for exp_path in exp_paths:
exp = ExperimentModel.load(exp_path)
results = loadResults(exp, 'return_summary.npy')
results = whereParameterEquals(results, 'epsilon', 0.05)
param_values = splitOverParameter(results, PARAM)
for key, param_results in param_values.items():
best = getBest(param_results)
print('best parameters:', exp_path, f'{PARAM}={key}')
print(best.params)
alg = exp.agent
b = plotBest(best, ax, label=alg + f' {PARAM}={key}', dashed=False)
bounds.append(b)
def generatePlot(ax, exp_path, bounds):
exp = loadExperiment(exp_path)
results = loadResults(exp, 'variance_summary.npy')
best = getBest(results)
best.reducer(lambda m: m[:, 0])
color = colors[exp.agent]
label = exp.agent
b = plotBest(best,
ax,
window=window,
smoothing=smoothing,
label=label,
color=color,
dashed=False)
bounds.append(b)
def generatePlotSSA(ax, exp_paths, bounds):
for exp_path in exp_paths:
if 'amsgrad' in exp_path:
continue
exp = loadExperiment(exp_path)
results = loadResults(exp, errorfile)
stepsizes = loadResults(exp, 'stepsize_summary.npy')
color = colors[exp.agent]
label = exp.agent
best_error = getBest(results)
best = find(stepsizes, best_error)
b = plotBest(best, ax, label=label, color=color, dashed=False)
bounds.append(b)
def generatePlotSSA(ax, exp_paths, bounds):
for exp_path in exp_paths:
exp = loadExperiment(exp_path)
results = loadResults(exp, errorfile)
color = colors[exp.agent]
label = exp.agent
if error == 'rmsve':
rmspbe = loadResults(exp, 'rmspbe_summary.npy')
best_rmspbe = getBest(rmspbe)
best = find(results, best_rmspbe)
elif error == 'rmspbe':
best = getBest(results)
b = plotBest(best, ax, label=label, color=color, dashed=False)
bounds.append(b)
def generatePlotTTA(ax, exp_paths, bounds):
for exp_path in exp_paths:
if 'amsgrad' in exp_path:
continue
exp = loadExperiment(exp_path)
results = loadResults(exp, errorfile)
stepsizes = loadResults(exp, 'stepsize_summary.npy')
results = whereParameterEquals(results, 'ratio', 1)
color = colors[exp.agent]
label = exp.agent
best_error = getBest(results)
best_stepsize = find(stepsizes, best_error)
b = plotBest(best_stepsize,
ax,
label=[label + '_w', label + '_h'],
color=color,
dashed=[False, True])
bounds.append(b)
if __name__ == "__main__":
ax = plt.gca()
f = plt.gcf()
bounds = []
if lstd_baseline:
path = f'experiments/batch/{problem}/lstd.json'
lstd_exp = loadExperiment(path)
LSTD_res = loadResults(lstd_exp, errorfile)
LSTD_best = getBest(LSTD_res)
b = plotBest(LSTD_best, ax, window=window, color=colors['LSTD'], label='LSTD', alphaMain=0.5, dashed=True)
bounds.append(b)
for alg in algorithms:
if stepsize == 'constant':
exp_path = f'experiments/batch/{problem}/{alg}/{alg}.json'
else:
exp_path = f'experiments/batch/{problem}/{alg}/{alg}{stepsize}.json'
if '_h' in alg or alg == 'td' or alg == 'vtrace':
generatePlotSSA(ax, exp_path, bounds)
else:
generatePlotTTA(ax, exp_path, bounds)
lower = min(map(lambda x: x[0], bounds)) * 0.9
upper = max(map(lambda x: x[1], bounds)) * 1.05
def generatePlot(exp_paths):
f, axes = plt.subplots(2, 2)
# get LSTD solution
path = up(up(first(exp_paths))) + '/lstd.json'
exp = loadExperiment(path)
LSTD_rmsve_results = loadResults(exp, 'errors_summary.npy')
LSTD_rmspbe_results = loadResults(exp, 'rmspbe_summary.npy')
LSTD_rmsve = metric(LSTD_rmsve_results)
LSTD_rmspbe = metric(LSTD_rmspbe_results)
rmspbe_bounds = []
rmsve_bounds = []
bounds = plotBest(LSTD_rmspbe,
axes[0, 0],
color=colors['LSTD'],
label='LSTD',
alphaMain=0.5)
rmspbe_bounds.append(bounds)
bounds = plotBest(LSTD_rmspbe,
axes[0, 1],
color=colors['LSTD'],
label='LSTD',
alphaMain=0.5)
rmspbe_bounds.append(bounds)
bounds = plotBest(LSTD_rmsve,
axes[1, 0],
color=colors['LSTD'],
label='LSTD',
alphaMain=0.5)
rmsve_bounds.append(bounds)
bounds = plotBest(LSTD_rmsve,
axes[1, 1],
color=colors['LSTD'],
label='LSTD',
alphaMain=0.5)
rmsve_bounds.append(bounds)
# RMSPBE plots
ax = axes[0, 0]
for exp_path in exp_paths:
exp = loadExperiment(exp_path)
results = loadResults(exp, 'rmspbe_summary.npy')
const, unconst = tee(results)
const = whereParameterGreaterEq(const, 'ratio', 1)
use_ideal_h = exp._d['metaParameters'].get('use_ideal_h', False)
agent = exp.agent
if 'SmoothTDC' in agent:
average = exp._d['metaParameters']['averageType']
agent += '_' + average
color = colors[agent]
label = agent.replace('adagrad', '')
if not (exp.agent in ['TDadagrad', 'TDschedule', 'TD', 'TDamsgrad']
or use_ideal_h):
if UNCONSTRAINED:
bounds = plot(unconst,
ax,
label=label + '_unc',
color=color,
dashed=True,
bestBy=bestBy)
rmspbe_bounds.append(bounds)
bounds = plot(const,
ax,
label=label,
color=color,
dashed=False,
bestBy=bestBy)
rmspbe_bounds.append(bounds)
else:
bounds = plot(unconst,
ax,
label=label,
color=color,
dashed=False,
bestBy=bestBy)
rmspbe_bounds.append(bounds)
ax.set_ylabel("MSPBE")
ax.set_title("MSPBE")
ax = axes[0, 1]
for exp_path in exp_paths:
exp = loadExperiment(exp_path)
rmsve = loadResults(exp, 'errors_summary.npy')
results = loadResults(exp, 'rmspbe_summary.npy')
const, unconst = tee(rmsve)
const_res, unconst_res = tee(results)
const = whereParameterGreaterEq(const, 'ratio', 1)
use_ideal_h = exp._d['metaParameters'].get('use_ideal_h', False)
agent = exp.agent
if 'SmoothTDC' in agent:
average = exp._d['metaParameters']['averageType']
agent += '_' + average
color = colors[agent]
label = agent.replace('adagrad', '')
if not (exp.agent in ['TDadagrad', 'TDschedule', 'TD', 'TDamsgrad']
or use_ideal_h):
best = metric(const)
best_unc = metric(unconst)
best_rmspbe = find(const_res, best)
best_rmspbe_unc = find(unconst_res, best_unc)
bounds = plotBest(best_rmspbe,
ax,
label=label,
color=color,
dashed=False)
rmspbe_bounds.append(bounds)
if UNCONSTRAINED:
bounds = plotBest(best_rmspbe_unc,
ax,
label=label + '_unc',
color=color,
dashed=True)
rmspbe_bounds.append(bounds)
else:
best = metric(unconst)
best_rmspbe = find(unconst_res, best)
bounds = plotBest(best_rmspbe,
ax,
label=label,
color=color,
dashed=False)
rmspbe_bounds.append(bounds)
ax.set_title("MSVE")
# RMSVE plots
ax = axes[1, 0]
for exp_path in exp_paths:
exp = loadExperiment(exp_path)
rmsve = loadResults(exp, 'errors_summary.npy')
rmspbe = loadResults(exp, 'rmspbe_summary.npy')
const, unconst = tee(rmspbe)
const_res, unconst_res = tee(rmsve)
const = whereParameterGreaterEq(const, 'ratio', 1)
use_ideal_h = exp._d['metaParameters'].get('use_ideal_h', False)
agent = exp.agent
if 'SmoothTDC' in agent:
average = exp._d['metaParameters']['averageType']
agent += '_' + average
color = colors[agent]
label = agent.replace('adagrad', '')
if not (exp.agent in ['TDadagrad', 'TDschedule', 'TD', 'TDamsgrad']
or use_ideal_h):
# best PBE using AUC
best = metric(const)
best_unc = metric(unconst)
best_rmsve = find(const_res, best)
best_rmsve_unc = find(unconst_res, best_unc)
print('rmsve_over_rmspbe')
print(label, best_rmsve.params)
print(label, best_rmsve_unc.params)
bounds = plotBest(best_rmsve,
ax,
label=label,
color=color,
dashed=False)
rmsve_bounds.append(bounds)
if UNCONSTRAINED:
bounds = plotBest(best_rmsve_unc,
ax,
label=label + '_unc',
color=color,
dashed=True)
rmsve_bounds.append(bounds)
else:
best = metric(unconst)
best_rmsve = find(unconst_res, best)
bounds = plotBest(best_rmsve,
ax,
label=label,
color=color,
dashed=False)
rmsve_bounds.append(bounds)
ax.set_ylabel("MSVE")
ax = axes[1, 1]
for exp_path in exp_paths:
exp = loadExperiment(exp_path)
results = loadResults(exp)
const, unconst = tee(results)
const = whereParameterGreaterEq(const, 'ratio', 1)
use_ideal_h = exp._d['metaParameters'].get('use_ideal_h', False)
agent = exp.agent
if 'SmoothTDC' in agent:
average = exp._d['metaParameters']['averageType']
agent += '_' + average
color = colors[agent]
label = agent.replace('adagrad', '')
if not (exp.agent in ['TDadagrad', 'TDschedule', 'TD', 'TDamsgrad']
or use_ideal_h):
bounds = plot(const,
ax,
label=label,
color=color,
dashed=False,
bestBy=bestBy)
rmsve_bounds.append(bounds)
if UNCONSTRAINED:
bounds = plot(unconst,
ax,
label=label + '_unc',
color=color,
dashed=True,
bestBy=bestBy)
rmsve_bounds.append(bounds)
else:
bounds = plot(unconst,
ax,
label=label,
color=color,
dashed=False,
bestBy=bestBy)
rmsve_bounds.append(bounds)
# rmspbe
rmspbe_lower = min(map(lambda x: x[0], rmspbe_bounds)) * 0.9
rmspbe_upper = max(map(lambda x: x[1], rmspbe_bounds)) * 1.05
if rmspbe_lower < 0.01:
rmspbe_lower = -0.01
axes[0, 0].set_ylim([rmspbe_lower, rmspbe_upper])
axes[0, 1].set_ylim([rmspbe_lower, rmspbe_upper])
# rmsve
rmsve_lower = min(map(lambda x: x[0], rmsve_bounds)) * 0.9
rmsve_upper = max(map(lambda x: x[1], rmsve_bounds)) * 1.05
if rmsve_lower < 0.01:
rmsve_lower = -0.01
axes[1, 0].set_ylim([0, 20])
axes[1, 1].set_ylim([0, 20])
def generatePlot(exp_paths):
f, axes = plt.subplots(2, 2)
# ax.semilogx()
# RMSPBE plots
ax = axes[0, 0]
rmspbe_bounds = []
for exp_path in exp_paths:
exp = loadExperiment(exp_path)
results = loadResults(exp, 'rmspbe_summary.npy')
use_ideal_h = exp._d['metaParameters'].get('use_ideal_h', False)
dashed = False
color = colors[exp.agent]
label = exp.agent.replace('adagrad', '')
# if use_ideal_h:
# label += '-h*'
bounds = plot(results, ax, label=label, color=color, dashed=dashed)
rmspbe_bounds.append(bounds)
ax.set_ylabel("RMSPBE")
ax.set_title("RMSPBE")
ax = axes[0, 1]
for exp_path in exp_paths:
exp = loadExperiment(exp_path)
rmsve = loadResults(exp, 'errors_summary.npy')
results = loadResults(exp, 'rmspbe_summary.npy')
best = getBestEnd(rmsve)
best_rmspbe = find(results, best)
use_ideal_h = exp._d['metaParameters'].get('use_ideal_h', False)
dashed = False
color = colors[exp.agent]
label = exp.agent.replace('adagrad', '')
# if use_ideal_h:
# label += '-h*'
bounds = plotBest(best_rmspbe,
ax,
label=label,
color=color,
dashed=dashed)
rmspbe_bounds.append(bounds)
ax.set_title("RMSVE")
# RMSVE plots
ax = axes[1, 0]
rmsve_bounds = []
for exp_path in exp_paths:
exp = loadExperiment(exp_path)
rmsve = loadResults(exp, 'errors_summary.npy')
rmspbe = loadResults(exp, 'rmspbe_summary.npy')
# if exp.agent == 'TDadagrad':
# continue
# best PBE using AUC
best = getBestEnd(rmspbe)
best_rmsve = find(rmsve, best)
use_ideal_h = exp._d['metaParameters'].get('use_ideal_h', False)
dashed = False
color = colors[exp.agent]
label = exp.agent.replace('adagrad', '')
# if use_ideal_h:
# label += '-h*'
bounds = plotBest(best_rmsve,
ax,
label=label,
color=color,
dashed=dashed)
rmsve_bounds.append(bounds)
ax.set_ylabel("RMSVE")
ax = axes[1, 1]
for exp_path in exp_paths:
exp = loadExperiment(exp_path)
results = loadResults(exp)
use_ideal_h = exp._d['metaParameters'].get('use_ideal_h', False)
dashed = False
color = colors[exp.agent]
label = exp.agent.replace('adagrad', '')
# if use_ideal_h:
# label += '-h*'
bounds = plot(results, ax, label=label, color=color, dashed=dashed)
rmsve_bounds.append(bounds)
# rmspbe
rmspbe_lower = min(map(lambda x: x[0], rmspbe_bounds)) * 0.9
rmspbe_upper = max(map(lambda x: x[1], rmspbe_bounds)) * 1.05
axes[0, 0].set_ylim([rmspbe_lower, rmspbe_upper])
axes[0, 1].set_ylim([rmspbe_lower, rmspbe_upper])
# rmsve
rmsve_lower = min(map(lambda x: x[0], rmsve_bounds)) * 0.9
rmsve_upper = max(map(lambda x: x[1], rmsve_bounds)) * 1.05
axes[1, 0].set_ylim([rmsve_lower, rmsve_upper])
axes[1, 1].set_ylim([rmsve_lower, rmsve_upper])
plt.show()
best = getBest(results)
all_results.append(best)
if best_result is None:
best_result = best
elif np.mean(best.mean()) < np.mean(best_result.mean()):
best_result = best
return all_results, best_result
tdc_results, best_tdc = getResultsAndBest(tdc)
gtd2_results, best_gtd2 = getResultsAndBest(gtd2)
for r in tdc_results:
if r == best_tdc:
plotBest(r, ax, 'blue', r.exp.agent, stderr=False)
else:
plotBest(r, ax, 'blue', '_', alphaMain=0.15, stderr=False)
for r in gtd2_results:
if r == best_gtd2:
plotBest(r, ax, 'red', r.exp.agent, stderr=False)
else:
plotBest(r, ax, 'red', '_', alphaMain=0.15, stderr=False)
# save(exp, 'lambda')
# ax.set_ylim([0, 1])
plt.show()
for i, alg in enumerate(algorithms):
for j, problem in enumerate(problems):
bounds = []
exp_paths = glob.glob(f'experiments/stepsizes/{problem}/{alg}/*.json')
if len(exp_paths) == 0:
continue
exp = loadExperiment(exp_paths[0])
path = up(up(first(exp_paths))) + '/lstd.json'
lstd_exp = loadExperiment(path)
LSTD_res = loadResults(lstd_exp, errorfile)
LSTD_best = getBest(LSTD_res)
b = plotBest(LSTD_best, axes[i, j], color=colors['LSTD'], label='LSTD', alphaMain=0.5)
bounds.append(b)
if '_h' in alg or alg == 'td':
generatePlotSSA(axes[i, j], exp_paths, bounds)
else:
generatePlotTTA(axes[i, j], exp_paths, bounds)
lower = min(map(lambda x: x[0], bounds)) * 0.9
upper = max(map(lambda x: x[1], bounds)) * 1.05
if lower < 0.01:
lower = -0.01
if i == 0:
axes[i, j].set_title(f'{problem}\n{alg}')
