def generatePlotTTA(ax, exp_path, bounds):
exp = loadExperiment(exp_path)
results = loadResults(exp, errorfile)
const, unconst = tee(results)
color = colors[exp.agent]
label = rename(exp.agent)
if 'ReghTDC' in exp.agent:
const = whereParameterEquals(const, 'ratio', 1)
const = whereParameterEquals(const, 'reg_h', 1)
elif 'TDRCC' in exp.agent:
const = whereParameterEquals(const, 'ratio', 1)
# const = whereParameterEquals(const, 'reg_h', 0.8)
const = whereParameterGreaterEq(const, 'reg_h', 0.01)
elif 'TDC' in exp.agent:
const = whereParameterGreaterEq(const, 'ratio', 1)
if show_unconst:
b = plotSensitivity(unconst, param, ax, stderr=stderr, color=color, label=label + '_unc', bestBy=bestBy, dashed=True)
bounds.append(b)
b = plotSensitivity(const, param, ax, stderr=stderr, color=color, label=label, bestBy=bestBy)
bounds.append(b)
def generatePlotTTA(ax, exp_paths, bounds):
for exp_path in exp_paths:
exp = loadExperiment(exp_path)
results = loadResults(exp, errorfile)
const, unconst = tee(results)
color = colors[exp.agent]
label = exp.agent
if error == 'rmsve':
rmspbe = loadResults(exp, 'rmspbe_summary.npy')
rmspbe_unconst, rmspbe_const = tee(rmspbe)
rmspbe_const = whereParameterGreaterEq(rmspbe_const, 'ratio', 1)
best_rmspbe_unconst = getBest(rmspbe_unconst)
best_rmspbe_const = getBest(rmspbe_const)
best_unconst = find(unconst, best_rmspbe_unconst)
best_const = find(const, best_rmspbe_const)
elif error == 'rmspbe':
const = whereParameterGreaterEq(const, 'ratio', 1)
best_unconst = getBest(unconst)
best_const = getBest(const)
b = plotBest(best_unconst, ax, label=label + '_unc', color=color, dashed=True)
bounds.append(b)
b = plotBest(best_const, ax, label=label, color=color, dashed=False)
bounds.append(b)
def generatePlotTTA(ax, exp_path, bounds):
exp = loadExperiment(exp_path)
results = loadResults(exp, errorfile)
const, unconst = tee(results)
color = colors[exp.agent]
label = exp.agent
const = whereParameterGreaterEq(const, 'ratio', 1)
if 'ReghTDC' in label:
const = whereParameterEquals(const, 'reg_h', 0.8)
best_const = getBest(const, bestBy=bestBy)
best_unconst = getBest(unconst, bestBy=bestBy)
if show_unconst and best_const != best_unconst:
b = plotBest(best_unconst,
ax,
window=window,
smoothing=smoothing,
label=label + '_unc',
color=color,
alpha=0.2,
dashed=True)
bounds.append(b)
b = plotBest(best_const,
ax,
window=window,
smoothing=smoothing,
label=label,
color=color,
alpha=0.2,
dashed=False)
bounds.append(b)
def generatePlotTTA(ax, exp_paths, bestBy, bounds):
for exp_path in exp_paths:
exp = loadExperiment(exp_path)
results = loadResults(exp, errorfile)
color = colors[exp.agent]
label = exp.agent
dashed = False
if 'TDC' in label:
dashed = True
results = whereParameterGreaterEq(results, 'ratio', 1)
b = plot(results, ax, label=label, color=color, dashed=dashed, bestBy=bestBy)
bounds.append(b)
def generatePlot(ax, exp_paths, bounds):
for exp_path in exp_paths:
exp = loadExperiment(exp_path)
results = loadResults(exp, errorfile)
color = colors[exp.agent]
label = exp.agent
results = whereParameterEquals(results, 'batch_size', 4)
results = whereParameterLesserEq(results, 'ratio', 8)
results = whereParameterLesserEq(results, 'alpha', 0.5)
results = where(results, lambda r: r.params.get('ratio', 1) * r.params['alpha'] <= 1)
if 'ReghTDC' in label:
results = whereParameterEquals(results, 'reg_h', 1)
results = whereParameterEquals(results, 'ratio', 1)
elif 'TDRCC' in label:
results = whereParameterEquals(results, 'reg_h', 0.8)
results = whereParameterEquals(results, 'ratio', 1)
elif 'TDC' in label:
results = whereParameterGreaterEq(results, 'ratio', 1)
left, right = tee(results)
best_line = getBest(right).mean()
best = np.mean(best_line)
for result in left:
# print(label, result.params)
shade = 0.12
line = result.mean()
if np.mean(line) == best:
shade = 1
plotBest(result, ax, label=label, color=color, alphaMain=shade, dashed=False)
bounds.append(best_line[0])
def generatePlotTTA(ax, exp_paths, bestBy, bounds):
ax.set_xscale("log", basex=2)
for exp_path in exp_paths:
exp = loadExperiment(exp_path)
results = loadResults(exp, errorfile)
agent = exp.agent
color = colors[agent]
label = agent
if exp.agent == 'ReghTDC':
results = whereParameterGreaterEq(results, 'ratio', 1.0)
# reducer='best' chooses the best value of other parameters *per value of 'replay'*
# reducer='slice' first chooses the best parameter setting, then sweeps over 'replay' with other parameters fixed
b = plotSensitivity(results,
'replay',
ax,
reducer='best',
color=color,
label=label,
bestBy=bestBy)
bounds.append(b)
if __name__ == "__main__":
table = np.zeros((len(algorithms), len(problems), 2))
for i, alg in enumerate(algorithms):
for j, problem in enumerate(problems):
exp_path = f'experiments/stepsizes/{problem}/{alg}/{alg}{stepsize}.json'
try:
exp = loadExperiment(exp_path)
except:
continue
results = loadResults(exp, errorfile)
if alg == 'td' or alg == 'vtrace':
const = results
else:
const = whereParameterGreaterEq(results, 'ratio', 1)
const = whereParameterEquals(const, 'reg_h', 0.8)
best = getBest(const)
metric = np.mean
# best = getBestEnd(const)
# metric = lambda m: np.mean(m[-(int(len(m))):])
mean = metric(best.mean())
stderr = metric(best.stderr())
table[i, j] = [mean, stderr]
htd_idx = indexOf(algorithms, 'htd')
vtrace_idx = indexOf(algorithms, 'vtrace')
def generatePlot(exp_paths):
f, axes = plt.subplots(2, 2)
# get LSTD solution
path = up(up(first(exp_paths))) + '/lstd.json'
exp = loadExperiment(path)
LSTD_rmsve_results = loadResults(exp, 'errors_summary.npy')
LSTD_rmspbe_results = loadResults(exp, 'rmspbe_summary.npy')
LSTD_rmsve = metric(LSTD_rmsve_results)
LSTD_rmspbe = metric(LSTD_rmspbe_results)
rmspbe_bounds = []
rmsve_bounds = []
bounds = plotBest(LSTD_rmspbe,
axes[0, 0],
color=colors['LSTD'],
label='LSTD',
alphaMain=0.5)
rmspbe_bounds.append(bounds)
bounds = plotBest(LSTD_rmspbe,
axes[0, 1],
color=colors['LSTD'],
label='LSTD',
alphaMain=0.5)
rmspbe_bounds.append(bounds)
bounds = plotBest(LSTD_rmsve,
axes[1, 0],
color=colors['LSTD'],
label='LSTD',
alphaMain=0.5)
rmsve_bounds.append(bounds)
bounds = plotBest(LSTD_rmsve,
axes[1, 1],
color=colors['LSTD'],
label='LSTD',
alphaMain=0.5)
rmsve_bounds.append(bounds)
# RMSPBE plots
ax = axes[0, 0]
for exp_path in exp_paths:
exp = loadExperiment(exp_path)
results = loadResults(exp, 'rmspbe_summary.npy')
const, unconst = tee(results)
const = whereParameterGreaterEq(const, 'ratio', 1)
use_ideal_h = exp._d['metaParameters'].get('use_ideal_h', False)
agent = exp.agent
if 'SmoothTDC' in agent:
average = exp._d['metaParameters']['averageType']
agent += '_' + average
color = colors[agent]
label = agent.replace('adagrad', '')
if not (exp.agent in ['TDadagrad', 'TDschedule', 'TD', 'TDamsgrad']
or use_ideal_h):
if UNCONSTRAINED:
bounds = plot(unconst,
ax,
label=label + '_unc',
color=color,
dashed=True,
bestBy=bestBy)
rmspbe_bounds.append(bounds)
bounds = plot(const,
ax,
label=label,
color=color,
dashed=False,
bestBy=bestBy)
rmspbe_bounds.append(bounds)
else:
bounds = plot(unconst,
ax,
label=label,
color=color,
dashed=False,
bestBy=bestBy)
rmspbe_bounds.append(bounds)
ax.set_ylabel("MSPBE")
ax.set_title("MSPBE")
ax = axes[0, 1]
for exp_path in exp_paths:
exp = loadExperiment(exp_path)
rmsve = loadResults(exp, 'errors_summary.npy')
results = loadResults(exp, 'rmspbe_summary.npy')
const, unconst = tee(rmsve)
const_res, unconst_res = tee(results)
const = whereParameterGreaterEq(const, 'ratio', 1)
use_ideal_h = exp._d['metaParameters'].get('use_ideal_h', False)
agent = exp.agent
if 'SmoothTDC' in agent:
average = exp._d['metaParameters']['averageType']
agent += '_' + average
color = colors[agent]
label = agent.replace('adagrad', '')
if not (exp.agent in ['TDadagrad', 'TDschedule', 'TD', 'TDamsgrad']
or use_ideal_h):
best = metric(const)
best_unc = metric(unconst)
best_rmspbe = find(const_res, best)
best_rmspbe_unc = find(unconst_res, best_unc)
bounds = plotBest(best_rmspbe,
ax,
label=label,
color=color,
dashed=False)
rmspbe_bounds.append(bounds)
if UNCONSTRAINED:
bounds = plotBest(best_rmspbe_unc,
ax,
label=label + '_unc',
color=color,
dashed=True)
rmspbe_bounds.append(bounds)
else:
best = metric(unconst)
best_rmspbe = find(unconst_res, best)
bounds = plotBest(best_rmspbe,
ax,
label=label,
color=color,
dashed=False)
rmspbe_bounds.append(bounds)
ax.set_title("MSVE")
# RMSVE plots
ax = axes[1, 0]
for exp_path in exp_paths:
exp = loadExperiment(exp_path)
rmsve = loadResults(exp, 'errors_summary.npy')
rmspbe = loadResults(exp, 'rmspbe_summary.npy')
const, unconst = tee(rmspbe)
const_res, unconst_res = tee(rmsve)
const = whereParameterGreaterEq(const, 'ratio', 1)
use_ideal_h = exp._d['metaParameters'].get('use_ideal_h', False)
agent = exp.agent
if 'SmoothTDC' in agent:
average = exp._d['metaParameters']['averageType']
agent += '_' + average
color = colors[agent]
label = agent.replace('adagrad', '')
if not (exp.agent in ['TDadagrad', 'TDschedule', 'TD', 'TDamsgrad']
or use_ideal_h):
# best PBE using AUC
best = metric(const)
best_unc = metric(unconst)
best_rmsve = find(const_res, best)
best_rmsve_unc = find(unconst_res, best_unc)
print('rmsve_over_rmspbe')
print(label, best_rmsve.params)
print(label, best_rmsve_unc.params)
bounds = plotBest(best_rmsve,
ax,
label=label,
color=color,
dashed=False)
rmsve_bounds.append(bounds)
if UNCONSTRAINED:
bounds = plotBest(best_rmsve_unc,
ax,
label=label + '_unc',
color=color,
dashed=True)
rmsve_bounds.append(bounds)
else:
best = metric(unconst)
best_rmsve = find(unconst_res, best)
bounds = plotBest(best_rmsve,
ax,
label=label,
color=color,
dashed=False)
rmsve_bounds.append(bounds)
ax.set_ylabel("MSVE")
ax = axes[1, 1]
for exp_path in exp_paths:
exp = loadExperiment(exp_path)
results = loadResults(exp)
const, unconst = tee(results)
const = whereParameterGreaterEq(const, 'ratio', 1)
use_ideal_h = exp._d['metaParameters'].get('use_ideal_h', False)
agent = exp.agent
if 'SmoothTDC' in agent:
average = exp._d['metaParameters']['averageType']
agent += '_' + average
color = colors[agent]
label = agent.replace('adagrad', '')
if not (exp.agent in ['TDadagrad', 'TDschedule', 'TD', 'TDamsgrad']
or use_ideal_h):
bounds = plot(const,
ax,
label=label,
color=color,
dashed=False,
bestBy=bestBy)
rmsve_bounds.append(bounds)
if UNCONSTRAINED:
bounds = plot(unconst,
ax,
label=label + '_unc',
color=color,
dashed=True,
bestBy=bestBy)
rmsve_bounds.append(bounds)
else:
bounds = plot(unconst,
ax,
label=label,
color=color,
dashed=False,
bestBy=bestBy)
rmsve_bounds.append(bounds)
# rmspbe
rmspbe_lower = min(map(lambda x: x[0], rmspbe_bounds)) * 0.9
rmspbe_upper = max(map(lambda x: x[1], rmspbe_bounds)) * 1.05
if rmspbe_lower < 0.01:
rmspbe_lower = -0.01
axes[0, 0].set_ylim([rmspbe_lower, rmspbe_upper])
axes[0, 1].set_ylim([rmspbe_lower, rmspbe_upper])
# rmsve
rmsve_lower = min(map(lambda x: x[0], rmsve_bounds)) * 0.9
rmsve_upper = max(map(lambda x: x[1], rmsve_bounds)) * 1.05
if rmsve_lower < 0.01:
rmsve_lower = -0.01
axes[1, 0].set_ylim([0, 20])
axes[1, 1].set_ylim([0, 20])
for i, alg in enumerate(algorithms):
for j, problem in enumerate(problems):
exp_path = f'experiments/icml_2020/{problem}/{alg}/{alg}{stepsize}.json'
try:
exp = loadExperiment(exp_path)
except:
continue
results = loadResults(exp, errorfile)
if alg == 'regh_tdc':
results = whereParameterEquals(results, 'ratio', 1)
results = whereParameterEquals(results, 'reg_h', 0.8)
if alg == 'tdc':
results = whereParameterGreaterEq(results, 'ratio', 1)
if alg == 'tdrcc':
results = whereParameterGreaterEq(results, 'reg_h', 0.1)
best = getBest(results)
metric = np.mean
# best = getBestEnd(results)
# metric = lambda m: np.mean(m[-(int(len(m))):])
mean = metric(best.mean())
stderr = metric(best.stderr())
print(alg, problem, mean, best.params)
table[i, j] = [mean, stderr]