def generatePlotTTA(ax, exp_path, bounds):
exp = loadExperiment(exp_path)
results = loadResults(exp, errorfile)
const, unconst = tee(results)
color = colors[exp.agent]
label = exp.agent
const = whereParameterGreaterEq(const, 'ratio', 1)
if 'ReghTDC' in label:
const = whereParameterEquals(const, 'reg_h', 0.8)
best_const = getBest(const, bestBy=bestBy)
best_unconst = getBest(unconst, bestBy=bestBy)
if show_unconst and best_const != best_unconst:
b = plotBest(best_unconst,
ax,
window=window,
smoothing=smoothing,
label=label + '_unc',
color=color,
alpha=0.2,
dashed=True)
bounds.append(b)
b = plotBest(best_const,
ax,
window=window,
smoothing=smoothing,
label=label,
color=color,
alpha=0.2,
dashed=False)
bounds.append(b)
def generatePlotTTA(ax, exp_paths, bounds):
for exp_path in exp_paths:
exp = loadExperiment(exp_path)
results = loadResults(exp, errorfile)
const, unconst = tee(results)
color = colors[exp.agent]
label = exp.agent
if error == 'rmsve':
rmspbe = loadResults(exp, 'rmspbe_summary.npy')
rmspbe_unconst, rmspbe_const = tee(rmspbe)
rmspbe_const = whereParameterGreaterEq(rmspbe_const, 'ratio', 1)
best_rmspbe_unconst = getBest(rmspbe_unconst)
best_rmspbe_const = getBest(rmspbe_const)
best_unconst = find(unconst, best_rmspbe_unconst)
best_const = find(const, best_rmspbe_const)
elif error == 'rmspbe':
const = whereParameterGreaterEq(const, 'ratio', 1)
best_unconst = getBest(unconst)
best_const = getBest(const)
b = plotBest(best_unconst, ax, label=label + '_unc', color=color, dashed=True)
bounds.append(b)
b = plotBest(best_const, ax, label=label, color=color, dashed=False)
bounds.append(b)
def plot(results,
ax,
window=1,
smoothing=0,
color=None,
alpha=0.4,
alphaMain=1,
label=None,
labelParams=None,
bestBy='end',
dashed=False):
if bestBy == 'end':
best = getBestEnd(results)
elif bestBy == 'auc':
best = getBest(results)
else:
raise Exception('I can only get best by "end" or "auc"')
print(best.exp.agent, best.params)
return plotBest(best,
ax,
smoothing=smoothing,
color=color,
alpha=alpha,
alphaMain=alphaMain,
label=label,
window=window,
labelParams=labelParams,
dashed=dashed)
def generatePlot(exp_paths):
ax = plt.gca()
# ax.semilogx()
for exp_path in exp_paths:
exp = loadExperiment(exp_path)
rmsve = loadResults(exp, 'errors_summary.npy')
rmspbe = loadResults(exp, 'rmspbe_summary.npy')
# if exp.agent == 'TDadagrad':
# continue
# best PBE using AUC
best = getBest(rmspbe)
best_rmsve = find(rmsve, best)
use_ideal_h = exp._d['metaParameters'].get('use_ideal_h', False)
dashed = use_ideal_h
color = colors[exp.agent]
label = exp.agent.replace('adagrad', '')
if use_ideal_h:
label += '-h*'
plotBest(best_rmsve, ax, label=label, color=color, dashed=dashed)
# plt.show()
save(exp, f'rmsve_over_rmspbe', type='svg')
plt.clf()
def generatePlotSTA(ax, exp_paths, bestBy, bounds):
for exp_path in exp_paths:
exp = loadExperiment(exp_path)
rmsve = loadResults(exp, 'errors_summary.npy')
rmspbe = loadResults(exp, 'rmspbe_summary.npy')
agent = exp.agent
if 'SmoothTDC' in agent:
average = exp._d['metaParameters']['averageType']
agent += '_' + average
color = colors[agent]
label = agent
if bestBy == 'end':
metric = lambda m: np.mean(m[-int(m.shape[0] * .1):])
best_rmspbe = getBestEnd(rmspbe)
best = find(rmsve, best_rmspbe)
elif bestBy == 'auc':
metric = np.mean
best_rmspbe = getBest(rmspbe)
best = find(rmsve, best_rmspbe)
m = metric(best.mean())
ax.hlines(m, 2**-6, 2**6, color=color, label=label)
bounds.append([m, m])
def baseline(ax, exp_path, values, bounds):
exp = loadExperiment(path)
results = loadResults(exp, errorfile)
if 'Regh' in exp.agent:
results = whereParameterEquals(results, 'reg_h', 0.8)
results = whereParameterEquals(results, 'ratio', 1)
elif 'TDC' in exp.agent or 'GTD2' in exp.agent:
results = whereParameterEquals(results, 'ratio', 1)
if bestBy == 'end':
metric = lambda m: np.mean(m[-int(m.shape[0] * .1):])
best = getBestEnd(results)
elif bestBy == 'auc':
metric = np.mean
best = getBest(results)
color = colors[exp.agent]
label = exp.agent
m = metric(best.mean())
low = min(values)
high = max(values)
ax.hlines(m, low, high, color=color, label=label, linewidth=4, linestyle=':')
bounds.append((m, m))
def generatePlotSSA(ax, exp_paths, bounds):
for exp_path in exp_paths:
exp = loadExperiment(exp_path)
results = loadResults(exp, errorfile)
color = colors[exp.agent]
label = exp.agent
if error == 'rmsve':
rmspbe = loadResults(exp, 'rmspbe_summary.npy')
best_rmspbe = getBest(rmspbe)
best = find(results, best_rmspbe)
elif error == 'rmspbe':
best = getBest(results)
b = plotBest(best, ax, label=label, color=color, dashed=False)
bounds.append(b)
def getResultsAndBest(exps):
all_results = []
best_result = None
for exp in exps:
results = loadResults(exp)
results = whereParameterEquals(results, 'lambda', LAMBDA)
best = getBest(results)
all_results.append(best)
if best_result is None:
best_result = best
elif np.mean(best.mean()) < np.mean(best_result.mean()):
best_result = best
return all_results, best_result
def generatePlot(ax, exp_paths, bounds):
for exp_path in exp_paths:
exp = loadExperiment(exp_path)
variance = loadResults(exp, 'expupd_summary.npy')
agent = exp.agent
color = colors[agent]
label = agent
best = getBest(variance)
if 'GTD2' in agent or 'TDC' in agent:
best.reducer(lambda m: m[:, 1])
else:
best.reducer(lambda m: m[:, 1])
b = plotBest(best, ax, color=color, label=label)
bounds.append(b)
def getSensitivityData(results, param, reducer='best', overStream=None, bestBy='end'):
useOtherStream = overStream is not None
overStream = overStream if useOtherStream else results
bestByStr = 'auc'
if not callable(bestBy):
bestByStr = bestBy
if reducer == 'best':
bestStream = getBestOverParameter(overStream, param, bestBy=bestByStr)
elif reducer == 'slice':
l, r = tee(overStream)
if bestByStr == 'end':
best = getBestEnd(l)
elif bestBy == 'auc':
best = getBest(l)
bestStream = sliceOverParameter(r, best, param)
x = sorted(list(bestStream))
if useOtherStream:
best = {}
teed = tee(results, len(x))
for i, k in enumerate(x):
best[k] = find(teed[i], bestStream[k])
else:
best = bestStream
if bestBy == 'end':
metric = lambda m: np.mean(m[-int(m.shape[0] * .1):])
elif bestBy == 'auc':
metric = np.mean
elif callable(bestBy):
metric = bestBy
y = np.array([metric(best[k].mean()) for k in x])
e = np.array([metric(best[k].stderr()) for k in x])
e[np.isnan(y)] = 0.000001
y[np.isnan(y)] = 100000
return x, y, e
def generatePlot(ax, exp_path, bounds):
exp = loadExperiment(exp_path)
results = loadResults(exp, 'variance_summary.npy')
best = getBest(results)
best.reducer(lambda m: m[:, 0])
color = colors[exp.agent]
label = exp.agent
b = plotBest(best,
ax,
window=window,
smoothing=smoothing,
label=label,
color=color,
dashed=False)
bounds.append(b)
def generatePlotSSA(ax, exp_paths, bounds):
for exp_path in exp_paths:
if 'amsgrad' in exp_path:
continue
exp = loadExperiment(exp_path)
results = loadResults(exp, errorfile)
stepsizes = loadResults(exp, 'stepsize_summary.npy')
color = colors[exp.agent]
label = exp.agent
best_error = getBest(results)
best = find(stepsizes, best_error)
b = plotBest(best, ax, label=label, color=color, dashed=False)
bounds.append(b)
def generatePlotTTA(ax, exp_paths, bounds):
for exp_path in exp_paths:
if 'amsgrad' in exp_path:
continue
exp = loadExperiment(exp_path)
results = loadResults(exp, errorfile)
stepsizes = loadResults(exp, 'stepsize_summary.npy')
results = whereParameterEquals(results, 'ratio', 1)
color = colors[exp.agent]
label = exp.agent
best_error = getBest(results)
best_stepsize = find(stepsizes, best_error)
b = plotBest(best_stepsize,
ax,
label=[label + '_w', label + '_h'],
color=color,
dashed=[False, True])
bounds.append(b)
def generatePlot(ax, exp_paths, bounds):
for exp_path in exp_paths:
exp = loadExperiment(exp_path)
results = loadResults(exp, errorfile)
color = colors[exp.agent]
label = exp.agent
results = whereParameterEquals(results, 'batch_size', 4)
results = whereParameterLesserEq(results, 'ratio', 8)
results = whereParameterLesserEq(results, 'alpha', 0.5)
results = where(results, lambda r: r.params.get('ratio', 1) * r.params['alpha'] <= 1)
if 'ReghTDC' in label:
results = whereParameterEquals(results, 'reg_h', 1)
results = whereParameterEquals(results, 'ratio', 1)
elif 'TDRCC' in label:
results = whereParameterEquals(results, 'reg_h', 0.8)
results = whereParameterEquals(results, 'ratio', 1)
elif 'TDC' in label:
results = whereParameterGreaterEq(results, 'ratio', 1)
left, right = tee(results)
best_line = getBest(right).mean()
best = np.mean(best_line)
for result in left:
# print(label, result.params)
shade = 0.12
line = result.mean()
if np.mean(line) == best:
shade = 1
plotBest(result, ax, label=label, color=color, alphaMain=shade, dashed=False)
bounds.append(best_line[0])
b = plot(results, ax, window=window, smoothing=smoothing, label=label, color=color, alpha=0.2, dashed=False, bestBy=bestBy)
bounds.append(b)
if __name__ == "__main__":
ax = plt.gca()
f = plt.gcf()
bounds = []
if lstd_baseline:
path = f'experiments/batch/{problem}/lstd.json'
lstd_exp = loadExperiment(path)
LSTD_res = loadResults(lstd_exp, errorfile)
LSTD_best = getBest(LSTD_res)
b = plotBest(LSTD_best, ax, window=window, color=colors['LSTD'], label='LSTD', alphaMain=0.5, dashed=True)
bounds.append(b)
for alg in algorithms:
if stepsize == 'constant':
exp_path = f'experiments/batch/{problem}/{alg}/{alg}.json'
else:
exp_path = f'experiments/batch/{problem}/{alg}/{alg}{stepsize}.json'
if '_h' in alg or alg == 'td' or alg == 'vtrace':
generatePlotSSA(ax, exp_path, bounds)
else:
generatePlotTTA(ax, exp_path, bounds)
curves = np.zeros(
(len(stepsizes), len(algorithms) + 1, len(problems), params, 2))
total_runs = 0
for k, problem in enumerate(problems):
for i, ss in enumerate(stepsizes):
td_idx = len(algorithms)
if ss == 'constant':
td_path = f'experiments/stepsizes/{problem}/td/td.json'
else:
td_path = f'experiments/stepsizes/{problem}/td/td{ss}.json'
exp = loadExperiment(td_path)
results = loadResults(exp, errorfile)
best = getBest(results)
curve = best.mean()
curves[i, td_idx, k, :, 0] = np.mean(curve) / curve[0]
curves[i, td_idx, k, :,
1] = np.mean(best.stderr()) * np.sqrt(best.runs())
for j, alg in enumerate(algorithms):
print(problem, alg, ss)
if alg == 'htd' and problem in on_policy_problems:
curves[i, j, k] = curves[i, td_idx, k]
continue
if ss == 'constant':
exp_paths = glob.glob(
f'experiments/stepsizes/{problem}/{alg}/{alg}.json')
for i, alg in enumerate(algorithms):
for j, problem in enumerate(problems):
exp_path = f'experiments/stepsizes/{problem}/{alg}/{alg}{stepsize}.json'
try:
exp = loadExperiment(exp_path)
except:
continue
results = loadResults(exp, errorfile)
if alg == 'td' or alg == 'vtrace':
const = results
else:
const = whereParameterGreaterEq(results, 'ratio', 1)
const = whereParameterEquals(const, 'reg_h', 0.8)
best = getBest(const)
metric = np.mean
# best = getBestEnd(const)
# metric = lambda m: np.mean(m[-(int(len(m))):])
mean = metric(best.mean())
stderr = metric(best.stderr())
table[i, j] = [mean, stderr]
htd_idx = indexOf(algorithms, 'htd')
vtrace_idx = indexOf(algorithms, 'vtrace')
td_idx = indexOf(algorithms, 'td')
for j, problem in enumerate(problems):
if htd_idx is not None and table[htd_idx, j, 0] == 0:
td_exp = loadExperiment(f'experiments/reward_scale/{problem}/td/td.json')
scales = td_exp._d['metaParameters']['reward_scale']
tdrc_exp = loadExperiment(
f'experiments/reward_scale/{problem}/regh_tdc/regh_tdc.json')
betas = tdrc_exp._d['metaParameters']['reg_h']
mat = np.zeros((len(scales), len(betas)))
xs = []
ys = []
alphas = []
for i, scale in enumerate(scales):
td_results = loadResults(td_exp, errorfile)
td_results = whereParameterEquals(td_results, 'reward_scale', scale)
best_td = getBest(td_results)
best_td_mean = np.mean(best_td.mean())
best_td_std = np.mean(
np.sqrt(best_td.stderr() * np.sqrt(best_td.runs())))
tdrc_results = loadResults(tdrc_exp, errorfile)
tdrc_results = whereParameterEquals(tdrc_results, 'reward_scale',
scale)
tdrc_split = splitOverParameter(tdrc_results, 'reg_h')
for j, beta in enumerate(betas):
results = tdrc_split[beta]
best_tdrc = getBest(results)
mean = np.mean(best_tdrc.mean())
import os
import sys
import json
import numpy as np
sys.path.append(os.getcwd())
from src.analysis.results import loadResults, getBest
from src.utils.model import loadExperiment
from src.utils.path import up, fileName
exp_paths = sys.argv[1:]
for exp_path in exp_paths:
exp = loadExperiment(exp_path)
results = loadResults(exp, 'rmspbe_summary.npy')
best = getBest(results)
print('---------------------')
print('agent:', exp.agent)
print(best.params)
f = fileName(exp_path)
new_path = up(exp_path) + '/best_rmspbe_auc/' + f
d = exp._d
d['metaParameters'] = best.params
os.makedirs(up(new_path), exist_ok=True)
with open(new_path, 'w') as f:
json.dump(d, f, indent=4)
