def generatePlotSTA(ax, exp_paths, bestBy, bounds):
for exp_path in exp_paths:
exp = loadExperiment(exp_path)
rmsve = loadResults(exp, 'errors_summary.npy')
rmspbe = loadResults(exp, 'rmspbe_summary.npy')
agent = exp.agent
if 'SmoothTDC' in agent:
average = exp._d['metaParameters']['averageType']
agent += '_' + average
color = colors[agent]
label = agent
if bestBy == 'end':
metric = lambda m: np.mean(m[-int(m.shape[0] * .1):])
best_rmspbe = getBestEnd(rmspbe)
best = find(rmsve, best_rmspbe)
elif bestBy == 'auc':
metric = np.mean
best_rmspbe = getBest(rmspbe)
best = find(rmsve, best_rmspbe)
m = metric(best.mean())
ax.hlines(m, 2**-6, 2**6, color=color, label=label)
bounds.append([m, m])
def generatePlot(exp_paths):
ax = plt.gca()
# ax.semilogx()
for exp_path in exp_paths:
exp = loadExperiment(exp_path)
if exp.agent == 'TDadagrad':
continue
use_ideal_h = exp._d['metaParameters'].get('use_ideal_h', False)
dashed = use_ideal_h
color = colors[exp.agent]
# load the errors and hnorm files
errors = loadResults(exp, 'errors_summary.npy')
results = loadResults(exp, 'ndh_summary.npy')
# choose the best parameters from the _errors_
best = getBestEnd(errors)
best_ndh = find(results, best)
label = exp.agent.replace('adagrad', '')
if use_ideal_h:
label += '-h*'
plotBest(best_ndh, ax, label=label, color=color, dashed=dashed)
# plt.show()
save(exp, f'norm_delta-hat')
plt.clf()
def baseline(ax, exp_path, values, bounds):
exp = loadExperiment(path)
results = loadResults(exp, errorfile)
if 'Regh' in exp.agent:
results = whereParameterEquals(results, 'reg_h', 0.8)
results = whereParameterEquals(results, 'ratio', 1)
elif 'TDC' in exp.agent or 'GTD2' in exp.agent:
results = whereParameterEquals(results, 'ratio', 1)
if bestBy == 'end':
metric = lambda m: np.mean(m[-int(m.shape[0] * .1):])
best = getBestEnd(results)
elif bestBy == 'auc':
metric = np.mean
best = getBest(results)
color = colors[exp.agent]
label = exp.agent
m = metric(best.mean())
low = min(values)
high = max(values)
ax.hlines(m, low, high, color=color, label=label, linewidth=4, linestyle=':')
bounds.append((m, m))
def plot(results,
ax,
window=1,
smoothing=0,
color=None,
alpha=0.4,
alphaMain=1,
label=None,
labelParams=None,
bestBy='end',
dashed=False):
if bestBy == 'end':
best = getBestEnd(results)
elif bestBy == 'auc':
best = getBest(results)
else:
raise Exception('I can only get best by "end" or "auc"')
print(best.exp.agent, best.params)
return plotBest(best,
ax,
smoothing=smoothing,
color=color,
alpha=alpha,
alphaMain=alphaMain,
label=label,
window=window,
labelParams=labelParams,
dashed=dashed)
def getSensitivityData(results, param, reducer='best', overStream=None, bestBy='end'):
useOtherStream = overStream is not None
overStream = overStream if useOtherStream else results
bestByStr = 'auc'
if not callable(bestBy):
bestByStr = bestBy
if reducer == 'best':
bestStream = getBestOverParameter(overStream, param, bestBy=bestByStr)
elif reducer == 'slice':
l, r = tee(overStream)
if bestByStr == 'end':
best = getBestEnd(l)
elif bestBy == 'auc':
best = getBest(l)
bestStream = sliceOverParameter(r, best, param)
x = sorted(list(bestStream))
if useOtherStream:
best = {}
teed = tee(results, len(x))
for i, k in enumerate(x):
best[k] = find(teed[i], bestStream[k])
else:
best = bestStream
if bestBy == 'end':
metric = lambda m: np.mean(m[-int(m.shape[0] * .1):])
elif bestBy == 'auc':
metric = np.mean
elif callable(bestBy):
metric = bestBy
y = np.array([metric(best[k].mean()) for k in x])
e = np.array([metric(best[k].stderr()) for k in x])
e[np.isnan(y)] = 0.000001
y[np.isnan(y)] = 100000
return x, y, e
def generatePlot(exp_paths):
ax = plt.gca()
# ax.semilogx()
for exp_path in exp_paths:
exp = loadExperiment(exp_path)
# load the errors and hnorm files
errors = loadResults(exp, 'errors_summary.npy')
results = loadResults(exp, 'hnorm_summary.npy')
# choose the best parameters from the _errors_
best = getBestEnd(errors)
best_hnorm = find(results, best)
label = fileName(exp_path).replace('.json', '')
plotBest(best_hnorm, ax, label=label)
plt.show()
# save(exp, f'learning-curve')
plt.clf()
def generatePlot(exp_paths):
ax = plt.gca()
# ax.semilogx()
for exp_path in exp_paths:
exp = loadExperiment(exp_path)
rmsve = loadResults(exp, 'errors_summary.npy')
results = loadResults(exp, 'rmspbe_summary.npy')
best = getBestEnd(rmsve)
best_rmspbe = find(results, best)
use_ideal_h = exp._d['metaParameters'].get('use_ideal_h', False)
dashed = use_ideal_h
color = colors[exp.agent]
label = exp.agent.replace('adagrad', '')
if use_ideal_h:
label += '-h*'
plotBest(best_rmspbe, ax, label=label, color=color, dashed=dashed)
# plt.show()
save(exp, f'rmspbe_over_rmsve', type='svg')
plt.clf()
def generatePlot(exp_path):
ax = plt.gca()
# ax.semilogx()
exp = loadExperiment(exp_path)
# load the errors and hnorm files
errors = loadResults(exp, 'errors_summary.npy')
results = loadResults(exp, 'stepsize_summary.npy')
# choose the best parameters from the _errors_
best = getBestEnd(errors)
best_ss = find(results, best)
alg = exp.agent.replace('adagrad', '')
plotBest(best_ss, ax, label=['w', 'h'])
ax.set_ylim([0, 4])
print(alg)
# plt.show()
save(exp, f'stepsizes-{alg}')
plt.clf()
def generatePlot(exp_paths):
f, axes = plt.subplots(2, 2)
# ax.semilogx()
# RMSPBE plots
ax = axes[0, 0]
rmspbe_bounds = []
for exp_path in exp_paths:
exp = loadExperiment(exp_path)
results = loadResults(exp, 'rmspbe_summary.npy')
use_ideal_h = exp._d['metaParameters'].get('use_ideal_h', False)
dashed = False
color = colors[exp.agent]
label = exp.agent.replace('adagrad', '')
# if use_ideal_h:
# label += '-h*'
bounds = plot(results, ax, label=label, color=color, dashed=dashed)
rmspbe_bounds.append(bounds)
ax.set_ylabel("RMSPBE")
ax.set_title("RMSPBE")
ax = axes[0, 1]
for exp_path in exp_paths:
exp = loadExperiment(exp_path)
rmsve = loadResults(exp, 'errors_summary.npy')
results = loadResults(exp, 'rmspbe_summary.npy')
best = getBestEnd(rmsve)
best_rmspbe = find(results, best)
use_ideal_h = exp._d['metaParameters'].get('use_ideal_h', False)
dashed = False
color = colors[exp.agent]
label = exp.agent.replace('adagrad', '')
# if use_ideal_h:
# label += '-h*'
bounds = plotBest(best_rmspbe,
ax,
label=label,
color=color,
dashed=dashed)
rmspbe_bounds.append(bounds)
ax.set_title("RMSVE")
# RMSVE plots
ax = axes[1, 0]
rmsve_bounds = []
for exp_path in exp_paths:
exp = loadExperiment(exp_path)
rmsve = loadResults(exp, 'errors_summary.npy')
rmspbe = loadResults(exp, 'rmspbe_summary.npy')
# if exp.agent == 'TDadagrad':
# continue
# best PBE using AUC
best = getBestEnd(rmspbe)
best_rmsve = find(rmsve, best)
use_ideal_h = exp._d['metaParameters'].get('use_ideal_h', False)
dashed = False
color = colors[exp.agent]
label = exp.agent.replace('adagrad', '')
# if use_ideal_h:
# label += '-h*'
bounds = plotBest(best_rmsve,
ax,
label=label,
color=color,
dashed=dashed)
rmsve_bounds.append(bounds)
ax.set_ylabel("RMSVE")
ax = axes[1, 1]
for exp_path in exp_paths:
exp = loadExperiment(exp_path)
results = loadResults(exp)
use_ideal_h = exp._d['metaParameters'].get('use_ideal_h', False)
dashed = False
color = colors[exp.agent]
label = exp.agent.replace('adagrad', '')
# if use_ideal_h:
# label += '-h*'
bounds = plot(results, ax, label=label, color=color, dashed=dashed)
rmsve_bounds.append(bounds)
# rmspbe
rmspbe_lower = min(map(lambda x: x[0], rmspbe_bounds)) * 0.9
rmspbe_upper = max(map(lambda x: x[1], rmspbe_bounds)) * 1.05
axes[0, 0].set_ylim([rmspbe_lower, rmspbe_upper])
axes[0, 1].set_ylim([rmspbe_lower, rmspbe_upper])
# rmsve
rmsve_lower = min(map(lambda x: x[0], rmsve_bounds)) * 0.9
rmsve_upper = max(map(lambda x: x[1], rmsve_bounds)) * 1.05
axes[1, 0].set_ylim([rmsve_lower, rmsve_upper])
axes[1, 1].set_ylim([rmsve_lower, rmsve_upper])
plt.show()