def generatePlotTTA(ax, exp_path, bounds): exp = loadExperiment(exp_path) results = loadResults(exp, errorfile) const, unconst = tee(results) color = colors[exp.agent] label = exp.agent const = whereParameterGreaterEq(const, 'ratio', 1) if 'ReghTDC' in label: const = whereParameterEquals(const, 'reg_h', 0.8) best_const = getBest(const, bestBy=bestBy) best_unconst = getBest(unconst, bestBy=bestBy) if show_unconst and best_const != best_unconst: b = plotBest(best_unconst, ax, window=window, smoothing=smoothing, label=label + '_unc', color=color, alpha=0.2, dashed=True) bounds.append(b) b = plotBest(best_const, ax, window=window, smoothing=smoothing, label=label, color=color, alpha=0.2, dashed=False) bounds.append(b)
def generatePlotTTA(ax, exp_paths, bounds): for exp_path in exp_paths: exp = loadExperiment(exp_path) results = loadResults(exp, errorfile) const, unconst = tee(results) color = colors[exp.agent] label = exp.agent if error == 'rmsve': rmspbe = loadResults(exp, 'rmspbe_summary.npy') rmspbe_unconst, rmspbe_const = tee(rmspbe) rmspbe_const = whereParameterGreaterEq(rmspbe_const, 'ratio', 1) best_rmspbe_unconst = getBest(rmspbe_unconst) best_rmspbe_const = getBest(rmspbe_const) best_unconst = find(unconst, best_rmspbe_unconst) best_const = find(const, best_rmspbe_const) elif error == 'rmspbe': const = whereParameterGreaterEq(const, 'ratio', 1) best_unconst = getBest(unconst) best_const = getBest(const) b = plotBest(best_unconst, ax, label=label + '_unc', color=color, dashed=True) bounds.append(b) b = plotBest(best_const, ax, label=label, color=color, dashed=False) bounds.append(b)
def plot(results, ax, window=1, smoothing=0, color=None, alpha=0.4, alphaMain=1, label=None, labelParams=None, bestBy='end', dashed=False): if bestBy == 'end': best = getBestEnd(results) elif bestBy == 'auc': best = getBest(results) else: raise Exception('I can only get best by "end" or "auc"') print(best.exp.agent, best.params) return plotBest(best, ax, smoothing=smoothing, color=color, alpha=alpha, alphaMain=alphaMain, label=label, window=window, labelParams=labelParams, dashed=dashed)
def generatePlot(exp_paths): ax = plt.gca() # ax.semilogx() for exp_path in exp_paths: exp = loadExperiment(exp_path) rmsve = loadResults(exp, 'errors_summary.npy') rmspbe = loadResults(exp, 'rmspbe_summary.npy') # if exp.agent == 'TDadagrad': # continue # best PBE using AUC best = getBest(rmspbe) best_rmsve = find(rmsve, best) use_ideal_h = exp._d['metaParameters'].get('use_ideal_h', False) dashed = use_ideal_h color = colors[exp.agent] label = exp.agent.replace('adagrad', '') if use_ideal_h: label += '-h*' plotBest(best_rmsve, ax, label=label, color=color, dashed=dashed) # plt.show() save(exp, f'rmsve_over_rmspbe', type='svg') plt.clf()
def generatePlotSTA(ax, exp_paths, bestBy, bounds): for exp_path in exp_paths: exp = loadExperiment(exp_path) rmsve = loadResults(exp, 'errors_summary.npy') rmspbe = loadResults(exp, 'rmspbe_summary.npy') agent = exp.agent if 'SmoothTDC' in agent: average = exp._d['metaParameters']['averageType'] agent += '_' + average color = colors[agent] label = agent if bestBy == 'end': metric = lambda m: np.mean(m[-int(m.shape[0] * .1):]) best_rmspbe = getBestEnd(rmspbe) best = find(rmsve, best_rmspbe) elif bestBy == 'auc': metric = np.mean best_rmspbe = getBest(rmspbe) best = find(rmsve, best_rmspbe) m = metric(best.mean()) ax.hlines(m, 2**-6, 2**6, color=color, label=label) bounds.append([m, m])
def baseline(ax, exp_path, values, bounds): exp = loadExperiment(path) results = loadResults(exp, errorfile) if 'Regh' in exp.agent: results = whereParameterEquals(results, 'reg_h', 0.8) results = whereParameterEquals(results, 'ratio', 1) elif 'TDC' in exp.agent or 'GTD2' in exp.agent: results = whereParameterEquals(results, 'ratio', 1) if bestBy == 'end': metric = lambda m: np.mean(m[-int(m.shape[0] * .1):]) best = getBestEnd(results) elif bestBy == 'auc': metric = np.mean best = getBest(results) color = colors[exp.agent] label = exp.agent m = metric(best.mean()) low = min(values) high = max(values) ax.hlines(m, low, high, color=color, label=label, linewidth=4, linestyle=':') bounds.append((m, m))
def generatePlotSSA(ax, exp_paths, bounds): for exp_path in exp_paths: exp = loadExperiment(exp_path) results = loadResults(exp, errorfile) color = colors[exp.agent] label = exp.agent if error == 'rmsve': rmspbe = loadResults(exp, 'rmspbe_summary.npy') best_rmspbe = getBest(rmspbe) best = find(results, best_rmspbe) elif error == 'rmspbe': best = getBest(results) b = plotBest(best, ax, label=label, color=color, dashed=False) bounds.append(b)
def getResultsAndBest(exps): all_results = [] best_result = None for exp in exps: results = loadResults(exp) results = whereParameterEquals(results, 'lambda', LAMBDA) best = getBest(results) all_results.append(best) if best_result is None: best_result = best elif np.mean(best.mean()) < np.mean(best_result.mean()): best_result = best return all_results, best_result
def generatePlot(ax, exp_paths, bounds): for exp_path in exp_paths: exp = loadExperiment(exp_path) variance = loadResults(exp, 'expupd_summary.npy') agent = exp.agent color = colors[agent] label = agent best = getBest(variance) if 'GTD2' in agent or 'TDC' in agent: best.reducer(lambda m: m[:, 1]) else: best.reducer(lambda m: m[:, 1]) b = plotBest(best, ax, color=color, label=label) bounds.append(b)
def getSensitivityData(results, param, reducer='best', overStream=None, bestBy='end'): useOtherStream = overStream is not None overStream = overStream if useOtherStream else results bestByStr = 'auc' if not callable(bestBy): bestByStr = bestBy if reducer == 'best': bestStream = getBestOverParameter(overStream, param, bestBy=bestByStr) elif reducer == 'slice': l, r = tee(overStream) if bestByStr == 'end': best = getBestEnd(l) elif bestBy == 'auc': best = getBest(l) bestStream = sliceOverParameter(r, best, param) x = sorted(list(bestStream)) if useOtherStream: best = {} teed = tee(results, len(x)) for i, k in enumerate(x): best[k] = find(teed[i], bestStream[k]) else: best = bestStream if bestBy == 'end': metric = lambda m: np.mean(m[-int(m.shape[0] * .1):]) elif bestBy == 'auc': metric = np.mean elif callable(bestBy): metric = bestBy y = np.array([metric(best[k].mean()) for k in x]) e = np.array([metric(best[k].stderr()) for k in x]) e[np.isnan(y)] = 0.000001 y[np.isnan(y)] = 100000 return x, y, e
def generatePlot(ax, exp_path, bounds): exp = loadExperiment(exp_path) results = loadResults(exp, 'variance_summary.npy') best = getBest(results) best.reducer(lambda m: m[:, 0]) color = colors[exp.agent] label = exp.agent b = plotBest(best, ax, window=window, smoothing=smoothing, label=label, color=color, dashed=False) bounds.append(b)
def generatePlotSSA(ax, exp_paths, bounds): for exp_path in exp_paths: if 'amsgrad' in exp_path: continue exp = loadExperiment(exp_path) results = loadResults(exp, errorfile) stepsizes = loadResults(exp, 'stepsize_summary.npy') color = colors[exp.agent] label = exp.agent best_error = getBest(results) best = find(stepsizes, best_error) b = plotBest(best, ax, label=label, color=color, dashed=False) bounds.append(b)
def generatePlotTTA(ax, exp_paths, bounds): for exp_path in exp_paths: if 'amsgrad' in exp_path: continue exp = loadExperiment(exp_path) results = loadResults(exp, errorfile) stepsizes = loadResults(exp, 'stepsize_summary.npy') results = whereParameterEquals(results, 'ratio', 1) color = colors[exp.agent] label = exp.agent best_error = getBest(results) best_stepsize = find(stepsizes, best_error) b = plotBest(best_stepsize, ax, label=[label + '_w', label + '_h'], color=color, dashed=[False, True]) bounds.append(b)
def generatePlot(ax, exp_paths, bounds): for exp_path in exp_paths: exp = loadExperiment(exp_path) results = loadResults(exp, errorfile) color = colors[exp.agent] label = exp.agent results = whereParameterEquals(results, 'batch_size', 4) results = whereParameterLesserEq(results, 'ratio', 8) results = whereParameterLesserEq(results, 'alpha', 0.5) results = where(results, lambda r: r.params.get('ratio', 1) * r.params['alpha'] <= 1) if 'ReghTDC' in label: results = whereParameterEquals(results, 'reg_h', 1) results = whereParameterEquals(results, 'ratio', 1) elif 'TDRCC' in label: results = whereParameterEquals(results, 'reg_h', 0.8) results = whereParameterEquals(results, 'ratio', 1) elif 'TDC' in label: results = whereParameterGreaterEq(results, 'ratio', 1) left, right = tee(results) best_line = getBest(right).mean() best = np.mean(best_line) for result in left: # print(label, result.params) shade = 0.12 line = result.mean() if np.mean(line) == best: shade = 1 plotBest(result, ax, label=label, color=color, alphaMain=shade, dashed=False) bounds.append(best_line[0])
b = plot(results, ax, window=window, smoothing=smoothing, label=label, color=color, alpha=0.2, dashed=False, bestBy=bestBy) bounds.append(b) if __name__ == "__main__": ax = plt.gca() f = plt.gcf() bounds = [] if lstd_baseline: path = f'experiments/batch/{problem}/lstd.json' lstd_exp = loadExperiment(path) LSTD_res = loadResults(lstd_exp, errorfile) LSTD_best = getBest(LSTD_res) b = plotBest(LSTD_best, ax, window=window, color=colors['LSTD'], label='LSTD', alphaMain=0.5, dashed=True) bounds.append(b) for alg in algorithms: if stepsize == 'constant': exp_path = f'experiments/batch/{problem}/{alg}/{alg}.json' else: exp_path = f'experiments/batch/{problem}/{alg}/{alg}{stepsize}.json' if '_h' in alg or alg == 'td' or alg == 'vtrace': generatePlotSSA(ax, exp_path, bounds) else: generatePlotTTA(ax, exp_path, bounds)
curves = np.zeros( (len(stepsizes), len(algorithms) + 1, len(problems), params, 2)) total_runs = 0 for k, problem in enumerate(problems): for i, ss in enumerate(stepsizes): td_idx = len(algorithms) if ss == 'constant': td_path = f'experiments/stepsizes/{problem}/td/td.json' else: td_path = f'experiments/stepsizes/{problem}/td/td{ss}.json' exp = loadExperiment(td_path) results = loadResults(exp, errorfile) best = getBest(results) curve = best.mean() curves[i, td_idx, k, :, 0] = np.mean(curve) / curve[0] curves[i, td_idx, k, :, 1] = np.mean(best.stderr()) * np.sqrt(best.runs()) for j, alg in enumerate(algorithms): print(problem, alg, ss) if alg == 'htd' and problem in on_policy_problems: curves[i, j, k] = curves[i, td_idx, k] continue if ss == 'constant': exp_paths = glob.glob( f'experiments/stepsizes/{problem}/{alg}/{alg}.json')
for i, alg in enumerate(algorithms): for j, problem in enumerate(problems): exp_path = f'experiments/stepsizes/{problem}/{alg}/{alg}{stepsize}.json' try: exp = loadExperiment(exp_path) except: continue results = loadResults(exp, errorfile) if alg == 'td' or alg == 'vtrace': const = results else: const = whereParameterGreaterEq(results, 'ratio', 1) const = whereParameterEquals(const, 'reg_h', 0.8) best = getBest(const) metric = np.mean # best = getBestEnd(const) # metric = lambda m: np.mean(m[-(int(len(m))):]) mean = metric(best.mean()) stderr = metric(best.stderr()) table[i, j] = [mean, stderr] htd_idx = indexOf(algorithms, 'htd') vtrace_idx = indexOf(algorithms, 'vtrace') td_idx = indexOf(algorithms, 'td') for j, problem in enumerate(problems): if htd_idx is not None and table[htd_idx, j, 0] == 0:
td_exp = loadExperiment(f'experiments/reward_scale/{problem}/td/td.json') scales = td_exp._d['metaParameters']['reward_scale'] tdrc_exp = loadExperiment( f'experiments/reward_scale/{problem}/regh_tdc/regh_tdc.json') betas = tdrc_exp._d['metaParameters']['reg_h'] mat = np.zeros((len(scales), len(betas))) xs = [] ys = [] alphas = [] for i, scale in enumerate(scales): td_results = loadResults(td_exp, errorfile) td_results = whereParameterEquals(td_results, 'reward_scale', scale) best_td = getBest(td_results) best_td_mean = np.mean(best_td.mean()) best_td_std = np.mean( np.sqrt(best_td.stderr() * np.sqrt(best_td.runs()))) tdrc_results = loadResults(tdrc_exp, errorfile) tdrc_results = whereParameterEquals(tdrc_results, 'reward_scale', scale) tdrc_split = splitOverParameter(tdrc_results, 'reg_h') for j, beta in enumerate(betas): results = tdrc_split[beta] best_tdrc = getBest(results) mean = np.mean(best_tdrc.mean())
import os import sys import json import numpy as np sys.path.append(os.getcwd()) from src.analysis.results import loadResults, getBest from src.utils.model import loadExperiment from src.utils.path import up, fileName exp_paths = sys.argv[1:] for exp_path in exp_paths: exp = loadExperiment(exp_path) results = loadResults(exp, 'rmspbe_summary.npy') best = getBest(results) print('---------------------') print('agent:', exp.agent) print(best.params) f = fileName(exp_path) new_path = up(exp_path) + '/best_rmspbe_auc/' + f d = exp._d d['metaParameters'] = best.params os.makedirs(up(new_path), exist_ok=True) with open(new_path, 'w') as f: json.dump(d, f, indent=4)