def generatePlot(exp_paths): ax = plt.gca() # ax.semilogx() for exp_path in exp_paths: exp = loadExperiment(exp_path) if exp.agent == 'TDadagrad': continue use_ideal_h = exp._d['metaParameters'].get('use_ideal_h', False) dashed = use_ideal_h color = colors[exp.agent] # load the errors and hnorm files errors = loadResults(exp, 'errors_summary.npy') results = loadResults(exp, 'ndh_summary.npy') # choose the best parameters from the _errors_ best = getBestEnd(errors) best_ndh = find(results, best) label = exp.agent.replace('adagrad', '') if use_ideal_h: label += '-h*' plotBest(best_ndh, ax, label=label, color=color, dashed=dashed) # plt.show() save(exp, f'norm_delta-hat') plt.clf()
def generatePlotTTA(ax, exp_paths, bounds): for exp_path in exp_paths: exp = loadExperiment(exp_path) results = loadResults(exp, errorfile) const, unconst = tee(results) color = colors[exp.agent] label = exp.agent if error == 'rmsve': rmspbe = loadResults(exp, 'rmspbe_summary.npy') rmspbe_unconst, rmspbe_const = tee(rmspbe) rmspbe_const = whereParameterGreaterEq(rmspbe_const, 'ratio', 1) best_rmspbe_unconst = getBest(rmspbe_unconst) best_rmspbe_const = getBest(rmspbe_const) best_unconst = find(unconst, best_rmspbe_unconst) best_const = find(const, best_rmspbe_const) elif error == 'rmspbe': const = whereParameterGreaterEq(const, 'ratio', 1) best_unconst = getBest(unconst) best_const = getBest(const) b = plotBest(best_unconst, ax, label=label + '_unc', color=color, dashed=True) bounds.append(b) b = plotBest(best_const, ax, label=label, color=color, dashed=False) bounds.append(b)
def generatePlotTTA(ax, exp_paths, bestBy, bounds): ax.set_xscale("log", basex=2) for exp_path in exp_paths: if 'amsgrad' in exp_path: continue exp = loadExperiment(exp_path) rmsve = loadResults(exp, 'errors_summary.npy') rmspbe = loadResults(exp, 'rmspbe_summary.npy') agent = exp.agent if 'SmoothTDC' in agent: average = exp._d['metaParameters']['averageType'] agent += '_' + average color = colors[agent] label = agent b = plotSensitivity(rmsve, 'ratio', ax, overStream=rmspbe, color=color, label=label, bestBy=bestBy) bounds.append(b)
def generatePlotSTA(ax, exp_paths, bestBy, bounds): for exp_path in exp_paths: exp = loadExperiment(exp_path) rmsve = loadResults(exp, 'errors_summary.npy') rmspbe = loadResults(exp, 'rmspbe_summary.npy') agent = exp.agent if 'SmoothTDC' in agent: average = exp._d['metaParameters']['averageType'] agent += '_' + average color = colors[agent] label = agent if bestBy == 'end': metric = lambda m: np.mean(m[-int(m.shape[0] * .1):]) best_rmspbe = getBestEnd(rmspbe) best = find(rmsve, best_rmspbe) elif bestBy == 'auc': metric = np.mean best_rmspbe = getBest(rmspbe) best = find(rmsve, best_rmspbe) m = metric(best.mean()) ax.hlines(m, 2**-6, 2**6, color=color, label=label) bounds.append([m, m])
def generatePlot(exp_paths): ax = plt.gca() # ax.semilogx() for exp_path in exp_paths: exp = loadExperiment(exp_path) rmsve = loadResults(exp, 'errors_summary.npy') rmspbe = loadResults(exp, 'rmspbe_summary.npy') # if exp.agent == 'TDadagrad': # continue # best PBE using AUC best = getBest(rmspbe) best_rmsve = find(rmsve, best) use_ideal_h = exp._d['metaParameters'].get('use_ideal_h', False) dashed = use_ideal_h color = colors[exp.agent] label = exp.agent.replace('adagrad', '') if use_ideal_h: label += '-h*' plotBest(best_rmsve, ax, label=label, color=color, dashed=dashed) # plt.show() save(exp, f'rmsve_over_rmspbe', type='svg') plt.clf()
def generatePlotTTA(ax, exp_path, bounds): exp = loadExperiment(exp_path) results = loadResults(exp, errorfile) const, unconst = tee(results) color = colors[exp.agent] label = exp.agent const = whereParameterGreaterEq(const, 'ratio', 1) if 'ReghTDC' in label: const = whereParameterEquals(const, 'reg_h', 0.8) best_const = getBest(const, bestBy=bestBy) best_unconst = getBest(unconst, bestBy=bestBy) if show_unconst and best_const != best_unconst: b = plotBest(best_unconst, ax, window=window, smoothing=smoothing, label=label + '_unc', color=color, alpha=0.2, dashed=True) bounds.append(b) b = plotBest(best_const, ax, window=window, smoothing=smoothing, label=label, color=color, alpha=0.2, dashed=False) bounds.append(b)
def generatePlotTTA(ax, exp_path, bounds): exp = loadExperiment(exp_path) results = loadResults(exp, errorfile) const, unconst = tee(results) color = colors[exp.agent] label = rename(exp.agent) if 'ReghTDC' in exp.agent: const = whereParameterEquals(const, 'ratio', 1) const = whereParameterEquals(const, 'reg_h', 1) elif 'TDRCC' in exp.agent: const = whereParameterEquals(const, 'ratio', 1) # const = whereParameterEquals(const, 'reg_h', 0.8) const = whereParameterGreaterEq(const, 'reg_h', 0.01) elif 'TDC' in exp.agent: const = whereParameterGreaterEq(const, 'ratio', 1) if show_unconst: b = plotSensitivity(unconst, param, ax, stderr=stderr, color=color, label=label + '_unc', bestBy=bestBy, dashed=True) bounds.append(b) b = plotSensitivity(const, param, ax, stderr=stderr, color=color, label=label, bestBy=bestBy) bounds.append(b)
def baseline(ax, exp_path, values, bounds): exp = loadExperiment(path) results = loadResults(exp, errorfile) if 'Regh' in exp.agent: results = whereParameterEquals(results, 'reg_h', 0.8) results = whereParameterEquals(results, 'ratio', 1) elif 'TDC' in exp.agent or 'GTD2' in exp.agent: results = whereParameterEquals(results, 'ratio', 1) if bestBy == 'end': metric = lambda m: np.mean(m[-int(m.shape[0] * .1):]) best = getBestEnd(results) elif bestBy == 'auc': metric = np.mean best = getBest(results) color = colors[exp.agent] label = exp.agent m = metric(best.mean()) low = min(values) high = max(values) ax.hlines(m, low, high, color=color, label=label, linewidth=4, linestyle=':') bounds.append((m, m))
def generatePlot(exp_paths): ax = plt.gca() # ax.semilogx() for exp_path in exp_paths: exp = loadExperiment(exp_path) results = loadResults(exp) use_ideal_h = exp._d['metaParameters'].get('use_ideal_h', False) dashed = use_ideal_h color = colors[exp.agent] label = exp.agent.replace('adagrad', '') if use_ideal_h: label += '-h*' plot(results, ax, label=label, color=color, dashed=dashed, bestBy='end') # plt.show() save(exp, f'rmsve_learning-curve', type='svg') plt.clf()
def generatePlot(exp_paths): ax = plt.gca() bounds = [] i = -1 for exp_path in exp_paths: i += 1 exp = loadExperiment(exp_path) results = loadResults(exp, 'errors_summary.npy') param_dict = splitOverParameter(results, 'ratio') for key in param_dict: bound = plotSensitivity(param_dict[key], 'alpha', ax, color=colors[i], bestBy='end') bounds.append(bound) lower = min(map(lambda x: x[0], bounds)) upper = max(map(lambda x: x[1], bounds)) ax.set_ylim([lower, upper]) ax.set_xscale("log", basex=2) # plt.show() save(exp, f'alpha_over_beta_rmsve', type='svg') plt.clf()
def generatePlot(exp_paths): b_fig = plt.figure("Mean Update Variance") w_fig = plt.figure("W Update Variance") h_fig = plt.figure("H Update Variance") b_ax = b_fig.add_subplot(1, 1, 1) w_ax = w_fig.add_subplot(1, 1, 1) h_ax = h_fig.add_subplot(1, 1, 1) for exp_path in exp_paths: exp = loadExperiment(exp_path) results = loadResults(exp, 'variance_summary.npy') both, w, h = tee(results, 3) both = map(lambda r: r.reducer(columnReducer(0)), both) w = map(lambda r: r.reducer(columnReducer(1)), w) h = map(lambda r: r.reducer(columnReducer(2)), h) plot(both, b_ax) plot(w, w_ax) plot(h, h_ax) plt.show() exit() exp_name = exp.getExperimentName() save_path = f'experiments/{exp_name}' os.makedirs(save_path, exist_ok=True) b_fig.savefig(f'{save_path}/both_variance-curve.pdf') w_fig.savefig(f'{save_path}/w_variance-curve.pdf') h_fig.savefig(f'{save_path}/h_variance-curve.pdf')
def generatePlot(exp_paths): ax = plt.gca() bounds = [] for exp_path in exp_paths: exp = loadExperiment(exp_path) results = loadResults(exp) if exp.agent == 'TDadagrad': continue color = colors[exp.agent] label = exp.agent.replace('adagrad', '') bound = plotSensitivity(results, 'h_variance', ax, label=label, color=color, bestBy='end') bounds.append(bound) # lower = min(map(lambda x: x[0], bounds)) # upper = max(map(lambda x: x[1], bounds)) # ax.set_ylim([lower, upper]) ax.set_xscale("log", basex=2) plt.show() # save(exp, f'alpha-sensitivity') plt.clf()
def generatePlot(exp_paths): ax = plt.gca() bounds = [] for exp_path in exp_paths: exp = loadExperiment(exp_path) results = loadResults(exp) if exp.agent == 'TDadagrad': continue use_ideal_h = exp._d['metaParameters'].get('use_ideal_h', False) dashed = use_ideal_h color = colors[exp.agent] label = exp.agent.replace('adagrad', '') if use_ideal_h: label += '-h*' bound = plotSensitivity(results, 'h_variance', ax, label=label, color=color, dashed=dashed, bestBy='end') bounds.append(bound) lower = min(map(lambda x: x[0], bounds)) upper = max(map(lambda x: x[1], bounds)) ax.set_ylim([lower, upper]) ax.set_xscale("log", basex=2) # plt.show() save(exp, f'h_variance-sensitivity') plt.clf()
def generatePlotSSA(ax, exp_paths, bounds): for exp_path in exp_paths: if 'amsgrad' in exp_path: continue exp = loadExperiment(exp_path) results = loadResults(exp, errorfile) stepsizes = loadResults(exp, 'stepsize_summary.npy') color = colors[exp.agent] label = exp.agent best_error = getBest(results) best = find(stepsizes, best_error) b = plotBest(best, ax, label=label, color=color, dashed=False) bounds.append(b)
def generatePlotSSA(ax, exp_path, bounds): exp = loadExperiment(exp_path) results = loadResults(exp, errorfile) color = colors[exp.agent] label = exp.agent b = plot(results, ax, window=window, smoothing=smoothing, label=label, color=color, alpha=0.2, dashed=False, bestBy=bestBy) bounds.append(b)
def generatePlot(ax, exp_path, bounds): exp = loadExperiment(exp_path) results = loadResults(exp, errorfile) color = colors[exp.agent] label = exp.agent b = plotSensitivity(results, param, ax, color=color, label=label, bestBy=bestBy) bounds.append(b)
def generatePlotSSA(ax, exp_paths, bounds): for exp_path in exp_paths: exp = loadExperiment(exp_path) results = loadResults(exp, errorfile) color = colors[exp.agent] label = exp.agent if error == 'rmsve': rmspbe = loadResults(exp, 'rmspbe_summary.npy') best_rmspbe = getBest(rmspbe) best = find(results, best_rmspbe) elif error == 'rmspbe': best = getBest(results) b = plotBest(best, ax, label=label, color=color, dashed=False) bounds.append(b)
def generatePlot(exp_paths): ax = plt.gca() # ax.semilogx() for exp_path in exp_paths: exp = loadExperiment(exp_path) results = loadResults(exp, 'rmspbe_summary.npy') use_ideal_h = exp._d['metaParameters'].get('use_ideal_h', False) dashed = use_ideal_h label = exp.agent if use_ideal_h: label += '-h*' if use_ideal_h: continue if 'SmoothTDC' in exp.agent: agents = splitOverParameter(results, 'averageType') smooth_colors = { 'ema': 'pink', 'buffer': 'grey', 'window': 'black', } plot(agents['ema'], ax, label=label + '_ema', bestBy='auc', color=smooth_colors['ema'], dashed=dashed) # plot(agents['buffer'], ax, label=label + '_buffer', bestBy='auc', color=smooth_colors['buffer'], dashed=dashed) # plot(agents['window'], ax, label=label + '_window', bestBy='auc', color=smooth_colors['window'], dashed=dashed) else: color = colors[exp.agent] plot(results, ax, label=label, bestBy='auc', color=color, dashed=dashed) plt.show() exit() # save(exp, f'rmspbe', type='png') problem = fileName(exp.getExperimentName()) save_path = f'plots/' os.makedirs(save_path, exist_ok=True) fig = plt.gcf() fig.set_size_inches((13, 12), forward=False) plt.savefig(f'{save_path}/{problem}_rmspbe.png')
def generatePlot(exp_paths): ax = plt.gca() # ax.semilogx() for exp_path in exp_paths: exp = loadExperiment(exp_path) # load the errors and hnorm files errors = loadResults(exp, 'errors_summary.npy') results = loadResults(exp, 'hnorm_summary.npy') # choose the best parameters from the _errors_ best = getBestEnd(errors) best_hnorm = find(results, best) label = fileName(exp_path).replace('.json', '') plotBest(best_hnorm, ax, label=label) plt.show() # save(exp, f'learning-curve') plt.clf()
def generatePlotTTA(ax, exp_paths, bounds): for exp_path in exp_paths: if 'amsgrad' in exp_path: continue exp = loadExperiment(exp_path) results = loadResults(exp, errorfile) stepsizes = loadResults(exp, 'stepsize_summary.npy') results = whereParameterEquals(results, 'ratio', 1) color = colors[exp.agent] label = exp.agent best_error = getBest(results) best_stepsize = find(stepsizes, best_error) b = plotBest(best_stepsize, ax, label=[label + '_w', label + '_h'], color=color, dashed=[False, True]) bounds.append(b)
def generatePlot(exp_path): ax = plt.gca() # ax.semilogx() exp = loadExperiment(exp_path) # load the errors and hnorm files errors = loadResults(exp, 'errors_summary.npy') results = loadResults(exp, 'stepsize_summary.npy') # choose the best parameters from the _errors_ best = getBestEnd(errors) best_ss = find(results, best) alg = exp.agent.replace('adagrad', '') plotBest(best_ss, ax, label=['w', 'h']) ax.set_ylim([0, 4]) print(alg) # plt.show() save(exp, f'stepsizes-{alg}') plt.clf()
def generatePlot(ax, exp_path, bounds): exp = loadExperiment(exp_path) results = loadResults(exp, errorfile) results = addUpdateParam(results) color = colors[exp.agent] label = rename(exp.agent) b = plotSensitivity(results, "updates", ax, color=color, label=label, bestBy=bestBy) bounds.append(b)
def generatePlotTTA(ax, exp_paths, bestBy, bounds): for exp_path in exp_paths: exp = loadExperiment(exp_path) results = loadResults(exp, errorfile) color = colors[exp.agent] label = exp.agent dashed = False if 'TDC' in label: dashed = True results = whereParameterGreaterEq(results, 'ratio', 1) b = plot(results, ax, label=label, color=color, dashed=dashed, bestBy=bestBy) bounds.append(b)
def getResultsAndBest(exps): all_results = [] best_result = None for exp in exps: results = loadResults(exp) results = whereParameterEquals(results, 'lambda', LAMBDA) best = getBest(results) all_results.append(best) if best_result is None: best_result = best elif np.mean(best.mean()) < np.mean(best_result.mean()): best_result = best return all_results, best_result
def generatePlot(ax, exp_paths, bounds): for exp_path in exp_paths: exp = loadExperiment(exp_path) variance = loadResults(exp, 'expupd_summary.npy') agent = exp.agent color = colors[agent] label = agent best = getBest(variance) if 'GTD2' in agent or 'TDC' in agent: best.reducer(lambda m: m[:, 1]) else: best.reducer(lambda m: m[:, 1]) b = plotBest(best, ax, color=color, label=label) bounds.append(b)
def generatePlotTTA(ax, exp_paths, bestBy, bounds): ax.set_xscale("log", basex=2) for exp_path in exp_paths: exp = loadExperiment(exp_path) results = loadResults(exp, errorfile) agent = exp.agent color = colors[agent] label = agent b = plotSensitivity(results, 'reg_h', ax, reducer='best', color=color, label=label, bestBy=bestBy) bounds.append(b)
def generatePlot(ax, exp_path, bounds): exp = loadExperiment(exp_path) results = loadResults(exp, 'variance_summary.npy') best = getBest(results) best.reducer(lambda m: m[:, 0]) color = colors[exp.agent] label = exp.agent b = plotBest(best, ax, window=window, smoothing=smoothing, label=label, color=color, dashed=False) bounds.append(b)
def generatePlot(exp_paths): ax = plt.gca() # ax.semilogx() for exp_path in exp_paths: exp = loadExperiment(exp_path) results = loadResults(exp) is_h_star = exp.getPermutation(0)['metaParameters']['use_ideal_h'] agent = exp.agent.replace('adagrad', '') label = agent if is_h_star: label += '-h*' plot(results, ax, label=label) plt.show() # save(exp, f'learning-curve') plt.clf()
def generatePlotTTA(ax, exp_paths, bestBy, bounds): ax.set_xscale("log", basex=2) for exp_path in exp_paths: exp = loadExperiment(exp_path) results = loadResults(exp, errorfile) agent = exp.agent if 'SmoothTDC' in agent: average = exp._d['metaParameters']['averageType'] agent += '_' + average color = colors[agent] label = agent b = plotSensitivity(results, 'buffer', ax, color=color, label=label, bestBy=bestBy) bounds.append(b)
def generatePlot(ax, exp_paths, bounds): for exp_path in exp_paths: exp = loadExperiment(exp_path) results = loadResults(exp, errorfile) color = colors[exp.agent] label = exp.agent results = whereParameterEquals(results, 'batch_size', 4) results = whereParameterLesserEq(results, 'ratio', 8) results = whereParameterLesserEq(results, 'alpha', 0.5) results = where(results, lambda r: r.params.get('ratio', 1) * r.params['alpha'] <= 1) if 'ReghTDC' in label: results = whereParameterEquals(results, 'reg_h', 1) results = whereParameterEquals(results, 'ratio', 1) elif 'TDRCC' in label: results = whereParameterEquals(results, 'reg_h', 0.8) results = whereParameterEquals(results, 'ratio', 1) elif 'TDC' in label: results = whereParameterGreaterEq(results, 'ratio', 1) left, right = tee(results) best_line = getBest(right).mean() best = np.mean(best_line) for result in left: # print(label, result.params) shade = 0.12 line = result.mean() if np.mean(line) == best: shade = 1 plotBest(result, ax, label=label, color=color, alphaMain=shade, dashed=False) bounds.append(best_line[0])