def generatePlotTTA(ax, exp_paths, bounds): for exp_path in exp_paths: exp = loadExperiment(exp_path) results = loadResults(exp, errorfile) const, unconst = tee(results) color = colors[exp.agent] label = exp.agent if error == 'rmsve': rmspbe = loadResults(exp, 'rmspbe_summary.npy') rmspbe_unconst, rmspbe_const = tee(rmspbe) rmspbe_const = whereParameterGreaterEq(rmspbe_const, 'ratio', 1) best_rmspbe_unconst = getBest(rmspbe_unconst) best_rmspbe_const = getBest(rmspbe_const) best_unconst = find(unconst, best_rmspbe_unconst) best_const = find(const, best_rmspbe_const) elif error == 'rmspbe': const = whereParameterGreaterEq(const, 'ratio', 1) best_unconst = getBest(unconst) best_const = getBest(const) b = plotBest(best_unconst, ax, label=label + '_unc', color=color, dashed=True) bounds.append(b) b = plotBest(best_const, ax, label=label, color=color, dashed=False) bounds.append(b)
def generatePlot(exp_paths): ax = plt.gca() # ax.semilogx() for exp_path in exp_paths: exp = loadExperiment(exp_path) rmsve = loadResults(exp, 'errors_summary.npy') rmspbe = loadResults(exp, 'rmspbe_summary.npy') # if exp.agent == 'TDadagrad': # continue # best PBE using AUC best = getBest(rmspbe) best_rmsve = find(rmsve, best) use_ideal_h = exp._d['metaParameters'].get('use_ideal_h', False) dashed = use_ideal_h color = colors[exp.agent] label = exp.agent.replace('adagrad', '') if use_ideal_h: label += '-h*' plotBest(best_rmsve, ax, label=label, color=color, dashed=dashed) # plt.show() save(exp, f'rmsve_over_rmspbe', type='svg') plt.clf()
def generatePlot(exp_paths): ax = plt.gca() # ax.semilogx() for exp_path in exp_paths: exp = loadExperiment(exp_path) if exp.agent == 'TDadagrad': continue use_ideal_h = exp._d['metaParameters'].get('use_ideal_h', False) dashed = use_ideal_h color = colors[exp.agent] # load the errors and hnorm files errors = loadResults(exp, 'errors_summary.npy') results = loadResults(exp, 'ndh_summary.npy') # choose the best parameters from the _errors_ best = getBestEnd(errors) best_ndh = find(results, best) label = exp.agent.replace('adagrad', '') if use_ideal_h: label += '-h*' plotBest(best_ndh, ax, label=label, color=color, dashed=dashed) # plt.show() save(exp, f'norm_delta-hat') plt.clf()
def generatePlotTTA(ax, exp_path, bounds): exp = loadExperiment(exp_path) results = loadResults(exp, errorfile) const, unconst = tee(results) color = colors[exp.agent] label = exp.agent const = whereParameterGreaterEq(const, 'ratio', 1) if 'ReghTDC' in label: const = whereParameterEquals(const, 'reg_h', 0.8) best_const = getBest(const, bestBy=bestBy) best_unconst = getBest(unconst, bestBy=bestBy) if show_unconst and best_const != best_unconst: b = plotBest(best_unconst, ax, window=window, smoothing=smoothing, label=label + '_unc', color=color, alpha=0.2, dashed=True) bounds.append(b) b = plotBest(best_const, ax, window=window, smoothing=smoothing, label=label, color=color, alpha=0.2, dashed=False) bounds.append(b)
def generatePlot(ax, exp_paths, bounds): for exp_path in exp_paths: exp = ExperimentModel.load(exp_path) raise Exception( 'Set the name of the results file saved from these experiments') results = loadResults(exp, 'results.npy') results = whereParameterEquals(results, 'initial_value', 0) best = getBest(results) b = plotBest(best, ax, label='TD', color='yellow', dashed=False) bounds.append(b)
def generatePlot(exp_paths): ax = plt.gca() # ax.semilogx() for exp_path in exp_paths: exp = loadExperiment(exp_path) # load the errors and hnorm files errors = loadResults(exp, 'errors_summary.npy') results = loadResults(exp, 'hnorm_summary.npy') # choose the best parameters from the _errors_ best = getBestEnd(errors) best_hnorm = find(results, best) label = fileName(exp_path).replace('.json', '') plotBest(best_hnorm, ax, label=label) plt.show() # save(exp, f'learning-curve') plt.clf()
def generatePlot(ax, exp_paths, bounds): for exp_path in exp_paths: exp = loadExperiment(exp_path) results = loadResults(exp, errorfile) color = colors[exp.agent] label = exp.agent results = whereParameterEquals(results, 'batch_size', 4) results = whereParameterLesserEq(results, 'ratio', 8) results = whereParameterLesserEq(results, 'alpha', 0.5) results = where(results, lambda r: r.params.get('ratio', 1) * r.params['alpha'] <= 1) if 'ReghTDC' in label: results = whereParameterEquals(results, 'reg_h', 1) results = whereParameterEquals(results, 'ratio', 1) elif 'TDRCC' in label: results = whereParameterEquals(results, 'reg_h', 0.8) results = whereParameterEquals(results, 'ratio', 1) elif 'TDC' in label: results = whereParameterGreaterEq(results, 'ratio', 1) left, right = tee(results) best_line = getBest(right).mean() best = np.mean(best_line) for result in left: # print(label, result.params) shade = 0.12 line = result.mean() if np.mean(line) == best: shade = 1 plotBest(result, ax, label=label, color=color, alphaMain=shade, dashed=False) bounds.append(best_line[0])
def generatePlot(exp_path): ax = plt.gca() # ax.semilogx() exp = loadExperiment(exp_path) # load the errors and hnorm files errors = loadResults(exp, 'errors_summary.npy') results = loadResults(exp, 'stepsize_summary.npy') # choose the best parameters from the _errors_ best = getBestEnd(errors) best_ss = find(results, best) alg = exp.agent.replace('adagrad', '') plotBest(best_ss, ax, label=['w', 'h']) ax.set_ylim([0, 4]) print(alg) # plt.show() save(exp, f'stepsizes-{alg}') plt.clf()
def generatePlot(ax, exp_paths, bounds): for exp_path in exp_paths: exp = ExperimentModel.load(exp_path) results = loadResults(exp, 'return_summary.npy') # optionally force epsilon to be 0.15 # results = whereParameterEquals(results, 'epsilon', 0.15) best = getBest(results) print('best parameters:', exp_path) print(best.params) alg = exp.agent b = plotBest(best, ax, label=alg, color=colors[alg], dashed=False) bounds.append(b)
def generatePlot(ax, exp_paths, bounds): for exp_path in exp_paths: exp = loadExperiment(exp_path) variance = loadResults(exp, 'expupd_summary.npy') agent = exp.agent color = colors[agent] label = agent best = getBest(variance) if 'GTD2' in agent or 'TDC' in agent: best.reducer(lambda m: m[:, 1]) else: best.reducer(lambda m: m[:, 1]) b = plotBest(best, ax, color=color, label=label) bounds.append(b)
def generatePlot(ax, exp_paths, bounds): for exp_path in exp_paths: exp = ExperimentModel.load(exp_path) results = loadResults(exp, 'return_summary.npy') results = whereParameterEquals(results, 'epsilon', 0.05) param_values = splitOverParameter(results, PARAM) for key, param_results in param_values.items(): best = getBest(param_results) print('best parameters:', exp_path, f'{PARAM}={key}') print(best.params) alg = exp.agent b = plotBest(best, ax, label=alg + f' {PARAM}={key}', dashed=False) bounds.append(b)
def generatePlot(ax, exp_path, bounds): exp = loadExperiment(exp_path) results = loadResults(exp, 'variance_summary.npy') best = getBest(results) best.reducer(lambda m: m[:, 0]) color = colors[exp.agent] label = exp.agent b = plotBest(best, ax, window=window, smoothing=smoothing, label=label, color=color, dashed=False) bounds.append(b)
def generatePlotSSA(ax, exp_paths, bounds): for exp_path in exp_paths: if 'amsgrad' in exp_path: continue exp = loadExperiment(exp_path) results = loadResults(exp, errorfile) stepsizes = loadResults(exp, 'stepsize_summary.npy') color = colors[exp.agent] label = exp.agent best_error = getBest(results) best = find(stepsizes, best_error) b = plotBest(best, ax, label=label, color=color, dashed=False) bounds.append(b)
def generatePlotSSA(ax, exp_paths, bounds): for exp_path in exp_paths: exp = loadExperiment(exp_path) results = loadResults(exp, errorfile) color = colors[exp.agent] label = exp.agent if error == 'rmsve': rmspbe = loadResults(exp, 'rmspbe_summary.npy') best_rmspbe = getBest(rmspbe) best = find(results, best_rmspbe) elif error == 'rmspbe': best = getBest(results) b = plotBest(best, ax, label=label, color=color, dashed=False) bounds.append(b)
def generatePlotTTA(ax, exp_paths, bounds): for exp_path in exp_paths: if 'amsgrad' in exp_path: continue exp = loadExperiment(exp_path) results = loadResults(exp, errorfile) stepsizes = loadResults(exp, 'stepsize_summary.npy') results = whereParameterEquals(results, 'ratio', 1) color = colors[exp.agent] label = exp.agent best_error = getBest(results) best_stepsize = find(stepsizes, best_error) b = plotBest(best_stepsize, ax, label=[label + '_w', label + '_h'], color=color, dashed=[False, True]) bounds.append(b)
if __name__ == "__main__": ax = plt.gca() f = plt.gcf() bounds = [] if lstd_baseline: path = f'experiments/batch/{problem}/lstd.json' lstd_exp = loadExperiment(path) LSTD_res = loadResults(lstd_exp, errorfile) LSTD_best = getBest(LSTD_res) b = plotBest(LSTD_best, ax, window=window, color=colors['LSTD'], label='LSTD', alphaMain=0.5, dashed=True) bounds.append(b) for alg in algorithms: if stepsize == 'constant': exp_path = f'experiments/batch/{problem}/{alg}/{alg}.json' else: exp_path = f'experiments/batch/{problem}/{alg}/{alg}{stepsize}.json' if '_h' in alg or alg == 'td' or alg == 'vtrace': generatePlotSSA(ax, exp_path, bounds) else: generatePlotTTA(ax, exp_path, bounds) lower = min(map(lambda x: x[0], bounds)) * 0.9 upper = max(map(lambda x: x[1], bounds)) * 1.05
def generatePlot(exp_paths): f, axes = plt.subplots(2, 2) # get LSTD solution path = up(up(first(exp_paths))) + '/lstd.json' exp = loadExperiment(path) LSTD_rmsve_results = loadResults(exp, 'errors_summary.npy') LSTD_rmspbe_results = loadResults(exp, 'rmspbe_summary.npy') LSTD_rmsve = metric(LSTD_rmsve_results) LSTD_rmspbe = metric(LSTD_rmspbe_results) rmspbe_bounds = [] rmsve_bounds = [] bounds = plotBest(LSTD_rmspbe, axes[0, 0], color=colors['LSTD'], label='LSTD', alphaMain=0.5) rmspbe_bounds.append(bounds) bounds = plotBest(LSTD_rmspbe, axes[0, 1], color=colors['LSTD'], label='LSTD', alphaMain=0.5) rmspbe_bounds.append(bounds) bounds = plotBest(LSTD_rmsve, axes[1, 0], color=colors['LSTD'], label='LSTD', alphaMain=0.5) rmsve_bounds.append(bounds) bounds = plotBest(LSTD_rmsve, axes[1, 1], color=colors['LSTD'], label='LSTD', alphaMain=0.5) rmsve_bounds.append(bounds) # RMSPBE plots ax = axes[0, 0] for exp_path in exp_paths: exp = loadExperiment(exp_path) results = loadResults(exp, 'rmspbe_summary.npy') const, unconst = tee(results) const = whereParameterGreaterEq(const, 'ratio', 1) use_ideal_h = exp._d['metaParameters'].get('use_ideal_h', False) agent = exp.agent if 'SmoothTDC' in agent: average = exp._d['metaParameters']['averageType'] agent += '_' + average color = colors[agent] label = agent.replace('adagrad', '') if not (exp.agent in ['TDadagrad', 'TDschedule', 'TD', 'TDamsgrad'] or use_ideal_h): if UNCONSTRAINED: bounds = plot(unconst, ax, label=label + '_unc', color=color, dashed=True, bestBy=bestBy) rmspbe_bounds.append(bounds) bounds = plot(const, ax, label=label, color=color, dashed=False, bestBy=bestBy) rmspbe_bounds.append(bounds) else: bounds = plot(unconst, ax, label=label, color=color, dashed=False, bestBy=bestBy) rmspbe_bounds.append(bounds) ax.set_ylabel("MSPBE") ax.set_title("MSPBE") ax = axes[0, 1] for exp_path in exp_paths: exp = loadExperiment(exp_path) rmsve = loadResults(exp, 'errors_summary.npy') results = loadResults(exp, 'rmspbe_summary.npy') const, unconst = tee(rmsve) const_res, unconst_res = tee(results) const = whereParameterGreaterEq(const, 'ratio', 1) use_ideal_h = exp._d['metaParameters'].get('use_ideal_h', False) agent = exp.agent if 'SmoothTDC' in agent: average = exp._d['metaParameters']['averageType'] agent += '_' + average color = colors[agent] label = agent.replace('adagrad', '') if not (exp.agent in ['TDadagrad', 'TDschedule', 'TD', 'TDamsgrad'] or use_ideal_h): best = metric(const) best_unc = metric(unconst) best_rmspbe = find(const_res, best) best_rmspbe_unc = find(unconst_res, best_unc) bounds = plotBest(best_rmspbe, ax, label=label, color=color, dashed=False) rmspbe_bounds.append(bounds) if UNCONSTRAINED: bounds = plotBest(best_rmspbe_unc, ax, label=label + '_unc', color=color, dashed=True) rmspbe_bounds.append(bounds) else: best = metric(unconst) best_rmspbe = find(unconst_res, best) bounds = plotBest(best_rmspbe, ax, label=label, color=color, dashed=False) rmspbe_bounds.append(bounds) ax.set_title("MSVE") # RMSVE plots ax = axes[1, 0] for exp_path in exp_paths: exp = loadExperiment(exp_path) rmsve = loadResults(exp, 'errors_summary.npy') rmspbe = loadResults(exp, 'rmspbe_summary.npy') const, unconst = tee(rmspbe) const_res, unconst_res = tee(rmsve) const = whereParameterGreaterEq(const, 'ratio', 1) use_ideal_h = exp._d['metaParameters'].get('use_ideal_h', False) agent = exp.agent if 'SmoothTDC' in agent: average = exp._d['metaParameters']['averageType'] agent += '_' + average color = colors[agent] label = agent.replace('adagrad', '') if not (exp.agent in ['TDadagrad', 'TDschedule', 'TD', 'TDamsgrad'] or use_ideal_h): # best PBE using AUC best = metric(const) best_unc = metric(unconst) best_rmsve = find(const_res, best) best_rmsve_unc = find(unconst_res, best_unc) print('rmsve_over_rmspbe') print(label, best_rmsve.params) print(label, best_rmsve_unc.params) bounds = plotBest(best_rmsve, ax, label=label, color=color, dashed=False) rmsve_bounds.append(bounds) if UNCONSTRAINED: bounds = plotBest(best_rmsve_unc, ax, label=label + '_unc', color=color, dashed=True) rmsve_bounds.append(bounds) else: best = metric(unconst) best_rmsve = find(unconst_res, best) bounds = plotBest(best_rmsve, ax, label=label, color=color, dashed=False) rmsve_bounds.append(bounds) ax.set_ylabel("MSVE") ax = axes[1, 1] for exp_path in exp_paths: exp = loadExperiment(exp_path) results = loadResults(exp) const, unconst = tee(results) const = whereParameterGreaterEq(const, 'ratio', 1) use_ideal_h = exp._d['metaParameters'].get('use_ideal_h', False) agent = exp.agent if 'SmoothTDC' in agent: average = exp._d['metaParameters']['averageType'] agent += '_' + average color = colors[agent] label = agent.replace('adagrad', '') if not (exp.agent in ['TDadagrad', 'TDschedule', 'TD', 'TDamsgrad'] or use_ideal_h): bounds = plot(const, ax, label=label, color=color, dashed=False, bestBy=bestBy) rmsve_bounds.append(bounds) if UNCONSTRAINED: bounds = plot(unconst, ax, label=label + '_unc', color=color, dashed=True, bestBy=bestBy) rmsve_bounds.append(bounds) else: bounds = plot(unconst, ax, label=label, color=color, dashed=False, bestBy=bestBy) rmsve_bounds.append(bounds) # rmspbe rmspbe_lower = min(map(lambda x: x[0], rmspbe_bounds)) * 0.9 rmspbe_upper = max(map(lambda x: x[1], rmspbe_bounds)) * 1.05 if rmspbe_lower < 0.01: rmspbe_lower = -0.01 axes[0, 0].set_ylim([rmspbe_lower, rmspbe_upper]) axes[0, 1].set_ylim([rmspbe_lower, rmspbe_upper]) # rmsve rmsve_lower = min(map(lambda x: x[0], rmsve_bounds)) * 0.9 rmsve_upper = max(map(lambda x: x[1], rmsve_bounds)) * 1.05 if rmsve_lower < 0.01: rmsve_lower = -0.01 axes[1, 0].set_ylim([0, 20]) axes[1, 1].set_ylim([0, 20])
def generatePlot(exp_paths): f, axes = plt.subplots(2, 2) # ax.semilogx() # RMSPBE plots ax = axes[0, 0] rmspbe_bounds = [] for exp_path in exp_paths: exp = loadExperiment(exp_path) results = loadResults(exp, 'rmspbe_summary.npy') use_ideal_h = exp._d['metaParameters'].get('use_ideal_h', False) dashed = False color = colors[exp.agent] label = exp.agent.replace('adagrad', '') # if use_ideal_h: # label += '-h*' bounds = plot(results, ax, label=label, color=color, dashed=dashed) rmspbe_bounds.append(bounds) ax.set_ylabel("RMSPBE") ax.set_title("RMSPBE") ax = axes[0, 1] for exp_path in exp_paths: exp = loadExperiment(exp_path) rmsve = loadResults(exp, 'errors_summary.npy') results = loadResults(exp, 'rmspbe_summary.npy') best = getBestEnd(rmsve) best_rmspbe = find(results, best) use_ideal_h = exp._d['metaParameters'].get('use_ideal_h', False) dashed = False color = colors[exp.agent] label = exp.agent.replace('adagrad', '') # if use_ideal_h: # label += '-h*' bounds = plotBest(best_rmspbe, ax, label=label, color=color, dashed=dashed) rmspbe_bounds.append(bounds) ax.set_title("RMSVE") # RMSVE plots ax = axes[1, 0] rmsve_bounds = [] for exp_path in exp_paths: exp = loadExperiment(exp_path) rmsve = loadResults(exp, 'errors_summary.npy') rmspbe = loadResults(exp, 'rmspbe_summary.npy') # if exp.agent == 'TDadagrad': # continue # best PBE using AUC best = getBestEnd(rmspbe) best_rmsve = find(rmsve, best) use_ideal_h = exp._d['metaParameters'].get('use_ideal_h', False) dashed = False color = colors[exp.agent] label = exp.agent.replace('adagrad', '') # if use_ideal_h: # label += '-h*' bounds = plotBest(best_rmsve, ax, label=label, color=color, dashed=dashed) rmsve_bounds.append(bounds) ax.set_ylabel("RMSVE") ax = axes[1, 1] for exp_path in exp_paths: exp = loadExperiment(exp_path) results = loadResults(exp) use_ideal_h = exp._d['metaParameters'].get('use_ideal_h', False) dashed = False color = colors[exp.agent] label = exp.agent.replace('adagrad', '') # if use_ideal_h: # label += '-h*' bounds = plot(results, ax, label=label, color=color, dashed=dashed) rmsve_bounds.append(bounds) # rmspbe rmspbe_lower = min(map(lambda x: x[0], rmspbe_bounds)) * 0.9 rmspbe_upper = max(map(lambda x: x[1], rmspbe_bounds)) * 1.05 axes[0, 0].set_ylim([rmspbe_lower, rmspbe_upper]) axes[0, 1].set_ylim([rmspbe_lower, rmspbe_upper]) # rmsve rmsve_lower = min(map(lambda x: x[0], rmsve_bounds)) * 0.9 rmsve_upper = max(map(lambda x: x[1], rmsve_bounds)) * 1.05 axes[1, 0].set_ylim([rmsve_lower, rmsve_upper]) axes[1, 1].set_ylim([rmsve_lower, rmsve_upper]) plt.show()
best = getBest(results) all_results.append(best) if best_result is None: best_result = best elif np.mean(best.mean()) < np.mean(best_result.mean()): best_result = best return all_results, best_result tdc_results, best_tdc = getResultsAndBest(tdc) gtd2_results, best_gtd2 = getResultsAndBest(gtd2) for r in tdc_results: if r == best_tdc: plotBest(r, ax, 'blue', r.exp.agent, stderr=False) else: plotBest(r, ax, 'blue', '_', alphaMain=0.15, stderr=False) for r in gtd2_results: if r == best_gtd2: plotBest(r, ax, 'red', r.exp.agent, stderr=False) else: plotBest(r, ax, 'red', '_', alphaMain=0.15, stderr=False) # save(exp, 'lambda') # ax.set_ylim([0, 1]) plt.show()
for i, alg in enumerate(algorithms): for j, problem in enumerate(problems): bounds = [] exp_paths = glob.glob(f'experiments/stepsizes/{problem}/{alg}/*.json') if len(exp_paths) == 0: continue exp = loadExperiment(exp_paths[0]) path = up(up(first(exp_paths))) + '/lstd.json' lstd_exp = loadExperiment(path) LSTD_res = loadResults(lstd_exp, errorfile) LSTD_best = getBest(LSTD_res) b = plotBest(LSTD_best, axes[i, j], color=colors['LSTD'], label='LSTD', alphaMain=0.5) bounds.append(b) if '_h' in alg or alg == 'td': generatePlotSSA(axes[i, j], exp_paths, bounds) else: generatePlotTTA(axes[i, j], exp_paths, bounds) lower = min(map(lambda x: x[0], bounds)) * 0.9 upper = max(map(lambda x: x[1], bounds)) * 1.05 if lower < 0.01: lower = -0.01 if i == 0: axes[i, j].set_title(f'{problem}\n{alg}')