type=str) parser.add_argument('--title', help='Plot title', default='Learning Curve', type=str) parser.add_argument('--smooth', action='store_true', default=False, help='Smooth Learning Curve') args = parser.parse_args() results = [] algos = [] for folder in args.log_dirs: timesteps = load_results(folder) results.append(timesteps) if folder.endswith('/'): folder = folder[:-1] algos.append(folder.split('/')[-1]) min_timesteps = np.inf # 'walltime_hrs', 'episodes' for plot_type in ['timesteps']: xy_list = [] for result in results: x, y = ts2xy(result, plot_type) if args.smooth: x, y = smooth((x, y), window=50) n_timesteps = x[-1]
def callback(_locals, _globals): """ Callback called at each step (for DQN an others) or after n steps (see ACER or PPO2) :param _locals: (dict) :param _globals: (dict) """ global n_steps, best_mean_reward, best_eval_mean_reward, prev_dt_eval # Print stats every 1000 calls total_reward = 0 mean_reward = 0 if (n_steps + 1) % 1000 == 0: for i in range(100): dones = False timesteps = 0 obs = test_env.reset() while not dones: action, _states = model.predict(obs) if model.use_action_repeat: for _ in range(1): obs, rewards, dones, info = test_env.step(action) total_reward += rewards timesteps += 1 if (timesteps == max_eval_timesteps): dones = True if (dones): break else: timesteps += 1 obs, rewards, dones, info = test_env.step(action) total_reward += rewards if (timesteps == max_eval_timesteps): dones = True if (dones): break mean_reward = total_reward / 100.0 dt_reward = mean_reward - prev_dt_eval dt_agent.update(dt_reward / 100.0) print("Mean reward last dt: {}".format(dt_reward)) prev_dt_eval = mean_reward new_dt = int(dt_agent.play()) + 1 model.action_repetition = new_dt print("Action repetition is :{}".format(model.action_repetition)) print("Steps: {} 100 Episode eval: {} Best eval {} ".format( n_steps, mean_reward, best_eval_mean_reward)) f.write("Steps: {} 100 Episode eval: {} Best eval {}\n".format( n_steps, mean_reward, best_eval_mean_reward)) if mean_reward > best_eval_mean_reward: best_eval_mean_reward = mean_reward # Example for saving best model print("Saving new best model") _locals['self'].save(log_dir + 'best_model_eval.pkl') log_data['dt'].append(model.action_repetition) log_data['eval'].append(mean_reward) log_data['timesteps'].append(model.num_timesteps) # Evaluate policy training performance if (n_steps + 1) % 1000 == 0: x, y = ts2xy(load_results(log_dir), 'timesteps') if len(x) > 0: mean_reward = np.mean(y[-100:]) print(x[-1], 'timesteps') print( "Best mean reward: {:.2f} - Last mean reward per episode: {:.2f}" .format(best_mean_reward, mean_reward)) # New best model, you could save the agent here if mean_reward > best_mean_reward: best_mean_reward = mean_reward # Example for saving best model print("Saving new best model") _locals['self'].save(log_dir + 'best_model.pkl') log_data['train'].append(mean_reward) n_steps += 1 # Returning False will stop training early return True
if __name__ == '__main__': parser = argparse.ArgumentParser() parser.add_argument('-f', '--folder', help='Log folder', type=str, default='trained_agents') args = parser.parse_args() log_dir = args.folder timesteps = 1e10 window_size = 50 W = load_results(log_dir) print("results: ", W) # save walltime to stats.csv df = pd.read_csv(log_dir + 'stats.csv') df["Train walltime (s)"] = W["t"].max() df.to_csv(log_dir + "stats.csv", index=False) print(df) # plot all training rewards results_plotter.plot_results([log_dir], timesteps, results_plotter.X_TIMESTEPS, "") plt.savefig(log_dir + "reward_vs_timesteps.png") # plt.show()
shifted_reward = np.nan * np.ones((int(2e6 / 51), 8)) shifted_reward[int(1e6 / 51):int(1e6 / 51) + ssac_size] = rewards fig, ax = smooth_bounded_curve( shifted_reward, time_steps=[51 * i for i in range(shifted_reward.shape[0])]) color_iter = iter(['b', 'g', 'y', 'm', 'c']) log_dir = script_path + '../rl-baselines-zoo/baseline_log2/' for algo in os.scandir(log_dir): try: df_list = [] min_length = float('inf') for entry in os.scandir(algo.path): df = load_results(entry.path) if len(df['r']) < min_length: min_length = len(df['r']) df_list.append(df) min_length = int(min_length) rewards = np.zeros((min_length, len(df_list))) for i, df in enumerate(df_list): rewards[:, i] = np.array(df['r'][:min_length]) print(print(algo.path), rewards[-1, :].mean(), rewards[-1, :].std()) smooth_bounded_curve(rewards[:min_length], time_steps=[51 * i for i in range(min_length)],
'-f', args.log_dir, '--algo', algo, '--env', env_id, '--no-render', '--seed', str(args.seed), '--verbose', '0', '--reward-log', reward_log ] if args.verbose >= 1: print('{}/{}'.format(idx + 1, n_experiments)) print("Evaluating {} on {}...".format(algo, env_id)) skip_eval = False if os.path.isdir(reward_log): try: x, y = ts2xy(load_results(reward_log), 'timesteps') skip_eval = len(x) > 0 except: pass if skip_eval: print("Skipping eval...") else: return_code = subprocess.call(['python', 'enjoy.py'] + arguments) x, y = ts2xy(load_results(reward_log), 'timesteps') if len(x) > 0: mean_reward = np.mean(y) std_reward = np.std(y) results['algo'].append(algo) results['env_id'].append(env_id)
# 'NovelGridworld-v2_8beams0filled40range3items_in_360degrees'] agents = [ 'NovelGridworld-Bow-v0_8beams0filled11hypotenuserange3items_in_360degrees' ] # log_dir = r"C:\Users\GyanT\Documents\GitHub\Reinforcement-Learning\5_DQN\experiments\\" # agents = ['NovelGridworld-v0_1_DQN', 'NovelGridworld-v0_2_Dueling_DQN', 'NovelGridworld-v0_3_Dueling_Double_DQN', # 'NovelGridworld-v0_4_Double_PER_DQN', 'NovelGridworld-v0_5_Dueling_Double_PER_DQN'] # agents = ['NovelGridworld-v0_3.1_Double_PER_DQN'] plot_after_steps = 1 # 1 for all points for agent in agents: print("agent: ", agent) x, y = ts2xy(load_results(log_dir + os.sep + agent), 'timesteps') print("# of Episodes: ", len(y)) # plt.plot(x, y, label=agent) plt.plot(x[0::plot_after_steps], y[0::plot_after_steps], label=agent + ' (' + str(len(y)) + ' eps)') plt.title('Learning Curve') plt.ylabel("Episodes Rewards") plt.xlabel("Timesteps") plt.legend() plt.savefig(log_dir + os.sep + "learning_curve.png", bbox_inches='tight', dpi=100)
model = A2C("MlpPolicy", env, verbose=1).learn(int(timesteps)) #plot results results_plotter.plot_results([log_dir], timesteps, results_plotter.X_TIMESTEPS, "My title") plt.show() results_plotter.plot_results([log_dir], timesteps, results_plotter.X_EPISODES, "My title") plt.show() results_plotter.plot_results([log_dir], timesteps, results_plotter.X_WALLTIME, "My title") plt.show() print(load_results(log_dir)) #### hand-made plotting #### def moving_average(values, window): """ Smooth values by doing a moving average :param values: (numpy array) :param window: (int) :return: (numpy array) """ weights = np.repeat(1.0, window) / window return np.convolve(values, weights, 'valid')
# plot_data(pd_data, X_EPISODES, 'block_pos_dist', window=500, max_idx=20000, label="PD") # plot_data(id_data, X_EPISODES, 'block_pos_dist', window=500, max_idx=20000 ,label="ID") # plot_data(impedance_data, X_EPISODES, 'block_pos_dist', window=500, max_idx=20000, label="impedance") # # plt.legend(fontsize=24) # plt.xlabel('Episodes', fontsize=24) # plt.ylabel('Goal Distance (m)', fontsize=24) # plt.title('Pushing (PPO)', fontsize=30) # plt.tight_layout() # Load the SAC data. # Torque directory = os.path.join(os.environ['HOME'], 'Google Drive', 'DriveSync', 'DirectTorqueController_PushingEnv_SAC_0') directory = os.path.abspath(directory) experiment_dir = get_latest_experiment_dir(directory) torque_data = load_results(experiment_dir) print(experiment_dir) # PD directory = os.path.join(os.environ['HOME'], 'Google Drive', 'DriveSync', 'RelativePDController_PushingEnv_SAC_0') directory = os.path.abspath(directory) experiment_dir = get_latest_experiment_dir(directory) print(experiment_dir) pd_data = load_results(experiment_dir) # ID directory = os.path.join(os.environ['HOME'], 'Google Drive', 'DriveSync', 'RelativeInverseDynamicsController_PushingEnv_SAC_0') directory = os.path.abspath(directory) experiment_dir = get_latest_experiment_dir(directory)
def callback(_locals, _globals, data_dir, freq=None, low_level_data_dir=None, checkpoint_freq=None): """ Callback called at each step (for DQN an others) or after n steps (see ACER or PPO2) :param _locals: (dict) :param _globals: (dict) """ if not freq: freq = 100000 global n_steps, best_mean_reward # Print stats every freq calls if (n_steps + 1) % freq == 0: if low_level_data_dir: x, y = ts2xy(load_results(data_dir), 'timesteps') if len(x) > 0: mean_reward = np.mean(y[-200:]) print(x[-1], 'timesteps') print("Best 200 mean reward: {:.2f} - Last 2000 mean reward per episode: {:.2f}".format(best_mean_reward, mean_reward)) # New best model, you could save the agent here if mean_reward > best_mean_reward: best_mean_reward = mean_reward # Example for saving best model print("Saving new best model.") _locals['self'].save(low_level_data_dir + '/best_model', data_dir + '/best_model') else: params = ModelParams.load(data_dir) env = get_env(params) ep_rewards = list() for _ in range(4): rewards = list() obs = env.reset() while True: ac = _locals['self'].predict(obs) obs, reward, done, _ = env.step(ac[0]) rewards.append(reward) if done: break ep_rewards.append(sum(rewards)) mean_reward = sum(ep_rewards) / 100.0 print("Best 100 mean reward: {:.2f} - Last mean 100 Ep reward: {:.2f}".format(best_mean_reward, mean_reward)) if mean_reward > best_mean_reward: best_mean_reward = mean_reward # Example for saving best model print("Saving new best model.") _locals['self'].save(data_dir + '/best_model') del env ''' # Evaluate policy training performance x, y = ts2xy(load_results(data_dir), 'timesteps') if len(x) > 0: mean_reward = np.mean(y[-200:]) print(x[-1], 'timesteps') print("Best 200 mean reward: {:.2f} - Last 2000 mean reward per episode: {:.2f}".format(best_mean_reward, mean_reward)) # New best model, you could save the agent here if mean_reward > best_mean_reward: best_mean_reward = mean_reward # Example for saving best model print("Saving new best model") if low_level_data_dir: _locals['self'].save(low_level_data_dir + '/best_model', data_dir + '/best_model') else: _locals['self'].save(data_dir + '/best_model') ''' if not checkpoint_freq is None and (n_steps + 1) % checkpoint_freq == 0: print("Saving Model Checkpoint") name = "/checkpoint_" + str(n_steps + 1) if low_level_data_dir: _locals['self'].save(low_level_data_dir + name, data_dir + name) else: _locals['self'].save(data_dir + name) n_steps += 1 return True
def eval_save_plot(self): eval_start_time = time.time() returns, lengths, scores, idle_steps, ghosts, levels, n_wins, eval_eps = evaluate_policy(self.model, self.eval_env, 10, # not used self.deterministic) eval_elapsed_time = get_formated_time(time.time() - eval_start_time) returns_columns = get_results_columns(returns) lengths_columns = get_results_columns(lengths) scores_columns = get_results_columns(scores) self.evals_idle_mean.append(np.mean(idle_steps)) self.evals_idle_eps.append(np.sum(np.asarray(idle_steps) > 0)) self.evals_ghosts_mean.append(np.mean(ghosts)) self.evals_levels_mean.append(np.mean(levels)) self.evals_n_wins.append(n_wins) self.evals_eps.append(eval_eps) mean_score = scores_columns[0] if mean_score > self.best_mean_score: self.model.save(os.path.join(self.log_dir, 'best_model')) self.best_mean_score = mean_score self.best_train_step = self.num_timesteps self.train_steps.append(self.num_timesteps) self.returns_columns.append(returns_columns) self.lengths_columns.append(lengths_columns) self.scores_columns.append(scores_columns) self.evals_elapsed_time.append(eval_elapsed_time) mean_returns, std_returns, max_returns, min_returns = map(list, zip(*self.returns_columns)) mean_lengths, std_lengths, max_lengths, min_lengths = map(list, zip(*self.lengths_columns)) mean_scores, std_scores, max_scores, min_scores = map(list, zip(*self.scores_columns)) train_results = load_results(self.log_dir) train_ghosts = train_results['ghosts'].tolist() self.train_ghosts.append(np.mean(train_ghosts[self.last_n_episodes:])) train_levels = train_results['level'].tolist() self.train_levels.append(np.mean(train_levels[self.last_n_episodes:])) self.last_n_episodes = len(train_ghosts) rows = zip(self.train_steps, mean_scores, std_scores, max_scores, min_scores, mean_returns, std_returns, max_returns, min_returns, mean_lengths, std_lengths, max_lengths, min_lengths, self.evals_n_wins, self.evals_eps, self.evals_idle_mean, self.evals_idle_eps, self.evals_ghosts_mean, self.evals_levels_mean, self.evals_elapsed_time, self.train_ghosts, self.train_levels) write_rows(os.path.join(self.log_dir, 'evaluations.csv'), rows, EVAL_HEADER) plot_error_bar(self.train_steps, mean_scores, std_scores, max_scores, min_scores, 'Evaluations Mean Score on {} Episodes | Best: {:.1f}'.format( eval_eps, self.best_mean_score), 'Training Step', 'Evaluation Mean Score', os.path.join(self.log_dir, 'eval_scores.png')) plot_error_bar(self.train_steps, mean_returns, std_returns, max_returns, min_returns, 'Evaluations Mean Return on {} Episodes'.format(eval_eps), 'Training Step', 'Evaluation Mean Return', os.path.join(self.log_dir, 'eval_returns.png')) plot_error_bar(self.train_steps, mean_lengths, std_lengths, max_lengths, min_lengths, 'Evaluations Mean Length on {} Episodes'.format(eval_eps), 'Training Step', 'Evaluation Mean Length', os.path.join(self.log_dir, 'eval_lengths.png')) # Train returns x, train_returns = ts2xy(train_results, 'timesteps') plot_line(x, train_returns, 'Training Episodes Return | Total Episodes: {}'.format(len(train_returns)), 'Training Step', 'Episode Return', os.path.join(self.log_dir, 'train_returns.png')) moving_n = 100 if len(train_returns) < moving_n * 2: moving_n = 10 moving_returns = moving_average(train_returns, n=moving_n) plot_line(x[moving_n-1:], moving_returns, 'Training Episodes Return Moving Mean | Total Episodes: {}'.format(len(train_returns)), 'Training Step', '{} Last Episodes Mean Return'.format(moving_n), os.path.join(self.log_dir, 'train_returns_MM.png')) # Train scores train_scores = train_results['score'].tolist() plot_line(x, train_scores, 'Training Episodes Score | Total Episodes: {}'.format(len(train_scores)), 'Training Step', 'Episode Score', os.path.join(self.log_dir, 'train_scores.png')) moving_scores = moving_average(train_scores, n=moving_n) plot_line(x[moving_n-1:], moving_scores, 'Training Episodes Score Moving Mean | Total Episodes: {}'.format(len(train_scores)), 'Training Step', '{} Last Episodes Mean Score'.format(moving_n), os.path.join(self.log_dir, 'train_scores_MM.png')) # Train ghosts plot_line(x, train_ghosts, 'Training Episodes N Ghosts | Total Episodes: {}'.format(len(train_ghosts)), 'Training Step', 'Episode N Ghosts', os.path.join(self.log_dir, 'train_ghosts.png')) moving_ghosts = moving_average(train_ghosts, n=moving_n) plot_line(x[moving_n-1:], moving_ghosts, 'Training Episodes Ghosts Moving Mean | Total Episodes: {}'.format(len(train_ghosts)), 'Training Step', '{} Last Episodes Mean Ghosts'.format(moving_n), os.path.join(self.log_dir, 'train_ghosts_MM.png')) # Train difficulty train_diff = train_results['d'].tolist() plot_line(x, train_diff, 'Training Episodes Difficulty | Total Episodes: {}'.format(len(train_diff)), 'Training Step', 'Episode Difficulty', os.path.join(self.log_dir, 'train_diff.png'))
for path in Path(log_dir).rglob(env_id + '_*'): res_file_list.append(path) res_file_list = sorted(res_file_list) # print(res_file_list) df_list = [] col_list = [] count = 1 for filename in res_file_list: # print(filename) filename = str(filename) # convert from Posixpath to string W = load_results(filename) print(W['r']) df_list.append(W['r']) col_list.append("seed " + str(count)) count += 1 # plot_results(filename, 'timesteps', "seed nb "+str(count)) # # plot_results(filename, 'episodes') # # plot_results(filename, 'walltime_hrs') all_rewards = pd.concat(df_list, axis=1) all_rewards.columns = col_list all_rewards_copy = all_rewards.copy() all_rewards["mean_reward"] = all_rewards_copy.mean(axis=1)
#model_created = False counter = 0 all_x = [] all_y = [] vert_x = [] counter += 1 env = gym.make(env_name) env = Monitor(env, log_dir, allow_early_resets=True) model = DDPG(MlpPolicy, env, verbose=1) start = time.time() model.learn(total_timesteps=step_total) model_loc = os.path.join(models_dir, 'hand') x, y = ts2xy(load_results(log_dir), 'timesteps') y = moving_average(y, window=50) x = x[len(x) - len(y):] for i in x: all_x.append(i) appended_val = x[-1] vert_x.append(appended_val) for i in y: all_y.append(i) #os.remove(os.path.join(log_dir, "monitor.csv")) model.save(model_loc) env.close() model_created = True