def plot(cfg): log.info("============= Configuration =============") log.info(f"Config:\n{cfg.pretty()}") log.info("=========================================") hv_characterization() quit() # Yaw control yaw_dir = "/Users/nato/Documents/Berkeley/Research/Codebases/dynamics-learn/sweeps/2020-05-13/08-01-09/metric.name=Yaw,robot=iono_sim/" ex = "0/trial_33.dat" yaw_ex = yaw_dir+ex # plot_sweep_1(yaw_dir) plot_rollout_dat(yaw_ex) quit() # dir=2020-02-10/15-39-36 files = glob.glob(hydra.utils.get_original_cwd() + '/outputs/' + cfg.dir + '/*/**.dat') ms = [] cl = [] for g in files: mse, clust = torch.load(g) if clust < 500: continue ms.append(mse) cl.append(clust) # ms = np.array(ms) # cl = np.array(cl) # Non clustered data full_size = [4000] base = [0.6844194601919266, 0.6426670856359498, 0.6760970001662061, 0.7867345088097977, 0.6402819700817463, 0.6432612884414582, 0.614643476721318, 0.673518857099874, 0.5565854257191823, 0.9437187183401807] # # for b in base: # ms.append(b) # cl.append(full_size[0]) cl, ms = zip(*sorted(zip(cl, ms))) ids = np.unique(cl) cl_arr = np.stack(cl).reshape((len(ids), -1)) ms_arr = np.stack(ms).reshape((len(ids), -1)) import matplotlib.pyplot as plt import plotly.graph_objects as go colors = plt.get_cmap('tab10').colors traces = [] i = 1 cs_str = 'rgb' + str(colors[i]) err_traces, xs, ys = generate_errorbar_traces(ms_arr.T, xs=cl_arr.T.tolist(), color=cs_str, name=f"Clustered Training") for t in err_traces: traces.append(t) layout = dict( #title=f"Test Set Prediction Error", # (Env: {env_name})", xaxis={'title': 'Cluster Size (Log Scale)', 'autorange': 'reversed', 'range':[3.7, 2.6] }, yaxis={'title': 'Prediction Mean Squared Error', 'range':[.3,1]}, font=dict(family='Times New Roman', size=33, color='#7f7f7f'), xaxis_type="log", # yaxis_type="log", height=600, width=1300, margin=dict(l=0, r=0, b=0, t=0), plot_bgcolor='white', legend={'x': .6, 'y': .05, 'bgcolor': 'rgba(50, 50, 50, .03)'}) fig = go.Figure( data=traces, layout=layout, ) fig.add_trace( # Line Horizontal go.Scatter( mode="lines", x=[max(ids), min(ids)], y=[np.mean(base), np.mean(base)], line=dict( color="gray", width=4, dash="dashdot", ), name='Default Training (4000 Datapoints)' )) import plotly.io as pio pio.show(fig) fig.write_image('clustering_thin.pdf') quit() hv_characterization() ###################################################################### logs = defaultdict(list) configs = defaultdict(list) logs_dirs = ['/Users/nol/Documents/code-bases/dynamicslearn/multirun/2019-12-16/20-01-04/', ] def load_log(directory, trial_file=None): if '.hydra' in os.listdir(directory): full_conf = OmegaConf.load(f"{directory}/.hydra/config.yaml") else: full_conf = OmegaConf.load(f"{directory}/config.yaml") trial_files = glob.glob(f"{directory}/trial_*.dat") if len(trial_files) > 1: if trial_file is not None: last_trial_log = f"{directory}/{trial_file}" else: last_trial_log = max(trial_files, key=os.path.getctime) vis_log = torch.load(last_trial_log) logs[log_dir].append(vis_log) configs[log_dir].append(full_conf) for log_dir in logs_dirs: if os.path.exists(os.path.join(log_dir, 'config.yaml')): log.info(f"Loading latest trial from {log_dir}") d = os.path.join(log_dir) load_log(d) else: # Assuming directory with multiple identical experiments (dir/0, dir/1 ..) latest = defaultdict(list) for ld in os.listdir(log_dir): directory = os.path.join(log_dir, ld) if os.path.isdir(directory): trial_files = glob.glob(f"{directory}/trial_*.dat") if len(trial_files) == 0: continue last_trial_log = max(trial_files, key=os.path.getctime) last_trial_log = last_trial_log[len(directory) + 1:] latest[log_dir].append(last_trial_log) for ld in os.listdir(log_dir): if ld == '.slurm': continue log_subdir = os.path.join(log_dir, ld) if os.path.isdir(log_subdir): # Load data for the smallest trial number from all sub directories if len(latest[log_dir]) == 0: log.warn(f"No trial files found under {log_dir}") break trial_file = natsorted(latest[log_dir])[0] load_log(log_subdir, trial_file) # To display the figure defined by this dict, use the low-level plotly.io.show function plot_rewards_over_trials(logs)
def mpc(cfg): log.info("============= Configuration =============") log.info(f"Config:\n{cfg.pretty()}") log.info("=========================================") env_name = cfg.env.params.name env = gym.make(env_name) env.reset() full_rewards = [] if cfg.metric.name == 'Living': metric = living_reward elif cfg.metric.name == 'Rotation': metric = rotation_mat elif cfg.metric.name == 'Square': metric = squ_cost elif cfg.metric.name == 'Yaw': metric = yaw_r else: raise ValueError("Improper metric name passed") for s in range(cfg.experiment.seeds): log.info(f"Random Seed: {s}") total_costs = [] data_rand = [] total_steps = [] r = 0 while r < cfg.experiment.random: data_r = rollout(env, RandomController(env, cfg), cfg.experiment, metric=metric) plot_rollout(data_r[0], data_r[1], pry=cfg.pid.params.pry, save=cfg.save, loc=f"/R_{r}") rews = data_r[-2] sim_error = data_r[-1] if sim_error: print("Repeating strange simulation") continue # rand_costs.append(np.sum(rews) / len(rews)) # for minimization total_costs.append(np.sum(rews)) # for minimization # log.info(f" - Cost {np.sum(rews) / cfg.experiment.r_len}") r += 1 # data_sample = subsample(data_r, cfg.policy.params.period) data_rand.append(data_r) total_steps.append(0) X, dX, U = to_XUdX(data_r) X, dX, U = combine_data(data_rand[:-1], (X, dX, U)) msg = "Random Rollouts completed of " msg += f"Mean Cumulative reward {np.mean(total_costs)}, " msg += f"Mean Flight length {cfg.policy.params.period * np.mean([np.shape(d[0])[0] for d in data_rand])}" log.info(msg) trial_log = dict( env_name=cfg.env.params.name, model=None, seed=cfg.random_seed, raw_data=data_rand, trial_num=-1, rewards=total_costs, steps=total_steps, nll=None, ) save_log(cfg, -1, trial_log) model, train_log = train_model(X, U, dX, cfg.model) for i in range(cfg.experiment.num_roll-cfg.experiment.random): controller = MPController(env, model, cfg) r = 0 cum_costs = [] data_rs = [] while r < cfg.experiment.repeat: data_r = rollout(env, controller, cfg.experiment, metric=metric) plot_rollout(data_r[0], data_r[1], pry=cfg.pid.params.pry, save=cfg.save, loc=f"/{str(i)}_{r}") rews = data_r[-2] sim_error = data_r[-1] if sim_error: print("Repeating strange simulation") continue # cum_costs.append(np.sum(rews) / len(rews)) # for minimization total_costs.append(np.sum(rews)) # for minimization # log.info(f" - Cost {np.sum(rews) / cfg.experiment.r_len}") r += 1 # data_sample = subsample(data_r, cfg.policy.params.period) data_rs.append(data_r) total_steps.append(np.shape(X)[0]) X, dX, U = combine_data(data_rs, (X, dX, U)) msg = "Rollouts completed of " msg += f"Mean Cumulative reward {np.mean(total_costs)}, " #/ cfg.experiment.r_len msg += f"Mean Flight length {cfg.policy.params.period * np.mean([np.shape(d[0])[0] for d in data_rs])}" log.info(msg) trial_log = dict( env_name=cfg.env.params.name, model=model, seed=cfg.random_seed, raw_data=data_rs, trial_num=i, rewards=total_costs, steps=total_steps, nll=train_log, ) save_log(cfg, i, trial_log) model, train_log = train_model(X, U, dX, cfg.model) fig = plot_rewards_over_trials(np.transpose(np.stack([total_costs])), env_name, save=True) fig.write_image(os.getcwd() + "/learning-curve.pdf")
def sac_experiment(cfg): log.info("============= Configuration =============") log.info(f"Config:\n{cfg.pretty()}") log.info("=========================================") real_env = gym.make(cfg.env.params.name) set_seed_everywhere(cfg.random_seed) obs_dim = cfg.model.params.dx action_dim = cfg.model.params.du target_entropy_coef = 1 batch_size = cfg.alg.params.batch_size # 512 discount = cfg.alg.trainer.discount # .99 tau = cfg.alg.trainer.tau # .005 policy_freq = cfg.alg.trainer.target_update_period # 2 replay_buffer_size = int(cfg.alg.replay_buffer_size) # 1000000 start_steps = cfg.alg.params.start_steps # 10000 eval_freq = cfg.alg.params.eval_freq # 10000 max_steps = int(cfg.alg.params.max_steps) # 2E6 num_eval_episodes = cfg.alg.params.num_eval_episodes # 5 num_eval_timesteps = cfg.alg.params.num_eval_timesteps # 1000 num_rl_updates = 1 model_dir = None replay_buffer = ReplayBuffer(obs_dim, action_dim, cfg.device, replay_buffer_size) policy = SAC( cfg.device, obs_dim, action_dim, hidden_dim=cfg.alg.layer_size, hidden_depth=cfg.alg.num_layers, initial_temperature=cfg.alg.trainer.initial_temp, actor_lr=cfg.alg.trainer.actor_lr, # 1E-3, critic_lr=cfg.alg.trainer.critic_lr, # 1E-3, actor_beta=cfg.alg.trainer.actor_beta, # 0.9, critic_beta=cfg.alg.trainer.critic_beta, # 0.9, log_std_min=cfg.alg.trainer.log_std_min, # -10, log_std_max=cfg.alg.trainer.log_std_max, period=cfg.policy.params.period) # 2) step = 0 steps_since_eval = 0 episode_num = 0 episode_reward = 0 episode_success = 0 episode_step = 0 saved_idx = 0 done = True returns = None target_entropy = -action_dim * target_entropy_coef if cfg.metric.name == 'Living': metric = living_reward elif cfg.metric.name == 'Rotation': metric = rotation_mat elif cfg.metric.name == 'Square': metric = squ_cost elif cfg.metric.name == 'Yaw': metric = yaw_r else: raise ValueError("Improper metric name passed") to_plot_rewards = [] total_steps = [] rewards = evaluate_policy(real_env, policy, step, log, num_eval_episodes, num_eval_timesteps, None, metric=metric) to_plot_rewards.append(rewards) total_steps.append(0) env = gym.make(cfg.env.params.name) # from gym import spaces # env.action_space = spaces.Box(low=np.array([0, 0, 0, 0]), # high=np.array([65535, 65535, 65535, 65535]), # dtype=np.int32) layout = dict( title= f"Learning Curve Reward vs Number of Steps Trials (Env: {cfg.env.params.name}, Alg: {cfg.policy.mode})", xaxis={'title': f"Steps*{eval_freq}"}, yaxis={'title': f"Avg Reward Num:{num_eval_episodes}"}, font=dict(family='Times New Roman', size=18, color='#7f7f7f'), legend={ 'x': .83, 'y': .05, 'bgcolor': 'rgba(50, 50, 50, .03)' }) while step < max_steps: # log.info(f"===================================") if step % 1000 == 0: log.info(f"Step {step}") if done: # Evaluate episode if steps_since_eval >= eval_freq: steps_since_eval %= eval_freq log.info(f"eval/episode: {episode_num}") returns = evaluate_policy(env, policy, step, log, num_eval_episodes, num_eval_timesteps, None, metric=metric) to_plot_rewards.append(returns) total_steps.append(step) if model_dir is not None: policy.save(model_dir, step) # log.info(f"train/episode_reward', episode_reward, step) obs = env.reset() done = False episode_reward = 0 episode_success = 0 episode_step = 0 episode_num += 1 # log.info(f"train/episode', episode_num, step) # Select action randomly or according to policy if step < start_steps: action = env.action_space.sample() action_scale = action else: with torch.no_grad(): with eval_mode(policy): action = policy.sample_action(obs) action_scale = env.action_space.high * (action + 1) / 2 if step >= start_steps: num_updates = start_steps if step == start_steps else num_rl_updates for _ in range(num_updates): policy.update(replay_buffer, step, log, batch_size, discount, tau, policy_freq, target_entropy=target_entropy) next_obs, reward, done, _ = env.step(action_scale) # print(next_obs[:3]) # done_bool = 0 if episode_step + 1 == env._max_episode_steps else float(done) done = 1 if episode_step + 1 == num_eval_timesteps else float(done) reward = metric(next_obs, action) episode_reward += reward replay_buffer.add(obs, action_scale, reward, next_obs, done) obs = next_obs episode_step += 1 step += 1 steps_since_eval += 1 if (step % eval_freq) == 0: trial_log = dict( env_name=cfg.env.params.name, trial_num=saved_idx, replay_buffer=replay_buffer if cfg.save_replay else [], steps=total_steps, policy=policy, rewards=to_plot_rewards, ) save_log(cfg, step, trial_log) saved_idx += 1 plot_rewards_over_trials(to_plot_rewards, cfg.env.params.name, save=True)
def mpc(cfg): log.info("============= Configuration =============") log.info(f"Config:\n{cfg.pretty()}") log.info("=========================================") # plot_results_yaw(pts=cfg.data) # quit() env_name = cfg.env.params.name env = gym.make(env_name) env.reset() env.seed(cfg.random_seed, inertial=cfg.experiment.inertial) if cfg.experiment.inertial: log.info( f"Running experiment with interial prop x:{env.Ixx}, y:{env.Iyy}") # full_rewards = [] # temp = hydra.utils.get_original_cwd() + '/outputs/2020-07-11/17-17-05/trial_3.dat' # dat = torch.load(temp) # actions = dat['raw_data'][0][1] # l = [] # # yaw_actions = np.array([ # [1500, 1500, 1500, 1500], # [2000, 1000, 1000, 2000], # [1000, 2000, 2000, 1000], # [2000, 2000, 1000, 1000], # [1000, 1000, 2000, 2000], # ]) # # def find_ind(arr): # if np.all(np.equal(arr, [1500, 1500, 1500, 1500])): # return 0 # elif np.all(np.equal(arr, [2000, 1000, 1000, 2000])): # return 1 # elif np.all(np.equal(arr, [1000, 2000, 2000, 1000])): # return 3 # elif np.all(np.equal(arr, [2000, 2000, 1000, 1000])): # return 2 # else: # [1000, 1000, 2000, 2000] # return 4 # # for act in actions: # act = act.numpy() # id = find_ind(act) # l.append(id) # # initial = l[:24] # states = dat['raw_data'][0][0][:25] # yaw_value = np.rad2deg(states[-1][0])-np.rad2deg(states[0][0]) # print(f"Yaw after 25 steps{yaw_value}") # plot_lie(initial) # # plot_rollout(np.stack(dat['raw_data'][0][0])[:500,:3], dat['raw_data'][0][1], loc="/yaw_plt", save=True, only_x=True, legend=False) # quit() if cfg.metric.name == 'Living': metric = living_reward log.info(f"Using metric living reward") elif cfg.metric.name == 'Rotation': metric = rotation_mat log.info(f"Using metric rotation matrix") elif cfg.metric.name == 'Square': metric = squ_cost log.info(f"Using metric square cost") elif cfg.metric.name == 'Yaw': metric = yaw_r log.info(f"Using metric yaw sliding mode") elif cfg.metric.name == 'Yaw2': metric = yaw_r2 log.info(f"Using metric yaw base") elif cfg.metric.name == 'Yaw3': metric = yaw_r3 log.info(f"Using metric yaw rate") else: raise ValueError("Improper metric name passed") for s in range(cfg.experiment.seeds): log.info(f"Random Seed: {s}") total_costs = [] data_rand = [] total_steps = [] r = 0 while r < cfg.experiment.random: data_r = rollout(env, RandomController(env, cfg), cfg.experiment, metric=metric) if env_name != 'CartPoleContEnv-v0': plot_rollout(data_r[0], data_r[1], pry=cfg.pid.params.pry, save=cfg.save, loc=f"/R_{r}") rews = data_r[-2] sim_error = data_r[-1] if sim_error: print("Repeating strange simulation") continue # rand_costs.append(np.sum(rews) / len(rews)) # for minimization total_costs.append(np.sum(rews)) # for minimization # log.info(f" - Cost {np.sum(rews) / cfg.experiment.r_len}") r += 1 # data_sample = subsample(data_r, cfg.policy.params.period) data_rand.append(data_r) total_steps.append(0) X, dX, U = to_XUdX(data_r) X, dX, U = combine_data(data_rand[:-1], (X, dX, U)) msg = "Random Rollouts completed of " msg += f"Mean Cumulative reward {np.mean(total_costs)}, " msg += f"Mean length {np.mean([len(a[0]) for a in data_rand])}" log.info(msg) last_yaw = np.max(np.abs(np.stack(data_r[0])[:, 2])) #data_r[0][-1][2] trial_log = dict( env_name=cfg.env.params.name, # model=model, seed=cfg.random_seed, raw_data=data_r, # yaw_num=last_yaw, trial_num=-1, rewards=total_costs, steps=total_steps, # nll=train_log, ) save_log(cfg, -1, trial_log) model, train_log = train_model(X.squeeze(), U, dX.squeeze(), cfg.model) for i in range(cfg.experiment.num_roll - cfg.experiment.random): controller = MPController(env, model, cfg) r = 0 # cum_costs = [] data_rs = [] while r < cfg.experiment.repeat: data_r = rollout(env, controller, cfg.experiment, metric=metric) plot_rollout(data_r[0], data_r[1], pry=cfg.pid.params.pry, save=cfg.save, loc=f"/{str(i)}_{r}") rews = data_r[-2] sim_error = data_r[-1] if sim_error: print("Repeating strange simulation") continue # cum_costs.append(np.sum(rews) / len(rews)) # for minimization total_costs.append(np.sum(rews)) # for minimization # log.info(f" - Cost {np.sum(rews) / cfg.experiment.r_len}") r += 1 # data_sample = subsample(data_r, cfg.policy.params.period) data_rs.append(data_r) total_steps.append(np.shape(X)[0]) X, dX, U = combine_data(data_rs, (X, dX, U)) msg = "Rollouts completed of " msg += f"Cumulative reward {total_costs[-1]}, " # / cfg.experiment.r_len msg += f"length {len(data_r[0])}" # log.info(f"Final yaw {180*np.array(data_r[0][-1][2])/np.pi}") log.info(msg) last_yaw = np.max(np.abs(np.stack( data_r[0])[:, 2])) #data_r[0][-1][2] trial_log = dict( env_name=cfg.env.params.name, # model=model, seed=cfg.random_seed, raw_data=data_r, # yaw_num=last_yaw, trial_num=i, rewards=total_costs, steps=total_steps, nll=train_log, ) save_log(cfg, i, trial_log) model, train_log = train_model(X, U, dX, cfg.model) fig = plot_rewards_over_trials(np.transpose(np.stack([total_costs])), env_name, save=True) fig.write_image(os.getcwd() + "/learning-curve.pdf")