def meil(WORKING_DIR, EXPERT_DIR, args): expert_distribution = Gaussian_Density() with tf.Session() as sess: env = gym.make(args.env) expert = load_policy(sess, EXPERT_DIR) expert_distribution.train(env, expert, args.trajects, args.distr_gamma, args.iter_length) env.close() expert_density = expert_distribution.density() env = gym.make(args.env) policy_distr = Gaussian_Density() policy = lambda s: np.random.uniform(-2.0, 2.0, size=env.action_space.shape) # random policy policy_distr.train(env, policy, args.trajects, args.distr_gamma, args.iter_length) density = policy_distr.density() for i in range(args.rounds): reward = lambda s: expert_density(s) / (density(s) + args.eps) logger_kwargs = setup_logger_kwargs(str(i), data_dir=WORKING_DIR) message = "\nRound {} out of {}\n".format(i + 1, args.rounds) ppo(message, lambda : gym.make(args.env), custom_reward=reward, actor_critic=core.mlp_actor_critic, ac_kwargs=dict(hidden_sizes=[args.hid]*args.l), gamma=args.gamma, steps_per_epoch=args.steps, epochs=args.epochs, logger_kwargs=logger_kwargs) with tf.Session() as sess: policy = load_policy(sess, os.path.join(WORKING_DIR, str(i))) policy_distr.train(env, policy, args.trajects, args.distr_gamma, args.iter_length) density = policy_distr.density() env.close() opt_dir = reward_validation(WORKING_DIR, args) return opt_dir
def main(): # Environment Fcn env_fn = lambda: \ NormalizedBoxEnv( CentauroTrayEnv(**env_params), # normalize_obs=True, normalize_obs=False, online_normalization=False, obs_mean=None, obs_var=None, obs_alpha=0.001, ) # Logger kwargs logger_kwargs = setup_logger_kwargs(EXP_NAME, SEED) with tf.Graph().as_default(): sac( env_fn, actor_critic=mlp_actor_critic, ac_kwargs=dict(hidden_sizes=(128, 128, 128)), seed=SEED, steps_per_epoch=PATHS_PER_EPOCH * PATH_LENGTH, epochs=EPOCHS, replay_size=int(1e6), gamma=0.99, polyak=0.995, # Polyak avg target pol (0-1) lr=1e-3, alpha=0.2, # entropy regularization coefficient (inv rew scale) batch_size=BATCH_SIZE, start_steps=10000, max_ep_len=PATH_LENGTH, # Max length for trajectory logger_kwargs=logger_kwargs, save_freq=1)
def main(): # Environment Fcn env_fn = lambda: \ Reacher2D3DofGoalCompoEnv(**env_params) # Logger kwargs logger_kwargs = setup_logger_kwargs(EXP_NAME, SEED) with tf.Graph().as_default(): ppo( env_fn, actor_critic=mlp_actor_critic, ac_kwargs=dict(hidden_sizes=(128, 128, 128)), seed=SEED, steps_per_epoch=PATHS_PER_EPOCH * PATH_LENGTH, epochs=10000, gamma=0.99, # Discount factor (0-1) clip_ratio=0.2, # clip pol objective (0.1-0.3) pi_lr=3e-4, vf_lr=1e-3, train_pi_iters=80, # Max grad steps in pol loss per epoch train_v_iters=80, # Max grad steps in val loss per epoch lam=0.97, # Lambda for GAE-Lambda (0-1) max_ep_len=PATH_LENGTH, # Max length for trajectory target_kl=0.01, # KLdiv between new and old policies logger_kwargs=logger_kwargs, save_freq=10, )
def __init__(self, config): self.log_dir = config["output_dir"] logger_kwargs = setup_logger_kwargs(config["exp_name"], config["seed"]) logger_kwargs["output_dir"] = config["output_dir"] self.csv_logger = EpochLogger(**logger_kwargs) self.csv_logger.save_config(config) self.tf_logger = SummaryWriter(os.path.join(self.log_dir))
def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) logger_kwargs = setup_logger_kwargs('MultiTaskDDPG') self.logger = EpochLogger(**logger_kwargs) self.logger.save_config(globals()) self.start_steps = 10000
def solve(self, env1, env2): msglogger.info("AMC: Using Spinningup") # training_noise_duration = amc_cfg.num_training_episodes * steps_per_episode heatup_duration = env1.amc_cfg.ddpg_cfg.num_heatup_episodes * env1.steps_per_episode exp_name = "Test" seed = 0 # The number and size of the Actor-Critic MLP hidden layers layers, hid = 2, 300 logger_kwargs = setup_logger_kwargs(exp_name) ddpg.ddpg( env=env1, test_env=env2, actor_critic=core.mlp_actor_critic, ac_kwargs=dict(hidden_sizes=[hid] * layers, output_activation=tf.sigmoid), gamma=1, # discount rate #seed=seed, epochs=400, replay_size=env1.amc_cfg.ddpg_cfg.replay_buffer_size, batch_size=64, start_steps=heatup_duration, steps_per_epoch=env1.steps_per_episode, act_noise=env1.amc_cfg.ddpg_cfg.initial_training_noise, pi_lr=env1.amc_cfg.ddpg_cfg.actor_lr, q_lr=env1.amc_cfg.ddpg_cfg.critic_lr, logger_kwargs=logger_kwargs, noise_decay=env1.amc_cfg.ddpg_cfg.training_noise_decay)
def ddpg_spinup(env1, env2): from spinup.utils.run_utils import setup_logger_kwargs exp_name = "Test" seed = 0 # The number and size of the Actor-Critic MLP hidden layers layers, hid = 2, 300 logger_kwargs = setup_logger_kwargs(exp_name) # , seed) ddpg.ddpg( env=env1, test_env=env2, actor_critic=core.mlp_actor_critic, ac_kwargs=dict(hidden_sizes=[hid] * layers, output_activation=tf.sigmoid), gamma=1, # discount rate seed=seed, epochs=400, replay_size=2000, batch_size=64, start_steps=env1.amc_cfg.num_heatup_epochs, steps_per_epoch=800 * env1.num_layers( ), # every 50 episodes perform 10 episodes of testing act_noise=0.5, pi_lr=1e-4, q_lr=1e-3, logger_kwargs=logger_kwargs)
def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) logger_kwargs = setup_logger_kwargs('MultiTaskDDPGAutoQuery') self.logger = EpochLogger(**logger_kwargs) self.logger.save_config(globals()) self.init_query = False self.init_reward = False self.query_reward = 0
def __init__(self, action_space, observation_space, rng, eps=0.9, discount_factor=0.99, alpha=1e-3): self.rng = rng logger_kwargs = setup_logger_kwargs('SingleTaskDDPG', self.rng) self.logger = EpochLogger(**logger_kwargs) self.logger.save_config(locals()) self.actor_critic = MLPActorCritic # ac_kwargs=dict() ****?????***** # seed=0 self.replay_size = int(1e6) self.polyak = 0.995 self.gamma = discount_factor self.pi_lr = alpha self.q_lr = alpha self.batch_size = 100 self.start_steps = 10000 self.update_after = 1000 self.update_every = 50 self.act_noise = 0.1 self.step_count = 0 self.action_space = action_space self.observation_space = observation_space # self.observation_space = spaces.Box(-np.inf, np.inf, shape=(17,), dtype=np.float32) #fix # torch.manual_seed(seed) # np.random.seed(seed) # self.obs_dim = self.observation_space.shape self.act_dim = self.action_space.shape[0] # act_dim = self.action_space.n # Action limit for clamping: critically, assumes all dimensions share the same bound! self.act_limit = self.action_space.high[0] self.net = False
def worker_test(ps, start_time): from spinup.utils.run_utils import setup_logger_kwargs logger_kwargs = setup_logger_kwargs(args.exp_name, args.seed) logger = EpochLogger(**logger_kwargs) config = locals() del config['ps'] logger.save_config(config) agent = Model(args) keys = agent.get_weights()[0] weights = ray.get(ps.pull.remote(keys)) agent.set_weights(keys, weights) test_env = gym.make(args.env) while True: ave_ret = agent.test_agent(test_env, args) # print("test Average Ret:", ave_ret, "time:", time.time()-start_time) logger.log_tabular('AverageTestEpRet', ave_ret) logger.log_tabular('Time', time.time() - start_time) logger.dump_tabular() weights = ray.get(ps.pull.remote(keys)) agent.set_weights(keys, weights)
'use_HER':True, 'use_prev_a':True, 'gamma':0.95, 'polyak':0.995, 'act_lr' :0.0005, 'crit_lr':0.0005, 'alph_lr':0.001, # ==== noise params ==== 'alpha': 'auto', # fixed or auto balance 'target_entropy':-6 # fixed or auto define with act_dim } saved_model_dir = os.path.join('../saved_models/', rl_params['platform'], rl_params['env_type'], rl_params['env_mode']) logger_kwargs = setup_logger_kwargs(exp_name='cont_sac', seed=rl_params['seed'], data_dir=saved_model_dir, datestamp=False) if 'sim' in rl_params['platform']: from braille_rl.envs.sim.cont_sim_braille_env.mockKBGymEnv import mockKBGymEnv env = mockKBGymEnv(mode=rl_params['env_mode'], max_steps=rl_params['max_ep_len']) elif 'robot' in rl_params['platform']: from braille_rl.envs.robot.cont_ur5_braille_env.ur5GymEnv import UR5GymEnv env = UR5GymEnv(mode=rl_params['env_mode'], max_steps=rl_params['max_ep_len']) sac(env, logger_kwargs=logger_kwargs, network_params=network_params, rl_params=rl_params, resume_training=False, resume_params=dict())
# Perform VPG update! update() # Log info about epoch #logger.log_tabular('Epoch', epoch) logger.log_tabular('EpRet', average_only=True) #logger.log_tabular('EpLen', average_only=True) #logger.log_tabular('VVals', with_min_and_max=True) #logger.log_tabular('TotalEnvInteracts', (epoch+1)*steps_per_epoch) #logger.log_tabular('LossPi', average_only=True) #logger.log_tabular('LossV', average_only=True) #logger.log_tabular('DeltaLossPi', average_only=True) #logger.log_tabular('DeltaLossV', average_only=True) #logger.log_tabular('Entropy', average_only=True) #logger.log_tabular('KL', average_only=True) #logger.log_tabular('Time', time.time()-start_time) logger.dump_tabular() from spinup.utils.run_utils import setup_logger_kwargs logger_kwargs = setup_logger_kwargs(exp_name, seed) vpg(lambda: gym.make(env), actor_critic=core.mlp_actor_critic, ac_kwargs=dict(hidden_sizes=[layer_size] * layers), gamma=gamma, seed=seed, steps_per_epoch=steps, epochs=epochs, logger_kwargs=logger_kwargs)
parser.add_argument('--pi_lr', type=float, default=3e-4) parser.add_argument('--clp_ratio', type=float, default=0.2) parser.add_argument('--jahThresh', type=int, default=100000) args = parser.parse_args() mpi_fork(args.cpu) # run parallel code with mpi # from spinup.utils.run_utils import setup_logger_kwargs # logger_kwargs = setup_logger_kwargs(args.exp_name, args.seed) from spinup.utils.run_utils import setup_logger_kwargs import time currTime = round(time.time()) logger_kwargs = setup_logger_kwargs( args.exp_name, str(args.seed) + "_e" + str(args.epochs) + "_" + args.objective + "_" + str(currTime) + "_PPO") ppo(lambda: gym.make(args.env, netname=args.exp_name, objective=args.objective, seed=args.seed, jahThresh=args.jahThresh), actor_critic=core.mlp_actor_critic, ac_kwargs=dict(hidden_sizes=[args.hid] * args.l), gamma=args.gamma, seed=args.seed, steps_per_epoch=args.steps, epochs=args.epochs, logger_kwargs=logger_kwargs, clip_ratio=args.clp_ratio,
seed = 125259 num_epochs = 100 ep_per_epoch = 5000 random_steps = 10000 max_ep_len = 1000 hidden_sizes = [400, 300] buffer_size = int(1e6) batch_size = 100 gamma = 0.99 lr_a = 1e-3 lr_q = 1e-3 p = 0.995 save_freq = 1 # Set logger logger_kwargs = setup_logger_kwargs('test_own', seed) logger = EpochLogger(**logger_kwargs) logger.save_config(locals()) # Setting seed np.random.seed(seed) tf.set_random_seed(seed) # Create environment env = gym.make('BipedalWalker-v2') test_env = gym.make('BipedalWalker-v2') #env.render() # Get imporant variables action_space = env.action_space obs_space = env.observation_space
pi_lr = params["actor_learning_rate"] vf_lr = params["critic_learning_rate"] minibatch_size = params["batch_size"] discountFactor = params["discount_factor"] explorationRate = params["exploration_rate"] memorySize = params["memory_size"] hidden_sizes = params['hidden_sizes'] act_noise_amount = params["action_noise"] policy_net_training_steps = params["training_steps"] # print("hidden_sizes: ", hidden_sizes) DEFAULT_DATA_DIR = osp.join( osp.abspath(osp.dirname(osp.dirname(__file__))), 'data') # PROJECT_ROOT = os.path.dirname(os.path.realpath(__file__)) logger_kwargs = setup_logger_kwargs(args.exp_name, seed=args.seed, data_dir=DEFAULT_DATA_DIR) # PROJECT_ROOT = logger_kwargs['output_dir'] # saver = tf.train.Saver(max_to_keep=10, keep_checkpoint_every_n_hours=0.5) sess = tf.Session() with tf.variable_scope('main'): policy_net = actor_critic_core.AC_Net( env, EP_MAX=epochs, EP_LEN=episode_steps, GAMMA=discountFactor, AR=pi_lr, CR=vf_lr, BATCH=minibatch_size, UPDATE_STEP=policy_net_training_steps, hidden_sizes=hidden_sizes,
parser.add_argument("--noise1", default=0.1, type=float) # Probability of selecting random action parser.add_argument("--noise2", default=0.1, type=float) # Std of Gaussian exploration noise args = parser.parse_args() file_name = "DDPG_%s_%s" % (args.env_name, str(args.seed)) buffer_name = "ExpertN%sN%s_%s_%s" % (str(args.noise1), str( args.noise2), args.env_name, str(args.seed)) print("---------------------------------------") print("Settings: " + file_name) print("---------------------------------------") from spinup.utils.logx import EpochLogger from spinup.utils.run_utils import setup_logger_kwargs logger_kwargs = setup_logger_kwargs('BufferQuality_' + buffer_name, args.seed) """set up logger""" logger = EpochLogger(**logger_kwargs) logger_save_freq = 10 if not os.path.exists("./buffers"): os.makedirs("./buffers") env = gym.make(args.env_name) env.seed(args.seed) torch.manual_seed(args.seed) np.random.seed(args.seed) state_dim = env.observation_space.shape[0] action_dim = env.action_space.shape[0]
def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) logger_kwargs = setup_logger_kwargs('MultiTaskDDPGAugmentedOracle') self.logger = EpochLogger(**logger_kwargs) self.logger.save_config(globals())
'max_ep_len':18000, 'max_noop':10, 'save_freq':5, 'render':True, # rl params 'gamma':0.99, 'polyak':0.995, 'lr':0.00025, 'grad_clip_val':5.0, # entropy params 'alpha': 'auto', 'target_entropy_start':0.5, # proportion of max_entropy 'target_entropy_stop':0.5, 'target_entropy_steps':1e6, } saved_model_dir = '../../saved_models' logger_kwargs = setup_logger_kwargs(exp_name='sac_discrete_gb_atari_' + rl_params['env_name'], seed=rl_params['seed'], data_dir=saved_model_dir, datestamp=False) env = gym.make(rl_params['env_name']) # avoids crash when later rendering the environment if rl_params['render']: test_env(lambda:env) sac(lambda:env, logger_kwargs=logger_kwargs, network_params=network_params, rl_params=rl_params)
def ppo(BASE_DIR, expert_density, env_fn, actor_critic=core.mlp_actor_critic, ac_kwargs=dict(), steps_per_epoch=1000, epochs=10, gamma=0.99, clip_ratio=0.2, pi_lr=3e-4, vf_lr=1e-3, train_pi_iters=50, train_v_iters=50, lam=0.97, max_ep_len=1000, target_kl=0.01, data_n=10): data = {} # ALL THE DATA logger_kwargs = setup_logger_kwargs(args.dir_name, data_dir=BASE_DIR) logger = EpochLogger(**logger_kwargs) logger.save_config(locals()) env = env_fn() obs_dim = env.observation_space.shape act_dim = env.action_space.shape # Share information about action space with policy architecture ac_kwargs['action_space'] = env.action_space # Inputs to computation graph x_ph, a_ph = core.placeholders_from_spaces(env.observation_space, env.action_space) adv_ph, ret_ph, logp_old_ph = core.placeholders(None, None, None) # Main outputs from computation graph pi, logp, logp_pi, v = actor_critic(x_ph, a_ph, **ac_kwargs) # Need all placeholders in *this* order later (to zip with data from buffer) all_phs = [x_ph, a_ph, adv_ph, ret_ph, logp_old_ph] # Every step, get: action, value, and logprob get_action_ops = [pi, v, logp_pi] # Experience buffer local_steps_per_epoch = int(steps_per_epoch / num_procs()) buf = PPOBuffer(obs_dim, act_dim, local_steps_per_epoch, gamma, lam) # Count variables var_counts = tuple(core.count_vars(scope) for scope in ['pi', 'v']) logger.log('\nNumber of parameters: \t pi: %d, \t v: %d\n' % var_counts) # PPO objectives ratio = tf.exp(logp - logp_old_ph) # pi(a|s) / pi_old(a|s) min_adv = tf.where(adv_ph > 0, (1 + clip_ratio) * adv_ph, (1 - clip_ratio) * adv_ph) pi_loss = -tf.reduce_mean(tf.minimum(ratio * adv_ph, min_adv)) v_loss = tf.reduce_mean((ret_ph - v)**2) # Info (useful to watch during learning) approx_kl = tf.reduce_mean( logp_old_ph - logp) # a sample estimate for KL-divergence, easy to compute approx_ent = tf.reduce_mean( -logp) # a sample estimate for entropy, also easy to compute clipped = tf.logical_or(ratio > (1 + clip_ratio), ratio < (1 - clip_ratio)) clipfrac = tf.reduce_mean(tf.cast(clipped, tf.float32)) # Optimizers train_pi = MpiAdamOptimizer(learning_rate=pi_lr).minimize(pi_loss) train_v = MpiAdamOptimizer(learning_rate=vf_lr).minimize(v_loss) sess = tf.Session() sess.run(tf.global_variables_initializer()) # Sync params across processes sess.run(sync_all_params()) # Setup model saving logger.setup_tf_saver(sess, inputs={'x': x_ph}, outputs={'pi': pi, 'v': v}) # update rule def update(): inputs = {k: v for k, v in zip(all_phs, buf.get())} pi_l_old, v_l_old, ent = sess.run([pi_loss, v_loss, approx_ent], feed_dict=inputs) # Training for i in range(train_pi_iters): _, kl = sess.run([train_pi, approx_kl], feed_dict=inputs) kl = mpi_avg(kl) if kl > 1.5 * target_kl: logger.log( 'Early stopping at step %d due to reaching max kl.' % i) break logger.store(StopIter=i) for _ in range(train_v_iters): sess.run(train_v, feed_dict=inputs) # Log changes from update pi_l_new, v_l_new, kl, cf = sess.run( [pi_loss, v_loss, approx_kl, clipfrac], feed_dict=inputs) logger.store(LossPi=pi_l_old, LossV=v_l_old, KL=kl, Entropy=ent, ClipFrac=cf, DeltaLossPi=(pi_l_new - pi_l_old), DeltaLossV=(v_l_new - v_l_old)) policy_distr = Gaussian_Density() policy = lambda s: np.random.uniform( -2.0, 2.0, size=env.action_space.shape) # random policy policy_distr.train(env, policy, args.trajects, args.distr_gamma, args.iter_length) density = policy_distr.density() data[0] = { 'pol_s': policy_distr.num_samples, 'pol_t': policy_distr.num_trajects } dist_rewards = [] # repeat REIL for given number of rounds for i in range(args.rounds): message = "\nRound {} out of {}\n".format(i + 1, args.rounds) reward = lambda s: expert_density(s) / (density(s) + args.eps) dist_rewards.append(reward) start_time = time.time() o, old_r, d, ep_ret, ep_len = env.reset(), 0, False, 0, 0 r = reward(o) # custom reward # Main loop: collect experience in env and update/log each epoch for epoch in range(epochs): for t in range(local_steps_per_epoch): a, v_t, logp_t = sess.run(get_action_ops, feed_dict={x_ph: o.reshape(1, -1)}) # save and log buf.store(o, a, r, v_t, logp_t) logger.store(VVals=v_t) o, old_r, d, _ = env.step(a[0]) r = reward(o) ep_ret += r ep_len += 1 terminal = d or (ep_len == max_ep_len) if terminal or (t == local_steps_per_epoch - 1): if not (terminal): print( 'Warning: trajectory cut off by epoch at %d steps.' % ep_len) # if trajectory didn't reach terminal state, bootstrap value target last_val = old_r if d else sess.run( v, feed_dict={x_ph: o.reshape(1, -1)}) last_val = reward(o) buf.finish_path(last_val) if terminal: # only save EpRet / EpLen if trajectory finished logger.store(EpRet=ep_ret, EpLen=ep_len) o, old_r, d, ep_ret, ep_len = env.reset(), 0, False, 0, 0 r = reward(o) # store model! if (epoch == epochs - 1): logger.save_state({'env': env}, None) # Perform PPO update! update() # Log info about epoch logger.log_tabular('Epoch', epoch) logger.log_tabular('EpRet', with_min_and_max=True) logger.log_tabular('EpLen', average_only=True) logger.log_tabular('VVals', with_min_and_max=True) logger.log_tabular('TotalEnvInteracts', (epoch + 1) * steps_per_epoch) logger.log_tabular('LossPi', average_only=True) logger.log_tabular('LossV', average_only=True) logger.log_tabular('DeltaLossPi', average_only=True) logger.log_tabular('DeltaLossV', average_only=True) logger.log_tabular('Entropy', average_only=True) logger.log_tabular('KL', average_only=True) logger.log_tabular('ClipFrac', average_only=True) logger.log_tabular('StopIter', average_only=True) logger.log_tabular('Time', time.time() - start_time) logger.dump_tabular() print(message) policy = lambda state: sess.run( get_action_ops, feed_dict={x_ph: state.reshape(1, -1)})[0][0] data[i] = { 'pol_s': policy_distr.num_samples, 'pol_t': policy_distr.num_trajects } data[i]['rewards'] = evaluate_reward(env, policy, data_n) if i != args.rounds - 1: policy_distr.train(env, policy, args.trajects, args.distr_gamma, args.iter_length) density = policy_distr.density() return data, dist_rewards
def trpo(env_fn, actor_critic=core.mlp_actor_critic, ac_kwargs=dict(), seed=0, steps_per_epoch=4000, epochs=50, gamma=0.99, delta=0.01, vf_lr=1e-3, train_v_iters=80, damping_coeff=0.1, cg_iters=10, backtrack_iters=10, backtrack_coeff=0.8, lam=0.97, max_ep_len=1000, logger_kwargs=dict(), save_freq=10, algo='trpo'): """ Args: env_fn : A function which creates a copy of the environment. The environment must satisfy the OpenAI Gym API. actor_critic: A function which takes in placeholder symbols for state, ``x_ph``, and action, ``a_ph``, and returns the main outputs from the agent's Tensorflow computation graph: ============ ================ ======================================== Symbol Shape Description ============ ================ ======================================== ``pi`` (batch, act_dim) | Samples actions from policy given | states. ``logp`` (batch,) | Gives log probability, according to | the policy, of taking actions ``a_ph`` | in states ``x_ph``. ``logp_pi`` (batch,) | Gives log probability, according to | the policy, of the action sampled by | ``pi``. ``info`` N/A | A dict of any intermediate quantities | (from calculating the policy or log | probabilities) which are needed for | analytically computing KL divergence. | (eg sufficient statistics of the | distributions) ``info_phs`` N/A | A dict of placeholders for old values | of the entries in ``info``. ``d_kl`` () | A symbol for computing the mean KL | divergence between the current policy | (``pi``) and the old policy (as | specified by the inputs to | ``info_phs``) over the batch of | states given in ``x_ph``. ``v`` (batch,) | Gives the value estimate for states | in ``x_ph``. (Critical: make sure | to flatten this!) ============ ================ ======================================== ac_kwargs (dict): Any kwargs appropriate for the actor_critic function you provided to TRPO. seed (int): Seed for random number generators. steps_per_epoch (int): Number of steps of interaction (state-action pairs) for the agent and the environment in each epoch. epochs (int): Number of epochs of interaction (equivalent to number of policy updates) to perform. gamma (float): Discount factor. (Always between 0 and 1.) delta (float): KL-divergence limit for TRPO / NPG update. (Should be small for stability. Values like 0.01, 0.05.) vf_lr (float): Learning rate for value function optimizer. train_v_iters (int): Number of gradient descent steps to take on value function per epoch. damping_coeff (float): Artifact for numerical stability, should be smallish. Adjusts Hessian-vector product calculation: .. math:: Hv \\rightarrow (\\alpha I + H)v where :math:`\\alpha` is the damping coefficient. Probably don't play with this hyperparameter. cg_iters (int): Number of iterations of conjugate gradient to perform. Increasing this will lead to a more accurate approximation to :math:`H^{-1} g`, and possibly slightly-improved performance, but at the cost of slowing things down. Also probably don't play with this hyperparameter. backtrack_iters (int): Maximum number of steps allowed in the backtracking line search. Since the line search usually doesn't backtrack, and usually only steps back once when it does, this hyperparameter doesn't often matter. backtrack_coeff (float): How far back to step during backtracking line search. (Always between 0 and 1, usually above 0.5.) lam (float): Lambda for GAE-Lambda. (Always between 0 and 1, close to 1.) max_ep_len (int): Maximum length of trajectory / episode / rollout. logger_kwargs (dict): Keyword args for EpochLogger. save_freq (int): How often (in terms of gap between epochs) to save the current policy and value function. algo: Either 'trpo' or 'npg': this code supports both, since they are almost the same. """ # initialize logger and save it # initialize seed, and set tf and np # get the env function, observation dimensions, and action dimensions # Share information about action space with policy architecture # Inputs to computation graph # Main outputs from computation graph, plus placeholders for old pdist (for KL) # Need all placeholders in *this* order later (to zip with data from buffer) # Every step, get: action, value, logprob, & info for pdist (for computing kl div) # Experience buffer # calculate the number of steps per epoch per process # get the info shapes # initialize the bugger # Count variables # TRPO losses # ratio of pi / pi_old # pi loss # v loss # Optimizer for value function # Symbols needed for CG solver # pi params # gradient # v_ph and hvp # check if the damping coeff is needed # if so, update hvp (damping_coeff * v_ph) # Symbols for getting and setting params # get pi params # set pi params # create a tf session and initialize it's variables # Sync params across processes # Setup model saving def cg(Ax, b): """ Conjugate gradient algorithm (see https://en.wikipedia.org/wiki/Conjugate_gradient_method) """ # initialize x as 0s of shape b # Note: should be 'b - Ax(x)', but for x=0, Ax(x)=0. Change if doing warm start. # make a copy of b and r as r and p # calculate r dot old (r dot r) # for cg_iterations # calc z as Ax(p) # calculate alpha # increment x # decrement r # calculate r dot new (r dot r) # calculate p # update r dot old with r dot new def update(): # Prepare hessian func, gradient eval # get inputs as a dictionary, all phs and buffer # calculate Hx # get g, pi_l_old, v_l_old # get g and pi_l_old averages # Core calculations for TRPO or NPG # get x # get alpha # get old paramers def set_and_eval(step): # set pi params with v_ph # old_params - alpha * x * step # return average of d_kl and pi_loss operation # handle npg # npg has no backtracking or hard kl constraint enforcement # handle trpo # trpo augments npg with backtracking line search, hard kl # for backtrack iterations # Value function updates # for train_v_iterations # update v_l_new with v_loss operation # Log changes from update # Update start time # reset variables # o, r, d, ep_ret, ep_len # Main loop: collect experience in env and update/log each epoch for epoch in range(epochs): for t in range(local_steps_per_epoch): # get agent outputs # decontruct the above to a, v_t, logp_t, info_t # save and log # take an action # update ep rewards and length # check if the episode is done # check if terminal or at max t for local epoch # if trajectory didn't reach terminal state, bootstrap value target # add the finish path to buffer # only save EpRet / EpLen if trajectory finished # reset environment variables # o, r, d, ep_ret, ep_len # Save model # Perform TRPO or NPG update! # Log info about epoch logger.log_tabular('Epoch', epoch) logger.log_tabular('EpRet', with_min_and_max=True) logger.log_tabular('EpLen', average_only=True) logger.log_tabular('VVals', with_min_and_max=True) logger.log_tabular('TotalEnvInteracts', (epoch+1)*steps_per_epoch) logger.log_tabular('LossPi', average_only=True) logger.log_tabular('LossV', average_only=True) logger.log_tabular('DeltaLossPi', average_only=True) logger.log_tabular('DeltaLossV', average_only=True) logger.log_tabular('KL', average_only=True) if algo=='trpo': logger.log_tabular('BacktrackIters', average_only=True) logger.log_tabular('Time', time.time()-start_time) logger.dump_tabular() if __name__ == '__main__': import argparse parser = argparse.ArgumentParser() parser.add_argument('--env', type=str, default='HalfCheetah-v2') parser.add_argument('--hid', type=int, default=64) parser.add_argument('--l', type=int, default=2) parser.add_argument('--gamma', type=float, default=0.99) parser.add_argument('--seed', '-s', type=int, default=0) parser.add_argument('--cpu', type=int, default=4) parser.add_argument('--steps', type=int, default=4000) parser.add_argument('--epochs', type=int, default=50) parser.add_argument('--exp_name', type=str, default='trpo') args = parser.parse_args() mpi_fork(args.cpu) # run parallel code with mpi from spinup.utils.run_utils import setup_logger_kwargs logger_kwargs = setup_logger_kwargs(args.exp_name, args.seed) trpo(lambda : gym.make(args.env), actor_critic=core.mlp_actor_critic, ac_kwargs=dict(hidden_sizes=[args.hid]*args.l), gamma=args.gamma, seed=args.seed, steps_per_epoch=args.steps, epochs=args.epochs, logger_kwargs=logger_kwargs)
parser.add_argument('--backfil', type=int, default=0) # this parameter is deprecated, and this model will have skip function by default. parser.add_argument('--skip', type=int, default=0) parser.add_argument('--score_type', type=int, default=0) parser.add_argument('--batch_job_slice', type=int, default=10000) parser.add_argument('--sched_algo', type=int, default=4) args = parser.parse_args() from spinup.utils.run_utils import setup_logger_kwargs # build absolute path for using in hpc_env. current_dir = os.getcwd() workload_file = os.path.join(current_dir, args.workload) log_data_dir = os.path.join(current_dir, './data/logs/') logger_kwargs = setup_logger_kwargs(args.exp_name, seed=args.seed, data_dir=log_data_dir) if args.pre_trained: model_file = os.path.join(current_dir, args.trained_model) # get_probs, get_value = load_policy(model_file, 'last') ppo(workload_file, args.model, gamma=args.gamma, seed=args.seed, traj_per_epoch=args.trajs, epochs=args.epochs, logger_kwargs=logger_kwargs, pre_trained=1, trained_model=os.path.join(model_file, "simple_save"), attn=args.attn,
logger.log_tabular('LossQ', average_only=True) logger.log_tabular('Time', time.time() - start_time) logger.dump_tabular() if __name__ == '__main__': import argparse parser = argparse.ArgumentParser() parser.add_argument('--env', type=str, default='HalfCheetah-v2') parser.add_argument('--seed', '-s', type=int, default=0) parser.add_argument('--exp_name', type=str, default='ex13-td3') parser.add_argument('--use_soln', action='store_true') args = parser.parse_args() from spinup.utils.run_utils import setup_logger_kwargs logger_kwargs = setup_logger_kwargs(args.exp_name + '-' + args.env.lower(), args.seed) all_kwargs = dict(env_fn=lambda: gym.make(args.env), actor_critic=core.mlp_actor_critic, ac_kwargs=dict(hidden_sizes=[128, 128]), max_ep_len=150, seed=args.seed, logger_kwargs=logger_kwargs, epochs=10) if args.use_soln: true_td3(**all_kwargs) else: td3(**all_kwargs)
args = parser.parse_args() #mpi_fork(args.cpu) # run parallel code with mpi ##make env dir to arrange files to each env dir from spinup.user_config import DEFAULT_DATA_DIR import os datadir = os.path.join(DEFAULT_DATA_DIR, args.env) os.makedirs(datadir, exist_ok=True) ## from spinup.utils.run_utils import setup_logger_kwargs logger_kwargs = setup_logger_kwargs(args.exp_name, args.seed, data_dir=datadir) sigail(lambda: gym.make(args.env), actor_critic=core.mlp_actor_critic_add, ac_kwargs=dict(hidden_sizes=[args.hid] * args.l, policy_logvar=args.logvar), gamma=args.gamma, d_hidden_size=args.d_hid, seed=args.seed, steps_per_epoch=args.steps, epochs=args.epochs, train_pi_iters=args.g_itr, train_v_iters=args.g_itr, beta=args.beta, logger_kwargs=logger_kwargs,
rospy.init_node('pelican_attitude_controller_sac_training', anonymous=True, log_level=rospy.WARN) import argparse parser = argparse.ArgumentParser() parser.add_argument('--env', type=str, default='PelicanAttControllerEnv-v0') parser.add_argument('--hid', type=int, default=64) parser.add_argument('--l', type=int, default=2) parser.add_argument('--gamma', type=float, default=0.99) parser.add_argument('--seed', '-s', type=int, default=0) parser.add_argument('--epochs', type=int, default=10000) parser.add_argument('--exp_name', type=str, default='sac') args = parser.parse_args() from spinup.utils.run_utils import setup_logger_kwargs DEFAULT_DATA_DIR = osp.join( osp.abspath(osp.dirname(osp.dirname(__file__))), 'data') logger_kwargs = setup_logger_kwargs(args.exp_name, args.seed, DEFAULT_DATA_DIR) outdir = '/tmp/openai_ros_experiments/' env = gym.make(args.env) # env = gym.wrappers.Monitor(env, outdir, force=True) sac(lambda: env, actor_critic=core.mlp_actor_critic, ac_kwargs=dict(hidden_sizes=[args.hid] * args.l), gamma=args.gamma, seed=args.seed, epochs=args.epochs, logger_kwargs=logger_kwargs)
'batch_size': 64, 'start_steps': 4000, 'max_ep_len': 1000, 'save_freq': 5, 'render': True, # rl params 'gamma': 0.99, 'polyak': 0.995, 'lr': 0.001, 'grad_clip_val': None, # entropy params 'alpha': 'auto', # fixed or auto balance 'target_entropy': 'auto', # fixed or auto define with act_dim } saved_model_dir = '../../saved_models' logger_kwargs = setup_logger_kwargs(exp_name='sac_cont_image_' + rl_params['env_name'], seed=rl_params['seed'], data_dir=saved_model_dir, datestamp=False) env = gym.make(rl_params['env_name']) sac(lambda: env, logger_kwargs=logger_kwargs, network_params=network_params, rl_params=rl_params)
def run_exp(alg="sac",alpha=None,add_penalty=1,mult_penalty=None,cost_penalty=0,buckets=None, epochs=30,start_steps=10000,split_policy=False,ac_kwargs={"hidden_sizes":(256,256)}, safe_policy=False,entropy_constraint=-1,collector_policy=None,filename="",steps_per_epoch=10001, num_test_episodes=10,act_noise=0.1,data_aug=False,env_name='Safexp-PointGoal1-v0',batch_size=100): # alg determines whether sac, ppo or td3 is used. # alpha is the exploration parameter in sac. Add_parameter is Beta from the proposal. # If mult_penalty is not None, all rewards get multiplied by it once the constraint is violated (1-alpha from the proposal) # cost_penalty is equal to zeta from the proposal. buckets determines how the accumulated cost is discretized for the agent: #if it is None, cost is a continouos variable, else there are buckets indicator variables for a partition of [0,constraint] # (with the last only activating if the constraint is violated). Epochs indicates, how many epochs to train for, start_steps indicates # how many random exploratory actions to perform before using the trained policy. Split_policy changes the network architecture # such that a second network is used for the policy and q-values when the constraint is violated. ac_kwargs is a dict containing # the arguments for the actor-critic class. Hidden sizes is a tuple containing the sizes for all hidden layers. # safe_policy indicates the saving location for a trained safe policy. If provided, the safe policy will take over whenever the constraint # is violated. filename determines, where in the results folder the res29541.pts-3.tensorflow-1-vmults and trained policy get saved to. # Steps_per epoch determines the amount of environment interaction per epoch. Num_test episodes the amount of test_episodes # (only for evaluation) that are performed after each epoch. act_noise controls the exploration noise used in the td3 algorithm. # Entropy_constraint is the entropy to aim for (if sac is used with trainable alpha) # Collector_policy specifies if mult_penalty == -1: mult_penalty = None if buckets == -1: buckets = None if entropy_constraint == 0: entropy_constraint = None if alpha == 0: alpha = None env = gym.make(env_name) # Create an instance of the safety-gym environment. # Create an instance of the constrained environment. env = constraint_wrapper(env,add_penalty=add_penalty,mult_penalty=mult_penalty, cost_penalty=cost_penalty,buckets=buckets,safe_policy=safe_policy) logger_kwargs = setup_logger_kwargs(filename+"policy",data_dir="results/") assert alg == "sac" or alg == "td3" or alg == "ppo" # Select learning method if alg == "sac": import spinup.algos.pytorch.sac.core as core if split_policy: actor_critic = core.MLPActorCriticSplit else: actor_critic = core.MLPActorCritic # Start training with SAC sac_pytorch(lambda: env,epochs=epochs,alpha=alpha,steps_per_epoch=steps_per_epoch,start_steps=start_steps, logger_kwargs=logger_kwargs,num_test_episodes=num_test_episodes,actor_critic=actor_critic,ac_kwargs=ac_kwargs,entropy_constraint=entropy_constraint,collector_policy=collector_policy,data_aug=data_aug,batch_size=batch_size) elif alg == "td3": import spinup.algos.pytorch.td3.core as core if split_policy: actor_critic = core.MLPActorCriticSplit else: actor_critic = core.MLPActorCritic # Start training with TD3 td3_pytorch(lambda: env,epochs=epochs,steps_per_epoch=steps_per_epoch,start_steps=start_steps,logger_kwargs=logger_kwargs, actor_critic=actor_critic,act_noise=act_noise,ac_kwargs=ac_kwargs,collector_policy=collector_policy,data_aug=data_aug,num_test_episodes=num_test_episodes,batch_size=batch_size) elif alg == "ppo": import spinup.algos.pytorch.ppo.core as core assert collector_policy==None if split_policy: actor_critic = core.MLPActorCriticSplit else: actor_critic = core.MLPActorCritic # Start training with PPO ppo_pytorch(lambda: env, epochs=epochs, steps_per_epoch=steps_per_epoch, logger_kwargs=logger_kwargs, actor_critic=actor_critic, ac_kwargs=ac_kwargs) #Ideally, you would separate train and test runs more directly here rather than reylying on the alg to work exactly as described... # Store training results in pickle file with open("results/"+filename+"rews.pkl", 'wb') as f: pickle.dump(env.total_rews, f) with open("results/"+filename+"costs.pkl", 'wb') as f: pickle.dump(env.total_costs, f)
parser.add_argument('--render', action='store_true') parser.add_argument('--vm_checkpoint', type=str) parser.add_argument('--pim_checkpoint', type=str, default=None) args = parser.parse_args() if args.cpu > 1: mpi_tools.mpi_fork(args.cpu, allow_run_as_root=args.allow_run_as_root) # run parallel code with mpi # Setup experiment name env = make_env(args.seed) from spinup.utils.run_utils import setup_logger_kwargs experiment_name = args.exp_name or env.spec.id logger_kwargs = setup_logger_kwargs(experiment_name, args.seed) # Load or create model saved_model_file = None if args.saved_model_file: saved_model_file = pathlib.Path(args.saved_model_file) elif args.continue_training: save_dir = pathlib.Path(logger_kwargs['output_dir'], 'pyt_save') if save_dir.exists(): saved_model_file = get_latest_saved_file(save_dir, prefix='model') if saved_model_file: assert saved_model_file.exists() model = torch.load(saved_model_file, map_location=DEVICE) for p in model.parameters(): p.requires_grad_()
def collect_entropy_policies(env, epochs, T, MODEL_DIR=''): video_dir = 'videos/' + args.exp_name direct = os.getcwd() + '/data/' experiment_directory = direct + args.exp_name print(experiment_directory) print(sys.argv) if not os.path.exists(experiment_directory): os.makedirs(experiment_directory) f = open(experiment_directory + '/args', 'w') f.write(' '.join(sys.argv)) f.flush() indexes = [1, 5, 10, 15] states_visited_indexes = [0, 5, 10, 15] states_visited_cumulative = [] states_visited_cumulative_baseline = [] running_avg_p = np.zeros(shape=(tuple(ant_utils.num_states))) running_avg_p_xy = np.zeros(shape=(tuple(ant_utils.num_states_2d))) running_avg_ent = 0 running_avg_ent_xy = 0 running_avg_p_baseline = np.zeros(shape=(tuple(ant_utils.num_states))) running_avg_p_baseline_xy = np.zeros( shape=(tuple(ant_utils.num_states_2d))) running_avg_ent_baseline = 0 running_avg_ent_baseline_xy = 0 pct_visited = [] pct_visited_baseline = [] pct_visited_xy = [] pct_visited_xy_baseline = [] running_avg_entropies = [] running_avg_entropies_xy = [] running_avg_ps_xy = [] avg_ps_xy = [] running_avg_entropies_baseline = [] running_avg_entropies_baseline_xy = [] running_avg_ps_baseline_xy = [] avg_ps_baseline_xy = [] policies = [] distributions = [] initial_state = init_state(env) prebuf = ExperienceBuffer() env.reset() for t in range(10000): action = env.action_space.sample() obs, reward, done, _ = env.step(action) prebuf.store(get_state(env, obs)) if done: env.reset() done = False prebuf.normalize() normalization_factors = prebuf.normalization_factors utils.log_statement(normalization_factors) prebuf = None if not args.gaussian: normalization_factors = [] reward_fn = np.zeros(shape=(tuple(ant_utils.num_states))) for i in range(epochs): utils.log_statement("*** ------- EPOCH %d ------- ***" % i) # clear initial state if applicable. if not args.initial_state: initial_state = [] else: utils.log_statement(initial_state) utils.log_statement("max reward: " + str(np.max(reward_fn))) logger_kwargs = setup_logger_kwargs("model%02d" % i, data_dir=experiment_directory) # Learn policy that maximizes current reward function. print("Learning new oracle...") seed = random.randint(1, 100000) sac = AntSoftActorCritic(lambda: gym.make(args.env), reward_fn=reward_fn, xid=i + 1, seed=seed, gamma=args.gamma, ac_kwargs=dict(hidden_sizes=[args.hid] * args.l), logger_kwargs=logger_kwargs, normalization_factors=normalization_factors) # The first policy is random if i == 0: sac.soft_actor_critic(epochs=0) else: sac.soft_actor_critic(epochs=args.episodes, initial_state=initial_state, start_steps=args.start_steps) policies.append(sac) p, _ = sac.test_agent(T, normalization_factors=normalization_factors) distributions.append(p) weights = utils.get_weights(distributions) epoch = 'epoch_%02d' % (i) if args.render: if i < 10: sac.record(T=args.record_steps, n=1, video_dir=video_dir + '/baseline/' + epoch, on_policy=False) sac.record(T=args.record_steps, n=1, video_dir=video_dir + '/entropy/' + epoch, on_policy=True) # Execute the cumulative average policy thus far. # Estimate distribution and entropy. print("Executing mixed policy...") average_p, average_p_xy, initial_state, states_visited, states_visited_xy = \ execute_average_policy(env, policies, T, weights, reward_fn=reward_fn, norm=normalization_factors, initial_state=initial_state, n=args.n, render=args.render, video_dir=video_dir+'/mixed/'+epoch, epoch=i, record_steps=args.record_steps) print("Calculating maxEnt entropy...") round_entropy = entropy(average_p.ravel()) round_entropy_xy = entropy(average_p_xy.ravel()) # Update running averages for maxEnt. print("Updating maxEnt running averages...") running_avg_ent = running_avg_ent * ( i) / float(i + 1) + round_entropy / float(i + 1) running_avg_ent_xy = running_avg_ent_xy * ( i) / float(i + 1) + round_entropy_xy / float(i + 1) running_avg_p *= (i) / float(i + 1) running_avg_p += average_p / float(i + 1) running_avg_p_xy *= (i) / float(i + 1) running_avg_p_xy += average_p_xy / float(i + 1) # update reward function print("Update reward function") eps = 1 / np.sqrt(ant_utils.total_state_space) if args.cumulative: reward_fn = grad_ent(running_avg_p) else: reward_fn = 1. average_p += eps reward_fn /= average_p average_p = None # delete big array # (save for plotting) running_avg_entropies.append(running_avg_ent) running_avg_entropies_xy.append(running_avg_ent_xy) if i in indexes: running_avg_ps_xy.append(np.copy(running_avg_p_xy)) avg_ps_xy.append(np.copy(average_p_xy)) print("Collecting baseline experience....") p_baseline, p_baseline_xy, states_visited_baseline, states_visited_xy_baseline = sac.test_agent_random( T, normalization_factors=normalization_factors, n=args.n) plotting.states_visited_over_time(states_visited, states_visited_baseline, i) plotting.states_visited_over_time(states_visited_xy, states_visited_xy_baseline, i, ext='_xy') # save for cumulative plot. if i in states_visited_indexes: # average over a whole bunch of rollouts # slow: so only do this when needed. print("Averaging unique xy states visited....") states_visited_xy = compute_states_visited_xy( env, policies, norm=normalization_factors, T=T, n=args.n, N=args.avg_N) states_visited_xy_baseline = compute_states_visited_xy( env, policies, norm=normalization_factors, T=T, n=args.n, N=args.avg_N, initial_state=initial_state, baseline=True) states_visited_cumulative.append(states_visited_xy) states_visited_cumulative_baseline.append( states_visited_xy_baseline) print("Compute baseline entropy....") round_entropy_baseline = entropy(p_baseline.ravel()) round_entropy_baseline_xy = entropy(p_baseline_xy.ravel()) # Update baseline running averages. print("Updating baseline running averages...") running_avg_ent_baseline = running_avg_ent_baseline * ( i) / float(i + 1) + round_entropy_baseline / float(i + 1) running_avg_ent_baseline_xy = running_avg_ent_baseline_xy * ( i) / float(i + 1) + round_entropy_baseline_xy / float(i + 1) running_avg_p_baseline *= (i) / float(i + 1) running_avg_p_baseline += p_baseline / float(i + 1) running_avg_p_baseline_xy *= (i) / float(i + 1) running_avg_p_baseline_xy += p_baseline_xy / float(i + 1) p_baseline = None # (save for plotting) running_avg_entropies_baseline.append(running_avg_ent_baseline) running_avg_entropies_baseline_xy.append(running_avg_ent_baseline_xy) if i in indexes: running_avg_ps_baseline_xy.append( np.copy(running_avg_p_baseline_xy)) avg_ps_baseline_xy.append(np.copy(p_baseline_xy)) utils.log_statement(average_p_xy) utils.log_statement(p_baseline_xy) # Calculate percent of state space visited. pct = np.count_nonzero(running_avg_p) / float(running_avg_p.size) pct_visited.append(pct) pct_xy = np.count_nonzero(running_avg_p_xy) / float( running_avg_p_xy.size) pct_visited_xy.append(pct_xy) pct_baseline = np.count_nonzero(running_avg_p_baseline) / float( running_avg_p_baseline.size) pct_visited_baseline.append(pct_baseline) pct_xy_baseline = np.count_nonzero(running_avg_p_baseline_xy) / float( running_avg_p_baseline_xy.size) pct_visited_xy_baseline.append(pct_xy_baseline) # Print round summary. col_headers = ["", "baseline", "maxEnt"] col1 = [ "round_entropy_xy", "running_avg_ent_xy", "round_entropy", "running_avg_ent", "% state space xy", "% total state space" ] col2 = [ round_entropy_baseline_xy, running_avg_ent_baseline_xy, round_entropy_baseline, running_avg_ent_baseline, pct_xy_baseline, pct_baseline ] col3 = [ round_entropy_xy, running_avg_ent_xy, round_entropy, running_avg_ent, pct_xy, pct ] table = tabulate(np.transpose([col1, col2, col3]), col_headers, tablefmt="fancy_grid", floatfmt=".4f") utils.log_statement(table) # Plot from round. plotting.heatmap(running_avg_p_xy, average_p_xy, i) plotting.heatmap1(running_avg_p_baseline_xy, i) if i == states_visited_indexes[3]: plotting.states_visited_over_time_multi( states_visited_cumulative, states_visited_cumulative_baseline, states_visited_indexes) # save final expert weights to use with the trained oracles. weights_file = experiment_directory + '/policy_weights' np.save(weights_file, weights) # cumulative plots. plotting.running_average_entropy(running_avg_entropies, running_avg_entropies_baseline) plotting.running_average_entropy(running_avg_entropies_xy, running_avg_entropies_baseline_xy, ext='_xy') plotting.heatmap4(running_avg_ps_xy, running_avg_ps_baseline_xy, indexes, ext="cumulative") plotting.heatmap4(avg_ps_xy, avg_ps_baseline_xy, indexes, ext="epoch") plotting.percent_state_space_reached(pct_visited, pct_visited_baseline, ext='_total') plotting.percent_state_space_reached(pct_visited_xy, pct_visited_xy_baseline, ext="_xy") return policies
logger.dump_tabular() if __name__ == '__main__': import argparse parser = argparse.ArgumentParser() parser.add_argument('--env', type=str, default='Pendulum-v0') parser.add_argument('--hid', type=int, default=64) parser.add_argument('--l', type=int, default=2) parser.add_argument('--gamma', type=float, default=0.99) parser.add_argument('--seed', '-s', type=int, default=0) parser.add_argument('--cpu', type=int, default=4) parser.add_argument('--steps', type=int, default=4000) parser.add_argument('--epochs', type=int, default=200) parser.add_argument('--exp_name', type=str, default='ppo') args = parser.parse_args() mpi_fork(args.cpu) # run parallel code with mpi from spinup.utils.run_utils import setup_logger_kwargs logger_kwargs = setup_logger_kwargs(args.exp_name, args.seed) ppo(lambda: gym.make(args.env), actor_critic=core.MLPActorCritic, ac_kwargs=dict(hidden_sizes=[args.hid] * args.l), gamma=args.gamma, seed=args.seed, steps_per_epoch=args.steps, epochs=args.epochs, logger_kwargs=logger_kwargs)
parser.add_argument('--ensemble_size', type=int, default=20) parser.add_argument('--replay_buf_bootstrap_p', type=float, default=0.75) parser.add_argument('--hardcopy_target_nn', action="store_true", help='Target network update method: hard copy') parser.add_argument('--act_noise',type=float, default=0.1) parser.add_argument("--exploration-strategy", type=str, choices=["action_noise", "epsilon_greedy"], default='epsilon_greedy', help='action_noise or epsilon_greedy') parser.add_argument("--epsilon-max", type=float, default=1.0, help='maximum of epsilon') parser.add_argument("--epsilon-min", type=float, default=.01, help='minimum of epsilon') parser.add_argument("--epsilon-decay", type=float, default=.001, help='epsilon decay') parser.add_argument("--data_dir", type=str, default=None) args = parser.parse_args() # Set log data saving directory from spinup.utils.run_utils import setup_logger_kwargs data_dir = osp.join(osp.dirname(osp.dirname(osp.dirname(osp.dirname(osp.dirname(osp.abspath(__file__)))))), 'spinup_data') logger_kwargs = setup_logger_kwargs(args.exp_name, args.seed, data_dir, datestamp=True) dbedpg(env_name=args.env, render_env=args.render_env, act_noise=args.act_noise, ac_kwargs=dict(hidden_sizes=[args.hid]*args.l), gamma=args.gamma, seed=args.seed, ensemble_size=args.ensemble_size, replay_buf_bootstrap_p=args.replay_buf_bootstrap_p, epochs=args.epochs, steps_per_epoch=args.steps_per_epoch, start_steps=args.start_steps, logger_kwargs=logger_kwargs)