def __init__(self, env, policy, baseline, max_kl): """ env = only structural info of env is used here; you need to pass the 'mode' to functions of this class max_kl = constraint for determining step-size (suggested: 1e-2 or 5e-3) """ self.policy = policy self.env = env self.baseline = baseline self.optimizer = ConjugateGradientOptimizer(**dict()) # Define symbolic variables self.observations_var = self.env.observation_space.new_tensor_variable( 'observations', extra_dims=1) self.actions_var = self.env.action_space.new_tensor_variable( 'actions', extra_dims=1) self.advantages_var = TT.vector('advantages') self.dist = self.policy.distribution self.old_dist_info_vars = { k: ext.new_tensor('old_%s' % k, ndim=2, dtype=theano.config.floatX) for k in self.dist.dist_info_keys } self.old_dist_info_vars_list = [ self.old_dist_info_vars[k] for k in self.dist.dist_info_keys ] self.state_info_vars = { k: ext.new_tensor(k, ndim=2, dtype=theano.config.floatX) for k in self.policy.state_info_keys } self.state_info_vars_list = [ self.state_info_vars[k] for k in self.policy.state_info_keys ] self.dist_info_vars = self.policy.dist_info_sym( self.observations_var, self.state_info_vars) # distribution info variable (symbolic) -- interpret as pi self.KL = self.dist.kl_sym(self.old_dist_info_vars, self.dist_info_vars) self.LR = self.dist.likelihood_ratio_sym(self.actions_var, self.old_dist_info_vars, self.dist_info_vars) self.mean_KL = TT.mean(self.KL) self.surr = -TT.mean(self.LR * self.advantages_var) self.input_list = [self.observations_var, self.actions_var, self.advantages_var] + \ self.state_info_vars_list + self.old_dist_info_vars_list self.optimizer.update_opt(loss=self.surr, target=self.policy, \ leq_constraint=(self.mean_KL, max_kl), \ inputs=self.input_list, constraint_name="mean_kl")
def __init__(self, optimizer=None, optimizer_args=None, optimizer_low=None, **kwargs): if optimizer is None: if optimizer_args is None: optimizer_args = dict() optimizer = ConjugateGradientOptimizer(**optimizer_args) optimizer_low = ConjugateGradientOptimizer(**optimizer_args) super(TRPO, self).__init__(optimizer=optimizer, optimizer_low=optimizer_low, **kwargs)
def __init__(self, optimizer=None, optimizer_args=None, **kwargs): print("init in TRPO_snn") if optimizer is None: if optimizer_args is None: optimizer_args = dict() optimizer = ConjugateGradientOptimizer(**optimizer_args) super(TRPO_snn, self).__init__(optimizer=optimizer, **kwargs)
def __init__( self, optimizer=None, optimizer_args=None, observation_permutation = None, action_permutation = None, sym_loss_weight = 0.0001, action_reg_weight = 0.0, **kwargs): if optimizer is None: if optimizer_args is None: optimizer_args = dict() optimizer = ConjugateGradientOptimizer(**optimizer_args) super(TRPO_Symmetry, self).__init__(optimizer=optimizer, **kwargs) self.observation_permutation = observation_permutation self.action_permutation = action_permutation self.sym_loss_weight = sym_loss_weight self.action_reg_weight = action_reg_weight self.obs_perm_mat = np.zeros((len(observation_permutation), len(observation_permutation))) self.act_per_mat = np.zeros((len(action_permutation), len(action_permutation))) for i, perm in enumerate(self.observation_permutation): self.obs_perm_mat[i][int(np.abs(perm))] = np.sign(perm) for i, perm in enumerate(self.action_permutation): self.act_per_mat[i][int(np.abs(perm))] = np.sign(perm)
def __init__(self, optimizer=None, optimizer_args=None, **kwargs): Serializable.quick_init(self, locals()) if optimizer is None: if optimizer_args is None: optimizer_args = dict() optimizer = ConjugateGradientOptimizer(**optimizer_args) super(TRPO, self).__init__(optimizer=optimizer, **kwargs)
def run_task(*_): # Please note that different environments with different action spaces may # require different policies. For example with a Discrete action space, a # CategoricalMLPPolicy works, but for a Box action space may need to use # a GaussianMLPPolicy (see the trpo_gym_pendulum.py example) env = normalize(GymEnv("CartPole-v0", record_video=False, force_reset=True)) policy = CategoricalMLPPolicy( env_spec=env.spec, # The neural network policy should have two hidden layers, each with 32 hidden units. hidden_sizes=(32, 32) ) baseline = LinearFeatureBaseline(env_spec=env.spec) algo = TRPO( env=env, policy=policy, baseline=baseline, batch_size=4000, max_path_length=env.horizon, n_itr=50, discount=0.99, step_size=0.01, optimizer=ConjugateGradientOptimizer(hvp_approach=FiniteDifferenceHvp(base_eps=1e-5, symmetric=False)) # Uncomment both lines (this and the plot parameter below) to enable plotting # plot=True, ) algo.train()
def __init__( self, optimizer=None, optimizer_args=None, guiding_policies=[], # guiding policies for training up guiding_policy_mps=[], guiding_policy_weight=0.0, guiding_policy_batch_sizes=[], guiding_policy_pool_size=0, guiding_policy_sample_size=0, **kwargs): if optimizer is None: if optimizer_args is None: optimizer_args = dict() optimizer = ConjugateGradientOptimizer(**optimizer_args) super(TRPOGuide, self).__init__(optimizer=optimizer, **kwargs) self.guiding_policies = guiding_policies if len(self.guiding_policies) != 0: self.guiding_policy_mps = guiding_policy_mps self.guiding_policy_weight = guiding_policy_weight self.guiding_policy_batch_sizes = guiding_policy_batch_sizes self.guiding_policy_sample_pool = SimpleGuidingSamplePool( guiding_policy_pool_size, self.env.observation_space.shape[0], self.env.action_space.shape[0]) self.guiding_policy_sample_size = guiding_policy_sample_size
def __init__(self, optimizer=None, optimizer_args=None, **kwargs): if optimizer is None: if optimizer_args is None: optimizer_args = dict() optimizer = ConjugateGradientOptimizer(**optimizer_args) #optimizer = PenaltyLbfgsOptimizer(**optimizer_args) super(TRPO, self).__init__(optimizer=optimizer, **kwargs)
def __init__(self, optimizer=None, optimizer_args=None, **kwargs): n = len(kwargs['policy'].policies) if optimizer is None: if optimizer_args is None: optimizer_args = dict() optimizers = [ ConjugateGradientOptimizer(**optimizer_args) for _ in range(n) ] super(MultiTRPO, self).__init__(optimizer=optimizers, **kwargs)
def __init__(self, optimizer=None, optimizer_args=None, **kwargs): if optimizer is None: default_args = dict(max_backtracks=1) if optimizer_args is None: optimizer_args = default_args else: optimizer_args = dict(default_args, **optimizer_args) optimizer = ConjugateGradientOptimizer(**optimizer_args) super(TNPG, self).__init__(optimizer=optimizer, **kwargs)
def __init__(self, optimizer=None, optimizer_args=None, task_num=1, **kwargs): if optimizer is None: if optimizer_args is None: optimizer_args = dict() optimizer = ConjugateGradientOptimizer(**optimizer_args) self.task_num = task_num self.kl_weights = np.ones(task_num) super(TRPO_MultiTask, self).__init__(optimizer=optimizer, **kwargs)
def __init__( self, optimizers=None, #dictionary of optimizers optimizer_args=None, #dictionary of optimizer parameters agent_names = ['hide', 'seek'], #names of agents **kwargs): if optimizers is None: optimizers = {} for name in agent_names: if optimizer_args is None: optimizer_args = dict() optimizers[name] = ConjugateGradientOptimizer(**optimizer_args) super(TRPO, self).__init__(optimizers=optimizers, **kwargs)
def __init__(self, env, policy, baseline, max_kl): """ env = only structural info of env is used here; you need to pass the 'mode' to functions of this class max_kl = constraint for determining step-size (suggested: 1e-2 or 5e-3) """ self.policy = policy self.env = env self.baseline = baseline self.optimizer = ConjugateGradientOptimizer(**dict()) # Define symbolic variables self.observations_var = self.env.observation_space.new_tensor_variable('observations', extra_dims=1) self.actions_var = self.env.action_space.new_tensor_variable('actions', extra_dims=1) self.advantages_var = TT.vector('advantages') self.dist = self.policy.distribution self.old_dist_info_vars = { k: ext.new_tensor( 'old_%s' % k, ndim=2, dtype=theano.config.floatX ) for k in self.dist.dist_info_keys } self.old_dist_info_vars_list = [self.old_dist_info_vars[k] for k in self.dist.dist_info_keys] self.state_info_vars = { k: ext.new_tensor( k, ndim=2, dtype=theano.config.floatX ) for k in self.policy.state_info_keys } self.state_info_vars_list = [self.state_info_vars[k] for k in self.policy.state_info_keys] self.dist_info_vars = self.policy.dist_info_sym(self.observations_var, self.state_info_vars) # distribution info variable (symbolic) -- interpret as pi self.KL = self.dist.kl_sym(self.old_dist_info_vars, self.dist_info_vars) self.LR = self.dist.likelihood_ratio_sym(self.actions_var, self.old_dist_info_vars, self.dist_info_vars) self.mean_KL = TT.mean(self.KL) self.surr = - TT.mean(self.LR * self.advantages_var) self.input_list = [self.observations_var, self.actions_var, self.advantages_var] + \ self.state_info_vars_list + self.old_dist_info_vars_list self.optimizer.update_opt(loss=self.surr, target=self.policy, \ leq_constraint=(self.mean_KL, max_kl), \ inputs=self.input_list, constraint_name="mean_kl")
def run_task(*_): env = normalize(CartpoleEnv()) policy = GaussianGRUPolicy(env_spec=env.spec, ) baseline = LinearFeatureBaseline(env_spec=env.spec) algo = TRPO(env=env, policy=policy, baseline=baseline, batch_size=4000, max_path_length=100, n_itr=10, discount=0.99, step_size=0.01, optimizer=ConjugateGradientOptimizer( hvp_approach=FiniteDifferenceHvp(base_eps=1e-5))) algo.train()
def __init__(self, optimizer=None, optimizer_args=None, aux_pred_step=3, aux_pred_dim=4, pool_batch_size=50000, **kwargs): if optimizer is None: if optimizer_args is None: optimizer_args = dict() optimizer = ConjugateGradientOptimizer(**optimizer_args) self.pool_batch_size = pool_batch_size self.aux_pred_step = aux_pred_step super(TRPOAux, self).__init__(optimizer=optimizer, **kwargs) self.aux_pred_dim = aux_pred_dim self.aux_pred_pool = SimpleReplayPoolAux( 50000, self.env.observation_space.shape[0] * self.aux_pred_step, aux_pred_dim)
def _buildRLAlg(useCG, algDict, optimizerArgs=None): #RL Algorithm if optimizerArgs is None: optimizerArgs = dict() if useCG: #either use CG optimizer == TRPO optimizer = ConjugateGradientOptimizer(**optimizerArgs) #or use BFGS optimzier == penalized policy optimization TODO can this be an avenue to PPO? does it not require also liklihood truncation? else: optimizer = PenaltyLbfgsOptimizer(**optimizerArgs) #NPO is expecting in ctor : #self.optimizer = optimizer - need to specify this or else defaults to PenaltyLbfgsOptimizer #self.step_size = step_size : defaults to 0.01 #truncate_local_is_ratio means to truncate distribution likelihood ration, which is defined as # lr = dist.likelihood_ratio_sym(action_var, old_dist_info_vars, dist_info_vars) # if truncation is not none : lr = TT.minimum(self.truncate_local_is_ratio, lr) #self.truncate_local_is_ratio = truncate_local_is_ratio algo = NPO(optimizer=optimizer, **algDict) return algo
def __init__(self, optimizer=None, optimizer_args=None, mp_dim=2, **kwargs): if optimizer is None: if optimizer_args is None: optimizer_args = dict() optimizer = ConjugateGradientOptimizer(**optimizer_args) self.mp_dim = mp_dim # genearte data for weight entropy related objective terms self.base = np.zeros(4) ent_input = [] for i in range(1000): ent_input.append( np.concatenate([self.base, np.random.random(self.mp_dim)]).tolist()) self.ent_input = [np.array(ent_input)] super(TRPOMPSel, self).__init__(optimizer=optimizer, **kwargs)
def run_task(*_): # Non-registration of this custom environment is an rllab bug # See https://github.com/openai/rllab/issues/68 # At the moment I'm bypassing this problem by adding the # import statement in gym_env.py import gym_follower_2d import lasagne.nonlinearities as NL gymenv = GymEnv(args.env, force_reset=True, record_video=False, record_log=True) env = normalize(gymenv) logger.log("Training Policy on %s" % args.env) policy = GaussianMLPPolicy(env_spec=env.spec, hidden_sizes=(100, 50, 25), hidden_nonlinearity=NL.tanh) baseline = LinearFeatureBaseline(env_spec=env.spec) algo = TRPO( env=env, policy=policy, baseline=baseline, batch_size=args.batch_size, max_path_length=100, n_itr=args.num_epochs, discount=0.99, step_size=args.step_size, optimizer=ConjugateGradientOptimizer( reg_coeff=args.reg_coeff, hvp_approach=FiniteDifferenceHvp(base_eps=args.reg_coeff)), plot=False, ) algo.train()
decent_portion=0.8, disc_window=2, iteration=3000, disc_joints_dim=16, hidden_sizes=(128, 64, 32)) # baseline #env = normalize(HumanEnv_v2(discriminator=None), normalize_obs=True) # GAN imitaion env = normalize(HumanEnv_v2(discriminator=discriminator), normalize_obs=True) # print(env.action_space.bounds) # print(env.observation_space.bounds) policy = GaussianMLPPolicy(env_spec=env.spec, hidden_sizes=(100, 50, 25)) base_line_optimizer = ConjugateGradientOptimizer() baseline = GaussianMLPBaseline(env.spec, regressor_args={ "mean_network": None, "hidden_sizes": (100, 50, 25), "hidden_nonlinearity": NL.tanh, "optimizer": base_line_optimizer, "use_trust_region": True, "step_size": 0.01, "learn_std": True, "init_std": 1.0, "adaptive_std": False, "std_share_network": False, "std_hidden_sizes": (32, 32), "std_nonlinearity": None, "normalize_inputs": True,
from rllab.baselines.linear_feature_baseline import LinearFeatureBaseline from rllab.envs.box2d.cartpole_env import CartpoleEnv from rllab.envs.normalized_env import normalize from rllab.policies.gaussian_gru_policy import GaussianGRUPolicy from rllab.optimizers.conjugate_gradient_optimizer import ConjugateGradientOptimizer, FiniteDifferenceHvp from rllab.misc.instrument import stub, run_experiment_lite stub(globals()) env = normalize(CartpoleEnv()) policy = GaussianGRUPolicy(env_spec=env.spec, ) baseline = LinearFeatureBaseline(env_spec=env.spec) algo = TRPO(env=env, policy=policy, baseline=baseline, batch_size=4000, max_path_length=100, n_itr=10, discount=0.99, step_size=0.01, optimizer=ConjugateGradientOptimizer( hvp_approach=FiniteDifferenceHvp(base_eps=1e-5))) run_experiment_lite( algo.train(), n_parallel=1, seed=1, )
def run_trpo_vase(env,nRuns = 20,seed_base=0, sigma_c=0.5, ablation_mode=False): now = datetime.datetime.now(dateutil.tz.tzlocal()) timestamp = now.strftime('%Y_%m_%d_%H_%M_%S') for seed in range(seed_base,nRuns): if env == 'mountaincar': mdp = MountainCarEnvX() n_itr = 50 max_path_length = 500 type = 'classic' elif env == 'cartpole': mdp = NormalizedEnv(env=CartpoleSwingupEnvX()) n_itr = 400 max_path_length = 500 type = 'classic' elif env == 'doublependulum': mdp = NormalizedEnv(env=DoublePendulumEnvX()) n_itr = 400 max_path_length = 500 type = 'classic' elif env == 'halfcheetah': mdp = NormalizedEnv(env=HalfCheetahEnvX()) n_itr = 600 max_path_length = 500 type = 'locomotion' elif env == 'ant': mdp = NormalizedEnv(env=AntEnv()) n_itr = 600 max_path_length = 500 type = 'locomotion' elif env == 'lunarlander': mdp = NormalizedEnv(env=LunarLanderContinuous()) n_itr = 100 max_path_length = 1000 type = 'classic' else: sys.stderr.write("Error! Environment '%s' not recognised\n" % env) sys.exit(-1) if type == 'classic': step_size = 0.01 replay_pool_size = 100000 policy_hidden_sizes = (32,) unn_n_hidden = [32] unn_layers_type=[1, 1] baseline = GaussianMLPBaseline( env_spec=mdp.spec, regressor_args={ 'hidden_sizes': (32,), 'learn_std': False, 'hidden_nonlinearity': NL.rectify, 'optimizer': ConjugateGradientOptimizer(subsample_factor=1.0) } ) else: step_size = 0.05 replay_pool_size = 5000000 policy_hidden_sizes = (64, 32) unn_n_hidden = [64, 64] unn_layers_type=[1, 1, 1] baseline = LinearFeatureBaseline( mdp.spec, ) policy = GaussianMLPPolicy( env_spec=mdp.spec, hidden_sizes=policy_hidden_sizes, hidden_nonlinearity=NL.tanh ) algo = TRPO( env=mdp, policy=policy, baseline=baseline, n_itr=n_itr, batch_size=5000, max_path_length = max_path_length, discount = 0.995, gae_lambda = 0.95, whole_paths=True, step_size=step_size, eta=1e-4, snn_n_samples=10, prior_sd=0.5, likelihood_sd=sigma_c, subsample_factor=1.0, use_replay_pool=True, replay_pool_size=replay_pool_size, n_updates_per_sample=500, unn_n_hidden=unn_n_hidden, unn_layers_type=unn_layers_type, unn_learning_rate=0.001 ) exp_name = "trpo-vase_%s_%04d" % (timestamp, seed + 1) if ablation_mode: cwd = os.getcwd() log_dir = cwd + "/data/local/sigmas/" + env + ("/%.3f/" % sigma_c) + exp_name else: log_dir = config.LOG_DIR + "/local/" + env + "/" + exp_name run_experiment_lite( algo.train(), exp_name = exp_name, log_dir= log_dir, n_parallel=0, snapshot_mode="last", seed=seed, mode="local", script="sandbox/vase/experiments/run_experiment_lite.py" )
def main(): parser = argparse.ArgumentParser() parser.add_argument('--resume_from', type=str) parser.add_argument('--encoding_levels', type=int, nargs='+') parser.add_argument('--num_encoding_levels', type=int, default=5) parser.add_argument('--conv_filters', nargs='*', type=int, default=[16, 16]) parser.add_argument('--conv_filter_sizes', nargs='*', type=int, default=[4, 4]) parser.add_argument('--conv_strides', nargs='*', type=int, default=[2, 2]) parser.add_argument('--hidden_sizes', nargs='*', type=int, default=[32, 32]) parser.add_argument('--init_std', type=float, default=1.0) parser.add_argument('--n_itr', type=int, default=500) parser.add_argument('--step_size', type=float, default=0.01) parser.add_argument('--batch_size', type=int, default=4000) parser.add_argument('--seed', type=int, default=0) parser.add_argument('--custom_local_flags', type=str, default=None) args = parser.parse_args() np.random.seed(args.seed) env = SimpleQuadPanda3dEnv(action_space=TranslationAxisAngleSpace( low=[-10., -10., -10., -1.5707963267948966], high=[10., 10., 10., 1.5707963267948966], axis=[0., 0., 1.]), sensor_names=['image'], camera_size=[256, 256], camera_hfov=26.007823885645635, car_env_class=GeometricCarPanda3dEnv, car_action_space=BoxSpace(low=[0., 0.], high=[0., 0.]), car_model_names=[ 'mazda6', 'chevrolet_camaro', 'nissan_gt_r_nismo', 'lamborghini_aventador', 'golf5' ], dt=0.1) env = ServoingEnv(env) transformers = { 'image': CompositionTransformer([ ImageTransformer(scale_size=0.5), OpsTransformer(transpose=(2, 0, 1)) ]), 'action': NormalizerTransformer(space=env.action_space) } env = RllabEnv(env, transformers=transformers) env = normalize(env) assert len(args.conv_filters) == len(args.conv_filter_sizes) assert len(args.conv_filters) == len(args.conv_strides) network_kwargs = dict(encoding_levels=args.encoding_levels, num_encoding_levels=args.num_encoding_levels, conv_filters=args.conv_filters, conv_filter_sizes=args.conv_filter_sizes, conv_strides=args.conv_strides, conv_pads=[0] * len(args.conv_filters), hidden_sizes=args.hidden_sizes, hidden_nonlinearity=LN.rectify, output_nonlinearity=None, name="mean_network") mean_network = VggConvNetwork(input_shape=env.observation_space.shape, output_dim=env.action_space.flat_dim, **network_kwargs) policy = GaussianConvPolicy( env_spec=env.spec, init_std=args.init_std, mean_network=mean_network, ) conv_baseline_kwargs = dict( env_spec=env.spec, regressor_args=dict( mean_network=VggConvNetwork( input_shape=env.observation_space.shape, output_dim=1, **network_kwargs), use_trust_region=True, step_size=args.step_size, normalize_inputs=True, normalize_outputs=True, hidden_sizes=None, conv_filters=None, conv_filter_sizes=None, conv_strides=None, conv_pads=None, batchsize=200, optimizer=PenaltyLbfgsOptimizer(n_slices=50), )) baseline = GaussianConvBaseline(**conv_baseline_kwargs) algo = TRPO( env=env, policy=policy, baseline=baseline, batch_size=args.batch_size, max_path_length=100, n_itr=args.n_itr, discount=0.9, step_size=args.step_size, optimizer=ConjugateGradientOptimizer(num_slices=50), ) if args.resume_from: run_experiment_lite(algo.train(), snapshot_mode='gap', snapshot_gap=10, seed=args.seed, custom_local_flags=args.custom_local_flags, resume_from=args.resume_from) else: run_experiment_lite(algo.train(), snapshot_mode='gap', snapshot_gap=10, seed=args.seed, custom_local_flags=args.custom_local_flags)
def run_task(v): expDict = v ############################### #Env if (expDict['isNormalized']): if (expDict['isGymEnv']): env = normalize( GymEnv(expDict['envName'], record_video=False, record_log=False)) else: env = normalize(expDict['envName']) #if env is normalized then it is wrapped #dartEnv = env.wrapped_env.env.unwrapped else: #if not normalized, needs to be gym environment env = GymEnv(expDict['envName'], record_video=False, record_log=False) if (expDict['blType'] == 'linear'): bl = LinearFeatureBaseline(env_spec=env.spec) elif (expDict['blType'] == 'MLP'): #use regressor_args as dict to define regressor arguments like layers regArgs = dict() regArgs['hidden_sizes'] = expDict['blMlpArch'] #only used if adaptive_std == True regArgs['std_hidden_sizes'] = expDict['blMlpArch'] #defaults to normalizing regArgs['normalize_inputs'] = False regArgs['normalize_outputs'] = False #regArgs['adaptive_std'] = True #regArgs['learn_std']= False #ignored if adaptive_std == true - sets global value which is required for all thread instances bl = GaussianMLPBaseline(env_spec=env.spec, regressor_args=regArgs) else: print('unknown baseline type : ' + expDict['blType']) bl = None ############################### #Policy pol = GaussianMLPPolicy( env_spec=env.spec, hidden_sizes=expDict[ 'polNetArch'] #must be tuple - if only 1 value should be followed by comma i.e. (8,) ) ############################### #RL Algorithm #allow for either trpo or ppo optimizerArgs = expDict['optimizerArgs'] if optimizerArgs is None: optimizerArgs = dict() if expDict['useCG']: #either use CG optimizer == TRPO optimizer = ConjugateGradientOptimizer(**optimizerArgs) print('Using CG optimizer (TRPO)') #or use BFGS optimzier -> ppo? not really else: optimizer = PenaltyLbfgsOptimizer(**optimizerArgs) print('Using LBFGS optimizer (PPO-like ?)') #NPO is expecting in ctor : #self.optimizer = optimizer - need to specify this or else defaults to PenaltyLbfgsOptimizer #self.step_size = step_size : defaults to 0.01 #truncate_local_is_ratio means to truncate distribution likelihood ration, which is defined as # lr = dist.likelihood_ratio_sym(action_var, old_dist_info_vars, dist_info_vars) # if truncation is not none : lr = TT.minimum(self.truncate_local_is_ratio, lr) #self.truncate_local_is_ratio = truncate_local_is_ratio algo = NPO(optimizer=optimizer, env=env, policy=pol, baseline=bl, batch_size=int(expDict['numBatches']), whole_paths=True, gae_lambda=float(expDict['gae_lambda']), max_path_length=int(expDict['maxPathLength']), n_itr=int(expDict['numIters']), discount=0.99, step_size=0.01, start_itr=1) algo.train()
def run_trpo(env, nRuns=20, seed_base=0): now = datetime.datetime.now(dateutil.tz.tzlocal()) timestamp = now.strftime('%Y_%m_%d_%H_%M_%S') for seed in range(seed_base, nRuns): if env == 'mountaincar': mdp = MountainCarEnvX() n_itr = 50 max_path_length = 500 type = 'classic' elif env == 'cartpole': mdp = NormalizedEnv(env=CartpoleSwingupEnvX()) n_itr = 400 max_path_length = 500 type = 'classic' elif env == 'doublependulum': mdp = NormalizedEnv(env=DoublePendulumEnvX()) n_itr = 400 max_path_length = 500 type = 'classic' elif env == 'halfcheetah': mdp = NormalizedEnv(env=HalfCheetahEnvX()) n_itr = 600 max_path_length = 500 type = 'locomotion' elif env == 'ant': mdp = NormalizedEnv(env=AntEnv()) n_itr = 600 max_path_length = 500 type = 'locomotion' elif env == 'lunarlander': mdp = NormalizedEnv(env=LunarLanderContinuous()) n_itr = 100 max_path_length = 1000 type = 'classic' else: sys.stderr.write("Error! Environment '%s' not recognised\n" % env) sys.exit(-1) if type == 'classic': step_size = 0.01 policy_hidden_sizes = (32, ) baseline = GaussianMLPBaseline( env_spec=mdp.spec, regressor_args={ 'hidden_sizes': (32, ), 'learn_std': False, 'hidden_nonlinearity': NL.rectify, 'optimizer': ConjugateGradientOptimizer(subsample_factor=1.0) }) else: step_size = 0.05 policy_hidden_sizes = (64, 32) baseline = LinearFeatureBaseline(mdp.spec, ) policy = GaussianMLPPolicy(env_spec=mdp.spec, hidden_sizes=policy_hidden_sizes, hidden_nonlinearity=NL.tanh) algo = TRPO( env=mdp, policy=policy, baseline=baseline, batch_size=5000, whole_paths=True, max_path_length=max_path_length, n_itr=n_itr, step_size=step_size, subsample_factor=1.0, ) exp_name = "trpo_%s_%04d" % (timestamp, seed + 1) log_dir = config.LOG_DIR + "/local/" + env + "/" + exp_name run_experiment_lite(algo.train(), exp_name=exp_name, log_dir=log_dir, n_parallel=0, snapshot_mode="last", seed=seed, mode="local")
class TRPO: def __init__(self, env, policy, baseline, max_kl): """ env = only structural info of env is used here; you need to pass the 'mode' to functions of this class max_kl = constraint for determining step-size (suggested: 1e-2 or 5e-3) """ self.policy = policy self.env = env self.baseline = baseline self.optimizer = ConjugateGradientOptimizer(**dict()) # Define symbolic variables self.observations_var = self.env.observation_space.new_tensor_variable('observations', extra_dims=1) self.actions_var = self.env.action_space.new_tensor_variable('actions', extra_dims=1) self.advantages_var = TT.vector('advantages') self.dist = self.policy.distribution self.old_dist_info_vars = { k: ext.new_tensor( 'old_%s' % k, ndim=2, dtype=theano.config.floatX ) for k in self.dist.dist_info_keys } self.old_dist_info_vars_list = [self.old_dist_info_vars[k] for k in self.dist.dist_info_keys] self.state_info_vars = { k: ext.new_tensor( k, ndim=2, dtype=theano.config.floatX ) for k in self.policy.state_info_keys } self.state_info_vars_list = [self.state_info_vars[k] for k in self.policy.state_info_keys] self.dist_info_vars = self.policy.dist_info_sym(self.observations_var, self.state_info_vars) # distribution info variable (symbolic) -- interpret as pi self.KL = self.dist.kl_sym(self.old_dist_info_vars, self.dist_info_vars) self.LR = self.dist.likelihood_ratio_sym(self.actions_var, self.old_dist_info_vars, self.dist_info_vars) self.mean_KL = TT.mean(self.KL) self.surr = - TT.mean(self.LR * self.advantages_var) self.input_list = [self.observations_var, self.actions_var, self.advantages_var] + \ self.state_info_vars_list + self.old_dist_info_vars_list self.optimizer.update_opt(loss=self.surr, target=self.policy, \ leq_constraint=(self.mean_KL, max_kl), \ inputs=self.input_list, constraint_name="mean_kl") def train(self, N, T, gamma, niter, env_mode='train'): """ N = number of trajectories T = horizon niter = number of iterations to update the policy env_mode = can be 'train', 'test' or something else. You need to write the appropriate function in MDP_funcs """ eval_statistics = [] for iter in range(niter): curr_iter_stats = self.train_step(N, T, gamma, env_mode) eval_statistics.append(curr_iter_stats) return eval_statistics def train_step(self, N, T, gamma, env_mode='train', num_cpu='max', save_paths=False, idx=None, mujoco_env=True, normalized_env=False, sub_sample=None, train_env=None): """ N = number of trajectories T = horizon env_mode = can be 'train', 'test' or something else. You need to write the appropriate function in MDP_funcs """ if train_env == None: paths = sample_paths_parallel(N, self.policy, self.baseline, env_mode, T, gamma, num_cpu=num_cpu, mujoco_env=mujoco_env, normalized_env=normalized_env) else: paths = sample_paths(N, self.policy, self.baseline, env=train_env, T=T, gamma=gamma, mujoco_env=mujoco_env, normalized_env=normalized_env) # save the paths used to make the policy update if save_paths == True and idx != None: robustRL.utils.save_paths(paths, idx) eval_statistics = self.train_from_paths(paths, sub_sample=sub_sample) eval_statistics[0].append(N) return eval_statistics def train_from_paths(self, paths, sub_sample=None, path_percentile=[10,15,33,50,66,85,90]): if sub_sample != None: # Pick subset of paths whose returns are in the sub_sample percentile range path_returns = [sum(p["rewards"]) for p in paths] sub_range = [np.percentile(path_returns, sub_sample[i]) for i in range(2)] # Find paths which satisfy criteria idx = [i for i,ret in enumerate(path_returns) if sub_range[0]<=ret and ret<=sub_range[1]] chosen_paths = [paths[i] for i in idx] else: chosen_paths = paths self.baseline.fit(paths) # concatenate from all the trajectories observations = tensor_utils.concat_tensor_list([path["observations"] for path in chosen_paths]) actions = tensor_utils.concat_tensor_list([path["actions"] for path in chosen_paths]) rewards = tensor_utils.concat_tensor_list([path["rewards"] for path in chosen_paths]) advantages = tensor_utils.concat_tensor_list([path["advantages"] for path in chosen_paths]) env_infos = tensor_utils.concat_tensor_dict_list([path["env_infos"] for path in chosen_paths]) agent_infos = tensor_utils.concat_tensor_dict_list([path["agent_infos"] for path in chosen_paths]) samples_data = dict( observations=observations, actions=actions, rewards=rewards, advantages=advantages, env_infos=env_infos, agent_infos=agent_infos, ) all_input_values = tuple(ext.extract( samples_data, "observations", "actions", "advantages" )) agent_infos = samples_data["agent_infos"] state_info_list = [agent_infos[k] for k in self.policy.state_info_keys] dist_info_list = [agent_infos[k] for k in self.policy.distribution.dist_info_keys] all_input_values += tuple(state_info_list) + tuple(dist_info_list) # Take a step with optimizer self.optimizer.optimize(all_input_values) # cache return distributions for the paths path_returns = [sum(p["rewards"]) for p in paths] mean_return = np.mean(path_returns) std_return = np.std(path_returns) min_return = np.amin(path_returns) max_return = np.amax(path_returns) sub_mean = np.mean([sum(p["rewards"]) for p in chosen_paths]) base_stats = [mean_return, std_return, min_return, max_return, sub_mean] percetile_stats = [] for p in path_percentile: percetile_stats.append(np.percentile(path_returns, p)) return [base_stats, percetile_stats]
env_spec=env.spec, hidden_sizes=(32, ), ) baseline = GaussianMLPBaseline( env_spec=env.spec, regressor_args={ 'hidden_sizes': (32, ), 'hidden_nonlinearity': NL.tanh, 'learn_std': False, 'step_size': 0.01, 'optimizer': ConjugateGradientOptimizer( subsample_factor=trpo_subsample_factor) }) elif task_type == 'locomotion': trpo_max_path_length = 500 trpo_batch_size = 5000 trpo_subsample_factor = 1 trpo_step_size = 0.05 expl_lambda = 0.001 policy = GaussianMLPPolicy( env_spec=env.spec, hidden_sizes=(64, 32), )