def run_linear_ocm_exp(variant):
from sandbox.rocky.tf.algos.trpo import TRPO
from rllab.baselines.linear_feature_baseline import LinearFeatureBaseline
from sandbox.rocky.tf.policies.gaussian_lstm_policy import GaussianLSTMPolicy
import sandbox.rocky.tf.core.layers as L
from sandbox.rocky.tf.optimizers.conjugate_gradient_optimizer import (
ConjugateGradientOptimizer,
FiniteDifferenceHvp,
)
from railrl.envs.flattened_product_box import FlattenedProductBox
from railrl.envs.memory.continuous_memory_augmented import (
ContinuousMemoryAugmented)
from railrl.envs.memory.one_char_memory import (
OneCharMemoryEndOnly, )
from railrl.envs.memory.high_low import HighLow
from railrl.launchers.launcher_util import (
set_seed, )
"""
Set up experiment variants.
"""
H = variant['H']
seed = variant['seed']
num_values = variant['num_values']
set_seed(seed)
onehot_dim = num_values + 1
"""
Code for running the experiment.
"""
# env = OneCharMemoryEndOnly(n=num_values, num_steps=H, softmax_action=True)
env = HighLow(num_steps=H)
env = ContinuousMemoryAugmented(
env,
num_memory_states=onehot_dim,
)
env = FlattenedProductBox(env)
policy = GaussianLSTMPolicy(
name="policy",
env_spec=env.spec,
lstm_layer_cls=L.LSTMLayer,
)
baseline = LinearFeatureBaseline(env_spec=env.spec)
optimizer_params = variant['optimizer_params']
trpo_params = variant['trpo_params']
algo = TRPO(env=env,
policy=policy,
baseline=baseline,
optimizer=ConjugateGradientOptimizer(
hvp_approach=FiniteDifferenceHvp(**optimizer_params)),
**trpo_params)
algo.train()
from sandbox.rocky.tf.algos.trpo import TRPO
from rllab.baselines.linear_feature_baseline import LinearFeatureBaseline
from rllab.envs.box2d.cartpole_env import CartpoleEnv
from rllab.envs.normalized_env import normalize
from sandbox.rocky.tf.policies.gaussian_gru_policy import GaussianGRUPolicy
from sandbox.rocky.tf.policies.gaussian_lstm_policy import GaussianLSTMPolicy
from sandbox.rocky.tf.envs.base import TfEnv
import sandbox.rocky.tf.core.layers as L
from sandbox.rocky.tf.optimizers.conjugate_gradient_optimizer import ConjugateGradientOptimizer, FiniteDifferenceHvp
from rllab.misc.instrument import stub, run_experiment_lite
env = TfEnv(normalize(CartpoleEnv()))
policy = GaussianLSTMPolicy(
name="policy",
env_spec=env.spec,
lstm_layer_cls=L.TfBasicLSTMLayer,
# gru_layer_cls=L.GRULayer,
)
baseline = LinearFeatureBaseline(env_spec=env.spec)
algo = TRPO(
env=env,
policy=policy,
baseline=baseline,
batch_size=4000,
max_path_length=100,
n_itr=10,
discount=0.99,
step_size=0.01,
optimizer=ConjugateGradientOptimizer(hvp_approach=FiniteDifferenceHvp(base_eps=1e-5))
def rllab_envpolicy_parser(env, args):
if isinstance(args, dict):
args = tonamedtuple(args)
env = RLLabEnv(env, mode=args.control)
if args.algo[:2] == 'tf':
env = TfEnv(env)
# Policy
if args.recurrent:
if args.feature_net:
feature_network = MLP(
name='feature_net',
input_shape=(env.spec.observation_space.flat_dim +
env.spec.action_space.flat_dim, ),
output_dim=args.feature_output,
hidden_sizes=tuple(args.feature_hidden),
hidden_nonlinearity=tf.nn.tanh,
output_nonlinearity=None)
elif args.conv:
strides = tuple(args.conv_strides)
chans = tuple(args.conv_channels)
filts = tuple(args.conv_filters)
assert len(strides) == len(chans) == len(
filts), "strides, chans and filts not equal"
# only discrete actions supported, should be straightforward to extend to continuous
assert isinstance(
env.spec.action_space,
Discrete), "Only discrete action spaces support conv"
feature_network = ConvNetwork(
name='feature_net',
input_shape=env.spec.observation_space.shape,
output_dim=args.feature_output,
conv_filters=chans,
conv_filter_sizes=filts,
conv_strides=strides,
conv_pads=('VALID', ) * len(chans),
hidden_sizes=tuple(args.feature_hidden),
hidden_nonlinearity=tf.nn.relu,
output_nonlinearity=None)
else:
feature_network = None
if args.recurrent == 'gru':
if isinstance(env.spec.action_space, Box):
policy = GaussianGRUPolicy(env_spec=env.spec,
feature_network=feature_network,
hidden_dim=int(
args.policy_hidden[0]),
name='policy')
elif isinstance(env.spec.action_space, Discrete):
policy = CategoricalGRUPolicy(
env_spec=env.spec,
feature_network=feature_network,
hidden_dim=int(args.policy_hidden[0]),
name='policy',
state_include_action=False if args.conv else True)
else:
raise NotImplementedError(env.spec.observation_space)
elif args.recurrent == 'lstm':
if isinstance(env.spec.action_space, Box):
policy = GaussianLSTMPolicy(
env_spec=env.spec,
feature_network=feature_network,
hidden_dim=int(args.policy_hidden),
name='policy')
elif isinstance(env.spec.action_space, Discrete):
policy = CategoricalLSTMPolicy(
env_spec=env.spec,
feature_network=feature_network,
hidden_dim=int(args.policy_hidden),
name='policy')
else:
raise NotImplementedError(env.spec.action_space)
else:
raise NotImplementedError(args.recurrent)
elif args.conv:
strides = tuple(args.conv_strides)
chans = tuple(args.conv_channels)
filts = tuple(args.conv_filters)
assert len(strides) == len(chans) == len(
filts), "strides, chans and filts not equal"
# only discrete actions supported, should be straightforward to extend to continuous
assert isinstance(
env.spec.action_space,
Discrete), "Only discrete action spaces support conv"
feature_network = ConvNetwork(
name='feature_net',
input_shape=env.spec.observation_space.shape,
output_dim=env.spec.action_space.n,
conv_filters=chans,
conv_filter_sizes=filts,
conv_strides=strides,
conv_pads=('VALID', ) * len(chans),
hidden_sizes=tuple(args.policy_hidden),
hidden_nonlinearity=tf.nn.relu,
output_nonlinearity=tf.nn.softmax)
policy = CategoricalMLPPolicy(name='policy',
env_spec=env.spec,
prob_network=feature_network)
else:
if isinstance(env.spec.action_space, Box):
policy = GaussianMLPPolicy(env_spec=env.spec,
hidden_sizes=tuple(
args.policy_hidden),
min_std=args.min_std,
name='policy')
elif isinstance(env.spec.action_space, Discrete):
policy = CategoricalMLPPolicy(env_spec=env.spec,
hidden_sizes=tuple(
args.policy_hidden),
name='policy')
else:
raise NotImplementedError(env.spec.action_space)
elif args.algo[:2] == 'th':
# Policy
if args.recurrent:
if args.feature_net:
feature_network = thMLP(
input_shape=(env.spec.observation_space.flat_dim +
env.spec.action_space.flat_dim, ),
output_dim=args.feature_output,
hidden_sizes=tuple(args.feature_hidden),
hidden_nonlinearity=tf.nn.tanh,
output_nonlinearity=None)
else:
feature_network = None
if args.recurrent == 'gru':
if isinstance(env.spec.observation_space, thBox):
policy = thGaussianGRUPolicy(
env_spec=env.spec,
feature_network=feature_network,
hidden_dim=int(args.policy_hidden[0]),
)
elif isinstance(env.spec.observation_space, thDiscrete):
policy = thCategoricalGRUPolicy(
env_spec=env.spec,
feature_network=feature_network,
hidden_dim=int(args.policy_hidden[0]),
)
else:
raise NotImplementedError(env.spec.observation_space)
# elif args.recurrent == 'lstm':
# if isinstance(env.spec.action_space, thBox):
# policy = thGaussianLSTMPolicy(env_spec=env.spec,
# feature_network=feature_network,
# hidden_dim=int(args.policy_hidden),
# name='policy')
# elif isinstance(env.spec.action_space, thDiscrete):
# policy = thCategoricalLSTMPolicy(env_spec=env.spec,
# feature_network=feature_network,
# hidden_dim=int(args.policy_hidden),
# name='policy')
# else:
# raise NotImplementedError(env.spec.action_space)
else:
raise NotImplementedError(args.recurrent)
else:
if args.algo == 'thddpg':
assert isinstance(env.spec.action_space, thBox)
policy = thDeterministicMLPPolicy(
env_spec=env.spec,
hidden_sizes=tuple(args.policy_hidden),
)
else:
if isinstance(env.spec.action_space, thBox):
policy = thGaussianMLPPolicy(env_spec=env.spec,
hidden_sizes=tuple(
args.policy_hidden),
min_std=args.min_std)
elif isinstance(env.spec.action_space, thDiscrete):
policy = thCategoricalMLPPolicy(env_spec=env.spec,
hidden_sizes=tuple(
args.policy_hidden),
min_std=args.min_std)
else:
raise NotImplementedError(env.spec.action_space)
if args.control == 'concurrent':
return env, policies
else:
return env, policy
def get_policy(env, algo_name, info, policy_hidden_sizes,
policy_hidden_nonlinearity, policy_output_nonlinearity,
recurrent, **kwargs):
policy = None
policy_class = None
hidden_sizes = get_hidden_sizes(policy_hidden_sizes)
hidden_nonlinearity = get_nonlinearity(policy_hidden_nonlinearity)
output_nonlinearity = get_nonlinearity(policy_output_nonlinearity)
if algo_name in [
'trpo',
'actrpo',
'acqftrpo',
'qprop',
'mqprop',
'qfqprop',
'trpg',
'trpgoff',
'nuqprop',
'nuqfqprop',
'nafqprop',
'vpg',
'qvpg',
'dspg',
'dspgoff',
]:
if not info['is_action_discrete']:
if recurrent:
policy = GaussianLSTMPolicy(
name="gauss_lstm_policy",
env_spec=env.spec,
lstm_layer_cls=L.TfBasicLSTMLayer,
# gru_layer_cls=L.GRULayer,
output_nonlinearity=output_nonlinearity, # None
)
policy_class = 'GaussianLSTMPolicy'
else:
policy = GaussianMLPPolicy(
name="gauss_policy",
env_spec=env.spec,
hidden_sizes=hidden_sizes,
hidden_nonlinearity=hidden_nonlinearity, # tf.nn.tanh
output_nonlinearity=output_nonlinearity, # None
)
policy_class = 'GaussianMLPPolicy'
else:
if recurrent:
policy = CategoricalLSTMPolicy(
name="cat_lstm_policy",
env_spec=env.spec,
lstm_layer_cls=L.TfBasicLSTMLayer,
# gru_layer_cls=L.GRULayer,
)
policy_class = 'CategoricalLSTMPolicy'
else:
policy = CategoricalMLPPolicy(
name="cat_policy",
env_spec=env.spec,
hidden_sizes=hidden_sizes,
hidden_nonlinearity=hidden_nonlinearity, # tf.nn.tanh
)
policy_class = 'CategoricalMLPPolicy'
elif algo_name in [
'ddpg',
]:
assert not info['is_action_discrete']
policy = DeterministicMLPPolicy(
name="det_policy",
env_spec=env.spec,
hidden_sizes=hidden_sizes,
hidden_nonlinearity=hidden_nonlinearity, # tf.nn.relu
output_nonlinearity=output_nonlinearity, # tf.nn.tanh
)
policy_class = 'DeterministicMLPPolicy'
print(
'[get_policy] Instantiating %s, with sizes=%s, hidden_nonlinearity=%s.'
% (policy_class, str(hidden_sizes), policy_hidden_nonlinearity))
print('[get_policy] output_nonlinearity=%s.' %
(policy_output_nonlinearity))
return policy
def main():
now = datetime.datetime.now(dateutil.tz.tzlocal())
rand_id = str(uuid.uuid4())[:5]
timestamp = now.strftime('%Y_%m_%d_%H_%M_%S_%f_%Z')
default_exp_name = 'experiment_%s_%s' % (timestamp, rand_id)
parser = argparse.ArgumentParser()
parser.add_argument('--exp_name',
type=str,
default=default_exp_name,
help='Name of the experiment.')
parser.add_argument('--discount', type=float, default=0.95)
parser.add_argument('--gae_lambda', type=float, default=0.99)
parser.add_argument('--reward_scale', type=float, default=1.0)
parser.add_argument('--enable_obsnorm', action='store_true', default=False)
parser.add_argument('--chunked', action='store_true', default=False)
parser.add_argument('--n_iter', type=int, default=250)
parser.add_argument('--sampler_workers', type=int, default=1)
parser.add_argument('--max_traj_len', type=int, default=250)
parser.add_argument('--update_curriculum',
action='store_true',
default=False)
parser.add_argument('--anneal_step_size', type=int, default=0)
parser.add_argument('--n_timesteps', type=int, default=8000)
parser.add_argument('--control', type=str, default='centralized')
parser.add_argument('--buffer_size', type=int, default=1)
parser.add_argument('--radius', type=float, default=0.015)
parser.add_argument('--n_evaders', type=int, default=10)
parser.add_argument('--n_pursuers', type=int, default=8)
parser.add_argument('--n_poison', type=int, default=10)
parser.add_argument('--n_coop', type=int, default=4)
parser.add_argument('--n_sensors', type=int, default=30)
parser.add_argument('--sensor_range', type=str, default='0.2')
parser.add_argument('--food_reward', type=float, default=5)
parser.add_argument('--poison_reward', type=float, default=-1)
parser.add_argument('--encounter_reward', type=float, default=0.05)
parser.add_argument('--reward_mech', type=str, default='local')
parser.add_argument('--recurrent', type=str, default=None)
parser.add_argument('--baseline_type', type=str, default='linear')
parser.add_argument('--policy_hidden_sizes', type=str, default='128,128')
parser.add_argument('--baseline_hidden_sizes', type=str, default='128,128')
parser.add_argument('--max_kl', type=float, default=0.01)
parser.add_argument('--log_dir', type=str, required=False)
parser.add_argument('--tabular_log_file',
type=str,
default='progress.csv',
help='Name of the tabular log file (in csv).')
parser.add_argument('--text_log_file',
type=str,
default='debug.log',
help='Name of the text log file (in pure text).')
parser.add_argument('--params_log_file',
type=str,
default='params.json',
help='Name of the parameter log file (in json).')
parser.add_argument('--seed', type=int, help='Random seed for numpy')
parser.add_argument('--args_data',
type=str,
help='Pickled data for stub objects')
parser.add_argument('--snapshot_mode',
type=str,
default='all',
help='Mode to save the snapshot. Can be either "all" '
'(all iterations will be saved), "last" (only '
'the last iteration will be saved), or "none" '
'(do not save snapshots)')
parser.add_argument(
'--log_tabular_only',
type=ast.literal_eval,
default=False,
help=
'Whether to only print the tabular log information (in a horizontal format)'
)
args = parser.parse_args()
parallel_sampler.initialize(n_parallel=args.sampler_workers)
if args.seed is not None:
set_seed(args.seed)
parallel_sampler.set_seed(args.seed)
args.hidden_sizes = tuple(map(int, args.policy_hidden_sizes.split(',')))
centralized = True if args.control == 'centralized' else False
sensor_range = np.array(map(float, args.sensor_range.split(',')))
if len(sensor_range) == 1:
sensor_range = sensor_range[0]
else:
assert sensor_range.shape == (args.n_pursuers, )
env = MAWaterWorld(args.n_pursuers,
args.n_evaders,
args.n_coop,
args.n_poison,
radius=args.radius,
n_sensors=args.n_sensors,
food_reward=args.food_reward,
poison_reward=args.poison_reward,
encounter_reward=args.encounter_reward,
reward_mech=args.reward_mech,
sensor_range=sensor_range,
obstacle_loc=None)
env = TfEnv(
RLLabEnv(StandardizedEnv(env,
scale_reward=args.reward_scale,
enable_obsnorm=args.enable_obsnorm),
mode=args.control))
if args.buffer_size > 1:
env = ObservationBuffer(env, args.buffer_size)
if args.recurrent:
feature_network = MLP(
name='feature_net',
input_shape=(env.spec.observation_space.flat_dim +
env.spec.action_space.flat_dim, ),
output_dim=16,
hidden_sizes=(128, 64, 32),
hidden_nonlinearity=tf.nn.tanh,
output_nonlinearity=None)
if args.recurrent == 'gru':
policy = GaussianGRUPolicy(env_spec=env.spec,
feature_network=feature_network,
hidden_dim=int(
args.policy_hidden_sizes),
name='policy')
elif args.recurrent == 'lstm':
policy = GaussianLSTMPolicy(env_spec=env.spec,
feature_network=feature_network,
hidden_dim=int(
args.policy_hidden_sizes),
name='policy')
else:
policy = GaussianMLPPolicy(
name='policy',
env_spec=env.spec,
hidden_sizes=tuple(map(int, args.policy_hidden_sizes.split(','))),
min_std=10e-5)
if args.baseline_type == 'linear':
baseline = LinearFeatureBaseline(env_spec=env.spec)
elif args.baseline_type == 'mlp':
raise NotImplementedError()
# baseline = GaussianMLPBaseline(
# env_spec=env.spec, hidden_sizes=tuple(map(int, args.baseline_hidden_sizes.split(','))))
else:
baseline = ZeroBaseline(env_spec=env.spec)
# logger
default_log_dir = config.LOG_DIR
if args.log_dir is None:
log_dir = osp.join(default_log_dir, args.exp_name)
else:
log_dir = args.log_dir
tabular_log_file = osp.join(log_dir, args.tabular_log_file)
text_log_file = osp.join(log_dir, args.text_log_file)
params_log_file = osp.join(log_dir, args.params_log_file)
logger.log_parameters_lite(params_log_file, args)
logger.add_text_output(text_log_file)
logger.add_tabular_output(tabular_log_file)
prev_snapshot_dir = logger.get_snapshot_dir()
prev_mode = logger.get_snapshot_mode()
logger.set_snapshot_dir(log_dir)
logger.set_snapshot_mode(args.snapshot_mode)
logger.set_log_tabular_only(args.log_tabular_only)
logger.push_prefix("[%s] " % args.exp_name)
algo = TRPO(
env=env,
policy=policy,
baseline=baseline,
batch_size=args.n_timesteps,
max_path_length=args.max_traj_len,
#max_path_length_limit=args.max_path_length_limit,
update_max_path_length=args.update_curriculum,
anneal_step_size=args.anneal_step_size,
n_itr=args.n_iter,
discount=args.discount,
gae_lambda=args.gae_lambda,
step_size=args.max_kl,
optimizer=ConjugateGradientOptimizer(hvp_approach=FiniteDifferenceHvp(
base_eps=1e-5)) if args.recurrent else None,
mode=args.control
if not args.chunked else 'chunk_{}'.format(args.control),
)
algo.train()
def parse_env_args(self, env, args):
if isinstance(args, dict):
args = to_named_tuple(args)
# Multi-agent wrapper
env = RLLabEnv(env, ma_mode=args.control)
env = MATfEnv(env)
# Policy
if args.recurrent:
if args.feature_net:
feature_network = MLP(
name='feature_net',
input_shape=(env.spec.observation_space.flat_dim +
env.spec.action_space.flat_dim, ),
output_dim=args.feature_output,
hidden_sizes=tuple(args.feature_hidden),
hidden_nonlinearity=tf.nn.tanh,
output_nonlinearity=None)
elif args.conv:
strides = tuple(args.conv_strides)
chans = tuple(args.conv_channels)
filts = tuple(args.conv_filters)
assert len(strides) == len(chans) == len(
filts), "strides, chans and filts not equal"
# only discrete actions supported, should be straightforward to extend to continuous
assert isinstance(
env.spec.action_space,
Discrete), "Only discrete action spaces support conv"
feature_network = ConvNetwork(
name='feature_net',
input_shape=env.spec.observation_space.shape,
output_dim=args.feature_output,
conv_filters=chans,
conv_filter_sizes=filts,
conv_strides=strides,
conv_pads=('VALID', ) * len(chans),
hidden_sizes=tuple(args.feature_hidden),
hidden_nonlinearity=tf.nn.relu,
output_nonlinearity=None)
else:
feature_network = None
if args.recurrent == 'gru':
if isinstance(env.spec.action_space, Box):
if args.control == 'concurrent':
policies = [
GaussianGRUPolicy(env_spec=env.spec,
feature_network=feature_network,
hidden_dim=int(
args.policy_hidden[0]),
name='policy_{}'.format(agid))
for agid in range(len(env.agents))
]
policy = GaussianGRUPolicy(env_spec=env.spec,
feature_network=feature_network,
hidden_dim=int(
args.policy_hidden[0]),
name='policy')
elif isinstance(env.spec.action_space, Discrete):
if args.control == 'concurrent':
policies = [
CategoricalGRUPolicy(
env_spec=env.spec,
feature_network=feature_network,
hidden_dim=int(args.policy_hidden[0]),
name='policy_{}'.format(agid),
state_include_action=False
if args.conv else True)
for agid in range(len(env.agents))
]
q_network = CategoricalGRUPolicy(
env_spec=env.spec,
feature_network=feature_network,
hidden_dim=int(args.policy_hidden[0]),
name='q_network',
state_include_action=False if args.conv else True)
target_q_network = CategoricalGRUPolicy(
env_spec=env.spec,
feature_network=feature_network,
hidden_dim=int(args.policy_hidden[0]),
name='target_q_network',
state_include_action=False if args.conv else True)
policy = {
'q_network': q_network,
'target_q_network': target_q_network
}
else:
raise NotImplementedError(env.spec.observation_space)
elif args.recurrent == 'lstm':
if isinstance(env.spec.action_space, Box):
if args.control == 'concurrent':
policies = [
GaussianLSTMPolicy(env_spec=env.spec,
feature_network=feature_network,
hidden_dim=int(
args.policy_hidden),
name='policy_{}'.format(agid))
for agid in range(len(env.agents))
]
policy = GaussianLSTMPolicy(
env_spec=env.spec,
feature_network=feature_network,
hidden_dim=int(args.policy_hidden),
name='policy')
elif isinstance(env.spec.action_space, Discrete):
if args.control == 'concurrent':
policies = [
CategoricalLSTMPolicy(
env_spec=env.spec,
feature_network=feature_network,
hidden_dim=int(args.policy_hidden),
name='policy_{}'.format(agid))
for agid in range(len(env.agents))
]
q_network = CategoricalLSTMPolicy(
env_spec=env.spec,
feature_network=feature_network,
hidden_dim=int(args.policy_hidden),
name='q_network')
target_q_network = CategoricalLSTMPolicy(
env_spec=env.spec,
feature_network=feature_network,
hidden_dim=int(args.policy_hidden),
name='target_q_network')
policy = {
'q_network': q_network,
'target_q_network': target_q_network
}
else:
raise NotImplementedError(env.spec.action_space)
else:
raise NotImplementedError(args.recurrent)
elif args.conv:
strides = tuple(args.conv_strides)
chans = tuple(args.conv_channels)
filts = tuple(args.conv_filters)
assert len(strides) == len(chans) == len(
filts), "strides, chans and filts not equal"
# only discrete actions supported, should be straightforward to extend to continuous
assert isinstance(
env.spec.action_space,
Discrete), "Only discrete action spaces support conv"
feature_network = ConvNetwork(
name='feature_net',
input_shape=env.spec.observation_space.shape,
output_dim=env.spec.action_space.n,
conv_filters=chans,
conv_filter_sizes=filts,
conv_strides=strides,
conv_pads=(args.conv_pads, ) * len(chans),
hidden_sizes=tuple(args.policy_hidden),
hidden_nonlinearity=tf.nn.relu,
output_nonlinearity=tf.nn.softmax,
batch_normalization=args.batch_normalization)
if args.algo == 'dqn':
q_network = CategoricalMLPPolicy(name='q_network',
env_spec=env.spec,
prob_network=feature_network)
target_q_network = CategoricalMLPPolicy(
name='target_q_network',
env_spec=env.spec,
prob_network=feature_network)
policy = {
'q_network': q_network,
'target_q_network': target_q_network
}
else:
policy = CategoricalMLPPolicy(name='policy',
env_spec=env.spec,
prob_network=feature_network)
else:
if env.spec is None:
networks = [
DQNNetwork(i,
env,
target_network_update_freq=self.args.
target_network_update,
discount_factor=self.args.discount,
batch_size=self.args.batch_size,
learning_rate=self.args.qfunc_lr)
for i in range(env.n)
]
policy = networks
elif isinstance(env.spec.action_space, Box):
policy = GaussianMLPPolicy(env_spec=env.spec,
hidden_sizes=tuple(
args.policy_hidden),
min_std=args.min_std,
name='policy')
elif isinstance(env.spec.action_space, Discrete):
policy = CategoricalMLPPolicy(env_spec=env.spec,
hidden_sizes=tuple(
args.policy_hidden),
name='policy')
else:
raise NotImplementedError(env.spec.action_space)
return env, policy
def run_task(vv, log_dir=None, exp_name=None):
global policy
global baseline
policy = None
baseline = None
trpo_stepsize = 0.01
trpo_subsample_factor = 0.2
# Check if variant is available
if vv['model_type'] not in ['BrushTireModel', 'LinearTireModel']:
raise ValueError('Unrecognized model type for simulating robot')
if vv['robot_type'] not in ['MRZR', 'RCCar']:
raise ValueError('Unrecognized robot type')
# Load environment
if not vv['use_ros']:
env = StraightEnv(
target_velocity=vv['target_velocity'],
dt=vv['dt'],
model_type=vv['model_type'],
robot_type=vv['robot_type'],
mu_s=vv['mu_s'],
mu_k=vv['mu_k']
)
env=TfEnv(env)
else:
from aa_simulation.envs.straight.straight_env_ros import StraightEnvROS
env = StraightEnvROS(
target_velocity=vv['target_velocity'],
dt=vv['dt'],
model_type=vv['model_type'],
robot_type=vv['robot_type']
)
# Save variant information for comparison plots
# variant_file = logger.get_snapshot_dir() + '/variant.json'
# logger.log_variant(variant_file, vv)
# Set variance for each action component separately for exploration
# Note: We set the variance manually because we are not scaling our
# action space during training.
init_std_speed = vv['target_velocity'] / 4
init_std_steer = np.pi / 6
init_std = [init_std_speed, init_std_steer]
# Build policy and baseline networks
# Note: Mean of policy network set to analytically computed values for
# faster training (rough estimates for RL to fine-tune).
if policy is None or baseline is None:
target_velocity = vv['target_velocity']
target_steering = 0
output_mean = np.array([target_velocity, target_steering])
hidden_sizes = (32, 32)
# In mean network, allow output b values to dominate final output
# value by constraining the magnitude of the output W matrix. This is
# to allow faster learning. These numbers are arbitrarily chosen.
W_gain = min(vv['target_velocity'] / 5, np.pi / 15)
policy = GaussianLSTMPolicy(
name="policy",
env_spec=env.spec,
# input_shape=(env.spec.observation_space.flat_dim,),
# output_dim=env.spec.action_space.flat_dim,
# gru_layer_cls=L.GRULayer,
)
# mean_network = MLP(
# input_shape=(env.spec.observation_space.flat_dim,),
# output_dim=env.spec.action_space.flat_dim,
# hidden_sizes=hidden_sizes,
# hidden_nonlinearity=LN.rectify,
# output_nonlinearity=None,
# output_W_init=LI.GlorotUniform(gain=W_gain),
# output_b_init=output_mean
# )
# policy = GaussianMLPPolicy(
# env_spec=env.spec,
# hidden_sizes=(32, 32),
# init_std=init_std,
# mean_network=mean_network
# )
baseline = LinearFeatureBaseline(
env_spec=env.spec,
target_key='returns'
)
# Reset variance to re-enable exploration when using pre-trained networks
else:
policy._l_log_std = ParamLayer(
policy._mean_network.input_layer,
num_units=env.spec.action_space.flat_dim,
param=LI.Constant(np.log(init_std)),
name='output_log_std',
trainable=True
)
obs_var = policy._mean_network.input_layer.input_var
mean_var, log_std_var = L.get_output([policy._l_mean, policy._l_log_std])
policy._log_std_var = log_std_var
LasagnePowered.__init__(policy, [policy._l_mean, policy._l_log_std])
policy._f_dist = ext.compile_function(
inputs=[obs_var],
outputs=[mean_var, log_std_var]
)
safety_baseline = LinearFeatureBaseline(
env_spec=env.spec,
target_key='safety_returns'
)
safety_constraint = StraightSafetyConstraint(
max_value=1.0,
baseline=safety_baseline
)
if vv['algo'] == 'TRPO':
algo = Trpo(
env=env,
policy=policy,
baseline=baseline,
batch_size=600,
max_path_length=env.horizon,
n_itr=2000,
discount=0.99,
step_size=trpo_stepsize,
plot=False,
optimizer=ConjugateGradientOptimizer(hvp_approach=FiniteDifferenceHvp(base_eps=1e-5)),
)
else:
algo = CPO(
env=env,
policy=policy,
baseline=baseline,
safety_constraint=safety_constraint,
batch_size=600,
max_path_length=env.horizon,
n_itr=2000,
discount=0.99,
step_size=trpo_stepsize,
gae_lambda=0.95,
safety_gae_lambda=1,
optimizer_args={'subsample_factor': trpo_subsample_factor},
plot=False
)
algo.train()
for seed in seeds:
mdp = TfEnv(normalize(env=GymEnv('Box3dReach-v10',record_video=False, \
log_dir='/tmp/gym_test',record_log=False)))
# policy = GaussianMLPPolicy(
# "mlp_policy",
# env_spec=mdp.spec,
# hidden_sizes=(64, 32),
# output_nonlinearity=tf.nn.tanh,
# clip_action=True,
# )
policy = GaussianLSTMPolicy(
"lstm_policy",
env_spec=mdp.spec,
hidden_dim=64,
output_nonlinearity=tf.tanh,
)
baseline = LinearFeatureBaseline(mdp.spec, )
batch_size = 5000
algo = TRPO(
env=mdp,
policy=policy,
baseline=baseline,
batch_size=batch_size,
whole_paths=True,
max_path_length=500,
n_itr=2000,
step_size=0.01,