def net_for_problem(self,
problem_ppddl_paths,
prob_name=None,
dropout=0.0):
"""Instantiate a network and environment for a specific problem. Also
creates a handler for the problem (maybe this is a bad idea? IDK)."""
all_ppddl_paths = self.extra_ppddl + list(problem_ppddl_paths)
if prob_name is None:
# get it automatically (and slowly…)
prob_name = get_problem_names(all_ppddl_paths)[0]
service_config = ProblemServiceConfig(
all_ppddl_paths,
prob_name,
heuristic_name=self.heuristic_data_gen_name,
# doesn't matter, use a cheap one
teacher_heur='h-add',
use_lm_cuts=self.use_lm_cuts)
problem_server = ProblemServer(service_config)
single_problem = SingleProblem(prob_name, problem_server)
planner_exts = PlannerExtensions(service_config.pddl_files,
service_config.init_problem_name)
raw_env, flat_env = create_environment(
single_problem.prob_meta,
planner_exts,
heuristic_name=self.heuristic_data_gen_name)
problem_network = PropNetwork(self.weight_manager,
single_problem.prob_meta,
dropout=dropout,
norm_response=self.norm_response)
# env_spec is SingleProblem.env_spec
policy = CategoricalMLPPolicy(env_spec=single_problem.env_spec,
prob_network=problem_network,
name='policy')
# returns policy, RLLab environment, problem server handles
return InstantiatedNetwork(self, policy, flat_env, single_problem)
def train_gail(session,
env,
dataset,
obs_dim=1,
act_dim=2,
n_itr=20,
use_env_rewards=False,
discount=.99,
batch_size=4000,
critic_scale=1.,
gail_step_size=.01,
critic_learning_rate=.001,
policy_hid_layer_dims=[32, 32],
gradient_penalty=.1,
critic_n_train_epochs=1,
sampler_args=dict(),
return_algo=False):
network = CriticNetwork(hidden_layer_dims=[32, 32])
critic = WassersteinCritic(obs_dim=obs_dim,
act_dim=act_dim,
dataset=dataset,
network=network,
verbose=2,
gradient_penalty=gradient_penalty,
optimizer=tf.train.AdamOptimizer(
critic_learning_rate, beta1=.5, beta2=.9),
n_train_epochs=critic_n_train_epochs)
policy = CategoricalMLPPolicy(name="policy",
env_spec=env.spec,
hidden_sizes=policy_hid_layer_dims)
baseline = LinearFeatureBaseline(env_spec=env.spec)
reward_handler = RewardHandler(use_env_rewards=use_env_rewards,
critic_final_scale=critic_scale)
algo = GAIL(critic=critic,
reward_handler=reward_handler,
env=env,
policy=policy,
baseline=baseline,
batch_size=batch_size,
max_path_length=200,
n_itr=n_itr,
discount=discount,
step_size=gail_step_size,
sampler_args=sampler_args)
session.run(tf.global_variables_initializer())
if return_algo:
return algo
algo.train(sess=session)
return policy, critic
def main(
log_dir,
env_name,
ent_coef,
n_steps,
total_timesteps,
num_vec,
):
tf.reset_default_graph()
# n_steps is the `batch_size // num_vec` in `imitation`.
batch_size = n_steps * num_vec
n_itr = int(math.ceil(total_timesteps / batch_size))
if env_name.startswith("airl/"):
env_cls = CustomGymEnv
else:
env_cls = GymEnv
env = TfEnv(env_cls(env_name, record_video=False, record_log=False))
# NOTE: Haven't yet checked if hidden_sizes=(32, 32) matches the settings in
# the `imitation` repo. We use the default Stable Baselines MLP policy.
if isinstance(env.spec.action_space, Box):
policy = GaussianMLPPolicy(name='policy',
env_spec=env.spec,
hidden_sizes=(32, 32))
else:
policy = CategoricalMLPPolicy(name='policy',
env_spec=env.spec,
hidden_sizes=(32, 32))
with tf.Session(config=get_session_config()) as sess:
algo = TRPO(
env=env,
policy=policy,
n_itr=n_itr,
batch_size=batch_size,
max_path_length=500,
discount=0.99,
store_paths=True,
entropy_weight=ent_coef,
baseline=LinearFeatureBaseline(env_spec=env.spec),
# Maybe it will be the case the not every policy is compatible with
# the VectorizedSampler. In that case, consider changing to
# `sampler_cls=None` and adding a dummy `n_envs` kwargs to BatchSampler.
sampler_cls=VectorizedSampler,
sampler_args=dict(n_envs=num_vec),
)
with rllab_logdir(algo=algo, dirname=log_dir):
algo.train(sess)
def run_experiment(**params):
base_params = copy.copy(DEFAULTS)
base_params.update(params)
params = base_params
grid_world = SlaveGridWorldEnv("3x3", goal_reward=params["goal_reward"])
env = normalize(grid_world)
baseline = LinearFeatureBaseline(env)
policy = CategoricalMLPPolicy(
name="policy",
env_spec=env.spec,
hidden_sizes=params["policy_hidden_dims"],
)
optimizer = ConjugateGradientOptimizer(hvp_approach=FiniteDifferenceHvp(
base_eps=1e-5))
algo = TRPO(
env=env,
policy=policy,
baseline=baseline,
batch_size=params["batch_size"],
max_path_length=5,
n_itr=params["n_itr"],
discount=0.99,
step_size=params["step_size"],
optimizer=optimizer,
)
run_experiment_lite(
algo.train(),
n_parallel=5,
snapshot_mode="last",
exp_prefix="grid_world_silent",
variant=params,
)
def test_saver(self):
savepath = 'data/model'
env = TfEnv(TwoRoundNondeterministicRewardEnv())
with tf.Session() as session:
policy = CategoricalMLPPolicy(name="policy",
env_spec=env.spec,
hidden_sizes=[2, 2])
session.run(tf.global_variables_initializer())
saver = tf.train.Saver(max_to_keep=100,
keep_checkpoint_every_n_hours=.5)
saver.save(session, savepath, global_step=0)
params = policy.get_params()
initial_values = policy.get_param_values()
assign = tf.group(
*[tf.assign(p, tf.zeros_like(p)) for p in params])
session.run(assign)
self.assertEqual(np.sum(policy.get_param_values()), 0)
latest = tf.train.latest_checkpoint('data')
saver.restore(session, latest)
final_values = policy.get_param_values()
np.testing.assert_array_equal(initial_values, final_values)
def rllab_envpolicy_parser(env, args):
if isinstance(args, dict):
args = tonamedtuple(args)
env = RLLabEnv(env, mode=args.control)
if args.algo[:2] == 'tf':
env = TfEnv(env)
# Policy
if args.recurrent:
if args.feature_net:
feature_network = MLP(
name='feature_net',
input_shape=(env.spec.observation_space.flat_dim +
env.spec.action_space.flat_dim, ),
output_dim=args.feature_output,
hidden_sizes=tuple(args.feature_hidden),
hidden_nonlinearity=tf.nn.tanh,
output_nonlinearity=None)
elif args.conv:
strides = tuple(args.conv_strides)
chans = tuple(args.conv_channels)
filts = tuple(args.conv_filters)
assert len(strides) == len(chans) == len(
filts), "strides, chans and filts not equal"
# only discrete actions supported, should be straightforward to extend to continuous
assert isinstance(
env.spec.action_space,
Discrete), "Only discrete action spaces support conv"
feature_network = ConvNetwork(
name='feature_net',
input_shape=env.spec.observation_space.shape,
output_dim=args.feature_output,
conv_filters=chans,
conv_filter_sizes=filts,
conv_strides=strides,
conv_pads=('VALID', ) * len(chans),
hidden_sizes=tuple(args.feature_hidden),
hidden_nonlinearity=tf.nn.relu,
output_nonlinearity=None)
else:
feature_network = None
if args.recurrent == 'gru':
if isinstance(env.spec.action_space, Box):
policy = GaussianGRUPolicy(env_spec=env.spec,
feature_network=feature_network,
hidden_dim=int(
args.policy_hidden[0]),
name='policy')
elif isinstance(env.spec.action_space, Discrete):
policy = CategoricalGRUPolicy(
env_spec=env.spec,
feature_network=feature_network,
hidden_dim=int(args.policy_hidden[0]),
name='policy',
state_include_action=False if args.conv else True)
else:
raise NotImplementedError(env.spec.observation_space)
elif args.recurrent == 'lstm':
if isinstance(env.spec.action_space, Box):
policy = GaussianLSTMPolicy(
env_spec=env.spec,
feature_network=feature_network,
hidden_dim=int(args.policy_hidden),
name='policy')
elif isinstance(env.spec.action_space, Discrete):
policy = CategoricalLSTMPolicy(
env_spec=env.spec,
feature_network=feature_network,
hidden_dim=int(args.policy_hidden),
name='policy')
else:
raise NotImplementedError(env.spec.action_space)
else:
raise NotImplementedError(args.recurrent)
elif args.conv:
strides = tuple(args.conv_strides)
chans = tuple(args.conv_channels)
filts = tuple(args.conv_filters)
assert len(strides) == len(chans) == len(
filts), "strides, chans and filts not equal"
# only discrete actions supported, should be straightforward to extend to continuous
assert isinstance(
env.spec.action_space,
Discrete), "Only discrete action spaces support conv"
feature_network = ConvNetwork(
name='feature_net',
input_shape=env.spec.observation_space.shape,
output_dim=env.spec.action_space.n,
conv_filters=chans,
conv_filter_sizes=filts,
conv_strides=strides,
conv_pads=('VALID', ) * len(chans),
hidden_sizes=tuple(args.policy_hidden),
hidden_nonlinearity=tf.nn.relu,
output_nonlinearity=tf.nn.softmax)
policy = CategoricalMLPPolicy(name='policy',
env_spec=env.spec,
prob_network=feature_network)
else:
if isinstance(env.spec.action_space, Box):
policy = GaussianMLPPolicy(env_spec=env.spec,
hidden_sizes=tuple(
args.policy_hidden),
min_std=args.min_std,
name='policy')
elif isinstance(env.spec.action_space, Discrete):
policy = CategoricalMLPPolicy(env_spec=env.spec,
hidden_sizes=tuple(
args.policy_hidden),
name='policy')
else:
raise NotImplementedError(env.spec.action_space)
elif args.algo[:2] == 'th':
# Policy
if args.recurrent:
if args.feature_net:
feature_network = thMLP(
input_shape=(env.spec.observation_space.flat_dim +
env.spec.action_space.flat_dim, ),
output_dim=args.feature_output,
hidden_sizes=tuple(args.feature_hidden),
hidden_nonlinearity=tf.nn.tanh,
output_nonlinearity=None)
else:
feature_network = None
if args.recurrent == 'gru':
if isinstance(env.spec.observation_space, thBox):
policy = thGaussianGRUPolicy(
env_spec=env.spec,
feature_network=feature_network,
hidden_dim=int(args.policy_hidden[0]),
)
elif isinstance(env.spec.observation_space, thDiscrete):
policy = thCategoricalGRUPolicy(
env_spec=env.spec,
feature_network=feature_network,
hidden_dim=int(args.policy_hidden[0]),
)
else:
raise NotImplementedError(env.spec.observation_space)
# elif args.recurrent == 'lstm':
# if isinstance(env.spec.action_space, thBox):
# policy = thGaussianLSTMPolicy(env_spec=env.spec,
# feature_network=feature_network,
# hidden_dim=int(args.policy_hidden),
# name='policy')
# elif isinstance(env.spec.action_space, thDiscrete):
# policy = thCategoricalLSTMPolicy(env_spec=env.spec,
# feature_network=feature_network,
# hidden_dim=int(args.policy_hidden),
# name='policy')
# else:
# raise NotImplementedError(env.spec.action_space)
else:
raise NotImplementedError(args.recurrent)
else:
if args.algo == 'thddpg':
assert isinstance(env.spec.action_space, thBox)
policy = thDeterministicMLPPolicy(
env_spec=env.spec,
hidden_sizes=tuple(args.policy_hidden),
)
else:
if isinstance(env.spec.action_space, thBox):
policy = thGaussianMLPPolicy(env_spec=env.spec,
hidden_sizes=tuple(
args.policy_hidden),
min_std=args.min_std)
elif isinstance(env.spec.action_space, thDiscrete):
policy = thCategoricalMLPPolicy(env_spec=env.spec,
hidden_sizes=tuple(
args.policy_hidden),
min_std=args.min_std)
else:
raise NotImplementedError(env.spec.action_space)
if args.control == 'concurrent':
return env, policies
else:
return env, policy
def make_policy(args,
env_spec,
dom_meta,
prob_meta,
dg_extra_dim=None,
weight_manager=None):
# size of input and output
obs_dim = int(env_spec.observation_space.flat_dim)
act_dim = int(env_spec.action_space.flat_dim)
# can make normal FC MLP or an action/proposition network
known_mo = set() # type: Set[str]
if args.model == 'simple':
known_mo = {'hidden_size', 'num_layers'}
mod = args.model_opts
hidden_size = int(mod.get('hidden_size', 32))
num_layers = int(mod.get('num_layers', 2))
print('Layer size: %d' % hidden_size)
print('Number of layers: %d' % num_layers)
# The dense policy network should have two hidden layers, each with
# <obs dim> units (thereabouts, anyway)
custom_network = make_masked_mlp('simple_masked_mlp', obs_dim, act_dim,
(hidden_size, ) * num_layers)
elif args.model == 'actprop':
known_mo = {'hidden_size', 'num_layers', 'dropout', 'norm_response'}
mod = args.model_opts
hs = int(mod.get('hidden_size', 8))
num_layers = int(mod.get('num_layers', 2))
dropout = float(mod.get('dropout', 0.0))
norm_response = int(mod.get('norm_response', '0')) != 0
print(
'hidden_size: %d, num_layers: %d, dropout: %f, norm_response: %d' %
(hs, num_layers, dropout, int(norm_response)))
if weight_manager is not None:
print('Re-using same weight manager')
elif args.resume_from:
print('Reloading weight manager (resuming training)')
weight_manager = joblib.load(args.resume_from)
else:
print('Creating new weight manager (not resuming)')
# TODO: should save all network metadata, including heuristic
# configuration
# extra_dim = sum([g.extra_dim for g in data_gens])
weight_manager = PropNetworkWeights(
dom_meta,
hidden_sizes=[(hs, hs)] * num_layers,
# extra inputs to each action module from data generators
extra_dim=dg_extra_dim)
custom_network = PropNetwork(weight_manager,
prob_meta,
dropout=dropout,
norm_response=norm_response)
else:
raise ValueError('Unknown network type "%s"' % args.model)
# What if a model option wasn't used?
unknown_mo = args.model_opts.keys() - known_mo
if unknown_mo:
print('WARNING: model options not understood by "%s" network: %s' %
(args.model, ', '.join(unknown_mo)),
file=sys.stderr)
policy = CategoricalMLPPolicy(env_spec=env_spec,
prob_network=custom_network,
name='policy')
# weight_manager will sometimes be None
return policy, weight_manager
def fu_irl(
venv,
is_airl,
expert=None,
expert_venv=None,
expert_trajectories=None,
total_timesteps=10000,
gen_batch_size=200,
policy_lr=1e-3,
callback=None,
**kwargs,
):
# Disable algorithm's internal prints
old_stdout = sys.stdout
sys.stdout = open(os.devnull, 'w')
raw_env = get_raw_env(venv)
tf_env = TfEnv(GymEnv(env=raw_env, record_video=False, record_log=False))
if expert_trajectories is None:
expert_trajectories = sample_trajectories(
expert_venv, expert, n_episodes=total_timesteps
)
expert_trajectories = to_rllab_trajectories(expert_trajectories, venv)
if is_airl:
irl_model = AIRLStateAction(
env_spec=tf_env.spec, expert_trajs=expert_trajectories
)
entropy_weight = 1.0
else:
irl_model = GAIL(env_spec=tf_env.spec, expert_trajs=expert_trajectories)
entropy_weight = 0.0
if isinstance(venv.action_space, Discrete):
policy = CategoricalMLPPolicy(
name="policy", env_spec=tf_env.spec, hidden_sizes=(32, 32)
)
else:
policy = GaussianMLPPolicy(
name="policy", env_spec=tf_env.spec, hidden_sizes=(32, 32)
)
num_epochs = int(total_timesteps // gen_batch_size)
algo = IRLTRPO(
env=tf_env,
policy=policy,
irl_model=irl_model,
n_itr=num_epochs,
batch_size=gen_batch_size,
max_path_length=100,
discount=0.99,
discrim_train_itrs=50,
irl_model_wt=1.0,
entropy_weight=entropy_weight,
zero_environment_reward=True,
baseline=LinearFeatureBaseline(env_spec=tf_env.spec),
)
algo.train()
sys.stdout = old_stdout
def predict_fn(ob, state=None, deterministic=False):
act, _ = algo.policy.get_action(ob)
return act, state
results = {}
results["policy"] = LightweightRLModel(predict_fn=predict_fn, env=venv)
return results
# if args.env not in supported_envs:
# raise Exception("Env not supported! Try it out though?")
# Need to wrap in a tf environment and force_reset to true
# see https://github.com/openai/rllab/issues/87#issuecomment-282519288
register_custom_envs()
gymenv = GymEnv(args.env, force_reset=True)
# gymenv.env.seed(124)
env = TfEnv(normalize(gymenv, normalize_obs=False))
if type(env.spec.action_space) is Discrete:
policy = CategoricalMLPPolicy(
name="policy",
env_spec=env.spec,
# The neural network policy should have two hidden layers, each with 32 hidden units.
hidden_sizes=(32, 32)
)
else:
policy = GaussianMLPPolicy(
name="policy",
env_spec=env.spec,
hidden_sizes=(100, 50, 25)
)
baseline = LinearFeatureBaseline(env_spec=env.spec)
iters = args.num_iters
algo = TRPO(
env=env,
def get_policy(env):
policy_network = get_policy_network(env)
policy = CategoricalMLPPolicy(name='policy',
env_spec=env.spec,
prob_network=policy_network)
return policy
def main(_):
env = TfEnv(
AtariEnv(args.env,
force_reset=True,
record_video=False,
record_log=False,
resize_size=args.resize_size,
atari_noop=args.atari_noop,
atari_eplife=args.atari_eplife,
atari_firereset=args.atari_firereset))
policy_network = ConvNetwork(
name='prob_network',
input_shape=env.observation_space.shape,
output_dim=env.action_space.n,
# number of channels/filters for each conv layer
conv_filters=(16, 32),
# filter size
conv_filter_sizes=(8, 4),
conv_strides=(4, 2),
conv_pads=('VALID', 'VALID'),
hidden_sizes=(256, ),
hidden_nonlinearity=tf.nn.relu,
output_nonlinearity=tf.nn.softmax,
batch_normalization=False)
policy = CategoricalMLPPolicy(name='policy',
env_spec=env.spec,
prob_network=policy_network)
if (args.value_function == 'zero'):
baseline = ZeroBaseline(env.spec)
else:
value_network = get_value_network(env)
baseline_batch_size = args.batch_size * 10
if (args.value_function == 'conj'):
baseline_optimizer = ConjugateGradientOptimizer(
subsample_factor=1.0, num_slices=args.num_slices)
elif (args.value_function == 'adam'):
baseline_optimizer = FirstOrderOptimizer(
max_epochs=3,
batch_size=512,
num_slices=args.num_slices,
verbose=True)
else:
logger.log("Inappropirate value function")
exit(0)
'''
baseline = GaussianMLPBaseline(
env.spec,
num_slices=args.num_slices,
regressor_args=dict(
step_size=0.01,
mean_network=value_network,
optimizer=baseline_optimizer,
subsample_factor=1.0,
batchsize=baseline_batch_size,
use_trust_region=False
)
)
'''
baseline = DeterministicMLPBaseline(env.spec,
num_slices=args.num_slices,
regressor_args=dict(
network=value_network,
optimizer=baseline_optimizer,
normalize_inputs=False))
algo = TRPO(env=env,
policy=policy,
baseline=baseline,
batch_size=args.batch_size,
max_path_length=4500,
n_itr=args.n_itr,
discount=args.discount_factor,
step_size=args.step_size,
clip_reward=(not args.reward_no_scale),
optimizer_args={
"subsample_factor": 1.0,
"num_slices": args.num_slices
}
# plot=True
)
config = tf.ConfigProto(allow_soft_placement=True,
intra_op_parallelism_threads=args.n_cpu,
inter_op_parallelism_threads=args.n_cpu)
config.gpu_options.allow_growth = True # pylint: disable=E1101
sess = tf.Session(config=config)
sess.__enter__()
algo.train(sess)
def main():
now = datetime.datetime.now(dateutil.tz.tzlocal())
rand_id = str(uuid.uuid4())[:5]
timestamp = now.strftime('%Y_%m_%d_%H_%M_%S_%f_%Z')
default_exp_name = 'experiment_%s_%s' % (timestamp, rand_id)
parser = argparse.ArgumentParser()
parser.add_argument('--exp_name',
type=str,
default=default_exp_name,
help='Name of the experiment.')
parser.add_argument('--discount', type=float, default=0.99)
parser.add_argument('--gae_lambda', type=float, default=1.0)
parser.add_argument('--reward_scale', type=float, default=1.0)
parser.add_argument('--n_iter', type=int, default=250)
parser.add_argument('--sampler_workers', type=int, default=1)
parser.add_argument('--max_traj_len', type=int, default=250)
parser.add_argument('--update_curriculum',
action='store_true',
default=False)
parser.add_argument('--n_timesteps', type=int, default=8000)
parser.add_argument('--control', type=str, default='centralized')
parser.add_argument('--rectangle', type=str, default='10,10')
parser.add_argument('--map_type', type=str, default='rectangle')
parser.add_argument('--n_evaders', type=int, default=5)
parser.add_argument('--n_pursuers', type=int, default=2)
parser.add_argument('--obs_range', type=int, default=3)
parser.add_argument('--n_catch', type=int, default=2)
parser.add_argument('--urgency', type=float, default=0.0)
parser.add_argument('--pursuit', dest='train_pursuit', action='store_true')
parser.add_argument('--evade', dest='train_pursuit', action='store_false')
parser.set_defaults(train_pursuit=True)
parser.add_argument('--surround', action='store_true', default=False)
parser.add_argument('--constraint_window', type=float, default=1.0)
parser.add_argument('--sample_maps', action='store_true', default=False)
parser.add_argument('--map_file', type=str, default='../maps/map_pool.npy')
parser.add_argument('--flatten', action='store_true', default=False)
parser.add_argument('--reward_mech', type=str, default='global')
parser.add_argument('--catchr', type=float, default=0.1)
parser.add_argument('--term_pursuit', type=float, default=5.0)
parser.add_argument('--recurrent', type=str, default=None)
parser.add_argument('--policy_hidden_sizes', type=str, default='128,128')
parser.add_argument('--baselin_hidden_sizes', type=str, default='128,128')
parser.add_argument('--baseline_type', type=str, default='linear')
parser.add_argument('--conv', action='store_true', default=False)
parser.add_argument('--max_kl', type=float, default=0.01)
parser.add_argument('--checkpoint', type=str, default=None)
parser.add_argument('--log_dir', type=str, required=False)
parser.add_argument('--tabular_log_file',
type=str,
default='progress.csv',
help='Name of the tabular log file (in csv).')
parser.add_argument('--text_log_file',
type=str,
default='debug.log',
help='Name of the text log file (in pure text).')
parser.add_argument('--params_log_file',
type=str,
default='params.json',
help='Name of the parameter log file (in json).')
parser.add_argument('--seed', type=int, help='Random seed for numpy')
parser.add_argument('--args_data',
type=str,
help='Pickled data for stub objects')
parser.add_argument('--snapshot_mode',
type=str,
default='all',
help='Mode to save the snapshot. Can be either "all" '
'(all iterations will be saved), "last" (only '
'the last iteration will be saved), or "none" '
'(do not save snapshots)')
parser.add_argument(
'--log_tabular_only',
type=ast.literal_eval,
default=False,
help=
'Whether to only print the tabular log information (in a horizontal format)'
)
args = parser.parse_args()
parallel_sampler.initialize(n_parallel=args.sampler_workers)
if args.seed is not None:
set_seed(args.seed)
parallel_sampler.set_seed(args.seed)
args.hidden_sizes = tuple(map(int, args.policy_hidden_sizes.split(',')))
if args.checkpoint:
with tf.Session() as sess:
data = joblib.load(args.checkpoint)
policy = data['policy']
env = data['env']
else:
if args.sample_maps:
map_pool = np.load(args.map_file)
else:
if args.map_type == 'rectangle':
env_map = TwoDMaps.rectangle_map(
*map(int, args.rectangle.split(',')))
elif args.map_type == 'complex':
env_map = TwoDMaps.complex_map(
*map(int, args.rectangle.split(',')))
else:
raise NotImplementedError()
map_pool = [env_map]
env = PursuitEvade(map_pool,
n_evaders=args.n_evaders,
n_pursuers=args.n_pursuers,
obs_range=args.obs_range,
n_catch=args.n_catch,
train_pursuit=args.train_pursuit,
urgency_reward=args.urgency,
surround=args.surround,
sample_maps=args.sample_maps,
constraint_window=args.constraint_window,
flatten=args.flatten,
reward_mech=args.reward_mech,
catchr=args.catchr,
term_pursuit=args.term_pursuit)
env = TfEnv(
RLLabEnv(StandardizedEnv(env,
scale_reward=args.reward_scale,
enable_obsnorm=False),
mode=args.control))
if args.recurrent:
if args.conv:
feature_network = ConvNetwork(
name='feature_net',
input_shape=emv.spec.observation_space.shape,
output_dim=5,
conv_filters=(16, 32, 32),
conv_filter_sizes=(3, 3, 3),
conv_strides=(1, 1, 1),
conv_pads=('VALID', 'VALID', 'VALID'),
hidden_sizes=(64, ),
hidden_nonlinearity=tf.nn.relu,
output_nonlinearity=tf.nn.softmax)
else:
feature_network = MLP(
name='feature_net',
input_shape=(env.spec.observation_space.flat_dim +
env.spec.action_space.flat_dim, ),
output_dim=5,
hidden_sizes=(256, 128, 64),
hidden_nonlinearity=tf.nn.tanh,
output_nonlinearity=None)
if args.recurrent == 'gru':
policy = CategoricalGRUPolicy(env_spec=env.spec,
feature_network=feature_network,
hidden_dim=int(
args.policy_hidden_sizes),
name='policy')
elif args.recurrent == 'lstm':
policy = CategoricalLSTMPolicy(env_spec=env.spec,
feature_network=feature_network,
hidden_dim=int(
args.policy_hidden_sizes),
name='policy')
elif args.conv:
feature_network = ConvNetwork(
name='feature_net',
input_shape=env.spec.observation_space.shape,
output_dim=5,
conv_filters=(8, 16),
conv_filter_sizes=(3, 3),
conv_strides=(2, 1),
conv_pads=('VALID', 'VALID'),
hidden_sizes=(32, ),
hidden_nonlinearity=tf.nn.relu,
output_nonlinearity=tf.nn.softmax)
policy = CategoricalMLPPolicy(name='policy',
env_spec=env.spec,
prob_network=feature_network)
else:
policy = CategoricalMLPPolicy(name='policy',
env_spec=env.spec,
hidden_sizes=args.hidden_sizes)
if args.baseline_type == 'linear':
baseline = LinearFeatureBaseline(env_spec=env.spec)
else:
baseline = ZeroBaseline(env_spec=env.spec)
# logger
default_log_dir = config.LOG_DIR
if args.log_dir is None:
log_dir = osp.join(default_log_dir, args.exp_name)
else:
log_dir = args.log_dir
tabular_log_file = osp.join(log_dir, args.tabular_log_file)
text_log_file = osp.join(log_dir, args.text_log_file)
params_log_file = osp.join(log_dir, args.params_log_file)
logger.log_parameters_lite(params_log_file, args)
logger.add_text_output(text_log_file)
logger.add_tabular_output(tabular_log_file)
prev_snapshot_dir = logger.get_snapshot_dir()
prev_mode = logger.get_snapshot_mode()
logger.set_snapshot_dir(log_dir)
logger.set_snapshot_mode(args.snapshot_mode)
logger.set_log_tabular_only(args.log_tabular_only)
logger.push_prefix("[%s] " % args.exp_name)
algo = TRPO(
env=env,
policy=policy,
baseline=baseline,
batch_size=args.n_timesteps,
max_path_length=args.max_traj_len,
n_itr=args.n_iter,
discount=args.discount,
gae_lambda=args.gae_lambda,
step_size=args.max_kl,
optimizer=ConjugateGradientOptimizer(hvp_approach=FiniteDifferenceHvp(
base_eps=1e-5)) if args.recurrent else None,
mode=args.control,
)
algo.train()
def parse_env_args(self, env, args):
if isinstance(args, dict):
args = to_named_tuple(args)
# Multi-agent wrapper
env = RLLabEnv(env, ma_mode=args.control)
env = MATfEnv(env)
# Policy
if args.recurrent:
if args.feature_net:
feature_network = MLP(
name='feature_net',
input_shape=(env.spec.observation_space.flat_dim +
env.spec.action_space.flat_dim, ),
output_dim=args.feature_output,
hidden_sizes=tuple(args.feature_hidden),
hidden_nonlinearity=tf.nn.tanh,
output_nonlinearity=None)
elif args.conv:
strides = tuple(args.conv_strides)
chans = tuple(args.conv_channels)
filts = tuple(args.conv_filters)
assert len(strides) == len(chans) == len(
filts), "strides, chans and filts not equal"
# only discrete actions supported, should be straightforward to extend to continuous
assert isinstance(
env.spec.action_space,
Discrete), "Only discrete action spaces support conv"
feature_network = ConvNetwork(
name='feature_net',
input_shape=env.spec.observation_space.shape,
output_dim=args.feature_output,
conv_filters=chans,
conv_filter_sizes=filts,
conv_strides=strides,
conv_pads=('VALID', ) * len(chans),
hidden_sizes=tuple(args.feature_hidden),
hidden_nonlinearity=tf.nn.relu,
output_nonlinearity=None)
else:
feature_network = None
if args.recurrent == 'gru':
if isinstance(env.spec.action_space, Box):
if args.control == 'concurrent':
policies = [
GaussianGRUPolicy(env_spec=env.spec,
feature_network=feature_network,
hidden_dim=int(
args.policy_hidden[0]),
name='policy_{}'.format(agid))
for agid in range(len(env.agents))
]
policy = GaussianGRUPolicy(env_spec=env.spec,
feature_network=feature_network,
hidden_dim=int(
args.policy_hidden[0]),
name='policy')
elif isinstance(env.spec.action_space, Discrete):
if args.control == 'concurrent':
policies = [
CategoricalGRUPolicy(
env_spec=env.spec,
feature_network=feature_network,
hidden_dim=int(args.policy_hidden[0]),
name='policy_{}'.format(agid),
state_include_action=False
if args.conv else True)
for agid in range(len(env.agents))
]
q_network = CategoricalGRUPolicy(
env_spec=env.spec,
feature_network=feature_network,
hidden_dim=int(args.policy_hidden[0]),
name='q_network',
state_include_action=False if args.conv else True)
target_q_network = CategoricalGRUPolicy(
env_spec=env.spec,
feature_network=feature_network,
hidden_dim=int(args.policy_hidden[0]),
name='target_q_network',
state_include_action=False if args.conv else True)
policy = {
'q_network': q_network,
'target_q_network': target_q_network
}
else:
raise NotImplementedError(env.spec.observation_space)
elif args.recurrent == 'lstm':
if isinstance(env.spec.action_space, Box):
if args.control == 'concurrent':
policies = [
GaussianLSTMPolicy(env_spec=env.spec,
feature_network=feature_network,
hidden_dim=int(
args.policy_hidden),
name='policy_{}'.format(agid))
for agid in range(len(env.agents))
]
policy = GaussianLSTMPolicy(
env_spec=env.spec,
feature_network=feature_network,
hidden_dim=int(args.policy_hidden),
name='policy')
elif isinstance(env.spec.action_space, Discrete):
if args.control == 'concurrent':
policies = [
CategoricalLSTMPolicy(
env_spec=env.spec,
feature_network=feature_network,
hidden_dim=int(args.policy_hidden),
name='policy_{}'.format(agid))
for agid in range(len(env.agents))
]
q_network = CategoricalLSTMPolicy(
env_spec=env.spec,
feature_network=feature_network,
hidden_dim=int(args.policy_hidden),
name='q_network')
target_q_network = CategoricalLSTMPolicy(
env_spec=env.spec,
feature_network=feature_network,
hidden_dim=int(args.policy_hidden),
name='target_q_network')
policy = {
'q_network': q_network,
'target_q_network': target_q_network
}
else:
raise NotImplementedError(env.spec.action_space)
else:
raise NotImplementedError(args.recurrent)
elif args.conv:
strides = tuple(args.conv_strides)
chans = tuple(args.conv_channels)
filts = tuple(args.conv_filters)
assert len(strides) == len(chans) == len(
filts), "strides, chans and filts not equal"
# only discrete actions supported, should be straightforward to extend to continuous
assert isinstance(
env.spec.action_space,
Discrete), "Only discrete action spaces support conv"
feature_network = ConvNetwork(
name='feature_net',
input_shape=env.spec.observation_space.shape,
output_dim=env.spec.action_space.n,
conv_filters=chans,
conv_filter_sizes=filts,
conv_strides=strides,
conv_pads=(args.conv_pads, ) * len(chans),
hidden_sizes=tuple(args.policy_hidden),
hidden_nonlinearity=tf.nn.relu,
output_nonlinearity=tf.nn.softmax,
batch_normalization=args.batch_normalization)
if args.algo == 'dqn':
q_network = CategoricalMLPPolicy(name='q_network',
env_spec=env.spec,
prob_network=feature_network)
target_q_network = CategoricalMLPPolicy(
name='target_q_network',
env_spec=env.spec,
prob_network=feature_network)
policy = {
'q_network': q_network,
'target_q_network': target_q_network
}
else:
policy = CategoricalMLPPolicy(name='policy',
env_spec=env.spec,
prob_network=feature_network)
else:
if env.spec is None:
networks = [
DQNNetwork(i,
env,
target_network_update_freq=self.args.
target_network_update,
discount_factor=self.args.discount,
batch_size=self.args.batch_size,
learning_rate=self.args.qfunc_lr)
for i in range(env.n)
]
policy = networks
elif isinstance(env.spec.action_space, Box):
policy = GaussianMLPPolicy(env_spec=env.spec,
hidden_sizes=tuple(
args.policy_hidden),
min_std=args.min_std,
name='policy')
elif isinstance(env.spec.action_space, Discrete):
policy = CategoricalMLPPolicy(env_spec=env.spec,
hidden_sizes=tuple(
args.policy_hidden),
name='policy')
else:
raise NotImplementedError(env.spec.action_space)
return env, policy
# env = TfEnv(normalize(gymenv))
# policy = GaussianMLPPolicy(
# name="policy",
# env_spec=env.spec,
# # The neural network policy should have two hidden layers, each with 32 hidden units.
# hidden_sizes=(100, 50, 25),
# hidden_nonlinearity=tf.nn.relu,
# )
"""
Use CategoricalMLPPolicy for GridWorld Environment
"""
policy = CategoricalMLPPolicy(
name="policy",
env_spec=env.spec,
# The neural network policy should have two hidden layers, each with 32 hidden units.
hidden_sizes=(100, 50, 25),
hidden_nonlinearity=tf.nn.relu,
)
baseline = LinearFeatureBaseline(env_spec=env.spec)
algo = TRPO(
env=env,
policy=policy,
baseline=baseline,
batch_size=5000,
max_path_length=2000,
#max_path_length=env.horizon,
n_itr=1000,
discount=0.99,
def run_experiment(expert_rollout_pickle_path,
trained_policy_pickle_path,
env,
cost_trainer_type,
iterations=30,
num_frames=1,
traj_len=200,
config={}):
# Load the expert rollouts into memory
expert_rollouts = load_expert_rollouts(expert_rollout_pickle_path)
# In the case that we only have one expert rollout in the file
if type(expert_rollouts) is dict:
expert_rollouts = [expert_rollouts]
#TODO: make this configurable
expert_rollouts = [
shorten_tensor_dict(x, traj_len) for x in expert_rollouts
]
# import pdb; pdb.set_trace()
# Sanity check, TODO: should prune any "expert" rollouts with suboptimal reward?
print("Average reward for expert rollouts: %f" %
np.mean([np.sum(p['rewards']) for p in expert_rollouts]))
if "transformers" in config and len(config["transformers"]) > 0:
print("Transforming expert rollouts...")
for rollout in tqdm(expert_rollouts):
transformed_observations = []
for ob in tqdm(rollout["observations"]):
for transformer in config["transformers"]:
ob = transformer.transform(ob)
transformed_observations.append(ob)
rollout["observations"] = np.array(transformed_observations)
# Handle both flattened state input and image input
# TODO: this could be done better by looking at just the shape and determining from that
if config["img_input"]:
obs_dims = expert_rollouts[0]['observations'][0].shape
else:
# import pdb; pdb.set_trace()
obs_dims = len(expert_rollouts[0]['observations'][0])
if "num_novice_rollouts" in config:
number_of_sample_trajectories = config["num_novice_rollouts"]
else:
number_of_sample_trajectories = len(expert_rollouts)
print(number_of_sample_trajectories)
# Choose a policy (Conv based on images, mlp based on states)
# TODO: may also have to switch out categorical for something else in continuous state spaces??
# Let's just avoid that for now?
if config[
"img_input"]: # TODO: unclear right now if this even works ok. get poor results early on.
policy = CategoricalConvPolicy(
name="policy",
env_spec=env.spec,
conv_filters=[32, 64, 64],
conv_filter_sizes=[3, 3, 3],
conv_strides=[1, 1, 1],
conv_pads=['SAME', 'SAME', 'SAME'],
# The neural network policy should have two hidden layers, each with 100 hidden units each (see RLGAN paper)
hidden_sizes=[200, 200])
elif type(env.spec.action_space) == Discrete:
policy = CategoricalMLPPolicy(
name="policy",
env_spec=env.spec,
# The neural network policy should have two hidden layers, each with 100 hidden units each (see RLGAN paper)
hidden_sizes=(400, 300))
else:
policy = GaussianMLPPolicy(name="policy",
env_spec=env.spec,
hidden_sizes=(100, 50, 25))
if config["img_input"]:
# TODO: right now the linear feature baseline is too computationally expensive to actually use
# with full image inputs, so for now just use the zero baseline
baseline = ZeroBaseline(env_spec=env.spec)
else:
baseline = LinearFeatureBaseline(env_spec=env.spec)
algo = TRPO(
env=env,
policy=policy,
baseline=baseline,
batch_size=number_of_sample_trajectories *
traj_len, # This is actually used internally by the sampler. We make use of this sampler to generate our samples, hence we pass it here
max_path_length=
traj_len, # same with this value. A cleaner way may be to create our own sampler, but for now doing it this way..
n_itr=40,
discount=0.995,
step_size=0.01,
optimizer=ConjugateGradientOptimizer(
hvp_approach=FiniteDifferenceHvp(base_eps=1e-5),
max_backtracks=40))
# Prune the number of rollouts if that option is enabled
if "num_expert_rollouts" in config:
rollouts_to_use = min(config["num_expert_rollouts"],
len(expert_rollouts))
expert_rollouts = expert_rollouts[:rollouts_to_use]
print("Only using %d expert rollouts" % rollouts_to_use)
true_rewards = []
actual_rewards = []
# Extract observations to a tensor
expert_rollouts_tensor = tensor_utils.stack_tensor_list(
[path["observations"] for path in expert_rollouts])
if "oversample" in config and config["oversample"]:
oversample_rate = max(
int(number_of_sample_trajectories / len(expert_rollouts_tensor)),
1.)
expert_rollouts_tensor = expert_rollouts_tensor.repeat(oversample_rate,
axis=0)
print("oversampling %d times to %d" %
(oversample_rate, len(expert_rollouts_tensor)))
with tf.Session() as sess:
algo.start_worker()
cost_trainer = cost_trainer_type([num_frames, obs_dims], config=config)
trainer = Trainer(env=env,
sess=sess,
cost_approximator=cost_trainer,
cost_trainer=cost_trainer,
novice_policy=policy,
novice_policy_optimizer=algo,
num_frames=num_frames)
sess.run(tf.global_variables_initializer())
for iter_step in range(0, iterations):
dump_data = (iter_step == (
iterations -
1)) and config["generate_option_graphs"] # is last iteration
true_reward, actual_reward = trainer.step(
dump_datapoints=dump_data,
config=config,
expert_horizon=traj_len,
number_of_sample_trajectories=number_of_sample_trajectories)
true_rewards.append(true_reward)
actual_rewards.append(actual_reward)
# run a rollout for the video
if "recording_env" in config:
novice_rollouts = rollout_policy(policy,
config["recording_env"],
get_image_observations=False,
max_path_length=200)
novice_rollouts = algo.obtain_samples(iter_step)
rollout_rewards = [np.sum(x['rewards']) for x in novice_rollouts]
print("Reward stats for final policy: %f +/- %f " %
(np.mean(rollout_rewards), np.std(rollout_rewards)))
# save the novice policy learned
with open(trained_policy_pickle_path, "wb") as output_file:
pickle.dump(policy, output_file)
# TODO: also save the reward function?
algo.shutdown_worker()
second_true_rewards = []
second_actual_rewards = []
# Do our transfer learning task here:
# TODO: move this to a separate script and save the learned weights
if config['second_env'] is not None:
with tf.variable_scope("second_policy"):
#TODO: remove gross copypasta
if not config["reset_second_policy"]:
second_policy = Serializable.clone(
policy) # TODO: start with a fresh policy
else:
if config[
"img_input"]: # TODO: unclear right now if this even works ok. get poor results early on.
second_policy = CategoricalConvPolicy(
name="policy",
env_spec=config["second_env"].spec,
conv_filters=[32, 64, 64],
conv_filter_sizes=[3, 3, 3],
conv_strides=[1, 1, 1],
conv_pads=['SAME', 'SAME', 'SAME'],
# The neural network policy should have two hidden layers, each with 100 hidden units each (see RLGAN paper)
hidden_sizes=[200, 200])
elif type(env.spec.action_space) == Discrete:
second_policy = CategoricalMLPPolicy(
name="policy",
env_spec=config["second_env"].spec,
# The neural network policy should have two hidden layers, each with 100 hidden units each (see RLGAN paper)
hidden_sizes=(400, 300))
else:
second_policy = GaussianMLPPolicy(
name="policy",
env_spec=config["second_env"].spec,
hidden_sizes=(100, 50, 25))
if config["img_input"]:
# TODO: right now the linear feature baseline is too computationally expensive to actually use
# with full image inputs, so for now just use the zero baseline
baseline = ZeroBaseline(env_spec=config["second_env"].spec)
else:
baseline = LinearFeatureBaseline(
env_spec=config["second_env"].spec)
algo = TRPO(
env=config["second_env"],
policy=second_policy,
baseline=baseline,
batch_size=number_of_sample_trajectories *
traj_len, # This is actually used internally by the sampler. We make use of this sampler to generate our samples, hence we pass it here
max_path_length=
traj_len, # same with this value. A cleaner way may be to create our own sampler, but for now doing it this way..
n_itr=40,
discount=0.995,
step_size=0.01,
optimizer=ConjugateGradientOptimizer(
hvp_approach=FiniteDifferenceHvp(base_eps=1e-5),
max_backtracks=40))
if not config["stop_disc_training_on_second_run"] and config[
"use_prev_options_relearn_mixing_func"]:
# If we're not retraining the discriminator at all in the transfer learning step,
# just keep the old network
options = cost_trainer.disc.discriminator_options
cost_trainer.disc._remake_network_from_disc_options(
options,
stop_gradients=(not config["retrain_options"]),
num_extra_options=config["num_extra_options_on_transfer"])
trainer = Trainer(
env=config['second_env'],
sess=sess,
cost_approximator=cost_trainer,
cost_trainer=cost_trainer,
novice_policy=second_policy,
novice_policy_optimizer=algo,
num_frames=num_frames,
train_disc=(not config["stop_disc_training_on_second_run"]))
algo.start_worker()
initialize_uninitialized(sess)
for iter_step in range(0, iterations):
# import pdb; pdb.set_trace()
dump_data = (iter_step == (iterations - 1)) and config[
"generate_option_graphs"] # is last iteration
true_reward, actual_reward = trainer.step(
expert_rollouts_tensor=expert_rollouts_tensor,
dump_datapoints=dump_data,
config=config,
expert_horizon=traj_len,
number_of_sample_trajectories=number_of_sample_trajectories
)
second_true_rewards.append(true_reward)
second_actual_rewards.append(actual_reward)
# run a rollout for the video
if "recording_env" in config:
novice_rollouts = rollout_policy(
second_policy,
config["recording_env"],
get_image_observations=False,
max_path_length=traj_len)
novice_rollouts = algo.obtain_samples(iter_step)
rollout_rewards = [np.sum(x['rewards']) for x in novice_rollouts]
print("Reward stats for final policy: %f +/- %f " %
(np.mean(rollout_rewards), np.std(rollout_rewards)))
# save the novice policy learned
with open(trained_policy_pickle_path, "wb") as output_file:
pickle.dump(second_policy, output_file)
algo.shutdown_worker()
return true_rewards, actual_rewards, second_true_rewards, second_actual_rewards
@property
def observation_space(self):
return Discrete(4)
if __name__ == "__main__":
env = GridWorldEnv()
from sandbox.rocky.tf.policies.categorical_mlp_policy import CategoricalMLPPolicy
from sandbox.rocky.tf.core.network import MLP
from sandbox.rocky.tf.envs.base import TfEnv
from sandbox.rocky.tf.algos.trpo import TRPO
import tensorflow as tf
env = TfEnv(env)
from rllab.baselines.linear_feature_baseline import LinearFeatureBaseline
policy = CategoricalMLPPolicy(env_spec=env.spec, name="policy")
baseline = LinearFeatureBaseline(env_spec=env.spec)
algo = TRPO(env=env,
policy=policy,
baseline=baseline,
whole_paths=True,
max_path_length=50,
n_itr=40,
discount=0.40)
algo.train()
summary_writer=summary_writer,
verbose=2)
# build the policy
latent_sampler = UniformlyRandomLatentSampler(
scheduler=ConstantIntervalScheduler(k=scheduler_k),
name='latent_sampler',
dim=latent_dim)
policy = CategoricalLatentVarMLPPolicy(policy_name="policy",
latent_sampler=latent_sampler,
env_spec=env.spec,
hidden_sizes=(64, 64))
else:
# build the policy
policy = CategoricalMLPPolicy(name="policy",
env_spec=env.spec,
hidden_sizes=(32, 32))
recognition_model = None
# build gail
baseline = LinearFeatureBaseline(env_spec=env.spec)
reward_handler = hgail.misc.utils.RewardHandler(
use_env_rewards=False,
max_epochs=50, # epoch at which final scales are used
critic_final_scale=1.,
recognition_initial_scale=0.)
session.run(tf.global_variables_initializer())
saver = tf.train.Saver(max_to_keep=100, keep_checkpoint_every_n_hours=.5)
if initial_filepath:
def get_policy(env, algo_name, info, policy_hidden_sizes,
policy_hidden_nonlinearity, policy_output_nonlinearity,
recurrent, **kwargs):
policy = None
policy_class = None
hidden_sizes = get_hidden_sizes(policy_hidden_sizes)
hidden_nonlinearity = get_nonlinearity(policy_hidden_nonlinearity)
output_nonlinearity = get_nonlinearity(policy_output_nonlinearity)
if algo_name in [
'trpo',
'actrpo',
'acqftrpo',
'qprop',
'mqprop',
'qfqprop',
'trpg',
'trpgoff',
'nuqprop',
'nuqfqprop',
'nafqprop',
'vpg',
'qvpg',
'dspg',
'dspgoff',
]:
if not info['is_action_discrete']:
if recurrent:
policy = GaussianLSTMPolicy(
name="gauss_lstm_policy",
env_spec=env.spec,
lstm_layer_cls=L.TfBasicLSTMLayer,
# gru_layer_cls=L.GRULayer,
output_nonlinearity=output_nonlinearity, # None
)
policy_class = 'GaussianLSTMPolicy'
else:
policy = GaussianMLPPolicy(
name="gauss_policy",
env_spec=env.spec,
hidden_sizes=hidden_sizes,
hidden_nonlinearity=hidden_nonlinearity, # tf.nn.tanh
output_nonlinearity=output_nonlinearity, # None
)
policy_class = 'GaussianMLPPolicy'
else:
if recurrent:
policy = CategoricalLSTMPolicy(
name="cat_lstm_policy",
env_spec=env.spec,
lstm_layer_cls=L.TfBasicLSTMLayer,
# gru_layer_cls=L.GRULayer,
)
policy_class = 'CategoricalLSTMPolicy'
else:
policy = CategoricalMLPPolicy(
name="cat_policy",
env_spec=env.spec,
hidden_sizes=hidden_sizes,
hidden_nonlinearity=hidden_nonlinearity, # tf.nn.tanh
)
policy_class = 'CategoricalMLPPolicy'
elif algo_name in [
'ddpg',
]:
assert not info['is_action_discrete']
policy = DeterministicMLPPolicy(
name="det_policy",
env_spec=env.spec,
hidden_sizes=hidden_sizes,
hidden_nonlinearity=hidden_nonlinearity, # tf.nn.relu
output_nonlinearity=output_nonlinearity, # tf.nn.tanh
)
policy_class = 'DeterministicMLPPolicy'
print(
'[get_policy] Instantiating %s, with sizes=%s, hidden_nonlinearity=%s.'
% (policy_class, str(hidden_sizes), policy_hidden_nonlinearity))
print('[get_policy] output_nonlinearity=%s.' %
(policy_output_nonlinearity))
return policy