class GenDiagGaussian:
def __init__(self,
name,
abstract_dim,
hidden_sizes=(32),
min_std=1e-6,
hidden_nonlinearity=tf.nn.tanh,
output_nonlinearity=None,
optim=tf.train.AdamOptimizer(learning_rate=0.001)):
self.obs_dim = abstract_dim
self.min_std = min_std
self.distribution = DiagonalGaussian(self.obs_dim)
with tf.variable_scope(name):
self.net = MLP(
name="mu_log_sigma",
input_shape=self.obs_dim,
output_dim=2 * self.obs_dim,
hidden_sizes=hidden_sizes,
hidden_nonlinearity=hidden_nonlinearity,
output_nonlinearity=output_nonlinearity,
)
self.obs_var = self.net.input_layer.input_var
self.output = L.get_output(self.net.output_layer, self.obs_var)
self.mu, unstable_log_sigma = tf.split(self.output,
[self.obs_dim, self.obs_dim], 1)
self.log_sigma = tf.maximum(unstable_log_sigma, self.min_std)
self.nexts = tf.placeholder(tf.float32, shape=(None, self.obs_dim))
self.loss = -self.distribution.log_likelihood(
self.nexts, dist_info=dict(mean=self.mu, log_stds=self.log_sigma))
self.optimizer = optim
self.train_op = optim.minimize(self.loss)
config = tf.ConfigProto()
config.gpu_options.allow_growth = True
self.sess = tf.Session(config=config)
self.sess.run(tf.global_variables_initializer())
def forward(self, obs):
return self.sess.run([self.mu, self.log_sigma],
feed_dict={self.obs_var: obs})
def get_loglikelihood(self, obs, next):
mu, sigma = self.forward(obs)
return self.distribution.log_likelihood(next,
dist_info=dict(mean=mu,
log_stds=sigma))
def fit(self, obs, nexts, n_steps=25):
for _ in range(n_steps):
self.sess.run(self.train_op,
feed_dict={
self.obs_var: obs,
self.nexts: nexts
})
return True
def __init__(
self,
name,
env_spec,
arch=gaussian_ff_arch,
arch_args=None,
min_std=1e-6,
):
"""
:param env_spec:
:param hidden_sizes: list of sizes for the fully-connected hidden layers
:param learn_std: Is std trainable
:param init_std: Initial std
:param adaptive_std:
:param std_share_network:
:param std_hidden_sizes: list of sizes for the fully-connected layers for std
:param min_std: whether to make sure that the std is at least some threshold value, to avoid numerical issues
:param std_hidden_nonlinearity:
:param hidden_nonlinearity: nonlinearity used for each hidden layer
:param output_nonlinearity: nonlinearity for the output layer
:param mean_network: custom network for the output mean
:param std_network: custom network for the output log std
:param std_parametrization: how the std should be parametrized. There are a few options:
- exp: the logarithm of the std will be stored, and applied a exponential transformation
- softplus: the std will be computed as log(1+exp(x))
:return:
"""
Serializable.quick_init(self, locals())
super(GaussianTFPolicy, self).__init__(env_spec)
assert isinstance(env_spec.action_space, Box)
if arch_args is None:
arch_args = {}
self.arch_args = arch_args
self.name = name
self.min_std_param = np.log(min_std)
obs_dim = env_spec.observation_space.flat_dim
action_dim = env_spec.action_space.flat_dim
self._dist = DiagonalGaussian(action_dim)
self.arch = arch
with tf.variable_scope(self.name):
self.sample_obs = tf.placeholder(tf.float32, [None, obs_dim])
self.sample_mu, self.sample_log_std = self.arch(self.sample_obs,
env_spec=env_spec,
**self.arch_args)
def __init__(self, env_spec, name=None, mean=0.0, log_std=1.0):
with tf.variable_scope(name):
assert isinstance(env_spec.action_space, Box)
Serializable.quick_init(self, locals())
self.action_dim = env_spec.action_space.flat_dim
self._dist = DiagonalGaussian(self.action_dim)
self.mean = mean * np.ones(self.action_dim)
self.log_std = log_std * np.ones(self.action_dim)
self.mean_tf = tf.constant(self.mean, dtype=tf.float32)
self.log_std_tf = tf.constant(self.log_std, dtype=tf.float32)
self.dummy_var = tf.get_variable(name='dummy',
shape=self.action_dim)
super(GaussianPolicy, self).__init__(env_spec=env_spec)
def __init__(
self,
name,
env_spec,
hidden_sizes=(32, 32),
learn_std=True,
init_std=1.0,
adaptive_std=False,
std_share_network=False,
std_hidden_sizes=(32, 32),
min_std=1e-6,
std_hidden_nonlinearity=tf.nn.tanh,
hidden_nonlinearity=tf.nn.tanh,
output_nonlinearity=None,
mean_network=None,
std_network=None,
std_parametrization='exp'
):
"""
:param env_spec:
:param hidden_sizes: list of sizes for the fully-connected hidden layers
:param learn_std: Is std trainable
:param init_std: Initial std
:param adaptive_std:
:param std_share_network:
:param std_hidden_sizes: list of sizes for the fully-connected layers for std
:param min_std: whether to make sure that the std is at least some threshold value, to avoid numerical issues
:param std_hidden_nonlinearity:
:param hidden_nonlinearity: nonlinearity used for each hidden layer
:param output_nonlinearity: nonlinearity for the output layer
:param mean_network: custom network for the output mean
:param std_network: custom network for the output log std
:param std_parametrization: how the std should be parametrized. There are a few options:
- exp: the logarithm of the std will be stored, and applied a exponential transformation
- softplus: the std will be computed as log(1+exp(x))
:return:
"""
Serializable.quick_init(self, locals())
assert isinstance(env_spec.action_space, Box)
with tf.variable_scope(name):
obs_dim = env_spec.observation_space.flat_dim
action_dim = env_spec.action_space.flat_dim
# create network
if mean_network is None:
mean_network = MLP(
name="mean_network",
input_shape=(obs_dim,),
output_dim=action_dim,
hidden_sizes=hidden_sizes,
hidden_nonlinearity=hidden_nonlinearity,
output_nonlinearity=output_nonlinearity,
)
self._mean_network = mean_network
l_mean = mean_network.output_layer
obs_var = mean_network.input_layer.input_var
if std_network is not None:
l_std_param = std_network.output_layer
else:
if adaptive_std:
std_network = MLP(
name="std_network",
input_shape=(obs_dim,),
input_layer=mean_network.input_layer,
output_dim=action_dim,
hidden_sizes=std_hidden_sizes,
hidden_nonlinearity=std_hidden_nonlinearity,
output_nonlinearity=None,
)
l_std_param = std_network.output_layer
else:
if std_parametrization == 'exp':
init_std_param = np.log(init_std)
elif std_parametrization == 'softplus':
init_std_param = np.log(np.exp(init_std) - 1)
else:
raise NotImplementedError
l_std_param = L.ParamLayer(
mean_network.input_layer,
num_units=action_dim,
param=tf.constant_initializer(init_std_param),
name="output_std_param",
trainable=learn_std,
)
self.std_parametrization = std_parametrization
if std_parametrization == 'exp':
min_std_param = np.log(min_std)
elif std_parametrization == 'softplus':
min_std_param = np.log(np.exp(min_std) - 1)
else:
raise NotImplementedError
self.min_std_param = min_std_param
# mean_var, log_std_var = L.get_output([l_mean, l_std_param])
#
# if self.min_std_param is not None:
# log_std_var = tf.maximum(log_std_var, np.log(min_std))
#
# self._mean_var, self._log_std_var = mean_var, log_std_var
self._l_mean = l_mean
self._l_std_param = l_std_param
self._dist = DiagonalGaussian(action_dim)
LayersPowered.__init__(self, [l_mean, l_std_param])
super(GaussianMLPPolicy, self).__init__(env_spec)
dist_info_sym = self.dist_info_sym(mean_network.input_layer.input_var, dict())
mean_var = dist_info_sym["mean"]
log_std_var = dist_info_sym["log_std"]
self._f_dist = tensor_utils.compile_function(
inputs=[obs_var],
outputs=[mean_var, log_std_var],
)
def __init__(
self,
name,
env_spec,
hidden_sizes=(32, 32),
learn_std=True,
init_std=1.0,
adaptive_std=False,
std_share_network=False,
std_hidden_sizes=(32, 32),
min_std=1e-6,
max_std=1000.0,
std_modifier=1.0,
std_hidden_nonlinearity=tf.nn.tanh,
hidden_nonlinearity=tf.nn.tanh,
output_nonlinearity=tf.identity,
mean_network=None,
std_network=None,
std_parametrization='exp',
grad_step_size=1.0,
stop_grad=False,
extra_input_dim=0,
# metalearn_baseline=False,
):
"""
:param env_spec:
:param hidden_sizes: list of sizes for the fully-connected hidden layers
:param learn_std: Is std trainable
:param init_std: Initial std
:param adaptive_std:
:param std_share_network:
:param std_hidden_sizes: list of sizes for the fully-connected layers for std
:param min_std: whether to make sure that the std is at least some threshold value, to avoid numerical issues
:param std_hidden_nonlinearity:
:param hidden_nonlinearity: nonlinearity used for each hidden layer
:param output_nonlinearity: nonlinearity for the output layer
:param mean_network: custom network for the output mean
:param std_network: custom network for the output log std
:param std_parametrization: how the std should be parametrized. There are a few options:
- exp: the logarithm of the std will be stored, and applied a exponential transformation
- softplus: the std will be computed as log(1+exp(x))
:param grad_step_size: the step size taken in the learner's gradient update, sample uniformly if it is a range e.g. [0.1,1]
:param stop_grad: whether or not to stop the gradient through the gradient.
:return:
"""
Serializable.quick_init(self, locals())
#assert isinstance(env_spec.action_space, Box)
obs_dim = env_spec.observation_space.flat_dim
self.action_dim = env_spec.action_space.flat_dim
self.n_hidden = len(hidden_sizes)
self.hidden_nonlinearity = hidden_nonlinearity
self.output_nonlinearity = output_nonlinearity
self.input_shape = (
None,
obs_dim + extra_input_dim,
)
self.extra_input_dim = extra_input_dim
self.step_size = grad_step_size
self.stop_grad = stop_grad
# self.metalearn_baseline = metalearn_baseline
if type(self.step_size) == list:
raise NotImplementedError('removing this since it didnt work well')
# create network
if mean_network is None:
self.all_params = self.create_MLP( # TODO: this should not be a method of the policy! --> helper
name="mean_network",
output_dim=self.action_dim,
hidden_sizes=hidden_sizes,
)
self.input_tensor, _ = self.forward_MLP(
'mean_network',
self.all_params,
reuse=None # Need to run this for batch norm
)
forward_mean = lambda x, params, is_train: self.forward_MLP(
'mean_network',
all_params=params,
input_tensor=x,
is_training=is_train)[1]
else:
raise NotImplementedError('Not supported.')
if std_network is not None:
raise NotImplementedError('Not supported.')
else:
if adaptive_std:
raise NotImplementedError('Not supported.')
else:
if std_parametrization == 'exp':
init_std_param = np.log(init_std)
elif std_parametrization == 'softplus':
init_std_param = np.log(np.exp(init_std) - 1)
else:
raise NotImplementedError
self.all_params['std_param'] = make_param_layer(
num_units=self.action_dim,
param=tf.constant_initializer(init_std_param),
name="output_std_param",
trainable=learn_std,
)
forward_std = lambda x, params: forward_param_layer(
x, params['std_param'])
self.all_param_vals = None
# unify forward mean and forward std into a single function
self._forward = lambda obs, params, is_train: (forward_mean(
obs, params, is_train), forward_std(obs, params))
self.std_parametrization = std_parametrization
if std_parametrization == 'exp':
min_std_param = np.log(min_std)
max_std_param = np.log(max_std)
elif std_parametrization == 'softplus':
min_std_param = np.log(np.exp(min_std) - 1)
max_std_param = np.log(np.exp(max_std) - 1)
else:
raise NotImplementedError
self.min_std_param = min_std_param # TODO: change these to min_std_param_raw
self.max_std_param = max_std_param
self.std_modifier = np.float64(std_modifier)
#print("initializing max_std debug4", self.min_std_param, self.max_std_param)
self._dist = DiagonalGaussian(self.action_dim)
self._cached_params = {}
super(MAMLGaussianMLPPolicy, self).__init__(env_spec)
dist_info_sym = self.dist_info_sym(self.input_tensor,
dict(),
is_training=False)
mean_var = dist_info_sym["mean"]
log_std_var = dist_info_sym["log_std"]
# pre-update policy
self._init_f_dist = tensor_utils.compile_function(
inputs=[self.input_tensor],
outputs=[mean_var, log_std_var],
)
self._cur_f_dist = self._init_f_dist
def __init__(self,
name,
env_spec,
hidden_sizes=(32, 32),
learn_std=True,
init_std=1.0,
adaptive_std=False,
std_share_network=False,
std_hidden_sizes=(32, 32),
min_std=1e-6,
std_hidden_nonlinearity=tf.nn.tanh,
hidden_nonlinearity=tf.nn.tanh,
output_nonlinearity=None,
mean_network=None,
std_network=None,
std_parametrization='exp'):
"""
:param env_spec:
:param hidden_sizes: list of sizes for the fully-connected hidden layers
:param learn_std: Is std trainable
:param init_std: Initial std
:param adaptive_std:
:param std_share_network:
:param std_hidden_sizes: list of sizes for the fully-connected layers for std
:param min_std: whether to make sure that the std is at least some threshold value, to avoid numerical issues
:param std_hidden_nonlinearity:
:param hidden_nonlinearity: nonlinearity used for each hidden layer
:param output_nonlinearity: nonlinearity for the output layer
:param mean_network: custom network for the output mean
:param std_network: custom network for the output log std
:param std_parametrization: how the std should be parametrized. There are a few options:
- exp: the logarithm of the std will be stored, and applied a exponential transformation
- softplus: the std will be computed as log(1+exp(x))
:return:
"""
Serializable.quick_init(self, locals())
assert isinstance(env_spec.action_space, Box)
with tf.variable_scope(name):
obs_dim = env_spec.observation_space.flat_dim
action_dim = env_spec.action_space.flat_dim
# create network
if mean_network is None:
# returns layers(), input_layer, output_layer, input_var, output
mean_network = self.create_MLP(
name="mean_network",
input_shape=(obs_dim, ),
output_dim=action_dim,
hidden_sizes=hidden_sizes,
hidden_nonlinearity=hidden_nonlinearity,
output_nonlinearity=output_nonlinearity,
)
else:
raise NotImplementedError('Chelsea does not support this.')
l_mean = mean_network[2]
obs_var = mean_network[3]
if std_network is not None:
raise NotImplementedError(
'Contained Gaussian MLP does not support this.')
l_std_param = std_network.output_layer
else:
if adaptive_std:
# returns layers(), input_layer, output_layer, input_var, output
std_network = self.create_MLP(
name="std_network",
input_shape=(obs_dim, ),
input_layer=mean_network[1],
output_dim=action_dim,
hidden_sizes=std_hidden_sizes,
hidden_nonlinearity=std_hidden_nonlinearity,
output_nonlinearity=None,
)
l_std_param = std_network[2]
else:
if std_parametrization == 'exp':
init_std_param = np.log(init_std)
elif std_parametrization == 'softplus':
init_std_param = np.log(np.exp(init_std) - 1)
else:
raise NotImplementedError
l_std_param = L.ParamLayer(
mean_network[1],
num_units=action_dim,
param=tf.constant_initializer(init_std_param),
name="output_std_param",
trainable=learn_std,
)
self.std_parametrization = std_parametrization
if std_parametrization == 'exp':
min_std_param = np.log(min_std)
elif std_parametrization == 'softplus':
min_std_param = np.log(np.exp(min_std) - 1)
else:
raise NotImplementedError
self.min_std_param = min_std_param
self._l_mean = l_mean
self._l_std_param = l_std_param
self._dist = DiagonalGaussian(action_dim)
# LayersPowered.__init__(self, [l_mean, l_std_param])
self._output_layers = [l_mean, l_std_param]
#self._input_layers = None
# Parameterized.__init__(self)
self._cached_params = {}
#self._cached_param_dtypes = {} #self._cached_param_shapes = {} #self._cached_assign_ops = {} #self._cached_assign_placeholders = {}
super(GaussianMLPPolicy, self).__init__(env_spec)
dist_info_sym = self.dist_info_sym(mean_network[3], dict())
mean_var = dist_info_sym["mean"]
log_std_var = dist_info_sym["log_std"]
def f_dist(*input_vals):
sess = tf.get_default_session()
return sess.run([mean_var, log_std_var],
feed_dict=dict(list(zip([obs_var],
input_vals))))
self._f_dist = f_dist
def __init__(
self,
name,
input_shape,
output_dim,
# observation_space,
mean_network=None,
hidden_sizes=(32, 32),
hidden_nonlinearity=tf.nn.tanh,
optimizer=None,
use_trust_region=True,
step_size=0.01,
learn_std=True,
init_std=1.0,
adaptive_std=False,
std_share_network=False,
std_hidden_sizes=(32, 32),
std_nonlinearity=None,
normalize_inputs=True,
normalize_outputs=True,
subsample_factor=1.0
):
"""
:param input_shape: Shape of the input data.
:param output_dim: Dimension of output.
:param hidden_sizes: Number of hidden units of each layer of the mean network.
:param hidden_nonlinearity: Non-linearity used for each layer of the mean network.
:param optimizer: Optimizer for minimizing the negative log-likelihood.
:param use_trust_region: Whether to use trust region constraint.
:param step_size: KL divergence constraint for each iteration
:param learn_std: Whether to learn the standard deviations. Only effective if adaptive_std is False. If
adaptive_std is True, this parameter is ignored, and the weights for the std network are always learned.
:param adaptive_std: Whether to make the std a function of the states.
:param std_share_network: Whether to use the same network as the mean.
:param std_hidden_sizes: Number of hidden units of each layer of the std network. Only used if
`std_share_network` is False. It defaults to the same architecture as the mean.
:param std_nonlinearity: Non-linearity used for each layer of the std network. Only used if `std_share_network`
is False. It defaults to the same non-linearity as the mean.
"""
Serializable.quick_init(self, locals())
with tf.variable_scope(name):
if optimizer is None:
if use_trust_region:
optimizer = PenaltyLbfgsOptimizer("optimizer")
else:
optimizer = LbfgsOptimizer("optimizer")
self._optimizer = optimizer
self._subsample_factor = subsample_factor
if mean_network is None:
mean_network = MLP(
name="mean_network",
input_shape=input_shape,
output_dim=output_dim,
hidden_sizes=hidden_sizes,
hidden_nonlinearity=hidden_nonlinearity,
output_nonlinearity=None,
)
l_mean = mean_network.output_layer
if adaptive_std:
l_log_std = MLP(
name="log_std_network",
input_shape=input_shape,
input_var=mean_network.input_layer.input_var,
output_dim=output_dim,
hidden_sizes=std_hidden_sizes,
hidden_nonlinearity=std_nonlinearity,
output_nonlinearity=None,
).output_layer
else:
l_log_std = L.ParamLayer(
mean_network.input_layer,
num_units=output_dim,
param=tf.constant_initializer(np.log(init_std)),
name="output_log_std",
trainable=learn_std,
)
LayersPowered.__init__(self, [l_mean, l_log_std])
xs_var = mean_network.input_layer.input_var
ys_var = tf.placeholder(dtype=tf.float32, name="ys", shape=(None, output_dim))
old_means_var = tf.placeholder(dtype=tf.float32, name="ys", shape=(None, output_dim))
old_log_stds_var = tf.placeholder(dtype=tf.float32, name="old_log_stds", shape=(None, output_dim))
x_mean_var = tf.Variable(
np.zeros((1,) + input_shape, dtype=np.float32),
name="x_mean",
)
x_std_var = tf.Variable(
np.ones((1,) + input_shape, dtype=np.float32),
name="x_std",
)
y_mean_var = tf.Variable(
np.zeros((1, output_dim), dtype=np.float32),
name="y_mean",
)
y_std_var = tf.Variable(
np.ones((1, output_dim), dtype=np.float32),
name="y_std",
)
self.x_mean_var = x_mean_var
self.x_std_var = x_std_var
self.y_mean_var = y_mean_var
self.y_std_var = y_std_var
# self.observation_space = observation_space
normalized_xs_var = (xs_var - x_mean_var) / x_std_var
normalized_ys_var = (ys_var - y_mean_var) / y_std_var
normalized_means_var = L.get_output(l_mean, {mean_network.input_layer: normalized_xs_var})
normalized_log_stds_var = L.get_output(l_log_std, {mean_network.input_layer: normalized_xs_var})
means_var = normalized_means_var * y_std_var + y_mean_var
log_stds_var = normalized_log_stds_var + tf.log(y_std_var)
normalized_old_means_var = (old_means_var - y_mean_var) / y_std_var
normalized_old_log_stds_var = old_log_stds_var - tf.log(y_std_var)
dist = self._dist = DiagonalGaussian(output_dim)
normalized_dist_info_vars = dict(mean=normalized_means_var, log_std=normalized_log_stds_var)
mean_kl = tf.reduce_mean(dist.kl_sym(
dict(mean=normalized_old_means_var, log_std=normalized_old_log_stds_var),
normalized_dist_info_vars,
))
loss = - tf.reduce_mean(dist.log_likelihood_sym(normalized_ys_var, normalized_dist_info_vars))
self._f_predict = tensor_utils.compile_function([xs_var], means_var)
self._f_pdists = tensor_utils.compile_function([xs_var], [means_var, log_stds_var])
self._l_mean = l_mean
self._l_log_std = l_log_std
optimizer_args = dict(
loss=loss,
target=self,
network_outputs=[normalized_means_var, normalized_log_stds_var],
)
if use_trust_region:
optimizer_args["leq_constraint"] = (mean_kl, step_size)
optimizer_args["inputs"] = [xs_var, ys_var, old_means_var, old_log_stds_var]
else:
optimizer_args["inputs"] = [xs_var, ys_var]
self._optimizer.update_opt(**optimizer_args)
self._use_trust_region = use_trust_region
self._name = name
self._normalize_inputs = normalize_inputs
self._normalize_outputs = normalize_outputs
self._mean_network = mean_network
self._x_mean_var = x_mean_var
self._x_std_var = x_std_var
self._y_mean_var = y_mean_var
self._y_std_var = y_std_var
self.input_dim = input_shape[0]
self.output_dim = output_dim
def __init__(
self,
env,
policy,
baseline,
scope=None,
n_itr=500,
start_itr=0,
# Note that the number of trajectories for grad upate = batch_size
# Defaults are 10 trajectories of length 500 for gradient update
batch_size=100,
max_path_length=500,
meta_batch_size = 100,
num_grad_updates=1,
discount=0.99,
gae_lambda=1,
plot=False,
pause_for_plot=False,
center_adv=True,
positive_adv=False,
store_paths=False,
whole_paths=True,
fixed_horizon=False,
sampler_cls=None,
sampler_args=None,
force_batch_sampler=False,
load_policy=None,
latent_dim=4,
num_total_tasks=100,
kl_weighting=0,
kl_scheme=None,
plottingFolder = None,
visitationFolder = None,
visitationFile = None,
load_policy_vals = None,
reset_step = 'False',
**kwargs
):
"""
:param env: Environment
:param policy: Policy
:type policy: Policy
:param baseline: Baseline
:param scope: Scope for identifying the algorithm. Must be specified if running multiple algorithms
simultaneously, each using different environments and policies
:param n_itr: Number of iterations.
:param start_itr: Starting iteration.
:param batch_size: Number of samples per iteration. #
:param max_path_length: Maximum length of a single rollout.
:param meta_batch_size: Number of tasks sampled per meta-update
:param num_grad_updates: Number of fast gradient updates
:param discount: Discount.
:param gae_lambda: Lambda used for generalized advantage estimation.
:param plot: Plot evaluation run after each iteration.
:param pause_for_plot: Whether to pause before contiuing when plotting.
:param center_adv: Whether to rescale the advantages so that they have mean 0 and standard deviation 1.
:param positive_adv: Whether to shift the advantages so that they are always positive. When used in
conjunction with center_adv the advantages will be standardized before shifting.
:param store_paths: Whether to save all paths data to the snapshot.
:return:
"""
self.env = env
self.latent_dim = latent_dim
self.kl_weighting = kl_weighting
self.kl_scheme = kl_scheme
self.num_total_tasks = num_total_tasks
self.policy = policy
self.load_policy=load_policy
self.baseline = baseline
self.scope = scope
self.n_itr = n_itr
self.start_itr = start_itr
# batch_size is the number of trajectories for one fast grad update.
# self.batch_size is the number of total transitions to collect.
self.num_trajs = batch_size
self.batch_size = batch_size * max_path_length * meta_batch_size
self.max_path_length = max_path_length
self.discount = discount
self.gae_lambda = gae_lambda
self.plot = plot
self.pause_for_plot = pause_for_plot
self.center_adv = center_adv
self.positive_adv = positive_adv
self.store_paths = store_paths
self.whole_paths = whole_paths
self.fixed_horizon = fixed_horizon
self.meta_batch_size = meta_batch_size # number of tasks
self.num_grad_updates = num_grad_updates # number of gradient steps during training
self.latent_dist = DiagonalGaussian(self.latent_dim)
self.plottingFolder = plottingFolder
self.visitationFolder = visitationFolder
self.visitationFile = visitationFile
self.load_policy_vals = load_policy_vals
self.reset_step = reset_step
if sampler_cls is None:
if singleton_pool.n_parallel > 1:
assert False , 'parallel sampling not implemented'
else:
sampler_cls = VectorizedSampler
if sampler_args is None:
sampler_args = dict()
sampler_args['n_envs'] = self.meta_batch_size
sampler_args['latent_dim'] = self.latent_dim
self.sampler = sampler_cls(self, **sampler_args)
def __init__(
self,
name,
input_shape,
output_dim,
mean_network=None,
hidden_sizes=(32, 32),
hidden_nonlinearity=tf.nn.tanh,
output_nonlinearity=lambda x: x * 0.0 + tf.Variable(
initial_value=-1.0, dtype=tf.float32),
# output_nonlinearity=tf.identity,
optimizer=None,
use_trust_region=True,
step_size=0.01,
learn_std=True,
init_std=1.0,
adaptive_std=False,
std_share_network=False,
std_hidden_sizes=(32, 32),
std_nonlinearity=None,
normalize_inputs=True,
normalize_outputs=True,
subsample_factor=1.0):
"""
:param input_shape: Shape of the input data.
:param output_dim: Dimension of output.
:param hidden_sizes: Number of hidden units of each layer of the mean network.
:param hidden_nonlinearity: Non-linearity used for each layer of the mean network.
:param optimizer: Optimizer for minimizing the negative log-likelihood.
:param use_trust_region: Whether to use trust region constraint.
:param step_size: KL divergence constraint for each iteration
:param learn_std: Whether to learn the standard deviations. Only effective if adaptive_std is False. If
adaptive_std is True, this parameter is ignored, and the weights for the std network are always learned.
:param adaptive_std: Whether to make the std a function of the states.
:param std_share_network: Whether to use the same network as the mean.
:param std_hidden_sizes: Number of hidden units of each layer of the std network. Only used if
`std_share_network` is False. It defaults to the same architecture as the mean.
:param std_nonlinearity: Non-linearity used for each layer of the std network. Only used if `std_share_network`
is False. It defaults to the same non-linearity as the mean.
"""
Serializable.quick_init(self, locals())
with tf.variable_scope(name):
if optimizer is None:
if use_trust_region:
optimizer = PenaltyLbfgsOptimizer("optimizer")
else:
optimizer = LbfgsOptimizer("optimizer")
self._optimizer = optimizer
self._subsample_factor = subsample_factor
if mean_network is None:
mean_network = create_MLP(
name="mean_network",
output_dim=1,
hidden_sizes=hidden_sizes,
hidden_nonlinearity=hidden_nonlinearity,
output_nonlinearity=output_nonlinearity,
)
forward_mean = lambda x, params, is_train: self.forward_MLP(
'mean_network',
all_params=params,
input_tensor=x,
is_training=is_train)[1]
else:
raise NotImplementedError('Not supported.')
# print("Debug2, mean network is defined here")
# mean_network = L.ParamLayer(
# incoming=L.InputLayer(
# shape=(None,) + input_shape,
# name="input_layer"),
# num_units=1,
# param=tf.constant_initializer(-200.0),
# name="mean_network",
# trainable=True,
# ),
# print(mean_network.input_layer)
# print("debug4", isinstance(L.InputLayer(
# shape=(None,) + input_shape,
# name="input_layer"), tuple))
#
# l_mean = mean_network
# mean_network = MLP(
# name="mean_network",
# input_shape=input_shape,
# output_dim=output_dim,
# hidden_sizes=hidden_sizes,
# hidden_nonlinearity=hidden_nonlinearity,
# output_nonlinearity=output_nonlinearity,
# )
#
# l_mean = mean_network.output_layer
if adaptive_std:
# l_log_std = MLP(
# name="log_std_network",
# input_shape=input_shape,
# input_var=mean_network.input_layer.input_var,
# output_dim=output_dim,
# hidden_sizes=std_hidden_sizes,
# hidden_nonlinearity=std_nonlinearity,
# output_nonlinearity=None,
# ).output_layer
raise NotImplementedError('Not supported.')
else:
# l_log_std = L.ParamLayer(
# mean_network.input_layer,
# num_units=output_dim,
# param=tf.constant_initializer(np.log(init_std)),
# name="output_log_std",
# trainable=learn_std,
# )
self.all_params['std_param'] = make_param_layer(
num_units=1,
param=tf.constant_initializer(init_std),
name="output_std_param",
trainable=learn_std,
)
forward_std = lambda x, params: forward_param_layer(
x, params['std_param'])
self.all_param_vals = None
LayersPowered.__init__(self, [l_mean, l_log_std])
xs_var = mean_network.input_layer.input_var
ys_var = tf.placeholder(dtype=tf.float32,
name="ys",
shape=(None, output_dim))
old_means_var = tf.placeholder(dtype=tf.float32,
name="ys",
shape=(None, output_dim))
old_log_stds_var = tf.placeholder(dtype=tf.float32,
name="old_log_stds",
shape=(None, output_dim))
x_mean_var = tf.Variable(
np.zeros((1, ) + input_shape, dtype=np.float32),
name="x_mean",
)
x_std_var = tf.Variable(
np.ones((1, ) + input_shape, dtype=np.float32),
name="x_std",
)
y_mean_var = tf.Variable(
np.zeros((1, output_dim), dtype=np.float32),
name="y_mean",
)
y_std_var = tf.Variable(
np.ones((1, output_dim), dtype=np.float32),
name="y_std",
)
normalized_xs_var = (xs_var - x_mean_var) / x_std_var
normalized_ys_var = (ys_var - y_mean_var) / y_std_var
normalized_means_var = L.get_output(
l_mean, {mean_network.input_layer: normalized_xs_var})
normalized_log_stds_var = L.get_output(
l_log_std, {mean_network.input_layer: normalized_xs_var})
means_var = normalized_means_var * y_std_var + y_mean_var
log_stds_var = normalized_log_stds_var + tf.log(y_std_var)
normalized_old_means_var = (old_means_var - y_mean_var) / y_std_var
normalized_old_log_stds_var = old_log_stds_var - tf.log(y_std_var)
## code added for symbolic prediction, used in constructing the meta-learning objective
def normalized_means_var_sym(xs, params):
inputs = OrderedDict({mean_network.input_layer: xs})
inputs.update(params)
return L.get_output(layer_or_layers=l_mean, inputs=inputs)
# normalized_means_var_sym = lambda xs, params: L.get_output(layer_or_layers=l_mean, inputs=OrderedDict({mean_network.input_layer:xs}.) #mean_network.input_layer: (xs-x_mean_var)/x_std_var,
# normalized_log_stds_var_sym = L.get_output(l_log_std, {mean_network.input_layer: normalized_xs_var})
means_var_sym = lambda xs, params: normalized_means_var_sym(
xs=xs, params=params) * y_std_var + y_mean_var
# log_stds_var = normalized_log_stds_var + tf.log(y_std_var)
dist = self._dist = DiagonalGaussian(output_dim)
normalized_dist_info_vars = dict(mean=normalized_means_var,
log_std=normalized_log_stds_var)
mean_kl = tf.reduce_mean(
dist.kl_sym(
dict(mean=normalized_old_means_var,
log_std=normalized_old_log_stds_var),
normalized_dist_info_vars,
))
# loss = - tf.reduce_mean(dist.log_likelihood_sym(normalized_ys_var, normalized_dist_info_vars))
loss = tf.nn.l2_loss(normalized_ys_var - normalized_means_var
) + tf.nn.l2_loss(normalized_log_stds_var)
self._f_predict = tensor_utils.compile_function([xs_var],
means_var)
self._f_pdists = tensor_utils.compile_function(
[xs_var], [means_var, log_stds_var])
self._l_mean = l_mean
self._l_log_std = l_log_std
self._f_predict_sym = means_var_sym
self.loss_sym = loss
optimizer_args = dict(
loss=loss,
target=self,
network_outputs=[
normalized_means_var, normalized_log_stds_var
],
)
if use_trust_region:
optimizer_args["leq_constraint"] = (mean_kl, step_size)
optimizer_args["inputs"] = [
xs_var, ys_var, old_means_var, old_log_stds_var
]
else:
optimizer_args["inputs"] = [xs_var, ys_var]
self._optimizer.update_opt(**optimizer_args)
self._use_trust_region = use_trust_region
self._name = name
self._normalize_inputs = normalize_inputs
self._normalize_outputs = normalize_outputs
self._mean_network = mean_network
self._x_mean_var = x_mean_var
self._x_std_var = x_std_var
self._y_mean_var = y_mean_var
self._y_std_var = y_std_var
def __init__(self,
name,
env_spec,
hidden_sizes=(32, 32),
learn_std=True,
init_std=1.0,
adaptive_std=False,
std_share_network=False,
std_hidden_sizes=(32, 32),
min_std=1e-6,
std_hidden_nonlinearity=tf.nn.tanh,
hidden_nonlinearity=tf.nn.tanh,
output_nonlinearity=tf.identity,
mean_network=None,
std_network=None,
std_parametrization='exp'):
"""
:param env_spec:
:param hidden_sizes: list of sizes for the fully-connected hidden layers
:param learn_std: Is std trainable
:param init_std: Initial std
:param adaptive_std:
:param std_share_network:
:param std_hidden_sizes: list of sizes for the fully-connected layers for std
:param min_std: whether to make sure that the std is at least some threshold value, to avoid numerical issues
:param std_hidden_nonlinearity:
:param hidden_nonlinearity: nonlinearity used for each hidden layer
:param output_nonlinearity: nonlinearity for the output layer
:param mean_network: custom network for the output mean
:param std_network: custom network for the output log std
:param std_parametrization: how the std should be parametrized. There are a few options:
- exp: the logarithm of the std will be stored, and applied a exponential transformation
- softplus: the std will be computed as log(1+exp(x))
:return:
"""
Serializable.quick_init(self, locals())
assert isinstance(env_spec.action_space, Box)
obs_dim = env_spec.observation_space.flat_dim
action_dim = env_spec.action_space.flat_dim
self.all_param_vals = False
print('obs_dim ', obs_dim, flush=True)
# create network
if mean_network is None:
self.mean_params = mean_params = self.create_MLP(
name="mean_network",
input_shape=(
None,
obs_dim,
),
output_dim=action_dim,
hidden_sizes=hidden_sizes,
)
input_tensor, mean_tensor = self.forward_MLP(
'mean_network',
mean_params,
n_hidden=len(hidden_sizes),
input_shape=(obs_dim, ),
hidden_nonlinearity=hidden_nonlinearity,
output_nonlinearity=output_nonlinearity,
reuse=None # Needed for batch norm
)
# if you want to input your own thing.
self._forward_mean = lambda x, is_train: self.forward_MLP(
'mean_network',
mean_params,
n_hidden=len(hidden_sizes),
hidden_nonlinearity=hidden_nonlinearity,
output_nonlinearity=output_nonlinearity,
input_tensor=x,
is_training=is_train)[1]
else:
raise NotImplementedError('Chelsea does not support this.')
if std_network is not None:
raise NotImplementedError(
'Minimal Gaussian MLP does not support this.')
else:
if adaptive_std:
# NOTE - this branch isn't tested
raise NotImplementedError(
'Minimal Gaussian MLP doesnt have a tested version of this.'
)
self.std_params = std_params = self.create_MLP(
name="std_network",
input_shape=(
None,
obs_dim,
),
output_dim=action_dim,
hidden_sizes=std_hidden_sizes,
)
# if you want to input your own thing.
self._forward_std = lambda x: self.forward_MLP(
'std_network',
std_params,
n_hidden=len(hidden_sizes),
hidden_nonlinearity=std_hidden_nonlinearity,
output_nonlinearity=tf.identity,
input_tensor=x)[1]
else:
if std_parametrization == 'exp':
init_std_param = np.log(init_std)
elif std_parametrization == 'softplus':
init_std_param = np.log(np.exp(init_std) - 1)
else:
raise NotImplementedError
self.std_params = make_param_layer(
num_units=action_dim,
param=tf.constant_initializer(init_std_param),
name="output_std_param",
trainable=learn_std,
)
self._forward_std = lambda x: forward_param_layer(
x, self.std_params)
self.std_parametrization = std_parametrization
if std_parametrization == 'exp':
min_std_param = np.log(min_std)
elif std_parametrization == 'softplus':
min_std_param = np.log(np.exp(min_std) - 1)
else:
raise NotImplementedError
self.min_std_param = min_std_param
self._dist = DiagonalGaussian(action_dim)
self._cached_params = {}
super(GaussianMLPPolicy, self).__init__(env_spec)
dist_info_sym = self.dist_info_sym(input_tensor,
dict(),
is_training=False)
mean_var = dist_info_sym["mean"]
log_std_var = dist_info_sym["log_std"]
self._f_dist = tensor_utils.compile_function(
inputs=[input_tensor],
outputs=[mean_var, log_std_var],
)
class GaussianMLPPolicy(StochasticPolicy, Serializable):
def __init__(self,
name,
env_spec,
hidden_sizes=(32, 32),
learn_std=True,
init_std=1.0,
adaptive_std=False,
std_share_network=False,
std_hidden_sizes=(32, 32),
min_std=1e-6,
std_hidden_nonlinearity=tf.nn.tanh,
hidden_nonlinearity=tf.nn.tanh,
output_nonlinearity=tf.identity,
mean_network=None,
std_network=None,
std_parametrization='exp'):
"""
:param env_spec:
:param hidden_sizes: list of sizes for the fully-connected hidden layers
:param learn_std: Is std trainable
:param init_std: Initial std
:param adaptive_std:
:param std_share_network:
:param std_hidden_sizes: list of sizes for the fully-connected layers for std
:param min_std: whether to make sure that the std is at least some threshold value, to avoid numerical issues
:param std_hidden_nonlinearity:
:param hidden_nonlinearity: nonlinearity used for each hidden layer
:param output_nonlinearity: nonlinearity for the output layer
:param mean_network: custom network for the output mean
:param std_network: custom network for the output log std
:param std_parametrization: how the std should be parametrized. There are a few options:
- exp: the logarithm of the std will be stored, and applied a exponential transformation
- softplus: the std will be computed as log(1+exp(x))
:return:
"""
Serializable.quick_init(self, locals())
assert isinstance(env_spec.action_space, Box)
obs_dim = env_spec.observation_space.flat_dim
action_dim = env_spec.action_space.flat_dim
# create network
self.varScope = name
with tf.variable_scope(name):
if mean_network is None:
self.mean_params = mean_params = self.create_MLP(
name="mean_network",
input_shape=(
None,
obs_dim,
),
output_dim=action_dim,
hidden_sizes=hidden_sizes,
)
input_tensor, mean_tensor = self.forward_MLP(
'mean_network',
mean_params,
n_hidden=len(hidden_sizes),
input_shape=(obs_dim, ),
hidden_nonlinearity=hidden_nonlinearity,
output_nonlinearity=output_nonlinearity,
reuse=None # Needed for batch norm
)
# if you want to input your own thing.
self._forward_mean = lambda x, is_train: self.forward_MLP(
'mean_network',
mean_params,
n_hidden=len(hidden_sizes),
hidden_nonlinearity=hidden_nonlinearity,
output_nonlinearity=output_nonlinearity,
input_tensor=x,
is_training=is_train)[1]
else:
raise NotImplementedError('Chelsea does not support this.')
if std_network is not None:
raise NotImplementedError(
'Minimal Gaussian MLP does not support this.')
else:
if adaptive_std:
# NOTE - this branch isn't tested
raise NotImplementedError(
'Minimal Gaussian MLP doesnt have a tested version of this.'
)
self.std_params = std_params = self.create_MLP(
name="std_network",
input_shape=(
None,
obs_dim,
),
output_dim=action_dim,
hidden_sizes=std_hidden_sizes,
)
# if you want to input your own thing.
self._forward_std = lambda x: self.forward_MLP(
'std_network',
std_params,
n_hidden=len(hidden_sizes),
hidden_nonlinearity=std_hidden_nonlinearity,
output_nonlinearity=tf.identity,
input_tensor=x)[1]
else:
if std_parametrization == 'exp':
init_std_param = np.log(init_std)
elif std_parametrization == 'softplus':
init_std_param = np.log(np.exp(init_std) - 1)
else:
raise NotImplementedError
self.std_params = make_param_layer(
num_units=action_dim,
param=tf.constant_initializer(init_std_param),
name="output_std_param",
trainable=learn_std,
)
self._forward_std = lambda x: forward_param_layer(
x, self.std_params)
self.std_parametrization = std_parametrization
if std_parametrization == 'exp':
min_std_param = np.log(min_std)
elif std_parametrization == 'softplus':
min_std_param = np.log(np.exp(min_std) - 1)
else:
raise NotImplementedError
self.min_std_param = min_std_param
self._dist = DiagonalGaussian(action_dim)
self._cached_params = {}
super(GaussianMLPPolicy, self).__init__(env_spec)
dist_info_sym = self.dist_info_sym(input_tensor,
dict(),
is_training=False)
mean_var = dist_info_sym["mean"]
log_std_var = dist_info_sym["log_std"]
self._f_dist = tensor_utils.compile_function(
inputs=[input_tensor],
outputs=[mean_var, log_std_var],
)
@property
def vectorized(self):
return True
def compute_stateAction_log_likelihood(self, observations, actions):
flat_obs = self.observation_space.flatten_n(observations)
means, log_stds = self._f_dist(
flat_obs) # _f_dist runs the network forward.
return self._dist.log_likelihood(actions, {
"mean": means,
"log_std": log_stds
})
# rnd = np.random.normal(size=means.shape)
# actions = rnd * np.exp(log_stds) + means
# return actions, dict(mean=means, log_std=log_stds)
def dist_info_sym(self, obs_var, state_info_vars=None, is_training=True):
# This function constructs the tf graph, only called during beginning of training
# obs_var - observation tensor
# mean_var - tensor for policy mean
# std_param_var - tensor for policy std before output
mean_var = self._forward_mean(obs_var, is_training)
std_param_var = self._forward_std(obs_var)
if self.min_std_param is not None:
std_param_var = tf.maximum(std_param_var, self.min_std_param)
if self.std_parametrization == 'exp':
log_std_var = std_param_var
elif self.std_parametrization == 'softplus':
log_std_var = tf.log(tf.log(1. + tf.exp(std_param_var)))
else:
raise NotImplementedError
return dict(mean=mean_var, log_std=log_std_var)
@overrides
def get_action(self, observation):
flat_obs = self.observation_space.flatten(observation)
mean, log_std = [x[0] for x in self._f_dist([flat_obs])]
rnd = np.random.normal(size=mean.shape)
action = rnd * np.exp(log_std) + mean
return action, dict(mean=mean, log_std=log_std)
def get_actions(self, observations):
flat_obs = self.observation_space.flatten_n(observations)
means, log_stds = self._f_dist(
flat_obs) # _f_dist runs the network forward.
rnd = np.random.normal(size=means.shape)
actions = rnd * np.exp(log_stds) + means
return actions, dict(mean=means, log_std=log_stds)
@property
def distribution(self):
return self._dist
def get_params_internal(self, **tags):
if tags.get('trainable', False):
params = tf.trainable_variables()
else:
params = tf.all_variables()
# TODO - this is hacky...
params = [
p for p in params
if p.name.startswith(self.varScope + '/mean_network')
or p.name.startswith(self.varScope + '/output_std_param')
]
params = [p for p in params if 'Adam' not in p.name]
return params
# This makes all of the parameters.
def create_MLP(
self,
name,
output_dim,
hidden_sizes,
hidden_W_init=L.XavierUniformInitializer(),
hidden_b_init=tf.zeros_initializer,
output_W_init=L.XavierUniformInitializer(),
output_b_init=tf.zeros_initializer,
input_shape=None,
weight_normalization=False,
):
assert input_shape is not None
cur_shape = input_shape
with tf.variable_scope(name):
all_params = {}
for idx, hidden_size in enumerate(hidden_sizes):
W, b, cur_shape = make_dense_layer(
cur_shape,
num_units=hidden_size,
name="hidden_%d" % idx,
W=hidden_W_init,
b=hidden_b_init,
weight_norm=weight_normalization,
)
all_params['W' + str(idx)] = W
all_params['b' + str(idx)] = b
W, b, _ = make_dense_layer(
cur_shape,
num_units=output_dim,
name='output',
W=output_W_init,
b=output_b_init,
weight_norm=weight_normalization,
)
all_params['W' + str(len(hidden_sizes))] = W
all_params['b' + str(len(hidden_sizes))] = b
return all_params
def forward_MLP(self,
name,
all_params,
input_tensor=None,
input_shape=None,
n_hidden=-1,
hidden_nonlinearity=tf.identity,
output_nonlinearity=tf.identity,
batch_normalization=False,
reuse=True,
is_training=False):
# is_training and reuse are for batch norm, irrelevant if batch_norm set to False
# set reuse to False if the first time this func is called.
with tf.variable_scope(name):
if input_tensor is None:
assert input_shape is not None
l_in = make_input(shape=(None, ) + input_shape,
input_var=None,
name='input')
else:
l_in = input_tensor
l_hid = l_in
for idx in range(n_hidden):
l_hid = forward_dense_layer(l_hid,
all_params['W' + str(idx)],
all_params['b' + str(idx)],
batch_norm=batch_normalization,
nonlinearity=hidden_nonlinearity,
scope=str(idx),
reuse=reuse,
is_training=is_training)
output = forward_dense_layer(
l_hid,
all_params['W' + str(n_hidden)],
all_params['b' + str(n_hidden)],
batch_norm=False,
nonlinearity=output_nonlinearity,
)
return l_in, output
def get_params(self, **tags):
"""
Get the list of parameters (symbolically), filtered by the provided tags.
Some common tags include 'regularizable' and 'trainable'
"""
tag_tuple = tuple(sorted(list(tags.items()), key=lambda x: x[0]))
if tag_tuple not in self._cached_params:
self._cached_params[tag_tuple] = self.get_params_internal(**tags)
return self._cached_params[tag_tuple]
def get_param_values(self, **tags):
params = self.get_params(**tags)
param_values = tf.get_default_session().run(params)
return flatten_tensors(param_values)
def log_diagnostics(self, paths):
log_stds = np.vstack(
[path["agent_infos"]["log_std"] for path in paths])
logger.record_tabular('AveragePolicyStd', np.mean(np.exp(log_stds)))
#### code not used after here, uunless loading policy ####
def get_reparam_action_sym(self, obs_var, action_var, old_dist_info_vars):
"""
Given observations, old actions, and distribution of old actions, return a symbolically reparameterized
representation of the actions in terms of the policy parameters
:param obs_var:
:param action_var:
:param old_dist_info_vars:
:return:
"""
# Not used
print(
'--this really shouldnt be used, not updated from non-minimal policy--'
)
new_dist_info_vars = self.dist_info_sym(obs_var, action_var)
new_mean_var, new_log_std_var = new_dist_info_vars[
"mean"], new_dist_info_vars["log_std"]
old_mean_var, old_log_std_var = old_dist_info_vars[
"mean"], old_dist_info_vars["log_std"]
epsilon_var = (action_var - old_mean_var) / (tf.exp(old_log_std_var) +
1e-8)
new_action_var = new_mean_var + epsilon_var * tf.exp(new_log_std_var)
return new_action_var
def get_param_dtypes(self, **tags):
# Not used.
tag_tuple = tuple(sorted(list(tags.items()), key=lambda x: x[0]))
if tag_tuple not in self._cached_param_dtypes:
params = self.get_params(**tags)
param_values = tf.get_default_session().run(params)
self._cached_param_dtypes[tag_tuple] = [
val.dtype for val in param_values
]
return self._cached_param_dtypes[tag_tuple]
def get_param_shapes(self, **tags):
tag_tuple = tuple(sorted(list(tags.items()), key=lambda x: x[0]))
if tag_tuple not in self._cached_param_shapes:
params = self.get_params(**tags)
param_values = tf.get_default_session().run(params)
self._cached_param_shapes[tag_tuple] = [
val.shape for val in param_values
]
return self._cached_param_shapes[tag_tuple]
def set_param_values(self, flattened_params, **tags):
debug = tags.pop("debug", False)
param_values = unflatten_tensors(flattened_params,
self.get_param_shapes(**tags))
ops = []
feed_dict = dict()
for param, dtype, value in zip(self.get_params(**tags),
self.get_param_dtypes(**tags),
param_values):
if param not in self._cached_assign_ops:
assign_placeholder = tf.placeholder(
dtype=param.dtype.base_dtype)
assign_op = tf.assign(param, assign_placeholder)
self._cached_assign_ops[param] = assign_op
self._cached_assign_placeholders[param] = assign_placeholder
ops.append(self._cached_assign_ops[param])
feed_dict[self._cached_assign_placeholders[param]] = value.astype(
dtype)
if debug:
print("setting value of %s" % param.name)
tf.get_default_session().run(ops, feed_dict=feed_dict)
def flat_to_params(self, flattened_params, **tags):
return unflatten_tensors(flattened_params,
self.get_param_shapes(**tags))
def __getstate__(self):
d = Serializable.__getstate__(self)
global load_params
if load_params:
d["params"] = self.get_param_values()
return d
def __setstate__(self, d):
Serializable.__setstate__(self, d)
global load_params
if load_params:
tf.get_default_session().run(
tf.initialize_variables(self.get_params()))
self.set_param_values(d["params"])
def __init__(self,
env,
policy,
baseline,
scope=None,
n_itr=500,
start_itr=0,
batch_size=5000,
max_path_length=500,
discount=0.99,
gae_lambda=1,
plot=False,
pause_for_plot=False,
center_adv=True,
positive_adv=False,
store_paths=False,
whole_paths=True,
fixed_horizon=False,
sampler_cls=None,
sampler_args=None,
force_batch_sampler=False,
load_policy=None,
reset_arg=None,
latent_dim=4,
num_total_tasks=10,
noise_opt=False,
joint_opt=False,
improve=False,
**kwargs):
"""
:param env: Environment
:param policy: Policy
:type policy: Policy
:param baseline: Baseline
:param scope: Scope for identifying the algorithm. Must be specified if running multiple algorithms
simultaneously, each using different environments and policies
:param n_itr: Number of iterations.
:param start_itr: Starting iteration.
:param batch_size: Number of samples per iteration.
:param max_path_length: Maximum length of a single rollout.
:param discount: Discount.
:param gae_lambda: Lambda used for generalized advantage estimation.
:param plot: Plot evaluation run after each iteration.
:param pause_for_plot: Whether to pause before contiuing when plotting.
:param center_adv: Whether to rescale the advantages so that they have mean 0 and standard deviation 1.
:param positive_adv: Whether to shift the advantages so that they are always positive. When used in
conjunction with center_adv the advantages will be standardized before shifting.
:param store_paths: Whether to save all paths data to the snapshot.
:return:
"""
self.env = env
self.noise_opt = noise_opt
self.joint_opt = joint_opt
self.latent_dim = latent_dim
self.num_total_tasks = num_total_tasks
self.policy = policy
self.load_policy = load_policy
self.baseline = baseline
self.scope = scope
self.n_itr = n_itr
self.start_itr = start_itr
self.batch_size = batch_size
self.max_path_length = max_path_length
self.discount = discount
self.gae_lambda = gae_lambda
self.plot = plot
self.pause_for_plot = pause_for_plot
self.center_adv = center_adv
self.positive_adv = positive_adv
self.store_paths = store_paths
self.whole_paths = whole_paths
self.fixed_horizon = fixed_horizon
self.improve = improve
if sampler_cls is None:
#sampler_cls = VectorizedSampler
sampler_cls = BatchSampler
if sampler_args is None:
sampler_args = dict()
self.sampler = sampler_cls(self, **sampler_args)
self.reset_arg = reset_arg
self.latent_dist = DiagonalGaussian(self.latent_dim)
class MAMLGaussianMLPPolicy(StochasticPolicy, Serializable):
def __init__(
self,
name,
env_spec,
hidden_sizes=(32, 32),
learn_std=True,
init_std=1.0,
adaptive_std=False,
std_share_network=False,
std_hidden_sizes=(32, 32),
min_std=1e-6,
std_hidden_nonlinearity=tf.nn.tanh,
hidden_nonlinearity=tf.nn.tanh,
output_nonlinearity=tf.identity,
mean_network=None,
std_network=None,
std_parametrization='exp',
grad_step_size=1.0,
stop_grad=False,
):
"""
:param env_spec:
:param hidden_sizes: list of sizes for the fully-connected hidden layers
:param learn_std: Is std trainable
:param init_std: Initial std
:param adaptive_std:
:param std_share_network:
:param std_hidden_sizes: list of sizes for the fully-connected layers for std
:param min_std: whether to make sure that the std is at least some threshold value, to avoid numerical issues
:param std_hidden_nonlinearity:
:param hidden_nonlinearity: nonlinearity used for each hidden layer
:param output_nonlinearity: nonlinearity for the output layer
:param mean_network: custom network for the output mean
:param std_network: custom network for the output log std
:param std_parametrization: how the std should be parametrized. There are a few options:
- exp: the logarithm of the std will be stored, and applied a exponential transformation
- softplus: the std will be computed as log(1+exp(x))
:param grad_step_size: the step size taken in the learner's gradient update, sample uniformly if it is a range e.g. [0.1,1]
:param stop_grad: whether or not to stop the gradient through the gradient.
:return:
"""
Serializable.quick_init(self, locals())
assert isinstance(env_spec.action_space, Box)
obs_dim = env_spec.observation_space.flat_dim
self.action_dim = env_spec.action_space.flat_dim
self.n_hidden = len(hidden_sizes)
self.hidden_nonlinearity = hidden_nonlinearity
self.output_nonlinearity = output_nonlinearity
self.input_shape = (None, obs_dim,)
self.step_size = grad_step_size
self.stop_grad = stop_grad
if type(self.step_size) == list:
raise NotImplementedError('removing this since it didnt work well')
# create network
if mean_network is None:
self.all_params = self.create_MLP( # TODO: this should not be a method of the policy! --> helper
name="mean_network",
output_dim=self.action_dim,
hidden_sizes=hidden_sizes,
)
self.input_tensor, _ = self.forward_MLP('mean_network', self.all_params,
reuse=None # Need to run this for batch norm
)
forward_mean = lambda x, params, is_train: self.forward_MLP('mean_network', params,
input_tensor=x, is_training=is_train)[1]
else:
raise NotImplementedError('Not supported.')
if std_network is not None:
raise NotImplementedError('Not supported.')
else:
if adaptive_std:
raise NotImplementedError('Not supported.')
else:
if std_parametrization == 'exp':
init_std_param = np.log(init_std)
elif std_parametrization == 'softplus':
init_std_param = np.log(np.exp(init_std) - 1)
else:
raise NotImplementedError
self.all_params['std_param'] = make_param_layer(
num_units=self.action_dim,
param=tf.constant_initializer(init_std_param),
name="output_std_param",
trainable=learn_std,
)
forward_std = lambda x, params: forward_param_layer(x, params['std_param'])
self.all_param_vals = None
# unify forward mean and forward std into a single function
self._forward = lambda obs, params, is_train: (
forward_mean(obs, params, is_train), forward_std(obs, params))
self.std_parametrization = std_parametrization
if std_parametrization == 'exp':
min_std_param = np.log(min_std)
elif std_parametrization == 'softplus':
min_std_param = np.log(np.exp(min_std) - 1)
else:
raise NotImplementedError
self.min_std_param = min_std_param
self._dist = DiagonalGaussian(self.action_dim)
self._cached_params = {}
super(MAMLGaussianMLPPolicy, self).__init__(env_spec)
dist_info_sym = self.dist_info_sym(self.input_tensor, dict(), is_training=False)
mean_var = dist_info_sym["mean"]
log_std_var = dist_info_sym["log_std"]
# pre-update policy
self._init_f_dist = tensor_utils.compile_function(
inputs=[self.input_tensor],
outputs=[mean_var, log_std_var],
)
self._cur_f_dist = self._init_f_dist
@property
def vectorized(self):
return True
def set_init_surr_obj(self, input_list, surr_objs_tensor):
""" Set the surrogate objectives used the update the policy
"""
self.input_list_for_grad = input_list
self.surr_objs = surr_objs_tensor
def compute_updated_dists(self, samples):
""" Compute fast gradients once per iteration and pull them out of tensorflow for sampling with the post-update policy.
"""
start = time.time()
num_tasks = len(samples)
param_keys = self.all_params.keys()
update_param_keys = param_keys
no_update_param_keys = []
sess = tf.get_default_session()
obs_list, action_list, adv_list = [], [], []
for i in range(num_tasks):
inputs = ext.extract(samples[i],
'observations', 'actions', 'advantages')
obs_list.append(inputs[0])
action_list.append(inputs[1])
adv_list.append(inputs[2])
inputs = obs_list + action_list + adv_list
# To do a second update, replace self.all_params below with the params that were used to collect the policy.
init_param_values = None
if self.all_param_vals is not None:
init_param_values = self.get_variable_values(self.all_params)
step_size = self.step_size
for i in range(num_tasks):
if self.all_param_vals is not None:
self.assign_params(self.all_params, self.all_param_vals[i])
if 'all_fast_params_tensor' not in dir(self):
# make computation graph once
self.all_fast_params_tensor = []
for i in range(num_tasks):
gradients = dict(zip(update_param_keys, tf.gradients(self.surr_objs[i], [self.all_params[key] for key in update_param_keys])))
fast_params_tensor = OrderedDict(zip(update_param_keys, [self.all_params[key] - step_size*gradients[key] for key in update_param_keys]))
for k in no_update_param_keys:
fast_params_tensor[k] = self.all_params[k]
self.all_fast_params_tensor.append(fast_params_tensor)
# pull new param vals out of tensorflow, so gradient computation only done once ## first is the vars, second the values
# these are the updated values of the params after the gradient step
self.all_param_vals = sess.run(self.all_fast_params_tensor, feed_dict=dict(list(zip(self.input_list_for_grad, inputs))))
if init_param_values is not None:
self.assign_params(self.all_params, init_param_values)
outputs = []
self._cur_f_dist_i = {}
inputs = tf.split(self.input_tensor, num_tasks, 0)
for i in range(num_tasks):
# TODO - use a placeholder to feed in the params, so that we don't have to recompile every time.
task_inp = inputs[i]
info, _ = self.dist_info_sym(task_inp, dict(), all_params=self.all_param_vals[i],
is_training=False)
outputs.append([info['mean'], info['log_std']])
self._cur_f_dist = tensor_utils.compile_function(
inputs = [self.input_tensor],
outputs = outputs,
)
total_time = time.time() - start
logger.record_tabular("ComputeUpdatedDistTime", total_time)
def get_variable_values(self, tensor_dict):
sess = tf.get_default_session()
result = sess.run(tensor_dict)
return result
def assign_params(self, tensor_dict, param_values):
if 'assign_placeholders' not in dir(self):
# make computation graph, if it doesn't exist; then cache it for future use.
self.assign_placeholders = {}
self.assign_ops = {}
for key in tensor_dict.keys():
self.assign_placeholders[key] = tf.placeholder(tf.float32)
self.assign_ops[key] = tf.assign(tensor_dict[key], self.assign_placeholders[key])
feed_dict = {self.assign_placeholders[key]:param_values[key] for key in tensor_dict.keys()}
sess = tf.get_default_session()
sess.run(self.assign_ops, feed_dict)
def switch_to_init_dist(self):
# switch cur policy distribution to pre-update policy
self._cur_f_dist = self._init_f_dist
self._cur_f_dist_i = None
self.all_param_vals = None
def dist_info_sym(self, obs_var, state_info_vars=None, all_params=None, is_training=True):
# This function constructs the tf graph, only called during beginning of meta-training
# obs_var - observation tensor
# mean_var - tensor for policy mean
# std_param_var - tensor for policy std before output
return_params=True
if all_params is None:
return_params=False
all_params = self.all_params
mean_var, std_param_var = self._forward(obs_var, all_params, is_training)
if self.min_std_param is not None:
std_param_var = tf.maximum(std_param_var, self.min_std_param)
if self.std_parametrization == 'exp':
log_std_var = std_param_var
elif self.std_parametrization == 'softplus':
log_std_var = tf.log(tf.log(1. + tf.exp(std_param_var)))
else:
raise NotImplementedError
if return_params:
return dict(mean=mean_var, log_std=log_std_var), all_params
else:
return dict(mean=mean_var, log_std=log_std_var)
def updated_dist_info_sym(self, task_id, surr_obj, new_obs_var, params_dict=None, is_training=True):
""" symbolically create MAML graph, for the meta-optimization, only called at the beginning of meta-training.
Called more than once if you want to do more than one grad step.
"""
old_params_dict = params_dict
step_size = self.step_size
if old_params_dict == None:
old_params_dict = self.all_params
param_keys = self.all_params.keys()
update_param_keys = param_keys
no_update_param_keys = []
grads = tf.gradients(surr_obj, [old_params_dict[key] for key in update_param_keys])
if self.stop_grad:
grads = [tf.stop_gradient(grad) for grad in grads]
gradients = dict(zip(update_param_keys, grads))
params_dict = dict(zip(update_param_keys, [old_params_dict[key] - step_size*gradients[key] for key in update_param_keys]))
for k in no_update_param_keys:
params_dict[k] = old_params_dict[k]
return self.dist_info_sym(new_obs_var, all_params=params_dict, is_training=is_training)
def get_deterministic_action(self, observation, idx=None):
# this function takes a numpy array observations and outputs randomly sampled actions.
# idx: index corresponding to the task/updated policy.
flat_obs = self.observation_space.flatten(observation)
f_dist = self._cur_f_dist
mean, log_std = [x[0] for x in f_dist([flat_obs])]
rnd = np.random.normal(size=mean.shape)
action = mean
return action, dict(mean=mean, log_std=log_std)
@overrides
def get_action(self, observation, idx=None):
# this function takes a numpy array observations and outputs randomly sampled actions.
# idx: index corresponding to the task/updated policy.
flat_obs = self.observation_space.flatten(observation)
f_dist = self._cur_f_dist
mean, log_std = [x[0] for x in f_dist([flat_obs])]
rnd = np.random.normal(size=mean.shape)
action = rnd * np.exp(log_std) + mean
return action, dict(mean=mean, log_std=log_std)
def get_actions(self, observations):
# this function takes a numpy array observations and outputs sampled actions.
# Assumes that there is one observation per post-update policy distr
flat_obs = self.observation_space.flatten_n(observations)
result = self._cur_f_dist(flat_obs)
if len(result) == 2:
# NOTE - this code assumes that there aren't 2 meta tasks in a batch
means, log_stds = result
else:
means = np.array([res[0] for res in result])[:,0,:]
log_stds = np.array([res[1] for res in result])[:,0,:]
rnd = np.random.normal(size=means.shape)
actions = rnd * np.exp(log_stds) + means
return actions, dict(mean=means, log_std=log_stds)
def compute_preUpdatePolicy_logLikelihood(self, observations ,actions):
# this function takes a numpy array observations and outputs sampled actions.
# Assumes that there is one observation per post-update policy distr
flat_obs = self.observation_space.flatten_n(observations)
means, log_stds = self._cur_f_dist(flat_obs)
result = self._dist.log_likelihood(actions, {"mean" : means , "log_std" : log_stds})
return result
@property
def distribution(self):
return self._dist
def get_params_internal(self, all_params=False, **tags):
if tags.get('trainable', False):
params = tf.trainable_variables()
else:
params = tf.global_variables()
params = [p for p in params if p.name.startswith('mean_network') or p.name.startswith('output_std_param')]
params = [p for p in params if 'Adam' not in p.name]
return params
# This makes all of the parameters.
def create_MLP(self, name, output_dim, hidden_sizes,
hidden_W_init=tf_layers.xavier_initializer(), hidden_b_init=tf.zeros_initializer(),
output_W_init=tf_layers.xavier_initializer(), output_b_init=tf.zeros_initializer(),
weight_normalization=False,
):
all_params = OrderedDict()
cur_shape = self.input_shape
with tf.variable_scope(name):
for idx, hidden_size in enumerate(hidden_sizes):
W, b, cur_shape = make_dense_layer(
cur_shape,
num_units=hidden_size,
name="hidden_%d" % idx,
W=hidden_W_init,
b=hidden_b_init,
weight_norm=weight_normalization,
)
all_params['W' + str(idx)] = W
all_params['b' + str(idx)] = b
W, b, _ = make_dense_layer(
cur_shape,
num_units=output_dim,
name='output',
W=output_W_init,
b=output_b_init,
weight_norm=weight_normalization,
)
all_params['W' + str(len(hidden_sizes))] = W
all_params['b'+str(len(hidden_sizes))] = b
return all_params
def forward_MLP(self, name, all_params, input_tensor=None,
batch_normalization=False, reuse=True, is_training=False):
# is_training and reuse are for batch norm, irrelevant if batch_norm set to False
# set reuse to False if the first time this func is called.
with tf.variable_scope(name):
if input_tensor is None:
l_in = make_input(shape=self.input_shape, input_var=None, name='input')
else:
l_in = input_tensor
l_hid = l_in
for idx in range(self.n_hidden):
l_hid = forward_dense_layer(l_hid, all_params['W'+str(idx)], all_params['b'+str(idx)],
batch_norm=batch_normalization,
nonlinearity=self.hidden_nonlinearity,
scope=str(idx), reuse=reuse,
is_training=is_training
)
output = forward_dense_layer(l_hid, all_params['W'+str(self.n_hidden)], all_params['b'+str(self.n_hidden)],
batch_norm=False, nonlinearity=self.output_nonlinearity,
)
return l_in, output
def get_params(self, all_params=False, **tags):
"""
Get the list of parameters (symbolically), filtered by the provided tags.
Some common tags include 'regularizable' and 'trainable'
"""
tag_tuple = tuple(sorted(list(tags.items()), key=lambda x: x[0]))
if tag_tuple not in self._cached_params:
self._cached_params[tag_tuple] = self.get_params_internal(all_params, **tags)
return self._cached_params[tag_tuple]
def get_param_values(self, all_params=False, **tags):
params = self.get_params(all_params, **tags)
param_values = tf.get_default_session().run(params)
return flatten_tensors(param_values)
def log_diagnostics(self, paths, prefix=''):
log_stds = np.vstack([path["agent_infos"]["log_std"] for path in paths])
logger.record_tabular(prefix+'AveragePolicyStd', np.mean(np.exp(log_stds)))
#### code largely not used after here except when resuming/loading a policy. ####
def get_reparam_action_sym(self, obs_var, action_var, old_dist_info_vars):
"""
Given observations, old actions, and distribution of old actions, return a symbolically reparameterized
representation of the actions in terms of the policy parameters
:param obs_var:
:param action_var:
:param old_dist_info_vars:
:return:
"""
# Not used
import pdb; pdb.set_trace()
new_dist_info_vars = self.dist_info_sym(obs_var, action_var)
new_mean_var, new_log_std_var = new_dist_info_vars["mean"], new_dist_info_vars["log_std"]
old_mean_var, old_log_std_var = old_dist_info_vars["mean"], old_dist_info_vars["log_std"]
epsilon_var = (action_var - old_mean_var) / (tf.exp(old_log_std_var) + 1e-8)
new_action_var = new_mean_var + epsilon_var * tf.exp(new_log_std_var)
return new_action_var
def get_param_dtypes(self, all_params=False, **tags):
tag_tuple = tuple(sorted(list(tags.items()), key=lambda x: x[0]))
if tag_tuple not in self._cached_param_dtypes:
params = self.get_params(all_params, **tags)
param_values = tf.get_default_session().run(params)
self._cached_param_dtypes[tag_tuple] = [val.dtype for val in param_values]
return self._cached_param_dtypes[tag_tuple]
def get_param_shapes(self, all_params=False, **tags):
tag_tuple = tuple(sorted(list(tags.items()), key=lambda x: x[0]))
if tag_tuple not in self._cached_param_shapes:
params = self.get_params(all_params, **tags)
param_values = tf.get_default_session().run(params)
self._cached_param_shapes[tag_tuple] = [val.shape for val in param_values]
return self._cached_param_shapes[tag_tuple]
def set_param_values(self, flattened_params, all_params=False, **tags):
debug = tags.pop("debug", False)
param_values = unflatten_tensors(
flattened_params, self.get_param_shapes(all_params, **tags))
ops = []
feed_dict = dict()
for param, dtype, value in zip(
self.get_params(all_params, **tags),
self.get_param_dtypes(all_params, **tags),
param_values):
if param not in self._cached_assign_ops:
assign_placeholder = tf.placeholder(dtype=param.dtype.base_dtype)
assign_op = tf.assign(param, assign_placeholder)
self._cached_assign_ops[param] = assign_op
self._cached_assign_placeholders[param] = assign_placeholder
ops.append(self._cached_assign_ops[param])
feed_dict[self._cached_assign_placeholders[param]] = value.astype(dtype)
if debug:
print("setting value of %s" % param.name)
tf.get_default_session().run(ops, feed_dict=feed_dict)
def flat_to_params(self, flattened_params, all_params=False, **tags):
return unflatten_tensors(flattened_params, self.get_param_shapes(all_params, **tags))
def __getstate__(self):
d = Serializable.__getstate__(self)
global load_params
if load_params:
d["params"] = self.get_param_values(all_params=True)
return d
def __setstate__(self, d):
Serializable.__setstate__(self, d)
global load_params
if load_params:
tf.get_default_session().run(tf.variables_initializer(self.get_params(all_params=True)))
self.set_param_values(d["params"], all_params=True)