def __init__(
self,
env_spec,
hardcoded_q=None,
scope='policy',
ent_wt=1.0,
):
"""
:param env_spec: A spec for the env.
:param hidden_dim: dimension of hidden layer
:param hidden_nonlinearity: nonlinearity used for each hidden layer
:return:
"""
#self.graph = tf.get_default_graph()
assert isinstance(env_spec.action_space, Discrete)
Serializable.quick_init(self, locals())
super(CategoricalSoftQPolicy, self).__init__(env_spec)
obs_dim = env_spec.observation_space.flat_dim
action_dim = env_spec.action_space.flat_dim
self.dist = Categorical(action_dim)
self.ent_wt = ent_wt
self.hardcoded_q = hardcoded_q
with tf.variable_scope(scope) as vs:
self.vs = vs
self.q_func = tf.get_variable(
'q_func', shape=(obs_dim, action_dim))
self.q_func_plc = tf.placeholder(
tf.float32, shape=(obs_dim, action_dim))
self.q_func_assgn = tf.assign(self.q_func, self.q_func_plc)
def __init__(self,
*,
name,
policy_model,
num_envs,
env_spec,
wrapped_env_action_space,
action_space,
observation_space,
batching_config,
init_location=None,
encoder=None):
Serializable.quick_init(self, locals())
assert isinstance(wrapped_env_action_space, Box)
self._dist = Categorical(wrapped_env_action_space.shape[0])
# this is going to be serialized, so we can't add in the envs or
# wrappers
self.init_args = dict(name=name,
policy_model=policy_model,
init_location=init_location)
ent_coef = 0.01
vf_coef = 0.5
max_grad_norm = 0.5
model_args = dict(policy=policy_model,
ob_space=observation_space,
ac_space=action_space,
nbatch_act=batching_config.nenvs,
nbatch_train=batching_config.nbatch_train,
nsteps=batching_config.nsteps,
ent_coef=ent_coef,
vf_coef=vf_coef,
max_grad_norm=max_grad_norm)
self.num_envs = num_envs
with tf.variable_scope(name) as scope:
policy = policies.Policy(model_args)
self.model = policy.model
self.act_model = self.model.act_model
self.scope = scope
StochasticPolicy.__init__(self, env_spec)
self.name = name
self.probs = tf.nn.softmax(self.act_model.pd.logits)
obs_var = self.act_model.X
self.tensor_values = lambda **kwargs: tf.get_default_session().run(
self.get_params())
self._f_dist = tensor_utils.compile_function(inputs=[obs_var],
outputs=self.probs)
if init_location:
data = joblib.load(open(init_location, 'rb'))
self.restore_from_snapshot(data['policy_params'])
def __init__(
self,
name,
env_spec,
hidden_sizes=(32, 32),
hidden_nonlinearity=tf.nn.tanh,
prob_network=None,
):
"""
:param env_spec: A spec for the mdp.
:param hidden_sizes: list of sizes for the fully connected hidden layers
:param hidden_nonlinearity: nonlinearity used for each hidden layer
:param prob_network: manually specified network for this policy, other network params
are ignored
:return:
"""
Serializable.quick_init(self, locals())
assert isinstance(env_spec.action_space, Discrete)
obs_dim = env_spec.observation_space.flat_dim
action_dim = env_spec.action_space.flat_dim
with tf.variable_scope(name):
if prob_network is None:
prob_network = self.create_MLP(
input_shape=(obs_dim, ),
output_dim=env_spec.action_space.n,
hidden_sizes=hidden_sizes,
name="prob_network",
)
self._l_obs, self._l_prob = self.forward_MLP(
'prob_network',
prob_network,
n_hidden=len(hidden_sizes),
input_shape=(obs_dim, ),
hidden_nonlinearity=hidden_nonlinearity,
output_nonlinearity=tf.nn.softmax,
reuse=None)
# if you want to input your own tensor.
self._forward_out = lambda x, is_train: self.forward_MLP(
'prob_network',
prob_network,
n_hidden=len(hidden_sizes),
hidden_nonlinearity=hidden_nonlinearity,
output_nonlinearity=output_nonlinearity,
input_tensor=x,
is_training=is_train)[1]
self._f_prob = tensor_utils.compile_function([self._l_obs],
L.get_output(
self._l_prob))
self._dist = Categorical(env_spec.action_space.n)
def __init__(
self,
name,
env_spec,
hidden_sizes=(32, 32),
hidden_nonlinearity=tf.nn.tanh,
prob_network=None,
grad_step_size=1.0,
):
"""
:param env_spec: A spec for the mdp.
:param hidden_sizes: list of sizes for the fully connected hidden layers
:param hidden_nonlinearity: nonlinearity used for each hidden layer
:param prob_network: manually specified network for this policy, other network params
are ignored
:param grad_step_size: the step size taken in the learner's gradient update, sample uniformly if it is a range e.g. [0.1,1]
:return:
"""
Serializable.quick_init(self, locals())
assert isinstance(env_spec.action_space, Discrete)
obs_dim = env_spec.observation_space.flat_dim
self.action_dim = env_spec.action_space.n
self.n_hidden = len(hidden_sizes)
self.hidden_nonlinearity = hidden_nonlinearity
self.input_shape = (None, obs_dim,)
self.step_size = grad_step_size
if prob_network is None:
self.all_params = self.create_MLP(
output_dim=self.action_dim,
hidden_sizes=hidden_sizes,
name="prob_network",
)
self.all_param_vals = None
self._l_obs, self._l_prob = self.forward_MLP('prob_network', self.all_params,
n_hidden=len(hidden_sizes), input_shape=(obs_dim,),
hidden_nonlinearity=hidden_nonlinearity,
output_nonlinearity=tf.nn.softmax, reuse=None)
# if you want to input your own tensor.
self._forward_out = lambda x, params, is_train: self.forward_MLP('prob_network', params,
n_hidden=len(hidden_sizes), hidden_nonlinearity=hidden_nonlinearity,
output_nonlinearity=tf.nn.softmax, input_tensor=x, is_training=is_train)[1]
self._init_f_prob = tensor_utils.compile_function(
[self._l_obs],
[self._l_prob])
self._cur_f_prob = self._init_f_prob
self._dist = Categorical(self.action_dim)
self._cached_params = {}
super(MAMLCategoricalMLPPolicy, self).__init__(env_spec)
def init_policy(self):
output_vec = L.get_output(self._output_vec_layer,
deterministic=True) / self._c
prob = tf.nn.softmax(output_vec)
max_qval = tf.reduce_logsumexp(output_vec, [1])
self._f_prob = tensor_utils.compile_function(
[self._obs_layer.input_var], prob)
self._f_max_qvals = tensor_utils.compile_function(
[self._obs_layer.input_var], max_qval)
self._dist = Categorical(self._n)
def __init__(
self,
name,
env_spec,
conv_filters,
conv_filter_sizes,
conv_strides,
conv_pads,
hidden_sizes=[],
hidden_nonlinearity=tf.nn.relu,
output_nonlinearity=tf.nn.softmax,
prob_network=None,
):
"""
:param env_spec: A spec for the mdp.
:param hidden_sizes: list of sizes for the fully connected hidden layers
:param hidden_nonlinearity: nonlinearity used for each hidden layer
:param prob_network: manually specified network for this policy, other network params
are ignored
:return:
"""
Serializable.quick_init(self, locals())
assert isinstance(env_spec.action_space, Discrete)
self._env_spec = env_spec
# import pdb; pdb.set_trace()
if prob_network is None:
prob_network = ConvNetwork(
input_shape=env_spec.observation_space.shape,
output_dim=env_spec.action_space.n,
conv_filters=conv_filters,
conv_filter_sizes=conv_filter_sizes,
conv_strides=conv_strides,
conv_pads=conv_pads,
hidden_sizes=hidden_sizes,
hidden_nonlinearity=hidden_nonlinearity,
output_nonlinearity=output_nonlinearity,
name="prob_network",
)
self._l_prob = prob_network.output_layer
self._l_obs = prob_network.input_layer
self._f_prob = tensor_utils.compile_function(
[prob_network.input_layer.input_var],
L.get_output(prob_network.output_layer))
self._dist = Categorical(env_spec.action_space.n)
super(CategoricalConvPolicy, self).__init__(env_spec)
LayersPowered.__init__(self, [prob_network.output_layer])
def __init__(self,
name,
env_spec,
hidden_sizes=(32, 32),
hidden_nonlinearity=tf.nn.tanh,
gating_network=None,
input_layer=None,
num_options=4,
conv_filters=None,
conv_filter_sizes=None,
conv_strides=None,
conv_pads=None,
input_shape=None):
"""
:param env_spec: A spec for the mdp.
:param hidden_sizes: list of sizes for the fully connected hidden layers
:param hidden_nonlinearity: nonlinearity used for each hidden layer
:param prob_network: manually specified network for this policy, other network params
are ignored
:return:
"""
Serializable.quick_init(self, locals())
self.num_options = num_options
assert isinstance(env_spec.action_space, Discrete)
with tf.variable_scope(name):
input_layer, output_layer = self.make_network(
(env_spec.observation_space.flat_dim, ),
env_spec.action_space.n,
hidden_sizes,
hidden_nonlinearity=hidden_nonlinearity,
gating_network=gating_network,
l_in=input_layer,
conv_filters=conv_filters,
conv_filter_sizes=conv_filter_sizes,
conv_strides=conv_strides,
conv_pads=conv_pads,
input_shape=input_shape)
self._l_prob = output_layer
self._l_obs = input_layer
self._f_prob = tensor_utils.compile_function(
[input_layer.input_var], L.get_output(output_layer))
self._dist = Categorical(env_spec.action_space.n)
super(CategoricalDecomposedPolicy, self).__init__(env_spec)
LayersPowered.__init__(self, [output_layer])
def __init__(
self,
name,
env_spec,
hidden_sizes=(32, 32),
hidden_nonlinearity=tf.nn.tanh,
prob_network=None,
):
"""
:param env_spec: A spec for the mdp.
:param hidden_sizes: list of sizes for the fully connected hidden layers
:param hidden_nonlinearity: nonlinearity used for each hidden layer
:param prob_network: manually specified network for this policy, other network params
are ignored
:return:
"""
Serializable.quick_init(self, locals())
assert isinstance(env_spec.action_space, Discrete)
with tf.variable_scope(name):
if prob_network is None:
prob_network = MLP(
input_shape=(env_spec.observation_space.flat_dim,),
output_dim=env_spec.action_space.n,
hidden_sizes=hidden_sizes,
hidden_nonlinearity=hidden_nonlinearity,
output_nonlinearity=tf.nn.softmax,
name="prob_network",
)
self._l_prob = prob_network.output_layer
self._l_obs = prob_network.input_layer
self._f_prob = tensor_utils.compile_function(
[prob_network.input_layer.input_var],
L.get_output(prob_network.output_layer)
)
self._dist = Categorical(env_spec.action_space.n)
super(CategoricalMLPPolicy, self).__init__(env_spec)
LayersPowered.__init__(self, [prob_network.output_layer])
def __init__(self, dim):
self._cat = Categorical(dim)
self._dim = dim
class RecurrentCategorical(Distribution):
def __init__(self, dim):
self._cat = Categorical(dim)
self._dim = dim
@property
def dim(self):
return self._dim
def kl_sym(self, old_dist_info_vars, new_dist_info_vars):
"""
Compute the symbolic KL divergence of two categorical distributions
"""
old_prob_var = old_dist_info_vars["prob"]
new_prob_var = new_dist_info_vars["prob"]
# Assume layout is N * T * A
return tf.reduce_sum(
old_prob_var * (tf.log(old_prob_var + TINY) - tf.log(new_prob_var + TINY)),
reduction_indices=2
)
def kl(self, old_dist_info, new_dist_info):
"""
Compute the KL divergence of two categorical distributions
"""
old_prob = old_dist_info["prob"]
new_prob = new_dist_info["prob"]
return np.sum(
old_prob * (np.log(old_prob + TINY) - np.log(new_prob + TINY)),
axis=2
)
def likelihood_ratio_sym(self, x_var, old_dist_info_vars, new_dist_info_vars):
old_prob_var = old_dist_info_vars["prob"]
new_prob_var = new_dist_info_vars["prob"]
# Assume layout is N * T * A
a_dim = tf.shape(x_var)[2]
flat_ratios = self._cat.likelihood_ratio_sym(
tf.reshape(x_var, tf.pack([-1, a_dim])),
dict(prob=tf.reshape(old_prob_var, tf.pack([-1, a_dim]))),
dict(prob=tf.reshape(new_prob_var, tf.pack([-1, a_dim])))
)
return tf.reshape(flat_ratios, tf.shape(old_prob_var)[:2])
def entropy(self, dist_info):
probs = dist_info["prob"]
return -np.sum(probs * np.log(probs + TINY), axis=2)
def entropy_sym(self, dist_info_vars):
probs = dist_info_vars["prob"]
return -tf.reduce_sum(probs * tf.log(probs + TINY), 2)
def log_likelihood_sym(self, xs, dist_info_vars):
probs = dist_info_vars["prob"]
# Assume layout is N * T * A
a_dim = tf.shape(probs)[2]
# a_dim = TT.printing.Print("lala")(a_dim)
flat_logli = self._cat.log_likelihood_sym(
tf.reshape(xs, tf.pack([-1, a_dim])),
dict(prob=tf.reshape(probs, tf.pack((-1, a_dim))))
)
return tf.reshape(flat_logli, tf.shape(probs)[:2])
def log_likelihood(self, xs, dist_info):
probs = dist_info["prob"]
# Assume layout is N * T * A
a_dim = tf.shape(probs)[2]
flat_logli = self._cat.log_likelihood_sym(
xs.reshape((-1, a_dim)),
dict(prob=probs.reshape((-1, a_dim)))
)
return flat_logli.reshape(probs.shape[:2])
@property
def dist_info_specs(self):
return [("prob", (self.dim,))]
def __init__(
self,
name,
input_shape,
output_dim,
prob_network=None,
hidden_sizes=(32, 32),
hidden_nonlinearity=tf.nn.tanh,
optimizer=None,
tr_optimizer=None,
use_trust_region=True,
step_size=0.01,
normalize_inputs=True,
no_initial_trust_region=True,
):
"""
:param input_shape: Shape of the input data.
:param output_dim: Dimension of output.
:param hidden_sizes: Number of hidden units of each layer of the mean network.
:param hidden_nonlinearity: Non-linearity used for each layer of the mean network.
:param optimizer: Optimizer for minimizing the negative log-likelihood.
:param use_trust_region: Whether to use trust region constraint.
:param step_size: KL divergence constraint for each iteration
"""
Serializable.quick_init(self, locals())
with tf.variable_scope(name):
if optimizer is None:
optimizer = LbfgsOptimizer(name="optimizer")
if tr_optimizer is None:
tr_optimizer = ConjugateGradientOptimizer()
self.input_dim = input_shape[0]
self.observation_space = Discrete(self.input_dim)
self.action_space = Discrete(output_dim)
self.output_dim = output_dim
self.optimizer = optimizer
self.tr_optimizer = tr_optimizer
if prob_network is None:
prob_network = MLP(
input_shape=input_shape,
output_dim=output_dim,
hidden_sizes=hidden_sizes,
hidden_nonlinearity=hidden_nonlinearity,
output_nonlinearity=tf.nn.softmax,
name="prob_network"
)
l_prob = prob_network.output_layer
LayersPowered.__init__(self, [l_prob])
xs_var = prob_network.input_layer.input_var
ys_var = tf.placeholder(dtype=tf.float32, shape=[None, output_dim], name="ys")
old_prob_var = tf.placeholder(dtype=tf.float32, shape=[None, output_dim], name="old_prob")
x_mean_var = tf.get_variable(
name="x_mean",
shape=(1,) + input_shape,
initializer=tf.constant_initializer(0., dtype=tf.float32)
)
x_std_var = tf.get_variable(
name="x_std",
shape=(1,) + input_shape,
initializer=tf.constant_initializer(1., dtype=tf.float32)
)
self.x_mean_var = x_mean_var
self.x_std_var = x_std_var
normalized_xs_var = (xs_var - x_mean_var) / x_std_var
prob_var = L.get_output(l_prob, {prob_network.input_layer: normalized_xs_var})
old_info_vars = dict(prob=old_prob_var)
info_vars = dict(prob=prob_var)
dist = self._dist = Categorical(output_dim)
mean_kl = tf.reduce_mean(dist.kl_sym(old_info_vars, info_vars))
loss = - tf.reduce_mean(dist.log_likelihood_sym(ys_var, info_vars))
predicted = tensor_utils.to_onehot_sym(tf.argmax(prob_var, axis=1), output_dim)
self.prob_network = prob_network
self.f_predict = tensor_utils.compile_function([xs_var], predicted)
self.f_prob = tensor_utils.compile_function([xs_var], prob_var)
self.l_prob = l_prob
self.optimizer.update_opt(loss=loss, target=self, network_outputs=[prob_var], inputs=[xs_var, ys_var])
self.tr_optimizer.update_opt(loss=loss, target=self, network_outputs=[prob_var],
inputs=[xs_var, ys_var, old_prob_var],
leq_constraint=(mean_kl, step_size)
)
self.use_trust_region = use_trust_region
self.name = name
self.normalize_inputs = normalize_inputs
self.x_mean_var = x_mean_var
self.x_std_var = x_std_var
self.first_optimized = not no_initial_trust_region
def __init__(self, dim):
self._cat = Categorical(dim)
self._dim = dim
class RecurrentCategorical(Distribution):
def __init__(self, dim):
self._cat = Categorical(dim)
self._dim = dim
@property
def dim(self):
return self._dim
def kl_sym(self, old_dist_info_vars, new_dist_info_vars):
"""
Compute the symbolic KL divergence of two categorical distributions
"""
old_prob_var = old_dist_info_vars["prob"]
new_prob_var = new_dist_info_vars["prob"]
# Assume layout is N * T * A
return tf.reduce_sum(
old_prob_var *
(tf.log(old_prob_var + TINY) - tf.log(new_prob_var + TINY)),
reduction_indices=2)
def kl(self, old_dist_info, new_dist_info):
"""
Compute the KL divergence of two categorical distributions
"""
old_prob = old_dist_info["prob"]
new_prob = new_dist_info["prob"]
return np.sum(old_prob *
(np.log(old_prob + TINY) - np.log(new_prob + TINY)),
axis=2)
def likelihood_ratio_sym(self, x_var, old_dist_info_vars,
new_dist_info_vars):
old_prob_var = old_dist_info_vars["prob"]
new_prob_var = new_dist_info_vars["prob"]
# Assume layout is N * T * A
a_dim = tf.shape(x_var)[2]
flat_ratios = self._cat.likelihood_ratio_sym(
tf.reshape(x_var, tf.stack([-1, a_dim])),
dict(prob=tf.reshape(old_prob_var, tf.stack([-1, a_dim]))),
dict(prob=tf.reshape(new_prob_var, tf.stack([-1, a_dim]))))
return tf.reshape(flat_ratios, tf.shape(old_prob_var)[:2])
def entropy(self, dist_info):
probs = dist_info["prob"]
return -np.sum(probs * np.log(probs + TINY), axis=2)
def entropy_sym(self, dist_info_vars):
probs = dist_info_vars["prob"]
return -tf.reduce_sum(probs * tf.log(probs + TINY), 2)
def log_likelihood_sym(self, xs, dist_info_vars):
probs = dist_info_vars["prob"]
# Assume layout is N * T * A
a_dim = tf.shape(probs)[2]
# a_dim = TT.printing.Print("lala")(a_dim)
flat_logli = self._cat.log_likelihood_sym(
tf.reshape(xs, tf.stack([-1, a_dim])),
dict(prob=tf.reshape(probs, tf.stack((-1, a_dim)))))
return tf.reshape(flat_logli, tf.shape(probs)[:2])
def log_likelihood(self, xs, dist_info):
probs = dist_info["prob"]
# Assume layout is N * T * A
a_dim = tf.shape(probs)[2]
flat_logli = self._cat.log_likelihood_sym(
xs.reshape((-1, a_dim)), dict(prob=probs.reshape((-1, a_dim))))
return flat_logli.reshape(probs.shape[:2])
@property
def dist_info_specs(self):
return [("prob", (self.dim, ))]