def __init__(self, env_spec, expert_trajs=None,
discrim_arch=feedforward_energy,
discrim_arch_args={},
l2_reg = 0,
discount = 1.0,
init_itrs = None,
score_dtau=False,
name='trajprior'):
super(GCLDiscrimTrajectory, self).__init__()
self.dO = env_spec.observation_space.flat_dim
self.dU = env_spec.action_space.flat_dim
self.score_dtau = score_dtau
self.expert_trajs = expert_trajs
# build energy model
with tf.variable_scope(name) as vs:
# Should be batch_size x T x dO/dU
self.obs_t = tf.placeholder(tf.float32, [None, None, self.dO], name='obs')
self.act_t = tf.placeholder(tf.float32, [None, None, self.dU], name='act')
self.traj_logprobs = tf.placeholder(tf.float32, [None, None], name='traj_probs')
self.labels = tf.placeholder(tf.float32, [None, 1], name='labels')
self.lr = tf.placeholder(tf.float32, (), name='lr')
obs_act = tf.concat([self.obs_t, self.act_t], axis=2)
with tf.variable_scope('discrim') as vs2:
self.energy = discrim_arch(obs_act, **discrim_arch_args)
discrim_vars = tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES, scope=vs2.name)
self.energy_timestep = self.energy
# Don't train separate log Z because we can't fully separate it from the energy function
log_p_tau = discounted_reduce_sum(-self.energy, discount=discount, axis=1)
log_q_tau = tf.reduce_sum(self.traj_logprobs, axis=1, keep_dims=True)
# numerical stability trick
log_pq = tf.reduce_logsumexp([log_p_tau, log_q_tau], axis=0)
self.d_tau = tf.exp(log_p_tau-log_pq)
cent_loss = -tf.reduce_mean(self.labels*(log_p_tau-log_pq) + (1-self.labels)*(log_q_tau-log_pq))
if l2_reg > 0:
reg_loss = l2_reg*tf.reduce_sum([tf.reduce_sum(tf.square(var)) for var in discrim_vars])
else:
reg_loss = 0
#self.predictions = tf.nn.sigmoid(logits)
self.loss = cent_loss + reg_loss
self.step = tf.train.AdamOptimizer(learning_rate=self.lr).minimize(self.loss)
self._make_param_ops(vs)
def __init__(self, env_spec, expert_trajs=None,
discrim_arch=feedforward_energy,
discrim_arch_args={},
l2_reg = 0,
discount = 1.0,
init_itrs = None,
score_dtau=False,
name='trajprior'):
super(GAN_GCL, self).__init__()
self.dO = env_spec.observation_space.flat_dim
self.dU = env_spec.action_space.flat_dim
self.score_dtau = score_dtau
self.expert_trajs = expert_trajs
# build energy model
with tf.variable_scope(name) as vs:
# Should be batch_size x T x dO/dU
self.obs_t = tf.placeholder(tf.float32, [None, None, self.dO], name='obs')
self.act_t = tf.placeholder(tf.float32, [None, None, self.dU], name='act')
self.traj_logprobs = tf.placeholder(tf.float32, [None, None], name='traj_probs')
self.labels = tf.placeholder(tf.float32, [None, 1], name='labels')
self.lr = tf.placeholder(tf.float32, (), name='lr')
obs_act = tf.concat([self.obs_t, self.act_t], axis=2)
with tf.variable_scope('discrim') as vs2:
self.energy = discrim_arch(obs_act, **discrim_arch_args)
discrim_vars = tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES, scope=vs2.name)
self.energy_timestep = self.energy
# Don't train separate log Z because we can't fully separate it from the energy function
log_p_tau = discounted_reduce_sum(-self.energy, discount=discount, axis=1)
log_q_tau = tf.reduce_sum(self.traj_logprobs, axis=1, keep_dims=True)
# numerical stability trick
log_pq = tf.reduce_logsumexp([log_p_tau, log_q_tau], axis=0)
self.d_tau = tf.exp(log_p_tau-log_pq)
cent_loss = -tf.reduce_mean(self.labels*(log_p_tau-log_pq) + (1-self.labels)*(log_q_tau-log_pq))
if l2_reg > 0:
reg_loss = l2_reg*tf.reduce_sum([tf.reduce_sum(tf.square(var)) for var in discrim_vars])
else:
reg_loss = 0
#self.predictions = tf.nn.sigmoid(logits)
self.loss = cent_loss + reg_loss
self.step = tf.train.AdamOptimizer(learning_rate=self.lr).minimize(self.loss)
self._make_param_ops(vs)