def run(self):
self.env = self.make_env(self.env_id)
self.env.seed = self.seed
self.continuous_actions = hasattr(self.env.action_space, "shape")
# tensorflow variables (same as in model.py)
observation_size = list(self.env.observation_space.shape)
if self.stacked_frames > 0:
observation_size += [self.stacked_frames]
hidden_size = 64
self.action_size = np.prod(
self.env.action_space.shape
) if self.continuous_actions else self.env.action_space.n
# tensorflow model of the policy
self.obs = tf.placeholder(tf.float32, [None] + observation_size)
self.policy_vars, self.avg_action_dist, self.logstd_action_dist = make_network(
"policy-a", self.obs, hidden_size, self.action_size)
config = tf.ConfigProto(device_count={'GPU': 0})
self.session = tf.Session(config=config)
self.session.run(tf.global_variables_initializer())
while True:
next_task = self.task_q.get(block=True)
if next_task == "do_rollout":
# the task is an actor request to collect experience
path = self.rollout()
self.task_q.task_done()
self.result_q.put(path)
elif next_task == "kill":
print(
"Received kill message for rollout process. Shutting down..."
)
self.task_q.task_done()
break
else:
# the task is to set parameters of the actor policy
self.set_policy(next_task)
# super hacky method to make sure when we fill the queue with set parameter tasks,
# an actor doesn't finish updating before the other actors can accept their own tasks.
sleep(0.1)
self.task_q.task_done()
def run(self):
# tensorflow variables (same as in model.py)
observation_size = 37
hidden_size = 200
action_size = 2
self.env = AgentTorcs2(self.env_id,
bots=['scr_server'],
track='road/g-track-1',
text_mode=False,
laps=3,
torcsIdxOffset=100,
screen_capture=True)
# agent = AgentTorcs2(aidx, bots=['scr_server', 'olethros', 'berniw', 'bt', 'damned'], track='road/g-track-1', text_mode=True)
self.env.reset()
# tensorflow model of the policy
self.obs = tf.placeholder(tf.float32, [None, observation_size])
self.policy_vars, self.avg_action_dist, self.logstd_action_dist = make_network(
"policy-a", self.obs, hidden_size, action_size)
config = tf.ConfigProto(device_count={'GPU': 0})
self.session = tf.Session(config=config)
self.session.run(tf.global_variables_initializer())
while True:
next_task = self.task_q.get(block=True)
if next_task == "do_rollout":
# the task is an actor request to collect experience
path = self.rollout()
self.task_q.task_done()
self.result_q.put(path)
elif next_task == "kill":
print("Received kill message. Shutting down...")
self.task_q.task_done()
break
else:
# the task is to set parameters of the actor policy
self.set_policy(next_task)
# super hacky method to make sure when we fill the queue with set parameter tasks,
# an actor doesn't finish updating before the other actors can accept their own tasks.
sleep(0.1)
self.task_q.task_done()
def __init__(self, env_id, make_env, max_kl, discount_factor, cg_damping):
self.max_kl = max_kl
self.discount_factor = discount_factor
self.cg_damping = cg_damping
env = make_env(env_id)
observation_size = env.observation_space.shape[0]
hidden_size = 64
action_size = np.prod(env.action_space.shape)
self.obs = tf.placeholder(tf.float32, [None, observation_size])
self.action = tf.placeholder(tf.float32, [None, action_size])
self.advantage = tf.placeholder(tf.float32, [None])
self.old_avg_action_dist = tf.placeholder(tf.float32,
[None, action_size])
self.old_logstd_action_dist = tf.placeholder(tf.float32,
[None, action_size])
self.policy_vars, self.avg_action_dist, self.logstd_action_dist = utils.make_network(
"policy", self.obs, hidden_size, action_size)
batch_size = tf.shape(self.obs)[0]
# what are the probabilities of taking self.action, given new and old distributions
log_p_n = utils.gauss_log_prob(self.avg_action_dist,
self.logstd_action_dist, self.action)
log_oldp_n = utils.gauss_log_prob(self.old_avg_action_dist,
self.old_logstd_action_dist,
self.action)
# tf.exp(log_p_n) / tf.exp(log_oldp_n)
ratio = tf.exp(log_p_n - log_oldp_n)
# importance sampling of surrogate loss (L in paper)
surr = -tf.reduce_mean(ratio * self.advantage)
batch_size_float = tf.cast(batch_size, tf.float32)
# kl divergence and shannon entropy
kl = utils.gauss_KL(self.old_avg_action_dist,
self.old_logstd_action_dist, self.avg_action_dist,
self.logstd_action_dist) / batch_size_float
ent = utils.gauss_ent(self.avg_action_dist,
self.logstd_action_dist) / batch_size_float
self.losses = [surr, kl, ent]
self.policy_gradient = utils.flatgrad(surr, self.policy_vars)
# KL divergence w/ itself, with first argument kept constant.
kl_firstfixed = utils.gauss_selfKL_firstfixed(
self.avg_action_dist, self.logstd_action_dist) / batch_size_float
# gradient of KL w/ itself
grads = tf.gradients(kl_firstfixed, self.policy_vars)
# what vector we're multiplying by
self.flat_tangent = tf.placeholder(tf.float32, [None])
start = 0
tangents = []
for var in self.policy_vars:
size = tf.size(var)
param = tf.reshape(self.flat_tangent[start:(start + size)],
var.shape)
tangents.append(param)
start += size
# gradient of KL w/ itself * tangent
gvp = [tf.reduce_sum(g * t) for (g, t) in zip(grads, tangents)]
# 2nd gradient of KL w/ itself * tangent
self.fvp = utils.flatgrad(gvp, self.policy_vars)
config = tf.ConfigProto(device_count={'GPU': 0})
self.session = tf.Session(config=config)
self.session.run(tf.global_variables_initializer())
# value function
# self.vf = VF(self.session)
self.vf = LinearVF()
# Operations
# _get_flat_params
self._get_flat_params = tf.concat(
axis=0,
values=[tf.reshape(v, [tf.size(v)]) for v in self.policy_vars])
# _set_from_flat
total_size = sum(
np.prod(v.get_shape().as_list()) for v in self.policy_vars)
self._theta_op = tf.placeholder(tf.float32, [total_size])
start = 0
assigns = []
for var in self.policy_vars:
size = tf.size(var)
assigns.append(
tf.assign(
var,
tf.reshape(self._theta_op[start:start + size], var.shape)))
start += size
self._set_from_flat = tf.group(*assigns)
# get_policy
self.get_policy = [
var for var in self.policy_vars if 'policy' in var.name
]
def __init__(self, env_id, make_env, stacked_frames, max_kl,
discount_factor, cg_damping):
self.max_kl = max_kl
self.discount_factor = discount_factor
self.cg_damping = cg_damping
env = make_env(env_id)
continuous_actions = hasattr(env.action_space, "shape")
observation_size = list(env.observation_space.shape)
if stacked_frames > 0:
observation_size += [stacked_frames]
hidden_size = 64
action_size = np.prod(env.action_space.shape
) if continuous_actions else env.action_space.n
self.obs = tf.placeholder(tf.float32, [None] + observation_size)
self.action = tf.placeholder(tf.float32, [None, action_size])
self.advantage = tf.placeholder(tf.float32, [None])
self.old_avg_action_dist = tf.placeholder(tf.float32,
[None, action_size])
self.old_logstd_action_dist = tf.placeholder(tf.float32,
[None, action_size])
self.policy_vars, self.avg_action_dist, self.logstd_action_dist = utils.make_network(
"policy", self.obs, hidden_size, action_size)
batch_size = tf.shape(self.obs)[0]
# what are the probabilities of taking self.action, given new and old distributions
log_p_n = utils.gauss_log_prob(self.avg_action_dist,
self.logstd_action_dist, self.action)
log_oldp_n = utils.gauss_log_prob(self.old_avg_action_dist,
self.old_logstd_action_dist,
self.action)
# tf.exp(log_p_n) / tf.exp(log_oldp_n)
ratio = tf.exp(log_p_n - log_oldp_n)
# importance sampling of surrogate loss (L in paper)
surr = -tf.reduce_mean(ratio * self.advantage)
batch_size_float = tf.cast(batch_size, tf.float32)
# kl divergence and shannon entropy
kl = utils.gauss_KL(self.old_avg_action_dist,
self.old_logstd_action_dist, self.avg_action_dist,
self.logstd_action_dist) / batch_size_float
ent = utils.gauss_ent(self.avg_action_dist,
self.logstd_action_dist) / batch_size_float
self.losses = [surr, kl, ent]
self.policy_gradient = utils.flatgrad(surr, self.policy_vars)
# KL divergence w/ itself, with first argument kept constant.
kl_firstfixed = utils.gauss_selfKL_firstfixed(
self.avg_action_dist, self.logstd_action_dist) / batch_size_float
# gradient of KL w/ itself
grads = tf.gradients(kl_firstfixed, self.policy_vars)
# what vector we're multiplying by
self.flat_tangent = tf.placeholder(tf.float32, [None])
start = 0
tangents = []
for var in self.policy_vars:
size = tf.size(var)
param = tf.reshape(self.flat_tangent[start:(start + size)],
var.shape)
tangents.append(param)
start += size
# gradient of KL w/ itself * tangent
gvp = [tf.reduce_sum(g * t) for (g, t) in zip(grads, tangents)]
# 2nd gradient of KL w/ itself * tangent
self.fvp = utils.flatgrad(gvp, self.policy_vars)
config = tf.ConfigProto(device_count={'GPU': 0})
self.session = tf.Session(config=config)
self.session.run(tf.global_variables_initializer())
# value function
# self.vf = VF(self.session)
self.vf = LinearVF()