def __init__(self, name, obs_size=2, action_size=2, actor_hidden_size=32, critic_hidden_size=32, pow_learning_rate=0.001,
RB_learning_rate=0.001, entropy_cost=0.01, normalise_entropy=True, lambda_=0., baseline_cost=1.):
with tf.variable_scope(name):
# hyperparameter bootstrap_n determines the batch size
self.name=name
self.input_ = tf.placeholder(tf.float32, [None, obs_size], name='inputs')
self.action_RB_ = tf.placeholder(tf.int32, [None, 1], name='action_RB')
self.action_pow_ = tf.placeholder(tf.int32, [None, 1], name='action_pow')
self.reward_ = tf.placeholder(tf.float32, [None, 1], name='reward')
self.discount_ = tf.placeholder(tf.float32, [None, 1], name='discount')
self.bootstrap_ = tf.placeholder(tf.float32, [None], name='bootstrap')
# set up actor network
self.fc1_actor_ = tf.contrib.layers.fully_connected(self.input_, actor_hidden_size, activation_fn=tf.nn.elu)
self.fc2_actor_ = tf.contrib.layers.fully_connected(self.fc1_actor_, actor_hidden_size, activation_fn=tf.nn.elu)
self.fc3_actor_power_ = tf.contrib.layers.fully_connected(self.fc2_actor_, ch.D2D_tr_Power_levels, activation_fn=None)
self.fc3_actor_RB_ = tf.contrib.layers.fully_connected(self.fc2_actor_, ch.N_CU, activation_fn=None)
# reshape the policy logits
self.policy_logits_RB_ = tf.reshape(self.fc3_actor_RB_, (-1, 1, ch.N_CU))
self.policy_logits_power_ = tf.reshape(self.fc3_actor_power_, (-1, 1, ch.D2D_tr_Power_levels))
# self.policy_logits_ = self.combine_tensors(self.fc3_actor_power_, self.fc3_actor_RB_) # TO-DO figure out how to have 2 seperate outputs for each action type
# might need to do two seperate updates - policy logtis for both pow_sel and RB_sel
#print(self.policy_logits_)
#self.policy_logits_ = tf.transpose(self.policy_logits_)
#self.policy_logits_ = tf.expand_dims(self.policy_logits_, axis=0)
#self.policy_logits_ = tf.reshape(self.policy_logits_, [-1, 1, action_size]) # these resize it weirdly
# generate action probabilities for taking actions
self.action_prob_power_ = tf.nn.softmax(self.fc3_actor_power_)
self.action_prob_RB_ = tf.nn.softmax(self.fc3_actor_RB_)
# set up critic network
self.fc1_critic_ = tf.contrib.layers.fully_connected(self.input_, critic_hidden_size, activation_fn=tf.nn.elu)
self.fc2_critic_ = tf.contrib.layers.fully_connected(self.fc1_critic_, critic_hidden_size, activation_fn=tf.nn.elu)
self.baseline_ = tf.contrib.layers.fully_connected(self.fc2_critic_, 1, activation_fn=None)
# Define Loss with TRFL
self.seq_aac_return_pow_ = trfl.sequence_advantage_actor_critic_loss(self.policy_logits_power_, self.baseline_, self.action_pow_,
self.reward_, self.discount_, self.bootstrap_, lambda_=lambda_, entropy_cost=entropy_cost,
baseline_cost=baseline_cost, normalise_entropy=normalise_entropy)
self.seq_aac_return_RB_ = trfl.sequence_advantage_actor_critic_loss(self.policy_logits_RB_, self.baseline_, self.action_RB_,
self.reward_, self.discount_, self.bootstrap_, lambda_=lambda_, entropy_cost=entropy_cost,
baseline_cost=baseline_cost, normalise_entropy=normalise_entropy)
# Optimize the loss
self.ac_loss_pow_ = tf.reduce_mean(self.seq_aac_return_pow_.loss)
self.ac_loss_RB_ = tf.reduce_mean(self.seq_aac_return_RB_.loss)
self.ac_optim_pow_ = tf.train.AdamOptimizer(learning_rate=pow_learning_rate).minimize(self.ac_loss_pow_)
self.ac_optim_RB_ = tf.train.AdamOptimizer(learning_rate=RB_learning_rate).minimize(self.ac_loss_RB_)
def __init__(self,
name,
obs_size=2,
action_size=2,
actor_hidden_size=32,
critic_hidden_size=32,
ac_learning_rate=0.001,
entropy_cost=0.01,
normalise_entropy=True,
lambda_=0.,
baseline_cost=1.):
with tf.variable_scope(name):
# hyperparameter bootstrap_n determines the batch size
self.name = name
self.input_ = tf.placeholder(tf.float32, [None, obs_size],
name='inputs')
self.action_ = tf.placeholder(tf.int32, [None, 1], name='action')
self.reward_ = tf.placeholder(tf.float32, [None, 1], name='reward')
self.discount_ = tf.placeholder(tf.float32, [None, 1],
name='discount')
self.bootstrap_ = tf.placeholder(tf.float32, [None],
name='bootstrap')
# set up actor network
self.fc1_actor_ = tf.contrib.layers.fully_connected(
self.input_, actor_hidden_size, activation_fn=tf.nn.relu)
self.fc2_actor_ = tf.contrib.layers.fully_connected(
self.fc1_actor_, action_size, activation_fn=tf.nn.softmax)
# reshape the policy logits
self.policy_logits_ = tf.reshape(self.fc2_actor_,
[-1, 1, action_size])
# generate action probabilities for taking actions
self.action_prob_ = tf.nn.softmax(self.fc2_actor_)
# set up critic network
self.fc1_critic_ = tf.contrib.layers.fully_connected(
self.input_, critic_hidden_size, activation_fn=tf.nn.relu)
self.baseline_ = tf.contrib.layers.fully_connected(
self.fc1_critic_, 1, activation_fn=None)
# TRFL usage
self.seq_aac_return_ = trfl.sequence_advantage_actor_critic_loss(
self.policy_logits_,
self.baseline_,
self.action_,
self.reward_,
self.discount_,
self.bootstrap_,
lambda_=lambda_,
entropy_cost=entropy_cost,
baseline_cost=baseline_cost,
normalise_entropy=normalise_entropy)
# Optimize the loss
self.ac_loss_ = tf.reduce_mean(self.seq_aac_return_.loss)
self.ac_optim_ = tf.train.AdamOptimizer(
learning_rate=ac_learning_rate).minimize(self.ac_loss_)
def get_loss(self, x, at, r):
# TODO change to PPO?
T, B = tf.shape(r)
h = tf.map_fn(self.net, x)
policy_logits = h[:, :, :self.n_actions]
v = h[:, :, self.n_actions]
pg_loss, extra = trfl.sequence_advantage_actor_critic_loss(
policy_logits=policy_logits,
baseline_values=v,
actions=at,
rewards=r,
pcontinues=0.99 * tf.ones_like(r),
bootstrap_value=tf.ones(tf.shape(r)[1]),
entropy_cost=1e-1,
normalise_entropy=True)
loss = tf.reduce_mean(pg_loss)
tf.contrib.summary.scalar('loss/policy', loss)
return loss
def train_step(self, x, a, r):
"""
Train on a batch of data.
x: (B, T)
a: (B, T)
r: (B, T)
"""
x = tf.constant(x,
shape=[self.batch_size, self.time_steps, 1],
dtype=tf.float32)
a = tf.constant(a,
shape=[self.batch_size, self.time_steps, 1],
dtype=tf.float32)
r = tf.constant(r,
shape=[self.batch_size, self.time_steps, 1],
dtype=tf.float32)
# current obs, old action, old reward
inputs = tf.concat([x[:, 1:, :], a[:, :-1:, ], r[:, :-1, :]], axis=-1)
actions_taken = tf.transpose(tf.squeeze(tf.cast(a[:, 1:, :],
tf.int32)))
rewards_received = tf.transpose(r[:, 1:, 0])
returns = tf.reduce_sum(rewards_received, axis=0)
with tf.GradientTape() as tape:
logits, v = self.forward(inputs)
# NOTE does this loss fn correct for its off policy nature?
policy_loss, extra = trfl.sequence_advantage_actor_critic_loss(
policy_logits=logits,
baseline_values=
v[...,
0], # Q how can A2C be extended with distributional estimates of value?
actions=actions_taken,
rewards=rewards_received,
pcontinues=0.99 * tf.ones_like(rewards_received),
bootstrap_value=returns,
entropy_cost=1.0,
lambda_=0.5)
beta = tf.constant(1.0 * 0.5**(self.global_step / 2000))
beta = tf.cast(beta, tf.float32)
loss = tf.reduce_mean(0.05 * extra.baseline_loss +
extra.policy_gradient_loss +
beta * extra.entropy_loss)
variables = self.nn.variables + self.rnn.variables
grads = tape.gradient(loss, variables)
self.opt.apply_gradients(zip(grads, variables),
global_step=self.global_step)
with tf.contrib.summary.record_summaries_every_n_global_steps(10):
tf.contrib.summary.scalar('loss', loss)
tf.contrib.summary.scalar('extra/baseline_loss',
tf.reduce_mean(extra.baseline_loss))
tf.contrib.summary.scalar(
'extra/policy_gradient_loss',
tf.reduce_mean(extra.policy_gradient_loss))
tf.contrib.summary.scalar('extra/entropy_loss',
tf.reduce_mean(extra.entropy_loss))
tf.contrib.summary.scalar('total_R', tf.reduce_sum(r))
tf.contrib.summary.histogram('actions', a)
def __init__(self, name, num_envs=4, n=20, entropy_reg_term=.05,
normalise_entropy=False, max_steps=5, learning_rate=0.001):
# Network parameters
# Hardcoded so as not to be global now
self.state_size = (80,80,3)
action_size = 6
kernel_size_1 = [8,8,3]
output_filters_conv1 = 32
output_filters_conv2 = 64
output_filters_conv3 = 64
hidden_size = 512 # number of units in hidden layer
lstm_size = 256
tf.reset_default_graph()
with tf.variable_scope(name):
self.name = name
# Input images
self.inputs_ = tf.placeholder(tf.float32, [None, self.state_size[0],
self.state_size[1], self.state_size[2]], name='inputs_')
# One hot encode the actions:
# [look_left, look_right, strafe_left, strafe_right, forward, backward]
self.actions = tf.placeholder(tf.int32, [None, num_envs], name='actions')
self.rewards = tf.placeholder(tf.float32, [None, num_envs], name='rewards')
# Conv layers
self.conv1 = tf.contrib.layers.conv2d(self.inputs_, output_filters_conv1, kernel_size=8, stride=2)
self.conv2 = tf.contrib.layers.conv2d(self.conv1, output_filters_conv2, kernel_size=4, stride=2)
self.conv3 = tf.contrib.layers.conv2d(self.conv2, output_filters_conv3, kernel_size=4, stride=1)
# Constructing input to AC network
self.actions_input = tf.reshape(tf.one_hot(self.actions, action_size), [-1, action_size])
self.rewards_input = tf.reshape(self.rewards, [-1, 1])
self.vision_input = tf.reshape(self.conv3, [-1, self.conv3.shape[1]*self.conv3.shape[2]*self.conv3.shape[3]])
self.ac_input = tf.concat([self.actions_input, self.rewards_input, self.vision_input], axis=1)
# FC Layer
self.fc1 = tf.contrib.layers.fully_connected(self.ac_input, hidden_size)
# LSTM Layer
self.lstm_cell = tf.nn.rnn_cell.LSTMCell(lstm_size, state_is_tuple=False)
self.lstm_hidden_state_input = tf.placeholder_with_default(
self.lstm_cell.zero_state(batch_size=num_envs, dtype=tf.float32),
[num_envs, hidden_size])
# Should be lstm_size not hidden_size?
self.lstm_input = tf.reshape(self.fc1, [-1, num_envs, hidden_size])
# Dynamic RNN code - might not need to be dynamic
self.lstm_output, self.lstm_hidden_state_output = tf.nn.dynamic_rnn(
self.lstm_cell,
self.lstm_input,
initial_state=self.lstm_hidden_state_input,
dtype=tf.float32,
time_major=True,
# parallel_iterations=num_envs, # Note: not sure what these do
# swap_memory=True, # Note: not sure what these do
)
self.lstm_output_flat = tf.reshape(self.lstm_output, [-1, lstm_size])
# Value function - Linear output layer
self.value_output = tf.contrib.layers.fully_connected(self.lstm_output_flat, 1,
activation_fn=None)
# Policy - softmax output layer
self.policy_logits = tf.contrib.layers.fully_connected(self.lstm_output_flat, action_size, activation_fn=None)
self.policy_output = tf.contrib.layers.softmax(self.policy_logits)
# Action sampling op
self.action_output = tf.squeeze(tf.multinomial(logits=self.policy_logits,num_samples=1), axis=1)
# Used for TRFL stuff
self.value_output_unflat = tf.reshape(self.value_output, [n, num_envs])
self.policy_logits_unflat = tf.reshape(self.policy_logits, [n, num_envs, -1])
self.discounts = tf.placeholder(tf.float32,[n, num_envs],name="discounts")
self.initial_Rs = tf.placeholder(tf.float32, [num_envs], name="initial_Rs")
#TRFL loss
a2c_loss, extra = trfl.sequence_advantage_actor_critic_loss(
policy_logits = self.policy_logits_unflat,
baseline_values = self.value_output_unflat,
actions = self.actions,
rewards = self.rewards,
pcontinues = self.discounts,
bootstrap_value = self.initial_Rs,
entropy_cost = entropy_reg_term,
normalise_entropy = normalise_entropy)
self.loss = tf.reduce_mean(a2c_loss)
self.extra = extra
self.opt = tf.train.AdamOptimizer(learning_rate).minimize(self.loss)
print("Network shapes:")
print("inputs_: ", self.inputs_.shape)
print("actions: ", self.actions.shape)
print("conv1: ", self.conv1.shape)
print("conv2: ", self.conv2.shape)
print("conv3: ", self.conv3.shape)
print("ac_input: ", self.ac_input.shape)
print("fc1: ", self.fc1.shape)
print("lstm_hidden_state_input: ", self.lstm_hidden_state_input.shape)
print("lstm_input: ", self.lstm_input.shape)
print("lstm_hidden_state_output: ", self.lstm_hidden_state_output.shape)
print("lstm_output: ", self.lstm_output.shape)
print("lstm_output_flat: ", self.lstm_output_flat.shape)
print("value_output: ", self.value_output.shape)
print("policy_logits: ", self.policy_logits.shape)
print("policy_output: ", self.policy_output.shape)
print("value_output_unflat: ", self.value_output_unflat.shape)
print("policy_logits_unflat: ", self.policy_logits_unflat.shape)
print(tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES))
graph = tf.get_default_graph()
# TODO: get this some other way?
conv1_w_summary = tf.summary.histogram('conv1 weights',graph.get_tensor_by_name("main_acn/Conv/weights:0"))
conv1_b_summary = tf.summary.histogram('conv1 biases',graph.get_tensor_by_name("main_acn/Conv/biases:0"))
conv2_w_summary = tf.summary.histogram('conv2 weights',graph.get_tensor_by_name("main_acn/Conv_1/weights:0"))
conv2_b_summary = tf.summary.histogram('conv2 biases',graph.get_tensor_by_name("main_acn/Conv_1/biases:0"))
conv3_w_summary = tf.summary.histogram('conv2 weights',graph.get_tensor_by_name("main_acn/Conv_2/weights:0"))
conv3_b_summary = tf.summary.histogram('conv3 biases',graph.get_tensor_by_name("main_acn/Conv_2/biases:0"))
fc1_w_summary = tf.summary.histogram('fc1 weights',graph.get_tensor_by_name("main_acn/fully_connected/weights:0"))
fc1_b_summary = tf.summary.histogram('fc1 biases',graph.get_tensor_by_name("main_acn/fully_connected/biases:0"))
lstm_w_summary = tf.summary.histogram('lstm weights',graph.get_tensor_by_name("main_acn/rnn/lstm_cell/kernel:0"))
lstm_b_summary = tf.summary.histogram('lstm biases',graph.get_tensor_by_name("main_acn/rnn/lstm_cell/bias:0"))
value_w_summary = tf.summary.histogram('value weights',graph.get_tensor_by_name("main_acn/fully_connected_1/weights:0"))
value_b_summary = tf.summary.histogram('value biases',graph.get_tensor_by_name("main_acn/fully_connected_1/biases:0"))
policy_w_summary = tf.summary.histogram('policy weights',graph.get_tensor_by_name("main_acn/fully_connected_2/weights:0"))
policy_b_summary = tf.summary.histogram('policy biases',graph.get_tensor_by_name("main_acn/fully_connected_2/biases:0"))
# Tensorboard
self.average_reward_metric = tf.placeholder(tf.float32, name="average_reward")
# self.average_length_of_episode = tf.placeholder(tf.float32, name="average_length_of_episode")
conv1_summary = tf.summary.histogram('conv1', self.conv1)
conv2_summary = tf.summary.histogram('conv2', self.conv2)
conv3_summary = tf.summary.histogram('conv3', self.conv3)
fc1_summary = tf.summary.histogram('fc1', self.fc1)
# lstm_summary = tf.summary.histogram('lstm', self.lstm_cell)
policy_summary = tf.summary.tensor_summary('policy', self.policy_output)
reward_summary = tf.summary.scalar('average_reward_metric', self.average_reward_metric)
loss_summary = tf.summary.scalar('loss', self.loss)
entropy_summary = tf.summary.scalar('policy_entropy', tf.math.reduce_mean(self.extra.entropy))
baseline_loss_summary = tf.summary.scalar('baseline_loss', tf.math.reduce_mean(self.extra.baseline_loss))
entropy_loss_summary = tf.summary.scalar('entropy_loss', tf.math.reduce_mean(self.extra.entropy_loss))
policy_gradient_loss = tf.summary.scalar('policy_gradient_loss', tf.math.reduce_mean(self.extra.policy_gradient_loss))
self.train_step_summary = tf.summary.merge([
reward_summary,
loss_summary,
entropy_summary,
baseline_loss_summary,
entropy_loss_summary,
policy_gradient_loss,
conv1_w_summary,
conv1_b_summary,
conv2_w_summary,
conv2_b_summary,
conv3_w_summary,
conv3_b_summary,
fc1_w_summary,
fc1_b_summary,
lstm_w_summary,
lstm_b_summary,
value_w_summary,
value_b_summary,
policy_w_summary,
policy_b_summary
])
self.action_step_summary = tf.summary.merge([policy_summary])
def __init__(self,
name,
obs_size=2,
action_size=2,
actor_hidden_size=32,
critic_hidden_size=32,
ac_learning_rate=0.001,
entropy_cost=0.01,
normalise_entropy=True,
lambda_=0.,
baseline_cost=1.):
with tf.variable_scope(name):
# network variables
self.name = name
self.input_ = tf.placeholder(tf.float32, [None, obs_size],
name='inputs')
self.action_ = tf.placeholder(tf.int32, [None, 1], name='action')
self.reward_ = tf.placeholder(tf.float32, [None, 1], name='reward')
self.discount_ = tf.placeholder(tf.float32, [None, 1],
name='discount')
self.bootstrap_ = tf.placeholder(tf.float32, [None],
name='bootstrap')
# set up actor network (approximates optimal policy)
self.fc1_actor_ = tf.contrib.layers.fully_connected(
self.input_, actor_hidden_size, activation_fn=tf.nn.elu)
self.fc2_actor_ = tf.contrib.layers.fully_connected(
self.fc1_actor_, actor_hidden_size, activation_fn=tf.nn.elu)
self.fc3_actor_ = tf.contrib.layers.fully_connected(
self.fc2_actor_, action_size, activation_fn=None)
# reshape the policy logits
self.policy_logits_ = tf.reshape(self.fc3_actor_,
[-1, 1, action_size])
# generate action probabilities for taking actions
self.action_prob_ = tf.nn.softmax(self.fc3_actor_)
# set up critic network (approximates optimal state-value function (used as a baseline to reduce variance of loss gradient))
# - uses policy evaluation (e.g. Monte-Carlo / TD learning) to estimate the advantage
self.fc1_critic_ = tf.contrib.layers.fully_connected(
self.input_, critic_hidden_size, activation_fn=tf.nn.elu)
self.fc2_critic_ = tf.contrib.layers.fully_connected(
self.fc1_critic_, critic_hidden_size, activation_fn=tf.nn.elu)
self.baseline_ = tf.contrib.layers.fully_connected(
self.fc2_critic_, 1, activation_fn=None)
# Calculates the loss for an A2C update along a batch of trajectories. (TRFL)
self.seq_aac_return_ = trfl.sequence_advantage_actor_critic_loss(
self.policy_logits_,
self.baseline_,
self.action_,
self.reward_,
self.discount_,
self.bootstrap_,
lambda_=lambda_,
entropy_cost=entropy_cost,
baseline_cost=baseline_cost,
normalise_entropy=normalise_entropy)
# Optimize the loss
self.ac_loss_ = tf.reduce_mean(self.seq_aac_return_.loss)
self.ac_optim_ = tf.train.AdamOptimizer(
learning_rate=ac_learning_rate).minimize(self.ac_loss_)
def __init__(self, name):
# state inputs to the Q-network
with tf.variable_scope(name):
# self.inputs_ = tf.placeholder(tf.float32, [n, num_envs, state_size[0], state_size[1], state_size[2]], name='inputs')
# self.inputs_flat = tf.reshape(self.inputs_, [n * num_envs, state_size[0], state_size[1], state_size[2]])
self.inputs_ = tf.placeholder(
tf.float32,
[None, state_size[0], state_size[1], state_size[2]],
name='inputs')
# Actions for the QNetwork:
# One-hot vector, with each action being as follows:
# (look_left, look_right, strafe_left, strafe_right, forward, backward)
# These are mapped to the deepmind-lab (not one-hot) actions with the same names
# defined in ACTIONS
# One hot encode the actions to later choose the Q-value for the action
self.actions_ = tf.placeholder(tf.int32, [n, num_envs],
name='actions')
# ReLU hidden layers
self.conv1 = tf.contrib.layers.conv2d(self.inputs_,
output_filters_conv1,
kernel_size=8,
stride=2)
self.conv2 = tf.contrib.layers.conv2d(self.conv1,
output_filters_conv2,
kernel_size=4,
stride=2)
self.conv3 = tf.contrib.layers.conv2d(self.conv2,
output_filters_conv3,
kernel_size=4,
stride=1)
self.fc1 = tf.contrib.layers.fully_connected( \
tf.reshape(self.conv3, [-1, self.conv3.shape[1]*self.conv3.shape[2]*self.conv3.shape[3]]), \
hidden_size)
# Value function - Linear output layer
self.value_output = tf.contrib.layers.fully_connected(
self.fc1, 1, activation_fn=None)
# Policy - softmax output layer
self.policy_logits = tf.contrib.layers.fully_connected(
self.fc1, action_size, activation_fn=None)
self.policy_output = tf.contrib.layers.softmax(self.policy_logits)
self.action_output = tf.squeeze(tf.multinomial(
logits=self.policy_logits, num_samples=1),
axis=1)
self.name = name
self.rewards = tf.placeholder(tf.float32, [n, num_envs],
name="rewards")
self.discounts = tf.placeholder(tf.float32, [n, num_envs],
name="discounts")
self.initial_Rs = tf.placeholder(tf.float32, [num_envs],
name="initial_Rs")
# Used for trfl stuff
self.value_output_unflat = tf.reshape(self.value_output,
[n, num_envs])
self.policy_logits_unflat = tf.reshape(self.policy_logits,
[n, num_envs, -1])
print("Network shapes:")
print("actions_: ", self.actions_.shape)
print("conv1: ", self.conv1.shape)
print("conv2: ", self.conv2.shape)
print("conv3: ", self.conv3.shape)
print("fc1: ", self.fc1.shape)
print("value_output: ", self.value_output.shape)
print("policy_logits: ", self.policy_logits.shape)
print("policy_output: ", self.policy_output.shape)
print("policy_logits_unflat: ", self.policy_logits_unflat.shape)
print("value_output_unflat: ", self.value_output_unflat.shape)
#TRFL qlearning
a2c_loss, extra = trfl.sequence_advantage_actor_critic_loss(
policy_logits=self.policy_logits_unflat,
baseline_values=self.value_output_unflat,
actions=self.actions_,
rewards=self.rewards,
pcontinues=self.discounts,
bootstrap_value=self.initial_Rs,
entropy_cost=entropy_reg_term,
normalise_entropy=normalise_entropy)
self.loss = tf.reduce_mean(a2c_loss)
self.extra = extra
self.opt = tf.train.AdamOptimizer(learning_rate).minimize(
self.loss)
def __init__(self,
name,
state_size=2,
action_size=2,
actor_hidden_size=32,
critic_hidden_size=32,
ac_learning_rate=0.001,
entropy_cost=0.01,
normalise_entropy=True,
lambda_=0.,
baseline_cost=1.):
with tf.variable_scope(name):
# placeholders
self.name = name
self.input = tf.placeholder(tf.float32, [None, state_size],
name='input')
self.action = tf.placeholder(tf.int32, [None, 1], name='action')
self.reward = tf.placeholder(tf.float32, [None, 1], name='reward')
self.discount = tf.placeholder(tf.float32, [None, 1],
name='discount')
self.bootstrap = tf.placeholder(tf.float32, [None],
name='bootstrap')
# actor net
self.actor_hidden_1 = tf.contrib.layers.fully_connected(
self.input, actor_hidden_size, activation_fn=tf.nn.relu)
self.actor_hidden_2 = tf.contrib.layers.fully_connected(
self.actor_hidden_1,
actor_hidden_size,
activation_fn=tf.nn.relu)
self.actor_out = tf.contrib.layers.fully_connected(
self.actor_hidden_2, action_size, activation_fn=None)
# policy logits
self.policy_logits = tf.reshape(self.actor_out,
[-1, 1, action_size],
name='policy_logits')
# action choice
self.action_choice = tf.nn.softmax(self.actor_out)
# critic net
self.critic_hidden_1 = tf.contrib.layers.fully_connected(
self.input, critic_hidden_size, activation_fn=tf.nn.relu)
self.critic_hidden_2 = tf.contrib.layers.fully_connected(
self.critic_hidden_1,
critic_hidden_size,
activation_fn=tf.nn.relu)
self.critic_out = tf.contrib.layers.fully_connected(
self.critic_hidden_2, critic_hidden_size, activation_fn=None)
# loss function
self.acloss = trfl.sequence_advantage_actor_critic_loss(
self.policy_logits,
self.critic_out,
self.action,
self.reward,
self.discount,
self.bootstrap,
lambda_=lambda_,
entropy_cost=entropy_cost,
baseline_cost=baseline_cost,
normalise_entropy=normalise_entropy)
# optimizer
self.ac_optim = tf.reduce_mean(self.acloss.loss)
self.optimizer = tf.train.AdamOptimizer(
learning_rate=ac_learning_rate).minimize(self.ac_optim)
def loss(self, observations, rewards, actions, additional_rewards=None):
"""Compute the loss."""
dummy_zeroth_step_actions = tf.zeros_like(actions[:1])
all_actions = tf.concat([dummy_zeroth_step_actions, actions], axis=0)
inputs, features = snt.BatchApply(self._encode)(observations, rewards,
all_actions)
rewards = rewards[1:] # Zeroth step reward not correlated to actions.
if additional_rewards is not None:
# Additional rewards are not passed to the encoder (above) in order to be
# consistent with the step, nor to the recon loss so that recons are
# consistent with the observations. Thus, additional rewards only affect
# the returns used to learn the value function.
rewards += additional_rewards
initial_state = self._core.initial_state(self._batch_size)
rnn_inputs = features
core_outputs = unroll(self._core, initial_state, rnn_inputs)
# Remove final timestep of outputs.
core_outputs = nest.map_structure(lambda t: t[:-1], core_outputs)
if self._with_reconstructions:
recons = snt.BatchApply(self._decode)(core_outputs.z)
recon_targets = nest.map_structure(lambda t: t[:-1], inputs)
recon_loss, recon_logged_values = losses.reconstruction_losses(
recons=recons,
targets=recon_targets,
image_cost=self._image_cost_weight / self._total_num_pixels,
action_cost=1.,
reward_cost=1.)
else:
recon_loss = tf.constant(0.0)
recon_logged_values = dict()
if core_outputs.read_info is not tuple():
read_reg_loss, read_reg_logged_values = (
losses.read_regularization_loss(
read_info=core_outputs.read_info,
strength_cost=self._read_strength_cost,
strength_tolerance=self._read_strength_tolerance,
strength_reg_mode='L1',
key_norm_cost=0.,
key_norm_tolerance=1.))
else:
read_reg_loss = tf.constant(0.0)
read_reg_logged_values = dict()
# Bootstrap value is at end of episode so is zero.
bootstrap_value = tf.zeros(shape=(self._batch_size, ),
dtype=tf.float32)
discounts = self._gamma * tf.ones_like(rewards)
a2c_loss, a2c_loss_extra = trfl.sequence_advantage_actor_critic_loss(
policy_logits=core_outputs.policy,
baseline_values=core_outputs.baseline,
actions=actions,
rewards=rewards,
pcontinues=discounts,
bootstrap_value=bootstrap_value,
lambda_=self._gamma,
entropy_cost=self._entropy_cost,
baseline_cost=self._return_cost_weight,
name='SequenceA2CLoss')
a2c_loss = tf.reduce_mean(a2c_loss) # Average over batch.
total_loss = a2c_loss + recon_loss + read_reg_loss
a2c_loss_logged_values = dict(
pg_loss=tf.reduce_mean(a2c_loss_extra.policy_gradient_loss),
baseline_loss=tf.reduce_mean(a2c_loss_extra.baseline_loss),
entropy_loss=tf.reduce_mean(a2c_loss_extra.entropy_loss))
agent_loss_log = losses.combine_logged_values(a2c_loss_logged_values,
recon_logged_values,
read_reg_logged_values)
agent_loss_log['total_loss'] = total_loss
return total_loss, agent_loss_log
def __init__(self,
name,
obs_size=2,
action_size=2,
actor_hidden_size=32,
critic_hidden_size=32,
pow_learning_rate=0.001,
RB_learning_rate=0.001,
entropy_cost=0.01,
normalise_entropy=True,
lambda_=0.,
baseline_cost=1.):
with tf.variable_scope(name):
# define inputs placeholders for the networks
self.name = name
self.input_ = tf.placeholder(tf.float32, [None, obs_size],
name='inputs')
self.action_RB_ = tf.placeholder(tf.int32, [None, 1],
name='action_RB')
self.action_pow_ = tf.placeholder(tf.int32, [None, 1],
name='action_pow')
self.reward_ = tf.placeholder(tf.float32, [None, 1], name='reward')
self.discount_ = tf.placeholder(tf.float32, [None, 1],
name='discount')
self.bootstrap_ = tf.placeholder(tf.float32, [None],
name='bootstrap')
# set up actor network
self.fc1_actor_ = tf.contrib.layers.fully_connected(
self.input_, actor_hidden_size, activation_fn=tf.nn.elu)
self.fc2_actor_ = tf.contrib.layers.fully_connected(
self.fc1_actor_, actor_hidden_size, activation_fn=tf.nn.elu)
self.fc3_actor_ = tf.contrib.layers.fully_connected(
self.fc2_actor_, actor_hidden_size, activation_fn=tf.nn.elu)
self.fc4_actor_power_ = tf.contrib.layers.fully_connected(
self.fc3_actor_, ch.D2D_tr_Power_levels, activation_fn=None)
self.fc4_actor_RB_ = tf.contrib.layers.fully_connected(
self.fc3_actor_, ch.N_CU, activation_fn=None)
# reshape the policy logits
self.policy_logits_RB_ = tf.reshape(self.fc4_actor_RB_,
(-1, 1, ch.N_CU))
self.policy_logits_power_ = tf.reshape(
self.fc4_actor_power_, (-1, 1, ch.D2D_tr_Power_levels))
# generate action probabilities for taking actions
self.action_prob_power_ = tf.nn.softmax(self.fc4_actor_power_)
self.action_prob_RB_ = tf.nn.softmax(self.fc4_actor_RB_)
# set up critic network
self.fc1_critic_ = tf.contrib.layers.fully_connected(
self.input_, critic_hidden_size, activation_fn=tf.nn.elu)
self.fc2_critic_ = tf.contrib.layers.fully_connected(
self.fc1_critic_, critic_hidden_size, activation_fn=tf.nn.elu)
self.baseline_ = tf.contrib.layers.fully_connected(
self.fc2_critic_, 1, activation_fn=None)
# Define Loss with TRFL
self.seq_aac_return_pow_ = trfl.sequence_advantage_actor_critic_loss(
self.policy_logits_power_,
self.baseline_,
self.action_pow_,
self.reward_,
self.discount_,
self.bootstrap_,
lambda_=lambda_,
entropy_cost=entropy_cost,
baseline_cost=baseline_cost,
normalise_entropy=normalise_entropy)
self.seq_aac_return_RB_ = trfl.sequence_advantage_actor_critic_loss(
self.policy_logits_RB_,
self.baseline_,
self.action_RB_,
self.reward_,
self.discount_,
self.bootstrap_,
lambda_=lambda_,
entropy_cost=entropy_cost,
baseline_cost=baseline_cost,
normalise_entropy=normalise_entropy)
# Optimize the loss
self.ac_loss_pow_ = tf.reduce_mean(self.seq_aac_return_pow_.loss)
self.ac_loss_RB_ = tf.reduce_mean(self.seq_aac_return_RB_.loss)
self.ac_optim_pow_ = tf.train.AdamOptimizer(
learning_rate=pow_learning_rate).minimize(self.ac_loss_pow_)
self.ac_optim_RB_ = tf.train.AdamOptimizer(
learning_rate=RB_learning_rate).minimize(self.ac_loss_RB_)
def __init__(self, name):
with tf.variable_scope(name):
self.name = name
# Input images
self.inputs_ = tf.placeholder(
tf.float32,
[None, state_size[0], state_size[1], state_size[2]],
name='inputs')
# One hot encode the actions:
# [look_left, look_right, strafe_left, strafe_right, forward, backward]
self.actions = tf.placeholder(tf.int32, [None, num_envs],
name='actions')
self.rewards = tf.placeholder(tf.float32, [None, num_envs],
name='rewards')
# Conv layers
self.conv1 = tf.contrib.layers.conv2d(self.inputs_,
output_filters_conv1,
kernel_size=8,
stride=2)
self.conv2 = tf.contrib.layers.conv2d(self.conv1,
output_filters_conv2,
kernel_size=4,
stride=2)
self.conv3 = tf.contrib.layers.conv2d(self.conv2,
output_filters_conv3,
kernel_size=4,
stride=1)
# Constructing input to AC network
self.actions_input = tf.reshape(
tf.one_hot(self.actions, action_size), [-1, action_size])
self.rewards_input = tf.reshape(self.rewards, [-1, 1])
self.vision_input = tf.reshape(self.conv3, [
-1,
self.conv3.shape[1] * self.conv3.shape[2] * self.conv3.shape[3]
])
self.ac_input = tf.concat(
[self.actions_input, self.rewards_input, self.vision_input],
axis=1)
# FC Layer
self.fc1 = tf.contrib.layers.fully_connected(
self.ac_input, hidden_size)
# LSTM Layer
self.lstm_cell = tf.nn.rnn_cell.LSTMCell(lstm_size,
state_is_tuple=False)
self.lstm_hidden_state_input = tf.placeholder_with_default(
self.lstm_cell.zero_state(batch_size=num_envs,
dtype=tf.float32),
[num_envs, hidden_size])
# Should be lstm_size not hidden_size?
self.lstm_input = tf.reshape(self.fc1, [-1, num_envs, hidden_size])
# Dynamic RNN code - might not need to be dynamic
self.lstm_output, self.lstm_hidden_state_output = tf.nn.dynamic_rnn(
self.lstm_cell,
self.lstm_input,
initial_state=self.lstm_hidden_state_input,
dtype=tf.float32,
time_major=True,
# parallel_iterations=num_envs, # Note: not sure what these do
# swap_memory=True, # Note: not sure what these do
)
self.lstm_output_flat = tf.reshape(self.lstm_output,
[-1, lstm_size])
# TODO: rethink layer shapes?
# Value function - Linear output layer
self.value_output = tf.contrib.layers.fully_connected(
self.lstm_output_flat, 1, activation_fn=None)
# Policy - softmax output layer
self.policy_logits = tf.contrib.layers.fully_connected(
self.lstm_output_flat, action_size, activation_fn=None)
self.policy_output = tf.contrib.layers.softmax(self.policy_logits)
# Action sampling op
self.action_output = tf.squeeze(tf.multinomial(
logits=self.policy_logits, num_samples=1),
axis=1)
# Used for TRFL stuff
self.value_output_unflat = tf.reshape(self.value_output,
[n, num_envs])
self.policy_logits_unflat = tf.reshape(self.policy_logits,
[n, num_envs, -1])
self.discounts = tf.placeholder(tf.float32, [n, num_envs],
name="discounts")
self.initial_Rs = tf.placeholder(tf.float32, [num_envs],
name="initial_Rs")
#TRFL loss
a2c_loss, extra = trfl.sequence_advantage_actor_critic_loss(
policy_logits=self.policy_logits_unflat,
baseline_values=self.value_output_unflat,
actions=self.actions,
rewards=self.rewards,
pcontinues=self.discounts,
bootstrap_value=self.initial_Rs,
entropy_cost=entropy_reg_term,
normalise_entropy=normalise_entropy)
self.loss = tf.reduce_mean(a2c_loss)
self.extra = extra
self.opt = tf.train.AdamOptimizer(learning_rate).minimize(
self.loss)
print("Network shapes:")
print("inputs_: ", self.inputs_.shape)
print("actions: ", self.actions.shape)
print("conv1: ", self.conv1.shape)
print("conv2: ", self.conv2.shape)
print("conv3: ", self.conv3.shape)
print("ac_input: ", self.ac_input.shape)
print("fc1: ", self.fc1.shape)
print("lstm_hidden_state_input: ",
self.lstm_hidden_state_input.shape)
print("lstm_input: ", self.lstm_input.shape)
print("lstm_hidden_state_output: ",
self.lstm_hidden_state_output.shape)
print("lstm_output: ", self.lstm_output.shape)
print("lstm_output_flat: ", self.lstm_output_flat.shape)
print("value_output: ", self.value_output.shape)
print("policy_logits: ", self.policy_logits.shape)
print("policy_output: ", self.policy_output.shape)
print("value_output_unflat: ", self.value_output_unflat.shape)
print("policy_logits_unflat: ", self.policy_logits_unflat.shape)
# Tensorboard
self.average_reward_metric = tf.placeholder(tf.float32,
name="average_reward")
# self.average_length_of_episode = tf.placeholder(tf.float32, name="average_length_of_episode")
policy_summary = tf.summary.tensor_summary('policy',
self.policy_output)
reward_summary = tf.summary.scalar('average_reward_metric',
self.average_reward_metric)
loss_summary = tf.summary.scalar('loss', self.loss)
entropy_summary = tf.summary.scalar(
'policy_entropy', tf.math.reduce_mean(self.extra.entropy))
baseline_loss_summary = tf.summary.scalar(
'baseline_loss', tf.math.reduce_mean(self.extra.baseline_loss))
entropy_loss_summary = tf.summary.scalar(
'entropy_loss', tf.math.reduce_mean(self.extra.entropy_loss))
policy_gradient_loss = tf.summary.scalar(
'policy_gradient_loss',
tf.math.reduce_mean(self.extra.policy_gradient_loss))
self.train_step_summary = tf.summary.merge([
reward_summary, loss_summary, entropy_summary,
baseline_loss_summary, entropy_loss_summary,
policy_gradient_loss
])
self.action_step_summary = tf.summary.merge([policy_summary])