def test_summarizer(self):
# Bulk Tests
with tf.Graph().as_default():
x = tf.placeholder("float", [None, 4])
W = tf.Variable(tf.random_normal([4, 4]))
x = tf.nn.tanh(tf.matmul(x, W))
tf.add_to_collection(tf.GraphKeys.ACTIVATIONS, x)
import tflearn.helpers.summarizer as s
s.summarize_variables([W])
s.summarize_activations(tf.get_collection(tf.GraphKeys.ACTIVATIONS))
s.summarize(x, 'histogram', "test_summary")
def test_summarizer(self):
# Bulk Tests
with tf.Graph().as_default():
x = tf.placeholder("float", [None, 4])
W = tf.Variable(tf.random_normal([4, 4]))
x = tf.nn.tanh(tf.matmul(x, W))
tf.add_to_collection(tf.GraphKeys.ACTIVATIONS, x)
import tflearn.helpers.summarizer as s
s.summarize_variables([W])
s.summarize_activations(tf.get_collection(
tf.GraphKeys.ACTIVATIONS))
s.summarize(x, 'histogram', "test_summary")
def train(self, tf_record_dir, save_dir, num_epochs, batch_size, num_save_every,
model_file=None, early_stop=False, full_eval_every=0, learning_rate=1e-3, lr_steps=0, lr_decay=0.96, avg = False):
"""Trains the Model defined in module on the records in tf_record_dir"""
train_loss_iterations = {'iteration': [], 'epoch': [], 'train_loss': [], 'train_dice': [],
'train_mse': [], 'val_loss': [], 'val_dice': [], 'val_mse': []}
meta_data_filepath = os.path.join(tf_record_dir, 'meta_data.txt')
with open(meta_data_filepath, 'r') as f:
meta_data = json.load(f)
num_examples = sum([x[1] for x in meta_data['train_examples'].items()])
num_batches_per_epoch = num_examples//batch_size
num_batches = math.ceil(num_epochs*num_batches_per_epoch) if num_epochs else 0
num_full_validation_every = full_eval_every if full_eval_every else num_batches_per_epoch
validation_tups = meta_data['validation_tups']
full_validation_metrics = {k[0]:[] for k in validation_tups}
if not os.path.exists(save_dir):
os.makedirs(save_dir)
write_csv(os.path.join(save_dir, 'log.csv'), train_loss_iterations)
write_csv(os.path.join(save_dir, 'full_validation_log.csv'), full_validation_metrics)
with tf.Graph().as_default():
config = tf.compat.v1.ConfigProto()
config.gpu_options.allow_growth = True
tflearn.config.init_training_mode()
with tf.compat.v1.name_scope('training'):
model = self.module.Model(batch_size, False, tf_record_dir, num_epochs)
with tf.compat.v1.name_scope('eval'):
model_eval = self.module.Model(batch_size, True, tf_record_dir,
num_epochs) if validation_tups else None
model_test = self.module.Model(batch_size, True)
inferer = model_test.build_full_inferer()
avg_time = 0
global_step = model.global_step #tf.Variable(0, name='global_step', trainable=False)
lr = model.lr #tf.train.exponential_decay(learning_rate, global_step, lr_steps, lr_decay, staircase=True) if lr_steps else learning_rate
optimizer = model.optimizer #tf.train.AdamOptimizer(lr of decay_steps)
tf.compat.v1.summary.scalar('learning_rate', lr)
update_op = self._avg(model.loss_op, optimizer, global_step) if avg else optimizer.minimize(model.loss_op,
global_step=global_step)
#update_op = optimizer.minimize(model.loss_op,
# global_step=global_step)
#config = tf.ConfigProto()
#config.gpu_options.allow_growth = True
with tf.compat.v1.Session(config=config) as sess:
merged = s.summarize_variables()
merged = tf.compat.v1.summary.merge_all()
summary_writer = tf.compat.v1.summary.FileWriter(save_dir, sess.graph)
sess.run(tf.compat.v1.local_variables_initializer())
sess.run(tf.compat.v1.global_variables_initializer())
saver = tf.compat.v1.train.Saver()
saver_epoch = tf.compat.v1.train.Saver(max_to_keep=None)
coord = tf.train.Coordinator()
if model_file:
if avg:
variable_averages = tf.train.ExponentialMovingAverage(0.999)
vars_to_restore = variable_averages.variables_to_restore()
vars_to_restore = model.filter_vars(vars_to_restore)
restore_saver = (tf.compat.v1.train.Saver(vars_to_restore)
if vars_to_restore else tf.compat.v1.train.Saver())
restore_saver.restore(sess, model_file)
else:
vars_to_restore = model.filter_vars(tf.compat.v1.global_variables())
restore_saver = (tf.compat.v1.train.Saver(vars_to_restore)
if vars_to_restore else tf.compat.v1.train.Saver())
restore_saver.restore(sess, model_file)
try:
continue_training = True
epoch = previous_epoch = 0
while continue_training and not coord.should_stop():
tflearn.is_training(True)
cur_step = sess.run(global_step)
previous_epoch = epoch
epoch = cur_step//num_batches_per_epoch
start = time.time()
if epoch != previous_epoch: saver = tf.compat.v1.train.Saver()
#print(tf.get_collection('summaries'))
#if num_batches_val and cur_step % num_save_every == 0:
message_string = ''
if num_save_every and cur_step % num_save_every == 0:
tflearn.is_training(False)
start = time.time()
(train_loss, train_dice, val_loss, val_dice, summaries,
train_mse, val_mse) = sess.run([model.loss_op,
model.dice_op,
model_eval.loss_op,
model_eval.dice_op,
merged, model.mse,
model_eval.mse,
] +
model_eval.metric_update_ops +
model.metric_update_ops)[0:7]
summary_writer.add_summary(summaries, cur_step)
message_string = (' val_loss: {:.3f}, val_dice: {:.3f}, mse: {:.5f}'
.format(val_loss, val_dice, val_mse))
non_zero_losses = [x for x in train_loss_iterations['val_loss']
if x is not None]
non_zero_dices = [x for x in train_loss_iterations['val_dice']
if x is not None]
non_zero_val_mses = [x for x in train_loss_iterations['val_mse']
if x is not None and x > 0]
should_save = (val_loss < np.percentile(non_zero_losses, 25)
if non_zero_losses else True)
should_save = (should_save and val_dice > np.percentile(non_zero_dices, 75)
if non_zero_dices else True)
min_mse = 0 if not non_zero_val_mses else np.min(non_zero_val_mses)
should_save = should_save and val_mse < min_mse and min_mse
train_loss_iterations['val_loss'].append(val_loss)
train_loss_iterations['val_dice'].append(val_dice)
train_loss_iterations['val_mse'].append(val_mse)
if should_save:
checkpoint_path = os.path.join(save_dir, 'epoch_'
+ str(epoch) + '_model.ckpt')
saver.save(sess, checkpoint_path, global_step=global_step)
print("model saved to {}".format(checkpoint_path))
non_zero_val_mses = [x for x in train_loss_iterations['val_mse']
if x is not None and x > 0]
if early_stop:
continue_training = (np.median(non_zero_val_mses[-10:])
<= np.min(non_zero_val_mses[:-10])
if non_zero_val_mses[:-10] and epoch > 0
else True)
else:
train_loss_iterations['val_loss'].append(None)
train_loss_iterations['val_dice'].append(None)
train_loss_iterations['val_mse'].append(None)
train_loss, train_dice, train_mse = sess.run([model.loss_op,
model.dice_op,
model.mse
])[0:3]
train_loss_iterations['iteration'].append(cur_step)
train_loss_iterations['epoch'].append(epoch)
train_loss_iterations['train_loss'].append(train_loss)
train_loss_iterations['train_dice'].append(train_dice)
train_loss_iterations['train_mse'].append(train_mse)
write_csv(os.path.join(save_dir, 'log.csv'),
{k:([v[-1]] if v else [])
for k, v in train_loss_iterations.items()},
mode='a', header=False)
end = time.time()
avg_time = avg_time + ((end-start)-avg_time)/cur_step if cur_step else end - start
time_remaining_s = (num_batches - cur_step)*avg_time if num_epochs else 0
t = timedelta(seconds=time_remaining_s)
time_remaining_string = ("time left: {}m-{}d {} (h:mm:ss)"
.format(t.days/30, t.days%30,
timedelta(seconds=t.seconds)))
message_string = ("{}/{} (epoch {}), train_loss = {:.3f}, dice = {:.3f}, accuracy = {:.3f}, time/batch = {:.3f}, "
.format(cur_step, num_batches, epoch, train_loss, train_dice,
0, end - start) + time_remaining_string +
message_string)
print(message_string)
if all([num_full_validation_every,
cur_step % num_full_validation_every == 0,
cur_step]):
tflearn.is_training(False)
start = time.time()
test_save_dir = os.path.join(save_dir, 'val_preds_' + str(cur_step))
if not os.path.exists(test_save_dir): os.makedirs(test_save_dir)
for tup in validation_tups:
dice = inferer(sess, tup, test_save_dir, model_test)
full_validation_metrics[tup[0]].append(dice)
print(dice)
end = time.time()
avg_time = avg_time + ((end-start)/num_full_validation_every
- avg_time)/cur_step if cur_step else avg_time
checkpoint_path = os.path.join(save_dir, 'epoch_model.ckpt')
saver_epoch.save(sess, checkpoint_path, global_step=global_step)
print("epoch model saved to {}".format(checkpoint_path))
write_csv(os.path.join(save_dir, 'full_validation_log.csv'),
{k:([v[-1]] if v else [])
for k, v in full_validation_metrics.items()},
mode='a',
header=False)
#Run optimiser after saving/evaluating the model
sess.run(update_op)
except tf.errors.OutOfRangeError:
print('Done training')
finally:
checkpoint_path = os.path.join(save_dir, 'model.ckpt')
saver.save(sess, checkpoint_path, global_step=global_step)
coord.request_stop()
print('Finished')
def create_network_graph(self):
input_shape = self._input_shape
output_num = self._output_num
# Input placeholders
with tf.name_scope('input'):
# we need to fix the input shape from (batch, filter, height, width) to
# tensorflow which is (batch, height, width, filter)
self._t_x_input_channel_firstdim = tf.placeholder(tf.uint8, [None] + input_shape, name='x-input')
# transpose because tf wants channels on last dim and channels are passed in on 2nd dim
self._t_x_input = tf.cast(tf.transpose(self._t_x_input_channel_firstdim, perm=[0, 2, 3, 1]), tf.float32) / 255.0
self._t_x_input_tp1_channel_firstdim = tf.placeholder(tf.uint8, [None] + input_shape, name='x-input-tp1')
# transpose because tf wants channels on last dim and channels are passed in on 2nd dim
self._t_x_input_tp1 = tf.cast(tf.transpose(self._t_x_input_tp1_channel_firstdim, perm=[0, 2, 3, 1]), tf.float32) / 255.0
self._t_x_actions = tf.placeholder(tf.int32, shape=[None], name='x-actions')
self._t_x_rewards = tf.placeholder(tf.float32, shape=[None], name='x-rewards')
self._t_x_terminals = tf.placeholder(tf.bool, shape=[None], name='x-terminals')
self._t_x_discount = self._q_discount
# Target network does not reuse variables
with tf.variable_scope('network') as var_scope:
self._t_network_output = self._network_generator(self._t_x_input, output_num)
# get the trainable variables for this network, later used to overwrite target network vars
self._tf_network_trainables = tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES, scope='network')
# summarize activations
summarizer.summarize_activations(tf.get_collection(tf.GraphKeys.ACTIVATIONS, scope='network'))
# if double DQN then we need to create network output for s_tp1
if self.algorithm_type == 'double' or self.algorithm_type == 'doublenstep':
var_scope.reuse_variables()
self._t_network_output_tp1 = self._network_generator(self._t_x_input_tp1, output_num)
# summarize a histogram of each action output
for output_ind in range(output_num):
summarizer.summarize(self._t_network_output[:, output_ind], 'histogram', 'network-output/{0}'.format(output_ind))
# add network summaries
summarizer.summarize_variables(train_vars=self._tf_network_trainables)
with tf.variable_scope('target-network'):
self._t_target_network_output = self._network_generator(self._t_x_input_tp1, output_num)
# get trainables for target network, used in assign op for the update target network step
target_network_trainables = tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES, scope='target-network')
# update target network with network variables
with tf.name_scope('update-target-network'):
self._tf_update_target_network_ops = [target_v.assign(v) for v, target_v in zip(self._tf_network_trainables, target_network_trainables)]
# if double convience function to get target values for online action
if self.algorithm_type == 'double' or self.algorithm_type == 'doublenstep':
with tf.name_scope('double_target'):
# Target = target_Q(s_tp1, argmax(online_Q(s_tp1)))
argmax_tp1 = tf.argmax(self._t_network_output_tp1, axis=1)
self._t_target_value_online_action = tf_util.one_hot(self._t_target_network_output, argmax_tp1, output_num)
# caclulate QLoss
with tf.name_scope('loss'):
# nstep rewards are calculated outside the gpu/graph because it requires a loop
if self.algorithm_type != 'nstep' and self.algorithm_type != 'doublenstep':
with tf.name_scope('estimated-reward-tp1'):
if self.algorithm_type == 'double':
# Target = target_Q(s_tp1, argmax(online_Q(s_tp1)))
target = self._t_target_value_online_action
elif self.algorithm_type == 'dqn':
# Target = max(target_Q(s_tp1))
target = tf.reduce_max(self._t_target_network_output, axis=1)
# compute a mask that returns gamma (discount factor) or 0 if terminal
terminal_discount_mask = tf.multiply(1.0 - tf.cast(self._t_x_terminals, tf.float32), self._t_x_discount)
est_rew_tp1 = tf.multiply(terminal_discount_mask, target)
y = self._t_x_rewards + tf.stop_gradient(est_rew_tp1)
# else nstep
else:
y = self._t_x_rewards
with tf.name_scope('estimated-reward'):
est_rew = tf_util.one_hot(self._t_network_output, self._t_x_actions, output_num)
with tf.name_scope('qloss'):
# clip loss but keep linear past clip bounds (huber loss with customizable linear part)
# REFS: https://github.com/spragunr/deep_q_rl/blob/master/deep_q_rl/q_network.py#L108
# https://github.com/Jabberwockyll/deep_rl_ale/blob/master/q_network.py#L241
diff = y - est_rew
if self._loss_clipping > 0.0:
abs_diff = tf.abs(diff)
# same as min(diff, loss_clipping) because diff can never be negative (definition of abs value)
quadratic_part = tf.clip_by_value(abs_diff, 0.0, self._loss_clipping)
linear_part = abs_diff - quadratic_part
loss = (0.5 * tf.square(quadratic_part)) + (self._loss_clipping * linear_part)
else:
# But why multiply the loss by 0.5 when not clipping? https://groups.google.com/forum/#!topic/deep-q-learning/hKK0ZM_OWd4
loss = 0.5 * tf.square(diff)
# NOTICE: we are summing gradients
error = tf.reduce_sum(loss)
summarizer.summarize(error, 'scalar', 'loss')
# optimizer
with tf.name_scope('shared-optimizer'):
self._tf_learning_rate = tf.placeholder(tf.float32)
optimizer = self._optimizer_fn(learning_rate=self._tf_learning_rate)
# only train the network vars not the target network
gradients = optimizer.compute_gradients(error, var_list=self._tf_network_trainables)
# gradients are stored as a tuple, (gradient, tensor the gradient corresponds to)
# kinda lame that clip by global norm doesn't accept the list of tuples returned from compute_gradients
# so we unzip then zip
tensors = [tensor for gradient, tensor in gradients]
grads = [gradient for gradient, tensor in gradients]
clipped_gradients, _ = tf.clip_by_global_norm(grads, self.global_norm_clipping) # returns list[tensors], norm
clipped_grads_tensors = zip(clipped_gradients, tensors)
self._tf_train_step = optimizer.apply_gradients(clipped_grads_tensors)
# tflearn smartly knows how gradients are stored so we just pass in the list of tuples
summarizer.summarize_gradients(clipped_grads_tensors)
# tf learn auto merges all summaries so we just have to grab the last output
self._tf_summaries = summarizer.summarize(self._tf_learning_rate, 'scalar', 'learning-rate')
def create_network_graph(self, input_shape, output_num, network_generator, q_discount, optimizer, loss_clipping):
# Input placeholders
with tf.name_scope('input'):
# we need to fix the input shape from (batch, filter, height, width) to
# tensorflow which is (batch, height, width, filter)
x_input_channel_firstdim = tf.placeholder(tf.uint8, [None] + input_shape, name='x-input')
# transpose because tf wants channels on last dim and channels are passed in on 2nd dim
x_input = tf.cast(tf.transpose(x_input_channel_firstdim, perm=[0, 2, 3, 1]), tf.float32) / 255.0
x_input_tp1_channel_firstdim = tf.placeholder(tf.uint8, [None] + input_shape, name='x-input-tp1')
# transpose because tf wants channels on last dim and channels are passed in on 2nd dim
x_input_tp1 = tf.cast(tf.transpose(x_input_tp1_channel_firstdim, perm=[0, 2, 3, 1]), tf.float32) / 255.0
x_actions = tf.placeholder(tf.int32, shape=[None], name='x-actions')
x_rewards = tf.placeholder(tf.float32, shape=[None], name='x-rewards')
x_terminals = tf.placeholder(tf.bool, shape=[None], name='x-terminals')
x_discount = q_discount
# Target network does not reuse variables. so we use two different variable scopes
with tf.variable_scope('network'):
network_output = network_generator(x_input, output_num)
# summarize a histogram of each action output
for output_ind in range(output_num):
summarizer.summarize(network_output[:, output_ind], 'histogram', 'network-output/{0}'.format(output_ind))
# get the trainable variables for this network, later used to overwrite target network vars
network_trainables = tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES, scope='network')
# summarize activations
summarizer.summarize_activations(tf.get_collection(tf.GraphKeys.ACTIVATIONS, scope='network'))
# add network summaries
summarizer.summarize_variables(train_vars=network_trainables)
with tf.variable_scope('target-network'):
target_network_output = network_generator(x_input_tp1, output_num)
# get trainables for target network, used in assign op for the update target network step
target_network_trainables = tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES, scope='target-network')
# summarize activations
summarizer.summarize_activations(tf.get_collection(tf.GraphKeys.ACTIVATIONS, scope='target-network'))
# add network summaries
summarizer.summarize_variables(train_vars=target_network_trainables)
# update target network with network variables
with tf.name_scope('update-target-network'):
update_target_network_ops = [target_v.assign(v) for v, target_v in zip(network_trainables, target_network_trainables)]
# caclulate QLoss
with tf.name_scope('loss'):
with tf.name_scope('estimated-reward-tp1'):
one_minus_term = tf.mul(1.0 - tf.cast(x_terminals, tf.float32), x_discount)
est_rew_tp1 = tf.mul(one_minus_term, tf.reduce_max(target_network_output, reduction_indices=1))
y = x_rewards + tf.stop_gradient(est_rew_tp1)
with tf.name_scope('estimated-reward'):
# Because of https://github.com/tensorflow/tensorflow/issues/206
# we cannot use numpy like indexing so we convert to a one hot
# multiply then take the max over last dim
# NumPy/Theano est_rew = network_output[:, x_actions]
x_actions_one_hot = tf.one_hot(x_actions, depth=output_num, name='one-hot',
on_value=1.0, off_value=0.0, dtype=tf.float32)
# we reduce sum here because the output could be negative we can't take the max
# the other indecies will be 0
est_rew = tf.reduce_sum(tf.mul(network_output, x_actions_one_hot), reduction_indices=1)
with tf.name_scope('qloss'):
# clip loss but keep linear past clip bounds
# REFS: https://github.com/spragunr/deep_q_rl/blob/master/deep_q_rl/q_network.py#L108
# https://github.com/Jabberwockyll/deep_rl_ale/blob/master/q_network.py#L241
diff = y - est_rew
if loss_clipping > 0.0:
abs_diff = tf.abs(diff)
# same as min(diff, loss_clipping) because diff can never be negative (definition of abs value)
quadratic_part = tf.clip_by_value(abs_diff, 0.0, loss_clipping)
linear_part = abs_diff - quadratic_part
loss = (0.5 * tf.square(quadratic_part)) + (loss_clipping * linear_part)
else:
# But why multiply the loss by 0.5 when not clipping? https://groups.google.com/forum/#!topic/deep-q-learning/hKK0ZM_OWd4
loss = 0.5 * tf.square(diff)
# NOTICE: we are summing gradients
error = tf.reduce_sum(loss)
summarizer.summarize(error, 'scalar', 'loss')
# optimizer
with tf.name_scope('shared-optimizer'):
tf_learning_rate = tf.placeholder(tf.float32)
optimizer = optimizer(learning_rate=tf_learning_rate)
# only train the network vars not the target network
tf_train_step = optimizer.minimize(error, var_list=network_trainables)
# tf learn auto merges all summaries so we just have to grab the last output
tf_summaries = summarizer.summarize(tf_learning_rate, 'scalar', 'learning-rate')
# function to get network output
def get_output(sess, state):
feed_dict = {x_input_channel_firstdim: state}
return sess.run([network_output], feed_dict=feed_dict)
# function to get mse feed dict
def train_step(sess, current_learning_rate, state, action, reward, state_tp1, terminal, summaries=False):
feed_dict = {x_input_channel_firstdim: state, x_input_tp1_channel_firstdim: state_tp1,
x_actions: action, x_rewards: reward, x_terminals: terminal,
tf_learning_rate: current_learning_rate}
if summaries:
return sess.run([tf_summaries, tf_train_step], feed_dict=feed_dict)[0]
else:
return sess.run([tf_train_step], feed_dict=feed_dict)
def update_target_net(sess):
return sess.run([update_target_network_ops])
self._get_output = get_output
self._train_step = train_step
self._update_target_network = update_target_net
self.saver = tf.train.Saver(var_list=network_trainables)
def create_network_graph(self):
input_shape = self._input_shape
output_num = self._output_num
# Input placeholders
with tf.name_scope('input'):
# we need to fix the input shape from (batch, filter, height, width) to
# tensorflow which is (batch, height, width, filter)
self._t_x_input_channel_firstdim = tf.placeholder(
tf.uint8, [None] + input_shape, name='x-input')
# transpose because tf wants channels on last dim and channels are passed in on 2nd dim
self._t_x_input = tf.cast(
tf.transpose(self._t_x_input_channel_firstdim,
perm=[0, 2, 3, 1]), tf.float32) / 255.0
self._t_x_input_tp1_channel_firstdim = tf.placeholder(
tf.uint8, [None] + input_shape, name='x-input-tp1')
# transpose because tf wants channels on last dim and channels are passed in on 2nd dim
self._t_x_input_tp1 = tf.cast(
tf.transpose(self._t_x_input_tp1_channel_firstdim,
perm=[0, 2, 3, 1]), tf.float32) / 255.0
self._t_x_actions = tf.placeholder(tf.int32,
shape=[None],
name='x-actions')
self._t_x_rewards = tf.placeholder(tf.float32,
shape=[None],
name='x-rewards')
self._t_x_terminals = tf.placeholder(tf.bool,
shape=[None],
name='x-terminals')
self._t_x_discount = self._q_discount
# Target network does not reuse variables
with tf.variable_scope('network') as var_scope:
self._t_network_output = self._network_generator(
self._t_x_input, output_num)
# get the trainable variables for this network, later used to overwrite target network vars
self._tf_network_trainables = tf.get_collection(
tf.GraphKeys.TRAINABLE_VARIABLES, scope='network')
# summarize activations
summarizer.summarize_activations(
tf.get_collection(tf.GraphKeys.ACTIVATIONS, scope='network'))
# if double DQN then we need to create network output for s_tp1
if self.algorithm_type == 'double' or self.algorithm_type == 'doublenstep':
var_scope.reuse_variables()
self._t_network_output_tp1 = self._network_generator(
self._t_x_input_tp1, output_num)
# summarize a histogram of each action output
for output_ind in range(output_num):
summarizer.summarize(self._t_network_output[:, output_ind],
'histogram',
'network-output/{0}'.format(output_ind))
# add network summaries
summarizer.summarize_variables(
train_vars=self._tf_network_trainables)
with tf.variable_scope('target-network'):
self._t_target_network_output = self._network_generator(
self._t_x_input_tp1, output_num)
# get trainables for target network, used in assign op for the update target network step
target_network_trainables = tf.get_collection(
tf.GraphKeys.TRAINABLE_VARIABLES, scope='target-network')
# update target network with network variables
with tf.name_scope('update-target-network'):
self._tf_update_target_network_ops = [
target_v.assign(v) for v, target_v in zip(
self._tf_network_trainables, target_network_trainables)
]
# if double convience function to get target values for online action
if self.algorithm_type == 'double' or self.algorithm_type == 'doublenstep':
with tf.name_scope('double_target'):
# Target = target_Q(s_tp1, argmax(online_Q(s_tp1)))
argmax_tp1 = tf.argmax(self._t_network_output_tp1, axis=1)
self._t_target_value_online_action = tf_util.one_hot(
self._t_target_network_output, argmax_tp1, output_num)
# caclulate QLoss
with tf.name_scope('loss'):
# nstep rewards are calculated outside the gpu/graph because it requires a loop
if self.algorithm_type != 'nstep' and self.algorithm_type != 'doublenstep':
with tf.name_scope('estimated-reward-tp1'):
if self.algorithm_type == 'double':
# Target = target_Q(s_tp1, argmax(online_Q(s_tp1)))
target = self._t_target_value_online_action
elif self.algorithm_type == 'dqn':
# Target = max(target_Q(s_tp1))
target = tf.reduce_max(self._t_target_network_output,
axis=1)
# compute a mask that returns gamma (discount factor) or 0 if terminal
terminal_discount_mask = tf.multiply(
1.0 - tf.cast(self._t_x_terminals, tf.float32),
self._t_x_discount)
est_rew_tp1 = tf.multiply(terminal_discount_mask, target)
y = self._t_x_rewards + tf.stop_gradient(est_rew_tp1)
# else nstep
else:
y = self._t_x_rewards
with tf.name_scope('estimated-reward'):
est_rew = tf_util.one_hot(self._t_network_output,
self._t_x_actions, output_num)
with tf.name_scope('qloss'):
# clip loss but keep linear past clip bounds (huber loss with customizable linear part)
# REFS: https://github.com/spragunr/deep_q_rl/blob/master/deep_q_rl/q_network.py#L108
# https://github.com/Jabberwockyll/deep_rl_ale/blob/master/q_network.py#L241
diff = y - est_rew
if self._loss_clipping > 0.0:
abs_diff = tf.abs(diff)
# same as min(diff, loss_clipping) because diff can never be negative (definition of abs value)
quadratic_part = tf.clip_by_value(abs_diff, 0.0,
self._loss_clipping)
linear_part = abs_diff - quadratic_part
loss = (0.5 * tf.square(quadratic_part)) + (
self._loss_clipping * linear_part)
else:
# But why multiply the loss by 0.5 when not clipping? https://groups.google.com/forum/#!topic/deep-q-learning/hKK0ZM_OWd4
loss = 0.5 * tf.square(diff)
# NOTICE: we are summing gradients
error = tf.reduce_sum(loss)
summarizer.summarize(error, 'scalar', 'loss')
# optimizer
with tf.name_scope('shared-optimizer'):
self._tf_learning_rate = tf.placeholder(tf.float32)
optimizer = self._optimizer_fn(
learning_rate=self._tf_learning_rate)
# only train the network vars not the target network
gradients = optimizer.compute_gradients(
error, var_list=self._tf_network_trainables)
# gradients are stored as a tuple, (gradient, tensor the gradient corresponds to)
# kinda lame that clip by global norm doesn't accept the list of tuples returned from compute_gradients
# so we unzip then zip
tensors = [tensor for gradient, tensor in gradients]
grads = [gradient for gradient, tensor in gradients]
clipped_gradients, _ = tf.clip_by_global_norm(
grads,
self.global_norm_clipping) # returns list[tensors], norm
clipped_grads_tensors = zip(clipped_gradients, tensors)
self._tf_train_step = optimizer.apply_gradients(
clipped_grads_tensors)
# tflearn smartly knows how gradients are stored so we just pass in the list of tuples
summarizer.summarize_gradients(clipped_grads_tensors)
# tf learn auto merges all summaries so we just have to grab the last output
self._tf_summaries = summarizer.summarize(self._tf_learning_rate,
'scalar',
'learning-rate')
def create_network_graph(self, input_shape, output_num, network_generator, q_discount, optimizer, loss_clipping):
# Input placeholders
with tf.name_scope('input'):
# we need to fix the input shape from (batch, filter, height, width) to
# tensorflow which is (batch, height, width, filter)
x_input_channel_firstdim = tf.placeholder(tf.uint8, [None] + input_shape, name='x-input')
# transpose because tf wants channels on last dim and channels are passed in on 2nd dim
x_input = tf.cast(tf.transpose(x_input_channel_firstdim, perm=[0, 2, 3, 1]), tf.float32) / 255.0
x_input_tp1_channel_firstdim = tf.placeholder(tf.uint8, [None] + input_shape, name='x-input-tp1')
# transpose because tf wants channels on last dim and channels are passed in on 2nd dim
x_input_tp1 = tf.cast(tf.transpose(x_input_tp1_channel_firstdim, perm=[0, 2, 3, 1]), tf.float32) / 255.0
x_actions = tf.placeholder(tf.int32, shape=[None], name='x-actions')
# TODO: SARSA only change
x_actions_tp1 = tf.placeholder(tf.int32, shape=[None], name='x-actions-tp1')
x_rewards = tf.placeholder(tf.float32, shape=[None], name='x-rewards')
x_terminals = tf.placeholder(tf.bool, shape=[None], name='x-terminals')
x_discount = q_discount
# Target network does not reuse variables. so we use two different variable scopes
with tf.variable_scope('network'):
network_output = network_generator(x_input, output_num)
# summarize a histogram of each action output
for output_ind in range(output_num):
summarizer.summarize(network_output[:, output_ind], 'histogram', 'network-output/{0}'.format(output_ind))
# get the trainable variables for this network, later used to overwrite target network vars
network_trainables = tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES, scope='network')
# summarize activations
summarizer.summarize_activations(tf.get_collection(tf.GraphKeys.ACTIVATIONS, scope='network'))
# add network summaries
summarizer.summarize_variables(train_vars=network_trainables)
with tf.variable_scope('target-network'):
target_network_output = network_generator(x_input_tp1, output_num)
# get trainables for target network, used in assign op for the update target network step
target_network_trainables = tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES, scope='target-network')
# summarize activations
summarizer.summarize_activations(tf.get_collection(tf.GraphKeys.ACTIVATIONS, scope='target-network'))
# add network summaries
summarizer.summarize_variables(train_vars=target_network_trainables)
# update target network with network variables
with tf.name_scope('update-target-network'):
update_target_network_ops = [target_v.assign(v) for v, target_v in zip(network_trainables, target_network_trainables)]
# caclulate QLoss
with tf.name_scope('loss'):
with tf.name_scope('estimated-reward-tp1'):
one_minus_term = tf.multiply(1.0 - tf.cast(x_terminals, tf.float32), x_discount)
# TODO: SARSA only change
# Sarsa uses the q estimate of the next state given action_tp1. Not the max
# We must convert to one hot same as below
# NumPy/Theano est_rew_tp1 = network_output[:, x_actions_tp1]
x_actions_tp1_one_hot = tf.one_hot(x_actions_tp1, depth=output_num, name='one-hot-tp1',
on_value=1.0, off_value=0.0, dtype=tf.float32)
# we reduce sum here because the output could be negative we can't take the max
# the other indecies will be 0
network_est_rew_tp1 = tf.reduce_sum(tf.multiply(target_network_output, x_actions_tp1_one_hot), axis=1)
est_rew_tp1 = tf.multiply(one_minus_term, network_est_rew_tp1)
y = x_rewards + tf.stop_gradient(est_rew_tp1)
with tf.name_scope('estimated-reward'):
# Because of https://github.com/tensorflow/tensorflow/issues/206
# we cannot use numpy like indexing so we convert to a one hot
# multiply then take the max over last dim
# NumPy/Theano est_rew = network_output[:, x_actions]
x_actions_one_hot = tf.one_hot(x_actions, depth=output_num, name='one-hot',
on_value=1.0, off_value=0.0, dtype=tf.float32)
# we reduce sum here because the output could be negative we can't take the max
# the other indecies will be 0
est_rew = tf.reduce_sum(tf.multiply(network_output, x_actions_one_hot), axis=1)
with tf.name_scope('qloss'):
# clip loss but keep linear past clip bounds
# REFS: https://github.com/spragunr/deep_q_rl/blob/master/deep_q_rl/q_network.py#L108
# https://github.com/Jabberwockyll/deep_rl_ale/blob/master/q_network.py#L241
diff = y - est_rew
if loss_clipping > 0.0:
abs_diff = tf.abs(diff)
# same as min(diff, loss_clipping) because diff can never be negative (definition of abs value)
quadratic_part = tf.clip_by_value(abs_diff, 0.0, loss_clipping)
linear_part = abs_diff - quadratic_part
loss = (0.5 * tf.square(quadratic_part)) + (loss_clipping * linear_part)
else:
# But why multiply the loss by 0.5 when not clipping? https://groups.google.com/forum/#!topic/deep-q-learning/hKK0ZM_OWd4
loss = 0.5 * tf.square(diff)
# NOTICE: we are summing gradients
error = tf.reduce_sum(loss)
summarizer.summarize(error, 'scalar', 'loss')
# optimizer
with tf.name_scope('shared-optimizer'):
tf_learning_rate = tf.placeholder(tf.float32)
optimizer = optimizer(learning_rate=tf_learning_rate)
# only train the network vars not the target network
tf_train_step = optimizer.minimize(error, var_list=network_trainables)
# tf learn auto merges all summaries so we just have to grab the last output
tf_summaries = summarizer.summarize(tf_learning_rate, 'scalar', 'learning-rate')
# function to get network output
def get_output(sess, state):
feed_dict = {x_input_channel_firstdim: state}
return sess.run([network_output], feed_dict=feed_dict)
# function to get mse feed dict
def train_step(sess, current_learning_rate, state, action, reward, state_tp1, action_tp1, terminal, summaries=False):
feed_dict = {x_input_channel_firstdim: state, x_input_tp1_channel_firstdim: state_tp1,
x_actions: action, x_actions_tp1: action_tp1, x_rewards: reward, x_terminals: terminal,
# TODO: SARSA only change action_tp1
tf_learning_rate: current_learning_rate}
if summaries:
return sess.run([tf_summaries, tf_train_step], feed_dict=feed_dict)[0]
else:
return sess.run([tf_train_step], feed_dict=feed_dict)
def update_target_net(sess):
return sess.run([update_target_network_ops])
self._get_output = get_output
self._train_step = train_step
self._update_target_network = update_target_net
self.saver = tf.train.Saver(var_list=network_trainables)