def _g_recurrence_2(i, x_t, gen_x, h_tm1, h_tm1_manager, last_goal, real_goal):
# with tf.device('/cpu:0'):
cur_sen = tf.cond(i > 0, lambda:
tf.split(tf.concat([tf.transpose(gen_x.stack(), perm=[1, 0]), self.padding_array], 1),
[self.sequence_length, i - 1], 1)[0], lambda: self.padding_array)
with tf.variable_scope(self.scope):
feature = self.FeatureExtractor_unit(cur_sen, self.drop_out)
h_t_Worker = self.g_worker_recurrent_unit(x_t, h_tm1) # hidden_memory_tuple
o_t_Worker = self.g_worker_output_unit(h_t_Worker) # batch x vocab , logits not prob
o_t_Worker = tf.reshape(o_t_Worker, [self.batch_size, self.num_vocabulary, self.goal_size])
h_t_manager = self.g_manager_recurrent_unit(feature, h_tm1_manager)
sub_goal = self.g_manager_output_unit(h_t_manager)
sub_goal = tf.nn.l2_normalize(sub_goal, 1)
real_sub_goal = tf.add(last_goal, sub_goal)
w_g = tf.matmul(real_goal, self.g_change) # batch x goal_size
w_g = tf.nn.l2_normalize(w_g, 1)
w_g = tf.expand_dims(w_g, 2) # batch x goal_size x 1
x_logits = tf.matmul(o_t_Worker, w_g)
x_logits = tf.squeeze(x_logits)
log_prob = tf.log(tf.nn.softmax(x_logits))
next_token = tf.cast(tf.reshape(tf.multinomial(log_prob, 1), [self.batch_size]), tf.int32)
x_tp1 = tf.nn.embedding_lookup(self.g_embeddings, next_token) # batch x emb_dim
with tf.control_dependencies([cur_sen]):
gen_x = gen_x.write(i - 1, next_token) # indices, batch_size
return i + 1, x_tp1, gen_x, h_t_Worker, h_t_manager, \
tf.cond(((i) % self.step_size) > 0, lambda: real_sub_goal,
lambda: tf.constant(0.0, shape=[self.batch_size, self.goal_out_size])), \
tf.cond(((i) % self.step_size) > 0, lambda: real_goal, lambda: real_sub_goal)
def _define_step(self, done, score, summary):
"""Combine operations of a phase.
Keeps track of the mean score and when to report it.
Args:
done: Tensor indicating whether current score can be used.
score: Tensor holding the current, possibly intermediate, score.
summary: Tensor holding summary string to write if not an empty string.
Returns:
Tuple of summary tensor, mean score, and new global step. The mean score
is zero for non reporting steps.
"""
if done.shape.ndims == 0:
done = done[None]
if score.shape.ndims == 0:
score = score[None]
score_mean = streaming_mean.StreamingMean((), tf.float32)
with tf.control_dependencies([done, score, summary]):
done_score = tf.gather(score, tf.where(done)[:, 0])
submit_score = tf.cond(tf.reduce_any(done), lambda: score_mean.submit(done_score), tf.no_op)
with tf.control_dependencies([submit_score]):
mean_score = tf.cond(self._report, score_mean.clear, float)
steps_made = tf.shape(score)[0]
next_step = self._step.assign_add(steps_made)
with tf.control_dependencies([mean_score, next_step]):
return tf.identity(summary), mean_score, next_step, steps_made
def _create_learning_rate(hyperparams, step_var):
"""Creates learning rate var, with decay and switching for CompositeOptimizer.
Args:
hyperparams: a GridPoint proto containing optimizer spec, particularly
learning_method to determine optimizer class to use.
step_var: tf.Variable, global training step.
Raises:
ValueError: If the composite optimizer is set, but not correctly configured.
Returns:
a scalar `Tensor`, the learning rate based on current step and hyperparams.
"""
if hyperparams.learning_method != 'composite':
base_rate = hyperparams.learning_rate
adjusted_steps = step_var
else:
spec = hyperparams.composite_optimizer_spec
switch = tf.less(step_var, spec.switch_after_steps)
base_rate = tf.cond(switch, lambda: tf.constant(spec.method1.learning_rate),
lambda: tf.constant(spec.method2.learning_rate))
if spec.reset_learning_rate:
adjusted_steps = tf.cond(switch, lambda: step_var,
lambda: step_var - spec.switch_after_steps)
else:
adjusted_steps = step_var
return tf.train.exponential_decay(
learning_rate=base_rate,
global_step=adjusted_steps,
decay_steps=hyperparams.decay_steps,
decay_rate=hyperparams.decay_base,
staircase=hyperparams.decay_staircase)
def while_exit_cond(result, logits, loss): # pylint: disable=unused-argument
"""Exit the loop either if reach decode_length or EOS."""
length = common_layers.shape_list(result)[1]
not_overflow = length < decode_length
if self._problem_hparams.stop_at_eos:
def fn_not_eos():
return tf.not_equal( # Check if the last predicted element is a EOS
tf.squeeze(result[:, -1, :, :]), text_encoder.EOS_ID)
not_eos = tf.cond(
# We only check for early stoping if there is at least 1 element (
# otherwise not_eos will crash)
tf.not_equal(length, 0),
fn_not_eos,
lambda: True,
)
return tf.cond(
tf.equal(batch_size, 1),
# If batch_size == 1, we check EOS for early stoping
lambda: tf.logical_and(not_overflow, not_eos),
# Else, just wait for max length
lambda: not_overflow)
return not_overflow
def __imagenet_data_process_function(self, x, y):
with tf.name_scope("imagenet_data_aug") as scope:
#random scale
#apparently, this works better than what we have:
#https://github.com/facebook/fb.resnet.torch
#but let's use the 'original' formulation for now
#randomly sample a size in specified range
random_size = tf.squeeze(tf.random_uniform((1, 1), 256, 480, dtype=tf.int32, name="random_scale_size"))
#rescale smaller size with this factor
tf.cond(tf.greater(tf.shape(x)[0], tf.shape(x)[1]),
lambda: tf.image.resize_images(x, [tf.shape(x)[0] * (tf.shape(x)[1] / random_size), random_size]),
lambda: tf.image.resize_images(x, [random_size, tf.shape(x)[1] * (tf.shape(x)[0] / random_size)]))
x = tf.image.resize_images(x, [224, 224])
#random flip
x = tf.image.flip_left_right(x)
#random crop
x = tf.random_crop(x, [224, 224, 3])
#colour augmentation
#this is a little more involved than I first thought
#lets pick the inception colour distortion
#https://github.com/tensorflow/models/blob/master/inception/inception/image_processing.py
x = tf.image.random_brightness(x, max_delta=32. / 255.)
x = tf.image.random_saturation(x, lower=0.5, upper=1.5)
x = tf.image.random_hue(x, max_delta=0.2)
x = tf.image.random_contrast(x, lower=0.5, upper=1.5)
x = tf.clip_by_value(x, 0.0, 1.0)
#normalisation
x = tf.image.per_image_standardization(x)
return [x, y]
def loop_fn(time, cell_output, cell_state, loop_state):
emit_output = cell_output # == None for time == 0
if cell_output is None: # time == 0
next_cell_state = initial_state
else:
next_cell_state = cell_state
elements_finished = (time >= sequence_length)
finished = tf.reduce_all(elements_finished)
next_input = tf.cond(
finished,
lambda: tf.zeros([batch_size, input_size], dtype=tf.float32),
lambda: inputs_ta.read(time))
next_target = tf.cond(
finished,
lambda: tf.zeros([batch_size, target_size], dtype=tf.float32),
lambda: targets_ta.read(time))
if loop_state is None:
next_input = tf.expand_dims(next_input, 1)
next_input = tf.pad(next_input, [[0,0], [window-1, 0], [0,0]])
else:
next_input = cat_hist(loop_state, next_input, 1)
next_loop_state = next_input
return (elements_finished, RnnHistInputTuple(next_input, next_target), next_cell_state, emit_output, next_loop_state)
def encoder_body(time, old_state, output_ta_t):
x_t = input_ta.read(time)
con = tf.concat(1, [x_t, old_state])
z = tf.sigmoid(tf.matmul(con, W_z) + b_z)
r = tf.sigmoid(tf.matmul(con, W_r) + b_r)
con = tf.concat(1, [x_t, r*old_state])
h = tf.tanh(tf.matmul(con, W_h) + b_h)
new_state = (1-z)*h + z*old_state
output_ta_t = output_ta_t.write(time, new_state)
def updateall():
return new_state
def updatesome():
if reverse:
return tf.select(
tf.greater_equal(time, max_sequence_length-lengths),
new_state,
old_state)
else:
return tf.select(tf.less(time, lengths), new_state, old_state)
if reverse:
state = tf.cond(
tf.greater_equal(time, max_sequence_length-min_sequence_length),
updateall,
updatesome)
else:
state = tf.cond(tf.less(time, min_sequence_length), updateall, updatesome)
return (time + 1, state, output_ta_t)
def _augment(self, image, bboxes=None, default_prob=0.5):
"""Applies different data augmentation techniques.
Uses the list of data augmentation configurations, each data
augmentation technique has a probability assigned to it (or just uses
the default value for the dataset).
Procedures are applied sequentially on top of each other according to
the order defined in the config.
TODO: We need a better way to ensure order using YAML config without
ending up with a list of single-key dictionaries.
Args:
image: A Tensor of shape (height, width, 3).
bboxes: A Tensor of shape (total_bboxes, 5).
Returns:
image: A Tensor of shape (height, width, 3).
bboxes: A Tensor of shape (total_bboxes, 5) of type tf.int32.
"""
applied_data_augmentation = []
for aug_config in self._data_augmentation:
if len(aug_config.keys()) != 1:
raise ValueError(
'Invalid data_augmentation definition: "{}"'.format(
aug_config))
aug_type = list(aug_config.keys())[0]
if aug_type not in DATA_AUGMENTATION_STRATEGIES:
tf.logging.warning(
'Invalid data augmentation strategy "{}". Ignoring'.format(
aug_type))
continue
aug_config = aug_config[aug_type]
aug_fn = DATA_AUGMENTATION_STRATEGIES[aug_type]
random_number = tf.random_uniform([], seed=self._seed)
prob = tf.to_float(aug_config.pop('prob', default_prob))
apply_aug_strategy = tf.less(random_number, prob)
augmented = aug_fn(image, bboxes, **aug_config)
image = tf.cond(
apply_aug_strategy,
lambda: augmented['image'],
lambda: image
)
if bboxes is not None:
bboxes = tf.cond(
apply_aug_strategy,
lambda: augmented.get('bboxes'),
lambda: bboxes
)
applied_data_augmentation.append({aug_type: apply_aug_strategy})
return image, bboxes, applied_data_augmentation
def apply_stats(self, statsUpdates):
""" compute stats and update/apply the new stats to the running average
"""
def updateAccumStats():
if self._full_stats_init:
return tf.cond(tf.greater(self.sgd_step, self._cold_iter), lambda: tf.group(*self._apply_stats(statsUpdates, accumulate=True, accumulateCoeff=1. / self._stats_accum_iter)), tf.no_op)
else:
return tf.group(*self._apply_stats(statsUpdates, accumulate=True, accumulateCoeff=1. / self._stats_accum_iter))
def updateRunningAvgStats(statsUpdates, fac_iter=1):
# return tf.cond(tf.greater_equal(self.factor_step,
# tf.convert_to_tensor(fac_iter)), lambda:
# tf.group(*self._apply_stats(stats_list, varlist)), tf.no_op)
return tf.group(*self._apply_stats(statsUpdates))
if self._async_stats:
# asynchronous stats update
update_stats = self._apply_stats(statsUpdates)
queue = tf.FIFOQueue(1, [item.dtype for item in update_stats], shapes=[
item.get_shape() for item in update_stats])
enqueue_op = queue.enqueue(update_stats)
def dequeue_stats_op():
return queue.dequeue()
self.qr_stats = tf.train.QueueRunner(queue, [enqueue_op])
update_stats_op = tf.cond(tf.equal(queue.size(), tf.convert_to_tensor(
0)), tf.no_op, lambda: tf.group(*[dequeue_stats_op(), ]))
else:
# synchronous stats update
update_stats_op = tf.cond(tf.greater_equal(
self.stats_step, self._stats_accum_iter), lambda: updateRunningAvgStats(statsUpdates), updateAccumStats)
self._update_stats_op = update_stats_op
return update_stats_op
def enc_step(step):
"""Encoder step."""
if autoenc_decay < 1.0:
quant_step = autoenc_quantize(step, 16, nmaps, self.do_training)
if backward:
exp_glob = tf.train.exponential_decay(1.0, self.global_step - 10000,
1000, autoenc_decay)
dec_factor = 1.0 - exp_glob # * self.do_training
dec_factor = tf.cond(tf.less(self.global_step, 10500),
lambda: tf.constant(0.05), lambda: dec_factor)
else:
dec_factor = 1.0
cur = tf.cond(tf.less(tf.random_uniform([]), dec_factor),
lambda: quant_step, lambda: step)
else:
cur = step
if dropout > 0.0001:
cur = tf.nn.dropout(cur, keep_prob)
if act_noise > 0.00001:
cur += tf.truncated_normal(tf.shape(cur)) * act_noise_scale
# Do nconvs-many CGRU steps.
if do_jit and tf.get_variable_scope().reuse:
with jit_scope():
for layer in xrange(nconvs):
cur = conv_gru([], cur, kw, kh, nmaps, conv_rate(layer),
cutoff, "ecgru_%d" % layer, do_layer_norm)
else:
for layer in xrange(nconvs):
cur = conv_gru([], cur, kw, kh, nmaps, conv_rate(layer),
cutoff, "ecgru_%d" % layer, do_layer_norm)
return cur
def batchnorm(self, Ylogits, offset, convolutional=False):
"""batchnormalization.
Args:
Ylogits: 1D向量或者是3D的卷积结果。
num_updates: 迭代的global_step
offset：表示beta，全局均值；在 RELU 激活中一般初始化为 0.1。
scale：表示lambda，全局方差；在 sigmoid 激活中需要，这 RELU 激活中作用不大。
m: 表示batch均值；v:表示batch方差。
bnepsilon：一个很小的浮点数，防止除以 0.
Returns:
Ybn: 和 Ylogits 的维度一样，就是经过 Batch Normalization 处理的结果。
update_moving_everages：更新mean和variance，主要是给最后的 test 使用。
"""
exp_moving_avg = tf.train.ExponentialMovingAverage(0.999,
self._global_step) # adding the iteration prevents from averaging across non-existing iterations
bnepsilon = 1e-5
if convolutional:
mean, variance = tf.nn.moments(Ylogits, [0, 1, 2])
else:
mean, variance = tf.nn.moments(Ylogits, [0])
update_moving_everages = exp_moving_avg.apply([mean, variance])
m = tf.cond(self.tst, lambda: exp_moving_avg.average(mean), lambda: mean)
v = tf.cond(self.tst, lambda: exp_moving_avg.average(variance), lambda: variance)
Ybn = tf.nn.batch_normalization(Ylogits, m, v, offset, None, bnepsilon)
return Ybn, update_moving_everages
def perform(self, observ):
"""Compute batch of actions and a summary for a batch of observation.
Args:
observ: Tensor of a batch of observations for all agents.
Returns:
Tuple of action batch tensor and summary tensor.
"""
with tf.name_scope('perform/'):
observ = self._observ_filter.transform(observ)
network = self._network(observ[:, None], tf.ones(observ.shape[0]), self._last_state)
action = tf.cond(self._is_training, network.policy.sample, lambda: network.mean)
logprob = network.policy.log_prob(action)[:, 0]
# pylint: disable=g-long-lambda
summary = tf.cond(
self._should_log, lambda: tf.summary.merge([
tf.summary.histogram('mean', network.mean[:, 0]),
tf.summary.histogram('std', tf.exp(network.logstd[:, 0])),
tf.summary.histogram('action', action[:, 0]),
tf.summary.histogram('logprob', logprob)
]), str)
# Remember current policy to append to memory in the experience callback.
with tf.control_dependencies([
utility.assign_nested_vars(self._last_state, network.state),
self._last_action.assign(action[:, 0]),
self._last_mean.assign(network.mean[:, 0]),
self._last_logstd.assign(network.logstd[:, 0])
]):
return tf.check_numerics(action[:, 0], 'action'), tf.identity(summary)
def gibbs_step(count, k, xk, raw_xk, W=W):
hk = gibbs_forward(xk, W, bh, binary)
C1 = tf.cond(count + 1 >= k, lambda: tf.constant(c_1), lambda: tf.constant(0.5))
C2 = tf.cond(count + 1 >= k, lambda: tf.constant(c_2), lambda: tf.constant(0.5))
raw_xk = tf.sigmoid(tf.matmul(hk, tf.transpose(W)) + bv)
xk = sample(raw_xk, binary=binary, c_1=c_1, c_2=c_2)
return count+1, k, xk, raw_xk
def _get_valid_sample_fraction(labels, p=0):
"""return fraction of non-negative examples, the ignored examples have been marked as negative"""
num_valid = tf.reduce_sum(tf.cast(tf.greater_equal(labels, p), tf.float32))
num_example = tf.cast(tf.size(labels), tf.float32)
frac = tf.cond(tf.greater(num_example, 0), lambda:num_valid / num_example,
lambda: tf.cast(0, tf.float32))
frac_ = tf.cond(tf.greater(num_valid, 0), lambda:num_example / num_valid,
lambda: tf.cast(0, tf.float32))
return frac, frac_
def __init__(self, input_size, output_size):
self.graph = tf.Graph()
self.hyper_cnt = input_size
self.save_path = "fit_trend.ckpt"
self.collect_counter = 0
self.fit_loss_collect = list()
self.stable_loss_predict_collect = list()
self.hp_collect = [list() for _ in range(self.hyper_cnt)]
self.gradient_collect = [list() for _ in range(self.hyper_cnt)]
self.stable_loss_label_collect = list()
self.hp_norms = list()
self.has_init = False
with self.graph.as_default():
# 接收输入
self.ph_hypers = tf.placeholder(tf.float32, shape=[self.hyper_cnt], name='ph_hypers')
self.tf_hypers, self.reset_vars = assign_diffable_vars2tensor(self.ph_hypers, self.hyper_cnt)
rnn_step = 5
trend_input = tf.concat(0, [self.tf_hypers for _ in range(rnn_step)])
# 通过一个RNN
trend_outputs = rnn(trend_input, n_hidden=128)
print('rnn output')
print(tf.concat(0, trend_outputs))
# RNN接一个DNN
trend_output = dnn(tf.concat(0, trend_outputs), [1, output_size])
print('dnn output')
print(trend_output)
self.predict = trend_output
# 实际的trend
self.train_label = tf.placeholder(tf.float32, shape=[output_size], name='train_label')
# 预测准确率，predict和trend的几何距离
predict_accuracy = tf.sqrt(tf.reduce_sum(tf.square(tf.sub(trend_output, self.train_label)))) / output_size
# predict_accuracy /= tf.reduce_mean(tf.concat(0, self.train_label))
# 稳定时损失，最后一个损失
stable_loss = tf.unpack(tf.unpack(trend_output)[0])[-1]
print(stable_loss)
self.is_fit = tf.placeholder(tf.bool, name='is_fit')
self.loss = tf.cond(self.is_fit, lambda: predict_accuracy, lambda: stable_loss)
# 优化器
self.var_s = tf.trainable_variables()
self.v_hp_s = self.var_s[0: self.hyper_cnt]
self.v_fit_s = [v for v in self.var_s if v not in self.v_hp_s]
self.grads = var_gradient(self.v_hp_s, self.loss, start_rate=0.1, lrd=False)
def optimize_fit():
optimizer_fit = var_optimizer(self.v_fit_s, self.loss)
return optimizer_fit
def optimize_hp():
optimizer_hp = var_optimizer(self.v_hp_s, self.loss, start_rate=0.1, lrd=False)
return optimizer_hp
self.optimizer = tf.cond(self.is_fit, optimize_fit, optimize_hp)
self.saver = tf.train.Saver()
def batchnorm(Ylogits, Offset, Scale, is_test, iteration):
exp_moving_avg = tf.train.ExponentialMovingAverage(0.998, iteration) # adding the iteration prevents from averaging across non-existing iterations
bnepsilon = 1e-5
mean, variance = tf.nn.moments(Ylogits, [0])
update_moving_averages = exp_moving_avg.apply([mean, variance])
m = tf.cond(is_test, lambda: exp_moving_avg.average(mean), lambda: mean)
v = tf.cond(is_test, lambda: exp_moving_avg.average(variance), lambda: variance)
Ybn = tf.nn.batch_normalization(Ylogits, m, v, Offset, Scale, bnepsilon)
return Ybn, update_moving_averages
def style_loss(CNN_structure, const_layers, var_layers, content_segs, style_segs, weight):
loss_styles = []
layer_count = float(len(const_layers))
layer_index = 0
_, content_seg_height, content_seg_width, _ = content_segs[0].get_shape().as_list()
_, style_seg_height, style_seg_width, _ = style_segs[0].get_shape().as_list()
for layer_name in CNN_structure:
layer_name = layer_name[layer_name.find("/") + 1:]
# downsampling segmentation
if "pool" in layer_name:
content_seg_width, content_seg_height = int(math.ceil(content_seg_width / 2)), int(math.ceil(content_seg_height / 2))
style_seg_width, style_seg_height = int(math.ceil(style_seg_width / 2)), int(math.ceil(style_seg_height / 2))
for i in xrange(len(content_segs)):
content_segs[i] = tf.image.resize_bilinear(content_segs[i], tf.constant((content_seg_height, content_seg_width)))
style_segs[i] = tf.image.resize_bilinear(style_segs[i], tf.constant((style_seg_height, style_seg_width)))
elif "conv" in layer_name:
for i in xrange(len(content_segs)):
# have some differences on border with torch
content_segs[i] = tf.nn.avg_pool(tf.pad(content_segs[i], [[0, 0], [1, 1], [1, 1], [0, 0]], "CONSTANT"), \
ksize=[1, 3, 3, 1], strides=[1, 1, 1, 1], padding='VALID')
style_segs[i] = tf.nn.avg_pool(tf.pad(style_segs[i], [[0, 0], [1, 1], [1, 1], [0, 0]], "CONSTANT"), \
ksize=[1, 3, 3, 1], strides=[1, 1, 1, 1], padding='VALID')
if layer_name == var_layers[layer_index].name[var_layers[layer_index].name.find("/") + 1:]:
print("Setting up style layer: <{}>".format(layer_name))
const_layer = const_layers[layer_index]
var_layer = var_layers[layer_index]
layer_index = layer_index + 1
layer_style_loss = 0.0
for content_seg, style_seg in zip(content_segs, style_segs):
gram_matrix_const = gram_matrix(tf.multiply(const_layer, style_seg))
style_mask_mean = tf.reduce_mean(style_seg)
gram_matrix_const = tf.cond(tf.greater(style_mask_mean, 0.),
lambda: gram_matrix_const / (tf.to_float(tf.size(const_layer)) * style_mask_mean),
lambda: gram_matrix_const
)
gram_matrix_var = gram_matrix(tf.multiply(var_layer, content_seg))
content_mask_mean = tf.reduce_mean(content_seg)
gram_matrix_var = tf.cond(tf.greater(content_mask_mean, 0.),
lambda: gram_matrix_var / (tf.to_float(tf.size(var_layer)) * content_mask_mean),
lambda: gram_matrix_var
)
diff_style_sum = tf.reduce_mean(tf.squared_difference(gram_matrix_const, gram_matrix_var)) * content_mask_mean
layer_style_loss += diff_style_sum
loss_styles.append(layer_style_loss * weight)
return loss_styles
def distort_image(image, input_width, input_height, output_side):
"""Applies random distortion to the image.
The output image is output_side x output_side x 3
"""
def random_crop_it():
"""Random crops image, after resizing it to output_side +10 x output_side+10"""
resized_img = resize_bl(image, output_side + 10)
return tf.random_crop(resized_img, [output_side, output_side, 3])
def resize_it():
"""Resize the image using resize_bl"""
return resize_bl(image, output_side)
# if input.width >= output.side + 10 and input.heigth >= output.side + 10
# resize it to output.side + 10 x output.size + 10 and random crop it
# else resize it
increased_output_side = tf.constant(output_side + 10, dtype=tf.int64)
image = tf.cond(
tf.logical_and(
tf.greater_equal(input_width, increased_output_side),
tf.greater_equal(input_height, increased_output_side)),
random_crop_it, resize_it)
# Apply random distortions to the image
flipped_image = tf.image.random_flip_left_right(image)
# randomize the order of the random distortions
def fn1():
"""Applies random brightness, saturation, hue, contrast"""
distorted_image = tf.image.random_brightness(
flipped_image, max_delta=32. / 255.)
distorted_image = tf.image.random_saturation(
distorted_image, lower=0.5, upper=1.5)
distorted_image = tf.image.random_hue(distorted_image, max_delta=0.2)
distorted_image = tf.image.random_contrast(
distorted_image, lower=0.5, upper=1.5)
return distorted_image
def fn2():
"""Applies random brightness, contrast, saturation, hue"""
distorted_image = tf.image.random_brightness(
flipped_image, max_delta=32. / 255.)
distorted_image = tf.image.random_contrast(
distorted_image, lower=0.5, upper=1.5)
distorted_image = tf.image.random_saturation(
distorted_image, lower=0.5, upper=1.5)
distorted_image = tf.image.random_hue(distorted_image, max_delta=0.2)
return distorted_image
p_order = tf.random_uniform(
shape=[], minval=0.0, maxval=1.0, dtype=tf.float32)
distorted_image = tf.cond(tf.less(p_order, 0.5), fn1, fn2)
distorted_image = tf.clip_by_value(distorted_image, 0.0, 1.0)
return distorted_image
def optimOp():
def updateOptimOp():
if self._full_stats_init:
return tf.cond(tf.greater(self.factor_step, tf.convert_to_tensor(0)), lambda: optim.apply_gradients(list(zip(u, varlist))), tf.no_op)
else:
return optim.apply_gradients(list(zip(u, varlist)))
if self._full_stats_init:
return tf.cond(tf.greater_equal(self.stats_step, self._stats_accum_iter), updateOptimOp, tf.no_op)
else:
return tf.cond(tf.greater_equal(self.sgd_step, self._cold_iter), updateOptimOp, tf.no_op)
def summary(self):
"""Summary string of mean and standard deviation.
Returns:
Summary tensor.
"""
with tf.name_scope(self._name + '/summary'):
mean_summary = tf.cond(self._count > 0, lambda: self._summary('mean', self._mean), str)
std_summary = tf.cond(self._count > 1, lambda: self._summary('stddev', self._std()), str)
return tf.summary.merge([mean_summary, std_summary])
def _build_model(self):
self.X = tf.placeholder(tf.float32, [None, 440])
self.Y = tf.placeholder(tf.float32, [None, 1940])
self.domain = tf.placeholder(tf.float32, [None, 2])
self.l = tf.placeholder(tf.float32, [])
self.train = tf.placeholder(tf.bool, [])
self.W1 = tf.placeholder(tf.float32, [440, 1024])
self.b1 = tf.placeholder(tf.float32, [1024])
self.W2 = tf.placeholder(tf.float32, [1024, 1024])
self.b2 = tf.placeholder(tf.float32, [1024])
self.W3 = tf.placeholder(tf.float32, [1024, 1024])
self.b3 = tf.placeholder(tf.float32, [1024])
self.W4 = tf.placeholder(tf.float32, [1024, 1024])
self.b4 = tf.placeholder(tf.float32, [1024])
self.W_out = tf.placeholder(tf.float32, [1024, 1940])
self.b_out = tf.placeholder(tf.float32, [1940])
with tf.variable_scope('feature_extractor'):
# l_1 = tf.get_variable('l1', shape=[1024], initializer=tf.contrib.layers.xavier_initializer())
# l_2 = tf.get_variable('l2', shape=[1024], initializer=tf.contrib.layers.xavier_initializer())
# l_3 = tf.get_variable('l3', shape=[1024], initializer=tf.contrib.layers.xavier_initializer())
l_1 = bias_variable([1024])
l_2 = bias_variable([1024])
l_3 = bias_variable([1024])
hidden_1 = tf.mul(tf.nn.sigmoid(tf.matmul(self.X, self.W1) + self.b1), 2*tf.nn.sigmoid(l_1))
hidden_2 = tf.mul(tf.nn.sigmoid(tf.matmul(hidden_1, self.W2) + self.b1), 2*tf.nn.sigmoid(l_2))
hidden_3 = tf.mul(tf.nn.sigmoid(tf.matmul(hidden_2, self.W3) + self.b3), 2*tf.nn.sigmoid(l_3))
self.features = hidden_3
with tf.variable_scope('label_predictor'):
all_features = lambda : self.features
source_features = lambda : tf.slice(self.features, [0,0], [batch_size/2, -1])
classify_feats = tf.cond(self.train, source_features, all_features)
all_labels = lambda : self.Y
source_labels = lambda : tf.slice(self.Y, [0,0], [batch_size/2, -1])
self.classify_labels = tf.cond(self.train, source_labels, all_labels)
hidden_4 = tf.nn.sigmoid(tf.matmul(classify_feats, self.W4) + self.b4)
logits = tf.matmul(hidden_4, self.W_out) + self.b_out
self.pred = tf.nn.softmax(logits)
self.pred_loss = tf.nn.softmax_cross_entropy_with_logits(logits, self.classify_labels)
with tf.variable_scope('domain_predictord'):
feat = flip_gradient(self.features, self.l)
W_4d = weight_variable([1024, 1024])
b_4d = bias_variable([1024])
W_outd = weight_variable([1024, 2])
b_outd = bias_variable([2])
# W_4d = tf.get_variable('W_4d', shape=[1024,1024], initializer=tf.contrib.layers.xavier_initializer())
# b_4d = tf.get_variable('b_4d', shape=[1024], initializer=tf.contrib.layers.xavier_initializer())
# W_outd = tf.get_variable('W_outd', shape=[1024, 1940], initializer=tf.contrib.layers.xavier_initializer())
# b_outd = tf.get_variable('b_outd', shape=[1940], initializer=tf.contrib.layers.xavier_initializer())
hidden_4d = tf.nn.sigmoid(tf.matmul(feat, W_4d) + b_4d)
d_logits = tf.matmul(hidden_4d, W_outd) + b_outd
self.domain_pred = tf.nn.softmax(d_logits)
self.domain_loss = tf.nn.softmax_cross_entropy_with_logits(d_logits, self.domain)
def __init__(self, args, txt_file, num_classes, mode, batch_size, num_preprocess_threads=1, shuffle=True,
min_queue_examples=1):
self.args = args
self.txt_file = txt_file
self.num_preprocess_threads = num_preprocess_threads
self.min_queue_examples = min_queue_examples
self.batch_size = batch_size
self.mode = mode
self.imgShape = [self.args.imageHeight, self.args.imageWidth, self.args.imageChannels]
self.maskShape = tf.stack([self.args.imageHeight, self.args.imageWidth])
self.num_classes = int(num_classes)
input_queue = tf.train.string_input_producer([txt_file], shuffle=False)
line_reader = tf.TextLineReader()
_, line = line_reader.read(input_queue)
split_line = tf.string_split([line]).values
if (mode == 'training' or mode == 'validation'):
split_line = tf.string_split([line]).values
rgb_image_path = split_line[0]
label_image_path = split_line[1]
self.image_o = self.read_image(rgb_image_path, 0)
self.label_image_o = self.read_image(label_image_path, 1)
do_flip = tf.random_uniform([], 0, 1)
self.image = tf.cond(do_flip > 0.5, lambda: tf.image.flip_left_right(self.image_o), lambda: self.image_o)
self.label_image = tf.cond(do_flip > 0.5, lambda: tf.image.flip_left_right(self.label_image_o),
lambda: self.label_image_o)
self.image.set_shape((self.args.imageHeight, self.args.imageWidth, 3))
self.label_image.set_shape((self.args.imageHeight, self.args.imageWidth, 1))
self.img_batch, self.label_batch = tf.train.shuffle_batch([self.image, self.label_image],
batch_size=batch_size,
num_threads=num_preprocess_threads,
capacity=min_queue_examples + 3 * batch_size,
min_after_dequeue=min_queue_examples)
elif (mode == 'test'):
print('Generating test Image Batch')
split_line = tf.string_split([line]).values
rgb_image_path = split_line[0]
self.image = self.read_image(rgb_image_path, 0)
self.image.set_shape((self.args.imageHeight, self.args.imageWidth, 3))
self.img_batch = tf.train.batch([self.image],
batch_size=batch_size,
num_threads=num_preprocess_threads,
capacity=min_queue_examples + 1 * batch_size,
)
def build(self, rgb, train_mode=None):
"""
load variable from npy to build the VGG
:param rgb: rgb image [batch, height, width, 3] values scaled [0, 1]
:param train_mode: a bool tensor, usually a placeholder: if True, dropout will be turned on
"""
#这里边输入的RGB图像是已经是经过缩放的了
rgb_scaled = rgb * 1.0
# Convert RGB to BGR
red, green, blue = tf.split(axis=3, num_or_size_splits=3, value=rgb_scaled)
assert red.get_shape().as_list()[1:] == [224, 224, 1]
assert green.get_shape().as_list()[1:] == [224, 224, 1]
assert blue.get_shape().as_list()[1:] == [224, 224, 1]
bgr = tf.concat(axis=3, values=[blue - VGG_MEAN[0],green - VGG_MEAN[1],red - VGG_MEAN[2]])
assert bgr.get_shape().as_list()[1:] == [224, 224, 3]
self.conv1_1 = self.conv_layer(bgr, 3,64,"conv1_1")
self.conv1_2 = self.conv_layer(self.conv1_1, 64,64,"conv1_2")
self.pool1 = self.max_pool(self.conv1_2, 'pool1')
self.conv2_1 = self.conv_layer(self.pool1, 64,128,"conv2_1")
self.conv2_2 = self.conv_layer(self.conv2_1,128 ,128,"conv2_2")
self.pool2 = self.max_pool(self.conv2_2, 'pool2')
self.conv3_1 = self.conv_layer(self.pool2, 128,256,"conv3_1")
self.conv3_2 = self.conv_layer(self.conv3_1,256,256,"conv3_2")
self.conv3_3 = self.conv_layer(self.conv3_2, 256,256,"conv3_3")
self.pool3 = self.max_pool(self.conv3_3, 'pool3')
self.conv4_1 = self.conv_layer(self.pool3, 256,512, "conv4_1")
self.conv4_2 = self.conv_layer(self.conv4_1, 512,512,"conv4_2")
self.conv4_3 = self.conv_layer(self.conv4_2,512,512, "conv4_3")
self.pool4 = self.max_pool(self.conv4_3, 'pool4')
self.conv5_1 = self.conv_layer(self.pool4,512,512, "conv5_1")
self.conv5_2 = self.conv_layer(self.conv5_1,512,512, "conv5_2")
self.conv5_3 = self.conv_layer(self.conv5_2,512,512, "conv5_3")
self.pool5 = self.max_pool(self.conv5_3, 'pool5')
self.fc6 = self.fc_layer(self.pool5, 25088, 4096, "fc6") # 25088 = ((224 // (2 ** 5)) ** 2) * 512 这个计算步骤也懂了
self.relu6 = tf.nn.relu(self.fc6)
if train_mode is not None:
self.relu6 = tf.cond(train_mode, lambda: tf.nn.dropout(self.relu6, self.dropout), lambda: self.relu6)
elif self.trainable:
self.relu6 = tf.nn.dropout(self.relu6, self.dropout)
self.fc7 = self.fc_layer(self.relu6, 4096, 4096, "fc7")
self.relu7 = tf.nn.relu(self.fc7)
if train_mode is not None:
self.relu7 = tf.cond(train_mode, lambda: tf.nn.dropout(self.relu7, self.dropout), lambda: self.relu7)
elif self.trainable:
self.relu7 = tf.nn.dropout(self.relu7, self.dropout)
def __call__(self, inputs, state, scope=None):
outputs_do, new_state_do = super(SwitchableDropoutWrapper, self).__call__(inputs, state, scope=scope)
tf.get_variable_scope().reuse_variables()
outputs, new_state = self._cell(inputs, state, scope)
outputs = tf.cond(self.is_train, lambda: outputs_do, lambda: outputs)
if isinstance(state, tuple):
new_state = state.__class__(*[tf.cond(self.is_train, lambda: new_state_do_i, lambda: new_state_i)
for new_state_do_i, new_state_i in zip(new_state_do, new_state)])
else:
new_state = tf.cond(self.is_train, lambda: new_state_do, lambda: new_state)
return outputs, new_state
def build_act(make_obs_ph, q_func, num_actions, scope="deepq", reuse=None):
"""Creates the act function:
Parameters
----------
make_obs_ph: str -> tf.placeholder or TfInput
a function that take a name and creates a placeholder of input with that name
q_func: (tf.Variable, int, str, bool) -> tf.Variable
the model that takes the following inputs:
observation_in: object
the output of observation placeholder
num_actions: int
number of actions
scope: str
reuse: bool
should be passed to outer variable scope
and returns a tensor of shape (batch_size, num_actions) with values of every action.
num_actions: int
number of actions.
scope: str or VariableScope
optional scope for variable_scope.
reuse: bool or None
whether or not the variables should be reused. To be able to reuse the scope must be given.
Returns
-------
act: (tf.Variable, bool, float) -> tf.Variable
function to select and action given observation.
` See the top of the file for details.
"""
with tf.variable_scope(scope, reuse=reuse):
observations_ph = make_obs_ph("observation")
stochastic_ph = tf.placeholder(tf.bool, (), name="stochastic")
update_eps_ph = tf.placeholder(tf.float32, (), name="update_eps")
eps = tf.get_variable("eps", (), initializer=tf.constant_initializer(0))
q_values = q_func(observations_ph.get(), num_actions, scope="q_func")
deterministic_actions = tf.argmax(q_values, axis=1)
batch_size = tf.shape(observations_ph.get())[0]
random_actions = tf.random_uniform(tf.stack([batch_size]), minval=0, maxval=num_actions, dtype=tf.int64)
chose_random = tf.random_uniform(tf.stack([batch_size]), minval=0, maxval=1, dtype=tf.float32) < eps
stochastic_actions = tf.where(chose_random, random_actions, deterministic_actions)
output_actions = tf.cond(stochastic_ph, lambda: stochastic_actions, lambda: deterministic_actions)
update_eps_expr = eps.assign(tf.cond(update_eps_ph >= 0, lambda: update_eps_ph, lambda: eps))
_act = U.function(inputs=[observations_ph, stochastic_ph, update_eps_ph],
outputs=output_actions,
givens={update_eps_ph: -1.0, stochastic_ph: True},
updates=[update_eps_expr])
def act(ob, stochastic=True, update_eps=-1):
return _act(ob, stochastic, update_eps)
return act
def testCond_3(self):
with self.test_session():
x = tf.constant(10)
pred = tf.less(1, 2)
fn1 = lambda: tf.add(x, 1)
fn2 = lambda: tf.sub(x, 1)
fn3 = lambda: tf.add(tf.cond(pred, fn1, fn2), 1)
r = tf.cond(pred, fn3, fn2)
result = r.eval()
self.assertTrue(check_op_order(x.graph))
self.assertAllEqual(12, result)
def pad_pred_label(predictions, labels):
num_digit_predictions = tf.shape(predictions)[-1]
num_digit_labels = tf.shape(labels)[-1]
paddings_mask = tf.constant([[0,0], [0,1]], dtype=labels.dtype)
paddings = tf.cast(tf.fill([2,2], tf.abs(num_digit_predictions-num_digit_labels)),labels.dtype)
paddings = paddings * paddings_mask
# paddings = tf.constant([[0, 0,], [0, tf.abs(num_digit_predictions-num_digit_predictions)]])
predictions = tf.cond(num_digit_predictions< num_digit_labels, lambda: tf.pad(predictions, paddings, constant_values=-1), lambda: tf.identity(predictions))
labels = tf.cond(num_digit_labels< num_digit_predictions, lambda: tf.pad(labels, paddings, constant_values=-1), lambda: tf.identity(labels))
return predictions, labels
def conv_internal(conv_fn, inputs, filters, kernel_size, **kwargs):
"""Conditional conv_fn making kernel 1d or 2d depending on inputs shape."""
static_shape = inputs.get_shape()
if not static_shape or len(static_shape) != 4:
raise ValueError("Inputs to conv must have statically known rank 4.")
inputs.set_shape([static_shape[0], None, None, static_shape[3]])
# Add support for left padding.
if "padding" in kwargs and kwargs["padding"] == "LEFT":
dilation_rate = (1, 1)
if "dilation_rate" in kwargs:
dilation_rate = kwargs["dilation_rate"]
assert kernel_size[0] % 2 == 1 and kernel_size[1] % 2 == 1
height_padding = 2 * (kernel_size[0] // 2) * dilation_rate[0]
cond_padding = tf.cond(
tf.equal(tf.shape(inputs)[2], 1), lambda: tf.constant(0),
lambda: tf.constant(2 * (kernel_size[1] // 2) * dilation_rate[1]))
width_padding = 0 if static_shape[2] == 1 else cond_padding
padding = [[0, 0], [height_padding, 0], [width_padding, 0], [0, 0]]
inputs = tf.pad(inputs, padding)
kwargs["padding"] = "VALID"
force2d = False # Special argument we use to force 2d kernels (see below).
if "force2d" in kwargs:
force2d = kwargs["force2d"]
def conv2d_kernel(kernel_size_arg, name_suffix):
"""Call conv2d but add suffix to name."""
if "name" in kwargs:
original_name = kwargs["name"]
name = kwargs.pop("name") + "_" + name_suffix
else:
original_name = None
name = "conv_" + name_suffix
original_force2d = None
if "force2d" in kwargs:
original_force2d = kwargs.pop("force2d")
result = conv_fn(inputs, filters, kernel_size_arg, name=name, **kwargs)
if original_name is not None:
kwargs["name"] = original_name # Restore for other calls.
if original_force2d is not None:
kwargs["force2d"] = original_force2d
return result
# Manually setting the shape to be unknown in the middle two dimensions so
# that the `tf.cond` below won't throw an error based on the convolution
# kernels being too large for the data.
inputs._shape = tf.TensorShape([static_shape[0], None, None, static_shape[3]]) # pylint: disable=protected-access
if kernel_size[1] == 1 or force2d:
# Avoiding the cond below can speed up graph and gradient construction.
return conv2d_kernel(kernel_size, "single")
return tf.cond(
tf.equal(tf.shape(inputs)[2],
1), lambda: conv2d_kernel((kernel_size[0], 1), "small"),
lambda: conv2d_kernel(kernel_size, "std"))
def batchnorm(Ylogits, is_test, iteration, convolutional=False):
exp_moving_avg = tf.train.ExponentialMovingAverage(0.9999, iteration) # adding the iteration prevents from averaging across non-existing iterations
bnepsilon = 1e-5
if convolutional:
mean, variance = tf.nn.moments(Ylogits, [0, 1, 2])
else:
mean, variance = tf.nn.moments(Ylogits, [0])
update_moving_everages = exp_moving_avg.apply([mean, variance])
m = tf.cond(is_test, lambda: exp_moving_avg.average(mean), lambda: mean)
v = tf.cond(is_test, lambda: exp_moving_avg.average(variance), lambda: variance)
Ybn = tf.nn.batch_normalization(Ylogits, m, v, 0.0, 1.0, bnepsilon)
return Ybn, update_moving_everages
def __init__(self, image):
image = tf.cast(image, tf.float32)
image = tf.cond(
tf.shape(tf.shape(image))[0] > 2, lambda: image[:, :, 0], lambda: image)
self.tensor = image < self.threshold
self.is_flipped = (tf.reduce_sum(
tf.reshape(tf.cast(self.tensor, tf.int64), [-1]))
> tf.cast(tf.cast(tf.reduce_prod(tf.shape(self.tensor)), tf.float32)
* 0.5, tf.int64))
self.image = tf.cond(self.is_flipped, lambda: 255 - image, lambda: image)
self.tensor = tf.cond(
self.is_flipped, lambda: ~self.tensor, lambda: self.tensor)
def main(unused_argv=None):
tf.logging.set_verbosity(FLAGS.log)
if FLAGS.config is None:
raise RuntimeError("No config name specified.")
config = utils.get_module("wavenet." + FLAGS.config).Config(
FLAGS.train_path)
logdir = FLAGS.logdir
tf.logging.info("Saving to %s" % logdir)
with tf.Graph().as_default():
total_batch_size = FLAGS.total_batch_size
assert total_batch_size % FLAGS.worker_replicas == 0
worker_batch_size = total_batch_size / FLAGS.worker_replicas
# Run the Reader on the CPU
cpu_device = "/job:localhost/replica:0/task:0/cpu:0"
if FLAGS.ps_tasks:
cpu_device = "/job:worker/cpu:0"
with tf.device(cpu_device):
inputs_dict = config.get_batch(worker_batch_size)
with tf.device(
tf.train.replica_device_setter(ps_tasks=FLAGS.ps_tasks,
merge_devices=True)):
global_step = tf.get_variable(
"global_step", [],
tf.int32,
initializer=tf.constant_initializer(0),
trainable=False)
# pylint: disable=cell-var-from-loop
lr = tf.constant(config.learning_rate_schedule[0])
for key, value in config.learning_rate_schedule.iteritems():
lr = tf.cond(tf.less(global_step, key), lambda: lr,
lambda: tf.constant(value))
# pylint: enable=cell-var-from-loop
tf.summary.scalar("learning_rate", lr)
# build the model graph
outputs_dict = config.build(inputs_dict, is_training=True)
loss = outputs_dict["loss"]
tf.summary.scalar("train_loss", loss)
worker_replicas = FLAGS.worker_replicas
ema = tf.train.ExponentialMovingAverage(decay=0.9999,
num_updates=global_step)
opt = tf.train.SyncReplicasOptimizer(
tf.train.AdamOptimizer(lr, epsilon=1e-8),
worker_replicas,
total_num_replicas=worker_replicas,
variable_averages=ema,
variables_to_average=tf.trainable_variables())
train_op = opt.minimize(loss,
global_step=global_step,
name="train",
colocate_gradients_with_ops=True)
session_config = tf.ConfigProto(allow_soft_placement=True)
is_chief = (FLAGS.task == 0)
local_init_op = opt.chief_init_op if is_chief else opt.local_step_init_op
slim.learning.train(
train_op=train_op,
logdir=logdir,
is_chief=is_chief,
master=FLAGS.master,
number_of_steps=config.num_iters,
global_step=global_step,
log_every_n_steps=250,
local_init_op=local_init_op,
save_interval_secs=300,
sync_optimizer=opt,
session_config=session_config,
)
def random_crop_patch(image,
bboxes,
labels,
prob=conf.random_crop_patch_prob,
seed=None):
"""Random flip left-right of an image and its bounding boxes."""
def crop_bboxes(image, bboxes, labels, offset_height, offset_width,
target_height, target_width):
"""crop bounding boxes coordinates.
Args:
bboxes: [xmin, ymin, xmax, ymax]
"""
#print('>>>>',tf.shape(bboxes))
#print('>>>>',bboxes)
bboxes = tf.reshape(bboxes, [-1, 4])
nocrop_boxes = bboxes
img_H = tf.cast(tf.shape(image, out_type=tf.int32)[0], tf.float32)
img_W = tf.cast(tf.shape(image, out_type=tf.int32)[1], tf.float32)
offset_width = tf.cast(offset_width, tf.float32)
offset_height = tf.cast(offset_height, tf.float32)
target_width = tf.cast(target_width, tf.float32)
target_height = tf.cast(target_height, tf.float32)
img = [img_W, img_H]
offset = [offset_width, offset_height]
crop = [target_width - 1., target_height - 1.]
box_col = tf.split(axis=1, num_or_size_splits=4, value=bboxes)
for i in range(4):
box_col[i] = box_col[i] * img[i % 2] - offset[i % 2]
box_col[i] = tf.clip_by_value(box_col[i], 0.,
crop[i % 2]) / crop[i % 2]
bboxes = tf.concat(axis=1, values=box_col)
keep = tf.where(
tf.less_equal(bboxes[:, 0], bboxes[:, 2] - 0.05)
& tf.less_equal(bboxes[:, 1], bboxes[:, 3] - 0.05))
crop_cond = tf.greater(tf.shape(keep)[0], 100)
bboxes, labels = tf.cond(
crop_cond, lambda:
(tf.gather_nd(bboxes, keep), tf.gather_nd(labels, keep)), lambda:
(nocrop_boxes, labels))
#bboxes = tf.gather_nd(bboxes, keep)
#labels = tf.gather_nd(labels, keep)
return crop_cond, bboxes, labels
def crop_images(image, seed=None):
"""crop image coordinates.
Args:
bboxes: [xmin, ymin, xmax, ymax]
"""
img_H = tf.cast(tf.shape(image, out_type=tf.int32)[0], tf.float32)
img_W = tf.cast(tf.shape(image, out_type=tf.int32)[1], tf.float32)
uniform_random = tf.random_uniform([],
conf.random_crop_image_prob,
1.0,
seed=seed)
target_height = tf.cast(img_H * uniform_random, dtype=tf.int32)
target_width = tf.cast(img_W * uniform_random, dtype=tf.int32)
offset_height = tf.random_uniform(
[],
minval=0,
maxval=(tf.cast(img_H, dtype=tf.int32) - target_height),
dtype=tf.int32)
offset_width = tf.random_uniform(
[],
minval=0,
maxval=(tf.cast(img_W, dtype=tf.int32) - target_width),
dtype=tf.int32)
return offset_height, offset_width, target_height, target_width
def process(image, bboxes, labels):
""" process
"""
offset_height, offset_width, target_height, target_width = crop_images(
image)
crop_cond, crop_bbox, crop_label = crop_bboxes(image, bboxes, labels,
offset_height,
offset_width,
target_height,
target_width)
crop_bbox, crop_label = tf.cond(crop_cond, lambda:
(crop_bbox, crop_label), lambda:
(bboxes, labels))
crop_image = tf.image.crop_to_bounding_box(image, offset_height,
offset_width, target_height,
target_width)
return crop_image, crop_bbox, crop_label
def no_process(image, bboxes, labels):
""" without process
"""
return image, bboxes, labels
# Random crop. Tensorflow implementation.
with tf.name_scope('random_crop_patch'):
image = tf.convert_to_tensor(image, name='image')
uniform_random = tf.random_uniform([], 0, 1.0, seed=seed)
crop_cond = tf.less(uniform_random, prob)
# crop image.
image, bboxes, labels = tf.cond(
crop_cond, #tf.convert_to_tensor(crop_cond),
lambda: process(image, bboxes, labels),
lambda: no_process(image, bboxes, labels))
return image, bboxes, labels
def peaks(values, minval=None, invalidate_distance=0, name=None):
"""Labels peaks in values.
Args:
values: 1D tensor of a numeric type.
minval: Minimum value which is considered a peak.
invalidate_distance: Invalidates nearby potential peaks. The peaks are
searched sequentially by descending value, and from left to right for
equal values. Once a peak is found in this order, it invalidates any peaks
yet to be seen that are <= invalidate_distance away. A distance of 0
effectively produces no invalidation.
name: Optional name for the op.
Returns:
peak_centers: The (rounded) centers of each peak, which are locations where
the value is higher than the value before and after. If there is a run
of equal values at the peak, the rounded center of the run is returned.
int32 1D tensor.
"""
with tf.name_scope(name, "peaks", [values]):
values = tf.convert_to_tensor(values, name="values")
invalidate_distance = tf.convert_to_tensor(invalidate_distance,
name="invalidate_distance",
dtype=tf.int32)
# Segment the values and find local maxima.
# Take the center of each run of consecutive equal values.
segment_centers, _ = _segments_1d(values, mode=SegmentsMode.CENTERS)
segment_values = tf.gather(values, segment_centers)
# If we have zero or one segments, there are no peaks. Just use zeros as the
# edge values in that case.
first_val, second_val, penultimate_val, last_val = tf.cond(
tf.greater_equal(tf.shape(segment_values)[0], 2),
lambda: tuple(segment_values[i] for i in (0, 1, -2, -1)),
lambda: tuple(tf.constant(0, values.dtype) for i in range(4)))
# Each segment must be greater than the segment before and after it.
segment_is_peak = tf.concat(
[[first_val > second_val],
tf.greater(segment_values[1:-1],
tf.maximum(segment_values[:-2], segment_values[2:])),
[last_val > penultimate_val]],
axis=0)
if minval is not None:
# Filter the peaks by minval.
segment_is_peak = tf.logical_and(
segment_is_peak, tf.greater_equal(segment_values, minval))
# Get the center coordinates of each peak, and sort by descending value.
all_peaks = tf.boolean_mask(segment_centers, segment_is_peak)
num_peaks = tf.shape(all_peaks)[0]
peak_values = tf.boolean_mask(segment_values, segment_is_peak)
_, peak_order = tf.nn.top_k(peak_values, k=num_peaks, sorted=True)
all_peaks = tf.gather(all_peaks, peak_order)
all_peaks.set_shape([None])
# Loop over the peaks, accepting one at a time and possibly invalidating
# other ones.
def loop_condition(_, current_peaks):
return tf.shape(current_peaks)[0] > 0
def loop_body(accepted_peaks, current_peaks):
peak = current_peaks[0]
remaining_peaks = current_peaks[1:]
keep_peaks = tf.greater(tf.abs(remaining_peaks - peak),
invalidate_distance)
remaining_peaks = tf.boolean_mask(remaining_peaks, keep_peaks)
return tf.concat([accepted_peaks, [peak]], axis=0), remaining_peaks
accepted_peaks = tf.while_loop(
loop_condition,
loop_body, [tf.zeros([0], all_peaks.dtype), all_peaks],
shape_invariants=[tf.TensorShape([None]),
tf.TensorShape([None])])[0]
# Sort the peaks by index.
# TODO(ringw): Add a tf.sort op that sorts in ascending order.
sorted_negative_peaks, _ = tf.nn.top_k(-accepted_peaks,
k=tf.shape(accepted_peaks)[0],
sorted=True)
return -sorted_negative_peaks
def _decode_image(content, channels):
return tf.cond(tf.image.is_jpeg(content),
lambda: tf.image.decode_jpeg(content, channels),
lambda: tf.image.decode_png(content, channels))
def style_loss(CNN_structure, const_layers, var_layers, content_segs,
style_segs, weight):
loss_styles = []
layer_count = float(len(const_layers))
layer_index = 0
_, content_seg_height, content_seg_width, _ = content_segs[0].get_shape(
).as_list()
_, style_seg_height, style_seg_width, _ = style_segs[0].get_shape(
).as_list()
for layer_name in CNN_structure:
layer_name = layer_name[layer_name.find("/") + 1:]
# downsampling segmentation
if "pool" in layer_name:
content_seg_width, content_seg_height = int(
math.ceil(content_seg_width / 2)), int(
math.ceil(content_seg_height / 2))
style_seg_width, style_seg_height = int(
math.ceil(style_seg_width / 2)), int(
math.ceil(style_seg_height / 2))
for i in xrange(len(content_segs)):
content_segs[i] = tf.image.resize_bilinear(
content_segs[i],
tf.constant((content_seg_height, content_seg_width)))
style_segs[i] = tf.image.resize_bilinear(
style_segs[i],
tf.constant((style_seg_height, style_seg_width)))
elif "conv" in layer_name:
for i in xrange(len(content_segs)):
# have some differences on border with torch
content_segs[i] = tf.nn.avg_pool(tf.pad(content_segs[i], [[0, 0], [1, 1], [1, 1], [0, 0]], "CONSTANT"), \
ksize=[1, 3, 3, 1], strides=[1, 1, 1, 1], padding='VALID')
style_segs[i] = tf.nn.avg_pool(tf.pad(style_segs[i], [[0, 0], [1, 1], [1, 1], [0, 0]], "CONSTANT"), \
ksize=[1, 3, 3, 1], strides=[1, 1, 1, 1], padding='VALID')
if layer_name == var_layers[layer_index].name[var_layers[layer_index].
name.find("/") + 1:]:
print("Setting up style layer: <{}>".format(layer_name))
const_layer = const_layers[layer_index]
var_layer = var_layers[layer_index]
layer_index = layer_index + 1
layer_style_loss = 0.0
for content_seg, style_seg in zip(content_segs, style_segs):
gram_matrix_const = gram_matrix(
tf.multiply(const_layer, style_seg))
style_mask_mean = tf.reduce_mean(style_seg)
gram_matrix_const = tf.cond(
tf.greater(style_mask_mean,
0.), lambda: gram_matrix_const /
(tf.to_float(tf.size(const_layer)) * style_mask_mean),
lambda: gram_matrix_const)
gram_matrix_var = gram_matrix(
tf.multiply(var_layer, content_seg))
content_mask_mean = tf.reduce_mean(content_seg)
gram_matrix_var = tf.cond(
tf.greater(content_mask_mean,
0.), lambda: gram_matrix_var /
(tf.to_float(tf.size(var_layer)) * content_mask_mean),
lambda: gram_matrix_var)
diff_style_sum = tf.reduce_mean(
tf.squared_difference(gram_matrix_const,
gram_matrix_var)) * content_mask_mean
layer_style_loss += diff_style_sum
loss_styles.append(layer_style_loss * weight)
return loss_styles
def eval_metrics_host_call_fn(policy_output,
value_output,
pi_tensor,
policy_cost,
value_cost,
l2_cost,
combined_cost,
step,
est_mode=tf.estimator.ModeKeys.TRAIN):
policy_entropy = -tf.reduce_mean(
tf.reduce_sum(policy_output * tf.log(policy_output), axis=1))
# pi_tensor is one_hot when generated from sgfs (for supervised learning)
# and soft-max when using self-play records. argmax normalizes the two.
policy_target_top_1 = tf.argmax(pi_tensor, axis=1)
policy_output_in_top1 = tf.to_float(
tf.nn.in_top_k(policy_output, policy_target_top_1, k=1))
policy_output_in_top3 = tf.to_float(
tf.nn.in_top_k(policy_output, policy_target_top_1, k=3))
policy_top_1_confidence = tf.reduce_max(policy_output, axis=1)
policy_target_top_1_confidence = tf.boolean_mask(
policy_output,
tf.one_hot(policy_target_top_1,
tf.shape(policy_output)[1]))
with tf.variable_scope("metrics"):
metric_ops = {
'policy_cost':
tf.metrics.mean(policy_cost),
'value_cost':
tf.metrics.mean(value_cost),
'l2_cost':
tf.metrics.mean(l2_cost),
'policy_entropy':
tf.metrics.mean(policy_entropy),
'combined_cost':
tf.metrics.mean(combined_cost),
'policy_accuracy_top_1':
tf.metrics.mean(policy_output_in_top1),
'policy_accuracy_top_3':
tf.metrics.mean(policy_output_in_top3),
'policy_top_1_confidence':
tf.metrics.mean(policy_top_1_confidence),
'policy_target_top_1_confidence':
tf.metrics.mean(policy_target_top_1_confidence),
'value_confidence':
tf.metrics.mean(tf.abs(value_output)),
}
if est_mode == tf.estimator.ModeKeys.EVAL:
return metric_ops
# NOTE: global_step is rounded to a multiple of FLAGS.summary_steps.
eval_step = tf.reduce_min(step)
# Create summary ops so that they show up in SUMMARIES collection
# That way, they get logged automatically during training
summary_writer = summary.create_file_writer(FLAGS.work_dir)
with summary_writer.as_default(), \
summary.record_summaries_every_n_global_steps(
params['summary_steps'], eval_step):
for metric_name, metric_op in metric_ops.items():
summary.scalar(metric_name, metric_op[1], step=eval_step)
# Reset metrics occasionally so that they are mean of recent batches.
reset_op = tf.variables_initializer(tf.local_variables("metrics"))
cond_reset_op = tf.cond(
tf.equal(eval_step % params['summary_steps'], tf.to_int64(1)),
lambda: reset_op, lambda: tf.no_op())
return summary.all_summary_ops() + [cond_reset_op]
def call(self, x):
input_image, y_pred, y_true, true_boxes = x
# adjust the shape of the y_predict [batch, grid_h, grid_w, 3, 4+1+nb_class]
y_pred = tf.reshape(
y_pred,
tf.concat([tf.shape(y_pred)[:3],
tf.constant([3, -1])], axis=0))
# initialize the masks
object_mask = tf.expand_dims(y_true[..., 4], 4)
# the variable to keep track of number of batches processed
batch_seen = tf.Variable(0.)
# compute grid factor and net factor
grid_h = tf.shape(y_true)[1]
grid_w = tf.shape(y_true)[2]
grid_factor = tf.reshape(tf.cast([grid_w, grid_h], tf.float32),
[1, 1, 1, 1, 2])
net_h = tf.shape(input_image)[1]
net_w = tf.shape(input_image)[2]
net_factor = tf.reshape(tf.cast([net_w, net_h], tf.float32),
[1, 1, 1, 1, 2])
"""
Adjust prediction
"""
pred_box_xy = (self.cell_grid[:, :grid_h, :grid_w, :, :] +
tf.sigmoid(y_pred[..., :2])) # sigma(t_xy) + c_xy
pred_box_wh = y_pred[..., 2:4] # t_wh
pred_box_conf = tf.expand_dims(tf.sigmoid(y_pred[..., 4]),
4) # adjust confidence
pred_box_class = y_pred[..., 5:] # adjust class probabilities
"""
Adjust ground truth
"""
true_box_xy = y_true[..., 0:2] # (sigma(t_xy) + c_xy)
true_box_wh = y_true[..., 2:4] # t_wh
true_box_conf = tf.expand_dims(y_true[..., 4], 4)
true_box_class = tf.argmax(y_true[..., 5:], -1)
"""
Compare each predicted box to all true boxes
"""
# initially, drag all objectness of all boxes to 0
conf_delta = pred_box_conf - 0
# then, ignore the boxes which have good overlap with some true box
true_xy = true_boxes[..., 0:2] / grid_factor
true_wh = true_boxes[..., 2:4] / net_factor
true_wh_half = true_wh / 2.
true_mins = true_xy - true_wh_half
true_maxes = true_xy + true_wh_half
pred_xy = tf.expand_dims(pred_box_xy / grid_factor, 4)
pred_wh = tf.expand_dims(
tf.exp(pred_box_wh) * self.anchors / net_factor, 4)
pred_wh_half = pred_wh / 2.
pred_mins = pred_xy - pred_wh_half
pred_maxes = pred_xy + pred_wh_half
intersect_mins = tf.maximum(pred_mins, true_mins)
intersect_maxes = tf.minimum(pred_maxes, true_maxes)
intersect_wh = tf.maximum(intersect_maxes - intersect_mins, 0.)
intersect_areas = intersect_wh[..., 0] * intersect_wh[..., 1]
true_areas = true_wh[..., 0] * true_wh[..., 1]
pred_areas = pred_wh[..., 0] * pred_wh[..., 1]
union_areas = pred_areas + true_areas - intersect_areas
iou_scores = tf.truediv(intersect_areas, union_areas)
best_ious = tf.reduce_max(iou_scores, axis=4)
conf_delta *= tf.expand_dims(
tf.to_float(best_ious < self.ignore_thresh), 4)
"""
Compute some online statistics
"""
true_xy = true_box_xy / grid_factor
true_wh = tf.exp(true_box_wh) * self.anchors / net_factor
true_wh_half = true_wh / 2.
true_mins = true_xy - true_wh_half
true_maxes = true_xy + true_wh_half
pred_xy = pred_box_xy / grid_factor
pred_wh = tf.exp(pred_box_wh) * self.anchors / net_factor
pred_wh_half = pred_wh / 2.
pred_mins = pred_xy - pred_wh_half
pred_maxes = pred_xy + pred_wh_half
intersect_mins = tf.maximum(pred_mins, true_mins)
intersect_maxes = tf.minimum(pred_maxes, true_maxes)
intersect_wh = tf.maximum(intersect_maxes - intersect_mins, 0.)
intersect_areas = intersect_wh[..., 0] * intersect_wh[..., 1]
true_areas = true_wh[..., 0] * true_wh[..., 1]
pred_areas = pred_wh[..., 0] * pred_wh[..., 1]
union_areas = pred_areas + true_areas - intersect_areas
iou_scores = tf.truediv(intersect_areas, union_areas)
iou_scores = object_mask * tf.expand_dims(iou_scores, 4)
count = tf.reduce_sum(object_mask)
count_noobj = tf.reduce_sum(1 - object_mask)
detect_mask = tf.to_float((pred_box_conf * object_mask) >= 0.5)
class_mask = tf.expand_dims(
tf.to_float(tf.equal(tf.argmax(pred_box_class, -1),
true_box_class)), 4)
recall50 = tf.reduce_sum(
tf.to_float(iou_scores >= 0.5) * detect_mask *
class_mask) / (count + 1e-3)
recall75 = tf.reduce_sum(
tf.to_float(iou_scores >= 0.75) * detect_mask *
class_mask) / (count + 1e-3)
avg_iou = tf.reduce_sum(iou_scores) / (count + 1e-3)
avg_obj = tf.reduce_sum(pred_box_conf * object_mask) / (count + 1e-3)
avg_noobj = tf.reduce_sum(pred_box_conf *
(1 - object_mask)) / (count_noobj + 1e-3)
avg_cat = tf.reduce_sum(object_mask * class_mask) / (count + 1e-3)
"""
Warm-up training
"""
batch_seen = tf.compat.v1.assign_add(batch_seen, 1.)
true_box_xy, true_box_wh, xywh_mask = tf.cond(
tf.less(batch_seen, self.warmup_batches + 1), lambda: [
true_box_xy +
(0.5 + self.cell_grid[:, :grid_h, :grid_w, :, :]) *
(1 - object_mask), true_box_wh + tf.zeros_like(true_box_wh) *
(1 - object_mask),
tf.ones_like(object_mask)
], lambda: [true_box_xy, true_box_wh, object_mask])
"""
Compare each true box to all anchor boxes
"""
wh_scale = tf.exp(true_box_wh) * self.anchors / net_factor
wh_scale = tf.expand_dims(
2 - wh_scale[..., 0] * wh_scale[..., 1],
axis=4) # the smaller the box, the bigger the scale
xy_delta = xywh_mask * (pred_box_xy -
true_box_xy) * wh_scale * self.xywh_scale
wh_delta = xywh_mask * (pred_box_wh -
true_box_wh) * wh_scale * self.xywh_scale
conf_delta = object_mask * (
pred_box_conf - true_box_conf) * self.obj_scale + (
1 - object_mask) * conf_delta * self.noobj_scale
class_delta = object_mask * \
tf.expand_dims(tf.nn.sparse_softmax_cross_entropy_with_logits(labels=true_box_class, logits=pred_box_class), 4) * \
self.class_scale
loss_xy = tf.reduce_sum(tf.square(xy_delta), list(range(1, 5)))
loss_wh = tf.reduce_sum(tf.square(wh_delta), list(range(1, 5)))
loss_conf = tf.reduce_sum(tf.square(conf_delta), list(range(1, 5)))
loss_class = tf.reduce_sum(class_delta, list(range(1, 5)))
loss = loss_xy + loss_wh + loss_conf + loss_class
if debug:
loss = tf.Print(loss, [grid_h, avg_obj],
message='avg_obj \t\t',
summarize=1000)
loss = tf.Print(loss, [grid_h, avg_noobj],
message='avg_noobj \t\t',
summarize=1000)
loss = tf.Print(loss, [grid_h, avg_iou],
message='avg_iou \t\t',
summarize=1000)
loss = tf.Print(loss, [grid_h, avg_cat],
message='avg_cat \t\t',
summarize=1000)
loss = tf.Print(loss, [grid_h, recall50],
message='recall50 \t',
summarize=1000)
loss = tf.Print(loss, [grid_h, recall75],
message='recall75 \t',
summarize=1000)
loss = tf.Print(loss, [grid_h, count],
message='count \t',
summarize=1000)
loss = tf.Print(loss, [
grid_h,
tf.reduce_sum(loss_xy),
tf.reduce_sum(loss_wh),
tf.reduce_sum(loss_conf),
tf.reduce_sum(loss_class)
],
message='loss xy, wh, conf, class: \t',
summarize=1000)
return loss * self.grid_scale
def build(self, bgr, train_mode=None):
"""
load variable from npy to build the VGG
:param bgr: bgr image [batch, height, width, 3] values scaled [0, 255]
:param train_mode: a bool tensor, usually a placeholder: if True, dropout will be turned on
"""
# subtract mean
blue, green, red = tf.split(axis=3, num_or_size_splits=3, value=bgr)
assert blue.get_shape().as_list()[1:] == [224, 224, 1]
assert green.get_shape().as_list()[1:] == [224, 224, 1]
assert red.get_shape().as_list()[1:] == [224, 224, 1]
bgr = tf.concat(axis=3, values=[
blue - VGG_MEAN[0],
green - VGG_MEAN[1],
red - VGG_MEAN[2],
])
assert bgr.get_shape().as_list()[1:] == [224, 224, 3]
self.conv1_1 = self.conv_layer(bgr, 3, 64, "conv1_1")
self.conv1_2 = self.conv_layer(self.conv1_1, 64, 64, "conv1_2")
self.pool1 = self.max_pool(self.conv1_2, 'pool1')
self.conv2_1 = self.conv_layer(self.pool1, 64, 128, "conv2_1")
self.conv2_2 = self.conv_layer(self.conv2_1, 128, 128, "conv2_2")
self.pool2 = self.max_pool(self.conv2_2, 'pool2')
self.conv3_1 = self.conv_layer(self.pool2, 128, 256, "conv3_1")
self.conv3_2 = self.conv_layer(self.conv3_1, 256, 256, "conv3_2")
self.conv3_3 = self.conv_layer(self.conv3_2, 256, 256, "conv3_3")
self.conv3_4 = self.conv_layer(self.conv3_3, 256, 256, "conv3_4")
self.pool3 = self.max_pool(self.conv3_4, 'pool3')
self.conv4_1 = self.conv_layer(self.pool3, 256, 512, "conv4_1")
self.conv4_2 = self.conv_layer(self.conv4_1, 512, 512, "conv4_2")
self.conv4_3 = self.conv_layer(self.conv4_2, 512, 512, "conv4_3")
self.conv4_4 = self.conv_layer(self.conv4_3, 512, 512, "conv4_4")
self.pool4 = self.max_pool(self.conv4_4, 'pool4')
self.conv5_1 = self.conv_layer(self.pool4, 512, 512, "conv5_1")
self.conv5_2 = self.conv_layer(self.conv5_1, 512, 512, "conv5_2")
self.conv5_3 = self.conv_layer(self.conv5_2, 512, 512, "conv5_3")
self.conv5_4 = self.conv_layer(self.conv5_3, 512, 512, "conv5_4")
self.pool5 = self.max_pool(self.conv5_4, 'pool5')
self.fc6 = self.fc_layer(self.pool5, 25088, 4096, "fc6") # 25088 = ((224 // (2 ** 5)) ** 2) * 512
self.relu6 = tf.nn.relu(self.fc6)
if train_mode is not None:
self.relu6 = tf.cond(train_mode, lambda: tf.nn.dropout(self.relu6, self.dropout), lambda: self.relu6)
elif self.trainable:
self.relu6 = tf.nn.dropout(self.relu6, self.dropout)
self.fc7 = self.fc_layer(self.relu6, 4096, 4096, "fc7")
self.relu7 = tf.nn.relu(self.fc7)
if train_mode is not None:
self.relu7 = tf.cond(train_mode, lambda: tf.nn.dropout(self.relu7, self.dropout), lambda: self.relu7)
elif self.trainable:
self.relu7 = tf.nn.dropout(self.relu7, self.dropout)
self.fc8 = self.fc_layer(self.relu7, 4096, 1000, "fc8")
self.prob = tf.nn.softmax(self.fc8, name="prob")
self.data_dict = None
Simple TensorFlow exercises
You should thoroughly test your code
"""
import tensorflow as tf
###############################################################################
# 1a: Create two random 0-d tensors x and y of any distribution.
# Create a TensorFlow object that returns x + y if x > y, and x - y otherwise.
# Hint: look up tf.cond()
# I do the first problem for you
###############################################################################
x = tf.random_uniform([]) # Empty array as shape creates a scalar.
y = tf.random_uniform([])
out = tf.cond(tf.greater(x, y), lambda: tf.add(x, y),
lambda: tf.subtract(x, y))
###############################################################################
# 1b: Create two 0-d tensors x and y randomly selected from the range [-1, 1).
# Return x + y if x < y, x - y if x > y, 0 otherwise.
# Hint: Look up tf.case().
###############################################################################
# YOUR CODE
x = tf.random_uniform([], -1, 1)
y = tf.random_uniform([], -1, 1)
out = tf.case(
{
tf.less(x, y): lambda: tf.add(x, y),
tf.greater(x, y): lambda: tf.substract(x, y)
},
def connect_data_and_network(self,
outputs_collector=None,
gradients_collector=None):
def switch_sampler(for_training):
with tf.name_scope('train' if for_training else 'validation'):
sampler = self.get_sampler()[0][0 if for_training else -1]
return sampler.pop_batch_op()
if self.is_training:
if self.action_param.validation_every_n > 0:
data_dict = tf.cond(tf.logical_not(self.is_validation),
lambda: switch_sampler(True),
lambda: switch_sampler(False))
else:
data_dict = switch_sampler(for_training=True)
image = tf.cast(data_dict['image'], tf.float32)
net_output = self.net(image, is_training=self.is_training)
with tf.name_scope('Optimiser'):
optimiser_class = OptimiserFactory.create(
name=self.action_param.optimiser)
self.optimiser = optimiser_class.get_instance(
learning_rate=self.action_param.lr)
loss_func = LossFunction(loss_type=self.action_param.loss_type)
data_loss = loss_func(net_output)
loss = data_loss
if self.net_param.decay > 0.0:
reg_losses = tf.get_collection(
tf.GraphKeys.REGULARIZATION_LOSSES)
if reg_losses:
reg_loss = tf.reduce_mean(
[tf.reduce_mean(reg_loss) for reg_loss in reg_losses])
loss = loss + reg_loss
grads = self.optimiser.compute_gradients(loss)
# collecting gradients variables
gradients_collector.add_to_collection([grads])
outputs_collector.add_to_collection(var=data_loss,
name='variational_lower_bound',
average_over_devices=True,
collection=CONSOLE)
outputs_collector.add_to_collection(var=data_loss,
name='variational_lower_bound',
average_over_devices=True,
summary_type='scalar',
collection=TF_SUMMARIES)
outputs_collector.add_to_collection(var=net_output[4],
name='Originals',
average_over_devices=False,
summary_type='image3_coronal',
collection=TF_SUMMARIES)
outputs_collector.add_to_collection(var=net_output[2],
name='Means',
average_over_devices=False,
summary_type='image3_coronal',
collection=TF_SUMMARIES)
outputs_collector.add_to_collection(var=net_output[5],
name='Variances',
average_over_devices=False,
summary_type='image3_coronal',
collection=TF_SUMMARIES)
else:
if self._infer_type in ('encode', 'encode-decode'):
data_dict = self.get_sampler()[0][0].pop_batch_op()
image = tf.cast(data_dict['image'], dtype=tf.float32)
net_output = self.net(image, is_training=False)
outputs_collector.add_to_collection(
var=data_dict['image_location'],
name='location',
average_over_devices=True,
collection=NETWORK_OUTPUT)
if self._infer_type == 'encode-decode':
outputs_collector.add_to_collection(
var=net_output[2],
name='generated_image',
average_over_devices=True,
collection=NETWORK_OUTPUT)
if self._infer_type == 'encode':
outputs_collector.add_to_collection(
var=net_output[7],
name='embedded',
average_over_devices=True,
collection=NETWORK_OUTPUT)
self.output_decoder = WindowAsImageAggregator(
image_reader=self.readers[0],
output_path=self.action_param.save_seg_dir)
return
elif self._infer_type == 'sample':
image_size = (self.net_param.batch_size,) + \
self.action_param.spatial_window_size + (1,)
dummy_image = tf.zeros(image_size)
net_output = self.net(dummy_image, is_training=False)
noise_shape = net_output[-1].shape.as_list()
stddev = self.autoencoder_param.noise_stddev
noise = tf.random_normal(shape=noise_shape,
mean=0.0,
stddev=stddev,
dtype=tf.float32)
partially_decoded_sample = self.net.shared_decoder(
noise, is_training=False)
decoder_output = self.net.decoder_means(
partially_decoded_sample, is_training=False)
outputs_collector.add_to_collection(var=decoder_output,
name='generated_image',
average_over_devices=True,
collection=NETWORK_OUTPUT)
self.output_decoder = WindowAsImageAggregator(
image_reader=None,
output_path=self.action_param.save_seg_dir)
return
elif self._infer_type == 'linear_interpolation':
# construct the entire network
image_size = (self.net_param.batch_size,) + \
self.action_param.spatial_window_size + (1,)
dummy_image = tf.zeros(image_size)
net_output = self.net(dummy_image, is_training=False)
data_dict = self.get_sampler()[0][0].pop_batch_op()
real_code = data_dict['feature']
real_code = tf.reshape(real_code, net_output[-1].get_shape())
partially_decoded_sample = self.net.shared_decoder(
real_code, is_training=False)
decoder_output = self.net.decoder_means(
partially_decoded_sample, is_training=False)
outputs_collector.add_to_collection(var=decoder_output,
name='generated_image',
average_over_devices=True,
collection=NETWORK_OUTPUT)
outputs_collector.add_to_collection(
var=data_dict['feature_location'],
name='location',
average_over_devices=True,
collection=NETWORK_OUTPUT)
self.output_decoder = WindowAsImageAggregator(
image_reader=self.readers[0],
output_path=self.action_param.save_seg_dir)
else:
raise NotImplementedError
def decay_warmup(params, hyper_params, global_step):
"""
Setups an optimizer object and returns it.
:param params: dict
:param hyper_params: dict, with hyper-parameters
:return: optimizer
"""
NUM_EPOCHS_PER_DECAY = hyper_params['num_epochs_per_decay']
NUM_EPOCHS_PER_RAMP = hyper_params['num_epochs_per_ramp']
NUM_EPOCHS_IN_WARM_UP = hyper_params['num_epochs_in_warm_up']
NUM_STEPS_IN_WARM_UP = hyper_params['num_steps_in_warm_up']
NUM_STEPS_PER_WARM_UP = hyper_params['num_steps_per_warm_up']
INITIAL_LEARNING_RATE = hyper_params['initial_learning_rate']
LEARNING_RATE_DECAY_FACTOR = hyper_params['learning_rate_decay_factor']
WARM_UP_LEARNING_RATE_MAX = tf.constant(
hyper_params['warm_up_max_learning_rate'], tf.float32)
# Set parameters that affect the learning rate.
decay_steps = hyper_params['num_steps_per_decay']
ramp_steps = NUM_STEPS_PER_WARM_UP
ramp_up_steps = NUM_STEPS_IN_WARM_UP
# Decay/ramp the learning rate exponentially based on the number of steps.
def ramp():
lr = tf.train.exponential_decay(INITIAL_LEARNING_RATE,
global_step,
ramp_steps,
LEARNING_RATE_DECAY_FACTOR,
staircase=True)
lr = INITIAL_LEARNING_RATE**2 * tf.pow(lr, tf.constant(-1.))
lr = tf.minimum(lr, WARM_UP_LEARNING_RATE_MAX)
return lr
def linear_ramp():
slope = WARM_UP_LEARNING_RATE_MAX / tf.cast(NUM_STEPS_PER_WARM_UP,
tf.float32)
lr = tf.cast(INITIAL_LEARNING_RATE,
tf.float32) + tf.cast(global_step, tf.float32) * slope
lr = tf.math.minimum(tf.cast(WARM_UP_LEARNING_RATE_MAX, tf.float32),
lr)
return lr
def decay(lr_current):
lr = tf.train.exponential_decay(lr_current,
global_step,
decay_steps,
LEARNING_RATE_DECAY_FACTOR,
staircase=True)
return lr
warm_up_slope = hyper_params.get('warm_up_slope', 0)
def constant_ramp():
lr = tf.cast(INITIAL_LEARNING_RATE, tf.float32) * (
tf.cast(global_step, tf.float32) * warm_up_slope + 1)
lr = tf.math.minimum(tf.cast(WARM_UP_LEARNING_RATE_MAX, tf.float32),
lr)
return lr
if warm_up_slope > 1e-2:
# LEARNING_RATE = tf.cond(global_step < ramp_up_steps, ramp, lambda: decay(ramp()))
#LEARNING_RATE = tf.cond(global_step < ramp_up_steps, linear_ramp, lambda: decay(linear_ramp()))
ramp_up_steps = tf.cast(
WARM_UP_LEARNING_RATE_MAX / INITIAL_LEARNING_RATE,
global_step.dtype)
LEARNING_RATE = tf.cond(global_step < ramp_up_steps, constant_ramp,
lambda: decay(constant_ramp()))
else:
LEARNING_RATE = tf.train.exponential_decay(INITIAL_LEARNING_RATE,
global_step,
decay_steps,
LEARNING_RATE_DECAY_FACTOR,
staircase=True)
return LEARNING_RATE
def build(self, eval_flag):
params = self.params
batch_size = params.batch_size
vocab_size = self.vocab_size
num_layers = params.rnnlm_layers
hidden_size = params.rnnlm_hidden_size
num_steps = self.num_steps
# initialize self
# placeholders
inputs = tf.placeholder('int32', shape=[None, num_steps], name='x') # [num_batch, num_steps]
ground_truth = tf.placeholder('int32', shape=[None, num_steps], name='y') # [num_batch, num_steps] - right shift version of x
is_training = tf.placeholder(tf.bool)
keep_prob = tf.placeholder(tf.float32)
batch_size = tf.shape(inputs)[0]
normal_initializer = tf.random_normal_initializer(stddev=0.1)
ones_initializer = tf.constant_initializer(1.0)
with tf.variable_scope('RNNLM', initializer=normal_initializer):
# Embeddings
with tf.device('/cpu:0'):
embedding_params = tf.get_variable('embedding_params', [vocab_size, hidden_size])
inputs_embedding = tf.nn.embedding_lookup(embedding_params, inputs)
inputs_embedding = tf.cond(is_training,
lambda: tf.nn.dropout(inputs_embedding, keep_prob),
lambda: inputs_embedding)
# Recurrence
# Slightly better results can be obtained with forget gate biases
# initialized to 1 but the hyperparameters of the model would need to be
# different than reported in the paper.
def lstm_cell():
# With the latest TensorFlow source code (as of Mar 27, 2017),
# the BasicLSTMCell will need a reuse parameter which is unfortunately not
# defined in TensorFlow 1.0. To maintain backwards compatibility, we add
# an argument check here:
if 'reuse' in inspect.getargspec(
tf.contrib.rnn.BasicLSTMCell.__init__).args:
return tf.contrib.rnn.BasicLSTMCell(
hidden_size, forget_bias=0.0, state_is_tuple=True,
reuse=tf.get_variable_scope().reuse)
else:
return tf.contrib.rnn.BasicLSTMCell(
hidden_size, forget_bias=0.0, state_is_tuple=True)
def attn_cell():
return tf.contrib.rnn.DropoutWrapper(lstm_cell(), output_keep_prob=keep_prob)
cell = tf.contrib.rnn.MultiRNNCell(
[attn_cell() for _ in range(num_layers)], state_is_tuple=True)
self.initial_state = cell.zero_state(batch_size, 'float32')
### RNN ###
outputs = []
state = self.initial_state
with tf.variable_scope('core'):
for time_step in range(num_steps):
if time_step > 0:tf.get_variable_scope().reuse_variables()
(cell_output, state) = cell(inputs_embedding[:, time_step, :], state)
outputs.append(cell_output)
output = tf.reshape(tf.stack(axis=1, values=outputs), [-1, hidden_size]) #[batch_size * num_steps, hidden_size]
weight_o = tf.get_variable('output_weight', [hidden_size, vocab_size], 'float32')
bias_o = tf.get_variable('output_bias', [vocab_size], 'float32')
logits = tf.matmul(output, weight_o) + bias_o
loss = tf.contrib.legacy_seq2seq.sequence_loss_by_example(
[logits], [tf.reshape(ground_truth, [-1])],
[tf.ones([batch_size * num_steps], dtype='float32')])
batch_size = tf.cast(batch_size, tf.float32)
self.loss = tf.reduce_sum(loss) / batch_size
log_perp = loss
self.final_state = state # for recurrent purpose
if not eval_flag:
def learning_rate_decay_fn(lr, global_step):
return tf.train.exponential_decay(lr,
global_step,
decay_steps=3000,
decay_rate=0.8,
staircase=True)
OPTIMIZER_SUMMARIES = ["learning_rate",
"loss",
"gradients",
"gradient_norm"]
opt_op = tf.contrib.layers.optimize_loss(self.loss,
self.global_step,
learning_rate=params.learning_rate,
optimizer=tf.train.AdamOptimizer,
clip_gradients=10.,
learning_rate_decay_fn=learning_rate_decay_fn,
summaries=OPTIMIZER_SUMMARIES)
self.x = inputs
self.y = ground_truth
self.is_training = is_training
self.keep_prob = keep_prob
self.log_perp = log_perp #[batch_size, 1]
self.num_steps = num_steps
# Output Module
if not eval_flag:
self.opt_op = opt_op
else:
self.opt_op = None
def main():
if a.seed is None:
a.seed = random.randint(0, 2**31 - 1)
tf.set_random_seed(a.seed)
np.random.seed(a.seed)
random.seed(a.seed)
if not os.path.exists(a.output_dir):
os.makedirs(a.output_dir)
if a.mode == "test" or a.mode == "export":
if a.checkpoint is None:
raise Exception("checkpoint required for test mode")
# load some options from the checkpoint
options = {"which_direction", "ngf", "ndf", "lab_colorization"}
with open(os.path.join(a.checkpoint, "options.json")) as f:
for key, val in json.loads(f.read()).items():
if key in options:
print("loaded", key, "=", val)
setattr(a, key, val)
# disable these features in test mode
a.scale_size = CROP_SIZE
a.flip = False
for k, v in a._get_kwargs():
print(k, "=", v)
with open(os.path.join(a.output_dir, "options.json"), "w") as f:
f.write(json.dumps(vars(a), sort_keys=True, indent=4))
if a.mode == "export":
# export the generator to a meta graph that can be imported later for standalone generation
if a.lab_colorization:
raise Exception("export not supported for lab_colorization")
ipt = tf.placeholder(tf.string, shape=[1])
input_data = tf.decode_base64(ipt[0])
input_image = tf.image.decode_png(input_data)
# remove alpha channel if present
input_image = tf.cond(tf.equal(tf.shape(input_image)[2],
4), lambda: input_image[:, :, :3],
lambda: input_image)
# convert grayscale to RGB
input_image = tf.cond(tf.equal(tf.shape(input_image)[2], 1),
lambda: tf.image.grayscale_to_rgb(input_image),
lambda: input_image)
input_image = tf.image.convert_image_dtype(input_image,
dtype=tf.float32)
input_image.set_shape([CROP_SIZE, CROP_SIZE, 3])
batch_input = tf.expand_dims(input_image, axis=0)
with tf.variable_scope("generator"):
batch_output = deprocess(
create_generator(preprocess(batch_input), 3))
output_image = tf.image.convert_image_dtype(batch_output,
dtype=tf.uint8)[0]
if a.output_filetype == "png":
output_data = tf.image.encode_png(output_image)
elif a.output_filetype == "jpeg":
output_data = tf.image.encode_jpeg(output_image, quality=80)
else:
raise Exception("invalid filetype")
output = tf.convert_to_tensor([tf.encode_base64(output_data)])
key = tf.placeholder(tf.string, shape=[1])
inputs = {"key": key.name, "input": ipt.name}
tf.add_to_collection("inputs", json.dumps(inputs))
outputs = {
"key": tf.identity(key).name,
"output": output.name,
}
tf.add_to_collection("outputs", json.dumps(outputs))
init_op = tf.global_variables_initializer()
restore_saver = tf.train.Saver()
export_saver = tf.train.Saver()
with tf.Session() as sess:
sess.run(init_op)
print("loading model from checkpoint")
checkpoint = tf.train.latest_checkpoint(a.checkpoint)
restore_saver.restore(sess, checkpoint)
print("exporting model")
export_saver.export_meta_graph(
filename=os.path.join(a.output_dir, "export.meta"))
export_saver.save(sess,
os.path.join(a.output_dir, "export"),
write_meta_graph=False)
return
examples = load_examples()
print("examples count = %d" % examples.count)
# inputs and targets are [batch_size, height, width, channels]
model = create_model(examples.inputs, examples.targets)
# undo colorization splitting on images that we use for display/output
if a.lab_colorization:
if a.which_direction == "AtoB":
# inputs is brightness, this will be handled fine as a grayscale image
# need to augment targets and outputs with brightness
targets = augment(examples.targets, examples.inputs)
outputs = augment(model.outputs, examples.inputs)
# inputs can be deprocessed normally and handled as if they are single channel
# grayscale images
inputs = deprocess(examples.inputs)
elif a.which_direction == "BtoA":
# inputs will be color channels only, get brightness from targets
inputs = augment(examples.inputs, examples.targets)
targets = deprocess(examples.targets)
outputs = deprocess(model.outputs)
else:
raise Exception("invalid direction")
else:
inputs = deprocess(examples.inputs)
targets = deprocess(examples.targets)
outputs = deprocess(model.outputs)
if a.img_masks:
masks = examples.masks
def convert(image):
if a.aspect_ratio != 1.0:
# upscale to correct aspect ratio
size = [CROP_SIZE, int(round(CROP_SIZE * a.aspect_ratio))]
image = tf.image.resize_images(
image, size=size, method=tf.image.ResizeMethod.BICUBIC)
return tf.image.convert_image_dtype(image,
dtype=tf.uint8,
saturate=True)
# reverse any processing on images so they can be written to disk or displayed to user
with tf.name_scope("convert_inputs"):
converted_inputs = convert(inputs)
with tf.name_scope("convert_targets"):
converted_targets = convert(targets)
with tf.name_scope("convert_outputs"):
converted_outputs = convert(outputs)
if a.img_masks and a.use_cue:
with tf.name_scope("convert_masks"):
converted_masks = convert(masks)
# converted_masks = tf.expand_dims(converted_masks)
with tf.name_scope("encode_images"):
display_fetches = {
"paths":
examples.paths,
"inputs":
tf.map_fn(tf.image.encode_png,
converted_inputs,
dtype=tf.string,
name="input_pngs"),
"targets":
tf.map_fn(tf.image.encode_png,
converted_targets,
dtype=tf.string,
name="target_pngs"),
"outputs":
tf.map_fn(tf.image.encode_png,
converted_outputs,
dtype=tf.string,
name="output_pngs"),
}
if a.img_masks and a.use_cue:
# TODO add circle masks to this shit
display_fetches['mask'] = tf.map_fn(tf.image.encode_png,
converted_masks,
dtype=tf.string,
name="output_pngs"),
# summaries
with tf.name_scope("inputs_summary"):
tf.summary.image("inputs", converted_inputs)
with tf.name_scope("targets_summary"):
tf.summary.image("targets", converted_targets)
with tf.name_scope("outputs_summary"):
tf.summary.image("outputs", converted_outputs)
if a.img_masks and a.use_cue:
with tf.name_scope("masks_summary"):
tf.summary.image("masks", converted_masks)
with tf.name_scope("predict_real_summary"):
tf.summary.image(
"predict_real",
tf.image.convert_image_dtype(model.predict_real, dtype=tf.uint8))
with tf.name_scope("predict_fake_summary"):
tf.summary.image(
"predict_fake",
tf.image.convert_image_dtype(model.predict_fake, dtype=tf.uint8))
tf.summary.scalar("discriminator_loss", model.discrim_loss)
tf.summary.scalar("generator_loss_GAN", model.gen_loss_GAN)
tf.summary.scalar("generator_loss_L1", model.gen_loss_L1)
for var in tf.trainable_variables():
tf.summary.histogram(var.op.name + "/values", var)
for grad, var in model.discrim_grads_and_vars + model.gen_grads_and_vars:
tf.summary.histogram(var.op.name + "/gradients", grad)
with tf.name_scope("parameter_count"):
parameter_count = tf.reduce_sum(
[tf.reduce_prod(tf.shape(v)) for v in tf.trainable_variables()])
saver = tf.train.Saver(max_to_keep=1)
logdir = a.output_dir if (a.trace_freq > 0 or a.summary_freq > 0) else None
sv = tf.train.Supervisor(logdir=logdir, save_summaries_secs=0, saver=None)
with sv.managed_session() as sess:
print("parameter_count =", sess.run(parameter_count))
if a.checkpoint is not None:
print("loading model from checkpoint")
checkpoint = tf.train.latest_checkpoint(a.checkpoint)
saver.restore(sess, checkpoint)
max_steps = 2**32
if a.max_epochs is not None:
max_steps = examples.steps_per_epoch * a.max_epochs
if a.max_steps is not None:
max_steps = a.max_steps
if a.mode == "test":
# testing
# at most, process the test data once
start = time.time()
max_steps = min(examples.steps_per_epoch, max_steps)
for step in range(max_steps):
results = sess.run(display_fetches)
filesets = save_images(results)
for i, f in enumerate(filesets):
print("evaluated image", f["name"])
index_path = append_index(filesets)
print("wrote index at", index_path)
print("rate", (time.time() - start) / max_steps)
else:
# training
start = time.time()
for step in range(max_steps):
def should(freq):
return freq > 0 and ((step + 1) % freq == 0
or step == max_steps - 1)
options = None
run_metadata = None
if should(a.trace_freq):
options = tf.RunOptions(
trace_level=tf.RunOptions.FULL_TRACE)
run_metadata = tf.RunMetadata()
fetches = {
"train": model.train,
"global_step": sv.global_step,
}
if should(a.progress_freq):
fetches["discrim_loss"] = model.discrim_loss
fetches["gen_loss_GAN"] = model.gen_loss_GAN
fetches["gen_loss_L1"] = model.gen_loss_L1
if should(a.summary_freq):
fetches["summary"] = sv.summary_op
if should(a.display_freq):
fetches["display"] = display_fetches
results = sess.run(fetches,
options=options,
run_metadata=run_metadata)
if should(a.summary_freq):
print("recording summary")
sv.summary_writer.add_summary(results["summary"],
results["global_step"])
if should(a.display_freq):
filesets = save_images(results["display"],
step=results["global_step"])
append_index(filesets, step=True)
if should(a.trace_freq):
print("recording trace")
sv.summary_writer.add_run_metadata(
run_metadata, "step_%d" % results["global_step"])
if should(a.progress_freq):
# global_step will have the correct step count if we resume from a checkpoint
train_epoch = math.ceil(results["global_step"] /
examples.steps_per_epoch)
train_step = (results["global_step"] -
1) % examples.steps_per_epoch + 1
rate = (step + 1) * a.batch_size / (time.time() - start)
remaining = (max_steps - step) * a.batch_size / rate
print(
"progress epoch %d step %d image/sec %0.1f remaining %dm"
% (train_epoch, train_step, rate, remaining / 60))
print("discrim_loss", results["discrim_loss"])
print("gen_loss_GAN", results["gen_loss_GAN"])
print("gen_loss_L1", results["gen_loss_L1"])
if should(a.save_freq):
print("saving model")
saver.save(sess,
os.path.join(a.output_dir, "model"),
global_step=sv.global_step)
if sv.should_stop():
break
def true_segments_1d(segments,
mode=SegmentsMode.CENTERS,
max_gap=0,
min_length=0,
name=None):
"""Labels contiguous True runs in segments.
Args:
segments: 1D boolean tensor.
mode: The SegmentsMode. Returns the start of each segment (STARTS), or the
rounded center of each segment (CENTERS).
max_gap: Fill gaps of length at most `max_gap` between true segments. int.
min_length: Minimum length of a returned segment. int.
name: Optional name for the op.
Returns:
run_centers: int32 tensor. Depending on `mode`, either the start of each
True run, or the (rounded) center of each True run.
run_lengths: int32; the lengths of each True run.
"""
with tf.name_scope(name, "true_segments", [segments]):
segments = tf.convert_to_tensor(segments, tf.bool)
run_starts, run_lengths = _segments_1d(segments,
mode=SegmentsMode.STARTS)
# Take only the True runs. After whichever run is True first, the True runs
# are every other run.
first_run = tf.cond(
# First value is False, or all values are False. Handles empty segments
# correctly.
tf.logical_or(tf.reduce_any(segments[0:1]),
~tf.reduce_any(segments)),
lambda: tf.constant(0),
lambda: tf.constant(1))
num_runs = tf.shape(run_starts)[0]
run_nums = tf.range(num_runs)
is_true_run = tf.equal(run_nums % 2, first_run % 2)
# Find gaps between True runs that can be merged.
is_gap = tf.logical_and(
tf.not_equal(run_nums % 2, first_run % 2),
tf.logical_and(tf.greater(run_nums, first_run),
tf.less(run_nums, num_runs - 1)))
fill_gap = tf.logical_and(is_gap, tf.less_equal(run_lengths, max_gap))
# Segment the consecutive runs of True or False values based on whether they
# are True, or are a gap of False values that can be bridged. Then, flatten
# the runs of runs.
runs_to_merge = tf.logical_or(is_true_run, fill_gap)
run_of_run_starts, _ = _segments_1d(runs_to_merge,
mode=SegmentsMode.STARTS)
# Get the start of every new run from the original run starts.
merged_run_starts = tf.gather(run_starts, run_of_run_starts)
# Make an array mapping the original runs to their run of runs. Increment
# the number for every run of run start except for the first one, so that
# the array has values from 0 to num_run_of_runs.
merged_run_inds = tf.cumsum(
tf.sparse_to_dense(
sparse_indices=tf.cast(run_of_run_starts[1:, None], tf.int64),
output_shape=tf.cast(num_runs[None], tf.int64),
sparse_values=tf.ones_like(run_of_run_starts[1:])))
# Sum the lengths of the original runs that were merged.
merged_run_lengths = tf.segment_sum(run_lengths, merged_run_inds)
if mode is SegmentsMode.CENTERS:
merged_starts_or_centers = (merged_run_starts +
tf.floordiv(merged_run_lengths - 1, 2))
else:
merged_starts_or_centers = merged_run_starts
# If there are no true values, increment first_run to 1, so we will skip
# the single (false) run.
first_run += tf.to_int32(tf.logical_not(tf.reduce_any(segments)))
merged_starts_or_centers = merged_starts_or_centers[first_run::2]
merged_run_lengths = merged_run_lengths[first_run::2]
# Only take segments at least min_length long.
is_long_enough = tf.greater_equal(merged_run_lengths, min_length)
is_long_enough.set_shape([None])
merged_starts_or_centers = tf.boolean_mask(merged_starts_or_centers,
is_long_enough)
merged_run_lengths = tf.boolean_mask(merged_run_lengths,
is_long_enough)
return merged_starts_or_centers, merged_run_lengths
def conv_embed(label, it_same, it_diff):
next_same = it_same.get_next()
next_diff = it_diff.get_next()
input_layer = preprocess(
tf.cond(label, lambda: next_diff, lambda: next_same))
conv1 = tf.layers.conv2d(inputs=input_layer,
filters=64,
kernel_size=[10, 10],
padding="valid",
activation=tf.nn.relu)
pool1 = tf.layers.max_pooling2d(inputs=conv1, pool_size=[2, 2], strides=2)
conv2 = tf.layers.conv2d(inputs=pool1,
filters=128,
kernel_size=[7, 7],
padding="valid",
activation=tf.nn.relu)
pool2 = tf.layers.max_pooling2d(inputs=conv2, pool_size=[2, 2], strides=2)
conv3 = tf.layers.conv2d(inputs=pool2,
filters=128,
kernel_size=[4, 4],
padding="valid",
activation=tf.nn.relu)
pool3 = tf.layers.max_pooling2d(inputs=conv3, pool_size=[2, 2], strides=2)
conv4 = tf.layers.conv2d(inputs=pool3,
filters=256,
kernel_size=[4, 4],
padding="valid",
activation=tf.nn.relu)
pool4 = tf.layers.max_pooling2d(inputs=conv4, pool_size=[2, 2], strides=2)
conv5 = tf.layers.conv2d(inputs=pool4,
filters=256,
kernel_size=[3, 3],
padding="valid",
activation=tf.nn.relu)
pool5 = tf.layers.max_pooling2d(inputs=conv5, pool_size=[2, 2], strides=2)
flat = tf.contrib.layers.flatten(inputs=pool5)
dense = tf.layers.dense(inputs=flat, units=4096, activation=tf.nn.relu)
dropout = tf.layers.dropout(inputs=dense, rate=0.5)
sphere = tf.nn.l2_normalize(
dropout,
axis=1) #project outputs of previous layer onto surface of hypersphere
loss = pairwise_quadratic_loss(sphere, label)
optimizer = tf.train.AdamOptimizer()
train = optimizer.minimize(loss)
return {"embedding": sphere, "loss": loss, "train": train}
def _compute_loss(self,input_rnn,seq_len,ner_ids,scoreMatrix,is_train,dropout_embedding_keep,dropout_lstm_keep,
dropout_lstm_output_keep,dropout_fcl_ner_keep,dropout_fcl_rel_keep,mask,reuse=False):
"""计算损失"""
with tf.variable_scope("loss_conputation",reuse=reuse):
if self.config.use_dropout:
input_rnn=tf.nn.dropout(input_rnn,keep_prob=dropout_embedding_keep)
#BiLSTM底层编码
# for i in range(self.config.num_lstm_layers):
# if self.config.use_dropout and i>0:
# input_rnn=tf.nn.dropout(input_rnn,keep_prob=dropout_lstm_keep)
# lstm_fw_cell=tf.nn.rnn_cell.LSTMCell(self.config.hidden_size_lstm)
# lstm_bw_cell=tf.nn.rnn_cell.LSTMCell(self.config.hidden_size_lstm)
#
# outputs,states=tf.nn.bidirectional_dynamic_rnn(lstm_fw_cell,
# lstm_bw_cell,
# input_rnn,
# sequence_length=seq_len,
# dtype=tf.float32,
# scope="BiLSTM"+str(i))
# input_rnn = tf.concat(outputs, axis=-1)
# lstm_output = input_rnn
input_rnn=tf.keras.layers.Bidirectional(CuDNNLSTM(units=self.config.hidden_size_lstm,return_sequences=True))(inputs=input_rnn)
input_rnn=tf.keras.layers.Bidirectional(CuDNNLSTM(units=self.config.hidden_size_lstm,return_sequences=True))(inputs=input_rnn)
lstm_output=input_rnn
lstm_output = self._attention(lstm_output,mask)
if self.config.use_dropout:
lstm_output=tf.nn.dropout(lstm_output,keep_prob=dropout_lstm_output_keep)
#计算LSTM发射分数
nerScores=self._get_ner_score(lstm_output,self.config.num_ner_classes,dropout_fcl_ner_keep)
#CRF计算NER损失
log_likelihood,trans_params=tf.contrib.crf.crf_log_likelihood(nerScores,ner_ids,seq_len)
lossNER=tf.reduce_mean(-log_likelihood)
#viterbi解码预测NER标签
predNERS,viterbi_score=tf.contrib.crf.crf_decode(nerScores,trans_params,seq_len)
#RC输入
if self.config.label_embedding_size>0:
labels=tf.cond(is_train,lambda:ner_ids,lambda:predNERS)
label_matrix=tf.get_variable(name="label_embedding",shape=[self.config.num_ner_classes,
self.config.label_embedding_size],dtype=tf.float32)
label_embedding=tf.nn.embedding_lookup(label_matrix,labels)
rel_input=tf.concat([lstm_output,label_embedding],axis=2)
else:
rel_input=lstm_output
#关系抽取分数
relScore=self._get_head_selection_score(rel_input,dropout_keep_in_prob=dropout_fcl_rel_keep)
#交叉熵计算损失函数
lossRel=tf.reduce_mean(tf.nn.sigmoid_cross_entropy_with_logits(logits=relScore,labels=scoreMatrix))
#预测关系
probas=tf.nn.sigmoid(relScore)
predRel=tf.round(probas) #四舍五入
return lossNER,lossRel,predNERS,predRel,relScore
def _build_model(self):
self.X = tf.placeholder(tf.float32, [None, self.input_dim])
self.y = tf.placeholder(tf.float32, [None, self.label_count])
self.domain = tf.placeholder(tf.float32, [None, 2])
self.l = tf.placeholder(tf.float32, [])
self.train = tf.placeholder(tf.bool, [])
batch_size = self.batch_size
X_input = self.X
# X_input = (tf.cast(self.X, tf.float32) - pixel_mean) / 255.
# CNN model for feature extraction
with tf.variable_scope('feature_extractor'):
input = X_input
for layer in self.encoder_h_layers:
input = self._add_layer(input, layer)
# The domain-invariant feature
self.feature = input
# MLP for class prediction
with tf.variable_scope('label_predictor'):
# Switches to route target examples (second half of batch) differently
# depending on train or test mode.
all_features = lambda: self.feature
source_features = lambda: tf.slice(self.feature, [0, 0],
[batch_size // 2, -1])
classify_feats = tf.cond(self.train, source_features, all_features)
all_labels = lambda: self.y
source_labels = lambda: tf.slice(self.y, [0, 0],
[batch_size // 2, -1])
self.classify_labels = tf.cond(self.train, source_labels,
all_labels)
print("tf.shape(all_labels)=", tf.shape(all_labels()))
print("tf.shape(source_labels)=", tf.shape(source_labels()))
#tf.shape(all_labels)= Tensor("label_predictor/Shape:0", shape=(2,), dtype=int32)
#tf.shape(source_labels)= Tensor("label_predictor/Shape_1:0", shape=(2,), dtype=int32)
input = classify_feats
for layer in self.clf_h_layers:
input = self._add_layer(input, layer)
#h_fc0 = self._add_layer(classify_feats, 128)
h_fc1 = self._add_layer(input, self.label_count, None)
logits = h_fc1
self.pred = tf.nn.softmax(logits)
self.pred_loss = tf.nn.softmax_cross_entropy_with_logits(
logits=logits, labels=self.classify_labels)
# Small MLP for domain prediction with adversarial loss
with tf.variable_scope('domain_predictor'):
# Flip the gradient when backpropagating through this operation
feat = flip_gradient(self.feature, self.l)
#feat = self.feature
input = feat
for layer in self.da_h_layers:
input = self._add_layer(input, layer)
#d_h_fc0 = self._add_layer(feat, 32)
d_b_fc1 = self._add_layer(input, 2, None)
d_logits = d_b_fc1
self.domain_pred = tf.nn.softmax(d_logits)
self.domain_loss = tf.nn.softmax_cross_entropy_with_logits(
logits=d_logits, labels=self.domain)
def add_model(self):
#nn_in = self.input_x
with tf.name_scope("embedding"):
self.ent_embeddings = tf.get_variable(name = "ent_embedding", shape = [self.max_ent, self.p.inp_dim], initializer = tf.contrib.layers.xavier_initializer(uniform = False), regularizer=self.regularizer)
self.rel_embeddings = tf.get_variable(name = "rel_embedding", shape = [self.max_rel, self.p.inp_dim], initializer = tf.contrib.layers.xavier_initializer(uniform = False), regularizer=self.regularizer)
self.time_embeddings = tf.get_variable(name = "time_embedding",shape = [self.max_time, self.p.inp_dim], initializer = tf.contrib.layers.xavier_initializer(uniform =False))
transE_in_dim = self.p.inp_dim
transE_in = self.ent_embeddings
####################------------------------ time aware GCN MODEL ---------------------------##############
## Some transE style model ####
neutral = tf.constant(0) ## mode = 1 for train mode = -1 test
test_type = tf.constant(0) ## pred_mode = 1 for head -1 for tail
query_type = tf.constant(0) ## query mode =1 for head tail , -1 for rel
def f_train():
pos_h_e = tf.squeeze(tf.nn.embedding_lookup(transE_in, self.pos_head))
pos_t_e = tf.squeeze(tf.nn.embedding_lookup(transE_in, self.pos_tail))
pos_r_e = tf.squeeze(tf.nn.embedding_lookup(self.rel_embeddings, self.rel))
return pos_h_e, pos_t_e, pos_r_e
def f_test():
def head_tail_query():
def f_head():
e2 = tf.squeeze(tf.nn.embedding_lookup(transE_in, self.pos_tail))
pos_h_e = transE_in
pos_t_e = tf.reshape(tf.tile(e2,[self.max_ent]),(self.max_ent, transE_in_dim))
return pos_h_e, pos_t_e
def f_tail():
e1 = tf.squeeze(tf.nn.embedding_lookup(transE_in, self.pos_head))
pos_h_e = tf.reshape(tf.tile(e1,[self.max_ent]),(self.max_ent, transE_in_dim))
pos_t_e = transE_in
return pos_h_e, pos_t_e
pos_h_e, pos_t_e = tf.cond(self.pred_mode > test_type, f_head, f_tail)
r = tf.squeeze(tf.nn.embedding_lookup(self.rel_embeddings,self.rel))
pos_r_e = tf.reshape(tf.tile(r,[self.max_ent]),(self.max_ent,transE_in_dim))
return pos_h_e, pos_t_e, pos_r_e
def rel_query():
e1 = tf.squeeze(tf.nn.embedding_lookup(transE_in, self.pos_head))
e2 = tf.squeeze(tf.nn.embedding_lookup(transE_in, self.pos_tail))
pos_h_e = tf.reshape(tf.tile(e1,[self.max_rel]),(self.max_rel, transE_in_dim))
pos_t_e = tf.reshape(tf.tile(e2,[self.max_rel]),(self.max_rel, transE_in_dim))
pos_r_e = self.rel_embeddings
return pos_h_e, pos_t_e, pos_r_e
pos_h_e, pos_t_e, pos_r_e = tf.cond(self.query_mode > query_type, head_tail_query, rel_query)
return pos_h_e, pos_t_e, pos_r_e
pos_h_e, pos_t_e, pos_r_e = tf.cond(self.mode > neutral, f_train, f_test)
neg_h_e = tf.squeeze(tf.nn.embedding_lookup(transE_in, self.neg_head))
neg_t_e = tf.squeeze(tf.nn.embedding_lookup(transE_in, self.neg_tail))
#### ----- time -----###
t_1 = tf.squeeze(tf.nn.embedding_lookup(self.time_embeddings, self.start_year))
pos_h_e_t_1 = self.time_projection(pos_h_e,t_1)
neg_h_e_t_1 = self.time_projection(neg_h_e,t_1)
pos_t_e_t_1 = self.time_projection(pos_t_e,t_1)
neg_t_e_t_1 = self.time_projection(neg_t_e,t_1)
pos_r_e_t_1 = self.time_projection(pos_r_e,t_1)
# pos_r_e_t_1 = pos_r_e
if self.p.L1_flag:
pos = tf.reduce_sum(abs(pos_h_e_t_1 + pos_r_e_t_1 - pos_t_e_t_1), 1, keep_dims = True)
neg = tf.reduce_sum(abs(neg_h_e_t_1 + pos_r_e_t_1 - neg_t_e_t_1), 1, keep_dims = True)
#self.predict = pos
else:
pos = tf.reduce_sum((pos_h_e_t_1 + pos_r_e_t_1 - pos_t_e_t_1) ** 2, 1, keep_dims = True)
neg = tf.reduce_sum((neg_h_e_t_1 + pos_r_e_t_1 - neg_t_e_t_1) ** 2, 1, keep_dims = True)
#self.predict = pos
'''
debug_nn([self.pred_mode,self.mode], feed_dict = self.create_feed_dict(self.data[0:self.p.batch_size],dtype='test'))
'''
return pos, neg
def testNeuralLinUCBUpdateNumTrainSteps0(self,
batch_size=1,
context_dim=10):
"""Check NeuralLinUCBAgent updates when behaving like LinUCB."""
# Construct a `Trajectory` for the given action, observation, reward.
num_actions = 5
initial_step, final_step = _get_initial_and_final_steps(
batch_size, context_dim)
action = np.random.randint(num_actions,
size=batch_size,
dtype=np.int32)
action_step = _get_action_step(action)
experience = _get_experience(initial_step, action_step, final_step)
# Construct an agent and perform the update.
observation_spec = tensor_spec.TensorSpec([context_dim], tf.float32)
time_step_spec = time_step.time_step_spec(observation_spec)
action_spec = tensor_spec.BoundedTensorSpec(dtype=tf.int32,
shape=(),
minimum=0,
maximum=num_actions - 1)
encoder = DummyNet(obs_dim=context_dim)
encoding_dim = 10
agent = neural_linucb_agent.NeuralLinUCBAgent(
time_step_spec=time_step_spec,
action_spec=action_spec,
encoding_network=encoder,
encoding_network_num_train_steps=0,
encoding_dim=encoding_dim,
optimizer=tf.compat.v1.train.AdamOptimizer(learning_rate=1e-2))
loss_info = agent.train(experience)
self.evaluate(agent.initialize())
self.evaluate(tf.compat.v1.global_variables_initializer())
self.evaluate(loss_info)
final_a = self.evaluate(agent.cov_matrix)
final_b = self.evaluate(agent.data_vector)
# Compute the expected updated estimates.
observations_list = tf.dynamic_partition(
data=tf.reshape(tf.cast(experience.observation, tf.float64),
[batch_size, context_dim]),
partitions=tf.convert_to_tensor(action),
num_partitions=num_actions)
rewards_list = tf.dynamic_partition(
data=tf.reshape(tf.cast(experience.reward, tf.float64),
[batch_size]),
partitions=tf.convert_to_tensor(action),
num_partitions=num_actions)
expected_a_updated_list = []
expected_b_updated_list = []
for _, (observations_for_arm, rewards_for_arm) in enumerate(
zip(observations_list, rewards_list)):
encoded_observations_for_arm, _ = encoder(observations_for_arm)
encoded_observations_for_arm = tf.cast(
encoded_observations_for_arm, dtype=tf.float64)
num_samples_for_arm_current = tf.cast(
tf.shape(rewards_for_arm)[0], tf.float64)
num_samples_for_arm_total = num_samples_for_arm_current
# pylint: disable=cell-var-from-loop
def true_fn():
a_new = tf.matmul(encoded_observations_for_arm,
encoded_observations_for_arm,
transpose_a=True)
b_new = bandit_utils.sum_reward_weighted_observations(
rewards_for_arm, encoded_observations_for_arm)
return a_new, b_new
def false_fn():
return (tf.zeros([encoding_dim, encoding_dim],
dtype=tf.float64),
tf.zeros([encoding_dim], dtype=tf.float64))
a_new, b_new = tf.cond(
tf.squeeze(num_samples_for_arm_total) > 0, true_fn, false_fn)
expected_a_updated_list.append(self.evaluate(a_new))
expected_b_updated_list.append(self.evaluate(b_new))
# Check that the actual updated estimates match the expectations.
self.assertAllClose(expected_a_updated_list, final_a)
self.assertAllClose(expected_b_updated_list, final_b)
def _parse_line(line):
input_data = dict.fromkeys(input_keys)
line_split = tf.strings.split([line], '|').values
# process uid
input_data['uid'] = tf.strings.to_number(line_split[data_map['uid']],
out_type=tf.int64)
def _string_to_sparse(string, dense_shape):
number = tf.strings.to_number(tf.strings.split([string],
sep=',').values,
out_type=tf.int64)
return tf.sparse.SparseTensor(indices=tf.expand_dims(number,
axis=1),
values=tf.ones_like(number,
dtype=tf.int8),
dense_shape=dense_shape)
# process image
input_data['image'] = tf.sparse.to_dense(
tf.cond(
tf.strings.length(line_split[data_map['image']]) > 0,
true_fn=lambda: _string_to_sparse(
line_split[data_map['image']], dense_shape=[num_softid]),
false_fn=lambda: tf.sparse.SparseTensor(
indices=np.empty((0, 1), dtype=np.int64),
values=tf.constant([], dtype=tf.int8),
dense_shape=[num_softid])))
# process label
label = tf.sparse.to_dense(
tf.cond(
tf.strings.length(line_split[data_map['label']]) > 0,
true_fn=lambda: _string_to_sparse(
line_split[data_map['label']], dense_shape=[num_label]),
false_fn=lambda: tf.sparse.SparseTensor(
indices=np.empty((0, 1), dtype=np.int64),
values=tf.constant([], dtype=tf.int8),
dense_shape=[num_label])))
# process lists
def _string_to_list(string, padding):
number = tf.strings.to_number(tf.strings.split([string],
sep=',').values,
out_type=tf.int32)
return tf.concat([
tf.tile(tf.constant([padding], dtype=tf.int32),
tf.math.maximum(length_his - tf.shape(number), 0)),
number[-length_his:]
],
axis=0)
input_data['new_time_list'] = tf.cond(
tf.strings.length(line_split[data_map['new_time_list']]) > 0,
true_fn=lambda: _string_to_list(
line_split[data_map['new_time_list']], padding=-7) // 7,
false_fn=lambda: tf.tile(tf.constant([-1], dtype=tf.int32),
[length_his]))
input_data['new_soft_list'] = tf.cond(
tf.strings.length(line_split[data_map['new_soft_list']]) > 0,
true_fn=lambda: _string_to_list(
line_split[data_map['new_soft_list']], padding=-1),
false_fn=lambda: tf.tile(tf.constant([-1], dtype=tf.int32),
[length_his]))
input_data['new_mask_list'] = tf.to_int32(
tf.greater_equal(
input_data['new_time_list'],
tf.tile(tf.constant([0], dtype=tf.int32), [length_his])))
input_data['new_bert_mask_index'] = tf.reverse(tf.math.top_k(
tf.random.shuffle(
tf.constant(np.asarray(range(length_his)),
dtype=tf.int8))[:bert_mask_num],
k=bert_mask_num).values,
axis=[0])
input_data['new_bert_mask'] = tf.sparse.to_dense(
tf.sparse.SparseTensor(indices=tf.expand_dims(tf.to_int64(
input_data['new_bert_mask_index']),
axis=1),
values=tf.ones_like(
input_data['new_bert_mask_index'],
dtype=tf.int8),
dense_shape=[length_his]))
input_data['loss_time_list'] = tf.cond(
tf.strings.length(line_split[data_map['loss_time_list']]) > 0,
true_fn=lambda: _string_to_list(
line_split[data_map['loss_time_list']], padding=-7) // 7,
false_fn=lambda: tf.tile(tf.constant([-1], dtype=tf.int32),
[length_his]))
input_data['loss_soft_list'] = tf.cond(
tf.strings.length(line_split[data_map['loss_soft_list']]) > 0,
true_fn=lambda: _string_to_list(
line_split[data_map['loss_soft_list']], padding=-1),
false_fn=lambda: tf.tile(tf.constant([-1], dtype=tf.int32),
[length_his]))
input_data['loss_mask_list'] = tf.to_int32(
tf.greater_equal(
input_data['loss_time_list'],
tf.tile(tf.constant([0], dtype=tf.int32), [length_his])))
input_data['loss_bert_mask_index'] = tf.reverse(tf.math.top_k(
tf.random.shuffle(
tf.constant(np.asarray(range(length_his)),
dtype=tf.int8))[:bert_mask_num],
k=bert_mask_num).values,
axis=[0])
input_data['loss_bert_mask'] = tf.sparse.to_dense(
tf.sparse.SparseTensor(indices=tf.expand_dims(tf.to_int64(
input_data['loss_bert_mask_index']),
axis=1),
values=tf.ones_like(
input_data['loss_bert_mask_index'],
dtype=tf.int8),
dense_shape=[length_his]))
return input_data, label
def augment_rotate(self, image_a, image_b):
r = tf.unstack(tf.random_uniform([1], minval=0, maxval=2, dtype=tf.int32, seed=None, name=None))
rotate_boolean = tf.equal(0, r, name="check-rotate-boolean")
[image_a, image_b] = tf.cond(rotate_boolean[0], lambda: self.rotate_data(image_a, image_b),
lambda: [image_a, image_b])
return image_a, image_b
def build_model(self):
with tf.variable_scope('Model', reuse=tf.AUTO_REUSE):
# Model Feeds
self.ratings = tf.placeholder(dtype=tf.float32,
shape=[None, self.num_item],
name='ratings')
self.output_mask = tf.placeholder(dtype=tf.bool,
shape=[None, self.num_item],
name='output_mask')
self.istraining = tf.placeholder(dtype=tf.bool,
shape=[],
name='training_flag')
self.isnegsample = tf.placeholder(dtype=tf.bool,
shape=[],
name='negative_sample_flag')
self.add_hidden_noise = tf.placeholder(dtype=tf.bool,
shape=[],
name='add_hidden_noise')
self.add_weight_noise = tf.placeholder(dtype=tf.bool,
shape=[],
name='add_weight_noise')
self.layer1_dropout_rate = tf.placeholder(
dtype=tf.float32, shape=[], name='layer1_dropout_rate')
input = self.ratings
# Encoder Variables
layer1_w = tf.get_variable(
name='encoder_weights',
shape=[self.num_item, self.num_factors],
initializer=tf.truncated_normal_initializer(mean=0.0,
stddev=0.03))
layer1_b = tf.get_variable(name='encoder_bias',
shape=[self.num_factors],
initializer=tf.zeros_initializer())
# user_node = tf.get_variable(name='user_nodes',
# shape=[self.num_factors],
# initializer=tf.truncated_normal_initializer(mean=0.0, stddev=0.03)
# )
w2_noise = tf.get_variable(name='w2_noise',
shape=[self.num_factors, self.num_item],
initializer=tf.zeros_initializer(),
dtype=tf.float32,
trainable=False)
hidden_noise_tr = tf.get_variable(
name='hidden_noise_tr',
shape=[self.batch_size, self.num_factors],
initializer=tf.zeros_initializer(),
dtype=tf.float32,
trainable=False)
# Decoder Variables
layer2_w = tf.get_variable(
name='decoder_weights',
shape=[self.num_factors, self.num_item],
initializer=tf.truncated_normal_initializer(mean=0.0,
stddev=0.03))
layer2_b = tf.get_variable(name='decoder_bias',
shape=[self.num_item],
initializer=tf.zeros_initializer())
# Original AE Model
org_encoder = tf.sigmoid(tf.matmul(input, layer1_w) + layer1_b)
org_encoder = tf.cond(
self.istraining,
lambda: tf.layers.dropout(org_encoder,
rate=self.layer1_dropout_rate,
name='layer1_dropout'),
lambda: org_encoder)
if self.robust_test: # Robustness Testing on W2
org_decoder = tf.identity(
tf.matmul(org_encoder, layer2_w + w2_noise) + layer2_b)
self.w2 = layer2_w
self.w2_org = layer2_w - w2_noise
else:
org_decoder = tf.identity(
tf.matmul(org_encoder, layer2_w) + layer2_b)
mask = tf.cond(
self.isnegsample,
lambda: tf.cast(self.output_mask, dtype=org_decoder.dtype),
lambda: tf.sign(self.ratings))
self.org_output = tf.cond(self.istraining,
lambda: tf.multiply(org_decoder, mask),
lambda: org_decoder)
org_base_loss = tf.reduce_sum(
tf.nn.sigmoid_cross_entropy_with_logits(
labels=input, logits=self.org_output))
org_base_loss = org_base_loss / tf.cast(tf.shape(input)[0],
dtype=org_base_loss.dtype)
# org_reg_loss = self.ae_regs[0] * tf.nn.l2_loss(layer1_w) + self.ae_regs[1] * tf.nn.l2_loss(layer1_b) + \
# self.ae_regs[2] * tf.nn.l2_loss(layer2_w2) + self.ae_regs[3] * tf.nn.l2_loss(layer2_b)
# org_loss = org_base_loss + org_reg_loss
# Noisy Model
if not self.robust_test:
noisy_encoder = tf.sigmoid(
tf.matmul(input, layer1_w) + layer1_b)
noisy_encoder = tf.cond(
self.istraining, lambda: noisy_encoder + hidden_noise_tr,
lambda: noisy_encoder)
# layer1 = tf.cond(self.istraining,
# lambda : tf.sigmoid(tf.matmul(input, layer1_w + layer1_delta) + layer1_b),
# lambda : tf.sigmoid(tf.matmul(input, layer1_w) + layer1_b))
noisy_encoder = tf.cond(
self.istraining,
lambda: tf.layers.dropout(noisy_encoder,
rate=self.layer1_dropout_rate,
name='layer1_dropout'),
lambda: noisy_encoder)
self.w2 = layer2_w
self.w2_org = tf.cond(self.add_weight_noise,
lambda: layer2_w - w2_noise,
lambda: layer2_w)
noisy_decoder = tf.identity(
tf.matmul(noisy_encoder, layer2_w) + layer2_b)
# Output
# Determine whether negative samples should be considered
mask = tf.cond(
self.isnegsample, lambda: tf.cast(
self.output_mask, dtype=noisy_decoder.dtype),
lambda: tf.sign(self.ratings))
self.noisy_output = tf.cond(
self.istraining, lambda: tf.multiply(noisy_decoder, mask),
lambda: noisy_decoder)
# self.pred_y = tf.sigmoid(self.output)
# self.pred_y = self.output
# Noisy Model Loss
# base_loss = tf.nn.l2_loss(input - self.pred_y)
noisy_base_loss = tf.reduce_sum(
tf.nn.sigmoid_cross_entropy_with_logits(
labels=input, logits=self.noisy_output))
noisy_base_loss = noisy_base_loss / tf.cast(
tf.shape(input)[0],
dtype=noisy_base_loss.dtype) # Average over the batches
################## Mix the outputs
# self.mixed_output = (1-self.loss_ratio) * self.org_output + self.loss_ratio * self.output
self.pred_y = tf.sigmoid(self.org_output)
# self.pred_y = self.mixed_output
# Mix the losses
# base_loss = tf.cond(tf.logical_or(self.add_concat_noise, self.add_weight_noise),
# lambda : org_base_loss + self.loss_ratio * base_loss,
# lambda : base_loss)
if not self.robust_test:
base_loss = org_base_loss + self.loss_ratio * noisy_base_loss
else:
base_loss = org_base_loss
reg_loss = self.ae_regs[0] * tf.nn.l2_loss(layer1_w) + self.ae_regs[1] * tf.nn.l2_loss(layer1_b) + \
self.ae_regs[2] * tf.nn.l2_loss(layer2_w) + self.ae_regs[3] * tf.nn.l2_loss(layer2_b)
self.loss = base_loss + reg_loss
# Optimizer
self.opt = tf.train.AdagradOptimizer(self.lr).minimize(self.loss)
# Add Noise (Random or Adversial)
if self.robust_test: # Robust Testing in Evaluation
if self.random_noise:
random_noise = tf.random_normal(
shape=tf.shape(layer2_w),
mean=tf.reduce_mean(layer2_w),
stddev=0.01)
self.update_delta = w2_noise.assign(
self.eps * random_noise / tf.norm(random_noise))
if self.adv_noise:
self.grad_delta = tf.gradients(ys=org_base_loss,
xs=w2_noise)[0]
self.grad_delta_dense = tf.stop_gradient(self.grad_delta)
self.update_delta = w2_noise.assign(
self.eps * self.grad_delta_dense /
tf.norm(self.grad_delta_dense))
else:
# Gradients Computation of the Noise in Training
self.grad_delta = tf.gradients(ys=base_loss,
xs=hidden_noise_tr)[0]
# convert the IndexedSlice Data to Dense Tensor
self.grad_delta_dense = tf.stop_gradient(self.grad_delta)
# self.grad_shape = tf.shape(self.grad_delta_dense)
# normalization: new_grad = (grad / |grad|) * eps
self.update_delta = hidden_noise_tr.assign(
self.eps * self.grad_delta_dense /
tf.norm(self.grad_delta_dense))
print('Model Building Completed.')
def parse(self, serialized):
"""See `_RankingDataParser`."""
list_size = self._list_size
context_feature_spec = self._context_feature_spec
example_feature_spec = self._example_feature_spec
# Convert `FixedLenFeature` in `example_feature_spec` to
# `FixedLenSequenceFeature` to parse the `feature_lists` in SequenceExample.
# In addition, we collect non-trivial `default_value`s (neither "" nor 0)
# for post-processing. This is because no `default_value` except None is
# allowed for `FixedLenSequenceFeature`. Also, we set allow_missing=True and
# handle the missing feature_list later.
fixed_len_sequence_features = {}
padding_values = {}
non_trivial_padding_values = {}
for k, s in six.iteritems(example_feature_spec):
if not isinstance(s, tf.io.FixedLenFeature):
continue
fixed_len_sequence_features[k] = tf.io.FixedLenSequenceFeature(
s.shape, s.dtype, allow_missing=True)
scalar = _get_scalar_default_value(s.dtype, s.default_value)
padding_values[k] = scalar
if scalar:
non_trivial_padding_values[k] = scalar
sequence_features = example_feature_spec.copy()
sequence_features.update(fixed_len_sequence_features)
context, examples, sizes = tf.io.parse_sequence_example(
serialized,
context_features=context_feature_spec,
sequence_features=sequence_features)
# Reset to no trivial padding values for example features.
for k, v in six.iteritems(non_trivial_padding_values):
tensor = examples[k] # [batch_size, num_frames, feature_size]
tensor.get_shape().assert_has_rank(3)
size = tf.reshape(sizes[k], [-1, 1, 1]) # [batch_size, 1, 1]
rank = tf.reshape(
tf.tile(tf.range(tf.shape(input=tensor)[1]),
[tf.shape(input=tensor)[0]]), tf.shape(input=tensor))
tensor = tf.compat.v1.where(
tf.less(rank, tf.cast(size, tf.int32)), tensor,
tf.fill(tf.shape(input=tensor), tf.cast(v, tensor.dtype)))
examples[k] = tensor
list_size_arg = list_size
if list_size is None:
# Use dynamic list_size. This is needed to pad missing feature_list.
list_size_dynamic = tf.reduce_max(input_tensor=tf.stack(
[tf.shape(input=t)[1] for t in six.itervalues(examples)]))
list_size = list_size_dynamic
# Collect features. Truncate or pad example features to normalize the tensor
# shape: [batch_size, num_frames, ...] --> [batch_size, list_size, ...]
features = {}
features.update(context)
for k, t in six.iteritems(examples):
# Old shape: [batch_size, num_frames, ...]
shape = tf.shape(input=t)
ndims = t.get_shape().rank
num_frames = shape[1]
# New shape: [batch_size, list_size, ...]
new_shape = tf.concat([[shape[0], list_size], shape[2:]], 0)
def truncate_fn(t=t, ndims=ndims, new_shape=new_shape):
"""Truncates the tensor."""
if isinstance(t, tf.sparse.SparseTensor):
return tf.sparse.slice(t, [0] * ndims,
tf.cast(new_shape, dtype=tf.int64))
else:
return tf.slice(t, [0] * ndims, new_shape)
def pad_fn(k=k,
t=t,
ndims=ndims,
num_frames=num_frames,
new_shape=new_shape):
"""Pads the tensor."""
if isinstance(t, tf.sparse.SparseTensor):
return tf.sparse.reset_shape(t, new_shape)
else:
# Paddings has shape [n, 2] where n is the rank of the tensor.
paddings = tf.stack([[0, 0], [0, list_size - num_frames]] +
[[0, 0]] * (ndims - 2))
return tf.pad(tensor=t,
paddings=paddings,
constant_values=padding_values[k])
tensor = tf.cond(pred=num_frames > list_size,
true_fn=truncate_fn,
false_fn=pad_fn)
# Infer static shape for Tensor. Set the 2nd dim to None and set_shape
# merges `static_shape` with the existing static shape of the thensor.
if not isinstance(tensor, tf.sparse.SparseTensor):
static_shape = t.get_shape().as_list()
static_shape[1] = list_size_arg
tensor.set_shape(static_shape)
features[k] = tensor
return features
def slim_get_batch(num_classes, batch_size, split_name, file_pattern, num_readers, num_preprocessing_threads, image_preprocessing_fn, anchor_encoder, num_epochs=None, is_training=True):
"""Gets a dataset tuple with instructions for reading Pascal VOC dataset.
Args:
num_classes: total class numbers in dataset.
batch_size: the size of each batch.
split_name: 'train' of 'val'.
file_pattern: The file pattern to use when matching the dataset sources (full path).
num_readers: the max number of reader used for reading tfrecords.
num_preprocessing_threads: the max number of threads used to run preprocessing function.
image_preprocessing_fn: the function used to dataset augumentation.
anchor_encoder: the function used to encoder all anchors.
num_epochs: total epoches for iterate this dataset.
is_training: whether we are in traing phase.
Returns:
A batch of [image, shape, loc_targets, cls_targets, match_scores].
"""
if split_name not in data_splits_num:
raise ValueError('split name %s was not recognized.' % split_name)
# Features in Pascal VOC TFRecords.
keys_to_features = {
'image/encoded': tf.FixedLenFeature((), tf.string, default_value=''),
'image/format': tf.FixedLenFeature((), tf.string, default_value='jpeg'),
'image/filename': tf.FixedLenFeature((), tf.string, default_value=''),
'image/height': tf.FixedLenFeature([1], tf.int64),
'image/width': tf.FixedLenFeature([1], tf.int64),
'image/channels': tf.FixedLenFeature([1], tf.int64),
'image/shape': tf.FixedLenFeature([3], tf.int64),
'image/object/bbox/xmin': tf.VarLenFeature(dtype=tf.float32),
'image/object/bbox/ymin': tf.VarLenFeature(dtype=tf.float32),
'image/object/bbox/xmax': tf.VarLenFeature(dtype=tf.float32),
'image/object/bbox/ymax': tf.VarLenFeature(dtype=tf.float32),
'image/object/bbox/label': tf.VarLenFeature(dtype=tf.int64),
'image/object/bbox/difficult': tf.VarLenFeature(dtype=tf.int64),
'image/object/bbox/truncated': tf.VarLenFeature(dtype=tf.int64),
}
items_to_handlers = {
'image': slim.tfexample_decoder.Image('image/encoded', 'image/format'),
'filename': slim.tfexample_decoder.Tensor('image/filename'),
'shape': slim.tfexample_decoder.Tensor('image/shape'),
'object/bbox': slim.tfexample_decoder.BoundingBox(
['ymin', 'xmin', 'ymax', 'xmax'], 'image/object/bbox/'),
'object/label': slim.tfexample_decoder.Tensor('image/object/bbox/label'),
'object/difficult': slim.tfexample_decoder.Tensor('image/object/bbox/difficult'),
'object/truncated': slim.tfexample_decoder.Tensor('image/object/bbox/truncated'),
}
decoder = slim.tfexample_decoder.TFExampleDecoder(keys_to_features, items_to_handlers)
labels_to_names = {}
for name, pair in VOC_LABELS.items():
labels_to_names[pair[0]] = name
dataset = slim.dataset.Dataset(
data_sources=file_pattern,
reader=tf.TFRecordReader,
decoder=decoder,
num_samples=data_splits_num[split_name],
items_to_descriptions=None,
num_classes=num_classes,
labels_to_names=labels_to_names)
with tf.name_scope('dataset_data_provider'):
provider = slim.dataset_data_provider.DatasetDataProvider(
dataset,
num_readers=num_readers,
common_queue_capacity=32 * batch_size,
common_queue_min=8 * batch_size,
shuffle=is_training,
num_epochs=num_epochs)
[org_image, filename, shape, glabels_raw, gbboxes_raw, isdifficult] = provider.get(['image', 'filename', 'shape',
'object/label',
'object/bbox',
'object/difficult'])
if is_training:
# if all is difficult, then keep the first one
isdifficult_mask =tf.cond(tf.count_nonzero(isdifficult, dtype=tf.int32) < tf.shape(isdifficult)[0],
lambda : isdifficult < tf.ones_like(isdifficult),
lambda : tf.one_hot(0, tf.shape(isdifficult)[0], on_value=True, off_value=False, dtype=tf.bool))
glabels_raw = tf.boolean_mask(glabels_raw, isdifficult_mask)
gbboxes_raw = tf.boolean_mask(gbboxes_raw, isdifficult_mask)
# Pre-processing image, labels and bboxes.
tensors_to_batch = []
if is_training:
image, glabels, gbboxes = image_preprocessing_fn(org_image, glabels_raw, gbboxes_raw)
gt_targets, gt_labels, gt_scores = anchor_encoder(glabels, gbboxes)
tensors_to_batch = [image, filename, shape, gt_targets, gt_labels, gt_scores]
else:
image, output_shape = image_preprocessing_fn(org_image, glabels_raw, gbboxes_raw)
tensors_to_batch = [image, filename, shape, output_shape]
return tf.train.batch(tensors_to_batch,
dynamic_pad=(not is_training),
batch_size=batch_size,
allow_smaller_final_batch=(not is_training),
num_threads=num_preprocessing_threads,
capacity=64 * batch_size)
def cond(*args, **kwargs):
return tf.cond(*args, **kwargs)
def _inference(self):
with tf.variable_scope(self._name):
# Input module
# Ax_ij shape is (batch_size, sentence_size ,embedding_size)
query_word_emb = tf.nn.embedding_lookup(self._que_emb,
self._queries)
# Global question embedding. When the languages
ques_emb = tf.reduce_sum(
query_word_emb, 1) # shape is (batch_size, embedding_size)
loss = tf.reshape(self._zeros, [-1, 1],
name='loss') # (batch_size, 1)
# The first state (entity) is given.
init_state_index = tf.reshape(self._paths[:, 0],
[-1, 1]) # (batch_size, 1)
# KG1 is the first KG to be used in the path, KG2 the other.
# Only in answer module do we cross language for state embedding.
state_emb = tf.nn.embedding_lookup(
self._kg1_ent_emb, init_state_index) # (b,1)->(b,1,e)
state_emb = tf.squeeze(state_emb,
[1]) # (batch_size, embedding_size)
dir_align = tf.constant(self._is_direct_align, dtype=tf.bool)
# e_M21 = tf.matrix_inverse(self._e_M12)
# Reasoning module
path = pre_t_idx = init_state_index
for hop in range(self._hops):
step = self._steps[hop]
py_is_1 = self._lan_labels[hop] == self._lan_labels[0]
is_1 = tf.constant(py_is_1, dtype=tf.bool)
py_is_cross = hop != 0 and (self._lan_labels[hop] !=
self._lan_labels[hop - 1])
is_cross = tf.constant(py_is_cross, dtype=tf.bool)
# Cross-lingual processing
# head_emb = tf.cond(is_1,
# lambda: tf.matmul(tf.nn.embedding_lookup(self._kg2_ent_emb,
# tf.cast(pre_t_idx, tf.int64)),
# tf.cond(tf.constant(self._is_dual_matrices),
# lambda: self._e_M21,
# lambda: e_M21)),
# lambda: tf.matmul(tf.nn.embedding_lookup(self._kg1_ent_emb,
# tf.cast(pre_t_idx, tf.int64)), self._e_M12))
# head_logits = tf.cond(is_1,
# lambda: tf.matmul(head_emb, self._kg1_ent_emb, transpose_b=True),
# lambda: tf.matmul(head_emb, self._kg2_ent_emb, transpose_b=True))
if system() == "Windows":
# Windows compatible
if self._is_direct_align:
head_index = tf.cond(
is_1, lambda: self._cor_a_table_21.lookup(
tf.cast(pre_t_idx, tf.int32)),
lambda: self._cor_a_table_12.lookup(
tf.cast(pre_t_idx, tf.int32)))
else:
head_index = tf.cond(
is_1, lambda: self._pred_a_table_21.lookup(
tf.cast(pre_t_idx, tf.int32)),
lambda: self._pred_a_table_12.lookup(
tf.cast(pre_t_idx, tf.int32)))
else:
# Linux compatible
if self._is_direct_align:
head_index = tf.cond(
is_1, lambda: self._cor_a_table_21.lookup(
tf.cast(pre_t_idx, tf.int64)),
lambda: self._cor_a_table_12.lookup(
tf.cast(pre_t_idx, tf.int64)))
else:
head_index = tf.cond(
is_1, lambda: self._pred_a_table_21.lookup(
tf.cast(pre_t_idx, tf.int64)),
lambda: self._pred_a_table_12.lookup(
tf.cast(pre_t_idx, tf.int64)))
path = tf.cond(
is_cross, lambda: tf.concat(
axis=1,
values=[
path,
tf.reshape(tf.cast(head_index, tf.int32), [-1, 1])
]), lambda: path)
state_emb = tf.cond(
is_cross, lambda: tf.cond(
is_1, lambda: tf.nn.embedding_lookup(
self._kg1_ent_emb, head_index), lambda: tf.nn.
embedding_lookup(self._kg2_ent_emb, head_index)),
lambda: state_emb)
gate = tf.matmul(ques_emb,
tf.cond(is_1,
lambda: tf.matmul(self._kg1_rel_emb, self._kg1_Mrq),
lambda: tf.matmul(self._kg2_rel_emb, self._kg2_Mrq)), transpose_b=True) \
+ tf.matmul(state_emb,
tf.cond(is_1,
lambda: tf.matmul(self._kg1_rel_emb, self._kg1_Mrs),
lambda: tf.matmul(self._kg2_rel_emb, self._kg2_Mrs)), transpose_b=True)
relation_logits = gate
relation_index = tf.argmax(relation_logits, 1)
gate = tf.nn.softmax(gate)
real_rel_one_hot = tf.cond(
is_cross, lambda: tf.one_hot(
self._paths[:, step + 2],
tf.cond(is_1, lambda: tf.constant(self._rel_size_1),
lambda: tf.constant(self._rel_size_2)),
on_value=1.0,
off_value=0.0,
axis=-1),
lambda: tf.one_hot(
self._paths[:, step + 1],
tf.cond(is_1, lambda: tf.constant(self._rel_size_1),
lambda: tf.constant(self._rel_size_2)),
on_value=1.0,
off_value=0.0,
axis=-1))
state_emb = state_emb + tf.matmul(
gate,
tf.cond(
is_1,
lambda: tf.matmul(self._kg1_rel_emb, self._kg1_Mrs),
lambda: tf.matmul(self._kg2_rel_emb, self._kg2_Mrs)))
loss += tf.reshape(
tf.nn.softmax_cross_entropy_with_logits(
logits=relation_logits, labels=real_rel_one_hot),
[-1, 1]) # (b,1)
ques_emb = ques_emb - tf.matmul(
gate,
tf.cond(
is_1,
lambda: tf.matmul(self._kg1_rel_emb, self._kg1_Mrq),
lambda: tf.matmul(self._kg2_rel_emb, self._kg2_Mrq)))
# Answer module
tail_logits = tf.cond(
is_1,
lambda: tf.matmul(tf.matmul(state_emb, self._kg1_Mse),
self._kg1_ent_emb,
transpose_b=True),
lambda: tf.matmul(tf.matmul(state_emb, self._kg2_Mse),
self._kg2_ent_emb,
transpose_b=True))
tail_index = tf.argmax(tail_logits, 1)
# # (batch_size, embedding_size)
# tail_index = tf.cast(tail_index, tf.float32)
# relation_index = tf.cast(relation_index, tf.float32)
# # if relation_index == 0, stop inference, tail_index = previous tail; if not, tail won't change
# tail_index = relation_index / (relation_index + 1e-15) * tail_index \
# + (1 - relation_index / (relation_index + 1e-15)) * tf.cast(path[:, -1], tf.float32)
path = tf.concat(axis=1,
values=[
path,
tf.reshape(
tf.cast(relation_index, tf.int32),
[-1, 1])
])
path = tf.concat(axis=1,
values=[
path,
tf.reshape(tf.cast(tail_index, tf.int32),
[-1, 1])
])
# (b,self._rel_size_1)
real_tail_one_hot = tf.cond(
is_cross, lambda: tf.one_hot(
self._paths[:, step + 3],
tf.cond(is_1, lambda: tf.constant(self._ent_size_1),
lambda: tf.constant(self._ent_size_2)),
on_value=1.0,
off_value=0.0,
axis=-1),
lambda: tf.one_hot(
self._paths[:, step + 2],
tf.cond(is_1, lambda: tf.constant(self._ent_size_1),
lambda: tf.constant(self._ent_size_2)),
on_value=1.0,
off_value=0.0,
axis=-1))
loss += tf.reshape(
tf.nn.softmax_cross_entropy_with_logits(
logits=tail_logits, labels=real_tail_one_hot),
[-1, 1]) # (b,1)
pre_t_idx = tail_index
return loss, path
def setup_to_run(m, args, is_training, batch_norm_is_training, summary_mode):
# Set up the model.
tf.set_random_seed(args.solver.seed)
task_params = args.navtask.task_params
num_steps = task_params.num_steps
num_goals = task_params.num_goals
num_actions = task_params.num_actions
num_actions_ = num_actions
n_views = task_params.n_views
batch_norm_is_training_op = \
tf.placeholder_with_default(batch_norm_is_training, shape=[],
name='batch_norm_is_training_op')
# Setup the inputs
m.input_tensors = {}
lstm_states = []; lstm_state_dims = [];
state_names = []; updated_state_ops = []; init_state_ops = [];
if args.arch.lstm_output:
lstm_states += ['lstm_output']
lstm_state_dims += [args.arch.lstm_output_dim+task_params.num_actions]
if args.arch.lstm_ego:
lstm_states += ['lstm_ego']
lstm_state_dims += [args.arch.lstm_ego_dim + args.arch.lstm_ego_out]
lstm_states += ['lstm_img']
lstm_state_dims += [args.arch.lstm_img_dim + args.arch.lstm_img_out]
elif args.arch.lstm_img:
# An LSTM only on the image
lstm_states += ['lstm_img']
lstm_state_dims += [args.arch.lstm_img_dim + args.arch.lstm_img_out]
else:
# No LSTMs involved here.
None
m.input_tensors['common'], m.input_tensors['step'], m.input_tensors['train'] = \
_inputs(task_params, lstm_states, lstm_state_dims)
with tf.name_scope('check_size'):
is_single_step = tf.equal(tf.unstack(tf.shape(m.input_tensors['step']['imgs']),
num=6)[1], 1)
images_reshaped = tf.reshape(m.input_tensors['step']['imgs'],
shape=[-1, task_params.img_height, task_params.img_width,
task_params.img_channels], name='re_image')
rel_goal_loc_reshaped = tf.reshape(m.input_tensors['step']['rel_goal_loc'],
shape=[-1, task_params.rel_goal_loc_dim], name='re_rel_goal_loc')
x, vars_ = get_repr_from_image(
images_reshaped, task_params.modalities, task_params.data_augment,
args.arch.encoder, args.solver.freeze_conv, args.solver.wt_decay,
is_training)
# Reshape into nice things so that these can be accumulated over time steps
# for faster backprop.
sh_before = x.get_shape().as_list()
m.encoder_output = tf.reshape(
x, shape=[task_params.batch_size, -1, n_views] + sh_before[1:])
x = tf.reshape(m.encoder_output, shape=[-1] + sh_before[1:])
# Add a layer to reduce dimensions for a fc layer.
if args.arch.dim_reduce_neurons > 0:
ks = 1; neurons = args.arch.dim_reduce_neurons;
init_var = np.sqrt(2.0/(ks**2)/neurons)
batch_norm_param = args.arch.batch_norm_param
batch_norm_param['is_training'] = batch_norm_is_training_op
m.conv_feat = slim.conv2d(
x, neurons, kernel_size=ks, stride=1, normalizer_fn=slim.batch_norm,
normalizer_params=batch_norm_param, padding='SAME', scope='dim_reduce',
weights_regularizer=slim.l2_regularizer(args.solver.wt_decay),
weights_initializer=tf.random_normal_initializer(stddev=init_var))
reshape_conv_feat = slim.flatten(m.conv_feat)
sh = reshape_conv_feat.get_shape().as_list()
m.reshape_conv_feat = tf.reshape(reshape_conv_feat,
shape=[-1, sh[1]*n_views])
# Restore these from a checkpoint.
if args.solver.pretrained_path is not None:
m.init_fn = slim.assign_from_checkpoint_fn(args.solver.pretrained_path,
vars_)
else:
m.init_fn = None
# Hit the goal_location with a bunch of fully connected layers, to embed it
# into some space.
with tf.variable_scope('embed_goal'):
batch_norm_param = args.arch.batch_norm_param
batch_norm_param['is_training'] = batch_norm_is_training_op
m.embed_goal, _ = tf_utils.fc_network(
rel_goal_loc_reshaped, neurons=args.arch.goal_embed_neurons,
wt_decay=args.solver.wt_decay, name='goal_embed', offset=0,
batch_norm_param=batch_norm_param, dropout_ratio=args.arch.fc_dropout,
is_training=is_training)
if args.arch.embed_goal_for_state:
with tf.variable_scope('embed_goal_for_state'):
batch_norm_param = args.arch.batch_norm_param
batch_norm_param['is_training'] = batch_norm_is_training_op
m.embed_goal_for_state, _ = tf_utils.fc_network(
m.input_tensors['common']['rel_goal_loc_at_start'][:,0,:],
neurons=args.arch.goal_embed_neurons, wt_decay=args.solver.wt_decay,
name='goal_embed', offset=0, batch_norm_param=batch_norm_param,
dropout_ratio=args.arch.fc_dropout, is_training=is_training)
# Hit the goal_location with a bunch of fully connected layers, to embed it
# into some space.
with tf.variable_scope('embed_img'):
batch_norm_param = args.arch.batch_norm_param
batch_norm_param['is_training'] = batch_norm_is_training_op
m.embed_img, _ = tf_utils.fc_network(
m.reshape_conv_feat, neurons=args.arch.img_embed_neurons,
wt_decay=args.solver.wt_decay, name='img_embed', offset=0,
batch_norm_param=batch_norm_param, dropout_ratio=args.arch.fc_dropout,
is_training=is_training)
# For lstm_ego, and lstm_image, embed the ego motion, accumulate it into an
# LSTM, combine with image features and accumulate those in an LSTM. Finally
# combine what you get from the image LSTM with the goal to output an action.
if args.arch.lstm_ego:
ego_reshaped = preprocess_egomotion(m.input_tensors['step']['incremental_locs'],
m.input_tensors['step']['incremental_thetas'])
with tf.variable_scope('embed_ego'):
batch_norm_param = args.arch.batch_norm_param
batch_norm_param['is_training'] = batch_norm_is_training_op
m.embed_ego, _ = tf_utils.fc_network(
ego_reshaped, neurons=args.arch.ego_embed_neurons,
wt_decay=args.solver.wt_decay, name='ego_embed', offset=0,
batch_norm_param=batch_norm_param, dropout_ratio=args.arch.fc_dropout,
is_training=is_training)
state_name, state_init_op, updated_state_op, out_op = lstm_setup(
'lstm_ego', m.embed_ego, task_params.batch_size, is_single_step,
args.arch.lstm_ego_dim, args.arch.lstm_ego_out, num_steps*num_goals,
m.input_tensors['step']['lstm_ego'])
state_names += [state_name]
init_state_ops += [state_init_op]
updated_state_ops += [updated_state_op]
# Combine the output with the vision features.
m.img_ego_op = combine_setup('img_ego', args.arch.combine_type_ego,
m.embed_img, out_op,
args.arch.img_embed_neurons[-1],
args.arch.lstm_ego_out)
# LSTM on these vision features.
state_name, state_init_op, updated_state_op, out_op = lstm_setup(
'lstm_img', m.img_ego_op, task_params.batch_size, is_single_step,
args.arch.lstm_img_dim, args.arch.lstm_img_out, num_steps*num_goals,
m.input_tensors['step']['lstm_img'])
state_names += [state_name]
init_state_ops += [state_init_op]
updated_state_ops += [updated_state_op]
m.img_for_goal = out_op
num_img_for_goal_neurons = args.arch.lstm_img_out
elif args.arch.lstm_img:
# LSTM on just the image features.
state_name, state_init_op, updated_state_op, out_op = lstm_setup(
'lstm_img', m.embed_img, task_params.batch_size, is_single_step,
args.arch.lstm_img_dim, args.arch.lstm_img_out, num_steps*num_goals,
m.input_tensors['step']['lstm_img'])
state_names += [state_name]
init_state_ops += [state_init_op]
updated_state_ops += [updated_state_op]
m.img_for_goal = out_op
num_img_for_goal_neurons = args.arch.lstm_img_out
else:
m.img_for_goal = m.embed_img
num_img_for_goal_neurons = args.arch.img_embed_neurons[-1]
if args.arch.use_visit_count:
m.embed_visit_count = visit_count_fc(
m.input_tensors['step']['visit_count'],
m.input_tensors['step']['last_visit'], args.arch.goal_embed_neurons,
args.solver.wt_decay, args.arch.fc_dropout, is_training=is_training)
m.embed_goal = m.embed_goal + m.embed_visit_count
m.combined_f = combine_setup('img_goal', args.arch.combine_type,
m.img_for_goal, m.embed_goal,
num_img_for_goal_neurons,
args.arch.goal_embed_neurons[-1])
# LSTM on the combined representation.
if args.arch.lstm_output:
name = 'lstm_output'
# A few fully connected layers here.
with tf.variable_scope('action_pred'):
batch_norm_param = args.arch.batch_norm_param
batch_norm_param['is_training'] = batch_norm_is_training_op
x, _ = tf_utils.fc_network(
m.combined_f, neurons=args.arch.pred_neurons,
wt_decay=args.solver.wt_decay, name='pred', offset=0,
batch_norm_param=batch_norm_param, dropout_ratio=args.arch.fc_dropout)
if args.arch.lstm_output_init_state_from_goal:
# Use the goal embedding to initialize the LSTM state.
# UGLY CLUGGY HACK: if this is doing computation for a single time step
# then this will not involve back prop, so we can use the state input from
# the feed dict, otherwise we compute the state representation from the
# goal and feed that in. Necessary for using goal location to generate the
# state representation.
m.embed_goal_for_state = tf.expand_dims(m.embed_goal_for_state, dim=1)
state_op = tf.cond(is_single_step, lambda: m.input_tensors['step'][name],
lambda: m.embed_goal_for_state)
state_name, state_init_op, updated_state_op, out_op = lstm_setup(
name, x, task_params.batch_size, is_single_step,
args.arch.lstm_output_dim,
num_actions_,
num_steps*num_goals, state_op)
init_state_ops += [m.embed_goal_for_state]
else:
state_op = m.input_tensors['step'][name]
state_name, state_init_op, updated_state_op, out_op = lstm_setup(
name, x, task_params.batch_size, is_single_step,
args.arch.lstm_output_dim,
num_actions_, num_steps*num_goals, state_op)
init_state_ops += [state_init_op]
state_names += [state_name]
updated_state_ops += [updated_state_op]
out_op = tf.reshape(out_op, shape=[-1, num_actions_])
if num_actions_ > num_actions:
m.action_logits_op = out_op[:,:num_actions]
m.baseline_op = out_op[:,num_actions:]
else:
m.action_logits_op = out_op
m.baseline_op = None
m.action_prob_op = tf.nn.softmax(m.action_logits_op)
else:
# A few fully connected layers here.
with tf.variable_scope('action_pred'):
batch_norm_param = args.arch.batch_norm_param
batch_norm_param['is_training'] = batch_norm_is_training_op
out_op, _ = tf_utils.fc_network(
m.combined_f, neurons=args.arch.pred_neurons,
wt_decay=args.solver.wt_decay, name='pred', offset=0,
num_pred=num_actions_,
batch_norm_param=batch_norm_param,
dropout_ratio=args.arch.fc_dropout, is_training=is_training)
if num_actions_ > num_actions:
m.action_logits_op = out_op[:,:num_actions]
m.baseline_op = out_op[:,num_actions:]
else:
m.action_logits_op = out_op
m.baseline_op = None
m.action_prob_op = tf.nn.softmax(m.action_logits_op)
m.train_ops = {}
m.train_ops['step'] = m.action_prob_op
m.train_ops['common'] = [m.input_tensors['common']['orig_maps'],
m.input_tensors['common']['goal_loc'],
m.input_tensors['common']['rel_goal_loc_at_start']]
m.train_ops['state_names'] = state_names
m.train_ops['init_state'] = init_state_ops
m.train_ops['updated_state'] = updated_state_ops
m.train_ops['batch_norm_is_training_op'] = batch_norm_is_training_op
# Flat list of ops which cache the step data.
m.train_ops['step_data_cache'] = [tf.no_op()]
if args.solver.freeze_conv:
m.train_ops['step_data_cache'] = [m.encoder_output]
else:
m.train_ops['step_data_cache'] = []
ewma_decay = 0.99 if is_training else 0.0
weight = tf.ones_like(m.input_tensors['train']['action'], dtype=tf.float32,
name='weight')
m.reg_loss_op, m.data_loss_op, m.total_loss_op, m.acc_ops = \
compute_losses_multi_or(
m.action_logits_op, m.input_tensors['train']['action'],
weights=weight, num_actions=num_actions,
data_loss_wt=args.solver.data_loss_wt,
reg_loss_wt=args.solver.reg_loss_wt, ewma_decay=ewma_decay)
if args.solver.freeze_conv:
vars_to_optimize = list(set(tf.trainable_variables()) - set(vars_))
else:
vars_to_optimize = None
m.lr_op, m.global_step_op, m.train_op, m.should_stop_op, m.optimizer, \
m.sync_optimizer = tf_utils.setup_training(
m.total_loss_op,
args.solver.initial_learning_rate,
args.solver.steps_per_decay,
args.solver.learning_rate_decay,
args.solver.momentum,
args.solver.max_steps,
args.solver.sync,
args.solver.adjust_lr_sync,
args.solver.num_workers,
args.solver.task,
vars_to_optimize=vars_to_optimize,
clip_gradient_norm=args.solver.clip_gradient_norm,
typ=args.solver.typ, momentum2=args.solver.momentum2,
adam_eps=args.solver.adam_eps)
if args.arch.sample_gt_prob_type == 'inverse_sigmoid_decay':
m.sample_gt_prob_op = tf_utils.inverse_sigmoid_decay(args.arch.isd_k,
m.global_step_op)
elif args.arch.sample_gt_prob_type == 'zero':
m.sample_gt_prob_op = tf.constant(-1.0, dtype=tf.float32)
elif args.arch.sample_gt_prob_type.split('_')[0] == 'step':
step = int(args.arch.sample_gt_prob_type.split('_')[1])
m.sample_gt_prob_op = tf_utils.step_gt_prob(
step, m.input_tensors['step']['step_number'][0,0,0])
m.sample_action_type = args.arch.action_sample_type
m.sample_action_combine_type = args.arch.action_sample_combine_type
_add_summaries(m, summary_mode, args.summary.arop_full_summary_iters)
m.init_op = tf.group(tf.global_variables_initializer(),
tf.local_variables_initializer())
m.saver_op = tf.train.Saver(keep_checkpoint_every_n_hours=4,
write_version=tf.train.SaverDef.V2)
return m
def _randomly_negate_tensor(tensor): """With 50% prob turn the tensor negative.""" should_flip = tf.cast(tf.floor(tf.random_uniform([]) + 0.5), tf.bool) final_tensor = tf.cond(should_flip, lambda: tensor, lambda: -tensor) return final_tensor
def init_step(self):
# q : action for RL
# value_matrix : state for RL
self.q = tf.placeholder(tf.int32,
[self.args.batch_size, self.args.seq_len],
name='step_q')
self.a = tf.placeholder(tf.int32,
[self.args.batch_size, self.args.seq_len],
name='step_a')
self.value_matrix = tf.placeholder(
tf.float32,
[self.args.memory_size, self.args.memory_value_state_dim],
name='step_value_matrix')
slice_a = tf.split(self.a, self.args.seq_len, 1)
a = tf.squeeze(slice_a[0], 1)
slice_q = tf.split(self.q, self.args.seq_len, 1)
q = tf.squeeze(slice_q[0], 1)
q_embed = self.embedding_q(q)
correlation_weight = self.memory.attention(q_embed)
stacked_value_matrix = tf.tile(tf.expand_dims(self.value_matrix, 0),
tf.stack([self.args.batch_size, 1, 1]))
# -1 for sampling
# 0, 1 for given answer
self.qa = tf.cond(
tf.squeeze(a) < 0,
lambda: self.sampling_a_given_q(q, stacked_value_matrix),
lambda: q + tf.multiply(a, self.args.n_questions))
a = (self.qa - 1) // self.args.n_questions
qa_embed = self.embedding_qa(self.qa)
######### Before Step ##########
prev_read_content, prev_summary, prev_pred_logits, prev_pred_prob = self.inference(
q_embed, correlation_weight, stacked_value_matrix, reuse_flag=True)
######### STEP #####################
knowledge_growth = self.calculate_knowledge_growth(
stacked_value_matrix, correlation_weight, qa_embed,
prev_read_content, prev_summary, prev_pred_prob)
# TODO : refactor sampling_a_given_q to return a only for below function call
self.stepped_value_matrix = tf.squeeze(self.memory.value.write_given_a(
stacked_value_matrix, correlation_weight, knowledge_growth, a,
True),
axis=0)
#self.stepped_value_matrix = tf.squeeze(self.memory.value.write(stacked_value_matrix, correlation_weight, qa_embed, knowledge_growth, True), axis=0)
self.stepped_read_content, self.stepped_summary, self.stepped_pred_logits, self.stepped_pred_prob = self.inference(
q_embed,
correlation_weight,
self.stepped_value_matrix,
reuse_flag=True)
######### After Step #########
self.value_matrix_difference = tf.squeeze(
tf.reduce_sum(self.stepped_value_matrix - stacked_value_matrix))
self.read_content_difference = tf.squeeze(
tf.reduce_sum(self.stepped_read_content - prev_read_content))
self.summary_difference = tf.squeeze(
tf.reduce_sum(self.stepped_summary - prev_summary))
self.pred_logit_difference = tf.squeeze(
tf.reduce_sum(self.stepped_pred_logits - prev_pred_logits))
self.pred_prob_difference = tf.squeeze(
tf.reduce_sum(self.stepped_pred_prob - prev_pred_prob))
def generate_temporal_curves(self, seed=None):
# Set kernel parameters
# Either choose a set of random parameters for the mini-batch
l1 = tf.random_uniform([self._batch_size, self._y_size,
self._x_size], self._l1_min, self._l1_max,
seed=seed)
sigma_f = tf.random_uniform([self._batch_size, self._y_size],
self._sigma_min, self._sigma_max, seed=seed)
l1_vel = tf.random_uniform([self._batch_size, self._y_size,
self._x_size],
-1*self._l1_vel, self._l1_vel, seed=seed)
sigma_f_vel = tf.random_uniform([self._batch_size, self._y_size],
-1*self._sigma_vel, self._sigma_vel,
seed=seed)
if self._testing:
num_total_points = 400
else:
num_total_points = 100
y_value_base = tf.random_normal([self._batch_size, self._y_size,
num_total_points, 1],seed=seed)
curve_list = []
if (self._case==2) or (self._case==3):
# sparse time or long term tracking
idx = tf.random_shuffle(tf.range(self._len_seq),
seed=seed)[:(self._len_given)]
for t in range(self._len_seq):
if seed is not None:
_seed = seed * t
else:
_seed = seed
if self._case==1: # using len_given
if t < self._len_given:
num_context = tf.random_uniform(shape=[], minval=5,
maxval=self._max_num_context, dtype=tf.int32,
seed=_seed)
else:
num_context = tf.constant(0)
#num_context = tf.constant(1)
if self._case==2: # sparse time
nc_cond = tf.where(tf.equal(idx,t))
nc_cond = tf.reshape(nc_cond, [-1])
num_context = tf.cond(tf.equal(tf.size(nc_cond),0),
lambda:tf.constant(0),
lambda:tf.random_uniform(shape=[], minval=5,
maxval=self._max_num_context,
dtype=tf.int32, seed=_seed))
if self._case==3: # long term tracking
nc_cond = tf.where(tf.equal(idx,t))
nc_cond = tf.reshape(nc_cond, [-1])
num_context = tf.cond(tf.equal(tf.size(nc_cond),0),
lambda:tf.constant(0),
lambda:tf.constant(1))
if self._temporal:
encoded_t = None
else:
encoded_t = 0.25 + 0.5*t/self._len_seq
curve_list.append(self.generate_curves(l1, sigma_f, num_context,
y_value_base, _seed,
encoded_t))
vel_noise = l1_vel * self._noise_factor * tf. random_normal([
self._batch_size,self._y_size, self._x_size],
seed=_seed)
l1 += l1_vel + vel_noise
vel_noise = sigma_f_vel * self._noise_factor * tf. random_normal(
[self._batch_size, self._x_size], seed=_seed)
sigma_f += sigma_f_vel + vel_noise
if self._testing:
for t in range(self._len_seq,self._len_seq+self._len_gen):
if seed is not None:
_seed = seed * t
else:
_seed = seed
num_context = tf.constant(0)
if self._temporal:
encoded_t = None
else:
encoded_t = 0.25 + 0.5*t/self._len_seq
curve_list.append(self.generate_curves(l1, sigma_f,
num_context,
y_value_base, _seed,
encoded_t))
vel_noise = l1_vel * self._noise_factor * tf. random_normal([
self._batch_size,self._y_size, self._x_size],
seed=_seed)
l1 += l1_vel + vel_noise
vel_noise = sigma_f_vel*self._noise_factor*tf.random_normal(
[self._batch_size, self._x_size], seed=_seed)
sigma_f += sigma_f_vel + vel_noise
context_x_list, context_y_list = [], []
target_x_list, target_y_list = [], []
num_total_points_list = []
num_context_points_list = []
for t in range(len(curve_list)):
(context_x, context_y), target_x = curve_list[t].query
target_y = curve_list[t].target_y
num_total_points_list.append(curve_list[t].num_total_points)
num_context_points_list.append(curve_list[t].num_context_points)
context_x_list.append(context_x)
context_y_list.append(context_y)
target_x_list.append(target_x)
target_y_list.append(target_y)
query = ((context_x_list, context_y_list), target_x_list)
return NPRegressionDescription(
query=query,
target_y=target_y_list,
num_total_points=num_total_points_list,
num_context_points=num_context_points_list,
hyperparams=[tf.constant(0)])
