def set_logp_to_neg_inf(X, logp, bounds):
"""Set `logp` to negative infinity when `X` is outside the allowed bounds.
# Arguments
X: tensorflow.Tensor
The variable to apply the bounds to
logp: tensorflow.Tensor
The log probability corrosponding to `X`
bounds: list of `Region` objects
The regions corrosponding to allowed regions of `X`
# Returns
logp: tensorflow.Tensor
The newly bounded log probability
"""
conditions = []
for l, u in bounds:
lower_is_neg_inf = not isinstance(l, tf.Tensor) and np.isneginf(l)
upper_is_pos_inf = not isinstance(u, tf.Tensor) and np.isposinf(u)
if not lower_is_neg_inf and upper_is_pos_inf:
conditions.append(tf.greater(X, l))
elif lower_is_neg_inf and not upper_is_pos_inf:
conditions.append(tf.less(X, u))
elif not (lower_is_neg_inf or upper_is_pos_inf):
conditions.append(tf.logical_and(tf.greater(X, l), tf.less(X, u)))
if len(conditions) > 0:
is_inside_bounds = conditions[0]
for condition in conditions[1:]:
is_inside_bounds = tf.logical_or(is_inside_bounds, condition)
logp = tf.select(is_inside_bounds, logp, tf.fill(tf.shape(X), config.dtype(-np.inf)))
return logp
def loop(step_, beams_, beam_value_, golden_value_, golden_inside_, step_valid_, g_id_, golden_record, beam_record):
cur_feat_x_ = tf.gather(x, step_)
cur_golden_path_ = tf.gather(golden_path, tf.range(step_))
cur_golden_feat_ = self._add_tag_dynamic(cur_feat_x_, cur_golden_path_)
# cur_golden_output_ = self._build_cnn(cur_golden_feat_)
cur_golden_output_ = build(cur_golden_feat_)
cur_golden_node_ = tf.gather(golden_path, tf.reshape(step_, [1]))
golden_value_ = tf.add(golden_value_,
tf.slice(cur_golden_output_, tf.concat(0, [[0], cur_golden_node_]), [1, 1]))
cur_beam_ = tf.unpack(beams_, num=self.beam_size)
cur_beam_feat_ = tf.concat(0, [self._add_tag_dynamic(cur_feat_x_, tf.reshape(e, [-1])) for e in cur_beam_])
# cur_beam_output_ = self._build_cnn(cur_beam_feat_)
cur_beam_output_ = build(cur_beam_feat_)
golden_record = golden_record.write(step_, cur_golden_output_)
beam_record = beam_record.write(step_, cur_beam_output_)
beam_value_, beams_ = self._top_beams_new(cur_beam_output_, beam_value_, beams_)
new_golden_path_ = tf.gather(golden_path, tf.range(step_ + 1))
# golden_beam_id_ = index_of_tensor(new_golden_path_, beams_)
g_id_ = index_of_tensor(new_golden_path_, beams_)
golden_inside_ = tf.select(tf.less(tf.shape(g_id_)[0], 1),
tf.constant(False, tf.bool), tf.constant(True, tf.bool))
step_valid_ = tf.logical_and(tf.less(step_+1, length), tf.less(step_+1, self.max_step_tracked))
return [step_ + 1, beams_, beam_value_, golden_value_, golden_inside_, step_valid_, g_id_, golden_record, beam_record]
def enc_step(step):
"""Encoder step."""
if autoenc_decay < 1.0:
quant_step = autoenc_quantize(step, 16, nmaps, self.do_training)
if backward:
exp_glob = tf.train.exponential_decay(1.0, self.global_step - 10000,
1000, autoenc_decay)
dec_factor = 1.0 - exp_glob # * self.do_training
dec_factor = tf.cond(tf.less(self.global_step, 10500),
lambda: tf.constant(0.05), lambda: dec_factor)
else:
dec_factor = 1.0
cur = tf.cond(tf.less(tf.random_uniform([]), dec_factor),
lambda: quant_step, lambda: step)
else:
cur = step
if dropout > 0.0001:
cur = tf.nn.dropout(cur, keep_prob)
if act_noise > 0.00001:
cur += tf.truncated_normal(tf.shape(cur)) * act_noise_scale
# Do nconvs-many CGRU steps.
if do_jit and tf.get_variable_scope().reuse:
with jit_scope():
for layer in xrange(nconvs):
cur = conv_gru([], cur, kw, kh, nmaps, conv_rate(layer),
cutoff, "ecgru_%d" % layer, do_layer_norm)
else:
for layer in xrange(nconvs):
cur = conv_gru([], cur, kw, kh, nmaps, conv_rate(layer),
cutoff, "ecgru_%d" % layer, do_layer_norm)
return cur
def compute_IOU(bboxA, bboxB):
"""Compute the Intersection Over Union.
Args:
bboxA: [N X 4 tensor] format = [left, top, right, bottom]
bboxB: [N X 4 tensor]
Return:
IOU: [N X 1 tensor]
"""
x1A, y1A, x2A, y2A = tf.split(1, 4, bboxA)
x1B, y1B, x2B, y2B = tf.split(1, 4, bboxB)
# compute intersection
x1_max = tf.maximum(x1A, x1B)
y1_max = tf.maximum(y1A, y1B)
x2_min = tf.minimum(x2A, x2B)
y2_min = tf.minimum(y2A, y2B)
# overlap_flag = tf.logical_and( tf.less(x1_max, x2_min), tf.less(y1_max, y2_min))
overlap_flag = tf.to_float(tf.less(x1_max, x2_min)) * \
tf.to_float(tf.less(y1_max, y2_min))
overlap_area = tf.mul(overlap_flag, tf.mul(
x2_min - x1_max, y2_min - y1_max))
# compute union
areaA = tf.mul(x2A - x1A, y2A - y1A)
areaB = tf.mul(x2B - x1B, y2B - y1B)
union_area = areaA + areaB - overlap_area
return tf.div(overlap_area, union_area)
def prune_outside_window(boxlist, window, scope=None):
"""Prunes bounding boxes that fall outside a given window.
This function prunes bounding boxes that even partially fall outside the given
window. See also clip_to_window which only prunes bounding boxes that fall
completely outside the window, and clips any bounding boxes that partially
overflow.
Args:
boxlist: a BoxList holding M_in boxes.
window: a float tensor of shape [4] representing [ymin, xmin, ymax, xmax]
of the window
scope: name scope.
Returns:
pruned_corners: a tensor with shape [M_out, 4] where M_out <= M_in
valid_indices: a tensor with shape [M_out] indexing the valid bounding boxes
in the input tensor.
"""
with tf.name_scope(scope, 'PruneOutsideWindow'):
y_min, x_min, y_max, x_max = tf.split(
value=boxlist.get(), num_or_size_splits=4, axis=1)
win_y_min, win_x_min, win_y_max, win_x_max = tf.unstack(window)
coordinate_violations = tf.concat([
tf.less(y_min, win_y_min), tf.less(x_min, win_x_min),
tf.greater(y_max, win_y_max), tf.greater(x_max, win_x_max)
], 1)
valid_indices = tf.reshape(
tf.where(tf.logical_not(tf.reduce_any(coordinate_violations, 1))), [-1])
return gather(boxlist, valid_indices), valid_indices
def l1_smooth_losses(predict_boxes, gtboxes, object_weights, classes_weights=None):
'''
:param predict_boxes: [minibatch_size, -1]
:param gtboxes: [minibatch_size, -1]
:param object_weights: [minibatch_size, ]. 1.0 represent object, 0.0 represent others(ignored or background)
:return:
'''
diff = predict_boxes - gtboxes
abs_diff = tf.cast(tf.abs(diff), tf.float32)
if classes_weights is None:
'''
first_stage:
predict_boxes :[minibatch_size, 5]
gtboxes: [minibatchs_size, 5]
'''
anchorwise_smooth_l1norm = tf.reduce_sum(
tf.where(tf.less(abs_diff, 1), 0.5 * tf.square(abs_diff), abs_diff - 0.5),
axis=1) * object_weights
else:
'''
fast_rcnn stage:
predict_boxes: [minibatch_size, 5*num_classes]
gtboxes: [minibatch_size, 5*num_classes]
classes_weights : [minibatch_size, 5*num_classes]
'''
anchorwise_smooth_l1norm = tf.reduce_sum(
tf.where(tf.less(abs_diff, 1), 0.5*tf.square(abs_diff)*classes_weights,
(abs_diff - 0.5)*classes_weights),
axis=1)*object_weights
return tf.reduce_mean(anchorwise_smooth_l1norm, axis=0) # reduce mean
def image_distortions(image, distortions):
distort_left_right_random = distortions[0]
mirror = tf.less(tf.pack([1.0, distort_left_right_random, 1.0]), 0.5)
image = tf.reverse(image, mirror)
distort_up_down_random = distortions[1]
mirror = tf.less(tf.pack([distort_up_down_random, 1.0, 1.0]), 0.5)
image = tf.reverse(image, mirror)
return image
def learning_rate_schedule(): # Function which controls learning rate during training
import tensorflow as tf
step = tf.train.get_or_create_global_step()
lr = tf.case([(tf.less(step, 1000), lambda: tf.constant(0.0004)),
(tf.less(step, 10000), lambda: tf.constant(0.01)),
(tf.less(step, 40000), lambda: tf.constant(0.005)),
(tf.less(step, 55000), lambda: tf.constant(0.0005)),
(tf.less(step, 65000), lambda: tf.constant(0.00005))])
return lr
def testWhileCond_3(self):
with self.test_session():
n = tf.convert_to_tensor(0)
c = lambda x: tf.less(x, 10)
b = lambda x: control_flow_ops.cond(tf.less(0, 1), lambda: tf.add(x, 1), lambda: tf.sub(x, 1))
r = control_flow_ops.While(c, b, [n])
result = r.eval()
self.assertTrue(check_op_order(n.graph))
self.assertAllEqual(10, result)
def bottleneck(self, x):
hparams = self.hparams
x = tf.tanh(tf.layers.dense(x, hparams.bottleneck_bits, name="bottleneck"))
d = x + tf.stop_gradient(2.0 * tf.to_float(tf.less(0.0, x)) - 1.0 - x)
if hparams.mode == tf.estimator.ModeKeys.TRAIN:
noise = tf.random_uniform(common_layers.shape_list(x))
noise = 2.0 * tf.to_float(tf.less(hparams.bottleneck_noise, noise)) - 1.0
d *= noise
x = common_layers.mix(d, x, hparams.discretize_warmup_steps,
hparams.mode == tf.estimator.ModeKeys.TRAIN)
return x, 0.0
def testCondWhile_1(self):
with self.test_session():
n = tf.convert_to_tensor(0, name="n")
c = lambda x: tf.less(x, 10)
b = lambda x: tf.add(x, 1)
r = tf.cond(tf.less(0, 1),
lambda: control_flow_ops.While(c, b, [n]),
lambda: n)
result = r.eval()
self.assertTrue(check_op_order(n.graph))
self.assertAllEqual(10, result)
def augment_2d(inputs, rotation=0, horizontal_flip=False, vertical_flip=False):
"""Apply additive augmentation on 2D data.
# Arguments
rotation: A float, the degree range for rotation (0 <= rotation < 180),
e.g. 3 for random image rotation between (-3.0, 3.0).
horizontal_flip: A boolean, whether to allow random horizontal flip,
e.g. true for 50% possibility to flip image horizontally.
vertical_flip: A boolean, whether to allow random vertical flip,
e.g. true for 50% possibility to flip image vertically.
# Returns
input data after augmentation, whose shape is the same as its original.
"""
if inputs.dtype != tf.float32:
inputs = tf.image.convert_image_dtype(inputs, dtype=tf.float32)
with tf.name_scope('augmentation'):
shp = tf.shape(inputs)
batch_size, height, width = shp[0], shp[1], shp[2]
width = tf.cast(width, tf.float32)
height = tf.cast(height, tf.float32)
transforms = []
identity = tf.constant([1, 0, 0, 0, 1, 0, 0, 0], dtype=tf.float32)
if rotation > 0:
angle_rad = rotation * 3.141592653589793 / 180.0
angles = tf.random_uniform([batch_size], -angle_rad, angle_rad)
f = tf.contrib.image.angles_to_projective_transforms(angles,
height, width)
transforms.append(f)
if horizontal_flip:
coin = tf.less(tf.random_uniform([batch_size], 0, 1.0), 0.5)
shape = [-1., 0., width, 0., 1., 0., 0., 0.]
flip_transform = tf.convert_to_tensor(shape, dtype=tf.float32)
flip = tf.tile(tf.expand_dims(flip_transform, 0), [batch_size, 1])
noflip = tf.tile(tf.expand_dims(identity, 0), [batch_size, 1])
transforms.append(tf.where(coin, flip, noflip))
if vertical_flip:
coin = tf.less(tf.random_uniform([batch_size], 0, 1.0), 0.5)
shape = [1., 0., 0., 0., -1., height, 0., 0.]
flip_transform = tf.convert_to_tensor(shape, dtype=tf.float32)
flip = tf.tile(tf.expand_dims(flip_transform, 0), [batch_size, 1])
noflip = tf.tile(tf.expand_dims(identity, 0), [batch_size, 1])
transforms.append(tf.where(coin, flip, noflip))
if transforms:
f = tf.contrib.image.compose_transforms(*transforms)
inputs = tf.contrib.image.transform(inputs, f, interpolation='BILINEAR')
return inputs
def tanh_discrete_bottleneck(x, bottleneck_bits, bottleneck_noise,
discretize_warmup_steps, mode):
"""Simple discretization through tanh, flip bottleneck_noise many bits."""
x = tf.tanh(tf.layers.dense(x, bottleneck_bits,
name="tanh_discrete_bottleneck"))
d = x + tf.stop_gradient(2.0 * tf.to_float(tf.less(0.0, x)) - 1.0 - x)
if mode == tf.estimator.ModeKeys.TRAIN:
noise = tf.random_uniform(common_layers.shape_list(x))
noise = 2.0 * tf.to_float(tf.less(bottleneck_noise, noise)) - 1.0
d *= noise
d = common_layers.mix(d, x, discretize_warmup_steps,
mode == tf.estimator.ModeKeys.TRAIN)
return d, 0.0
def process_features(image, label, do_augment = False):
# Do any image preprocessing/augmentation here...
with tf.name_scope('process_features'):
# Crop driving view (Start from row 126, and slice 100 rows)
image = tf.slice(image, [126, 0, 0], [100, 256, 3])
# Resize image
image = tf.image.resize_images(image, [66, 200])
if do_augment == True:
# Change or not change colors
def do_color_changes():
distorted_image = tf.image.random_brightness(image, max_delta=32. / 255.)
distorted_image = tf.image.random_saturation(distorted_image, lower=0.5, upper=1.5)
distorted_image = tf.image.random_hue(distorted_image, max_delta=0.2)
distorted_image = tf.image.random_contrast(distorted_image, lower=0.5, upper=1.5)
return distorted_image
def no_color_change():
distorted_image = image
return distorted_image
# Uniform variable in [0,1)
flip_coin_color = tf.random_uniform(shape=[], minval=0., maxval=1., dtype=tf.float32)
pred_color = tf.less(flip_coin_color, 0.5)
# Randomically select doing color augmentation
image = tf.cond(pred_color, do_color_changes, no_color_change, name='if_color')
# Change or not change colors
def flip_image_steering():
distorted_image = tf.image.flip_left_right(image)
distorted_label = -label
return distorted_image, distorted_label
def no_flip_image_steering():
distorted_image = image
distorted_label = label
return distorted_image, distorted_label
# Uniform variable in [0,1)
flip_coin_flip = tf.random_uniform(shape=[], minval=0., maxval=1., dtype=tf.float32)
pred_flip = tf.less(flip_coin_flip, 0.5)
# Randomically select doing color augmentation
image, label = tf.cond(pred_flip, flip_image_steering, no_flip_image_steering, name='if_steering')
# Convert from [0, 255] -> [-0.5, 0.5] floats.
image = tf.cast(image, tf.float32) * (1. / 255.0)
#image = tf.image.per_image_standardization(image)
return image, label
def unwrap(p, discont=np.pi, axis=-1):
"""Unwrap a cyclical phase tensor.
Args:
p: Phase tensor.
discont: Float, size of the cyclic discontinuity.
axis: Axis of which to unwrap.
Returns:
unwrapped: Unwrapped tensor of same size as input.
"""
dd = diff(p, axis=axis)
ddmod = tf.mod(dd + np.pi, 2.0 * np.pi) - np.pi
idx = tf.logical_and(tf.equal(ddmod, -np.pi), tf.greater(dd, 0))
ddmod = tf.where(idx, tf.ones_like(ddmod) * np.pi, ddmod)
ph_correct = ddmod - dd
idx = tf.less(tf.abs(dd), discont)
ddmod = tf.where(idx, tf.zeros_like(ddmod), dd)
ph_cumsum = tf.cumsum(ph_correct, axis=axis)
shape = p.get_shape().as_list()
shape[axis] = 1
ph_cumsum = tf.concat([tf.zeros(shape, dtype=p.dtype), ph_cumsum], axis=axis)
unwrapped = p + ph_cumsum
return unwrapped
def _create_learning_rate(hyperparams, step_var):
"""Creates learning rate var, with decay and switching for CompositeOptimizer.
Args:
hyperparams: a GridPoint proto containing optimizer spec, particularly
learning_method to determine optimizer class to use.
step_var: tf.Variable, global training step.
Raises:
ValueError: If the composite optimizer is set, but not correctly configured.
Returns:
a scalar `Tensor`, the learning rate based on current step and hyperparams.
"""
if hyperparams.learning_method != 'composite':
base_rate = hyperparams.learning_rate
adjusted_steps = step_var
else:
spec = hyperparams.composite_optimizer_spec
switch = tf.less(step_var, spec.switch_after_steps)
base_rate = tf.cond(switch, lambda: tf.constant(spec.method1.learning_rate),
lambda: tf.constant(spec.method2.learning_rate))
if spec.reset_learning_rate:
adjusted_steps = tf.cond(switch, lambda: step_var,
lambda: step_var - spec.switch_after_steps)
else:
adjusted_steps = step_var
return tf.train.exponential_decay(
learning_rate=base_rate,
global_step=adjusted_steps,
decay_steps=hyperparams.decay_steps,
decay_rate=hyperparams.decay_base,
staircase=hyperparams.decay_staircase)
def gather(self, indices, keys=None):
"""Returns elements at the specified indices from the buffer.
Args:
indices: (list of integers or rank 1 int Tensor) indices in the buffer to
retrieve elements from.
keys: List of keys of tensors to retrieve. If None retrieve all.
Returns:
A list of tensors, where each element in the list is obtained by indexing
one of the tensors in the buffer.
Raises:
ValueError: If gather is called before calling the add function.
tf.errors.InvalidArgumentError: If indices are bigger than the number of
items in the buffer.
"""
if not self._tensors:
raise ValueError('The add function must be called before calling gather.')
if keys is None:
keys = self._tensors.keys()
with tf.name_scope('Gather'):
index_bound_assert = tf.Assert(
tf.less(
tf.to_int64(tf.reduce_max(indices)),
tf.minimum(self.get_num_adds(), self._buffer_size)),
['Index out of bounds.'])
with tf.control_dependencies([index_bound_assert]):
indices = tf.convert_to_tensor(indices)
batch = []
for key in keys:
batch.append(tf.gather(self._tensors[key], indices, name=key))
return batch
def testLoop_2(self):
with self.test_session():
zero = tf.constant(0)
one = tf.constant(1)
n = tf.constant(10)
enter_i = control_flow_ops.enter(zero, "foo", False)
enter_one = control_flow_ops.enter(one, "foo", True)
enter_n = control_flow_ops.enter(n, "foo", True)
merge_i = control_flow_ops.merge([enter_i, enter_i])[0]
less_op = tf.less(merge_i, enter_n)
cond_op = control_flow_ops.loop_cond(less_op)
switch_i = control_flow_ops.switch(merge_i, cond_op)
add_i = tf.add(switch_i[1], enter_one)
with tf.device("/gpu:0"):
next_i = control_flow_ops.next_iteration(add_i)
merge_i.op._update_input(1, next_i)
exit_i = control_flow_ops.exit(switch_i[0])
result = exit_i.eval()
self.assertAllEqual(10, result)
def ced(model, config, scope, connect, threshold = 1e-5):
with tf.variable_scope(scope), tf.name_scope(scope):
with tf.variable_scope('inputs'), tf.name_scope('inputs'):
model['%s_in0length' %scope] = model['%s_out0length' %connect]
model['%s_in1length' %scope] = model['%s_out1length' %connect]
model['%s_in2length' %scope] = model['%s_out2length' %connect]
model['%s_maxin2length' %scope] = model['%s_maxout2length' %connect]
model['%s_inputs' %scope] = tf.clip_by_value(model['%s_outputs' %connect], threshold, 1. - threshold, name = '%s_inputs' %scope)
model['%s_out0length' %scope] = model['%s_in0length' %scope]
model['%s_out1length' %scope] = model['%s_in1length' %scope]
model['%s_out2length' %scope] = tf.placeholder(tf.int32, [model['%s_in0length' %scope]], '%s_out2length' %scope)
model['%s_maxout2length' %scope] = model['%s_maxin2length' %scope]
with tf.variable_scope('labels'), tf.name_scope('labels'):
model['%s_labels_len' %scope] = tf.placeholder(tf.int32, [model['%s_in0length' %scope]], '%s_labels_len' %scope)
model['%s_labels_ind' %scope] = tf.placeholder(tf.int64, [None, 3], '%s_labels_ind' %scope)
model['%s_labels_val' %scope] = tf.placeholder(tf.float32, [None], '%s_labels_val' %scope)
model['%s_labels' %scope] = tf.sparse_to_dense(model['%s_labels_ind' %scope], [model['%s_in0length' %scope], model['%s_maxin2length' %scope], model['%s_maxin2length' %scope]], model['%s_labels_val' %scope], -1, name = '%s_labels' %scope)
with tf.variable_scope('loss'), tf.name_scope('loss'):
model['%s_loss' %scope] = tf.reduce_sum(tf.where(tf.less(model['%s_labels' %scope], tf.zeros([model['%s_in0length' %scope], model['%s_maxin2length' %scope], model['%s_maxin2length' %scope]], tf.float32)), tf.zeros([model['%s_in0length' %scope], model['%s_maxin2length' %scope], model['%s_maxin2length' %scope]], tf.float32), -tf.add(tf.multiply(model['%s_labels' %scope], tf.log(model['%s_inputs' %scope])), tf.multiply(tf.subtract(1., model['%s_labels' %scope]), tf.log(tf.subtract(1., model['%s_inputs' %scope]))))), name = '%s_loss' %scope)
with tf.variable_scope('outputs'), tf.name_scope('outputs'):
model['%s_output' %scope] = model['%s_inputs' %scope]
return model
def hinge_loss(self, y_true, y_pred):
# Custom loss function
margin = 0.1
# loop counter
i = tf.constant(0)
# loop condition function
c = lambda i, _: tf.less(i, tf.shape(y_true)[0])
outer_sum_loss = tf.constant(0.0)
def process_ele(i, outer_sum_loss):
# Get a subtensor from batch
y_true_one = y_true[i]
y_pred_one = y_pred[i]
# Stack margin to a num_class*1 matrix
margin_stack = tf.reshape(tf.stack([tf.constant(0.1)] * self.num_classes), [self.num_classes, 1])
# Stack true label to a word_dim*num_class matrix and transpose it
y_true_one_stack = tf.stack([tf.transpose(y_true_one)] * self.num_classes)
# Reshape predict from (word_dim,) to (word_dim,1)
y_pred_one_t = tf.reshape(y_pred_one, [self.word_dim, 1])
# Calculate loss
r = margin_stack - tf.matmul(y_true_one_stack, y_pred_one_t) + tf.matmul(self.label_vec_tensor, y_pred_one_t)
# Summation
# We did not exclude true label inside summation, so we subtract extra margin
sum_inner_loss = tf.reduce_sum(K.relu(r)) - margin
# Return counter++ and accumulated loss
return tf.add(i, 1), tf.add(outer_sum_loss, sum_inner_loss)
_, outer_sum_loss = tf.while_loop(c, process_ele, [i, outer_sum_loss])
# Return average loss over batch
return outer_sum_loss / tf.cast(tf.shape(y_true)[0], dtype=tf.float32)
def testChooseVector(self):
with self.test_session():
x = np.arange(10, 12)
y = np.arange(15, 18)
self.assertAllEqual(
x, distribution_util.pick_vector(
tf.less(0, 5), x, y).eval())
self.assertAllEqual(
y, distribution_util.pick_vector(
tf.less(5, 0), x, y).eval())
self.assertAllEqual(
x, distribution_util.pick_vector(
tf.constant(True), x, y)) # No eval.
self.assertAllEqual(
y, distribution_util.pick_vector(
tf.constant(False), x, y)) # No eval.
def testControlFlow(self):
with self.test_session() as sess:
v0 = tf.get_variable("v0", [], initializer=tf.constant_initializer(0))
var_dict = {}
# Call get_variable in each of the cond clauses.
def var_in_then_clause():
v1 = tf.get_variable("v1", [1], initializer=tf.constant_initializer(1))
var_dict["v1"] = v1
return v1 + v0
def var_in_else_clause():
v2 = tf.get_variable("v2", [1], initializer=tf.constant_initializer(2))
var_dict["v2"] = v2
return v2 + v0
add = control_flow_ops.cond(tf.less(v0, 10),
var_in_then_clause,
var_in_else_clause)
v1 = var_dict["v1"]
v2 = var_dict["v2"]
# We should be able to initialize and run v1 and v2 without initializing
# v0, even if the variable was created with a control dep on v0.
sess.run(v1.initializer)
self.assertEqual([1], sess.run(v1))
sess.run(v2.initializer)
self.assertEqual([2], sess.run(v2))
# v0 should still be uninitialized.
with self.assertRaisesRegexp(tf.OpError, "uninitialized"):
sess.run(v0)
# We should not be able to run 'add' yet.
with self.assertRaisesRegexp(tf.OpError, "uninitialized"):
sess.run(add)
# If we initialize v0 we should be able to run 'add'.
sess.run(v0.initializer)
sess.run(add)
def con1():
values, top_nodes = top_k_in_2dim_tensor(value_mat, cur_beam_size)
beam_new = self._concat_top_nodes(beams, top_nodes)
beam_size_tmp = tf.mul(cur_beam_size, self.output_size)
beam_size = tf.select(tf.less(beam_size_tmp, self.beam_size), beam_size_tmp, self.beam_size)
return values, beam_new, beam_size
def _teacher_forcing_ratio_decay(init_tfr, global_step, hparams): ################################################################# # Narrow Cosine Decay: # Phase 1: tfr = 1 # We only start learning rate decay after 10k steps # Phase 2: tfr in ]0, 1[ # decay reach minimal value at step ~280k # Phase 3: tfr = 0 # clip by minimal teacher forcing ratio value (step >~ 280k) ################################################################# #Compute natural cosine decay tfr = tf.train.cosine_decay(init_tfr, global_step=global_step - hparams.tacotron_teacher_forcing_start_decay, #tfr = 1 at step 10k decay_steps=hparams.tacotron_teacher_forcing_decay_steps, #tfr = 0 at step ~280k alpha=hparams.tacotron_teacher_forcing_decay_alpha, #tfr = 0% of init_tfr as final value name='tfr_cosine_decay') #force teacher forcing ratio to take initial value when global step < start decay step. narrow_tfr = tf.cond( tf.less(global_step, tf.convert_to_tensor(hparams.tacotron_teacher_forcing_start_decay)), lambda: tf.convert_to_tensor(init_tfr), lambda: tfr) return narrow_tfr
def beta_schedule(schedule, global_step, final_beta, decay_start, decay_end):
"""Get KL multiplier (beta) based on the schedule."""
if decay_start > decay_end:
raise ValueError("decay_end is smaller than decay_end.")
# Since some of the TF schedules do not support incrementing a value,
# in all of the schedules, we anneal the beta from final_beta to zero
# and then reverse it at the bottom.
if schedule == "constant":
decayed_value = 0.0
elif schedule == "linear":
decayed_value = tf.train.polynomial_decay(
learning_rate=final_beta,
global_step=global_step - decay_start,
decay_steps=decay_end - decay_start,
end_learning_rate=0.0)
elif schedule == "noisy_linear_cosine_decay":
decayed_value = tf.train.noisy_linear_cosine_decay(
learning_rate=final_beta,
global_step=global_step - decay_start,
decay_steps=decay_end - decay_start)
# TODO(mechcoder): Add log_annealing schedule.
else:
raise ValueError("Unknown beta schedule.")
increased_value = final_beta - decayed_value
increased_value = tf.maximum(0.0, increased_value)
beta = tf.case(
pred_fn_pairs={
tf.less(global_step, decay_start): lambda: 0.0,
tf.greater(global_step, decay_end): lambda: final_beta},
default=lambda: increased_value)
return beta
def loop(q_, mask, mass_, found_):
q_list = tf.dynamic_partition(q_, mask, 2)
condition_indices = tf.dynamic_partition(tf.range(tf.shape(q_)[0]), mask, 2) # 0 element it False,
# 1 element if true
p = q_list[1] * (1.0 - mass_) / tf.reduce_sum(q_list[1])
p_new = tf.dynamic_stitch(condition_indices, [q_list[0], p])
# condition verification and mask modification
less_mask = tf.cast(tf.less(u, p_new), tf.int32) # 0 when u is bigger than p, 1 when u is less than p
condition_indices = tf.dynamic_partition(tf.range(tf.shape(p_new)[0]), less_mask,
2) # 0 when u is bigger than p, 1 when u is less than p
split_p_new = tf.dynamic_partition(p_new, less_mask, 2)
split_u = tf.dynamic_partition(u, less_mask, 2)
alpha = tf.dynamic_stitch(condition_indices, [split_p_new[0], split_u[1]])
mass_ += tf.reduce_sum(split_u[1])
mask = mask * (tf.ones_like(less_mask) - less_mask)
found_ = tf.cond(tf.equal(tf.reduce_sum(less_mask), 0),
lambda: False,
lambda: True)
alpha = tf.reshape(alpha, q_.shape)
return alpha, mask, mass_, found_
def rpn_proposals(self):
with tf.variable_scope('rpn_proposals'):
rpn_decode_boxes = encode_and_decode.decode_boxes(encode_boxes=self.rpn_encode_boxes,
reference_boxes=self.anchors,
scale_factors=self.scale_factors)
rpn_softmax_scores = slim.softmax(self.rpn_scores)
rpn_object_score = rpn_softmax_scores[:, 1] # second column represent object
if self.top_k_nms:
rpn_object_score, top_k_indices = tf.nn.top_k(rpn_object_score, k=self.top_k_nms)
rpn_decode_boxes = tf.gather(rpn_decode_boxes, top_k_indices)
# NMS
valid_indices = tf_wrapper.nms_rotate_tf(boxes_list=rpn_decode_boxes,
scores=rpn_object_score,
iou_threshold=self.rpn_nms_iou_threshold,
max_output_size=self.max_proposals_num,
use_gpu=cfgs.NMS_USE_GPU)
valid_boxes = tf.gather(rpn_decode_boxes, valid_indices)
valid_scores = tf.gather(rpn_object_score, valid_indices)
# print_tensors(valid_scores, 'rpn_score')
rpn_proposals_boxes, rpn_proposals_scores = tf.cond(
tf.less(tf.shape(valid_boxes)[0], self.max_proposals_num),
lambda: boxes_utils.padd_boxes_with_zeros(valid_boxes, valid_scores,
self.max_proposals_num),
lambda: (valid_boxes, valid_scores))
return rpn_proposals_boxes, rpn_proposals_scores
def dae(x, hparams, name):
with tf.variable_scope(name):
m = tf.layers.dense(x, hparams.v_size, name="mask")
if hparams.softmax_k > 0:
m, kl = top_k_softmax(m, hparams.softmax_k)
return m, m, 1.0 - tf.reduce_mean(kl)
logsm = tf.nn.log_softmax(m)
# Gumbel-softmax sample.
gumbel_samples = gumbel_sample(common_layers.shape_list(m))
steps = hparams.kl_warmup_steps
gumbel_samples *= common_layers.inverse_exp_decay(steps // 5) * 0.5
temperature = 1.2 - common_layers.inverse_lin_decay(steps)
# 10% of the time keep reasonably high temperature to keep learning.
temperature = tf.cond(tf.less(tf.random_uniform([]), 0.9),
lambda: temperature,
lambda: tf.random_uniform([], minval=0.5, maxval=1.0))
s = tf.nn.softmax((logsm + gumbel_samples) / temperature)
m = tf.nn.softmax(m)
kl = - tf.reduce_max(logsm, axis=-1)
if _DO_SUMMARIES:
tf.summary.histogram("max-log", tf.reshape(kl, [-1]))
# Calculate the argmax and construct hot vectors.
maxvec = tf.reshape(tf.argmax(m, axis=-1), [-1])
maxvhot = tf.stop_gradient(tf.one_hot(maxvec, hparams.v_size))
# Add losses that prevent too few being used.
distrib = tf.reshape(logsm, [-1, hparams.v_size]) * maxvhot
d_mean = tf.reduce_mean(distrib, axis=[0], keep_dims=True)
d_variance = tf.reduce_mean(tf.square(distrib - d_mean), axis=[0])
d_dev = - tf.reduce_mean(d_variance)
ret = s
if hparams.mode != tf.contrib.learn.ModeKeys.TRAIN:
ret = tf.reshape(maxvhot, common_layers.shape_list(s)) # Just hot @eval.
return m, ret, d_dev * 5.0 + tf.reduce_mean(kl) * 0.002
def testCond_2(self):
with self.test_session():
x = tf.constant(10)
r = tf.cond(tf.less(1, 0), lambda: tf.add(x, 1), lambda: tf.sub(x, 1))
result = r.eval()
self.assertTrue(check_op_order(x.graph))
self.assertAllEqual(9, result)
def _get_values_from_start_and_end(self, input_tensor, num_start_samples,
num_end_samples, total_num_samples):
"""slices num_start_samples and last num_end_samples from input_tensor.
Args:
input_tensor: An int32 tensor of shape [N] to be sliced.
num_start_samples: Number of examples to be sliced from the beginning
of the input tensor.
num_end_samples: Number of examples to be sliced from the end of the
input tensor.
total_num_samples: Sum of is num_start_samples and num_end_samples. This
should be a scalar.
Returns:
A tensor containing the first num_start_samples and last num_end_samples
from input_tensor.
"""
input_length = tf.shape(input_tensor)[0]
start_positions = tf.less(tf.range(input_length), num_start_samples)
end_positions = tf.greater_equal(
tf.range(input_length), input_length - num_end_samples)
selected_positions = tf.logical_or(start_positions, end_positions)
selected_positions = tf.cast(selected_positions, tf.int32)
indexed_positions = tf.multiply(tf.cumsum(selected_positions),
selected_positions)
one_hot_selector = tf.one_hot(indexed_positions - 1,
total_num_samples,
dtype=tf.int32)
return tf.tensordot(input_tensor, one_hot_selector, axes=[0, 0])
def compute_eval_loss_and_metrics(logits, # type: tf.Tensor
softmax_logits, # type: tf.Tensor
duplicate_mask, # type: tf.Tensor
num_training_neg, # type: int
match_mlperf=False, # type: bool
use_tpu=False # type: bool
):
# type: (...) -> tf.estimator.EstimatorSpec
"""Model evaluation with HR and NDCG metrics.
The evaluation protocol is to rank the test interacted item (truth items)
among the randomly chosen 999 items that are not interacted by the user.
The performance of the ranked list is judged by Hit Ratio (HR) and Normalized
Discounted Cumulative Gain (NDCG).
For evaluation, the ranked list is truncated at 10 for both metrics. As such,
the HR intuitively measures whether the test item is present on the top-10
list, and the NDCG accounts for the position of the hit by assigning higher
scores to hits at top ranks. Both metrics are calculated for each test user,
and the average scores are reported.
If `match_mlperf` is True, then the HR and NDCG computations are done in a
slightly unusual way to match the MLPerf reference implementation.
Specifically, if the evaluation negatives contain duplicate items, it will be
treated as if the item only appeared once. Effectively, for duplicate items in
a row, the predicted score for all but one of the items will be set to
-infinity
For example, suppose we have that following inputs:
logits_by_user: [[ 2, 3, 3],
[ 5, 4, 4]]
items_by_user: [[10, 20, 20],
[30, 40, 40]]
# Note: items_by_user is not explicitly present. Instead the relevant \
information is contained within `duplicate_mask`
top_k: 2
Then with match_mlperf=True, the HR would be 2/2 = 1.0. With
match_mlperf=False, the HR would be 1/2 = 0.5. This is because each user has
predicted scores for only 2 unique items: 10 and 20 for the first user, and 30
and 40 for the second. Therefore, with match_mlperf=True, it's guaranteed the
first item's score is in the top 2. With match_mlperf=False, this function
would compute the first user's first item is not in the top 2, because item 20
has a higher score, and item 20 occurs twice.
Args:
logits: A tensor containing the predicted logits for each user. The shape
of logits is (num_users_per_batch * (1 + NUM_EVAL_NEGATIVES),) Logits
for a user are grouped, and the first element of the group is the true
element.
softmax_logits: The same tensor, but with zeros left-appended.
duplicate_mask: A vector with the same shape as logits, with a value of 1
if the item corresponding to the logit at that position has already
appeared for that user.
num_training_neg: The number of negatives per positive during training.
match_mlperf: Use the MLPerf reference convention for computing rank.
use_tpu: Should the evaluation be performed on a TPU.
Returns:
An EstimatorSpec for evaluation.
"""
logits_by_user = tf.reshape(logits, (-1, rconst.NUM_EVAL_NEGATIVES + 1))
duplicate_mask_by_user = tf.reshape(duplicate_mask,
(-1, rconst.NUM_EVAL_NEGATIVES + 1))
if match_mlperf:
# Set duplicate logits to the min value for that dtype. The MLPerf
# reference dedupes during evaluation.
logits_by_user *= (1 - duplicate_mask_by_user)
logits_by_user += duplicate_mask_by_user * logits_by_user.dtype.min
# Determine the location of the first element in each row after the elements
# are sorted.
sort_indices = tf.contrib.framework.argsort(
logits_by_user, axis=1, direction="DESCENDING")
# Use matrix multiplication to extract the position of the true item from the
# tensor of sorted indices. This approach is chosen because both GPUs and TPUs
# perform matrix multiplications very quickly. This is similar to np.argwhere.
# However this is a special case because the target will only appear in
# sort_indices once.
one_hot_position = tf.cast(tf.equal(sort_indices, 0), tf.int32)
sparse_positions = tf.multiply(
one_hot_position, tf.range(logits_by_user.shape[1])[tf.newaxis, :])
position_vector = tf.reduce_sum(sparse_positions, axis=1)
in_top_k = tf.cast(tf.less(position_vector, rconst.TOP_K), tf.float32)
ndcg = tf.log(2.) / tf.log(tf.cast(position_vector, tf.float32) + 2)
ndcg *= in_top_k
# If a row is a padded row, all but the first element will be a duplicate.
metric_weights = tf.not_equal(tf.reduce_sum(duplicate_mask_by_user, axis=1),
rconst.NUM_EVAL_NEGATIVES)
# Examples are provided by the eval Dataset in a structured format, so eval
# labels can be reconstructed on the fly.
eval_labels = tf.reshape(tf.one_hot(
tf.zeros(shape=(logits_by_user.shape[0],), dtype=tf.int32),
logits_by_user.shape[1], dtype=tf.int32), (-1,))
eval_labels_float = tf.cast(eval_labels, tf.float32)
# During evaluation, the ratio of negatives to positives is much higher
# than during training. (Typically 999 to 1 vs. 4 to 1) By adjusting the
# weights for the negative examples we compute a loss which is consistent with
# the training data. (And provides apples-to-apples comparison)
negative_scale_factor = num_training_neg / rconst.NUM_EVAL_NEGATIVES
example_weights = (
(eval_labels_float + (1 - eval_labels_float) * negative_scale_factor) *
(1 + rconst.NUM_EVAL_NEGATIVES) / (1 + num_training_neg))
# Tile metric weights back to logit dimensions
expanded_metric_weights = tf.reshape(tf.tile(
metric_weights[:, tf.newaxis], (1, rconst.NUM_EVAL_NEGATIVES + 1)), (-1,))
# ignore padded examples
example_weights *= tf.cast(expanded_metric_weights, tf.float32)
cross_entropy = tf.losses.sparse_softmax_cross_entropy(
logits=softmax_logits, labels=eval_labels, weights=example_weights)
def metric_fn(top_k_tensor, ndcg_tensor, weight_tensor):
return {
rconst.HR_KEY: tf.metrics.mean(top_k_tensor, weights=weight_tensor),
rconst.NDCG_KEY: tf.metrics.mean(ndcg_tensor, weights=weight_tensor),
}
if use_tpu:
return tf.contrib.tpu.TPUEstimatorSpec(
mode=tf.estimator.ModeKeys.EVAL, loss=cross_entropy,
eval_metrics=(metric_fn, [in_top_k, ndcg, metric_weights]))
return tf.estimator.EstimatorSpec(
mode=tf.estimator.ModeKeys.EVAL,
loss=cross_entropy,
eval_metric_ops=metric_fn(in_top_k, ndcg, metric_weights)
)
def eval_right_length(example, target):
l = tf.maximum(tf.shape(example['inputs'])[0], tf.shape(target)[0])
return tf.less(l, max_eval_length + 1)
def _continue_search(self, state) -> tf.Tensor:
i = state[decoding_module.StateKeys.CUR_INDEX]
return tf.less(i, self.max_decode_length)
def _relu(self, x, leakiness=0.0):
"""Relu, with optional leaky support."""
return tf.where(tf.less(x, 0.0), leakiness * x, x, name='leaky_relu')
def should_continue(self, threshold) -> bool:
return tf.reduce_any(tf.less(self.halting_probability, threshold))
def add_loss(self):
# Chi Squared
self.chiSq = tf.reduce_mean(
tf.divide(tf.square(self.prediction - self.fitData),
self.fitDataVar))
# Minimize large R contribution
self.rCut = tf.constant(1, tf.float32)
self.rNorm = tf.einsum('nij,jk->nik', tf.abs(self.fitCoeffs),
self.rVals)
self.lowR = tf.reduce_mean(
tf.pow((self.rNorm - self.rCut) / self.rCut,
tf.ones_like(self.rNorm) * 6))
if self.parameters["Rrange"][0] < 1:
zeroLowRcut = int(1. * self.parameters["NfitFxns"] /
float(self.parameters["Rrange"][1] -
self.parameters["Rrange"][0]))
self.zeroLowR = tf.reduce_sum(
tf.abs(self.fitCoeffs[:, :, :zeroLowRcut]))
self.zeroLowR *= 1E4
else:
self.zeroLowR = tf.constant(0, tf.float32)
# Smooth fit coefficient distribution
self.nnDiff = tf.divide(
self.fitCoeffs[:, :, :-1] - self.fitCoeffs[:, :, 1:],
(tf.abs(self.fitCoeffs[:, :, :-1]) +
tf.abs(self.fitCoeffs[:, :, 1:])) / 2 + 1e-7)
self.nnnDiff = tf.divide(
self.fitCoeffs[:, :, :-2] - self.fitCoeffs[:, :, 2:],
(tf.abs(self.fitCoeffs[:, :, :-2]) +
tf.abs(self.fitCoeffs[:, :, 2:])) / 2 + 1e-7)
self.nnVariance = (0.5)**2
self.nnnVariance = (0.7)**2
self.smooth = tf.reduce_mean(1 - tf.exp(-1 * tf.square(self.nnDiff) /
(2 * self.nnVariance)))
#self.smooth += tf.reduce_mean(
# 1 - tf.exp(-1*tf.square(self.nnnDiff)
# /(2*self.nnnVariance)))
self.smooth *= self.chiSq
# Remove negative coefficients
self.nonNeg = tf.reduce_mean(
tf.minimum(tf.zeros_like(self.fitCoeffs), self.fitCoeffs))
self.nonNeg *= -1000
# Dampen coefficients with small values (noise)
self.noise = tf.reduce_mean(tf.where(tf.less(
self.rNorm, 0.04 * tf.ones_like(self.rNorm)),
tf.abs(self.rNorm),
tf.zeros_like(self.rNorm),
name="noise_where"),
name="noise")
self.noise *= tf.sqrt(
tf.sqrt(tf.cast(self.global_step, dtype=tf.float32)))
"""
self.noise = tf.reduce_mean(
tf.square(
tf.where(
tf.less(self.fitCoeffs,
0.01*tf.ones_like(self.fitCoeffs)),
1 + tf.abs(self.fitCoeffs),
tf.zeros_like(self.fitCoeffs))))
"""
"""
self.noise = tf.reduce_mean(
tf.abs(
1./(0.0051 - tf.where(
tf.less(self.fitCoeffs,
0.005*tf.ones_like(self.fitCoeffs)),
tf.abs(self.fitCoeffs),
tf.zeros_like(self.fitCoeffs)))))
self.noise /= 10000
"""
if self.parameters["timeDepLoss"]:
self.tdVariance = 0.5**2
self.tdDiff = tf.divide(
self.fitCoeffs[:-1, :, :] - self.fitCoeffs[1:, :, :],
(tf.abs(self.fitCoeffs[:-1, :, :]) +
tf.abs(self.fitCoeffs[1:, :, :])) / 2 + 1e-7)
self.nnTdDiff = tf.divide(
self.fitCoeffs[:-1, :, :-1] - self.fitCoeffs[1:, :, 1:],
(tf.abs(self.fitCoeffs[:-1, :, :-1]) +
tf.abs(self.fitCoeffs[1:, :, 1:])) / 2 + 1e-7)
self.nnnTdDiff = tf.divide(
self.fitCoeffs[:-1, :, :-2] - self.fitCoeffs[1:, :, 2:],
(tf.abs(self.fitCoeffs[:-1, :, :-2]) +
tf.abs(self.fitCoeffs[1:, :, 2:])) / 2 + 1e-7)
self.smoothTD = tf.reduce_mean(1 -
tf.exp(-1 * tf.square(self.tdDiff) /
(2 * self.tdVariance)))
self.smoothTD += tf.reduce_mean(1 -
tf.exp(-1 *
tf.square(self.nnTdDiff) /
(2 * self.tdVariance)))
self.smoothTD += tf.reduce_mean(1 -
tf.exp(-1 *
tf.square(self.nnnTdDiff) /
(2 * self.tdVariance)))
self.smoothTD *= 0.5 * self.chiSq
else:
self.smoothTD = tf.constant(0, tf.float32)
self.L1 = tf.reduce_mean(tf.abs(self.rNorm))
self.L1NR = tf.reduce_mean(tf.abs(self.fitCoeffs))
self.L2 = tf.reduce_mean(tf.square(self.rNorm))
self.loss = self.chiSq #+ self.lowR + self.zeroLowR
if self.parameters["L1regularize"] is not None:
self.loss += self.parameters["L1regularize"] * self.L1
if self.parameters["L1NRregularize"] is not None:
self.loss += self.parameters["L1NRregularize"] * self.L1NR
if self.parameters["L2regularize"] is not None:
self.loss += self.parameters["L2regularize"] * self.L2
if self.parameters["smoothLoss"] is not None:
self.loss += self.parameters["smoothLoss"] * self.smooth
if self.parameters["nonNegLoss"]:
self.loss += self.nonNeg
if self.parameters["noiseLoss"] is not None:
self.loss += self.parameters["noiseLoss"] * self.noise
if self.parameters["timeDepLoss"]:
self.loss += self.smoothTD
def _rand_erase(image,
mask,
weight,
range_image,
range_mask,
range_weight,
probability=0.5,
min_size=0.02,
max_size=0.4,
min_aspect_ratio=0.3,
max_aspect_ratio=1 / 0.3,
pixel_wise=False,
seed=None):
# Generate seed for reproductivity
if seed is not None:
np.random.seed(seed)
seeds = np.random.randint(np.iinfo(np.int32).min,
np.iinfo(np.int32).max,
size=[5])
seed_size, seed_ratio, seed_left, seed_top, seed_prob = seeds
else:
seed_size, seed_ratio, seed_left, seed_top, seed_prob = [None
] * 5
height, width, channels = image.get_shape().as_list()
num_elems = tf.cast(height, dtype=tf.float32) * tf.cast(
width, dtype=tf.float32)
s = tf.random_uniform(
(), min_size, max_size, seed=seed_size) * num_elems
log_min_asp = tf.log(min_aspect_ratio)
log_max_asp = tf.log(max_aspect_ratio)
r = tf.exp(
tf.random_uniform((),
log_min_asp,
log_max_asp,
seed=seed_ratio))
w = tf.cast(tf.sqrt(s / r), dtype=tf.int32)
h = tf.cast(tf.sqrt(s * r), dtype=tf.int32)
w = tf.reduce_min([width, w])
h = tf.reduce_min([height, h])
left = tf.cond(
tf.equal(w, width), lambda: 0, lambda: tf.random_uniform(
(), 0, width - w, seed=seed_left, dtype=tf.int32))
top = tf.cond(
tf.equal(h, height), lambda: 0, lambda: tf.random_uniform(
(), 0, height - h, seed=seed_top, dtype=tf.int32))
erased_image = _rand_bbox(image,
h,
w,
top,
left,
min_val=range_image[0],
max_val=range_image[1],
pixel_wise=pixel_wise)
erased_mask = _rand_bbox(mask,
h,
w,
top,
left,
min_val=range_mask[0],
max_val=range_mask[1],
pixel_wise=False)
erased_weight = _rand_bbox(weight,
h,
w,
top,
left,
min_val=range_weight[0],
max_val=range_weight[1],
pixel_wise=False)
prob = tf.random_uniform((), seed=seed_prob)
image = tf.cond(tf.less(prob, probability),
true_fn=lambda: erased_image,
false_fn=lambda: image)
mask = tf.cond(tf.less(prob, probability),
true_fn=lambda: erased_mask,
false_fn=lambda: mask)
weight = tf.cond(tf.less(prob, probability),
true_fn=lambda: erased_weight,
false_fn=lambda: weight)
return image, mask, weight
def target_right_length(_, target):
return tf.less(tf.shape(target)[0], max_target_length + 1)
def condition2(id_, outputs_ta2_, states_ta2_,
e_ta_): # , outputs_ta2_, states_ta2_
return tf.less(id_, T_dim * S_dim)
def __init__(self,
n_channel,
n_classes,
image_size,
max_objects_per_image,
cell_size,
box_per_cell,
object_scale,
noobject_scale,
coord_scale,
class_scale,
batch_size,
noobject_thresh=0.6,
recall_thresh=0.5,
pred_thresh=0.5,
nms_thresh=0.4):
# 设置参数
self.n_classes = n_classes
self.image_size = image_size
self.n_channel = n_channel
self.max_objects = max_objects_per_image
self.cell_size = cell_size
self.n_boxes = box_per_cell
self.object_scale = float(object_scale)
self.noobject_scale = float(noobject_scale)
self.coord_scale = float(coord_scale)
self.class_scale = float(class_scale)
self.batch_size = batch_size
self.noobject_thresh = noobject_thresh
self.recall_thresh = recall_thresh
self.pred_thresh = pred_thresh
self.nms_thresh = nms_thresh
# 输入变量
self.images = tf.placeholder(dtype=tf.float32,
shape=[
self.batch_size, self.image_size,
self.image_size, self.n_channel
],
name='images')
self.box_labels = tf.placeholder(
dtype=tf.float32,
shape=[self.batch_size, self.max_objects, 5],
name='box_labels')
self.object_nums = tf.placeholder(dtype=tf.int32,
shape=[
self.batch_size,
],
name='object_nums')
self.keep_prob = tf.placeholder(dtype=tf.float32, name='keep_prob')
self.global_step = tf.Variable(0, dtype=tf.int32, name='global_step')
# 待输出的中间变量
self.logits = self.inference(self.images)
self.coord_loss, self.object_loss, self.noobject_loss, self.class_loss, \
self.iou_value, self.object_value, self.nobject_value, \
self.recall_value, self.class_value = \
self.calculate_loss(self.logits)
# 目标函数和优化器
tf.add_to_collection('losses', self.coord_loss)
tf.add_to_collection('losses', self.object_loss)
tf.add_to_collection('losses', self.noobject_loss)
tf.add_to_collection('losses', self.class_loss)
self.avg_loss = tf.add_n(tf.get_collection('losses'))
# 设置学习率
lr = tf.cond(
tf.less(self.global_step, 100), lambda: tf.constant(0.001),
lambda: tf.cond(
tf.less(self.global_step, 80000), lambda: tf.constant(0.01),
lambda: tf.cond(tf.less(self.global_step, 100000), lambda: tf.
constant(0.001), lambda: tf.constant(0.0001))))
self.optimizer = tf.train.MomentumOptimizer(
learning_rate=0.001,
momentum=0.9).minimize(self.avg_loss, global_step=self.global_step)
def _soft_sigmoid(x):
return tf.where(tf.greater(x, 1.), x*0.0001,
tf.where(tf.less(x, 0.), x*0.0001, x))
def range_prelu(x, upper_limit=4., outer_slope=0.0001):
'''Quatery ReLU (0,4)
'''
return tf.where(tf.logical_or(tf.greater(x, upper_limit), tf.less(x, 0.)), x*outer_slope, x)
def set_constants(self, constantArray, enzymeInit, activInitC, inhibInitC,
computedRescaleFactor):
"""
Define the ops assigning the values for the network constants.
:return:
"""
self.rescaleFactor.assign(computedRescaleFactor)
enzymeInitTensor = enzymeInit * (computedRescaleFactor**0.5)
self.k1.assign(
tf.cast(tf.fill(self.k1.shape, constantArray[0]),
dtype=tf.float64))
self.k1n.assign(
tf.cast(tf.fill(self.k1n.shape, constantArray[1]),
dtype=tf.float64))
self.k2.assign(
tf.cast(tf.fill(self.k2.shape, constantArray[2]),
dtype=tf.float64))
self.k3.assign(
tf.cast(tf.fill(self.k3.shape, constantArray[3]),
dtype=tf.float64))
self.k3n.assign(
tf.cast(tf.fill(self.k3n.shape, constantArray[4]),
dtype=tf.float64))
self.k4.assign(
tf.cast(tf.fill(self.k4.shape, constantArray[5]),
dtype=tf.float64))
self.k5.assign(
tf.cast(tf.fill(self.k5.shape, constantArray[6]),
dtype=tf.float64))
self.k5n.assign(
tf.cast(tf.fill(self.k5n.shape, constantArray[7]),
dtype=tf.float64))
self.k6.assign(
tf.cast(tf.fill(self.k6.shape, constantArray[8]),
dtype=tf.float64))
self.kdI.assign(
tf.cast(tf.fill(self.kdI.shape, constantArray[9]),
dtype=tf.float64))
self.kdT.assign(
tf.cast(tf.fill(self.kdT.shape, constantArray[10]),
dtype=tf.float64))
self.TA0.assign(
tf.cast(tf.fill(self.TA0.shape, activInitC), dtype=tf.float64))
self.TI0.assign(
tf.cast(tf.fill(self.TI0.shape, inhibInitC), dtype=tf.float64))
self.E0.assign(tf.constant(enzymeInitTensor, dtype=tf.float64))
#used in the first layer:
self.firstLayerTA0.assign(
tf.cast(tf.fill(self.firstLayerTA0.shape, activInitC),
dtype=tf.float64))
self.firstLayerK1M.assign(
tf.cast(tf.fill(
self.firstLayerK1M.shape,
constantArray[0] / (constantArray[1] + constantArray[2])),
dtype=tf.float64))
self.firstLayerkdT.assign(
tf.cast(tf.fill(self.firstLayerkdT.shape, constantArray[10]),
dtype=tf.float64))
self.firstLayerk2.assign(
tf.cast(tf.fill(self.firstLayerk2.shape, constantArray[2]),
dtype=tf.float64))
#intermediate values for faster computations:
self.k1M.assign(self.k1 / (self.k1n + self.k2))
self.Cactiv.assign(self.k2 * self.k1M * self.E0 * self.TA0)
self.k5M.assign(self.k5 / (self.k5n + self.k6))
self.k3M.assign(self.k3 / (self.k3n + self.k4))
self.Cinhib.assign(
tf.stack([self.k6 * self.k5M] * (self.k4.shape[0]), axis=0) *
self.k4 * self.k3M * self.E0 * self.E0 * self.TI0)
self.Kactiv.assign(self.k1M * self.TA0)
self.Kinhib.assign(self.k3M * self.TI0)
self.mask.assign(
tf.cast(tf.where(tf.less(self.kernel, -0.2), -1.,
tf.where(tf.less(0.2, self.kernel), 1., 0.)),
dtype=tf.float64))
self.cstList = [
self.k1, self.k1n, self.k2, self.k3, self.k3n, self.k4, self.k5,
self.k5n, self.k6, self.kdI, self.kdT, self.TA0, self.E0, self.k1M,
self.Cactiv, self.Cinhib, self.Kactiv, self.Kinhib, self.k5M,
self.k3M, self.firstLayerTA0, self.firstLayerK1M,
self.firstLayerkdT, self.firstLayerk2
]
self.cstListName = [
"self.k1", "self.k1n", "self.k2", "self.k3", "self.k3n", "self.k4",
"self.k5", "self.k5n", "self.k6", "self.kdI", "self.kdT",
"self.TA0", "self.E0", "self.k1M", "self.Cactiv", "self.Cinhib",
"self.Kactiv", "self.Kinhib", "self.k5M", "self.k3M",
"self.firstLayerTA0", "self.firstLayerK1M", "self.firstLayerkdT",
"self.firstLayerk2"
]
def replace_with_zero_values_below_threshold(x_row):
less_than_threshold = tf.less(x_row, threshold)
return tf.where(less_than_threshold,
tf.zeros((features_cnt, ), dtype=tf.float64),
x_row)
def do_dual_max_match(overlap_matrix,
low_thres,
high_thres,
ignore_between=True,
gt_max_first=True):
# overlap_matrix: num_gt * num_anchors
with tf.name_scope('dual_max_match', [overlap_matrix]):
anchors_to_gt = tf.argmax(overlap_matrix, axis=0) # anchor移动时第一个匹配
match_values = tf.reduce_max(overlap_matrix, axis=0) # 匹配的程度
less_mask = tf.less(match_values, low_thres)
between_mask = tf.logical_and(
tf.less(match_values, high_thres),
tf.greater_equal(match_values, low_thres))
negative_mask = less_mask if ignore_between else between_mask
ignore_mask = between_mask if ignore_between else less_mask
# 负样本位置为-1,所有忽略的样本为-2
match_indices = tf.where(negative_mask,
-1 * tf.ones_like(anchors_to_gt),
anchors_to_gt)
match_indices = tf.where(ignore_mask, -2 * tf.ones_like(match_indices),
match_indices)
anchors_to_gt_mask = tf.one_hot(tf.clip_by_value(
match_indices, -1, tf.cast(tf.shape(overlap_matrix)[0], tf.int64)),
tf.shape(overlap_matrix)[0],
on_value=1,
off_value=0,
axis=0,
dtype=tf.int32)
gt_to_anchors = tf.argmax(overlap_matrix, axis=1) # 真实数据的移动匹配
if gt_max_first:
# the max match from ground truth's side has higher priority
left_gt_to_anchors_mask = tf.one_hot(gt_to_anchors,
tf.shape(overlap_matrix)[1],
on_value=1,
off_value=0,
axis=1,
dtype=tf.int32)
else:
# the max match from anchors' side has higher priority
# use match result from ground truth's side only when the the matching degree from anchors' side is lower than position threshold
left_gt_to_anchors_mask = tf.cast(
tf.logical_and(
tf.reduce_max(anchors_to_gt_mask, axis=1, keep_dims=True) <
1,
tf.one_hot(gt_to_anchors,
tf.shape(overlap_matrix)[1],
on_value=True,
off_value=False,
axis=1,
dtype=tf.bool)), tf.int64)
# can not use left_gt_to_anchors_mask here, because there are many ground truthes match to one anchor, we should pick the highest one even when we are merging matching from ground truth side
left_gt_to_anchors_scores = overlap_matrix * tf.to_float(
left_gt_to_anchors_mask)
# merge matching results from ground truth's side with the original matching results from anchors' side
# then select all the overlap score of those matching pairs
selected_scores = tf.gather_nd(
overlap_matrix,
tf.stack([
tf.where(
tf.reduce_max(left_gt_to_anchors_mask, axis=0) > 0,
tf.argmax(left_gt_to_anchors_scores, axis=0),
anchors_to_gt),
tf.range(tf.cast(tf.shape(overlap_matrix)[1], tf.int64))
],
axis=1))
# return the matching results for both foreground anchors and background anchors, also with overlap scores
return tf.where(
tf.reduce_max(left_gt_to_anchors_mask, axis=0) > 0,
tf.argmax(left_gt_to_anchors_scores, axis=0),
match_indices), selected_scores
def set_mask(self, mask):
Tminus = tf.cast(tf.fill(self.kernel.shape, -1), dtype=tf.float64)
Tplus = tf.cast(tf.fill(self.kernel.shape, 1), dtype=tf.float64)
Tzero = tf.cast(tf.fill(self.kernel.shape, 0), dtype=tf.float64)
clippedKernel = tf.where(
tf.less(self.kernel, -0.2), Tminus,
tf.where(tf.less(0.2, self.kernel), Tplus, Tzero))
newClippedKernel = tf.where(
mask * clippedKernel > 0., clippedKernel,
tf.where(mask * clippedKernel < 0., (-1) * clippedKernel, mask))
newKernel = tf.where(
tf.less(self.kernel, -0.2),
tf.where(
tf.less(newClippedKernel, 0.), self.kernel,
tf.where(tf.less(0., newClippedKernel), (-1) * self.kernel,
0.)), # same or symetric
tf.where(
tf.less(0.2, self.kernel),
tf.where(
tf.less(0., newClippedKernel), self.kernel,
tf.where(tf.less(newClippedKernel, 0.), (-1) * self.kernel,
0.)),
tf.where(
tf.less(newClippedKernel, 0.), -1.,
tf.where(tf.less(0, newClippedKernel), 1., self.kernel))))
self.kernel.assign(newKernel)
self.mask.assign(
tf.cast(tf.where(tf.less(self.kernel, -0.2), -1.,
tf.where(tf.less(0.2, self.kernel), 1., 0.)),
dtype=tf.float64))
def __init__(self,
nmaps,
vec_size,
niclass,
noclass,
dropout,
max_grad_norm,
cutoff,
nconvs,
kw,
kh,
height,
mem_size,
learning_rate,
min_length,
num_gpus,
num_replicas,
grad_noise_scale,
sampling_rate,
act_noise=0.0,
do_rnn=False,
atrous=False,
beam_size=1,
backward=True,
do_layer_norm=False,
autoenc_decay=1.0):
# Feeds for parameters and ops to update them.
self.nmaps = nmaps
if backward:
self.global_step = tf.Variable(0,
trainable=False,
name="global_step")
self.cur_length = tf.Variable(min_length, trainable=False)
self.cur_length_incr_op = self.cur_length.assign_add(1)
self.lr = tf.Variable(learning_rate, trainable=False)
self.lr_decay_op = self.lr.assign(self.lr * 0.995)
self.do_training = tf.placeholder(tf.float32, name="do_training")
self.update_mem = tf.placeholder(tf.int32, name="update_mem")
self.noise_param = tf.placeholder(tf.float32, name="noise_param")
# Feeds for inputs, targets, outputs, losses, etc.
self.input = tf.placeholder(tf.int32, name="inp")
self.target = tf.placeholder(tf.int32, name="tgt")
self.prev_step = tf.placeholder(tf.float32, name="prev_step")
gpu_input = tf.split(axis=0,
num_or_size_splits=num_gpus,
value=self.input)
gpu_target = tf.split(axis=0,
num_or_size_splits=num_gpus,
value=self.target)
gpu_prev_step = tf.split(axis=0,
num_or_size_splits=num_gpus,
value=self.prev_step)
batch_size = tf.shape(gpu_input[0])[0]
if backward:
adam_lr = 0.005 * self.lr
adam = tf.train.AdamOptimizer(adam_lr, epsilon=1e-3)
def adam_update(grads):
return adam.apply_gradients(zip(grads,
tf.trainable_variables()),
global_step=self.global_step,
name="adam_update")
# When switching from Adam to SGD we perform reverse-decay.
if backward:
global_step_float = tf.cast(self.global_step, tf.float32)
sampling_decay_exponent = global_step_float / 100000.0
sampling_decay = tf.maximum(0.05,
tf.pow(0.5, sampling_decay_exponent))
self.sampling = sampling_rate * 0.05 / sampling_decay
else:
self.sampling = tf.constant(0.0)
# Cache variables on cpu if needed.
if num_replicas > 1 or num_gpus > 1:
with tf.device("/cpu:0"):
caching_const = tf.constant(0)
tf.get_variable_scope().set_caching_device(caching_const.op.device)
# partitioner = tf.variable_axis_size_partitioner(1024*256*4)
# tf.get_variable_scope().set_partitioner(partitioner)
def gpu_avg(l):
if l[0] is None:
for elem in l:
assert elem is None
return 0.0
if len(l) < 2:
return l[0]
return sum(l) / float(num_gpus)
self.length_tensor = tf.placeholder(tf.int32, name="length")
with tf.device("/cpu:0"):
emb_weights = tf.get_variable(
"embedding", [niclass, vec_size],
initializer=tf.random_uniform_initializer(-1.7, 1.7))
if beam_size > 0:
target_emb_weights = tf.get_variable(
"target_embedding", [noclass, nmaps],
initializer=tf.random_uniform_initializer(-1.7, 1.7))
e0 = tf.scatter_update(emb_weights,
tf.constant(0, dtype=tf.int32, shape=[1]),
tf.zeros([1, vec_size]))
output_w = tf.get_variable("output_w", [nmaps, noclass],
tf.float32)
def conv_rate(layer):
if atrous:
return 2**layer
return 1
# pylint: disable=cell-var-from-loop
def enc_step(step):
"""Encoder step."""
if autoenc_decay < 1.0:
quant_step = autoenc_quantize(step, 16, nmaps,
self.do_training)
if backward:
exp_glob = tf.train.exponential_decay(
1.0, self.global_step - 10000, 1000, autoenc_decay)
dec_factor = 1.0 - exp_glob # * self.do_training
dec_factor = tf.cond(tf.less(self.global_step, 10500),
lambda: tf.constant(0.05),
lambda: dec_factor)
else:
dec_factor = 1.0
cur = tf.cond(tf.less(tf.random_uniform([]), dec_factor),
lambda: quant_step, lambda: step)
else:
cur = step
if dropout > 0.0001:
cur = tf.nn.dropout(cur, keep_prob)
if act_noise > 0.00001:
cur += tf.truncated_normal(tf.shape(cur)) * act_noise_scale
# Do nconvs-many CGRU steps.
if do_jit and tf.get_variable_scope().reuse:
with jit_scope():
for layer in xrange(nconvs):
cur = conv_gru([], cur, kw, kh, nmaps,
conv_rate(layer), cutoff,
"ecgru_%d" % layer, do_layer_norm)
else:
for layer in xrange(nconvs):
cur = conv_gru([], cur, kw, kh, nmaps, conv_rate(layer),
cutoff, "ecgru_%d" % layer, do_layer_norm)
return cur
zero_tgt = tf.zeros([batch_size, nmaps, 1])
zero_tgt.set_shape([None, nmaps, 1])
def dec_substep(step, decided):
"""Decoder sub-step."""
cur = step
if dropout > 0.0001:
cur = tf.nn.dropout(cur, keep_prob)
if act_noise > 0.00001:
cur += tf.truncated_normal(tf.shape(cur)) * act_noise_scale
# Do nconvs-many CGRU steps.
if do_jit and tf.get_variable_scope().reuse:
with jit_scope():
for layer in xrange(nconvs):
cur = conv_gru([decided], cur, kw, kh, nmaps,
conv_rate(layer), cutoff,
"dcgru_%d" % layer, do_layer_norm)
else:
for layer in xrange(nconvs):
cur = conv_gru([decided], cur, kw, kh, nmaps,
conv_rate(layer), cutoff,
"dcgru_%d" % layer, do_layer_norm)
return cur
# pylint: enable=cell-var-from-loop
def dec_step(step, it, it_int, decided, output_ta, tgts, mloss,
nupd_in, out_idx, beam_cost):
"""Decoder step."""
nupd, mem_loss = 0, 0.0
if mem_size > 0:
it_incr = tf.minimum(it + 1, length - 1)
mem, mem_loss, nupd = memory_run(
step, nmaps, mem_size, batch_size, noclass,
self.global_step, self.do_training, self.update_mem, 10,
num_gpus, target_emb_weights, output_w, gpu_targets_tn,
it_incr)
step = dec_substep(step, decided)
output_l = tf.expand_dims(tf.expand_dims(step[:, it, 0, :], 1), 1)
# Calculate argmax output.
output = tf.reshape(output_l, [-1, nmaps])
# pylint: disable=cell-var-from-loop
output = tf.matmul(output, output_w)
if beam_size > 1:
beam_cost, output, out, reordered = reorder_beam(
beam_size, batch_size, beam_cost, output, it_int == 0,
[output_l, out_idx, step, decided])
[output_l, out_idx, step, decided] = reordered
else:
# Scheduled sampling.
out = tf.multinomial(tf.stop_gradient(output), 1)
out = tf.to_int32(tf.squeeze(out, [1]))
out_write = output_ta.write(it, output_l[:batch_size, :, :, :])
output = tf.gather(target_emb_weights, out)
output = tf.reshape(output, [-1, 1, nmaps])
output = tf.concat(axis=1, values=[output] * height)
tgt = tgts[it, :, :, :]
selected = tf.cond(tf.less(tf.random_uniform([]), self.sampling),
lambda: output, lambda: tgt)
# pylint: enable=cell-var-from-loop
dec_write = place_at14(decided, tf.expand_dims(selected, 1), it)
out_idx = place_at13(
out_idx, tf.reshape(out, [beam_size * batch_size, 1, 1]), it)
if mem_size > 0:
mem = tf.concat(axis=2, values=[mem] * height)
dec_write = place_at14(dec_write, mem, it_incr)
return (step, dec_write, out_write, mloss + mem_loss,
nupd_in + nupd, out_idx, beam_cost)
# Main model construction.
gpu_outputs = []
gpu_losses = []
gpu_grad_norms = []
grads_list = []
gpu_out_idx = []
self.after_enc_step = []
for gpu in xrange(
num_gpus): # Multi-GPU towers, average gradients later.
length = self.length_tensor
length_float = tf.cast(length, tf.float32)
if gpu > 0:
tf.get_variable_scope().reuse_variables()
gpu_outputs.append([])
gpu_losses.append([])
gpu_grad_norms.append([])
with tf.name_scope("gpu%d" % gpu), tf.device("/gpu:%d" % gpu):
# Main graph creation loop.
data.print_out("Creating model.")
start_time = time.time()
# Embed inputs and calculate mask.
with tf.device("/cpu:0"):
tgt_shape = tf.shape(tf.squeeze(gpu_target[gpu], [1]))
weights = tf.where(
tf.squeeze(gpu_target[gpu], [1]) > 0,
tf.ones(tgt_shape), tf.zeros(tgt_shape))
# Embed inputs and targets.
with tf.control_dependencies([e0]):
start = tf.gather(emb_weights,
gpu_input[gpu]) # b x h x l x nmaps
gpu_targets_tn = gpu_target[gpu] # b x 1 x len
if beam_size > 0:
embedded_targets_tn = tf.gather(
target_emb_weights, gpu_targets_tn)
embedded_targets_tn = tf.transpose(
embedded_targets_tn,
[2, 0, 1, 3]) # len x b x 1 x nmaps
embedded_targets_tn = tf.concat(
axis=2, values=[embedded_targets_tn] * height)
# First image comes from start by applying convolution and adding 0s.
start = tf.transpose(start,
[0, 2, 1, 3]) # Now b x len x h x vec_s
first = conv_linear(start, 1, 1, vec_size, nmaps, 1, True, 0.0,
"input")
first = layer_norm(first, nmaps, "input")
# Computation steps.
keep_prob = dropout * 3.0 / tf.sqrt(length_float)
keep_prob = 1.0 - self.do_training * keep_prob
act_noise_scale = act_noise * self.do_training
# Start with a convolutional gate merging previous step.
step = conv_gru([gpu_prev_step[gpu]], first, kw, kh, nmaps, 1,
cutoff, "first", do_layer_norm)
# This is just for running a baseline RNN seq2seq model.
if do_rnn:
self.after_enc_step.append(
step) # Not meaningful here, but needed.
def lstm_cell():
return tf.contrib.rnn.BasicLSTMCell(height * nmaps)
cell = tf.contrib.rnn.MultiRNNCell(
[lstm_cell() for _ in range(nconvs)])
with tf.variable_scope("encoder"):
encoder_outputs, encoder_state = tf.nn.dynamic_rnn(
cell,
tf.reshape(step,
[batch_size, length, height * nmaps]),
dtype=tf.float32,
time_major=False)
# Attention.
attn = tf.layers.dense(encoder_outputs,
height * nmaps,
name="attn1")
# pylint: disable=cell-var-from-loop
@function.Defun(noinline=True)
def attention_query(query, attn_v):
vecs = tf.tanh(attn + tf.expand_dims(query, 1))
mask = tf.reduce_sum(
vecs * tf.reshape(attn_v, [1, 1, -1]), 2)
mask = tf.nn.softmax(mask)
return tf.reduce_sum(
encoder_outputs * tf.expand_dims(mask, 2), 1)
with tf.variable_scope("decoder"):
def decoder_loop_fn(state__prev_cell_out__unused,
cell_inp__cur_tgt):
"""Decoder loop function."""
state, prev_cell_out, _ = state__prev_cell_out__unused
cell_inp, cur_tgt = cell_inp__cur_tgt
attn_q = tf.layers.dense(prev_cell_out,
height * nmaps,
name="attn_query")
attn_res = attention_query(
attn_q,
tf.get_variable(
"attn_v", [height * nmaps],
initializer=tf.random_uniform_initializer(
-0.1, 0.1)))
concatenated = tf.reshape(
tf.concat(axis=1, values=[cell_inp, attn_res]),
[batch_size, 2 * height * nmaps])
cell_inp = tf.layers.dense(concatenated,
height * nmaps,
name="attn_merge")
output, new_state = cell(cell_inp, state)
mem_loss = 0.0
if mem_size > 0:
res, mask, mem_loss = memory_call(
output, cur_tgt, height * nmaps, mem_size,
noclass, num_gpus, self.update_mem)
res = tf.gather(target_emb_weights, res)
res *= tf.expand_dims(mask[:, 0], 1)
output = tf.layers.dense(tf.concat(
axis=1, values=[output, res]),
height * nmaps,
name="rnnmem")
return new_state, output, mem_loss
# pylint: enable=cell-var-from-loop
gpu_targets = tf.squeeze(gpu_target[gpu],
[1]) # b x len
gpu_tgt_trans = tf.transpose(gpu_targets, [1, 0])
dec_zero = tf.zeros([batch_size, 1], dtype=tf.int32)
dec_inp = tf.concat(axis=1,
values=[dec_zero, gpu_targets])
dec_inp = dec_inp[:, :length]
embedded_dec_inp = tf.gather(target_emb_weights,
dec_inp)
embedded_dec_inp_proj = tf.layers.dense(
embedded_dec_inp, height * nmaps, name="dec_proj")
embedded_dec_inp_proj = tf.transpose(
embedded_dec_inp_proj, [1, 0, 2])
init_vals = (encoder_state,
tf.zeros([batch_size,
height * nmaps]), 0.0)
_, dec_outputs, mem_losses = tf.scan(
decoder_loop_fn,
(embedded_dec_inp_proj, gpu_tgt_trans),
initializer=init_vals)
mem_loss = tf.reduce_mean(mem_losses)
outputs = tf.layers.dense(dec_outputs,
nmaps,
name="out_proj")
# Final convolution to get logits, list outputs.
outputs = tf.matmul(tf.reshape(outputs, [-1, nmaps]),
output_w)
outputs = tf.reshape(outputs,
[length, batch_size, noclass])
gpu_out_idx.append(tf.argmax(outputs, 2))
else: # Here we go with the Neural GPU.
# Encoder.
enc_length = length
step = enc_step(step) # First step hard-coded.
# pylint: disable=cell-var-from-loop
i = tf.constant(1)
c = lambda i, _s: tf.less(i, enc_length)
def enc_step_lambda(i, step):
with tf.variable_scope(tf.get_variable_scope(),
reuse=True):
new_step = enc_step(step)
return (i + 1, new_step)
_, step = tf.while_loop(c,
enc_step_lambda, [i, step],
parallel_iterations=1,
swap_memory=True)
# pylint: enable=cell-var-from-loop
self.after_enc_step.append(step)
# Decoder.
if beam_size > 0:
output_ta = tf.TensorArray(dtype=tf.float32,
size=length,
dynamic_size=False,
infer_shape=False,
name="outputs")
out_idx = tf.zeros([beam_size * batch_size, length, 1],
dtype=tf.int32)
decided_t = tf.zeros(
[beam_size * batch_size, length, height, vec_size])
# Prepare for beam search.
tgts = tf.concat(axis=1,
values=[embedded_targets_tn] *
beam_size)
beam_cost = tf.zeros([batch_size, beam_size])
step = tf.concat(axis=0, values=[step] * beam_size)
# First step hard-coded.
step, decided_t, output_ta, mem_loss, nupd, oi, bc = dec_step(
step, 0, 0, decided_t, output_ta, tgts, 0.0, 0,
out_idx, beam_cost)
tf.get_variable_scope().reuse_variables()
# pylint: disable=cell-var-from-loop
def step_lambda(i, step, dec_t, out_ta, ml, nu, oi,
bc):
with tf.variable_scope(tf.get_variable_scope(),
reuse=True):
s, d, t, nml, nu, oi, bc = dec_step(
step, i, 1, dec_t, out_ta, tgts, ml, nu,
oi, bc)
return (i + 1, s, d, t, nml, nu, oi, bc)
i = tf.constant(1)
c = lambda i, _s, _d, _o, _ml, _nu, _oi, _bc: tf.less(
i, length)
_, step, _, output_ta, mem_loss, nupd, out_idx, _ = tf.while_loop(
c,
step_lambda, [
i, step, decided_t, output_ta, mem_loss, nupd,
oi, bc
],
parallel_iterations=1,
swap_memory=True)
# pylint: enable=cell-var-from-loop
gpu_out_idx.append(tf.squeeze(out_idx, [2]))
outputs = output_ta.stack()
outputs = tf.squeeze(outputs,
[2, 3]) # Now l x b x nmaps
else:
# If beam_size is 0 or less, we don't have a decoder.
mem_loss = 0.0
outputs = tf.transpose(step[:, :, 1, :], [1, 0, 2])
gpu_out_idx.append(tf.argmax(outputs, 2))
# Final convolution to get logits, list outputs.
outputs = tf.matmul(tf.reshape(outputs, [-1, nmaps]),
output_w)
outputs = tf.reshape(outputs,
[length, batch_size, noclass])
gpu_outputs[gpu] = tf.nn.softmax(outputs)
# Calculate cross-entropy loss and normalize it.
targets_soft = make_dense(tf.squeeze(gpu_target[gpu], [1]),
noclass, 0.1)
targets_soft = tf.reshape(targets_soft, [-1, noclass])
targets_hard = make_dense(tf.squeeze(gpu_target[gpu], [1]),
noclass, 0.0)
targets_hard = tf.reshape(targets_hard, [-1, noclass])
output = tf.transpose(outputs, [1, 0, 2])
xent_soft = tf.reshape(
tf.nn.softmax_cross_entropy_with_logits(
logits=tf.reshape(output, [-1, noclass]),
labels=targets_soft), [batch_size, length])
xent_hard = tf.reshape(
tf.nn.softmax_cross_entropy_with_logits(
logits=tf.reshape(output, [-1, noclass]),
labels=targets_hard), [batch_size, length])
low, high = 0.1 / float(noclass - 1), 0.9
const = high * tf.log(high) + float(noclass -
1) * low * tf.log(low)
weight_sum = tf.reduce_sum(weights) + 1e-20
true_perp = tf.reduce_sum(xent_hard * weights) / weight_sum
soft_loss = tf.reduce_sum(xent_soft * weights) / weight_sum
perp_loss = soft_loss + const
# Final loss: cross-entropy + shared parameter relaxation part + extra.
mem_loss = 0.5 * tf.reduce_mean(mem_loss) / length_float
total_loss = perp_loss + mem_loss
gpu_losses[gpu].append(true_perp)
# Gradients.
if backward:
data.print_out("Creating backward pass for the model.")
grads = tf.gradients(total_loss,
tf.trainable_variables(),
colocate_gradients_with_ops=True)
for g_i, g in enumerate(grads):
if isinstance(g, tf.IndexedSlices):
grads[g_i] = tf.convert_to_tensor(g)
grads, norm = tf.clip_by_global_norm(grads, max_grad_norm)
gpu_grad_norms[gpu].append(norm)
for g in grads:
if grad_noise_scale > 0.001:
g += tf.truncated_normal(
tf.shape(g)) * self.noise_param
grads_list.append(grads)
else:
gpu_grad_norms[gpu].append(0.0)
data.print_out("Created model for gpu %d in %.2f s." %
(gpu, time.time() - start_time))
self.updates = []
self.after_enc_step = tf.concat(
axis=0, values=self.after_enc_step) # Concat GPUs.
if backward:
tf.get_variable_scope()._reuse = False
tf.get_variable_scope().set_caching_device(None)
grads = [
gpu_avg([grads_list[g][i] for g in xrange(num_gpus)])
for i in xrange(len(grads_list[0]))
]
update = adam_update(grads)
self.updates.append(update)
else:
self.updates.append(tf.no_op())
self.losses = [
gpu_avg([gpu_losses[g][i] for g in xrange(num_gpus)])
for i in xrange(len(gpu_losses[0]))
]
self.out_idx = tf.concat(axis=0, values=gpu_out_idx)
self.grad_norms = [
gpu_avg([gpu_grad_norms[g][i] for g in xrange(num_gpus)])
for i in xrange(len(gpu_grad_norms[0]))
]
self.outputs = [
tf.concat(axis=1,
values=[gpu_outputs[g] for g in xrange(num_gpus)])
]
self.quantize_op = quantize_weights_op(512, 8)
if backward:
self.saver = tf.train.Saver(tf.global_variables(), max_to_keep=10)
def false_fn():
return tf.cast(tf.less(x[0], self._accuracy_threshold),
tf.int32)
def body(depth_index, initial_state0, initial_state1, initial_state2,
initial_state3, initial_state4, depth_image, max_prob_image,
exp_sum, incre):
"""Loop body."""
# calculate cost
ave_feature = ref_feature
ave_feature2 = tf.square(ref_feature)
warped_view_volumes = tf.zeros([batch_size, height, width, 1])
weight_sum = tf.zeros([batch_size, height, width, 1])
for view in range(0, FLAGS.view_num - 1):
homographies = view_homographies[view]
homographies = tf.transpose(homographies, perm=[1, 0, 2, 3])
homography = homographies[depth_index]
view_feature = tf.squeeze(
tf.slice(view_features, [0, view, 0, 0, 0],
[-1, 1, -1, -1, -1]), 1)
warped_view_feature = tf_transform_homography(
view_feature, homography)
warped_view_volume = tf.square(warped_view_feature - ref_feature)
weight = gateNet(warped_view_volume, 32, name='gate')
warped_view_volumes += (weight + 1) * warped_view_volume
weight_sum += (weight + 1)
cost = warped_view_volumes / weight_sum
with tf.name_scope('cost_volume_homography') as scope:
with tf.variable_scope("rnn/", reuse=tf.AUTO_REUSE):
cost0, initial_state0 = cell0(cost, state=initial_state0)
cost1 = tf.nn.max_pool2d(cost0, (2, 2), 2, 'SAME')
cost1, initial_state1 = cell1(cost1, state=initial_state1)
cost2 = tf.nn.max_pool2d(cost1, (2, 2), 2, 'SAME')
cost2, initial_state2 = cell2(cost2, state=initial_state2)
cost2 = deconv_gn(cost2,
16,
3,
padding='same',
strides=2,
reuse=tf.AUTO_REUSE,
name='cost_upconv0')
cost2 = tf.concat([cost2, cost1], -1)
cost3, initial_state3 = cell3(cost2, state=initial_state3)
cost3 = deconv_gn(cost3,
16,
3,
padding='same',
strides=2,
reuse=tf.AUTO_REUSE,
name='cost_upconv1')
cost3 = tf.concat([cost3, cost0], -1)
cost4, initial_state4 = cell4(cost3, state=initial_state4)
cost = tf.layers.conv2d(cost4,
1,
3,
padding='same',
reuse=tf.AUTO_REUSE,
name='prob_conv')
prob = tf.exp(-cost)
# index
d_idx = tf.cast(depth_index, tf.float32)
if inverse_depth:
inv_depth_start = tf.div(1.0, depth_start)
inv_depth_end = tf.div(1.0, depth_end)
inv_interval = (inv_depth_start - inv_depth_end) / (
tf.cast(depth_num, 'float32') - 1)
inv_depth = inv_depth_start - d_idx * inv_interval
depth = tf.div(1.0, inv_depth)
else:
depth = depth_start + d_idx * depth_interval
temp_depth_image = tf.reshape(depth, [FLAGS.batch_size, 1, 1, 1])
temp_depth_image = tf.tile(temp_depth_image,
[1, feature_shape[1], feature_shape[2], 1])
# update the best
update_flag_image = tf.cast(tf.less(max_prob_image, prob),
dtype='float32')
new_max_prob_image = update_flag_image * prob + (
1 - update_flag_image) * max_prob_image
new_depth_image = update_flag_image * temp_depth_image + (
1 - update_flag_image) * depth_image
max_prob_image = tf.assign(max_prob_image, new_max_prob_image)
depth_image = tf.assign(depth_image, new_depth_image)
# update counter
exp_sum = tf.assign_add(exp_sum, prob)
depth_index = tf.add(depth_index, incre)
return depth_index, initial_state0, initial_state1, initial_state2, initial_state3, initial_state4, depth_image, max_prob_image, exp_sum, incre
def cond(batch_i, point_i, b_inds):
return tf.less(batch_i, tf.shape(stacks_len)[0])
def train_parse_prepare_preprocess_mapillary(example, params):
_TRAIN_SIZE = (params.height_feature_extractor, params.width_feature_extractor)
# example: a serialized tf example
# proimage: 3D, tf.float32, _TRAIN_SIZE
# prolabel: 2D, tf.int32, _TRAIN_SIZE
## parse
keys_to_features = {
'image/encoded': tf.FixedLenFeature((), tf.string, default_value=None),
'image/format': tf.FixedLenFeature((), tf.string, default_value=b'jpeg'),
'image/dtype': tf.FixedLenFeature((), tf.string, default_value=b'uint8'),
'image/shape': tf.FixedLenFeature((3), tf.int64, default_value=None),
'image/path': tf.FixedLenFeature((), tf.string, default_value=None),
'label/encoded': tf.FixedLenFeature((), tf.string, default_value=None),
'label/format': tf.FixedLenFeature((), tf.string, default_value=b'png'),
'label/dtype': tf.FixedLenFeature((), tf.string, default_value=b'uint8'),
'label/shape': tf.FixedLenFeature((3), tf.int64, default_value=None),
'label/path': tf.FixedLenFeature((), tf.string, default_value=None)}
features = tf.parse_single_example(example, keys_to_features)
image = tf.image.decode_jpeg(features['image/encoded'])
label = tf.image.decode_png(features['label/encoded'])
# label = label[..., 0]
im_path = features['image/path']
la_path = features['label/path']
## prepare
# TF internal bug: resize_images semantics don't correspond to uint8 images
# it needs an input of [0,1] to work properly
image = tf.image.convert_image_dtype(image, dtype=tf.float32)
# label = tf.gather(tf.to_float(lids2cids), tf.to_int32(label))
label = tf.gather(tf.cast(params.training_lids2cids_mapil, tf.uint8), tf.to_int32(label))
print('debug: prepare:', image, label)
## preprocess
## rectify sizes before random cropping
# make the smaller dimension at least as large as the respective train dimension
def true_fn(image, label):
# the size which is smaller than the respective average size will have bigger scale
scales = _TRAIN_SIZE / tf.shape(image)[:2]
new_size = tf.to_int32(tf.reduce_max(scales) * tf.saturate_cast(tf.shape(image)[:2], tf.float64))
# + 1 to new_size just in case to_int32 does floor rounding
image = tf.image.resize_images(image, new_size + 1)
# image has to be 3D
label = tf.image.resize_images(label, new_size + 1,
method=tf.image.ResizeMethod.NEAREST_NEIGHBOR)
return image, label
def false_fn(image, label):
im_spatial_shape = tf.shape(image)[:2]
mult = tf.constant(_TRAIN_SIZE)[tf.argmin(im_spatial_shape)] / tf.reduce_min(im_spatial_shape)
new_spatial_size = tf.to_int32(tf.saturate_cast(im_spatial_shape, tf.float64) * mult)
# + 1 to new_size just in case to_int32 does floor rounding
image = tf.image.resize_images(image, new_spatial_size + 1)
# image has to be 3D
label = tf.image.resize_images(label, new_spatial_size + 1,
method=tf.image.ResizeMethod.NEAREST_NEIGHBOR)
return image, label
# if any dim is smaller than the respective _TRAIN_SIZE rectify it
image, label = tf.cond(tf.reduce_any(tf.less(tf.shape(image)[:2], _TRAIN_SIZE)),
true_fn=lambda: true_fn(image, label),
false_fn=lambda: false_fn(image, label))
# one more time same cond since false_fn can create smaller images
image, label = tf.cond(tf.reduce_any(tf.less(tf.shape(image)[:2], _TRAIN_SIZE)),
true_fn=lambda: true_fn(image, label),
false_fn=lambda: (image, label))
# random crop
# trick to randomly crop the same area from image and label
# print('debug: before random crop:', image, label)
# image = tf.Print(image, [tf.reduce_min(image), tf.reduce_max(image)], message='image min, max: ')
image = tf.image.convert_image_dtype(image, tf.uint8, saturate=True) # so it can be concated
combined = tf.concat([image, label], 2)
combined = tf.random_crop(combined, _TRAIN_SIZE + tuple([4]))
image = combined[..., :3]
label = combined[..., 3]
image = tf.image.convert_image_dtype(image, tf.float32)
label = tf.to_int32(label)
print('debug: crop:', image, label)
# after this point image and label has _TRAIN_SIZE size
# center input to [-1,1] is equivalent to assuming mean of 0.5
mean = 0.5
image = (image - mean)/mean
proimage = image
prolabel = label
# proimage, prolabel = preprocess_train(image, label, params)
return proimage, prolabel, im_path, la_path
def inference_winner_take_all(images,
cams,
depth_num,
depth_start,
depth_end,
is_master_gpu=True,
reg_type='GRU',
inverse_depth=False):
""" infer disparity image from stereo images and cameras """
if not inverse_depth:
depth_interval = (depth_end -
depth_start) / (tf.cast(depth_num, tf.float32) - 1)
# reference image
ref_image = tf.squeeze(tf.slice(images, [0, 0, 0, 0, 0],
[-1, 1, -1, -1, 3]),
axis=1)
ref_cam = tf.squeeze(tf.slice(cams, [0, 0, 0, 0, 0], [-1, 1, 2, 4, 4]),
axis=1)
height = tf.shape(images)[2]
width = tf.shape(images)[3]
images = tf.reshape(images, [-1, height, width, 3])
feature_tower = SNetDS2GN_1({
'data': images
},
is_training=True,
reuse=tf.AUTO_REUSE,
dilation=1).get_output()
height = tf.shape(feature_tower)[1]
width = tf.shape(feature_tower)[2]
features = tf.reshape(
feature_tower, [FLAGS.batch_size, FLAGS.view_num, height, width, 32])
ref_feature = tf.squeeze(
tf.slice(features, [0, 0, 0, 0, 0], [-1, 1, -1, -1, -1]), 1)
view_features = tf.slice(features, [0, 1, 0, 0, 0], [-1, -1, -1, -1, -1])
# get all homographies
view_homographies = []
for view in range(1, FLAGS.view_num):
view_cam = tf.squeeze(tf.slice(cams, [0, view, 0, 0, 0],
[-1, 1, 2, 4, 4]),
axis=1)
if inverse_depth:
homographies = get_homographies_inv_depth(ref_cam,
view_cam,
depth_num=depth_num,
depth_start=depth_start,
depth_end=depth_end)
else:
homographies = get_homographies(ref_cam,
view_cam,
depth_num=depth_num,
depth_start=depth_start,
depth_interval=depth_interval)
view_homographies.append(homographies)
feature_shape = [FLAGS.batch_size, FLAGS.max_h, FLAGS.max_w, 32]
batch_size, height, width, channel = feature_shape
gru_input_shape = [feature_shape[1], feature_shape[2]]
cell0 = ConvLSTMCell(conv_ndims=2,
input_shape=[height, width, 32],
output_channels=16,
kernel_shape=[3, 3],
name="conv_lstm_cell0")
cell1 = ConvLSTMCell(conv_ndims=2,
input_shape=[height / 2, width / 2, 16],
output_channels=16,
kernel_shape=[3, 3],
name="conv_lstm_cell1")
cell2 = ConvLSTMCell(conv_ndims=2,
input_shape=[height / 4, width / 4, 16],
output_channels=16,
kernel_shape=[3, 3],
name="conv_lstm_cell2")
cell3 = ConvLSTMCell(conv_ndims=2,
input_shape=[height / 2, width / 2, 32],
output_channels=16,
kernel_shape=[3, 3],
name="conv_lstm_cell3")
cell4 = ConvLSTMCell(conv_ndims=2,
input_shape=[height, width, 32],
output_channels=8,
kernel_shape=[3, 3],
name="conv_lstm_cell4")
initial_state0 = cell0.zero_state(batch_size, dtype=tf.float32)
initial_state1 = cell1.zero_state(batch_size, dtype=tf.float32)
initial_state2 = cell2.zero_state(batch_size, dtype=tf.float32)
initial_state3 = cell3.zero_state(batch_size, dtype=tf.float32)
initial_state4 = cell4.zero_state(batch_size, dtype=tf.float32)
# initialize variables
exp_sum = tf.Variable(tf.zeros(
[FLAGS.batch_size, feature_shape[1], feature_shape[2], 1]),
name='exp_sum',
trainable=False,
collections=[tf.GraphKeys.LOCAL_VARIABLES])
depth_image = tf.Variable(tf.zeros(
[FLAGS.batch_size, feature_shape[1], feature_shape[2], 1]),
name='depth_image',
trainable=False,
collections=[tf.GraphKeys.LOCAL_VARIABLES])
max_prob_image = tf.Variable(tf.zeros(
[FLAGS.batch_size, feature_shape[1], feature_shape[2], 1]),
name='max_prob_image',
trainable=False,
collections=[tf.GraphKeys.LOCAL_VARIABLES])
init_map = tf.zeros(
[FLAGS.batch_size, feature_shape[1], feature_shape[2], 1])
weights = tf.reshape(tf.constant([1.0, 0.5, 0.1], dtype=tf.float32),
[1, 1, 1, 1, 3])
# define winner take all loop
def body(depth_index, initial_state0, initial_state1, initial_state2,
initial_state3, initial_state4, depth_image, max_prob_image,
exp_sum, incre):
"""Loop body."""
# calculate cost
ave_feature = ref_feature
ave_feature2 = tf.square(ref_feature)
warped_view_volumes = tf.zeros([batch_size, height, width, 1])
weight_sum = tf.zeros([batch_size, height, width, 1])
for view in range(0, FLAGS.view_num - 1):
homographies = view_homographies[view]
homographies = tf.transpose(homographies, perm=[1, 0, 2, 3])
homography = homographies[depth_index]
view_feature = tf.squeeze(
tf.slice(view_features, [0, view, 0, 0, 0],
[-1, 1, -1, -1, -1]), 1)
warped_view_feature = tf_transform_homography(
view_feature, homography)
warped_view_volume = tf.square(warped_view_feature - ref_feature)
weight = gateNet(warped_view_volume, 32, name='gate')
warped_view_volumes += (weight + 1) * warped_view_volume
weight_sum += (weight + 1)
cost = warped_view_volumes / weight_sum
with tf.name_scope('cost_volume_homography') as scope:
with tf.variable_scope("rnn/", reuse=tf.AUTO_REUSE):
cost0, initial_state0 = cell0(cost, state=initial_state0)
cost1 = tf.nn.max_pool2d(cost0, (2, 2), 2, 'SAME')
cost1, initial_state1 = cell1(cost1, state=initial_state1)
cost2 = tf.nn.max_pool2d(cost1, (2, 2), 2, 'SAME')
cost2, initial_state2 = cell2(cost2, state=initial_state2)
cost2 = deconv_gn(cost2,
16,
3,
padding='same',
strides=2,
reuse=tf.AUTO_REUSE,
name='cost_upconv0')
cost2 = tf.concat([cost2, cost1], -1)
cost3, initial_state3 = cell3(cost2, state=initial_state3)
cost3 = deconv_gn(cost3,
16,
3,
padding='same',
strides=2,
reuse=tf.AUTO_REUSE,
name='cost_upconv1')
cost3 = tf.concat([cost3, cost0], -1)
cost4, initial_state4 = cell4(cost3, state=initial_state4)
cost = tf.layers.conv2d(cost4,
1,
3,
padding='same',
reuse=tf.AUTO_REUSE,
name='prob_conv')
prob = tf.exp(-cost)
# index
d_idx = tf.cast(depth_index, tf.float32)
if inverse_depth:
inv_depth_start = tf.div(1.0, depth_start)
inv_depth_end = tf.div(1.0, depth_end)
inv_interval = (inv_depth_start - inv_depth_end) / (
tf.cast(depth_num, 'float32') - 1)
inv_depth = inv_depth_start - d_idx * inv_interval
depth = tf.div(1.0, inv_depth)
else:
depth = depth_start + d_idx * depth_interval
temp_depth_image = tf.reshape(depth, [FLAGS.batch_size, 1, 1, 1])
temp_depth_image = tf.tile(temp_depth_image,
[1, feature_shape[1], feature_shape[2], 1])
# update the best
update_flag_image = tf.cast(tf.less(max_prob_image, prob),
dtype='float32')
new_max_prob_image = update_flag_image * prob + (
1 - update_flag_image) * max_prob_image
new_depth_image = update_flag_image * temp_depth_image + (
1 - update_flag_image) * depth_image
max_prob_image = tf.assign(max_prob_image, new_max_prob_image)
depth_image = tf.assign(depth_image, new_depth_image)
# update counter
exp_sum = tf.assign_add(exp_sum, prob)
depth_index = tf.add(depth_index, incre)
return depth_index, initial_state0, initial_state1, initial_state2, initial_state3, initial_state4, depth_image, max_prob_image, exp_sum, incre
# run forward loop
exp_sum = tf.assign(exp_sum, init_map)
depth_image = tf.assign(depth_image, init_map)
max_prob_image = tf.assign(max_prob_image, init_map)
depth_index = tf.constant(0)
incre = tf.constant(1)
cond = lambda depth_index, *_: tf.less(depth_index, depth_num)
_, initial_state0, initial_state1, initial_state2, initial_state3, initial_state4, depth_image, max_prob_image, exp_sum, incre = tf.while_loop(
cond,
body, [
depth_index, initial_state0, initial_state1, initial_state2,
initial_state3, initial_state4, depth_image, max_prob_image,
exp_sum, incre
],
back_prop=False,
parallel_iterations=1)
# get output
forward_exp_sum = exp_sum + 1e-7
forward_depth_map = depth_image
return forward_depth_map, max_prob_image / forward_exp_sum
y = tf.random_uniform([])
out = tf.cond(tf.greater(x, y), lambda: tf.add(x, y),
lambda: tf.subtract(x, y))
###############################################################################
# 1b: Create two 0-d tensors x and y randomly selected from the range [-1, 1).
# Return x + y if x < y, x - y if x > y, 0 otherwise.
# Hint: Look up tf.case().
###############################################################################
# YOUR CODE
x = tf.random_uniform([], minval=-1, maxval=1, dtype=tf.float32)
y = tf.random_uniform([], minval=-1, maxval=1, dtype=tf.float32)
out_1b = tf.case(
{
tf.less(x, y): lambda: tf.add(x, y),
tf.greater(x, y): lambda: tf.subtract(x, y)
},
default=lambda: tf.constant(0.0),
exclusive=True)
sess = tf.InteractiveSession()
print(x)
print(y)
print(sess.run(out_1b))
###############################################################################
# 1c: Create the tensor x of the value [[0, -2, -1], [0, 1, 2]]
# and y as a tensor of zeros with the same shape as x.
# Return a boolean tensor that yields Trues if x equals y element-wise.
# Hint: Look up tf.equal().
def smooth_l1_loss(x):
square_loss = 0.5*x**2
absolute_loss = tf.abs(x)
return tf.where(tf.less(absolute_loss, 1.), square_loss, absolute_loss-0.5)
def get_train_ops(loss,
tf_variables,
train_step,
clip_mode=None,
grad_bound=None,
l2_reg=1e-4,
lr_warmup_val=None,
lr_warmup_steps=100,
lr_init=0.1,
lr_dec_start=0,
lr_dec_every=10000,
lr_dec_rate=0.1,
lr_dec_min=None,
lr_cosine=False,
lr_max=None,
lr_min=None,
lr_T_0=None,
lr_T_mul=None,
num_train_batches=None,
optim_algo=None,
sync_replicas=False,
num_aggregate=None,
num_replicas=None,
get_grad_norms=False,
moving_average=None):
"""
Args:
clip_mode: "global", "norm", or None.
moving_average: store the moving average of parameters
"""
if l2_reg > 0:
l2_losses = []
for var in tf_variables:
l2_losses.append(tf.reduce_sum(var**2))
l2_loss = tf.add_n(l2_losses)
loss += l2_reg * l2_loss
grads = tf.gradients(loss, tf_variables)
grad_norm = tf.global_norm(grads)
grad_norms = {}
for v, g in zip(tf_variables, grads):
if v is None or g is None:
continue
if isinstance(g, tf.IndexedSlices):
grad_norms[v.name] = tf.sqrt(tf.reduce_sum(g.values**2))
else:
grad_norms[v.name] = tf.sqrt(tf.reduce_sum(g**2))
if clip_mode is not None:
assert grad_bound is not None, "Need grad_bound to clip gradients."
if clip_mode == "global":
grads, _ = tf.clip_by_global_norm(grads, grad_bound)
elif clip_mode == "norm":
clipped = []
for g in grads:
if isinstance(g, tf.IndexedSlices):
c_g = tf.clip_by_norm(g.values, grad_bound)
c_g = tf.IndexedSlices(g.indices, c_g)
else:
c_g = tf.clip_by_norm(g, grad_bound)
clipped.append(g)
grads = clipped
else:
raise NotImplementedError("Unknown clip_mode {}".format(clip_mode))
if lr_cosine:
assert lr_max is not None, "Need lr_max to use lr_cosine"
assert lr_min is not None, "Need lr_min to use lr_cosine"
assert lr_T_0 is not None, "Need lr_T_0 to use lr_cosine"
assert lr_T_mul is not None, "Need lr_T_mul to use lr_cosine"
assert num_train_batches is not None, ("Need num_train_batches to use"
" lr_cosine")
curr_epoch = train_step // num_train_batches
last_reset = tf.Variable(0,
dtype=tf.int32,
trainable=False,
name="last_reset")
T_i = tf.Variable(lr_T_0, dtype=tf.int32, trainable=False, name="T_i")
T_curr = curr_epoch - last_reset
def _update():
update_last_reset = tf.assign(last_reset,
curr_epoch,
use_locking=True)
update_T_i = tf.assign(T_i, T_i * lr_T_mul, use_locking=True)
with tf.control_dependencies([update_last_reset, update_T_i]):
rate = tf.to_float(T_curr) / tf.to_float(T_i) * 3.1415926
lr = lr_min + 0.5 * (lr_max - lr_min) * (1.0 + tf.cos(rate))
return lr
def _no_update():
rate = tf.to_float(T_curr) / tf.to_float(T_i) * 3.1415926
lr = lr_min + 0.5 * (lr_max - lr_min) * (1.0 + tf.cos(rate))
return lr
learning_rate = tf.cond(tf.greater_equal(T_curr, T_i), _update,
_no_update)
else:
learning_rate = tf.train.exponential_decay(
lr_init,
tf.maximum(train_step - lr_dec_start, 0),
lr_dec_every,
lr_dec_rate,
staircase=True)
if lr_dec_min is not None:
learning_rate = tf.maximum(learning_rate, lr_dec_min)
if lr_warmup_val is not None:
learning_rate = tf.cond(tf.less(train_step, lr_warmup_steps),
lambda: lr_warmup_val, lambda: learning_rate)
# if get_grad_norms:
# g_1, g_2 = 0.0001, 0.0001
# for v, g in zip(tf_variables, grads):
# if g is not None:
# if isinstance(g, tf.IndexedSlices):
# g_n = tf.reduce_sum(g.values ** 2)
# else:
# g_n = tf.reduce_sum(g ** 2)
# if "enas_cell" in v.name:
# print("g_1: {}".format(v.name))
# g_1 += g_n
# else:
# print("g_2: {}".format(v.name))
# g_2 += g_n
# learning_rate = tf.Print(learning_rate, [g_1, g_2, tf.sqrt(g_1 / g_2)],
# message="g_1, g_2, g_1/g_2: ", summarize=5)
if optim_algo == "momentum":
opt = tf.train.MomentumOptimizer(learning_rate,
0.9,
use_locking=True,
use_nesterov=True)
elif optim_algo == "sgd":
opt = tf.train.GradientDescentOptimizer(learning_rate,
use_locking=True)
elif optim_algo == "adam":
opt = tf.train.AdamOptimizer(learning_rate,
beta1=0.0,
epsilon=1e-3,
use_locking=True)
else:
raise ValueError("Unknown optim_algo {}".format(optim_algo))
if sync_replicas:
assert num_aggregate is not None, "Need num_aggregate to sync."
assert num_replicas is not None, "Need num_replicas to sync."
opt = tf.train.SyncReplicasOptimizer(
opt,
replicas_to_aggregate=num_aggregate,
total_num_replicas=num_replicas,
use_locking=True)
if moving_average is not None:
opt = tf.contrib.opt.MovingAverageOptimizer(
opt, average_decay=moving_average)
train_op = opt.apply_gradients(zip(grads, tf_variables),
global_step=train_step)
if get_grad_norms:
return train_op, learning_rate, grad_norm, opt, grad_norms
else:
return train_op, learning_rate, grad_norm, opt
def _leaky_relu(self, x, leakiness=0.0):
return tf.where(tf.less(x, 0.0), leakiness * x, x, name='leaky_relu')
def compile(self,
k=1,
optimizer='adam',
learning_rate=1e-4,
mu=0.5,
margin=1):
""" Builds the tensorflow graph that is evaluated in the fit method
Arguments:
k: integer, number of target neighbours
optimizer: string, name of optimizer to use. See dlmnn.helper.utility
for which optimizers that are supported
learning_rate: scalar, learning rate for optimizer
mu: scalar, weighting of the pull and push term. Should be between 0
and 1. High values put weight on the push term and vice verse for
the pull term
margin: scalar, size of margin inforcing between similar pairs and
imposters. Should be higher than 0.
"""
assert k > 0 and isinstance(
k, int), ''' k need to be a positive integer '''
assert learning_rate > 0, ''' learning rate needs to be a positive number '''
assert 0 <= mu and mu <= 1, ''' mu needs to be between 0 and 1 '''
assert margin > 0, ''' margin needs to be a positive number '''
assert len(
self.extractor.layers) != 0, '''Layers must be added with the
lmnn.add() method before this function is called '''
self.built = True
# Set number of neighbours
self.k = k
# Shapes
self.input_shape = self.extractor.input_shape
self.output_shape = self.extractor.output_shape
# Placeholders for data
self.global_step = tf.Variable(0, trainable=False)
self.Xp = tf.placeholder(tf.float32,
shape=self.input_shape,
name='In_features')
self.yp = tf.placeholder(tf.int32, shape=(None, ), name='In_targets')
self.tNp = tf.placeholder(tf.int32,
shape=(None, 2),
name='In_targetNeighbours')
# Feature extraction function and pairwise distance function
self.extractor_func = tf_featureExtractor(self.extractor)
self.dist_func = tf_makePairwiseFunc(self.extractor_func)
# Build graph
#D = self.dist_func(self.Xp, self.Xp)
D = self.dist_func(self.Xp)
tup = tf_findImposters(D, self.yp, self.tNp, margin=margin)
self._LMNN_loss, D_1, D_2, D_3 = tf_LMNN_loss(D,
self.tNp,
tup,
mu,
margin=margin)
# Construct training operation
self._optimizer = get_optimizer(optimizer)(learning_rate=learning_rate)
self._trainer = self._optimizer.minimize(self._LMNN_loss,
global_step=self.global_step)
# Summaries
self._n_tup = tf.shape(tup)[0]
self._true_imp = tf.cast(tf.less(D_3, D_2), tf.float32)
features = self.extractor_func(self.Xp)
tf.summary.scalar('Loss', self._LMNN_loss)
tf.summary.scalar('Num_imp', self._n_tup)
tf.summary.scalar('Loss_pull', tf.reduce_sum(D_1))
tf.summary.scalar('Loss_push', tf.reduce_sum(margin + D_2 - D_3))
tf.summary.scalar('True_imp', tf.reduce_sum(self._true_imp))
tf.summary.scalar('Frac_true_imp', tf.reduce_mean(self._true_imp))
tf.summary.scalar(
'Sparsity_tanh',
tf.reduce_mean(
tf.reduce_sum(tf.tanh(tf.pow(features, 2.0)), axis=1)))
tf.summary.scalar(
'Sparsity_l0',
tf.reduce_mean(
tf.reduce_sum(tf.cast(tf.equal(features, 0), tf.int32),
axis=1)))
self._summary = tf.summary.merge_all()
# Initilize session
init = tf.global_variables_initializer()
self.session.run(init)
# Create callable functions
self._transformer = self.session.make_callable(
self.extractor_func(self.Xp), [self.Xp])
# self._distances = self.session.make_callable(
# self.dist_func(self.Xp, self.Xp), [self.Xp])
self._distances = self.session.make_callable(self.dist_func(self.Xp),
[self.Xp])
def _cond(x_in, y_in, domain_in, i, cond_in):
# Repeat the loop until we have achieved misclassification or
# reaches the maximum iterations
return tf.logical_and(tf.less(i, self.iterations), cond_in)
def _is_valid_box(ymin, xmin, ymax, xmax):
return tf.logical_and(tf.less(ymin, ymax), tf.less(xmin, xmax))
def build_optimizer(self, learning_rate=0.001, weight_decay=0.0005,
momentum=0.9, global_step=None):
self.labels = tf.placeholder(tf.float32, name='labels',
shape=[None, None, self.num_vars])
with tf.variable_scope('ground_truth'):
#-------------------------------------------------------------------
# Split the ground truth tensor
#-------------------------------------------------------------------
# Classification ground truth tensor
# Shape: (batch_size, num_anchors, num_classes)
gt_cl = self.labels[:,:,:self.num_classes]
# Localization ground truth tensor
# Shape: (batch_size, num_anchors, 4)
gt_loc = self.labels[:,:,self.num_classes:]
# Batch size
# Shape: scalar
batch_size = tf.shape(gt_cl)[0]
#-----------------------------------------------------------------------
# Compute match counters
#-----------------------------------------------------------------------
with tf.variable_scope('match_counters'):
# Number of anchors per sample
# Shape: (batch_size)
total_num = tf.ones([batch_size], dtype=tf.int64) * \
tf.to_int64(self.preset.num_anchors)
# Number of negative (not-matched) anchors per sample, computed
# by counting boxes of the background class in each sample.
# Shape: (batch_size)
negatives_num = tf.count_nonzero(gt_cl[:,:,-1], axis=1)
# Number of positive (matched) anchors per sample
# Shape: (batch_size)
positives_num = total_num-negatives_num
# Number of positives per sample that is division-safe
# Shape: (batch_size)
positives_num_safe = tf.where(tf.equal(positives_num, 0),
tf.ones([batch_size])*10e-15,
tf.to_float(positives_num))
#-----------------------------------------------------------------------
# Compute masks
#-----------------------------------------------------------------------
with tf.variable_scope('match_masks'):
# Boolean tensor determining whether an anchor is a positive
# Shape: (batch_size, num_anchors)
positives_mask = tf.equal(gt_cl[:,:,-1], 0)
# Boolean tensor determining whether an anchor is a negative
# Shape: (batch_size, num_anchors)
negatives_mask = tf.logical_not(positives_mask)
#-----------------------------------------------------------------------
# Compute the confidence loss
#-----------------------------------------------------------------------
with tf.variable_scope('confidence_loss'):
# Cross-entropy tensor - all of the values are non-negative
# Shape: (batch_size, num_anchors)
ce = tf.nn.softmax_cross_entropy_with_logits_v2(labels=gt_cl,
logits=self.logits)
#-------------------------------------------------------------------
# Sum up the loss of all the positive anchors
#-------------------------------------------------------------------
# Positives - the loss of negative anchors is zeroed out
# Shape: (batch_size, num_anchors)
positives = tf.where(positives_mask, ce, tf.zeros_like(ce))
# Total loss of positive anchors
# Shape: (batch_size)
positives_sum = tf.reduce_sum(positives, axis=-1)
#-------------------------------------------------------------------
# Figure out what the negative anchors with highest confidence loss
# are
#-------------------------------------------------------------------
# Negatives - the loss of positive anchors is zeroed out
# Shape: (batch_size, num_anchors)
negatives = tf.where(negatives_mask, ce, tf.zeros_like(ce))
# Top negatives - sorted confience loss with the highest one first
# Shape: (batch_size, num_anchors)
negatives_top = tf.nn.top_k(negatives, self.preset.num_anchors)[0]
#-------------------------------------------------------------------
# Fugure out what the number of negatives we want to keep is
#-------------------------------------------------------------------
# Maximum number of negatives to keep per sample - we keep at most
# 3 times as many as we have positive anchors in the sample
# Shape: (batch_size)
negatives_num_max = tf.minimum(negatives_num, 3*positives_num)
#-------------------------------------------------------------------
# Mask out superfluous negatives and compute the sum of the loss
#-------------------------------------------------------------------
# Transposed vector of maximum negatives per sample
# Shape (batch_size, 1)
negatives_num_max_t = tf.expand_dims(negatives_num_max, 1)
# Range tensor: [0, 1, 2, ..., num_anchors-1]
# Shape: (num_anchors)
rng = tf.range(0, self.preset.num_anchors, 1)
# Row range, the same as above, but int64 and a row of a matrix
# Shape: (1, num_anchors)
range_row = tf.to_int64(tf.expand_dims(rng, 0))
# Mask of maximum negatives - first `negative_num_max` elements
# in corresponding row are `True`, the rest is false
# Shape: (batch_size, num_anchors)
negatives_max_mask = tf.less(range_row, negatives_num_max_t)
# Max negatives - all the positives and superfluous negatives are
# zeroed out.
# Shape: (batch_size, num_anchors)
negatives_max = tf.where(negatives_max_mask, negatives_top,
tf.zeros_like(negatives_top))
# Sum of max negatives for each sample
# Shape: (batch_size)
negatives_max_sum = tf.reduce_sum(negatives_max, axis=-1)
#-------------------------------------------------------------------
# Compute the confidence loss for each element
#-------------------------------------------------------------------
# Total confidence loss for each sample
# Shape: (batch_size)
confidence_loss = tf.add(positives_sum, negatives_max_sum)
# Total confidence loss normalized by the number of positives
# per sample
# Shape: (batch_size)
confidence_loss = tf.where(tf.equal(positives_num, 0),
tf.zeros([batch_size]),
tf.div(confidence_loss,
positives_num_safe))
# Mean confidence loss for the batch
# Shape: scalar
self.confidence_loss = tf.reduce_mean(confidence_loss,
name='confidence_loss')
#-----------------------------------------------------------------------
# Compute the localization loss
#-----------------------------------------------------------------------
with tf.variable_scope('localization_loss'):
# Element-wise difference between the predicted localization loss
# and the ground truth
# Shape: (batch_size, num_anchors, 4)
loc_diff = tf.subtract(self.locator, gt_loc)
# Smooth L1 loss
# Shape: (batch_size, num_anchors, 4)
loc_loss = smooth_l1_loss(loc_diff)
# Sum of localization losses for each anchor
# Shape: (batch_size, num_anchors)
loc_loss_sum = tf.reduce_sum(loc_loss, axis=-1)
# Positive locs - the loss of negative anchors is zeroed out
# Shape: (batch_size, num_anchors)
positive_locs = tf.where(positives_mask, loc_loss_sum,
tf.zeros_like(loc_loss_sum))
# Total loss of positive anchors
# Shape: (batch_size)
localization_loss = tf.reduce_sum(positive_locs, axis=-1)
# Total localization loss normalized by the number of positives
# per sample
# Shape: (batch_size)
localization_loss = tf.where(tf.equal(positives_num, 0),
tf.zeros([batch_size]),
tf.div(localization_loss,
positives_num_safe))
# Mean localization loss for the batch
# Shape: scalar
self.localization_loss = tf.reduce_mean(localization_loss,
name='localization_loss')
#-----------------------------------------------------------------------
# Compute total loss
#-----------------------------------------------------------------------
with tf.variable_scope('total_loss'):
# Sum of the localization and confidence loss
# Shape: (batch_size)
self.conf_and_loc_loss = tf.add(self.confidence_loss,
self.localization_loss,
name='sum_losses')
# L2 loss
# Shape: scalar
self.l2_loss = tf.multiply(weight_decay, self.l2_loss,
name='l2_loss')
# Final loss
# Shape: scalar
self.loss = tf.add(self.conf_and_loc_loss, self.l2_loss,
name='loss')
#-----------------------------------------------------------------------
# Build the optimizer
#-----------------------------------------------------------------------
with tf.variable_scope('optimizer'):
optimizer = tf.train.MomentumOptimizer(learning_rate, momentum)
optimizer = optimizer.minimize(self.loss, global_step=global_step,
name='optimizer')
#-----------------------------------------------------------------------
# Store the tensors
#-----------------------------------------------------------------------
self.optimizer = optimizer
self.losses = {
'total': self.loss,
'localization': self.localization_loss,
'confidence': self.confidence_loss,
'l2': self.l2_loss
}
