def set_logp_to_neg_inf(X, logp, bounds): """Set `logp` to negative infinity when `X` is outside the allowed bounds. # Arguments X: tensorflow.Tensor The variable to apply the bounds to logp: tensorflow.Tensor The log probability corrosponding to `X` bounds: list of `Region` objects The regions corrosponding to allowed regions of `X` # Returns logp: tensorflow.Tensor The newly bounded log probability """ conditions = [] for l, u in bounds: lower_is_neg_inf = not isinstance(l, tf.Tensor) and np.isneginf(l) upper_is_pos_inf = not isinstance(u, tf.Tensor) and np.isposinf(u) if not lower_is_neg_inf and upper_is_pos_inf: conditions.append(tf.greater(X, l)) elif lower_is_neg_inf and not upper_is_pos_inf: conditions.append(tf.less(X, u)) elif not (lower_is_neg_inf or upper_is_pos_inf): conditions.append(tf.logical_and(tf.greater(X, l), tf.less(X, u))) if len(conditions) > 0: is_inside_bounds = conditions[0] for condition in conditions[1:]: is_inside_bounds = tf.logical_or(is_inside_bounds, condition) logp = tf.select(is_inside_bounds, logp, tf.fill(tf.shape(X), config.dtype(-np.inf))) return logp
def loop(step_, beams_, beam_value_, golden_value_, golden_inside_, step_valid_, g_id_, golden_record, beam_record): cur_feat_x_ = tf.gather(x, step_) cur_golden_path_ = tf.gather(golden_path, tf.range(step_)) cur_golden_feat_ = self._add_tag_dynamic(cur_feat_x_, cur_golden_path_) # cur_golden_output_ = self._build_cnn(cur_golden_feat_) cur_golden_output_ = build(cur_golden_feat_) cur_golden_node_ = tf.gather(golden_path, tf.reshape(step_, [1])) golden_value_ = tf.add(golden_value_, tf.slice(cur_golden_output_, tf.concat(0, [[0], cur_golden_node_]), [1, 1])) cur_beam_ = tf.unpack(beams_, num=self.beam_size) cur_beam_feat_ = tf.concat(0, [self._add_tag_dynamic(cur_feat_x_, tf.reshape(e, [-1])) for e in cur_beam_]) # cur_beam_output_ = self._build_cnn(cur_beam_feat_) cur_beam_output_ = build(cur_beam_feat_) golden_record = golden_record.write(step_, cur_golden_output_) beam_record = beam_record.write(step_, cur_beam_output_) beam_value_, beams_ = self._top_beams_new(cur_beam_output_, beam_value_, beams_) new_golden_path_ = tf.gather(golden_path, tf.range(step_ + 1)) # golden_beam_id_ = index_of_tensor(new_golden_path_, beams_) g_id_ = index_of_tensor(new_golden_path_, beams_) golden_inside_ = tf.select(tf.less(tf.shape(g_id_)[0], 1), tf.constant(False, tf.bool), tf.constant(True, tf.bool)) step_valid_ = tf.logical_and(tf.less(step_+1, length), tf.less(step_+1, self.max_step_tracked)) return [step_ + 1, beams_, beam_value_, golden_value_, golden_inside_, step_valid_, g_id_, golden_record, beam_record]
def enc_step(step): """Encoder step.""" if autoenc_decay < 1.0: quant_step = autoenc_quantize(step, 16, nmaps, self.do_training) if backward: exp_glob = tf.train.exponential_decay(1.0, self.global_step - 10000, 1000, autoenc_decay) dec_factor = 1.0 - exp_glob # * self.do_training dec_factor = tf.cond(tf.less(self.global_step, 10500), lambda: tf.constant(0.05), lambda: dec_factor) else: dec_factor = 1.0 cur = tf.cond(tf.less(tf.random_uniform([]), dec_factor), lambda: quant_step, lambda: step) else: cur = step if dropout > 0.0001: cur = tf.nn.dropout(cur, keep_prob) if act_noise > 0.00001: cur += tf.truncated_normal(tf.shape(cur)) * act_noise_scale # Do nconvs-many CGRU steps. if do_jit and tf.get_variable_scope().reuse: with jit_scope(): for layer in xrange(nconvs): cur = conv_gru([], cur, kw, kh, nmaps, conv_rate(layer), cutoff, "ecgru_%d" % layer, do_layer_norm) else: for layer in xrange(nconvs): cur = conv_gru([], cur, kw, kh, nmaps, conv_rate(layer), cutoff, "ecgru_%d" % layer, do_layer_norm) return cur
def compute_IOU(bboxA, bboxB): """Compute the Intersection Over Union. Args: bboxA: [N X 4 tensor] format = [left, top, right, bottom] bboxB: [N X 4 tensor] Return: IOU: [N X 1 tensor] """ x1A, y1A, x2A, y2A = tf.split(1, 4, bboxA) x1B, y1B, x2B, y2B = tf.split(1, 4, bboxB) # compute intersection x1_max = tf.maximum(x1A, x1B) y1_max = tf.maximum(y1A, y1B) x2_min = tf.minimum(x2A, x2B) y2_min = tf.minimum(y2A, y2B) # overlap_flag = tf.logical_and( tf.less(x1_max, x2_min), tf.less(y1_max, y2_min)) overlap_flag = tf.to_float(tf.less(x1_max, x2_min)) * \ tf.to_float(tf.less(y1_max, y2_min)) overlap_area = tf.mul(overlap_flag, tf.mul( x2_min - x1_max, y2_min - y1_max)) # compute union areaA = tf.mul(x2A - x1A, y2A - y1A) areaB = tf.mul(x2B - x1B, y2B - y1B) union_area = areaA + areaB - overlap_area return tf.div(overlap_area, union_area)
def prune_outside_window(boxlist, window, scope=None): """Prunes bounding boxes that fall outside a given window. This function prunes bounding boxes that even partially fall outside the given window. See also clip_to_window which only prunes bounding boxes that fall completely outside the window, and clips any bounding boxes that partially overflow. Args: boxlist: a BoxList holding M_in boxes. window: a float tensor of shape [4] representing [ymin, xmin, ymax, xmax] of the window scope: name scope. Returns: pruned_corners: a tensor with shape [M_out, 4] where M_out <= M_in valid_indices: a tensor with shape [M_out] indexing the valid bounding boxes in the input tensor. """ with tf.name_scope(scope, 'PruneOutsideWindow'): y_min, x_min, y_max, x_max = tf.split( value=boxlist.get(), num_or_size_splits=4, axis=1) win_y_min, win_x_min, win_y_max, win_x_max = tf.unstack(window) coordinate_violations = tf.concat([ tf.less(y_min, win_y_min), tf.less(x_min, win_x_min), tf.greater(y_max, win_y_max), tf.greater(x_max, win_x_max) ], 1) valid_indices = tf.reshape( tf.where(tf.logical_not(tf.reduce_any(coordinate_violations, 1))), [-1]) return gather(boxlist, valid_indices), valid_indices
def l1_smooth_losses(predict_boxes, gtboxes, object_weights, classes_weights=None): ''' :param predict_boxes: [minibatch_size, -1] :param gtboxes: [minibatch_size, -1] :param object_weights: [minibatch_size, ]. 1.0 represent object, 0.0 represent others(ignored or background) :return: ''' diff = predict_boxes - gtboxes abs_diff = tf.cast(tf.abs(diff), tf.float32) if classes_weights is None: ''' first_stage: predict_boxes :[minibatch_size, 5] gtboxes: [minibatchs_size, 5] ''' anchorwise_smooth_l1norm = tf.reduce_sum( tf.where(tf.less(abs_diff, 1), 0.5 * tf.square(abs_diff), abs_diff - 0.5), axis=1) * object_weights else: ''' fast_rcnn stage: predict_boxes: [minibatch_size, 5*num_classes] gtboxes: [minibatch_size, 5*num_classes] classes_weights : [minibatch_size, 5*num_classes] ''' anchorwise_smooth_l1norm = tf.reduce_sum( tf.where(tf.less(abs_diff, 1), 0.5*tf.square(abs_diff)*classes_weights, (abs_diff - 0.5)*classes_weights), axis=1)*object_weights return tf.reduce_mean(anchorwise_smooth_l1norm, axis=0) # reduce mean
def image_distortions(image, distortions): distort_left_right_random = distortions[0] mirror = tf.less(tf.pack([1.0, distort_left_right_random, 1.0]), 0.5) image = tf.reverse(image, mirror) distort_up_down_random = distortions[1] mirror = tf.less(tf.pack([distort_up_down_random, 1.0, 1.0]), 0.5) image = tf.reverse(image, mirror) return image
def learning_rate_schedule(): # Function which controls learning rate during training import tensorflow as tf step = tf.train.get_or_create_global_step() lr = tf.case([(tf.less(step, 1000), lambda: tf.constant(0.0004)), (tf.less(step, 10000), lambda: tf.constant(0.01)), (tf.less(step, 40000), lambda: tf.constant(0.005)), (tf.less(step, 55000), lambda: tf.constant(0.0005)), (tf.less(step, 65000), lambda: tf.constant(0.00005))]) return lr
def testWhileCond_3(self): with self.test_session(): n = tf.convert_to_tensor(0) c = lambda x: tf.less(x, 10) b = lambda x: control_flow_ops.cond(tf.less(0, 1), lambda: tf.add(x, 1), lambda: tf.sub(x, 1)) r = control_flow_ops.While(c, b, [n]) result = r.eval() self.assertTrue(check_op_order(n.graph)) self.assertAllEqual(10, result)
def bottleneck(self, x): hparams = self.hparams x = tf.tanh(tf.layers.dense(x, hparams.bottleneck_bits, name="bottleneck")) d = x + tf.stop_gradient(2.0 * tf.to_float(tf.less(0.0, x)) - 1.0 - x) if hparams.mode == tf.estimator.ModeKeys.TRAIN: noise = tf.random_uniform(common_layers.shape_list(x)) noise = 2.0 * tf.to_float(tf.less(hparams.bottleneck_noise, noise)) - 1.0 d *= noise x = common_layers.mix(d, x, hparams.discretize_warmup_steps, hparams.mode == tf.estimator.ModeKeys.TRAIN) return x, 0.0
def testCondWhile_1(self): with self.test_session(): n = tf.convert_to_tensor(0, name="n") c = lambda x: tf.less(x, 10) b = lambda x: tf.add(x, 1) r = tf.cond(tf.less(0, 1), lambda: control_flow_ops.While(c, b, [n]), lambda: n) result = r.eval() self.assertTrue(check_op_order(n.graph)) self.assertAllEqual(10, result)
def augment_2d(inputs, rotation=0, horizontal_flip=False, vertical_flip=False): """Apply additive augmentation on 2D data. # Arguments rotation: A float, the degree range for rotation (0 <= rotation < 180), e.g. 3 for random image rotation between (-3.0, 3.0). horizontal_flip: A boolean, whether to allow random horizontal flip, e.g. true for 50% possibility to flip image horizontally. vertical_flip: A boolean, whether to allow random vertical flip, e.g. true for 50% possibility to flip image vertically. # Returns input data after augmentation, whose shape is the same as its original. """ if inputs.dtype != tf.float32: inputs = tf.image.convert_image_dtype(inputs, dtype=tf.float32) with tf.name_scope('augmentation'): shp = tf.shape(inputs) batch_size, height, width = shp[0], shp[1], shp[2] width = tf.cast(width, tf.float32) height = tf.cast(height, tf.float32) transforms = [] identity = tf.constant([1, 0, 0, 0, 1, 0, 0, 0], dtype=tf.float32) if rotation > 0: angle_rad = rotation * 3.141592653589793 / 180.0 angles = tf.random_uniform([batch_size], -angle_rad, angle_rad) f = tf.contrib.image.angles_to_projective_transforms(angles, height, width) transforms.append(f) if horizontal_flip: coin = tf.less(tf.random_uniform([batch_size], 0, 1.0), 0.5) shape = [-1., 0., width, 0., 1., 0., 0., 0.] flip_transform = tf.convert_to_tensor(shape, dtype=tf.float32) flip = tf.tile(tf.expand_dims(flip_transform, 0), [batch_size, 1]) noflip = tf.tile(tf.expand_dims(identity, 0), [batch_size, 1]) transforms.append(tf.where(coin, flip, noflip)) if vertical_flip: coin = tf.less(tf.random_uniform([batch_size], 0, 1.0), 0.5) shape = [1., 0., 0., 0., -1., height, 0., 0.] flip_transform = tf.convert_to_tensor(shape, dtype=tf.float32) flip = tf.tile(tf.expand_dims(flip_transform, 0), [batch_size, 1]) noflip = tf.tile(tf.expand_dims(identity, 0), [batch_size, 1]) transforms.append(tf.where(coin, flip, noflip)) if transforms: f = tf.contrib.image.compose_transforms(*transforms) inputs = tf.contrib.image.transform(inputs, f, interpolation='BILINEAR') return inputs
def tanh_discrete_bottleneck(x, bottleneck_bits, bottleneck_noise, discretize_warmup_steps, mode): """Simple discretization through tanh, flip bottleneck_noise many bits.""" x = tf.tanh(tf.layers.dense(x, bottleneck_bits, name="tanh_discrete_bottleneck")) d = x + tf.stop_gradient(2.0 * tf.to_float(tf.less(0.0, x)) - 1.0 - x) if mode == tf.estimator.ModeKeys.TRAIN: noise = tf.random_uniform(common_layers.shape_list(x)) noise = 2.0 * tf.to_float(tf.less(bottleneck_noise, noise)) - 1.0 d *= noise d = common_layers.mix(d, x, discretize_warmup_steps, mode == tf.estimator.ModeKeys.TRAIN) return d, 0.0
def process_features(image, label, do_augment = False): # Do any image preprocessing/augmentation here... with tf.name_scope('process_features'): # Crop driving view (Start from row 126, and slice 100 rows) image = tf.slice(image, [126, 0, 0], [100, 256, 3]) # Resize image image = tf.image.resize_images(image, [66, 200]) if do_augment == True: # Change or not change colors def do_color_changes(): distorted_image = tf.image.random_brightness(image, max_delta=32. / 255.) distorted_image = tf.image.random_saturation(distorted_image, lower=0.5, upper=1.5) distorted_image = tf.image.random_hue(distorted_image, max_delta=0.2) distorted_image = tf.image.random_contrast(distorted_image, lower=0.5, upper=1.5) return distorted_image def no_color_change(): distorted_image = image return distorted_image # Uniform variable in [0,1) flip_coin_color = tf.random_uniform(shape=[], minval=0., maxval=1., dtype=tf.float32) pred_color = tf.less(flip_coin_color, 0.5) # Randomically select doing color augmentation image = tf.cond(pred_color, do_color_changes, no_color_change, name='if_color') # Change or not change colors def flip_image_steering(): distorted_image = tf.image.flip_left_right(image) distorted_label = -label return distorted_image, distorted_label def no_flip_image_steering(): distorted_image = image distorted_label = label return distorted_image, distorted_label # Uniform variable in [0,1) flip_coin_flip = tf.random_uniform(shape=[], minval=0., maxval=1., dtype=tf.float32) pred_flip = tf.less(flip_coin_flip, 0.5) # Randomically select doing color augmentation image, label = tf.cond(pred_flip, flip_image_steering, no_flip_image_steering, name='if_steering') # Convert from [0, 255] -> [-0.5, 0.5] floats. image = tf.cast(image, tf.float32) * (1. / 255.0) #image = tf.image.per_image_standardization(image) return image, label
def unwrap(p, discont=np.pi, axis=-1): """Unwrap a cyclical phase tensor. Args: p: Phase tensor. discont: Float, size of the cyclic discontinuity. axis: Axis of which to unwrap. Returns: unwrapped: Unwrapped tensor of same size as input. """ dd = diff(p, axis=axis) ddmod = tf.mod(dd + np.pi, 2.0 * np.pi) - np.pi idx = tf.logical_and(tf.equal(ddmod, -np.pi), tf.greater(dd, 0)) ddmod = tf.where(idx, tf.ones_like(ddmod) * np.pi, ddmod) ph_correct = ddmod - dd idx = tf.less(tf.abs(dd), discont) ddmod = tf.where(idx, tf.zeros_like(ddmod), dd) ph_cumsum = tf.cumsum(ph_correct, axis=axis) shape = p.get_shape().as_list() shape[axis] = 1 ph_cumsum = tf.concat([tf.zeros(shape, dtype=p.dtype), ph_cumsum], axis=axis) unwrapped = p + ph_cumsum return unwrapped
def _create_learning_rate(hyperparams, step_var): """Creates learning rate var, with decay and switching for CompositeOptimizer. Args: hyperparams: a GridPoint proto containing optimizer spec, particularly learning_method to determine optimizer class to use. step_var: tf.Variable, global training step. Raises: ValueError: If the composite optimizer is set, but not correctly configured. Returns: a scalar `Tensor`, the learning rate based on current step and hyperparams. """ if hyperparams.learning_method != 'composite': base_rate = hyperparams.learning_rate adjusted_steps = step_var else: spec = hyperparams.composite_optimizer_spec switch = tf.less(step_var, spec.switch_after_steps) base_rate = tf.cond(switch, lambda: tf.constant(spec.method1.learning_rate), lambda: tf.constant(spec.method2.learning_rate)) if spec.reset_learning_rate: adjusted_steps = tf.cond(switch, lambda: step_var, lambda: step_var - spec.switch_after_steps) else: adjusted_steps = step_var return tf.train.exponential_decay( learning_rate=base_rate, global_step=adjusted_steps, decay_steps=hyperparams.decay_steps, decay_rate=hyperparams.decay_base, staircase=hyperparams.decay_staircase)
def gather(self, indices, keys=None): """Returns elements at the specified indices from the buffer. Args: indices: (list of integers or rank 1 int Tensor) indices in the buffer to retrieve elements from. keys: List of keys of tensors to retrieve. If None retrieve all. Returns: A list of tensors, where each element in the list is obtained by indexing one of the tensors in the buffer. Raises: ValueError: If gather is called before calling the add function. tf.errors.InvalidArgumentError: If indices are bigger than the number of items in the buffer. """ if not self._tensors: raise ValueError('The add function must be called before calling gather.') if keys is None: keys = self._tensors.keys() with tf.name_scope('Gather'): index_bound_assert = tf.Assert( tf.less( tf.to_int64(tf.reduce_max(indices)), tf.minimum(self.get_num_adds(), self._buffer_size)), ['Index out of bounds.']) with tf.control_dependencies([index_bound_assert]): indices = tf.convert_to_tensor(indices) batch = [] for key in keys: batch.append(tf.gather(self._tensors[key], indices, name=key)) return batch
def testLoop_2(self): with self.test_session(): zero = tf.constant(0) one = tf.constant(1) n = tf.constant(10) enter_i = control_flow_ops.enter(zero, "foo", False) enter_one = control_flow_ops.enter(one, "foo", True) enter_n = control_flow_ops.enter(n, "foo", True) merge_i = control_flow_ops.merge([enter_i, enter_i])[0] less_op = tf.less(merge_i, enter_n) cond_op = control_flow_ops.loop_cond(less_op) switch_i = control_flow_ops.switch(merge_i, cond_op) add_i = tf.add(switch_i[1], enter_one) with tf.device("/gpu:0"): next_i = control_flow_ops.next_iteration(add_i) merge_i.op._update_input(1, next_i) exit_i = control_flow_ops.exit(switch_i[0]) result = exit_i.eval() self.assertAllEqual(10, result)
def ced(model, config, scope, connect, threshold = 1e-5): with tf.variable_scope(scope), tf.name_scope(scope): with tf.variable_scope('inputs'), tf.name_scope('inputs'): model['%s_in0length' %scope] = model['%s_out0length' %connect] model['%s_in1length' %scope] = model['%s_out1length' %connect] model['%s_in2length' %scope] = model['%s_out2length' %connect] model['%s_maxin2length' %scope] = model['%s_maxout2length' %connect] model['%s_inputs' %scope] = tf.clip_by_value(model['%s_outputs' %connect], threshold, 1. - threshold, name = '%s_inputs' %scope) model['%s_out0length' %scope] = model['%s_in0length' %scope] model['%s_out1length' %scope] = model['%s_in1length' %scope] model['%s_out2length' %scope] = tf.placeholder(tf.int32, [model['%s_in0length' %scope]], '%s_out2length' %scope) model['%s_maxout2length' %scope] = model['%s_maxin2length' %scope] with tf.variable_scope('labels'), tf.name_scope('labels'): model['%s_labels_len' %scope] = tf.placeholder(tf.int32, [model['%s_in0length' %scope]], '%s_labels_len' %scope) model['%s_labels_ind' %scope] = tf.placeholder(tf.int64, [None, 3], '%s_labels_ind' %scope) model['%s_labels_val' %scope] = tf.placeholder(tf.float32, [None], '%s_labels_val' %scope) model['%s_labels' %scope] = tf.sparse_to_dense(model['%s_labels_ind' %scope], [model['%s_in0length' %scope], model['%s_maxin2length' %scope], model['%s_maxin2length' %scope]], model['%s_labels_val' %scope], -1, name = '%s_labels' %scope) with tf.variable_scope('loss'), tf.name_scope('loss'): model['%s_loss' %scope] = tf.reduce_sum(tf.where(tf.less(model['%s_labels' %scope], tf.zeros([model['%s_in0length' %scope], model['%s_maxin2length' %scope], model['%s_maxin2length' %scope]], tf.float32)), tf.zeros([model['%s_in0length' %scope], model['%s_maxin2length' %scope], model['%s_maxin2length' %scope]], tf.float32), -tf.add(tf.multiply(model['%s_labels' %scope], tf.log(model['%s_inputs' %scope])), tf.multiply(tf.subtract(1., model['%s_labels' %scope]), tf.log(tf.subtract(1., model['%s_inputs' %scope]))))), name = '%s_loss' %scope) with tf.variable_scope('outputs'), tf.name_scope('outputs'): model['%s_output' %scope] = model['%s_inputs' %scope] return model
def hinge_loss(self, y_true, y_pred): # Custom loss function margin = 0.1 # loop counter i = tf.constant(0) # loop condition function c = lambda i, _: tf.less(i, tf.shape(y_true)[0]) outer_sum_loss = tf.constant(0.0) def process_ele(i, outer_sum_loss): # Get a subtensor from batch y_true_one = y_true[i] y_pred_one = y_pred[i] # Stack margin to a num_class*1 matrix margin_stack = tf.reshape(tf.stack([tf.constant(0.1)] * self.num_classes), [self.num_classes, 1]) # Stack true label to a word_dim*num_class matrix and transpose it y_true_one_stack = tf.stack([tf.transpose(y_true_one)] * self.num_classes) # Reshape predict from (word_dim,) to (word_dim,1) y_pred_one_t = tf.reshape(y_pred_one, [self.word_dim, 1]) # Calculate loss r = margin_stack - tf.matmul(y_true_one_stack, y_pred_one_t) + tf.matmul(self.label_vec_tensor, y_pred_one_t) # Summation # We did not exclude true label inside summation, so we subtract extra margin sum_inner_loss = tf.reduce_sum(K.relu(r)) - margin # Return counter++ and accumulated loss return tf.add(i, 1), tf.add(outer_sum_loss, sum_inner_loss) _, outer_sum_loss = tf.while_loop(c, process_ele, [i, outer_sum_loss]) # Return average loss over batch return outer_sum_loss / tf.cast(tf.shape(y_true)[0], dtype=tf.float32)
def testChooseVector(self): with self.test_session(): x = np.arange(10, 12) y = np.arange(15, 18) self.assertAllEqual( x, distribution_util.pick_vector( tf.less(0, 5), x, y).eval()) self.assertAllEqual( y, distribution_util.pick_vector( tf.less(5, 0), x, y).eval()) self.assertAllEqual( x, distribution_util.pick_vector( tf.constant(True), x, y)) # No eval. self.assertAllEqual( y, distribution_util.pick_vector( tf.constant(False), x, y)) # No eval.
def testControlFlow(self): with self.test_session() as sess: v0 = tf.get_variable("v0", [], initializer=tf.constant_initializer(0)) var_dict = {} # Call get_variable in each of the cond clauses. def var_in_then_clause(): v1 = tf.get_variable("v1", [1], initializer=tf.constant_initializer(1)) var_dict["v1"] = v1 return v1 + v0 def var_in_else_clause(): v2 = tf.get_variable("v2", [1], initializer=tf.constant_initializer(2)) var_dict["v2"] = v2 return v2 + v0 add = control_flow_ops.cond(tf.less(v0, 10), var_in_then_clause, var_in_else_clause) v1 = var_dict["v1"] v2 = var_dict["v2"] # We should be able to initialize and run v1 and v2 without initializing # v0, even if the variable was created with a control dep on v0. sess.run(v1.initializer) self.assertEqual([1], sess.run(v1)) sess.run(v2.initializer) self.assertEqual([2], sess.run(v2)) # v0 should still be uninitialized. with self.assertRaisesRegexp(tf.OpError, "uninitialized"): sess.run(v0) # We should not be able to run 'add' yet. with self.assertRaisesRegexp(tf.OpError, "uninitialized"): sess.run(add) # If we initialize v0 we should be able to run 'add'. sess.run(v0.initializer) sess.run(add)
def con1(): values, top_nodes = top_k_in_2dim_tensor(value_mat, cur_beam_size) beam_new = self._concat_top_nodes(beams, top_nodes) beam_size_tmp = tf.mul(cur_beam_size, self.output_size) beam_size = tf.select(tf.less(beam_size_tmp, self.beam_size), beam_size_tmp, self.beam_size) return values, beam_new, beam_size
def _teacher_forcing_ratio_decay(init_tfr, global_step, hparams): ################################################################# # Narrow Cosine Decay: # Phase 1: tfr = 1 # We only start learning rate decay after 10k steps # Phase 2: tfr in ]0, 1[ # decay reach minimal value at step ~280k # Phase 3: tfr = 0 # clip by minimal teacher forcing ratio value (step >~ 280k) ################################################################# #Compute natural cosine decay tfr = tf.train.cosine_decay(init_tfr, global_step=global_step - hparams.tacotron_teacher_forcing_start_decay, #tfr = 1 at step 10k decay_steps=hparams.tacotron_teacher_forcing_decay_steps, #tfr = 0 at step ~280k alpha=hparams.tacotron_teacher_forcing_decay_alpha, #tfr = 0% of init_tfr as final value name='tfr_cosine_decay') #force teacher forcing ratio to take initial value when global step < start decay step. narrow_tfr = tf.cond( tf.less(global_step, tf.convert_to_tensor(hparams.tacotron_teacher_forcing_start_decay)), lambda: tf.convert_to_tensor(init_tfr), lambda: tfr) return narrow_tfr
def beta_schedule(schedule, global_step, final_beta, decay_start, decay_end): """Get KL multiplier (beta) based on the schedule.""" if decay_start > decay_end: raise ValueError("decay_end is smaller than decay_end.") # Since some of the TF schedules do not support incrementing a value, # in all of the schedules, we anneal the beta from final_beta to zero # and then reverse it at the bottom. if schedule == "constant": decayed_value = 0.0 elif schedule == "linear": decayed_value = tf.train.polynomial_decay( learning_rate=final_beta, global_step=global_step - decay_start, decay_steps=decay_end - decay_start, end_learning_rate=0.0) elif schedule == "noisy_linear_cosine_decay": decayed_value = tf.train.noisy_linear_cosine_decay( learning_rate=final_beta, global_step=global_step - decay_start, decay_steps=decay_end - decay_start) # TODO(mechcoder): Add log_annealing schedule. else: raise ValueError("Unknown beta schedule.") increased_value = final_beta - decayed_value increased_value = tf.maximum(0.0, increased_value) beta = tf.case( pred_fn_pairs={ tf.less(global_step, decay_start): lambda: 0.0, tf.greater(global_step, decay_end): lambda: final_beta}, default=lambda: increased_value) return beta
def loop(q_, mask, mass_, found_): q_list = tf.dynamic_partition(q_, mask, 2) condition_indices = tf.dynamic_partition(tf.range(tf.shape(q_)[0]), mask, 2) # 0 element it False, # 1 element if true p = q_list[1] * (1.0 - mass_) / tf.reduce_sum(q_list[1]) p_new = tf.dynamic_stitch(condition_indices, [q_list[0], p]) # condition verification and mask modification less_mask = tf.cast(tf.less(u, p_new), tf.int32) # 0 when u is bigger than p, 1 when u is less than p condition_indices = tf.dynamic_partition(tf.range(tf.shape(p_new)[0]), less_mask, 2) # 0 when u is bigger than p, 1 when u is less than p split_p_new = tf.dynamic_partition(p_new, less_mask, 2) split_u = tf.dynamic_partition(u, less_mask, 2) alpha = tf.dynamic_stitch(condition_indices, [split_p_new[0], split_u[1]]) mass_ += tf.reduce_sum(split_u[1]) mask = mask * (tf.ones_like(less_mask) - less_mask) found_ = tf.cond(tf.equal(tf.reduce_sum(less_mask), 0), lambda: False, lambda: True) alpha = tf.reshape(alpha, q_.shape) return alpha, mask, mass_, found_
def rpn_proposals(self): with tf.variable_scope('rpn_proposals'): rpn_decode_boxes = encode_and_decode.decode_boxes(encode_boxes=self.rpn_encode_boxes, reference_boxes=self.anchors, scale_factors=self.scale_factors) rpn_softmax_scores = slim.softmax(self.rpn_scores) rpn_object_score = rpn_softmax_scores[:, 1] # second column represent object if self.top_k_nms: rpn_object_score, top_k_indices = tf.nn.top_k(rpn_object_score, k=self.top_k_nms) rpn_decode_boxes = tf.gather(rpn_decode_boxes, top_k_indices) # NMS valid_indices = tf_wrapper.nms_rotate_tf(boxes_list=rpn_decode_boxes, scores=rpn_object_score, iou_threshold=self.rpn_nms_iou_threshold, max_output_size=self.max_proposals_num, use_gpu=cfgs.NMS_USE_GPU) valid_boxes = tf.gather(rpn_decode_boxes, valid_indices) valid_scores = tf.gather(rpn_object_score, valid_indices) # print_tensors(valid_scores, 'rpn_score') rpn_proposals_boxes, rpn_proposals_scores = tf.cond( tf.less(tf.shape(valid_boxes)[0], self.max_proposals_num), lambda: boxes_utils.padd_boxes_with_zeros(valid_boxes, valid_scores, self.max_proposals_num), lambda: (valid_boxes, valid_scores)) return rpn_proposals_boxes, rpn_proposals_scores
def dae(x, hparams, name): with tf.variable_scope(name): m = tf.layers.dense(x, hparams.v_size, name="mask") if hparams.softmax_k > 0: m, kl = top_k_softmax(m, hparams.softmax_k) return m, m, 1.0 - tf.reduce_mean(kl) logsm = tf.nn.log_softmax(m) # Gumbel-softmax sample. gumbel_samples = gumbel_sample(common_layers.shape_list(m)) steps = hparams.kl_warmup_steps gumbel_samples *= common_layers.inverse_exp_decay(steps // 5) * 0.5 temperature = 1.2 - common_layers.inverse_lin_decay(steps) # 10% of the time keep reasonably high temperature to keep learning. temperature = tf.cond(tf.less(tf.random_uniform([]), 0.9), lambda: temperature, lambda: tf.random_uniform([], minval=0.5, maxval=1.0)) s = tf.nn.softmax((logsm + gumbel_samples) / temperature) m = tf.nn.softmax(m) kl = - tf.reduce_max(logsm, axis=-1) if _DO_SUMMARIES: tf.summary.histogram("max-log", tf.reshape(kl, [-1])) # Calculate the argmax and construct hot vectors. maxvec = tf.reshape(tf.argmax(m, axis=-1), [-1]) maxvhot = tf.stop_gradient(tf.one_hot(maxvec, hparams.v_size)) # Add losses that prevent too few being used. distrib = tf.reshape(logsm, [-1, hparams.v_size]) * maxvhot d_mean = tf.reduce_mean(distrib, axis=[0], keep_dims=True) d_variance = tf.reduce_mean(tf.square(distrib - d_mean), axis=[0]) d_dev = - tf.reduce_mean(d_variance) ret = s if hparams.mode != tf.contrib.learn.ModeKeys.TRAIN: ret = tf.reshape(maxvhot, common_layers.shape_list(s)) # Just hot @eval. return m, ret, d_dev * 5.0 + tf.reduce_mean(kl) * 0.002
def testCond_2(self): with self.test_session(): x = tf.constant(10) r = tf.cond(tf.less(1, 0), lambda: tf.add(x, 1), lambda: tf.sub(x, 1)) result = r.eval() self.assertTrue(check_op_order(x.graph)) self.assertAllEqual(9, result)
def _get_values_from_start_and_end(self, input_tensor, num_start_samples, num_end_samples, total_num_samples): """slices num_start_samples and last num_end_samples from input_tensor. Args: input_tensor: An int32 tensor of shape [N] to be sliced. num_start_samples: Number of examples to be sliced from the beginning of the input tensor. num_end_samples: Number of examples to be sliced from the end of the input tensor. total_num_samples: Sum of is num_start_samples and num_end_samples. This should be a scalar. Returns: A tensor containing the first num_start_samples and last num_end_samples from input_tensor. """ input_length = tf.shape(input_tensor)[0] start_positions = tf.less(tf.range(input_length), num_start_samples) end_positions = tf.greater_equal( tf.range(input_length), input_length - num_end_samples) selected_positions = tf.logical_or(start_positions, end_positions) selected_positions = tf.cast(selected_positions, tf.int32) indexed_positions = tf.multiply(tf.cumsum(selected_positions), selected_positions) one_hot_selector = tf.one_hot(indexed_positions - 1, total_num_samples, dtype=tf.int32) return tf.tensordot(input_tensor, one_hot_selector, axes=[0, 0])
def compute_eval_loss_and_metrics(logits, # type: tf.Tensor softmax_logits, # type: tf.Tensor duplicate_mask, # type: tf.Tensor num_training_neg, # type: int match_mlperf=False, # type: bool use_tpu=False # type: bool ): # type: (...) -> tf.estimator.EstimatorSpec """Model evaluation with HR and NDCG metrics. The evaluation protocol is to rank the test interacted item (truth items) among the randomly chosen 999 items that are not interacted by the user. The performance of the ranked list is judged by Hit Ratio (HR) and Normalized Discounted Cumulative Gain (NDCG). For evaluation, the ranked list is truncated at 10 for both metrics. As such, the HR intuitively measures whether the test item is present on the top-10 list, and the NDCG accounts for the position of the hit by assigning higher scores to hits at top ranks. Both metrics are calculated for each test user, and the average scores are reported. If `match_mlperf` is True, then the HR and NDCG computations are done in a slightly unusual way to match the MLPerf reference implementation. Specifically, if the evaluation negatives contain duplicate items, it will be treated as if the item only appeared once. Effectively, for duplicate items in a row, the predicted score for all but one of the items will be set to -infinity For example, suppose we have that following inputs: logits_by_user: [[ 2, 3, 3], [ 5, 4, 4]] items_by_user: [[10, 20, 20], [30, 40, 40]] # Note: items_by_user is not explicitly present. Instead the relevant \ information is contained within `duplicate_mask` top_k: 2 Then with match_mlperf=True, the HR would be 2/2 = 1.0. With match_mlperf=False, the HR would be 1/2 = 0.5. This is because each user has predicted scores for only 2 unique items: 10 and 20 for the first user, and 30 and 40 for the second. Therefore, with match_mlperf=True, it's guaranteed the first item's score is in the top 2. With match_mlperf=False, this function would compute the first user's first item is not in the top 2, because item 20 has a higher score, and item 20 occurs twice. Args: logits: A tensor containing the predicted logits for each user. The shape of logits is (num_users_per_batch * (1 + NUM_EVAL_NEGATIVES),) Logits for a user are grouped, and the first element of the group is the true element. softmax_logits: The same tensor, but with zeros left-appended. duplicate_mask: A vector with the same shape as logits, with a value of 1 if the item corresponding to the logit at that position has already appeared for that user. num_training_neg: The number of negatives per positive during training. match_mlperf: Use the MLPerf reference convention for computing rank. use_tpu: Should the evaluation be performed on a TPU. Returns: An EstimatorSpec for evaluation. """ logits_by_user = tf.reshape(logits, (-1, rconst.NUM_EVAL_NEGATIVES + 1)) duplicate_mask_by_user = tf.reshape(duplicate_mask, (-1, rconst.NUM_EVAL_NEGATIVES + 1)) if match_mlperf: # Set duplicate logits to the min value for that dtype. The MLPerf # reference dedupes during evaluation. logits_by_user *= (1 - duplicate_mask_by_user) logits_by_user += duplicate_mask_by_user * logits_by_user.dtype.min # Determine the location of the first element in each row after the elements # are sorted. sort_indices = tf.contrib.framework.argsort( logits_by_user, axis=1, direction="DESCENDING") # Use matrix multiplication to extract the position of the true item from the # tensor of sorted indices. This approach is chosen because both GPUs and TPUs # perform matrix multiplications very quickly. This is similar to np.argwhere. # However this is a special case because the target will only appear in # sort_indices once. one_hot_position = tf.cast(tf.equal(sort_indices, 0), tf.int32) sparse_positions = tf.multiply( one_hot_position, tf.range(logits_by_user.shape[1])[tf.newaxis, :]) position_vector = tf.reduce_sum(sparse_positions, axis=1) in_top_k = tf.cast(tf.less(position_vector, rconst.TOP_K), tf.float32) ndcg = tf.log(2.) / tf.log(tf.cast(position_vector, tf.float32) + 2) ndcg *= in_top_k # If a row is a padded row, all but the first element will be a duplicate. metric_weights = tf.not_equal(tf.reduce_sum(duplicate_mask_by_user, axis=1), rconst.NUM_EVAL_NEGATIVES) # Examples are provided by the eval Dataset in a structured format, so eval # labels can be reconstructed on the fly. eval_labels = tf.reshape(tf.one_hot( tf.zeros(shape=(logits_by_user.shape[0],), dtype=tf.int32), logits_by_user.shape[1], dtype=tf.int32), (-1,)) eval_labels_float = tf.cast(eval_labels, tf.float32) # During evaluation, the ratio of negatives to positives is much higher # than during training. (Typically 999 to 1 vs. 4 to 1) By adjusting the # weights for the negative examples we compute a loss which is consistent with # the training data. (And provides apples-to-apples comparison) negative_scale_factor = num_training_neg / rconst.NUM_EVAL_NEGATIVES example_weights = ( (eval_labels_float + (1 - eval_labels_float) * negative_scale_factor) * (1 + rconst.NUM_EVAL_NEGATIVES) / (1 + num_training_neg)) # Tile metric weights back to logit dimensions expanded_metric_weights = tf.reshape(tf.tile( metric_weights[:, tf.newaxis], (1, rconst.NUM_EVAL_NEGATIVES + 1)), (-1,)) # ignore padded examples example_weights *= tf.cast(expanded_metric_weights, tf.float32) cross_entropy = tf.losses.sparse_softmax_cross_entropy( logits=softmax_logits, labels=eval_labels, weights=example_weights) def metric_fn(top_k_tensor, ndcg_tensor, weight_tensor): return { rconst.HR_KEY: tf.metrics.mean(top_k_tensor, weights=weight_tensor), rconst.NDCG_KEY: tf.metrics.mean(ndcg_tensor, weights=weight_tensor), } if use_tpu: return tf.contrib.tpu.TPUEstimatorSpec( mode=tf.estimator.ModeKeys.EVAL, loss=cross_entropy, eval_metrics=(metric_fn, [in_top_k, ndcg, metric_weights])) return tf.estimator.EstimatorSpec( mode=tf.estimator.ModeKeys.EVAL, loss=cross_entropy, eval_metric_ops=metric_fn(in_top_k, ndcg, metric_weights) )
def eval_right_length(example, target): l = tf.maximum(tf.shape(example['inputs'])[0], tf.shape(target)[0]) return tf.less(l, max_eval_length + 1)
def _continue_search(self, state) -> tf.Tensor: i = state[decoding_module.StateKeys.CUR_INDEX] return tf.less(i, self.max_decode_length)
def _relu(self, x, leakiness=0.0): """Relu, with optional leaky support.""" return tf.where(tf.less(x, 0.0), leakiness * x, x, name='leaky_relu')
def should_continue(self, threshold) -> bool: return tf.reduce_any(tf.less(self.halting_probability, threshold))
def add_loss(self): # Chi Squared self.chiSq = tf.reduce_mean( tf.divide(tf.square(self.prediction - self.fitData), self.fitDataVar)) # Minimize large R contribution self.rCut = tf.constant(1, tf.float32) self.rNorm = tf.einsum('nij,jk->nik', tf.abs(self.fitCoeffs), self.rVals) self.lowR = tf.reduce_mean( tf.pow((self.rNorm - self.rCut) / self.rCut, tf.ones_like(self.rNorm) * 6)) if self.parameters["Rrange"][0] < 1: zeroLowRcut = int(1. * self.parameters["NfitFxns"] / float(self.parameters["Rrange"][1] - self.parameters["Rrange"][0])) self.zeroLowR = tf.reduce_sum( tf.abs(self.fitCoeffs[:, :, :zeroLowRcut])) self.zeroLowR *= 1E4 else: self.zeroLowR = tf.constant(0, tf.float32) # Smooth fit coefficient distribution self.nnDiff = tf.divide( self.fitCoeffs[:, :, :-1] - self.fitCoeffs[:, :, 1:], (tf.abs(self.fitCoeffs[:, :, :-1]) + tf.abs(self.fitCoeffs[:, :, 1:])) / 2 + 1e-7) self.nnnDiff = tf.divide( self.fitCoeffs[:, :, :-2] - self.fitCoeffs[:, :, 2:], (tf.abs(self.fitCoeffs[:, :, :-2]) + tf.abs(self.fitCoeffs[:, :, 2:])) / 2 + 1e-7) self.nnVariance = (0.5)**2 self.nnnVariance = (0.7)**2 self.smooth = tf.reduce_mean(1 - tf.exp(-1 * tf.square(self.nnDiff) / (2 * self.nnVariance))) #self.smooth += tf.reduce_mean( # 1 - tf.exp(-1*tf.square(self.nnnDiff) # /(2*self.nnnVariance))) self.smooth *= self.chiSq # Remove negative coefficients self.nonNeg = tf.reduce_mean( tf.minimum(tf.zeros_like(self.fitCoeffs), self.fitCoeffs)) self.nonNeg *= -1000 # Dampen coefficients with small values (noise) self.noise = tf.reduce_mean(tf.where(tf.less( self.rNorm, 0.04 * tf.ones_like(self.rNorm)), tf.abs(self.rNorm), tf.zeros_like(self.rNorm), name="noise_where"), name="noise") self.noise *= tf.sqrt( tf.sqrt(tf.cast(self.global_step, dtype=tf.float32))) """ self.noise = tf.reduce_mean( tf.square( tf.where( tf.less(self.fitCoeffs, 0.01*tf.ones_like(self.fitCoeffs)), 1 + tf.abs(self.fitCoeffs), tf.zeros_like(self.fitCoeffs)))) """ """ self.noise = tf.reduce_mean( tf.abs( 1./(0.0051 - tf.where( tf.less(self.fitCoeffs, 0.005*tf.ones_like(self.fitCoeffs)), tf.abs(self.fitCoeffs), tf.zeros_like(self.fitCoeffs))))) self.noise /= 10000 """ if self.parameters["timeDepLoss"]: self.tdVariance = 0.5**2 self.tdDiff = tf.divide( self.fitCoeffs[:-1, :, :] - self.fitCoeffs[1:, :, :], (tf.abs(self.fitCoeffs[:-1, :, :]) + tf.abs(self.fitCoeffs[1:, :, :])) / 2 + 1e-7) self.nnTdDiff = tf.divide( self.fitCoeffs[:-1, :, :-1] - self.fitCoeffs[1:, :, 1:], (tf.abs(self.fitCoeffs[:-1, :, :-1]) + tf.abs(self.fitCoeffs[1:, :, 1:])) / 2 + 1e-7) self.nnnTdDiff = tf.divide( self.fitCoeffs[:-1, :, :-2] - self.fitCoeffs[1:, :, 2:], (tf.abs(self.fitCoeffs[:-1, :, :-2]) + tf.abs(self.fitCoeffs[1:, :, 2:])) / 2 + 1e-7) self.smoothTD = tf.reduce_mean(1 - tf.exp(-1 * tf.square(self.tdDiff) / (2 * self.tdVariance))) self.smoothTD += tf.reduce_mean(1 - tf.exp(-1 * tf.square(self.nnTdDiff) / (2 * self.tdVariance))) self.smoothTD += tf.reduce_mean(1 - tf.exp(-1 * tf.square(self.nnnTdDiff) / (2 * self.tdVariance))) self.smoothTD *= 0.5 * self.chiSq else: self.smoothTD = tf.constant(0, tf.float32) self.L1 = tf.reduce_mean(tf.abs(self.rNorm)) self.L1NR = tf.reduce_mean(tf.abs(self.fitCoeffs)) self.L2 = tf.reduce_mean(tf.square(self.rNorm)) self.loss = self.chiSq #+ self.lowR + self.zeroLowR if self.parameters["L1regularize"] is not None: self.loss += self.parameters["L1regularize"] * self.L1 if self.parameters["L1NRregularize"] is not None: self.loss += self.parameters["L1NRregularize"] * self.L1NR if self.parameters["L2regularize"] is not None: self.loss += self.parameters["L2regularize"] * self.L2 if self.parameters["smoothLoss"] is not None: self.loss += self.parameters["smoothLoss"] * self.smooth if self.parameters["nonNegLoss"]: self.loss += self.nonNeg if self.parameters["noiseLoss"] is not None: self.loss += self.parameters["noiseLoss"] * self.noise if self.parameters["timeDepLoss"]: self.loss += self.smoothTD
def _rand_erase(image, mask, weight, range_image, range_mask, range_weight, probability=0.5, min_size=0.02, max_size=0.4, min_aspect_ratio=0.3, max_aspect_ratio=1 / 0.3, pixel_wise=False, seed=None): # Generate seed for reproductivity if seed is not None: np.random.seed(seed) seeds = np.random.randint(np.iinfo(np.int32).min, np.iinfo(np.int32).max, size=[5]) seed_size, seed_ratio, seed_left, seed_top, seed_prob = seeds else: seed_size, seed_ratio, seed_left, seed_top, seed_prob = [None ] * 5 height, width, channels = image.get_shape().as_list() num_elems = tf.cast(height, dtype=tf.float32) * tf.cast( width, dtype=tf.float32) s = tf.random_uniform( (), min_size, max_size, seed=seed_size) * num_elems log_min_asp = tf.log(min_aspect_ratio) log_max_asp = tf.log(max_aspect_ratio) r = tf.exp( tf.random_uniform((), log_min_asp, log_max_asp, seed=seed_ratio)) w = tf.cast(tf.sqrt(s / r), dtype=tf.int32) h = tf.cast(tf.sqrt(s * r), dtype=tf.int32) w = tf.reduce_min([width, w]) h = tf.reduce_min([height, h]) left = tf.cond( tf.equal(w, width), lambda: 0, lambda: tf.random_uniform( (), 0, width - w, seed=seed_left, dtype=tf.int32)) top = tf.cond( tf.equal(h, height), lambda: 0, lambda: tf.random_uniform( (), 0, height - h, seed=seed_top, dtype=tf.int32)) erased_image = _rand_bbox(image, h, w, top, left, min_val=range_image[0], max_val=range_image[1], pixel_wise=pixel_wise) erased_mask = _rand_bbox(mask, h, w, top, left, min_val=range_mask[0], max_val=range_mask[1], pixel_wise=False) erased_weight = _rand_bbox(weight, h, w, top, left, min_val=range_weight[0], max_val=range_weight[1], pixel_wise=False) prob = tf.random_uniform((), seed=seed_prob) image = tf.cond(tf.less(prob, probability), true_fn=lambda: erased_image, false_fn=lambda: image) mask = tf.cond(tf.less(prob, probability), true_fn=lambda: erased_mask, false_fn=lambda: mask) weight = tf.cond(tf.less(prob, probability), true_fn=lambda: erased_weight, false_fn=lambda: weight) return image, mask, weight
def target_right_length(_, target): return tf.less(tf.shape(target)[0], max_target_length + 1)
def condition2(id_, outputs_ta2_, states_ta2_, e_ta_): # , outputs_ta2_, states_ta2_ return tf.less(id_, T_dim * S_dim)
def __init__(self, n_channel, n_classes, image_size, max_objects_per_image, cell_size, box_per_cell, object_scale, noobject_scale, coord_scale, class_scale, batch_size, noobject_thresh=0.6, recall_thresh=0.5, pred_thresh=0.5, nms_thresh=0.4): # 设置参数 self.n_classes = n_classes self.image_size = image_size self.n_channel = n_channel self.max_objects = max_objects_per_image self.cell_size = cell_size self.n_boxes = box_per_cell self.object_scale = float(object_scale) self.noobject_scale = float(noobject_scale) self.coord_scale = float(coord_scale) self.class_scale = float(class_scale) self.batch_size = batch_size self.noobject_thresh = noobject_thresh self.recall_thresh = recall_thresh self.pred_thresh = pred_thresh self.nms_thresh = nms_thresh # 输入变量 self.images = tf.placeholder(dtype=tf.float32, shape=[ self.batch_size, self.image_size, self.image_size, self.n_channel ], name='images') self.box_labels = tf.placeholder( dtype=tf.float32, shape=[self.batch_size, self.max_objects, 5], name='box_labels') self.object_nums = tf.placeholder(dtype=tf.int32, shape=[ self.batch_size, ], name='object_nums') self.keep_prob = tf.placeholder(dtype=tf.float32, name='keep_prob') self.global_step = tf.Variable(0, dtype=tf.int32, name='global_step') # 待输出的中间变量 self.logits = self.inference(self.images) self.coord_loss, self.object_loss, self.noobject_loss, self.class_loss, \ self.iou_value, self.object_value, self.nobject_value, \ self.recall_value, self.class_value = \ self.calculate_loss(self.logits) # 目标函数和优化器 tf.add_to_collection('losses', self.coord_loss) tf.add_to_collection('losses', self.object_loss) tf.add_to_collection('losses', self.noobject_loss) tf.add_to_collection('losses', self.class_loss) self.avg_loss = tf.add_n(tf.get_collection('losses')) # 设置学习率 lr = tf.cond( tf.less(self.global_step, 100), lambda: tf.constant(0.001), lambda: tf.cond( tf.less(self.global_step, 80000), lambda: tf.constant(0.01), lambda: tf.cond(tf.less(self.global_step, 100000), lambda: tf. constant(0.001), lambda: tf.constant(0.0001)))) self.optimizer = tf.train.MomentumOptimizer( learning_rate=0.001, momentum=0.9).minimize(self.avg_loss, global_step=self.global_step)
def _soft_sigmoid(x): return tf.where(tf.greater(x, 1.), x*0.0001, tf.where(tf.less(x, 0.), x*0.0001, x))
def range_prelu(x, upper_limit=4., outer_slope=0.0001): '''Quatery ReLU (0,4) ''' return tf.where(tf.logical_or(tf.greater(x, upper_limit), tf.less(x, 0.)), x*outer_slope, x)
def set_constants(self, constantArray, enzymeInit, activInitC, inhibInitC, computedRescaleFactor): """ Define the ops assigning the values for the network constants. :return: """ self.rescaleFactor.assign(computedRescaleFactor) enzymeInitTensor = enzymeInit * (computedRescaleFactor**0.5) self.k1.assign( tf.cast(tf.fill(self.k1.shape, constantArray[0]), dtype=tf.float64)) self.k1n.assign( tf.cast(tf.fill(self.k1n.shape, constantArray[1]), dtype=tf.float64)) self.k2.assign( tf.cast(tf.fill(self.k2.shape, constantArray[2]), dtype=tf.float64)) self.k3.assign( tf.cast(tf.fill(self.k3.shape, constantArray[3]), dtype=tf.float64)) self.k3n.assign( tf.cast(tf.fill(self.k3n.shape, constantArray[4]), dtype=tf.float64)) self.k4.assign( tf.cast(tf.fill(self.k4.shape, constantArray[5]), dtype=tf.float64)) self.k5.assign( tf.cast(tf.fill(self.k5.shape, constantArray[6]), dtype=tf.float64)) self.k5n.assign( tf.cast(tf.fill(self.k5n.shape, constantArray[7]), dtype=tf.float64)) self.k6.assign( tf.cast(tf.fill(self.k6.shape, constantArray[8]), dtype=tf.float64)) self.kdI.assign( tf.cast(tf.fill(self.kdI.shape, constantArray[9]), dtype=tf.float64)) self.kdT.assign( tf.cast(tf.fill(self.kdT.shape, constantArray[10]), dtype=tf.float64)) self.TA0.assign( tf.cast(tf.fill(self.TA0.shape, activInitC), dtype=tf.float64)) self.TI0.assign( tf.cast(tf.fill(self.TI0.shape, inhibInitC), dtype=tf.float64)) self.E0.assign(tf.constant(enzymeInitTensor, dtype=tf.float64)) #used in the first layer: self.firstLayerTA0.assign( tf.cast(tf.fill(self.firstLayerTA0.shape, activInitC), dtype=tf.float64)) self.firstLayerK1M.assign( tf.cast(tf.fill( self.firstLayerK1M.shape, constantArray[0] / (constantArray[1] + constantArray[2])), dtype=tf.float64)) self.firstLayerkdT.assign( tf.cast(tf.fill(self.firstLayerkdT.shape, constantArray[10]), dtype=tf.float64)) self.firstLayerk2.assign( tf.cast(tf.fill(self.firstLayerk2.shape, constantArray[2]), dtype=tf.float64)) #intermediate values for faster computations: self.k1M.assign(self.k1 / (self.k1n + self.k2)) self.Cactiv.assign(self.k2 * self.k1M * self.E0 * self.TA0) self.k5M.assign(self.k5 / (self.k5n + self.k6)) self.k3M.assign(self.k3 / (self.k3n + self.k4)) self.Cinhib.assign( tf.stack([self.k6 * self.k5M] * (self.k4.shape[0]), axis=0) * self.k4 * self.k3M * self.E0 * self.E0 * self.TI0) self.Kactiv.assign(self.k1M * self.TA0) self.Kinhib.assign(self.k3M * self.TI0) self.mask.assign( tf.cast(tf.where(tf.less(self.kernel, -0.2), -1., tf.where(tf.less(0.2, self.kernel), 1., 0.)), dtype=tf.float64)) self.cstList = [ self.k1, self.k1n, self.k2, self.k3, self.k3n, self.k4, self.k5, self.k5n, self.k6, self.kdI, self.kdT, self.TA0, self.E0, self.k1M, self.Cactiv, self.Cinhib, self.Kactiv, self.Kinhib, self.k5M, self.k3M, self.firstLayerTA0, self.firstLayerK1M, self.firstLayerkdT, self.firstLayerk2 ] self.cstListName = [ "self.k1", "self.k1n", "self.k2", "self.k3", "self.k3n", "self.k4", "self.k5", "self.k5n", "self.k6", "self.kdI", "self.kdT", "self.TA0", "self.E0", "self.k1M", "self.Cactiv", "self.Cinhib", "self.Kactiv", "self.Kinhib", "self.k5M", "self.k3M", "self.firstLayerTA0", "self.firstLayerK1M", "self.firstLayerkdT", "self.firstLayerk2" ]
def replace_with_zero_values_below_threshold(x_row): less_than_threshold = tf.less(x_row, threshold) return tf.where(less_than_threshold, tf.zeros((features_cnt, ), dtype=tf.float64), x_row)
def do_dual_max_match(overlap_matrix, low_thres, high_thres, ignore_between=True, gt_max_first=True): # overlap_matrix: num_gt * num_anchors with tf.name_scope('dual_max_match', [overlap_matrix]): anchors_to_gt = tf.argmax(overlap_matrix, axis=0) # anchor移动时第一个匹配 match_values = tf.reduce_max(overlap_matrix, axis=0) # 匹配的程度 less_mask = tf.less(match_values, low_thres) between_mask = tf.logical_and( tf.less(match_values, high_thres), tf.greater_equal(match_values, low_thres)) negative_mask = less_mask if ignore_between else between_mask ignore_mask = between_mask if ignore_between else less_mask # 负样本位置为-1,所有忽略的样本为-2 match_indices = tf.where(negative_mask, -1 * tf.ones_like(anchors_to_gt), anchors_to_gt) match_indices = tf.where(ignore_mask, -2 * tf.ones_like(match_indices), match_indices) anchors_to_gt_mask = tf.one_hot(tf.clip_by_value( match_indices, -1, tf.cast(tf.shape(overlap_matrix)[0], tf.int64)), tf.shape(overlap_matrix)[0], on_value=1, off_value=0, axis=0, dtype=tf.int32) gt_to_anchors = tf.argmax(overlap_matrix, axis=1) # 真实数据的移动匹配 if gt_max_first: # the max match from ground truth's side has higher priority left_gt_to_anchors_mask = tf.one_hot(gt_to_anchors, tf.shape(overlap_matrix)[1], on_value=1, off_value=0, axis=1, dtype=tf.int32) else: # the max match from anchors' side has higher priority # use match result from ground truth's side only when the the matching degree from anchors' side is lower than position threshold left_gt_to_anchors_mask = tf.cast( tf.logical_and( tf.reduce_max(anchors_to_gt_mask, axis=1, keep_dims=True) < 1, tf.one_hot(gt_to_anchors, tf.shape(overlap_matrix)[1], on_value=True, off_value=False, axis=1, dtype=tf.bool)), tf.int64) # can not use left_gt_to_anchors_mask here, because there are many ground truthes match to one anchor, we should pick the highest one even when we are merging matching from ground truth side left_gt_to_anchors_scores = overlap_matrix * tf.to_float( left_gt_to_anchors_mask) # merge matching results from ground truth's side with the original matching results from anchors' side # then select all the overlap score of those matching pairs selected_scores = tf.gather_nd( overlap_matrix, tf.stack([ tf.where( tf.reduce_max(left_gt_to_anchors_mask, axis=0) > 0, tf.argmax(left_gt_to_anchors_scores, axis=0), anchors_to_gt), tf.range(tf.cast(tf.shape(overlap_matrix)[1], tf.int64)) ], axis=1)) # return the matching results for both foreground anchors and background anchors, also with overlap scores return tf.where( tf.reduce_max(left_gt_to_anchors_mask, axis=0) > 0, tf.argmax(left_gt_to_anchors_scores, axis=0), match_indices), selected_scores
def set_mask(self, mask): Tminus = tf.cast(tf.fill(self.kernel.shape, -1), dtype=tf.float64) Tplus = tf.cast(tf.fill(self.kernel.shape, 1), dtype=tf.float64) Tzero = tf.cast(tf.fill(self.kernel.shape, 0), dtype=tf.float64) clippedKernel = tf.where( tf.less(self.kernel, -0.2), Tminus, tf.where(tf.less(0.2, self.kernel), Tplus, Tzero)) newClippedKernel = tf.where( mask * clippedKernel > 0., clippedKernel, tf.where(mask * clippedKernel < 0., (-1) * clippedKernel, mask)) newKernel = tf.where( tf.less(self.kernel, -0.2), tf.where( tf.less(newClippedKernel, 0.), self.kernel, tf.where(tf.less(0., newClippedKernel), (-1) * self.kernel, 0.)), # same or symetric tf.where( tf.less(0.2, self.kernel), tf.where( tf.less(0., newClippedKernel), self.kernel, tf.where(tf.less(newClippedKernel, 0.), (-1) * self.kernel, 0.)), tf.where( tf.less(newClippedKernel, 0.), -1., tf.where(tf.less(0, newClippedKernel), 1., self.kernel)))) self.kernel.assign(newKernel) self.mask.assign( tf.cast(tf.where(tf.less(self.kernel, -0.2), -1., tf.where(tf.less(0.2, self.kernel), 1., 0.)), dtype=tf.float64))
def __init__(self, nmaps, vec_size, niclass, noclass, dropout, max_grad_norm, cutoff, nconvs, kw, kh, height, mem_size, learning_rate, min_length, num_gpus, num_replicas, grad_noise_scale, sampling_rate, act_noise=0.0, do_rnn=False, atrous=False, beam_size=1, backward=True, do_layer_norm=False, autoenc_decay=1.0): # Feeds for parameters and ops to update them. self.nmaps = nmaps if backward: self.global_step = tf.Variable(0, trainable=False, name="global_step") self.cur_length = tf.Variable(min_length, trainable=False) self.cur_length_incr_op = self.cur_length.assign_add(1) self.lr = tf.Variable(learning_rate, trainable=False) self.lr_decay_op = self.lr.assign(self.lr * 0.995) self.do_training = tf.placeholder(tf.float32, name="do_training") self.update_mem = tf.placeholder(tf.int32, name="update_mem") self.noise_param = tf.placeholder(tf.float32, name="noise_param") # Feeds for inputs, targets, outputs, losses, etc. self.input = tf.placeholder(tf.int32, name="inp") self.target = tf.placeholder(tf.int32, name="tgt") self.prev_step = tf.placeholder(tf.float32, name="prev_step") gpu_input = tf.split(axis=0, num_or_size_splits=num_gpus, value=self.input) gpu_target = tf.split(axis=0, num_or_size_splits=num_gpus, value=self.target) gpu_prev_step = tf.split(axis=0, num_or_size_splits=num_gpus, value=self.prev_step) batch_size = tf.shape(gpu_input[0])[0] if backward: adam_lr = 0.005 * self.lr adam = tf.train.AdamOptimizer(adam_lr, epsilon=1e-3) def adam_update(grads): return adam.apply_gradients(zip(grads, tf.trainable_variables()), global_step=self.global_step, name="adam_update") # When switching from Adam to SGD we perform reverse-decay. if backward: global_step_float = tf.cast(self.global_step, tf.float32) sampling_decay_exponent = global_step_float / 100000.0 sampling_decay = tf.maximum(0.05, tf.pow(0.5, sampling_decay_exponent)) self.sampling = sampling_rate * 0.05 / sampling_decay else: self.sampling = tf.constant(0.0) # Cache variables on cpu if needed. if num_replicas > 1 or num_gpus > 1: with tf.device("/cpu:0"): caching_const = tf.constant(0) tf.get_variable_scope().set_caching_device(caching_const.op.device) # partitioner = tf.variable_axis_size_partitioner(1024*256*4) # tf.get_variable_scope().set_partitioner(partitioner) def gpu_avg(l): if l[0] is None: for elem in l: assert elem is None return 0.0 if len(l) < 2: return l[0] return sum(l) / float(num_gpus) self.length_tensor = tf.placeholder(tf.int32, name="length") with tf.device("/cpu:0"): emb_weights = tf.get_variable( "embedding", [niclass, vec_size], initializer=tf.random_uniform_initializer(-1.7, 1.7)) if beam_size > 0: target_emb_weights = tf.get_variable( "target_embedding", [noclass, nmaps], initializer=tf.random_uniform_initializer(-1.7, 1.7)) e0 = tf.scatter_update(emb_weights, tf.constant(0, dtype=tf.int32, shape=[1]), tf.zeros([1, vec_size])) output_w = tf.get_variable("output_w", [nmaps, noclass], tf.float32) def conv_rate(layer): if atrous: return 2**layer return 1 # pylint: disable=cell-var-from-loop def enc_step(step): """Encoder step.""" if autoenc_decay < 1.0: quant_step = autoenc_quantize(step, 16, nmaps, self.do_training) if backward: exp_glob = tf.train.exponential_decay( 1.0, self.global_step - 10000, 1000, autoenc_decay) dec_factor = 1.0 - exp_glob # * self.do_training dec_factor = tf.cond(tf.less(self.global_step, 10500), lambda: tf.constant(0.05), lambda: dec_factor) else: dec_factor = 1.0 cur = tf.cond(tf.less(tf.random_uniform([]), dec_factor), lambda: quant_step, lambda: step) else: cur = step if dropout > 0.0001: cur = tf.nn.dropout(cur, keep_prob) if act_noise > 0.00001: cur += tf.truncated_normal(tf.shape(cur)) * act_noise_scale # Do nconvs-many CGRU steps. if do_jit and tf.get_variable_scope().reuse: with jit_scope(): for layer in xrange(nconvs): cur = conv_gru([], cur, kw, kh, nmaps, conv_rate(layer), cutoff, "ecgru_%d" % layer, do_layer_norm) else: for layer in xrange(nconvs): cur = conv_gru([], cur, kw, kh, nmaps, conv_rate(layer), cutoff, "ecgru_%d" % layer, do_layer_norm) return cur zero_tgt = tf.zeros([batch_size, nmaps, 1]) zero_tgt.set_shape([None, nmaps, 1]) def dec_substep(step, decided): """Decoder sub-step.""" cur = step if dropout > 0.0001: cur = tf.nn.dropout(cur, keep_prob) if act_noise > 0.00001: cur += tf.truncated_normal(tf.shape(cur)) * act_noise_scale # Do nconvs-many CGRU steps. if do_jit and tf.get_variable_scope().reuse: with jit_scope(): for layer in xrange(nconvs): cur = conv_gru([decided], cur, kw, kh, nmaps, conv_rate(layer), cutoff, "dcgru_%d" % layer, do_layer_norm) else: for layer in xrange(nconvs): cur = conv_gru([decided], cur, kw, kh, nmaps, conv_rate(layer), cutoff, "dcgru_%d" % layer, do_layer_norm) return cur # pylint: enable=cell-var-from-loop def dec_step(step, it, it_int, decided, output_ta, tgts, mloss, nupd_in, out_idx, beam_cost): """Decoder step.""" nupd, mem_loss = 0, 0.0 if mem_size > 0: it_incr = tf.minimum(it + 1, length - 1) mem, mem_loss, nupd = memory_run( step, nmaps, mem_size, batch_size, noclass, self.global_step, self.do_training, self.update_mem, 10, num_gpus, target_emb_weights, output_w, gpu_targets_tn, it_incr) step = dec_substep(step, decided) output_l = tf.expand_dims(tf.expand_dims(step[:, it, 0, :], 1), 1) # Calculate argmax output. output = tf.reshape(output_l, [-1, nmaps]) # pylint: disable=cell-var-from-loop output = tf.matmul(output, output_w) if beam_size > 1: beam_cost, output, out, reordered = reorder_beam( beam_size, batch_size, beam_cost, output, it_int == 0, [output_l, out_idx, step, decided]) [output_l, out_idx, step, decided] = reordered else: # Scheduled sampling. out = tf.multinomial(tf.stop_gradient(output), 1) out = tf.to_int32(tf.squeeze(out, [1])) out_write = output_ta.write(it, output_l[:batch_size, :, :, :]) output = tf.gather(target_emb_weights, out) output = tf.reshape(output, [-1, 1, nmaps]) output = tf.concat(axis=1, values=[output] * height) tgt = tgts[it, :, :, :] selected = tf.cond(tf.less(tf.random_uniform([]), self.sampling), lambda: output, lambda: tgt) # pylint: enable=cell-var-from-loop dec_write = place_at14(decided, tf.expand_dims(selected, 1), it) out_idx = place_at13( out_idx, tf.reshape(out, [beam_size * batch_size, 1, 1]), it) if mem_size > 0: mem = tf.concat(axis=2, values=[mem] * height) dec_write = place_at14(dec_write, mem, it_incr) return (step, dec_write, out_write, mloss + mem_loss, nupd_in + nupd, out_idx, beam_cost) # Main model construction. gpu_outputs = [] gpu_losses = [] gpu_grad_norms = [] grads_list = [] gpu_out_idx = [] self.after_enc_step = [] for gpu in xrange( num_gpus): # Multi-GPU towers, average gradients later. length = self.length_tensor length_float = tf.cast(length, tf.float32) if gpu > 0: tf.get_variable_scope().reuse_variables() gpu_outputs.append([]) gpu_losses.append([]) gpu_grad_norms.append([]) with tf.name_scope("gpu%d" % gpu), tf.device("/gpu:%d" % gpu): # Main graph creation loop. data.print_out("Creating model.") start_time = time.time() # Embed inputs and calculate mask. with tf.device("/cpu:0"): tgt_shape = tf.shape(tf.squeeze(gpu_target[gpu], [1])) weights = tf.where( tf.squeeze(gpu_target[gpu], [1]) > 0, tf.ones(tgt_shape), tf.zeros(tgt_shape)) # Embed inputs and targets. with tf.control_dependencies([e0]): start = tf.gather(emb_weights, gpu_input[gpu]) # b x h x l x nmaps gpu_targets_tn = gpu_target[gpu] # b x 1 x len if beam_size > 0: embedded_targets_tn = tf.gather( target_emb_weights, gpu_targets_tn) embedded_targets_tn = tf.transpose( embedded_targets_tn, [2, 0, 1, 3]) # len x b x 1 x nmaps embedded_targets_tn = tf.concat( axis=2, values=[embedded_targets_tn] * height) # First image comes from start by applying convolution and adding 0s. start = tf.transpose(start, [0, 2, 1, 3]) # Now b x len x h x vec_s first = conv_linear(start, 1, 1, vec_size, nmaps, 1, True, 0.0, "input") first = layer_norm(first, nmaps, "input") # Computation steps. keep_prob = dropout * 3.0 / tf.sqrt(length_float) keep_prob = 1.0 - self.do_training * keep_prob act_noise_scale = act_noise * self.do_training # Start with a convolutional gate merging previous step. step = conv_gru([gpu_prev_step[gpu]], first, kw, kh, nmaps, 1, cutoff, "first", do_layer_norm) # This is just for running a baseline RNN seq2seq model. if do_rnn: self.after_enc_step.append( step) # Not meaningful here, but needed. def lstm_cell(): return tf.contrib.rnn.BasicLSTMCell(height * nmaps) cell = tf.contrib.rnn.MultiRNNCell( [lstm_cell() for _ in range(nconvs)]) with tf.variable_scope("encoder"): encoder_outputs, encoder_state = tf.nn.dynamic_rnn( cell, tf.reshape(step, [batch_size, length, height * nmaps]), dtype=tf.float32, time_major=False) # Attention. attn = tf.layers.dense(encoder_outputs, height * nmaps, name="attn1") # pylint: disable=cell-var-from-loop @function.Defun(noinline=True) def attention_query(query, attn_v): vecs = tf.tanh(attn + tf.expand_dims(query, 1)) mask = tf.reduce_sum( vecs * tf.reshape(attn_v, [1, 1, -1]), 2) mask = tf.nn.softmax(mask) return tf.reduce_sum( encoder_outputs * tf.expand_dims(mask, 2), 1) with tf.variable_scope("decoder"): def decoder_loop_fn(state__prev_cell_out__unused, cell_inp__cur_tgt): """Decoder loop function.""" state, prev_cell_out, _ = state__prev_cell_out__unused cell_inp, cur_tgt = cell_inp__cur_tgt attn_q = tf.layers.dense(prev_cell_out, height * nmaps, name="attn_query") attn_res = attention_query( attn_q, tf.get_variable( "attn_v", [height * nmaps], initializer=tf.random_uniform_initializer( -0.1, 0.1))) concatenated = tf.reshape( tf.concat(axis=1, values=[cell_inp, attn_res]), [batch_size, 2 * height * nmaps]) cell_inp = tf.layers.dense(concatenated, height * nmaps, name="attn_merge") output, new_state = cell(cell_inp, state) mem_loss = 0.0 if mem_size > 0: res, mask, mem_loss = memory_call( output, cur_tgt, height * nmaps, mem_size, noclass, num_gpus, self.update_mem) res = tf.gather(target_emb_weights, res) res *= tf.expand_dims(mask[:, 0], 1) output = tf.layers.dense(tf.concat( axis=1, values=[output, res]), height * nmaps, name="rnnmem") return new_state, output, mem_loss # pylint: enable=cell-var-from-loop gpu_targets = tf.squeeze(gpu_target[gpu], [1]) # b x len gpu_tgt_trans = tf.transpose(gpu_targets, [1, 0]) dec_zero = tf.zeros([batch_size, 1], dtype=tf.int32) dec_inp = tf.concat(axis=1, values=[dec_zero, gpu_targets]) dec_inp = dec_inp[:, :length] embedded_dec_inp = tf.gather(target_emb_weights, dec_inp) embedded_dec_inp_proj = tf.layers.dense( embedded_dec_inp, height * nmaps, name="dec_proj") embedded_dec_inp_proj = tf.transpose( embedded_dec_inp_proj, [1, 0, 2]) init_vals = (encoder_state, tf.zeros([batch_size, height * nmaps]), 0.0) _, dec_outputs, mem_losses = tf.scan( decoder_loop_fn, (embedded_dec_inp_proj, gpu_tgt_trans), initializer=init_vals) mem_loss = tf.reduce_mean(mem_losses) outputs = tf.layers.dense(dec_outputs, nmaps, name="out_proj") # Final convolution to get logits, list outputs. outputs = tf.matmul(tf.reshape(outputs, [-1, nmaps]), output_w) outputs = tf.reshape(outputs, [length, batch_size, noclass]) gpu_out_idx.append(tf.argmax(outputs, 2)) else: # Here we go with the Neural GPU. # Encoder. enc_length = length step = enc_step(step) # First step hard-coded. # pylint: disable=cell-var-from-loop i = tf.constant(1) c = lambda i, _s: tf.less(i, enc_length) def enc_step_lambda(i, step): with tf.variable_scope(tf.get_variable_scope(), reuse=True): new_step = enc_step(step) return (i + 1, new_step) _, step = tf.while_loop(c, enc_step_lambda, [i, step], parallel_iterations=1, swap_memory=True) # pylint: enable=cell-var-from-loop self.after_enc_step.append(step) # Decoder. if beam_size > 0: output_ta = tf.TensorArray(dtype=tf.float32, size=length, dynamic_size=False, infer_shape=False, name="outputs") out_idx = tf.zeros([beam_size * batch_size, length, 1], dtype=tf.int32) decided_t = tf.zeros( [beam_size * batch_size, length, height, vec_size]) # Prepare for beam search. tgts = tf.concat(axis=1, values=[embedded_targets_tn] * beam_size) beam_cost = tf.zeros([batch_size, beam_size]) step = tf.concat(axis=0, values=[step] * beam_size) # First step hard-coded. step, decided_t, output_ta, mem_loss, nupd, oi, bc = dec_step( step, 0, 0, decided_t, output_ta, tgts, 0.0, 0, out_idx, beam_cost) tf.get_variable_scope().reuse_variables() # pylint: disable=cell-var-from-loop def step_lambda(i, step, dec_t, out_ta, ml, nu, oi, bc): with tf.variable_scope(tf.get_variable_scope(), reuse=True): s, d, t, nml, nu, oi, bc = dec_step( step, i, 1, dec_t, out_ta, tgts, ml, nu, oi, bc) return (i + 1, s, d, t, nml, nu, oi, bc) i = tf.constant(1) c = lambda i, _s, _d, _o, _ml, _nu, _oi, _bc: tf.less( i, length) _, step, _, output_ta, mem_loss, nupd, out_idx, _ = tf.while_loop( c, step_lambda, [ i, step, decided_t, output_ta, mem_loss, nupd, oi, bc ], parallel_iterations=1, swap_memory=True) # pylint: enable=cell-var-from-loop gpu_out_idx.append(tf.squeeze(out_idx, [2])) outputs = output_ta.stack() outputs = tf.squeeze(outputs, [2, 3]) # Now l x b x nmaps else: # If beam_size is 0 or less, we don't have a decoder. mem_loss = 0.0 outputs = tf.transpose(step[:, :, 1, :], [1, 0, 2]) gpu_out_idx.append(tf.argmax(outputs, 2)) # Final convolution to get logits, list outputs. outputs = tf.matmul(tf.reshape(outputs, [-1, nmaps]), output_w) outputs = tf.reshape(outputs, [length, batch_size, noclass]) gpu_outputs[gpu] = tf.nn.softmax(outputs) # Calculate cross-entropy loss and normalize it. targets_soft = make_dense(tf.squeeze(gpu_target[gpu], [1]), noclass, 0.1) targets_soft = tf.reshape(targets_soft, [-1, noclass]) targets_hard = make_dense(tf.squeeze(gpu_target[gpu], [1]), noclass, 0.0) targets_hard = tf.reshape(targets_hard, [-1, noclass]) output = tf.transpose(outputs, [1, 0, 2]) xent_soft = tf.reshape( tf.nn.softmax_cross_entropy_with_logits( logits=tf.reshape(output, [-1, noclass]), labels=targets_soft), [batch_size, length]) xent_hard = tf.reshape( tf.nn.softmax_cross_entropy_with_logits( logits=tf.reshape(output, [-1, noclass]), labels=targets_hard), [batch_size, length]) low, high = 0.1 / float(noclass - 1), 0.9 const = high * tf.log(high) + float(noclass - 1) * low * tf.log(low) weight_sum = tf.reduce_sum(weights) + 1e-20 true_perp = tf.reduce_sum(xent_hard * weights) / weight_sum soft_loss = tf.reduce_sum(xent_soft * weights) / weight_sum perp_loss = soft_loss + const # Final loss: cross-entropy + shared parameter relaxation part + extra. mem_loss = 0.5 * tf.reduce_mean(mem_loss) / length_float total_loss = perp_loss + mem_loss gpu_losses[gpu].append(true_perp) # Gradients. if backward: data.print_out("Creating backward pass for the model.") grads = tf.gradients(total_loss, tf.trainable_variables(), colocate_gradients_with_ops=True) for g_i, g in enumerate(grads): if isinstance(g, tf.IndexedSlices): grads[g_i] = tf.convert_to_tensor(g) grads, norm = tf.clip_by_global_norm(grads, max_grad_norm) gpu_grad_norms[gpu].append(norm) for g in grads: if grad_noise_scale > 0.001: g += tf.truncated_normal( tf.shape(g)) * self.noise_param grads_list.append(grads) else: gpu_grad_norms[gpu].append(0.0) data.print_out("Created model for gpu %d in %.2f s." % (gpu, time.time() - start_time)) self.updates = [] self.after_enc_step = tf.concat( axis=0, values=self.after_enc_step) # Concat GPUs. if backward: tf.get_variable_scope()._reuse = False tf.get_variable_scope().set_caching_device(None) grads = [ gpu_avg([grads_list[g][i] for g in xrange(num_gpus)]) for i in xrange(len(grads_list[0])) ] update = adam_update(grads) self.updates.append(update) else: self.updates.append(tf.no_op()) self.losses = [ gpu_avg([gpu_losses[g][i] for g in xrange(num_gpus)]) for i in xrange(len(gpu_losses[0])) ] self.out_idx = tf.concat(axis=0, values=gpu_out_idx) self.grad_norms = [ gpu_avg([gpu_grad_norms[g][i] for g in xrange(num_gpus)]) for i in xrange(len(gpu_grad_norms[0])) ] self.outputs = [ tf.concat(axis=1, values=[gpu_outputs[g] for g in xrange(num_gpus)]) ] self.quantize_op = quantize_weights_op(512, 8) if backward: self.saver = tf.train.Saver(tf.global_variables(), max_to_keep=10)
def false_fn(): return tf.cast(tf.less(x[0], self._accuracy_threshold), tf.int32)
def body(depth_index, initial_state0, initial_state1, initial_state2, initial_state3, initial_state4, depth_image, max_prob_image, exp_sum, incre): """Loop body.""" # calculate cost ave_feature = ref_feature ave_feature2 = tf.square(ref_feature) warped_view_volumes = tf.zeros([batch_size, height, width, 1]) weight_sum = tf.zeros([batch_size, height, width, 1]) for view in range(0, FLAGS.view_num - 1): homographies = view_homographies[view] homographies = tf.transpose(homographies, perm=[1, 0, 2, 3]) homography = homographies[depth_index] view_feature = tf.squeeze( tf.slice(view_features, [0, view, 0, 0, 0], [-1, 1, -1, -1, -1]), 1) warped_view_feature = tf_transform_homography( view_feature, homography) warped_view_volume = tf.square(warped_view_feature - ref_feature) weight = gateNet(warped_view_volume, 32, name='gate') warped_view_volumes += (weight + 1) * warped_view_volume weight_sum += (weight + 1) cost = warped_view_volumes / weight_sum with tf.name_scope('cost_volume_homography') as scope: with tf.variable_scope("rnn/", reuse=tf.AUTO_REUSE): cost0, initial_state0 = cell0(cost, state=initial_state0) cost1 = tf.nn.max_pool2d(cost0, (2, 2), 2, 'SAME') cost1, initial_state1 = cell1(cost1, state=initial_state1) cost2 = tf.nn.max_pool2d(cost1, (2, 2), 2, 'SAME') cost2, initial_state2 = cell2(cost2, state=initial_state2) cost2 = deconv_gn(cost2, 16, 3, padding='same', strides=2, reuse=tf.AUTO_REUSE, name='cost_upconv0') cost2 = tf.concat([cost2, cost1], -1) cost3, initial_state3 = cell3(cost2, state=initial_state3) cost3 = deconv_gn(cost3, 16, 3, padding='same', strides=2, reuse=tf.AUTO_REUSE, name='cost_upconv1') cost3 = tf.concat([cost3, cost0], -1) cost4, initial_state4 = cell4(cost3, state=initial_state4) cost = tf.layers.conv2d(cost4, 1, 3, padding='same', reuse=tf.AUTO_REUSE, name='prob_conv') prob = tf.exp(-cost) # index d_idx = tf.cast(depth_index, tf.float32) if inverse_depth: inv_depth_start = tf.div(1.0, depth_start) inv_depth_end = tf.div(1.0, depth_end) inv_interval = (inv_depth_start - inv_depth_end) / ( tf.cast(depth_num, 'float32') - 1) inv_depth = inv_depth_start - d_idx * inv_interval depth = tf.div(1.0, inv_depth) else: depth = depth_start + d_idx * depth_interval temp_depth_image = tf.reshape(depth, [FLAGS.batch_size, 1, 1, 1]) temp_depth_image = tf.tile(temp_depth_image, [1, feature_shape[1], feature_shape[2], 1]) # update the best update_flag_image = tf.cast(tf.less(max_prob_image, prob), dtype='float32') new_max_prob_image = update_flag_image * prob + ( 1 - update_flag_image) * max_prob_image new_depth_image = update_flag_image * temp_depth_image + ( 1 - update_flag_image) * depth_image max_prob_image = tf.assign(max_prob_image, new_max_prob_image) depth_image = tf.assign(depth_image, new_depth_image) # update counter exp_sum = tf.assign_add(exp_sum, prob) depth_index = tf.add(depth_index, incre) return depth_index, initial_state0, initial_state1, initial_state2, initial_state3, initial_state4, depth_image, max_prob_image, exp_sum, incre
def cond(batch_i, point_i, b_inds): return tf.less(batch_i, tf.shape(stacks_len)[0])
def train_parse_prepare_preprocess_mapillary(example, params): _TRAIN_SIZE = (params.height_feature_extractor, params.width_feature_extractor) # example: a serialized tf example # proimage: 3D, tf.float32, _TRAIN_SIZE # prolabel: 2D, tf.int32, _TRAIN_SIZE ## parse keys_to_features = { 'image/encoded': tf.FixedLenFeature((), tf.string, default_value=None), 'image/format': tf.FixedLenFeature((), tf.string, default_value=b'jpeg'), 'image/dtype': tf.FixedLenFeature((), tf.string, default_value=b'uint8'), 'image/shape': tf.FixedLenFeature((3), tf.int64, default_value=None), 'image/path': tf.FixedLenFeature((), tf.string, default_value=None), 'label/encoded': tf.FixedLenFeature((), tf.string, default_value=None), 'label/format': tf.FixedLenFeature((), tf.string, default_value=b'png'), 'label/dtype': tf.FixedLenFeature((), tf.string, default_value=b'uint8'), 'label/shape': tf.FixedLenFeature((3), tf.int64, default_value=None), 'label/path': tf.FixedLenFeature((), tf.string, default_value=None)} features = tf.parse_single_example(example, keys_to_features) image = tf.image.decode_jpeg(features['image/encoded']) label = tf.image.decode_png(features['label/encoded']) # label = label[..., 0] im_path = features['image/path'] la_path = features['label/path'] ## prepare # TF internal bug: resize_images semantics don't correspond to uint8 images # it needs an input of [0,1] to work properly image = tf.image.convert_image_dtype(image, dtype=tf.float32) # label = tf.gather(tf.to_float(lids2cids), tf.to_int32(label)) label = tf.gather(tf.cast(params.training_lids2cids_mapil, tf.uint8), tf.to_int32(label)) print('debug: prepare:', image, label) ## preprocess ## rectify sizes before random cropping # make the smaller dimension at least as large as the respective train dimension def true_fn(image, label): # the size which is smaller than the respective average size will have bigger scale scales = _TRAIN_SIZE / tf.shape(image)[:2] new_size = tf.to_int32(tf.reduce_max(scales) * tf.saturate_cast(tf.shape(image)[:2], tf.float64)) # + 1 to new_size just in case to_int32 does floor rounding image = tf.image.resize_images(image, new_size + 1) # image has to be 3D label = tf.image.resize_images(label, new_size + 1, method=tf.image.ResizeMethod.NEAREST_NEIGHBOR) return image, label def false_fn(image, label): im_spatial_shape = tf.shape(image)[:2] mult = tf.constant(_TRAIN_SIZE)[tf.argmin(im_spatial_shape)] / tf.reduce_min(im_spatial_shape) new_spatial_size = tf.to_int32(tf.saturate_cast(im_spatial_shape, tf.float64) * mult) # + 1 to new_size just in case to_int32 does floor rounding image = tf.image.resize_images(image, new_spatial_size + 1) # image has to be 3D label = tf.image.resize_images(label, new_spatial_size + 1, method=tf.image.ResizeMethod.NEAREST_NEIGHBOR) return image, label # if any dim is smaller than the respective _TRAIN_SIZE rectify it image, label = tf.cond(tf.reduce_any(tf.less(tf.shape(image)[:2], _TRAIN_SIZE)), true_fn=lambda: true_fn(image, label), false_fn=lambda: false_fn(image, label)) # one more time same cond since false_fn can create smaller images image, label = tf.cond(tf.reduce_any(tf.less(tf.shape(image)[:2], _TRAIN_SIZE)), true_fn=lambda: true_fn(image, label), false_fn=lambda: (image, label)) # random crop # trick to randomly crop the same area from image and label # print('debug: before random crop:', image, label) # image = tf.Print(image, [tf.reduce_min(image), tf.reduce_max(image)], message='image min, max: ') image = tf.image.convert_image_dtype(image, tf.uint8, saturate=True) # so it can be concated combined = tf.concat([image, label], 2) combined = tf.random_crop(combined, _TRAIN_SIZE + tuple([4])) image = combined[..., :3] label = combined[..., 3] image = tf.image.convert_image_dtype(image, tf.float32) label = tf.to_int32(label) print('debug: crop:', image, label) # after this point image and label has _TRAIN_SIZE size # center input to [-1,1] is equivalent to assuming mean of 0.5 mean = 0.5 image = (image - mean)/mean proimage = image prolabel = label # proimage, prolabel = preprocess_train(image, label, params) return proimage, prolabel, im_path, la_path
def inference_winner_take_all(images, cams, depth_num, depth_start, depth_end, is_master_gpu=True, reg_type='GRU', inverse_depth=False): """ infer disparity image from stereo images and cameras """ if not inverse_depth: depth_interval = (depth_end - depth_start) / (tf.cast(depth_num, tf.float32) - 1) # reference image ref_image = tf.squeeze(tf.slice(images, [0, 0, 0, 0, 0], [-1, 1, -1, -1, 3]), axis=1) ref_cam = tf.squeeze(tf.slice(cams, [0, 0, 0, 0, 0], [-1, 1, 2, 4, 4]), axis=1) height = tf.shape(images)[2] width = tf.shape(images)[3] images = tf.reshape(images, [-1, height, width, 3]) feature_tower = SNetDS2GN_1({ 'data': images }, is_training=True, reuse=tf.AUTO_REUSE, dilation=1).get_output() height = tf.shape(feature_tower)[1] width = tf.shape(feature_tower)[2] features = tf.reshape( feature_tower, [FLAGS.batch_size, FLAGS.view_num, height, width, 32]) ref_feature = tf.squeeze( tf.slice(features, [0, 0, 0, 0, 0], [-1, 1, -1, -1, -1]), 1) view_features = tf.slice(features, [0, 1, 0, 0, 0], [-1, -1, -1, -1, -1]) # get all homographies view_homographies = [] for view in range(1, FLAGS.view_num): view_cam = tf.squeeze(tf.slice(cams, [0, view, 0, 0, 0], [-1, 1, 2, 4, 4]), axis=1) if inverse_depth: homographies = get_homographies_inv_depth(ref_cam, view_cam, depth_num=depth_num, depth_start=depth_start, depth_end=depth_end) else: homographies = get_homographies(ref_cam, view_cam, depth_num=depth_num, depth_start=depth_start, depth_interval=depth_interval) view_homographies.append(homographies) feature_shape = [FLAGS.batch_size, FLAGS.max_h, FLAGS.max_w, 32] batch_size, height, width, channel = feature_shape gru_input_shape = [feature_shape[1], feature_shape[2]] cell0 = ConvLSTMCell(conv_ndims=2, input_shape=[height, width, 32], output_channels=16, kernel_shape=[3, 3], name="conv_lstm_cell0") cell1 = ConvLSTMCell(conv_ndims=2, input_shape=[height / 2, width / 2, 16], output_channels=16, kernel_shape=[3, 3], name="conv_lstm_cell1") cell2 = ConvLSTMCell(conv_ndims=2, input_shape=[height / 4, width / 4, 16], output_channels=16, kernel_shape=[3, 3], name="conv_lstm_cell2") cell3 = ConvLSTMCell(conv_ndims=2, input_shape=[height / 2, width / 2, 32], output_channels=16, kernel_shape=[3, 3], name="conv_lstm_cell3") cell4 = ConvLSTMCell(conv_ndims=2, input_shape=[height, width, 32], output_channels=8, kernel_shape=[3, 3], name="conv_lstm_cell4") initial_state0 = cell0.zero_state(batch_size, dtype=tf.float32) initial_state1 = cell1.zero_state(batch_size, dtype=tf.float32) initial_state2 = cell2.zero_state(batch_size, dtype=tf.float32) initial_state3 = cell3.zero_state(batch_size, dtype=tf.float32) initial_state4 = cell4.zero_state(batch_size, dtype=tf.float32) # initialize variables exp_sum = tf.Variable(tf.zeros( [FLAGS.batch_size, feature_shape[1], feature_shape[2], 1]), name='exp_sum', trainable=False, collections=[tf.GraphKeys.LOCAL_VARIABLES]) depth_image = tf.Variable(tf.zeros( [FLAGS.batch_size, feature_shape[1], feature_shape[2], 1]), name='depth_image', trainable=False, collections=[tf.GraphKeys.LOCAL_VARIABLES]) max_prob_image = tf.Variable(tf.zeros( [FLAGS.batch_size, feature_shape[1], feature_shape[2], 1]), name='max_prob_image', trainable=False, collections=[tf.GraphKeys.LOCAL_VARIABLES]) init_map = tf.zeros( [FLAGS.batch_size, feature_shape[1], feature_shape[2], 1]) weights = tf.reshape(tf.constant([1.0, 0.5, 0.1], dtype=tf.float32), [1, 1, 1, 1, 3]) # define winner take all loop def body(depth_index, initial_state0, initial_state1, initial_state2, initial_state3, initial_state4, depth_image, max_prob_image, exp_sum, incre): """Loop body.""" # calculate cost ave_feature = ref_feature ave_feature2 = tf.square(ref_feature) warped_view_volumes = tf.zeros([batch_size, height, width, 1]) weight_sum = tf.zeros([batch_size, height, width, 1]) for view in range(0, FLAGS.view_num - 1): homographies = view_homographies[view] homographies = tf.transpose(homographies, perm=[1, 0, 2, 3]) homography = homographies[depth_index] view_feature = tf.squeeze( tf.slice(view_features, [0, view, 0, 0, 0], [-1, 1, -1, -1, -1]), 1) warped_view_feature = tf_transform_homography( view_feature, homography) warped_view_volume = tf.square(warped_view_feature - ref_feature) weight = gateNet(warped_view_volume, 32, name='gate') warped_view_volumes += (weight + 1) * warped_view_volume weight_sum += (weight + 1) cost = warped_view_volumes / weight_sum with tf.name_scope('cost_volume_homography') as scope: with tf.variable_scope("rnn/", reuse=tf.AUTO_REUSE): cost0, initial_state0 = cell0(cost, state=initial_state0) cost1 = tf.nn.max_pool2d(cost0, (2, 2), 2, 'SAME') cost1, initial_state1 = cell1(cost1, state=initial_state1) cost2 = tf.nn.max_pool2d(cost1, (2, 2), 2, 'SAME') cost2, initial_state2 = cell2(cost2, state=initial_state2) cost2 = deconv_gn(cost2, 16, 3, padding='same', strides=2, reuse=tf.AUTO_REUSE, name='cost_upconv0') cost2 = tf.concat([cost2, cost1], -1) cost3, initial_state3 = cell3(cost2, state=initial_state3) cost3 = deconv_gn(cost3, 16, 3, padding='same', strides=2, reuse=tf.AUTO_REUSE, name='cost_upconv1') cost3 = tf.concat([cost3, cost0], -1) cost4, initial_state4 = cell4(cost3, state=initial_state4) cost = tf.layers.conv2d(cost4, 1, 3, padding='same', reuse=tf.AUTO_REUSE, name='prob_conv') prob = tf.exp(-cost) # index d_idx = tf.cast(depth_index, tf.float32) if inverse_depth: inv_depth_start = tf.div(1.0, depth_start) inv_depth_end = tf.div(1.0, depth_end) inv_interval = (inv_depth_start - inv_depth_end) / ( tf.cast(depth_num, 'float32') - 1) inv_depth = inv_depth_start - d_idx * inv_interval depth = tf.div(1.0, inv_depth) else: depth = depth_start + d_idx * depth_interval temp_depth_image = tf.reshape(depth, [FLAGS.batch_size, 1, 1, 1]) temp_depth_image = tf.tile(temp_depth_image, [1, feature_shape[1], feature_shape[2], 1]) # update the best update_flag_image = tf.cast(tf.less(max_prob_image, prob), dtype='float32') new_max_prob_image = update_flag_image * prob + ( 1 - update_flag_image) * max_prob_image new_depth_image = update_flag_image * temp_depth_image + ( 1 - update_flag_image) * depth_image max_prob_image = tf.assign(max_prob_image, new_max_prob_image) depth_image = tf.assign(depth_image, new_depth_image) # update counter exp_sum = tf.assign_add(exp_sum, prob) depth_index = tf.add(depth_index, incre) return depth_index, initial_state0, initial_state1, initial_state2, initial_state3, initial_state4, depth_image, max_prob_image, exp_sum, incre # run forward loop exp_sum = tf.assign(exp_sum, init_map) depth_image = tf.assign(depth_image, init_map) max_prob_image = tf.assign(max_prob_image, init_map) depth_index = tf.constant(0) incre = tf.constant(1) cond = lambda depth_index, *_: tf.less(depth_index, depth_num) _, initial_state0, initial_state1, initial_state2, initial_state3, initial_state4, depth_image, max_prob_image, exp_sum, incre = tf.while_loop( cond, body, [ depth_index, initial_state0, initial_state1, initial_state2, initial_state3, initial_state4, depth_image, max_prob_image, exp_sum, incre ], back_prop=False, parallel_iterations=1) # get output forward_exp_sum = exp_sum + 1e-7 forward_depth_map = depth_image return forward_depth_map, max_prob_image / forward_exp_sum
y = tf.random_uniform([]) out = tf.cond(tf.greater(x, y), lambda: tf.add(x, y), lambda: tf.subtract(x, y)) ############################################################################### # 1b: Create two 0-d tensors x and y randomly selected from the range [-1, 1). # Return x + y if x < y, x - y if x > y, 0 otherwise. # Hint: Look up tf.case(). ############################################################################### # YOUR CODE x = tf.random_uniform([], minval=-1, maxval=1, dtype=tf.float32) y = tf.random_uniform([], minval=-1, maxval=1, dtype=tf.float32) out_1b = tf.case( { tf.less(x, y): lambda: tf.add(x, y), tf.greater(x, y): lambda: tf.subtract(x, y) }, default=lambda: tf.constant(0.0), exclusive=True) sess = tf.InteractiveSession() print(x) print(y) print(sess.run(out_1b)) ############################################################################### # 1c: Create the tensor x of the value [[0, -2, -1], [0, 1, 2]] # and y as a tensor of zeros with the same shape as x. # Return a boolean tensor that yields Trues if x equals y element-wise. # Hint: Look up tf.equal().
def smooth_l1_loss(x): square_loss = 0.5*x**2 absolute_loss = tf.abs(x) return tf.where(tf.less(absolute_loss, 1.), square_loss, absolute_loss-0.5)
def get_train_ops(loss, tf_variables, train_step, clip_mode=None, grad_bound=None, l2_reg=1e-4, lr_warmup_val=None, lr_warmup_steps=100, lr_init=0.1, lr_dec_start=0, lr_dec_every=10000, lr_dec_rate=0.1, lr_dec_min=None, lr_cosine=False, lr_max=None, lr_min=None, lr_T_0=None, lr_T_mul=None, num_train_batches=None, optim_algo=None, sync_replicas=False, num_aggregate=None, num_replicas=None, get_grad_norms=False, moving_average=None): """ Args: clip_mode: "global", "norm", or None. moving_average: store the moving average of parameters """ if l2_reg > 0: l2_losses = [] for var in tf_variables: l2_losses.append(tf.reduce_sum(var**2)) l2_loss = tf.add_n(l2_losses) loss += l2_reg * l2_loss grads = tf.gradients(loss, tf_variables) grad_norm = tf.global_norm(grads) grad_norms = {} for v, g in zip(tf_variables, grads): if v is None or g is None: continue if isinstance(g, tf.IndexedSlices): grad_norms[v.name] = tf.sqrt(tf.reduce_sum(g.values**2)) else: grad_norms[v.name] = tf.sqrt(tf.reduce_sum(g**2)) if clip_mode is not None: assert grad_bound is not None, "Need grad_bound to clip gradients." if clip_mode == "global": grads, _ = tf.clip_by_global_norm(grads, grad_bound) elif clip_mode == "norm": clipped = [] for g in grads: if isinstance(g, tf.IndexedSlices): c_g = tf.clip_by_norm(g.values, grad_bound) c_g = tf.IndexedSlices(g.indices, c_g) else: c_g = tf.clip_by_norm(g, grad_bound) clipped.append(g) grads = clipped else: raise NotImplementedError("Unknown clip_mode {}".format(clip_mode)) if lr_cosine: assert lr_max is not None, "Need lr_max to use lr_cosine" assert lr_min is not None, "Need lr_min to use lr_cosine" assert lr_T_0 is not None, "Need lr_T_0 to use lr_cosine" assert lr_T_mul is not None, "Need lr_T_mul to use lr_cosine" assert num_train_batches is not None, ("Need num_train_batches to use" " lr_cosine") curr_epoch = train_step // num_train_batches last_reset = tf.Variable(0, dtype=tf.int32, trainable=False, name="last_reset") T_i = tf.Variable(lr_T_0, dtype=tf.int32, trainable=False, name="T_i") T_curr = curr_epoch - last_reset def _update(): update_last_reset = tf.assign(last_reset, curr_epoch, use_locking=True) update_T_i = tf.assign(T_i, T_i * lr_T_mul, use_locking=True) with tf.control_dependencies([update_last_reset, update_T_i]): rate = tf.to_float(T_curr) / tf.to_float(T_i) * 3.1415926 lr = lr_min + 0.5 * (lr_max - lr_min) * (1.0 + tf.cos(rate)) return lr def _no_update(): rate = tf.to_float(T_curr) / tf.to_float(T_i) * 3.1415926 lr = lr_min + 0.5 * (lr_max - lr_min) * (1.0 + tf.cos(rate)) return lr learning_rate = tf.cond(tf.greater_equal(T_curr, T_i), _update, _no_update) else: learning_rate = tf.train.exponential_decay( lr_init, tf.maximum(train_step - lr_dec_start, 0), lr_dec_every, lr_dec_rate, staircase=True) if lr_dec_min is not None: learning_rate = tf.maximum(learning_rate, lr_dec_min) if lr_warmup_val is not None: learning_rate = tf.cond(tf.less(train_step, lr_warmup_steps), lambda: lr_warmup_val, lambda: learning_rate) # if get_grad_norms: # g_1, g_2 = 0.0001, 0.0001 # for v, g in zip(tf_variables, grads): # if g is not None: # if isinstance(g, tf.IndexedSlices): # g_n = tf.reduce_sum(g.values ** 2) # else: # g_n = tf.reduce_sum(g ** 2) # if "enas_cell" in v.name: # print("g_1: {}".format(v.name)) # g_1 += g_n # else: # print("g_2: {}".format(v.name)) # g_2 += g_n # learning_rate = tf.Print(learning_rate, [g_1, g_2, tf.sqrt(g_1 / g_2)], # message="g_1, g_2, g_1/g_2: ", summarize=5) if optim_algo == "momentum": opt = tf.train.MomentumOptimizer(learning_rate, 0.9, use_locking=True, use_nesterov=True) elif optim_algo == "sgd": opt = tf.train.GradientDescentOptimizer(learning_rate, use_locking=True) elif optim_algo == "adam": opt = tf.train.AdamOptimizer(learning_rate, beta1=0.0, epsilon=1e-3, use_locking=True) else: raise ValueError("Unknown optim_algo {}".format(optim_algo)) if sync_replicas: assert num_aggregate is not None, "Need num_aggregate to sync." assert num_replicas is not None, "Need num_replicas to sync." opt = tf.train.SyncReplicasOptimizer( opt, replicas_to_aggregate=num_aggregate, total_num_replicas=num_replicas, use_locking=True) if moving_average is not None: opt = tf.contrib.opt.MovingAverageOptimizer( opt, average_decay=moving_average) train_op = opt.apply_gradients(zip(grads, tf_variables), global_step=train_step) if get_grad_norms: return train_op, learning_rate, grad_norm, opt, grad_norms else: return train_op, learning_rate, grad_norm, opt
def _leaky_relu(self, x, leakiness=0.0): return tf.where(tf.less(x, 0.0), leakiness * x, x, name='leaky_relu')
def compile(self, k=1, optimizer='adam', learning_rate=1e-4, mu=0.5, margin=1): """ Builds the tensorflow graph that is evaluated in the fit method Arguments: k: integer, number of target neighbours optimizer: string, name of optimizer to use. See dlmnn.helper.utility for which optimizers that are supported learning_rate: scalar, learning rate for optimizer mu: scalar, weighting of the pull and push term. Should be between 0 and 1. High values put weight on the push term and vice verse for the pull term margin: scalar, size of margin inforcing between similar pairs and imposters. Should be higher than 0. """ assert k > 0 and isinstance( k, int), ''' k need to be a positive integer ''' assert learning_rate > 0, ''' learning rate needs to be a positive number ''' assert 0 <= mu and mu <= 1, ''' mu needs to be between 0 and 1 ''' assert margin > 0, ''' margin needs to be a positive number ''' assert len( self.extractor.layers) != 0, '''Layers must be added with the lmnn.add() method before this function is called ''' self.built = True # Set number of neighbours self.k = k # Shapes self.input_shape = self.extractor.input_shape self.output_shape = self.extractor.output_shape # Placeholders for data self.global_step = tf.Variable(0, trainable=False) self.Xp = tf.placeholder(tf.float32, shape=self.input_shape, name='In_features') self.yp = tf.placeholder(tf.int32, shape=(None, ), name='In_targets') self.tNp = tf.placeholder(tf.int32, shape=(None, 2), name='In_targetNeighbours') # Feature extraction function and pairwise distance function self.extractor_func = tf_featureExtractor(self.extractor) self.dist_func = tf_makePairwiseFunc(self.extractor_func) # Build graph #D = self.dist_func(self.Xp, self.Xp) D = self.dist_func(self.Xp) tup = tf_findImposters(D, self.yp, self.tNp, margin=margin) self._LMNN_loss, D_1, D_2, D_3 = tf_LMNN_loss(D, self.tNp, tup, mu, margin=margin) # Construct training operation self._optimizer = get_optimizer(optimizer)(learning_rate=learning_rate) self._trainer = self._optimizer.minimize(self._LMNN_loss, global_step=self.global_step) # Summaries self._n_tup = tf.shape(tup)[0] self._true_imp = tf.cast(tf.less(D_3, D_2), tf.float32) features = self.extractor_func(self.Xp) tf.summary.scalar('Loss', self._LMNN_loss) tf.summary.scalar('Num_imp', self._n_tup) tf.summary.scalar('Loss_pull', tf.reduce_sum(D_1)) tf.summary.scalar('Loss_push', tf.reduce_sum(margin + D_2 - D_3)) tf.summary.scalar('True_imp', tf.reduce_sum(self._true_imp)) tf.summary.scalar('Frac_true_imp', tf.reduce_mean(self._true_imp)) tf.summary.scalar( 'Sparsity_tanh', tf.reduce_mean( tf.reduce_sum(tf.tanh(tf.pow(features, 2.0)), axis=1))) tf.summary.scalar( 'Sparsity_l0', tf.reduce_mean( tf.reduce_sum(tf.cast(tf.equal(features, 0), tf.int32), axis=1))) self._summary = tf.summary.merge_all() # Initilize session init = tf.global_variables_initializer() self.session.run(init) # Create callable functions self._transformer = self.session.make_callable( self.extractor_func(self.Xp), [self.Xp]) # self._distances = self.session.make_callable( # self.dist_func(self.Xp, self.Xp), [self.Xp]) self._distances = self.session.make_callable(self.dist_func(self.Xp), [self.Xp])
def _cond(x_in, y_in, domain_in, i, cond_in): # Repeat the loop until we have achieved misclassification or # reaches the maximum iterations return tf.logical_and(tf.less(i, self.iterations), cond_in)
def _is_valid_box(ymin, xmin, ymax, xmax): return tf.logical_and(tf.less(ymin, ymax), tf.less(xmin, xmax))
def build_optimizer(self, learning_rate=0.001, weight_decay=0.0005, momentum=0.9, global_step=None): self.labels = tf.placeholder(tf.float32, name='labels', shape=[None, None, self.num_vars]) with tf.variable_scope('ground_truth'): #------------------------------------------------------------------- # Split the ground truth tensor #------------------------------------------------------------------- # Classification ground truth tensor # Shape: (batch_size, num_anchors, num_classes) gt_cl = self.labels[:,:,:self.num_classes] # Localization ground truth tensor # Shape: (batch_size, num_anchors, 4) gt_loc = self.labels[:,:,self.num_classes:] # Batch size # Shape: scalar batch_size = tf.shape(gt_cl)[0] #----------------------------------------------------------------------- # Compute match counters #----------------------------------------------------------------------- with tf.variable_scope('match_counters'): # Number of anchors per sample # Shape: (batch_size) total_num = tf.ones([batch_size], dtype=tf.int64) * \ tf.to_int64(self.preset.num_anchors) # Number of negative (not-matched) anchors per sample, computed # by counting boxes of the background class in each sample. # Shape: (batch_size) negatives_num = tf.count_nonzero(gt_cl[:,:,-1], axis=1) # Number of positive (matched) anchors per sample # Shape: (batch_size) positives_num = total_num-negatives_num # Number of positives per sample that is division-safe # Shape: (batch_size) positives_num_safe = tf.where(tf.equal(positives_num, 0), tf.ones([batch_size])*10e-15, tf.to_float(positives_num)) #----------------------------------------------------------------------- # Compute masks #----------------------------------------------------------------------- with tf.variable_scope('match_masks'): # Boolean tensor determining whether an anchor is a positive # Shape: (batch_size, num_anchors) positives_mask = tf.equal(gt_cl[:,:,-1], 0) # Boolean tensor determining whether an anchor is a negative # Shape: (batch_size, num_anchors) negatives_mask = tf.logical_not(positives_mask) #----------------------------------------------------------------------- # Compute the confidence loss #----------------------------------------------------------------------- with tf.variable_scope('confidence_loss'): # Cross-entropy tensor - all of the values are non-negative # Shape: (batch_size, num_anchors) ce = tf.nn.softmax_cross_entropy_with_logits_v2(labels=gt_cl, logits=self.logits) #------------------------------------------------------------------- # Sum up the loss of all the positive anchors #------------------------------------------------------------------- # Positives - the loss of negative anchors is zeroed out # Shape: (batch_size, num_anchors) positives = tf.where(positives_mask, ce, tf.zeros_like(ce)) # Total loss of positive anchors # Shape: (batch_size) positives_sum = tf.reduce_sum(positives, axis=-1) #------------------------------------------------------------------- # Figure out what the negative anchors with highest confidence loss # are #------------------------------------------------------------------- # Negatives - the loss of positive anchors is zeroed out # Shape: (batch_size, num_anchors) negatives = tf.where(negatives_mask, ce, tf.zeros_like(ce)) # Top negatives - sorted confience loss with the highest one first # Shape: (batch_size, num_anchors) negatives_top = tf.nn.top_k(negatives, self.preset.num_anchors)[0] #------------------------------------------------------------------- # Fugure out what the number of negatives we want to keep is #------------------------------------------------------------------- # Maximum number of negatives to keep per sample - we keep at most # 3 times as many as we have positive anchors in the sample # Shape: (batch_size) negatives_num_max = tf.minimum(negatives_num, 3*positives_num) #------------------------------------------------------------------- # Mask out superfluous negatives and compute the sum of the loss #------------------------------------------------------------------- # Transposed vector of maximum negatives per sample # Shape (batch_size, 1) negatives_num_max_t = tf.expand_dims(negatives_num_max, 1) # Range tensor: [0, 1, 2, ..., num_anchors-1] # Shape: (num_anchors) rng = tf.range(0, self.preset.num_anchors, 1) # Row range, the same as above, but int64 and a row of a matrix # Shape: (1, num_anchors) range_row = tf.to_int64(tf.expand_dims(rng, 0)) # Mask of maximum negatives - first `negative_num_max` elements # in corresponding row are `True`, the rest is false # Shape: (batch_size, num_anchors) negatives_max_mask = tf.less(range_row, negatives_num_max_t) # Max negatives - all the positives and superfluous negatives are # zeroed out. # Shape: (batch_size, num_anchors) negatives_max = tf.where(negatives_max_mask, negatives_top, tf.zeros_like(negatives_top)) # Sum of max negatives for each sample # Shape: (batch_size) negatives_max_sum = tf.reduce_sum(negatives_max, axis=-1) #------------------------------------------------------------------- # Compute the confidence loss for each element #------------------------------------------------------------------- # Total confidence loss for each sample # Shape: (batch_size) confidence_loss = tf.add(positives_sum, negatives_max_sum) # Total confidence loss normalized by the number of positives # per sample # Shape: (batch_size) confidence_loss = tf.where(tf.equal(positives_num, 0), tf.zeros([batch_size]), tf.div(confidence_loss, positives_num_safe)) # Mean confidence loss for the batch # Shape: scalar self.confidence_loss = tf.reduce_mean(confidence_loss, name='confidence_loss') #----------------------------------------------------------------------- # Compute the localization loss #----------------------------------------------------------------------- with tf.variable_scope('localization_loss'): # Element-wise difference between the predicted localization loss # and the ground truth # Shape: (batch_size, num_anchors, 4) loc_diff = tf.subtract(self.locator, gt_loc) # Smooth L1 loss # Shape: (batch_size, num_anchors, 4) loc_loss = smooth_l1_loss(loc_diff) # Sum of localization losses for each anchor # Shape: (batch_size, num_anchors) loc_loss_sum = tf.reduce_sum(loc_loss, axis=-1) # Positive locs - the loss of negative anchors is zeroed out # Shape: (batch_size, num_anchors) positive_locs = tf.where(positives_mask, loc_loss_sum, tf.zeros_like(loc_loss_sum)) # Total loss of positive anchors # Shape: (batch_size) localization_loss = tf.reduce_sum(positive_locs, axis=-1) # Total localization loss normalized by the number of positives # per sample # Shape: (batch_size) localization_loss = tf.where(tf.equal(positives_num, 0), tf.zeros([batch_size]), tf.div(localization_loss, positives_num_safe)) # Mean localization loss for the batch # Shape: scalar self.localization_loss = tf.reduce_mean(localization_loss, name='localization_loss') #----------------------------------------------------------------------- # Compute total loss #----------------------------------------------------------------------- with tf.variable_scope('total_loss'): # Sum of the localization and confidence loss # Shape: (batch_size) self.conf_and_loc_loss = tf.add(self.confidence_loss, self.localization_loss, name='sum_losses') # L2 loss # Shape: scalar self.l2_loss = tf.multiply(weight_decay, self.l2_loss, name='l2_loss') # Final loss # Shape: scalar self.loss = tf.add(self.conf_and_loc_loss, self.l2_loss, name='loss') #----------------------------------------------------------------------- # Build the optimizer #----------------------------------------------------------------------- with tf.variable_scope('optimizer'): optimizer = tf.train.MomentumOptimizer(learning_rate, momentum) optimizer = optimizer.minimize(self.loss, global_step=global_step, name='optimizer') #----------------------------------------------------------------------- # Store the tensors #----------------------------------------------------------------------- self.optimizer = optimizer self.losses = { 'total': self.loss, 'localization': self.localization_loss, 'confidence': self.confidence_loss, 'l2': self.l2_loss }