def _g_recurrence_2(i, x_t, gen_x, h_tm1, h_tm1_manager, last_goal, real_goal): # with tf.device('/cpu:0'): cur_sen = tf.cond(i > 0, lambda: tf.split(tf.concat([tf.transpose(gen_x.stack(), perm=[1, 0]), self.padding_array], 1), [self.sequence_length, i - 1], 1)[0], lambda: self.padding_array) with tf.variable_scope(self.scope): feature = self.FeatureExtractor_unit(cur_sen, self.drop_out) h_t_Worker = self.g_worker_recurrent_unit(x_t, h_tm1) # hidden_memory_tuple o_t_Worker = self.g_worker_output_unit(h_t_Worker) # batch x vocab , logits not prob o_t_Worker = tf.reshape(o_t_Worker, [self.batch_size, self.num_vocabulary, self.goal_size]) h_t_manager = self.g_manager_recurrent_unit(feature, h_tm1_manager) sub_goal = self.g_manager_output_unit(h_t_manager) sub_goal = tf.nn.l2_normalize(sub_goal, 1) real_sub_goal = tf.add(last_goal, sub_goal) w_g = tf.matmul(real_goal, self.g_change) # batch x goal_size w_g = tf.nn.l2_normalize(w_g, 1) w_g = tf.expand_dims(w_g, 2) # batch x goal_size x 1 x_logits = tf.matmul(o_t_Worker, w_g) x_logits = tf.squeeze(x_logits) log_prob = tf.log(tf.nn.softmax(x_logits)) next_token = tf.cast(tf.reshape(tf.multinomial(log_prob, 1), [self.batch_size]), tf.int32) x_tp1 = tf.nn.embedding_lookup(self.g_embeddings, next_token) # batch x emb_dim with tf.control_dependencies([cur_sen]): gen_x = gen_x.write(i - 1, next_token) # indices, batch_size return i + 1, x_tp1, gen_x, h_t_Worker, h_t_manager, \ tf.cond(((i) % self.step_size) > 0, lambda: real_sub_goal, lambda: tf.constant(0.0, shape=[self.batch_size, self.goal_out_size])), \ tf.cond(((i) % self.step_size) > 0, lambda: real_goal, lambda: real_sub_goal)
def _define_step(self, done, score, summary): """Combine operations of a phase. Keeps track of the mean score and when to report it. Args: done: Tensor indicating whether current score can be used. score: Tensor holding the current, possibly intermediate, score. summary: Tensor holding summary string to write if not an empty string. Returns: Tuple of summary tensor, mean score, and new global step. The mean score is zero for non reporting steps. """ if done.shape.ndims == 0: done = done[None] if score.shape.ndims == 0: score = score[None] score_mean = streaming_mean.StreamingMean((), tf.float32) with tf.control_dependencies([done, score, summary]): done_score = tf.gather(score, tf.where(done)[:, 0]) submit_score = tf.cond(tf.reduce_any(done), lambda: score_mean.submit(done_score), tf.no_op) with tf.control_dependencies([submit_score]): mean_score = tf.cond(self._report, score_mean.clear, float) steps_made = tf.shape(score)[0] next_step = self._step.assign_add(steps_made) with tf.control_dependencies([mean_score, next_step]): return tf.identity(summary), mean_score, next_step, steps_made
def _create_learning_rate(hyperparams, step_var): """Creates learning rate var, with decay and switching for CompositeOptimizer. Args: hyperparams: a GridPoint proto containing optimizer spec, particularly learning_method to determine optimizer class to use. step_var: tf.Variable, global training step. Raises: ValueError: If the composite optimizer is set, but not correctly configured. Returns: a scalar `Tensor`, the learning rate based on current step and hyperparams. """ if hyperparams.learning_method != 'composite': base_rate = hyperparams.learning_rate adjusted_steps = step_var else: spec = hyperparams.composite_optimizer_spec switch = tf.less(step_var, spec.switch_after_steps) base_rate = tf.cond(switch, lambda: tf.constant(spec.method1.learning_rate), lambda: tf.constant(spec.method2.learning_rate)) if spec.reset_learning_rate: adjusted_steps = tf.cond(switch, lambda: step_var, lambda: step_var - spec.switch_after_steps) else: adjusted_steps = step_var return tf.train.exponential_decay( learning_rate=base_rate, global_step=adjusted_steps, decay_steps=hyperparams.decay_steps, decay_rate=hyperparams.decay_base, staircase=hyperparams.decay_staircase)
def while_exit_cond(result, logits, loss): # pylint: disable=unused-argument """Exit the loop either if reach decode_length or EOS.""" length = common_layers.shape_list(result)[1] not_overflow = length < decode_length if self._problem_hparams.stop_at_eos: def fn_not_eos(): return tf.not_equal( # Check if the last predicted element is a EOS tf.squeeze(result[:, -1, :, :]), text_encoder.EOS_ID) not_eos = tf.cond( # We only check for early stoping if there is at least 1 element ( # otherwise not_eos will crash) tf.not_equal(length, 0), fn_not_eos, lambda: True, ) return tf.cond( tf.equal(batch_size, 1), # If batch_size == 1, we check EOS for early stoping lambda: tf.logical_and(not_overflow, not_eos), # Else, just wait for max length lambda: not_overflow) return not_overflow
def __imagenet_data_process_function(self, x, y): with tf.name_scope("imagenet_data_aug") as scope: #random scale #apparently, this works better than what we have: #https://github.com/facebook/fb.resnet.torch #but let's use the 'original' formulation for now #randomly sample a size in specified range random_size = tf.squeeze(tf.random_uniform((1, 1), 256, 480, dtype=tf.int32, name="random_scale_size")) #rescale smaller size with this factor tf.cond(tf.greater(tf.shape(x)[0], tf.shape(x)[1]), lambda: tf.image.resize_images(x, [tf.shape(x)[0] * (tf.shape(x)[1] / random_size), random_size]), lambda: tf.image.resize_images(x, [random_size, tf.shape(x)[1] * (tf.shape(x)[0] / random_size)])) x = tf.image.resize_images(x, [224, 224]) #random flip x = tf.image.flip_left_right(x) #random crop x = tf.random_crop(x, [224, 224, 3]) #colour augmentation #this is a little more involved than I first thought #lets pick the inception colour distortion #https://github.com/tensorflow/models/blob/master/inception/inception/image_processing.py x = tf.image.random_brightness(x, max_delta=32. / 255.) x = tf.image.random_saturation(x, lower=0.5, upper=1.5) x = tf.image.random_hue(x, max_delta=0.2) x = tf.image.random_contrast(x, lower=0.5, upper=1.5) x = tf.clip_by_value(x, 0.0, 1.0) #normalisation x = tf.image.per_image_standardization(x) return [x, y]
def loop_fn(time, cell_output, cell_state, loop_state): emit_output = cell_output # == None for time == 0 if cell_output is None: # time == 0 next_cell_state = initial_state else: next_cell_state = cell_state elements_finished = (time >= sequence_length) finished = tf.reduce_all(elements_finished) next_input = tf.cond( finished, lambda: tf.zeros([batch_size, input_size], dtype=tf.float32), lambda: inputs_ta.read(time)) next_target = tf.cond( finished, lambda: tf.zeros([batch_size, target_size], dtype=tf.float32), lambda: targets_ta.read(time)) if loop_state is None: next_input = tf.expand_dims(next_input, 1) next_input = tf.pad(next_input, [[0,0], [window-1, 0], [0,0]]) else: next_input = cat_hist(loop_state, next_input, 1) next_loop_state = next_input return (elements_finished, RnnHistInputTuple(next_input, next_target), next_cell_state, emit_output, next_loop_state)
def encoder_body(time, old_state, output_ta_t): x_t = input_ta.read(time) con = tf.concat(1, [x_t, old_state]) z = tf.sigmoid(tf.matmul(con, W_z) + b_z) r = tf.sigmoid(tf.matmul(con, W_r) + b_r) con = tf.concat(1, [x_t, r*old_state]) h = tf.tanh(tf.matmul(con, W_h) + b_h) new_state = (1-z)*h + z*old_state output_ta_t = output_ta_t.write(time, new_state) def updateall(): return new_state def updatesome(): if reverse: return tf.select( tf.greater_equal(time, max_sequence_length-lengths), new_state, old_state) else: return tf.select(tf.less(time, lengths), new_state, old_state) if reverse: state = tf.cond( tf.greater_equal(time, max_sequence_length-min_sequence_length), updateall, updatesome) else: state = tf.cond(tf.less(time, min_sequence_length), updateall, updatesome) return (time + 1, state, output_ta_t)
def _augment(self, image, bboxes=None, default_prob=0.5): """Applies different data augmentation techniques. Uses the list of data augmentation configurations, each data augmentation technique has a probability assigned to it (or just uses the default value for the dataset). Procedures are applied sequentially on top of each other according to the order defined in the config. TODO: We need a better way to ensure order using YAML config without ending up with a list of single-key dictionaries. Args: image: A Tensor of shape (height, width, 3). bboxes: A Tensor of shape (total_bboxes, 5). Returns: image: A Tensor of shape (height, width, 3). bboxes: A Tensor of shape (total_bboxes, 5) of type tf.int32. """ applied_data_augmentation = [] for aug_config in self._data_augmentation: if len(aug_config.keys()) != 1: raise ValueError( 'Invalid data_augmentation definition: "{}"'.format( aug_config)) aug_type = list(aug_config.keys())[0] if aug_type not in DATA_AUGMENTATION_STRATEGIES: tf.logging.warning( 'Invalid data augmentation strategy "{}". Ignoring'.format( aug_type)) continue aug_config = aug_config[aug_type] aug_fn = DATA_AUGMENTATION_STRATEGIES[aug_type] random_number = tf.random_uniform([], seed=self._seed) prob = tf.to_float(aug_config.pop('prob', default_prob)) apply_aug_strategy = tf.less(random_number, prob) augmented = aug_fn(image, bboxes, **aug_config) image = tf.cond( apply_aug_strategy, lambda: augmented['image'], lambda: image ) if bboxes is not None: bboxes = tf.cond( apply_aug_strategy, lambda: augmented.get('bboxes'), lambda: bboxes ) applied_data_augmentation.append({aug_type: apply_aug_strategy}) return image, bboxes, applied_data_augmentation
def apply_stats(self, statsUpdates): """ compute stats and update/apply the new stats to the running average """ def updateAccumStats(): if self._full_stats_init: return tf.cond(tf.greater(self.sgd_step, self._cold_iter), lambda: tf.group(*self._apply_stats(statsUpdates, accumulate=True, accumulateCoeff=1. / self._stats_accum_iter)), tf.no_op) else: return tf.group(*self._apply_stats(statsUpdates, accumulate=True, accumulateCoeff=1. / self._stats_accum_iter)) def updateRunningAvgStats(statsUpdates, fac_iter=1): # return tf.cond(tf.greater_equal(self.factor_step, # tf.convert_to_tensor(fac_iter)), lambda: # tf.group(*self._apply_stats(stats_list, varlist)), tf.no_op) return tf.group(*self._apply_stats(statsUpdates)) if self._async_stats: # asynchronous stats update update_stats = self._apply_stats(statsUpdates) queue = tf.FIFOQueue(1, [item.dtype for item in update_stats], shapes=[ item.get_shape() for item in update_stats]) enqueue_op = queue.enqueue(update_stats) def dequeue_stats_op(): return queue.dequeue() self.qr_stats = tf.train.QueueRunner(queue, [enqueue_op]) update_stats_op = tf.cond(tf.equal(queue.size(), tf.convert_to_tensor( 0)), tf.no_op, lambda: tf.group(*[dequeue_stats_op(), ])) else: # synchronous stats update update_stats_op = tf.cond(tf.greater_equal( self.stats_step, self._stats_accum_iter), lambda: updateRunningAvgStats(statsUpdates), updateAccumStats) self._update_stats_op = update_stats_op return update_stats_op
def enc_step(step): """Encoder step.""" if autoenc_decay < 1.0: quant_step = autoenc_quantize(step, 16, nmaps, self.do_training) if backward: exp_glob = tf.train.exponential_decay(1.0, self.global_step - 10000, 1000, autoenc_decay) dec_factor = 1.0 - exp_glob # * self.do_training dec_factor = tf.cond(tf.less(self.global_step, 10500), lambda: tf.constant(0.05), lambda: dec_factor) else: dec_factor = 1.0 cur = tf.cond(tf.less(tf.random_uniform([]), dec_factor), lambda: quant_step, lambda: step) else: cur = step if dropout > 0.0001: cur = tf.nn.dropout(cur, keep_prob) if act_noise > 0.00001: cur += tf.truncated_normal(tf.shape(cur)) * act_noise_scale # Do nconvs-many CGRU steps. if do_jit and tf.get_variable_scope().reuse: with jit_scope(): for layer in xrange(nconvs): cur = conv_gru([], cur, kw, kh, nmaps, conv_rate(layer), cutoff, "ecgru_%d" % layer, do_layer_norm) else: for layer in xrange(nconvs): cur = conv_gru([], cur, kw, kh, nmaps, conv_rate(layer), cutoff, "ecgru_%d" % layer, do_layer_norm) return cur
def batchnorm(self, Ylogits, offset, convolutional=False): """batchnormalization. Args: Ylogits: 1D向量或者是3D的卷积结果。 num_updates: 迭代的global_step offset:表示beta,全局均值;在 RELU 激活中一般初始化为 0.1。 scale:表示lambda,全局方差;在 sigmoid 激活中需要,这 RELU 激活中作用不大。 m: 表示batch均值;v:表示batch方差。 bnepsilon:一个很小的浮点数,防止除以 0. Returns: Ybn: 和 Ylogits 的维度一样,就是经过 Batch Normalization 处理的结果。 update_moving_everages:更新mean和variance,主要是给最后的 test 使用。 """ exp_moving_avg = tf.train.ExponentialMovingAverage(0.999, self._global_step) # adding the iteration prevents from averaging across non-existing iterations bnepsilon = 1e-5 if convolutional: mean, variance = tf.nn.moments(Ylogits, [0, 1, 2]) else: mean, variance = tf.nn.moments(Ylogits, [0]) update_moving_everages = exp_moving_avg.apply([mean, variance]) m = tf.cond(self.tst, lambda: exp_moving_avg.average(mean), lambda: mean) v = tf.cond(self.tst, lambda: exp_moving_avg.average(variance), lambda: variance) Ybn = tf.nn.batch_normalization(Ylogits, m, v, offset, None, bnepsilon) return Ybn, update_moving_everages
def perform(self, observ): """Compute batch of actions and a summary for a batch of observation. Args: observ: Tensor of a batch of observations for all agents. Returns: Tuple of action batch tensor and summary tensor. """ with tf.name_scope('perform/'): observ = self._observ_filter.transform(observ) network = self._network(observ[:, None], tf.ones(observ.shape[0]), self._last_state) action = tf.cond(self._is_training, network.policy.sample, lambda: network.mean) logprob = network.policy.log_prob(action)[:, 0] # pylint: disable=g-long-lambda summary = tf.cond( self._should_log, lambda: tf.summary.merge([ tf.summary.histogram('mean', network.mean[:, 0]), tf.summary.histogram('std', tf.exp(network.logstd[:, 0])), tf.summary.histogram('action', action[:, 0]), tf.summary.histogram('logprob', logprob) ]), str) # Remember current policy to append to memory in the experience callback. with tf.control_dependencies([ utility.assign_nested_vars(self._last_state, network.state), self._last_action.assign(action[:, 0]), self._last_mean.assign(network.mean[:, 0]), self._last_logstd.assign(network.logstd[:, 0]) ]): return tf.check_numerics(action[:, 0], 'action'), tf.identity(summary)
def gibbs_step(count, k, xk, raw_xk, W=W): hk = gibbs_forward(xk, W, bh, binary) C1 = tf.cond(count + 1 >= k, lambda: tf.constant(c_1), lambda: tf.constant(0.5)) C2 = tf.cond(count + 1 >= k, lambda: tf.constant(c_2), lambda: tf.constant(0.5)) raw_xk = tf.sigmoid(tf.matmul(hk, tf.transpose(W)) + bv) xk = sample(raw_xk, binary=binary, c_1=c_1, c_2=c_2) return count+1, k, xk, raw_xk
def _get_valid_sample_fraction(labels, p=0): """return fraction of non-negative examples, the ignored examples have been marked as negative""" num_valid = tf.reduce_sum(tf.cast(tf.greater_equal(labels, p), tf.float32)) num_example = tf.cast(tf.size(labels), tf.float32) frac = tf.cond(tf.greater(num_example, 0), lambda:num_valid / num_example, lambda: tf.cast(0, tf.float32)) frac_ = tf.cond(tf.greater(num_valid, 0), lambda:num_example / num_valid, lambda: tf.cast(0, tf.float32)) return frac, frac_
def __init__(self, input_size, output_size): self.graph = tf.Graph() self.hyper_cnt = input_size self.save_path = "fit_trend.ckpt" self.collect_counter = 0 self.fit_loss_collect = list() self.stable_loss_predict_collect = list() self.hp_collect = [list() for _ in range(self.hyper_cnt)] self.gradient_collect = [list() for _ in range(self.hyper_cnt)] self.stable_loss_label_collect = list() self.hp_norms = list() self.has_init = False with self.graph.as_default(): # 接收输入 self.ph_hypers = tf.placeholder(tf.float32, shape=[self.hyper_cnt], name='ph_hypers') self.tf_hypers, self.reset_vars = assign_diffable_vars2tensor(self.ph_hypers, self.hyper_cnt) rnn_step = 5 trend_input = tf.concat(0, [self.tf_hypers for _ in range(rnn_step)]) # 通过一个RNN trend_outputs = rnn(trend_input, n_hidden=128) print('rnn output') print(tf.concat(0, trend_outputs)) # RNN接一个DNN trend_output = dnn(tf.concat(0, trend_outputs), [1, output_size]) print('dnn output') print(trend_output) self.predict = trend_output # 实际的trend self.train_label = tf.placeholder(tf.float32, shape=[output_size], name='train_label') # 预测准确率,predict和trend的几何距离 predict_accuracy = tf.sqrt(tf.reduce_sum(tf.square(tf.sub(trend_output, self.train_label)))) / output_size # predict_accuracy /= tf.reduce_mean(tf.concat(0, self.train_label)) # 稳定时损失,最后一个损失 stable_loss = tf.unpack(tf.unpack(trend_output)[0])[-1] print(stable_loss) self.is_fit = tf.placeholder(tf.bool, name='is_fit') self.loss = tf.cond(self.is_fit, lambda: predict_accuracy, lambda: stable_loss) # 优化器 self.var_s = tf.trainable_variables() self.v_hp_s = self.var_s[0: self.hyper_cnt] self.v_fit_s = [v for v in self.var_s if v not in self.v_hp_s] self.grads = var_gradient(self.v_hp_s, self.loss, start_rate=0.1, lrd=False) def optimize_fit(): optimizer_fit = var_optimizer(self.v_fit_s, self.loss) return optimizer_fit def optimize_hp(): optimizer_hp = var_optimizer(self.v_hp_s, self.loss, start_rate=0.1, lrd=False) return optimizer_hp self.optimizer = tf.cond(self.is_fit, optimize_fit, optimize_hp) self.saver = tf.train.Saver()
def batchnorm(Ylogits, Offset, Scale, is_test, iteration): exp_moving_avg = tf.train.ExponentialMovingAverage(0.998, iteration) # adding the iteration prevents from averaging across non-existing iterations bnepsilon = 1e-5 mean, variance = tf.nn.moments(Ylogits, [0]) update_moving_averages = exp_moving_avg.apply([mean, variance]) m = tf.cond(is_test, lambda: exp_moving_avg.average(mean), lambda: mean) v = tf.cond(is_test, lambda: exp_moving_avg.average(variance), lambda: variance) Ybn = tf.nn.batch_normalization(Ylogits, m, v, Offset, Scale, bnepsilon) return Ybn, update_moving_averages
def style_loss(CNN_structure, const_layers, var_layers, content_segs, style_segs, weight): loss_styles = [] layer_count = float(len(const_layers)) layer_index = 0 _, content_seg_height, content_seg_width, _ = content_segs[0].get_shape().as_list() _, style_seg_height, style_seg_width, _ = style_segs[0].get_shape().as_list() for layer_name in CNN_structure: layer_name = layer_name[layer_name.find("/") + 1:] # downsampling segmentation if "pool" in layer_name: content_seg_width, content_seg_height = int(math.ceil(content_seg_width / 2)), int(math.ceil(content_seg_height / 2)) style_seg_width, style_seg_height = int(math.ceil(style_seg_width / 2)), int(math.ceil(style_seg_height / 2)) for i in xrange(len(content_segs)): content_segs[i] = tf.image.resize_bilinear(content_segs[i], tf.constant((content_seg_height, content_seg_width))) style_segs[i] = tf.image.resize_bilinear(style_segs[i], tf.constant((style_seg_height, style_seg_width))) elif "conv" in layer_name: for i in xrange(len(content_segs)): # have some differences on border with torch content_segs[i] = tf.nn.avg_pool(tf.pad(content_segs[i], [[0, 0], [1, 1], [1, 1], [0, 0]], "CONSTANT"), \ ksize=[1, 3, 3, 1], strides=[1, 1, 1, 1], padding='VALID') style_segs[i] = tf.nn.avg_pool(tf.pad(style_segs[i], [[0, 0], [1, 1], [1, 1], [0, 0]], "CONSTANT"), \ ksize=[1, 3, 3, 1], strides=[1, 1, 1, 1], padding='VALID') if layer_name == var_layers[layer_index].name[var_layers[layer_index].name.find("/") + 1:]: print("Setting up style layer: <{}>".format(layer_name)) const_layer = const_layers[layer_index] var_layer = var_layers[layer_index] layer_index = layer_index + 1 layer_style_loss = 0.0 for content_seg, style_seg in zip(content_segs, style_segs): gram_matrix_const = gram_matrix(tf.multiply(const_layer, style_seg)) style_mask_mean = tf.reduce_mean(style_seg) gram_matrix_const = tf.cond(tf.greater(style_mask_mean, 0.), lambda: gram_matrix_const / (tf.to_float(tf.size(const_layer)) * style_mask_mean), lambda: gram_matrix_const ) gram_matrix_var = gram_matrix(tf.multiply(var_layer, content_seg)) content_mask_mean = tf.reduce_mean(content_seg) gram_matrix_var = tf.cond(tf.greater(content_mask_mean, 0.), lambda: gram_matrix_var / (tf.to_float(tf.size(var_layer)) * content_mask_mean), lambda: gram_matrix_var ) diff_style_sum = tf.reduce_mean(tf.squared_difference(gram_matrix_const, gram_matrix_var)) * content_mask_mean layer_style_loss += diff_style_sum loss_styles.append(layer_style_loss * weight) return loss_styles
def distort_image(image, input_width, input_height, output_side): """Applies random distortion to the image. The output image is output_side x output_side x 3 """ def random_crop_it(): """Random crops image, after resizing it to output_side +10 x output_side+10""" resized_img = resize_bl(image, output_side + 10) return tf.random_crop(resized_img, [output_side, output_side, 3]) def resize_it(): """Resize the image using resize_bl""" return resize_bl(image, output_side) # if input.width >= output.side + 10 and input.heigth >= output.side + 10 # resize it to output.side + 10 x output.size + 10 and random crop it # else resize it increased_output_side = tf.constant(output_side + 10, dtype=tf.int64) image = tf.cond( tf.logical_and( tf.greater_equal(input_width, increased_output_side), tf.greater_equal(input_height, increased_output_side)), random_crop_it, resize_it) # Apply random distortions to the image flipped_image = tf.image.random_flip_left_right(image) # randomize the order of the random distortions def fn1(): """Applies random brightness, saturation, hue, contrast""" distorted_image = tf.image.random_brightness( flipped_image, max_delta=32. / 255.) distorted_image = tf.image.random_saturation( distorted_image, lower=0.5, upper=1.5) distorted_image = tf.image.random_hue(distorted_image, max_delta=0.2) distorted_image = tf.image.random_contrast( distorted_image, lower=0.5, upper=1.5) return distorted_image def fn2(): """Applies random brightness, contrast, saturation, hue""" distorted_image = tf.image.random_brightness( flipped_image, max_delta=32. / 255.) distorted_image = tf.image.random_contrast( distorted_image, lower=0.5, upper=1.5) distorted_image = tf.image.random_saturation( distorted_image, lower=0.5, upper=1.5) distorted_image = tf.image.random_hue(distorted_image, max_delta=0.2) return distorted_image p_order = tf.random_uniform( shape=[], minval=0.0, maxval=1.0, dtype=tf.float32) distorted_image = tf.cond(tf.less(p_order, 0.5), fn1, fn2) distorted_image = tf.clip_by_value(distorted_image, 0.0, 1.0) return distorted_image
def optimOp(): def updateOptimOp(): if self._full_stats_init: return tf.cond(tf.greater(self.factor_step, tf.convert_to_tensor(0)), lambda: optim.apply_gradients(list(zip(u, varlist))), tf.no_op) else: return optim.apply_gradients(list(zip(u, varlist))) if self._full_stats_init: return tf.cond(tf.greater_equal(self.stats_step, self._stats_accum_iter), updateOptimOp, tf.no_op) else: return tf.cond(tf.greater_equal(self.sgd_step, self._cold_iter), updateOptimOp, tf.no_op)
def summary(self): """Summary string of mean and standard deviation. Returns: Summary tensor. """ with tf.name_scope(self._name + '/summary'): mean_summary = tf.cond(self._count > 0, lambda: self._summary('mean', self._mean), str) std_summary = tf.cond(self._count > 1, lambda: self._summary('stddev', self._std()), str) return tf.summary.merge([mean_summary, std_summary])
def _build_model(self): self.X = tf.placeholder(tf.float32, [None, 440]) self.Y = tf.placeholder(tf.float32, [None, 1940]) self.domain = tf.placeholder(tf.float32, [None, 2]) self.l = tf.placeholder(tf.float32, []) self.train = tf.placeholder(tf.bool, []) self.W1 = tf.placeholder(tf.float32, [440, 1024]) self.b1 = tf.placeholder(tf.float32, [1024]) self.W2 = tf.placeholder(tf.float32, [1024, 1024]) self.b2 = tf.placeholder(tf.float32, [1024]) self.W3 = tf.placeholder(tf.float32, [1024, 1024]) self.b3 = tf.placeholder(tf.float32, [1024]) self.W4 = tf.placeholder(tf.float32, [1024, 1024]) self.b4 = tf.placeholder(tf.float32, [1024]) self.W_out = tf.placeholder(tf.float32, [1024, 1940]) self.b_out = tf.placeholder(tf.float32, [1940]) with tf.variable_scope('feature_extractor'): # l_1 = tf.get_variable('l1', shape=[1024], initializer=tf.contrib.layers.xavier_initializer()) # l_2 = tf.get_variable('l2', shape=[1024], initializer=tf.contrib.layers.xavier_initializer()) # l_3 = tf.get_variable('l3', shape=[1024], initializer=tf.contrib.layers.xavier_initializer()) l_1 = bias_variable([1024]) l_2 = bias_variable([1024]) l_3 = bias_variable([1024]) hidden_1 = tf.mul(tf.nn.sigmoid(tf.matmul(self.X, self.W1) + self.b1), 2*tf.nn.sigmoid(l_1)) hidden_2 = tf.mul(tf.nn.sigmoid(tf.matmul(hidden_1, self.W2) + self.b1), 2*tf.nn.sigmoid(l_2)) hidden_3 = tf.mul(tf.nn.sigmoid(tf.matmul(hidden_2, self.W3) + self.b3), 2*tf.nn.sigmoid(l_3)) self.features = hidden_3 with tf.variable_scope('label_predictor'): all_features = lambda : self.features source_features = lambda : tf.slice(self.features, [0,0], [batch_size/2, -1]) classify_feats = tf.cond(self.train, source_features, all_features) all_labels = lambda : self.Y source_labels = lambda : tf.slice(self.Y, [0,0], [batch_size/2, -1]) self.classify_labels = tf.cond(self.train, source_labels, all_labels) hidden_4 = tf.nn.sigmoid(tf.matmul(classify_feats, self.W4) + self.b4) logits = tf.matmul(hidden_4, self.W_out) + self.b_out self.pred = tf.nn.softmax(logits) self.pred_loss = tf.nn.softmax_cross_entropy_with_logits(logits, self.classify_labels) with tf.variable_scope('domain_predictord'): feat = flip_gradient(self.features, self.l) W_4d = weight_variable([1024, 1024]) b_4d = bias_variable([1024]) W_outd = weight_variable([1024, 2]) b_outd = bias_variable([2]) # W_4d = tf.get_variable('W_4d', shape=[1024,1024], initializer=tf.contrib.layers.xavier_initializer()) # b_4d = tf.get_variable('b_4d', shape=[1024], initializer=tf.contrib.layers.xavier_initializer()) # W_outd = tf.get_variable('W_outd', shape=[1024, 1940], initializer=tf.contrib.layers.xavier_initializer()) # b_outd = tf.get_variable('b_outd', shape=[1940], initializer=tf.contrib.layers.xavier_initializer()) hidden_4d = tf.nn.sigmoid(tf.matmul(feat, W_4d) + b_4d) d_logits = tf.matmul(hidden_4d, W_outd) + b_outd self.domain_pred = tf.nn.softmax(d_logits) self.domain_loss = tf.nn.softmax_cross_entropy_with_logits(d_logits, self.domain)
def __init__(self, args, txt_file, num_classes, mode, batch_size, num_preprocess_threads=1, shuffle=True, min_queue_examples=1): self.args = args self.txt_file = txt_file self.num_preprocess_threads = num_preprocess_threads self.min_queue_examples = min_queue_examples self.batch_size = batch_size self.mode = mode self.imgShape = [self.args.imageHeight, self.args.imageWidth, self.args.imageChannels] self.maskShape = tf.stack([self.args.imageHeight, self.args.imageWidth]) self.num_classes = int(num_classes) input_queue = tf.train.string_input_producer([txt_file], shuffle=False) line_reader = tf.TextLineReader() _, line = line_reader.read(input_queue) split_line = tf.string_split([line]).values if (mode == 'training' or mode == 'validation'): split_line = tf.string_split([line]).values rgb_image_path = split_line[0] label_image_path = split_line[1] self.image_o = self.read_image(rgb_image_path, 0) self.label_image_o = self.read_image(label_image_path, 1) do_flip = tf.random_uniform([], 0, 1) self.image = tf.cond(do_flip > 0.5, lambda: tf.image.flip_left_right(self.image_o), lambda: self.image_o) self.label_image = tf.cond(do_flip > 0.5, lambda: tf.image.flip_left_right(self.label_image_o), lambda: self.label_image_o) self.image.set_shape((self.args.imageHeight, self.args.imageWidth, 3)) self.label_image.set_shape((self.args.imageHeight, self.args.imageWidth, 1)) self.img_batch, self.label_batch = tf.train.shuffle_batch([self.image, self.label_image], batch_size=batch_size, num_threads=num_preprocess_threads, capacity=min_queue_examples + 3 * batch_size, min_after_dequeue=min_queue_examples) elif (mode == 'test'): print('Generating test Image Batch') split_line = tf.string_split([line]).values rgb_image_path = split_line[0] self.image = self.read_image(rgb_image_path, 0) self.image.set_shape((self.args.imageHeight, self.args.imageWidth, 3)) self.img_batch = tf.train.batch([self.image], batch_size=batch_size, num_threads=num_preprocess_threads, capacity=min_queue_examples + 1 * batch_size, )
def build(self, rgb, train_mode=None): """ load variable from npy to build the VGG :param rgb: rgb image [batch, height, width, 3] values scaled [0, 1] :param train_mode: a bool tensor, usually a placeholder: if True, dropout will be turned on """ #这里边输入的RGB图像是已经是经过缩放的了 rgb_scaled = rgb * 1.0 # Convert RGB to BGR red, green, blue = tf.split(axis=3, num_or_size_splits=3, value=rgb_scaled) assert red.get_shape().as_list()[1:] == [224, 224, 1] assert green.get_shape().as_list()[1:] == [224, 224, 1] assert blue.get_shape().as_list()[1:] == [224, 224, 1] bgr = tf.concat(axis=3, values=[blue - VGG_MEAN[0],green - VGG_MEAN[1],red - VGG_MEAN[2]]) assert bgr.get_shape().as_list()[1:] == [224, 224, 3] self.conv1_1 = self.conv_layer(bgr, 3,64,"conv1_1") self.conv1_2 = self.conv_layer(self.conv1_1, 64,64,"conv1_2") self.pool1 = self.max_pool(self.conv1_2, 'pool1') self.conv2_1 = self.conv_layer(self.pool1, 64,128,"conv2_1") self.conv2_2 = self.conv_layer(self.conv2_1,128 ,128,"conv2_2") self.pool2 = self.max_pool(self.conv2_2, 'pool2') self.conv3_1 = self.conv_layer(self.pool2, 128,256,"conv3_1") self.conv3_2 = self.conv_layer(self.conv3_1,256,256,"conv3_2") self.conv3_3 = self.conv_layer(self.conv3_2, 256,256,"conv3_3") self.pool3 = self.max_pool(self.conv3_3, 'pool3') self.conv4_1 = self.conv_layer(self.pool3, 256,512, "conv4_1") self.conv4_2 = self.conv_layer(self.conv4_1, 512,512,"conv4_2") self.conv4_3 = self.conv_layer(self.conv4_2,512,512, "conv4_3") self.pool4 = self.max_pool(self.conv4_3, 'pool4') self.conv5_1 = self.conv_layer(self.pool4,512,512, "conv5_1") self.conv5_2 = self.conv_layer(self.conv5_1,512,512, "conv5_2") self.conv5_3 = self.conv_layer(self.conv5_2,512,512, "conv5_3") self.pool5 = self.max_pool(self.conv5_3, 'pool5') self.fc6 = self.fc_layer(self.pool5, 25088, 4096, "fc6") # 25088 = ((224 // (2 ** 5)) ** 2) * 512 这个计算步骤也懂了 self.relu6 = tf.nn.relu(self.fc6) if train_mode is not None: self.relu6 = tf.cond(train_mode, lambda: tf.nn.dropout(self.relu6, self.dropout), lambda: self.relu6) elif self.trainable: self.relu6 = tf.nn.dropout(self.relu6, self.dropout) self.fc7 = self.fc_layer(self.relu6, 4096, 4096, "fc7") self.relu7 = tf.nn.relu(self.fc7) if train_mode is not None: self.relu7 = tf.cond(train_mode, lambda: tf.nn.dropout(self.relu7, self.dropout), lambda: self.relu7) elif self.trainable: self.relu7 = tf.nn.dropout(self.relu7, self.dropout)
def __call__(self, inputs, state, scope=None): outputs_do, new_state_do = super(SwitchableDropoutWrapper, self).__call__(inputs, state, scope=scope) tf.get_variable_scope().reuse_variables() outputs, new_state = self._cell(inputs, state, scope) outputs = tf.cond(self.is_train, lambda: outputs_do, lambda: outputs) if isinstance(state, tuple): new_state = state.__class__(*[tf.cond(self.is_train, lambda: new_state_do_i, lambda: new_state_i) for new_state_do_i, new_state_i in zip(new_state_do, new_state)]) else: new_state = tf.cond(self.is_train, lambda: new_state_do, lambda: new_state) return outputs, new_state
def build_act(make_obs_ph, q_func, num_actions, scope="deepq", reuse=None): """Creates the act function: Parameters ---------- make_obs_ph: str -> tf.placeholder or TfInput a function that take a name and creates a placeholder of input with that name q_func: (tf.Variable, int, str, bool) -> tf.Variable the model that takes the following inputs: observation_in: object the output of observation placeholder num_actions: int number of actions scope: str reuse: bool should be passed to outer variable scope and returns a tensor of shape (batch_size, num_actions) with values of every action. num_actions: int number of actions. scope: str or VariableScope optional scope for variable_scope. reuse: bool or None whether or not the variables should be reused. To be able to reuse the scope must be given. Returns ------- act: (tf.Variable, bool, float) -> tf.Variable function to select and action given observation. ` See the top of the file for details. """ with tf.variable_scope(scope, reuse=reuse): observations_ph = make_obs_ph("observation") stochastic_ph = tf.placeholder(tf.bool, (), name="stochastic") update_eps_ph = tf.placeholder(tf.float32, (), name="update_eps") eps = tf.get_variable("eps", (), initializer=tf.constant_initializer(0)) q_values = q_func(observations_ph.get(), num_actions, scope="q_func") deterministic_actions = tf.argmax(q_values, axis=1) batch_size = tf.shape(observations_ph.get())[0] random_actions = tf.random_uniform(tf.stack([batch_size]), minval=0, maxval=num_actions, dtype=tf.int64) chose_random = tf.random_uniform(tf.stack([batch_size]), minval=0, maxval=1, dtype=tf.float32) < eps stochastic_actions = tf.where(chose_random, random_actions, deterministic_actions) output_actions = tf.cond(stochastic_ph, lambda: stochastic_actions, lambda: deterministic_actions) update_eps_expr = eps.assign(tf.cond(update_eps_ph >= 0, lambda: update_eps_ph, lambda: eps)) _act = U.function(inputs=[observations_ph, stochastic_ph, update_eps_ph], outputs=output_actions, givens={update_eps_ph: -1.0, stochastic_ph: True}, updates=[update_eps_expr]) def act(ob, stochastic=True, update_eps=-1): return _act(ob, stochastic, update_eps) return act
def testCond_3(self): with self.test_session(): x = tf.constant(10) pred = tf.less(1, 2) fn1 = lambda: tf.add(x, 1) fn2 = lambda: tf.sub(x, 1) fn3 = lambda: tf.add(tf.cond(pred, fn1, fn2), 1) r = tf.cond(pred, fn3, fn2) result = r.eval() self.assertTrue(check_op_order(x.graph)) self.assertAllEqual(12, result)
def pad_pred_label(predictions, labels): num_digit_predictions = tf.shape(predictions)[-1] num_digit_labels = tf.shape(labels)[-1] paddings_mask = tf.constant([[0,0], [0,1]], dtype=labels.dtype) paddings = tf.cast(tf.fill([2,2], tf.abs(num_digit_predictions-num_digit_labels)),labels.dtype) paddings = paddings * paddings_mask # paddings = tf.constant([[0, 0,], [0, tf.abs(num_digit_predictions-num_digit_predictions)]]) predictions = tf.cond(num_digit_predictions< num_digit_labels, lambda: tf.pad(predictions, paddings, constant_values=-1), lambda: tf.identity(predictions)) labels = tf.cond(num_digit_labels< num_digit_predictions, lambda: tf.pad(labels, paddings, constant_values=-1), lambda: tf.identity(labels)) return predictions, labels
def conv_internal(conv_fn, inputs, filters, kernel_size, **kwargs): """Conditional conv_fn making kernel 1d or 2d depending on inputs shape.""" static_shape = inputs.get_shape() if not static_shape or len(static_shape) != 4: raise ValueError("Inputs to conv must have statically known rank 4.") inputs.set_shape([static_shape[0], None, None, static_shape[3]]) # Add support for left padding. if "padding" in kwargs and kwargs["padding"] == "LEFT": dilation_rate = (1, 1) if "dilation_rate" in kwargs: dilation_rate = kwargs["dilation_rate"] assert kernel_size[0] % 2 == 1 and kernel_size[1] % 2 == 1 height_padding = 2 * (kernel_size[0] // 2) * dilation_rate[0] cond_padding = tf.cond( tf.equal(tf.shape(inputs)[2], 1), lambda: tf.constant(0), lambda: tf.constant(2 * (kernel_size[1] // 2) * dilation_rate[1])) width_padding = 0 if static_shape[2] == 1 else cond_padding padding = [[0, 0], [height_padding, 0], [width_padding, 0], [0, 0]] inputs = tf.pad(inputs, padding) kwargs["padding"] = "VALID" force2d = False # Special argument we use to force 2d kernels (see below). if "force2d" in kwargs: force2d = kwargs["force2d"] def conv2d_kernel(kernel_size_arg, name_suffix): """Call conv2d but add suffix to name.""" if "name" in kwargs: original_name = kwargs["name"] name = kwargs.pop("name") + "_" + name_suffix else: original_name = None name = "conv_" + name_suffix original_force2d = None if "force2d" in kwargs: original_force2d = kwargs.pop("force2d") result = conv_fn(inputs, filters, kernel_size_arg, name=name, **kwargs) if original_name is not None: kwargs["name"] = original_name # Restore for other calls. if original_force2d is not None: kwargs["force2d"] = original_force2d return result # Manually setting the shape to be unknown in the middle two dimensions so # that the `tf.cond` below won't throw an error based on the convolution # kernels being too large for the data. inputs._shape = tf.TensorShape([static_shape[0], None, None, static_shape[3]]) # pylint: disable=protected-access if kernel_size[1] == 1 or force2d: # Avoiding the cond below can speed up graph and gradient construction. return conv2d_kernel(kernel_size, "single") return tf.cond( tf.equal(tf.shape(inputs)[2], 1), lambda: conv2d_kernel((kernel_size[0], 1), "small"), lambda: conv2d_kernel(kernel_size, "std"))
def batchnorm(Ylogits, is_test, iteration, convolutional=False): exp_moving_avg = tf.train.ExponentialMovingAverage(0.9999, iteration) # adding the iteration prevents from averaging across non-existing iterations bnepsilon = 1e-5 if convolutional: mean, variance = tf.nn.moments(Ylogits, [0, 1, 2]) else: mean, variance = tf.nn.moments(Ylogits, [0]) update_moving_everages = exp_moving_avg.apply([mean, variance]) m = tf.cond(is_test, lambda: exp_moving_avg.average(mean), lambda: mean) v = tf.cond(is_test, lambda: exp_moving_avg.average(variance), lambda: variance) Ybn = tf.nn.batch_normalization(Ylogits, m, v, 0.0, 1.0, bnepsilon) return Ybn, update_moving_everages
def __init__(self, image): image = tf.cast(image, tf.float32) image = tf.cond( tf.shape(tf.shape(image))[0] > 2, lambda: image[:, :, 0], lambda: image) self.tensor = image < self.threshold self.is_flipped = (tf.reduce_sum( tf.reshape(tf.cast(self.tensor, tf.int64), [-1])) > tf.cast(tf.cast(tf.reduce_prod(tf.shape(self.tensor)), tf.float32) * 0.5, tf.int64)) self.image = tf.cond(self.is_flipped, lambda: 255 - image, lambda: image) self.tensor = tf.cond( self.is_flipped, lambda: ~self.tensor, lambda: self.tensor)
def main(unused_argv=None): tf.logging.set_verbosity(FLAGS.log) if FLAGS.config is None: raise RuntimeError("No config name specified.") config = utils.get_module("wavenet." + FLAGS.config).Config( FLAGS.train_path) logdir = FLAGS.logdir tf.logging.info("Saving to %s" % logdir) with tf.Graph().as_default(): total_batch_size = FLAGS.total_batch_size assert total_batch_size % FLAGS.worker_replicas == 0 worker_batch_size = total_batch_size / FLAGS.worker_replicas # Run the Reader on the CPU cpu_device = "/job:localhost/replica:0/task:0/cpu:0" if FLAGS.ps_tasks: cpu_device = "/job:worker/cpu:0" with tf.device(cpu_device): inputs_dict = config.get_batch(worker_batch_size) with tf.device( tf.train.replica_device_setter(ps_tasks=FLAGS.ps_tasks, merge_devices=True)): global_step = tf.get_variable( "global_step", [], tf.int32, initializer=tf.constant_initializer(0), trainable=False) # pylint: disable=cell-var-from-loop lr = tf.constant(config.learning_rate_schedule[0]) for key, value in config.learning_rate_schedule.iteritems(): lr = tf.cond(tf.less(global_step, key), lambda: lr, lambda: tf.constant(value)) # pylint: enable=cell-var-from-loop tf.summary.scalar("learning_rate", lr) # build the model graph outputs_dict = config.build(inputs_dict, is_training=True) loss = outputs_dict["loss"] tf.summary.scalar("train_loss", loss) worker_replicas = FLAGS.worker_replicas ema = tf.train.ExponentialMovingAverage(decay=0.9999, num_updates=global_step) opt = tf.train.SyncReplicasOptimizer( tf.train.AdamOptimizer(lr, epsilon=1e-8), worker_replicas, total_num_replicas=worker_replicas, variable_averages=ema, variables_to_average=tf.trainable_variables()) train_op = opt.minimize(loss, global_step=global_step, name="train", colocate_gradients_with_ops=True) session_config = tf.ConfigProto(allow_soft_placement=True) is_chief = (FLAGS.task == 0) local_init_op = opt.chief_init_op if is_chief else opt.local_step_init_op slim.learning.train( train_op=train_op, logdir=logdir, is_chief=is_chief, master=FLAGS.master, number_of_steps=config.num_iters, global_step=global_step, log_every_n_steps=250, local_init_op=local_init_op, save_interval_secs=300, sync_optimizer=opt, session_config=session_config, )
def random_crop_patch(image, bboxes, labels, prob=conf.random_crop_patch_prob, seed=None): """Random flip left-right of an image and its bounding boxes.""" def crop_bboxes(image, bboxes, labels, offset_height, offset_width, target_height, target_width): """crop bounding boxes coordinates. Args: bboxes: [xmin, ymin, xmax, ymax] """ #print('>>>>',tf.shape(bboxes)) #print('>>>>',bboxes) bboxes = tf.reshape(bboxes, [-1, 4]) nocrop_boxes = bboxes img_H = tf.cast(tf.shape(image, out_type=tf.int32)[0], tf.float32) img_W = tf.cast(tf.shape(image, out_type=tf.int32)[1], tf.float32) offset_width = tf.cast(offset_width, tf.float32) offset_height = tf.cast(offset_height, tf.float32) target_width = tf.cast(target_width, tf.float32) target_height = tf.cast(target_height, tf.float32) img = [img_W, img_H] offset = [offset_width, offset_height] crop = [target_width - 1., target_height - 1.] box_col = tf.split(axis=1, num_or_size_splits=4, value=bboxes) for i in range(4): box_col[i] = box_col[i] * img[i % 2] - offset[i % 2] box_col[i] = tf.clip_by_value(box_col[i], 0., crop[i % 2]) / crop[i % 2] bboxes = tf.concat(axis=1, values=box_col) keep = tf.where( tf.less_equal(bboxes[:, 0], bboxes[:, 2] - 0.05) & tf.less_equal(bboxes[:, 1], bboxes[:, 3] - 0.05)) crop_cond = tf.greater(tf.shape(keep)[0], 100) bboxes, labels = tf.cond( crop_cond, lambda: (tf.gather_nd(bboxes, keep), tf.gather_nd(labels, keep)), lambda: (nocrop_boxes, labels)) #bboxes = tf.gather_nd(bboxes, keep) #labels = tf.gather_nd(labels, keep) return crop_cond, bboxes, labels def crop_images(image, seed=None): """crop image coordinates. Args: bboxes: [xmin, ymin, xmax, ymax] """ img_H = tf.cast(tf.shape(image, out_type=tf.int32)[0], tf.float32) img_W = tf.cast(tf.shape(image, out_type=tf.int32)[1], tf.float32) uniform_random = tf.random_uniform([], conf.random_crop_image_prob, 1.0, seed=seed) target_height = tf.cast(img_H * uniform_random, dtype=tf.int32) target_width = tf.cast(img_W * uniform_random, dtype=tf.int32) offset_height = tf.random_uniform( [], minval=0, maxval=(tf.cast(img_H, dtype=tf.int32) - target_height), dtype=tf.int32) offset_width = tf.random_uniform( [], minval=0, maxval=(tf.cast(img_W, dtype=tf.int32) - target_width), dtype=tf.int32) return offset_height, offset_width, target_height, target_width def process(image, bboxes, labels): """ process """ offset_height, offset_width, target_height, target_width = crop_images( image) crop_cond, crop_bbox, crop_label = crop_bboxes(image, bboxes, labels, offset_height, offset_width, target_height, target_width) crop_bbox, crop_label = tf.cond(crop_cond, lambda: (crop_bbox, crop_label), lambda: (bboxes, labels)) crop_image = tf.image.crop_to_bounding_box(image, offset_height, offset_width, target_height, target_width) return crop_image, crop_bbox, crop_label def no_process(image, bboxes, labels): """ without process """ return image, bboxes, labels # Random crop. Tensorflow implementation. with tf.name_scope('random_crop_patch'): image = tf.convert_to_tensor(image, name='image') uniform_random = tf.random_uniform([], 0, 1.0, seed=seed) crop_cond = tf.less(uniform_random, prob) # crop image. image, bboxes, labels = tf.cond( crop_cond, #tf.convert_to_tensor(crop_cond), lambda: process(image, bboxes, labels), lambda: no_process(image, bboxes, labels)) return image, bboxes, labels
def peaks(values, minval=None, invalidate_distance=0, name=None): """Labels peaks in values. Args: values: 1D tensor of a numeric type. minval: Minimum value which is considered a peak. invalidate_distance: Invalidates nearby potential peaks. The peaks are searched sequentially by descending value, and from left to right for equal values. Once a peak is found in this order, it invalidates any peaks yet to be seen that are <= invalidate_distance away. A distance of 0 effectively produces no invalidation. name: Optional name for the op. Returns: peak_centers: The (rounded) centers of each peak, which are locations where the value is higher than the value before and after. If there is a run of equal values at the peak, the rounded center of the run is returned. int32 1D tensor. """ with tf.name_scope(name, "peaks", [values]): values = tf.convert_to_tensor(values, name="values") invalidate_distance = tf.convert_to_tensor(invalidate_distance, name="invalidate_distance", dtype=tf.int32) # Segment the values and find local maxima. # Take the center of each run of consecutive equal values. segment_centers, _ = _segments_1d(values, mode=SegmentsMode.CENTERS) segment_values = tf.gather(values, segment_centers) # If we have zero or one segments, there are no peaks. Just use zeros as the # edge values in that case. first_val, second_val, penultimate_val, last_val = tf.cond( tf.greater_equal(tf.shape(segment_values)[0], 2), lambda: tuple(segment_values[i] for i in (0, 1, -2, -1)), lambda: tuple(tf.constant(0, values.dtype) for i in range(4))) # Each segment must be greater than the segment before and after it. segment_is_peak = tf.concat( [[first_val > second_val], tf.greater(segment_values[1:-1], tf.maximum(segment_values[:-2], segment_values[2:])), [last_val > penultimate_val]], axis=0) if minval is not None: # Filter the peaks by minval. segment_is_peak = tf.logical_and( segment_is_peak, tf.greater_equal(segment_values, minval)) # Get the center coordinates of each peak, and sort by descending value. all_peaks = tf.boolean_mask(segment_centers, segment_is_peak) num_peaks = tf.shape(all_peaks)[0] peak_values = tf.boolean_mask(segment_values, segment_is_peak) _, peak_order = tf.nn.top_k(peak_values, k=num_peaks, sorted=True) all_peaks = tf.gather(all_peaks, peak_order) all_peaks.set_shape([None]) # Loop over the peaks, accepting one at a time and possibly invalidating # other ones. def loop_condition(_, current_peaks): return tf.shape(current_peaks)[0] > 0 def loop_body(accepted_peaks, current_peaks): peak = current_peaks[0] remaining_peaks = current_peaks[1:] keep_peaks = tf.greater(tf.abs(remaining_peaks - peak), invalidate_distance) remaining_peaks = tf.boolean_mask(remaining_peaks, keep_peaks) return tf.concat([accepted_peaks, [peak]], axis=0), remaining_peaks accepted_peaks = tf.while_loop( loop_condition, loop_body, [tf.zeros([0], all_peaks.dtype), all_peaks], shape_invariants=[tf.TensorShape([None]), tf.TensorShape([None])])[0] # Sort the peaks by index. # TODO(ringw): Add a tf.sort op that sorts in ascending order. sorted_negative_peaks, _ = tf.nn.top_k(-accepted_peaks, k=tf.shape(accepted_peaks)[0], sorted=True) return -sorted_negative_peaks
def _decode_image(content, channels): return tf.cond(tf.image.is_jpeg(content), lambda: tf.image.decode_jpeg(content, channels), lambda: tf.image.decode_png(content, channels))
def style_loss(CNN_structure, const_layers, var_layers, content_segs, style_segs, weight): loss_styles = [] layer_count = float(len(const_layers)) layer_index = 0 _, content_seg_height, content_seg_width, _ = content_segs[0].get_shape( ).as_list() _, style_seg_height, style_seg_width, _ = style_segs[0].get_shape( ).as_list() for layer_name in CNN_structure: layer_name = layer_name[layer_name.find("/") + 1:] # downsampling segmentation if "pool" in layer_name: content_seg_width, content_seg_height = int( math.ceil(content_seg_width / 2)), int( math.ceil(content_seg_height / 2)) style_seg_width, style_seg_height = int( math.ceil(style_seg_width / 2)), int( math.ceil(style_seg_height / 2)) for i in xrange(len(content_segs)): content_segs[i] = tf.image.resize_bilinear( content_segs[i], tf.constant((content_seg_height, content_seg_width))) style_segs[i] = tf.image.resize_bilinear( style_segs[i], tf.constant((style_seg_height, style_seg_width))) elif "conv" in layer_name: for i in xrange(len(content_segs)): # have some differences on border with torch content_segs[i] = tf.nn.avg_pool(tf.pad(content_segs[i], [[0, 0], [1, 1], [1, 1], [0, 0]], "CONSTANT"), \ ksize=[1, 3, 3, 1], strides=[1, 1, 1, 1], padding='VALID') style_segs[i] = tf.nn.avg_pool(tf.pad(style_segs[i], [[0, 0], [1, 1], [1, 1], [0, 0]], "CONSTANT"), \ ksize=[1, 3, 3, 1], strides=[1, 1, 1, 1], padding='VALID') if layer_name == var_layers[layer_index].name[var_layers[layer_index]. name.find("/") + 1:]: print("Setting up style layer: <{}>".format(layer_name)) const_layer = const_layers[layer_index] var_layer = var_layers[layer_index] layer_index = layer_index + 1 layer_style_loss = 0.0 for content_seg, style_seg in zip(content_segs, style_segs): gram_matrix_const = gram_matrix( tf.multiply(const_layer, style_seg)) style_mask_mean = tf.reduce_mean(style_seg) gram_matrix_const = tf.cond( tf.greater(style_mask_mean, 0.), lambda: gram_matrix_const / (tf.to_float(tf.size(const_layer)) * style_mask_mean), lambda: gram_matrix_const) gram_matrix_var = gram_matrix( tf.multiply(var_layer, content_seg)) content_mask_mean = tf.reduce_mean(content_seg) gram_matrix_var = tf.cond( tf.greater(content_mask_mean, 0.), lambda: gram_matrix_var / (tf.to_float(tf.size(var_layer)) * content_mask_mean), lambda: gram_matrix_var) diff_style_sum = tf.reduce_mean( tf.squared_difference(gram_matrix_const, gram_matrix_var)) * content_mask_mean layer_style_loss += diff_style_sum loss_styles.append(layer_style_loss * weight) return loss_styles
def eval_metrics_host_call_fn(policy_output, value_output, pi_tensor, policy_cost, value_cost, l2_cost, combined_cost, step, est_mode=tf.estimator.ModeKeys.TRAIN): policy_entropy = -tf.reduce_mean( tf.reduce_sum(policy_output * tf.log(policy_output), axis=1)) # pi_tensor is one_hot when generated from sgfs (for supervised learning) # and soft-max when using self-play records. argmax normalizes the two. policy_target_top_1 = tf.argmax(pi_tensor, axis=1) policy_output_in_top1 = tf.to_float( tf.nn.in_top_k(policy_output, policy_target_top_1, k=1)) policy_output_in_top3 = tf.to_float( tf.nn.in_top_k(policy_output, policy_target_top_1, k=3)) policy_top_1_confidence = tf.reduce_max(policy_output, axis=1) policy_target_top_1_confidence = tf.boolean_mask( policy_output, tf.one_hot(policy_target_top_1, tf.shape(policy_output)[1])) with tf.variable_scope("metrics"): metric_ops = { 'policy_cost': tf.metrics.mean(policy_cost), 'value_cost': tf.metrics.mean(value_cost), 'l2_cost': tf.metrics.mean(l2_cost), 'policy_entropy': tf.metrics.mean(policy_entropy), 'combined_cost': tf.metrics.mean(combined_cost), 'policy_accuracy_top_1': tf.metrics.mean(policy_output_in_top1), 'policy_accuracy_top_3': tf.metrics.mean(policy_output_in_top3), 'policy_top_1_confidence': tf.metrics.mean(policy_top_1_confidence), 'policy_target_top_1_confidence': tf.metrics.mean(policy_target_top_1_confidence), 'value_confidence': tf.metrics.mean(tf.abs(value_output)), } if est_mode == tf.estimator.ModeKeys.EVAL: return metric_ops # NOTE: global_step is rounded to a multiple of FLAGS.summary_steps. eval_step = tf.reduce_min(step) # Create summary ops so that they show up in SUMMARIES collection # That way, they get logged automatically during training summary_writer = summary.create_file_writer(FLAGS.work_dir) with summary_writer.as_default(), \ summary.record_summaries_every_n_global_steps( params['summary_steps'], eval_step): for metric_name, metric_op in metric_ops.items(): summary.scalar(metric_name, metric_op[1], step=eval_step) # Reset metrics occasionally so that they are mean of recent batches. reset_op = tf.variables_initializer(tf.local_variables("metrics")) cond_reset_op = tf.cond( tf.equal(eval_step % params['summary_steps'], tf.to_int64(1)), lambda: reset_op, lambda: tf.no_op()) return summary.all_summary_ops() + [cond_reset_op]
def call(self, x): input_image, y_pred, y_true, true_boxes = x # adjust the shape of the y_predict [batch, grid_h, grid_w, 3, 4+1+nb_class] y_pred = tf.reshape( y_pred, tf.concat([tf.shape(y_pred)[:3], tf.constant([3, -1])], axis=0)) # initialize the masks object_mask = tf.expand_dims(y_true[..., 4], 4) # the variable to keep track of number of batches processed batch_seen = tf.Variable(0.) # compute grid factor and net factor grid_h = tf.shape(y_true)[1] grid_w = tf.shape(y_true)[2] grid_factor = tf.reshape(tf.cast([grid_w, grid_h], tf.float32), [1, 1, 1, 1, 2]) net_h = tf.shape(input_image)[1] net_w = tf.shape(input_image)[2] net_factor = tf.reshape(tf.cast([net_w, net_h], tf.float32), [1, 1, 1, 1, 2]) """ Adjust prediction """ pred_box_xy = (self.cell_grid[:, :grid_h, :grid_w, :, :] + tf.sigmoid(y_pred[..., :2])) # sigma(t_xy) + c_xy pred_box_wh = y_pred[..., 2:4] # t_wh pred_box_conf = tf.expand_dims(tf.sigmoid(y_pred[..., 4]), 4) # adjust confidence pred_box_class = y_pred[..., 5:] # adjust class probabilities """ Adjust ground truth """ true_box_xy = y_true[..., 0:2] # (sigma(t_xy) + c_xy) true_box_wh = y_true[..., 2:4] # t_wh true_box_conf = tf.expand_dims(y_true[..., 4], 4) true_box_class = tf.argmax(y_true[..., 5:], -1) """ Compare each predicted box to all true boxes """ # initially, drag all objectness of all boxes to 0 conf_delta = pred_box_conf - 0 # then, ignore the boxes which have good overlap with some true box true_xy = true_boxes[..., 0:2] / grid_factor true_wh = true_boxes[..., 2:4] / net_factor true_wh_half = true_wh / 2. true_mins = true_xy - true_wh_half true_maxes = true_xy + true_wh_half pred_xy = tf.expand_dims(pred_box_xy / grid_factor, 4) pred_wh = tf.expand_dims( tf.exp(pred_box_wh) * self.anchors / net_factor, 4) pred_wh_half = pred_wh / 2. pred_mins = pred_xy - pred_wh_half pred_maxes = pred_xy + pred_wh_half intersect_mins = tf.maximum(pred_mins, true_mins) intersect_maxes = tf.minimum(pred_maxes, true_maxes) intersect_wh = tf.maximum(intersect_maxes - intersect_mins, 0.) intersect_areas = intersect_wh[..., 0] * intersect_wh[..., 1] true_areas = true_wh[..., 0] * true_wh[..., 1] pred_areas = pred_wh[..., 0] * pred_wh[..., 1] union_areas = pred_areas + true_areas - intersect_areas iou_scores = tf.truediv(intersect_areas, union_areas) best_ious = tf.reduce_max(iou_scores, axis=4) conf_delta *= tf.expand_dims( tf.to_float(best_ious < self.ignore_thresh), 4) """ Compute some online statistics """ true_xy = true_box_xy / grid_factor true_wh = tf.exp(true_box_wh) * self.anchors / net_factor true_wh_half = true_wh / 2. true_mins = true_xy - true_wh_half true_maxes = true_xy + true_wh_half pred_xy = pred_box_xy / grid_factor pred_wh = tf.exp(pred_box_wh) * self.anchors / net_factor pred_wh_half = pred_wh / 2. pred_mins = pred_xy - pred_wh_half pred_maxes = pred_xy + pred_wh_half intersect_mins = tf.maximum(pred_mins, true_mins) intersect_maxes = tf.minimum(pred_maxes, true_maxes) intersect_wh = tf.maximum(intersect_maxes - intersect_mins, 0.) intersect_areas = intersect_wh[..., 0] * intersect_wh[..., 1] true_areas = true_wh[..., 0] * true_wh[..., 1] pred_areas = pred_wh[..., 0] * pred_wh[..., 1] union_areas = pred_areas + true_areas - intersect_areas iou_scores = tf.truediv(intersect_areas, union_areas) iou_scores = object_mask * tf.expand_dims(iou_scores, 4) count = tf.reduce_sum(object_mask) count_noobj = tf.reduce_sum(1 - object_mask) detect_mask = tf.to_float((pred_box_conf * object_mask) >= 0.5) class_mask = tf.expand_dims( tf.to_float(tf.equal(tf.argmax(pred_box_class, -1), true_box_class)), 4) recall50 = tf.reduce_sum( tf.to_float(iou_scores >= 0.5) * detect_mask * class_mask) / (count + 1e-3) recall75 = tf.reduce_sum( tf.to_float(iou_scores >= 0.75) * detect_mask * class_mask) / (count + 1e-3) avg_iou = tf.reduce_sum(iou_scores) / (count + 1e-3) avg_obj = tf.reduce_sum(pred_box_conf * object_mask) / (count + 1e-3) avg_noobj = tf.reduce_sum(pred_box_conf * (1 - object_mask)) / (count_noobj + 1e-3) avg_cat = tf.reduce_sum(object_mask * class_mask) / (count + 1e-3) """ Warm-up training """ batch_seen = tf.compat.v1.assign_add(batch_seen, 1.) true_box_xy, true_box_wh, xywh_mask = tf.cond( tf.less(batch_seen, self.warmup_batches + 1), lambda: [ true_box_xy + (0.5 + self.cell_grid[:, :grid_h, :grid_w, :, :]) * (1 - object_mask), true_box_wh + tf.zeros_like(true_box_wh) * (1 - object_mask), tf.ones_like(object_mask) ], lambda: [true_box_xy, true_box_wh, object_mask]) """ Compare each true box to all anchor boxes """ wh_scale = tf.exp(true_box_wh) * self.anchors / net_factor wh_scale = tf.expand_dims( 2 - wh_scale[..., 0] * wh_scale[..., 1], axis=4) # the smaller the box, the bigger the scale xy_delta = xywh_mask * (pred_box_xy - true_box_xy) * wh_scale * self.xywh_scale wh_delta = xywh_mask * (pred_box_wh - true_box_wh) * wh_scale * self.xywh_scale conf_delta = object_mask * ( pred_box_conf - true_box_conf) * self.obj_scale + ( 1 - object_mask) * conf_delta * self.noobj_scale class_delta = object_mask * \ tf.expand_dims(tf.nn.sparse_softmax_cross_entropy_with_logits(labels=true_box_class, logits=pred_box_class), 4) * \ self.class_scale loss_xy = tf.reduce_sum(tf.square(xy_delta), list(range(1, 5))) loss_wh = tf.reduce_sum(tf.square(wh_delta), list(range(1, 5))) loss_conf = tf.reduce_sum(tf.square(conf_delta), list(range(1, 5))) loss_class = tf.reduce_sum(class_delta, list(range(1, 5))) loss = loss_xy + loss_wh + loss_conf + loss_class if debug: loss = tf.Print(loss, [grid_h, avg_obj], message='avg_obj \t\t', summarize=1000) loss = tf.Print(loss, [grid_h, avg_noobj], message='avg_noobj \t\t', summarize=1000) loss = tf.Print(loss, [grid_h, avg_iou], message='avg_iou \t\t', summarize=1000) loss = tf.Print(loss, [grid_h, avg_cat], message='avg_cat \t\t', summarize=1000) loss = tf.Print(loss, [grid_h, recall50], message='recall50 \t', summarize=1000) loss = tf.Print(loss, [grid_h, recall75], message='recall75 \t', summarize=1000) loss = tf.Print(loss, [grid_h, count], message='count \t', summarize=1000) loss = tf.Print(loss, [ grid_h, tf.reduce_sum(loss_xy), tf.reduce_sum(loss_wh), tf.reduce_sum(loss_conf), tf.reduce_sum(loss_class) ], message='loss xy, wh, conf, class: \t', summarize=1000) return loss * self.grid_scale
def build(self, bgr, train_mode=None): """ load variable from npy to build the VGG :param bgr: bgr image [batch, height, width, 3] values scaled [0, 255] :param train_mode: a bool tensor, usually a placeholder: if True, dropout will be turned on """ # subtract mean blue, green, red = tf.split(axis=3, num_or_size_splits=3, value=bgr) assert blue.get_shape().as_list()[1:] == [224, 224, 1] assert green.get_shape().as_list()[1:] == [224, 224, 1] assert red.get_shape().as_list()[1:] == [224, 224, 1] bgr = tf.concat(axis=3, values=[ blue - VGG_MEAN[0], green - VGG_MEAN[1], red - VGG_MEAN[2], ]) assert bgr.get_shape().as_list()[1:] == [224, 224, 3] self.conv1_1 = self.conv_layer(bgr, 3, 64, "conv1_1") self.conv1_2 = self.conv_layer(self.conv1_1, 64, 64, "conv1_2") self.pool1 = self.max_pool(self.conv1_2, 'pool1') self.conv2_1 = self.conv_layer(self.pool1, 64, 128, "conv2_1") self.conv2_2 = self.conv_layer(self.conv2_1, 128, 128, "conv2_2") self.pool2 = self.max_pool(self.conv2_2, 'pool2') self.conv3_1 = self.conv_layer(self.pool2, 128, 256, "conv3_1") self.conv3_2 = self.conv_layer(self.conv3_1, 256, 256, "conv3_2") self.conv3_3 = self.conv_layer(self.conv3_2, 256, 256, "conv3_3") self.conv3_4 = self.conv_layer(self.conv3_3, 256, 256, "conv3_4") self.pool3 = self.max_pool(self.conv3_4, 'pool3') self.conv4_1 = self.conv_layer(self.pool3, 256, 512, "conv4_1") self.conv4_2 = self.conv_layer(self.conv4_1, 512, 512, "conv4_2") self.conv4_3 = self.conv_layer(self.conv4_2, 512, 512, "conv4_3") self.conv4_4 = self.conv_layer(self.conv4_3, 512, 512, "conv4_4") self.pool4 = self.max_pool(self.conv4_4, 'pool4') self.conv5_1 = self.conv_layer(self.pool4, 512, 512, "conv5_1") self.conv5_2 = self.conv_layer(self.conv5_1, 512, 512, "conv5_2") self.conv5_3 = self.conv_layer(self.conv5_2, 512, 512, "conv5_3") self.conv5_4 = self.conv_layer(self.conv5_3, 512, 512, "conv5_4") self.pool5 = self.max_pool(self.conv5_4, 'pool5') self.fc6 = self.fc_layer(self.pool5, 25088, 4096, "fc6") # 25088 = ((224 // (2 ** 5)) ** 2) * 512 self.relu6 = tf.nn.relu(self.fc6) if train_mode is not None: self.relu6 = tf.cond(train_mode, lambda: tf.nn.dropout(self.relu6, self.dropout), lambda: self.relu6) elif self.trainable: self.relu6 = tf.nn.dropout(self.relu6, self.dropout) self.fc7 = self.fc_layer(self.relu6, 4096, 4096, "fc7") self.relu7 = tf.nn.relu(self.fc7) if train_mode is not None: self.relu7 = tf.cond(train_mode, lambda: tf.nn.dropout(self.relu7, self.dropout), lambda: self.relu7) elif self.trainable: self.relu7 = tf.nn.dropout(self.relu7, self.dropout) self.fc8 = self.fc_layer(self.relu7, 4096, 1000, "fc8") self.prob = tf.nn.softmax(self.fc8, name="prob") self.data_dict = None
Simple TensorFlow exercises You should thoroughly test your code """ import tensorflow as tf ############################################################################### # 1a: Create two random 0-d tensors x and y of any distribution. # Create a TensorFlow object that returns x + y if x > y, and x - y otherwise. # Hint: look up tf.cond() # I do the first problem for you ############################################################################### x = tf.random_uniform([]) # Empty array as shape creates a scalar. y = tf.random_uniform([]) out = tf.cond(tf.greater(x, y), lambda: tf.add(x, y), lambda: tf.subtract(x, y)) ############################################################################### # 1b: Create two 0-d tensors x and y randomly selected from the range [-1, 1). # Return x + y if x < y, x - y if x > y, 0 otherwise. # Hint: Look up tf.case(). ############################################################################### # YOUR CODE x = tf.random_uniform([], -1, 1) y = tf.random_uniform([], -1, 1) out = tf.case( { tf.less(x, y): lambda: tf.add(x, y), tf.greater(x, y): lambda: tf.substract(x, y) },
def connect_data_and_network(self, outputs_collector=None, gradients_collector=None): def switch_sampler(for_training): with tf.name_scope('train' if for_training else 'validation'): sampler = self.get_sampler()[0][0 if for_training else -1] return sampler.pop_batch_op() if self.is_training: if self.action_param.validation_every_n > 0: data_dict = tf.cond(tf.logical_not(self.is_validation), lambda: switch_sampler(True), lambda: switch_sampler(False)) else: data_dict = switch_sampler(for_training=True) image = tf.cast(data_dict['image'], tf.float32) net_output = self.net(image, is_training=self.is_training) with tf.name_scope('Optimiser'): optimiser_class = OptimiserFactory.create( name=self.action_param.optimiser) self.optimiser = optimiser_class.get_instance( learning_rate=self.action_param.lr) loss_func = LossFunction(loss_type=self.action_param.loss_type) data_loss = loss_func(net_output) loss = data_loss if self.net_param.decay > 0.0: reg_losses = tf.get_collection( tf.GraphKeys.REGULARIZATION_LOSSES) if reg_losses: reg_loss = tf.reduce_mean( [tf.reduce_mean(reg_loss) for reg_loss in reg_losses]) loss = loss + reg_loss grads = self.optimiser.compute_gradients(loss) # collecting gradients variables gradients_collector.add_to_collection([grads]) outputs_collector.add_to_collection(var=data_loss, name='variational_lower_bound', average_over_devices=True, collection=CONSOLE) outputs_collector.add_to_collection(var=data_loss, name='variational_lower_bound', average_over_devices=True, summary_type='scalar', collection=TF_SUMMARIES) outputs_collector.add_to_collection(var=net_output[4], name='Originals', average_over_devices=False, summary_type='image3_coronal', collection=TF_SUMMARIES) outputs_collector.add_to_collection(var=net_output[2], name='Means', average_over_devices=False, summary_type='image3_coronal', collection=TF_SUMMARIES) outputs_collector.add_to_collection(var=net_output[5], name='Variances', average_over_devices=False, summary_type='image3_coronal', collection=TF_SUMMARIES) else: if self._infer_type in ('encode', 'encode-decode'): data_dict = self.get_sampler()[0][0].pop_batch_op() image = tf.cast(data_dict['image'], dtype=tf.float32) net_output = self.net(image, is_training=False) outputs_collector.add_to_collection( var=data_dict['image_location'], name='location', average_over_devices=True, collection=NETWORK_OUTPUT) if self._infer_type == 'encode-decode': outputs_collector.add_to_collection( var=net_output[2], name='generated_image', average_over_devices=True, collection=NETWORK_OUTPUT) if self._infer_type == 'encode': outputs_collector.add_to_collection( var=net_output[7], name='embedded', average_over_devices=True, collection=NETWORK_OUTPUT) self.output_decoder = WindowAsImageAggregator( image_reader=self.readers[0], output_path=self.action_param.save_seg_dir) return elif self._infer_type == 'sample': image_size = (self.net_param.batch_size,) + \ self.action_param.spatial_window_size + (1,) dummy_image = tf.zeros(image_size) net_output = self.net(dummy_image, is_training=False) noise_shape = net_output[-1].shape.as_list() stddev = self.autoencoder_param.noise_stddev noise = tf.random_normal(shape=noise_shape, mean=0.0, stddev=stddev, dtype=tf.float32) partially_decoded_sample = self.net.shared_decoder( noise, is_training=False) decoder_output = self.net.decoder_means( partially_decoded_sample, is_training=False) outputs_collector.add_to_collection(var=decoder_output, name='generated_image', average_over_devices=True, collection=NETWORK_OUTPUT) self.output_decoder = WindowAsImageAggregator( image_reader=None, output_path=self.action_param.save_seg_dir) return elif self._infer_type == 'linear_interpolation': # construct the entire network image_size = (self.net_param.batch_size,) + \ self.action_param.spatial_window_size + (1,) dummy_image = tf.zeros(image_size) net_output = self.net(dummy_image, is_training=False) data_dict = self.get_sampler()[0][0].pop_batch_op() real_code = data_dict['feature'] real_code = tf.reshape(real_code, net_output[-1].get_shape()) partially_decoded_sample = self.net.shared_decoder( real_code, is_training=False) decoder_output = self.net.decoder_means( partially_decoded_sample, is_training=False) outputs_collector.add_to_collection(var=decoder_output, name='generated_image', average_over_devices=True, collection=NETWORK_OUTPUT) outputs_collector.add_to_collection( var=data_dict['feature_location'], name='location', average_over_devices=True, collection=NETWORK_OUTPUT) self.output_decoder = WindowAsImageAggregator( image_reader=self.readers[0], output_path=self.action_param.save_seg_dir) else: raise NotImplementedError
def decay_warmup(params, hyper_params, global_step): """ Setups an optimizer object and returns it. :param params: dict :param hyper_params: dict, with hyper-parameters :return: optimizer """ NUM_EPOCHS_PER_DECAY = hyper_params['num_epochs_per_decay'] NUM_EPOCHS_PER_RAMP = hyper_params['num_epochs_per_ramp'] NUM_EPOCHS_IN_WARM_UP = hyper_params['num_epochs_in_warm_up'] NUM_STEPS_IN_WARM_UP = hyper_params['num_steps_in_warm_up'] NUM_STEPS_PER_WARM_UP = hyper_params['num_steps_per_warm_up'] INITIAL_LEARNING_RATE = hyper_params['initial_learning_rate'] LEARNING_RATE_DECAY_FACTOR = hyper_params['learning_rate_decay_factor'] WARM_UP_LEARNING_RATE_MAX = tf.constant( hyper_params['warm_up_max_learning_rate'], tf.float32) # Set parameters that affect the learning rate. decay_steps = hyper_params['num_steps_per_decay'] ramp_steps = NUM_STEPS_PER_WARM_UP ramp_up_steps = NUM_STEPS_IN_WARM_UP # Decay/ramp the learning rate exponentially based on the number of steps. def ramp(): lr = tf.train.exponential_decay(INITIAL_LEARNING_RATE, global_step, ramp_steps, LEARNING_RATE_DECAY_FACTOR, staircase=True) lr = INITIAL_LEARNING_RATE**2 * tf.pow(lr, tf.constant(-1.)) lr = tf.minimum(lr, WARM_UP_LEARNING_RATE_MAX) return lr def linear_ramp(): slope = WARM_UP_LEARNING_RATE_MAX / tf.cast(NUM_STEPS_PER_WARM_UP, tf.float32) lr = tf.cast(INITIAL_LEARNING_RATE, tf.float32) + tf.cast(global_step, tf.float32) * slope lr = tf.math.minimum(tf.cast(WARM_UP_LEARNING_RATE_MAX, tf.float32), lr) return lr def decay(lr_current): lr = tf.train.exponential_decay(lr_current, global_step, decay_steps, LEARNING_RATE_DECAY_FACTOR, staircase=True) return lr warm_up_slope = hyper_params.get('warm_up_slope', 0) def constant_ramp(): lr = tf.cast(INITIAL_LEARNING_RATE, tf.float32) * ( tf.cast(global_step, tf.float32) * warm_up_slope + 1) lr = tf.math.minimum(tf.cast(WARM_UP_LEARNING_RATE_MAX, tf.float32), lr) return lr if warm_up_slope > 1e-2: # LEARNING_RATE = tf.cond(global_step < ramp_up_steps, ramp, lambda: decay(ramp())) #LEARNING_RATE = tf.cond(global_step < ramp_up_steps, linear_ramp, lambda: decay(linear_ramp())) ramp_up_steps = tf.cast( WARM_UP_LEARNING_RATE_MAX / INITIAL_LEARNING_RATE, global_step.dtype) LEARNING_RATE = tf.cond(global_step < ramp_up_steps, constant_ramp, lambda: decay(constant_ramp())) else: LEARNING_RATE = tf.train.exponential_decay(INITIAL_LEARNING_RATE, global_step, decay_steps, LEARNING_RATE_DECAY_FACTOR, staircase=True) return LEARNING_RATE
def build(self, eval_flag): params = self.params batch_size = params.batch_size vocab_size = self.vocab_size num_layers = params.rnnlm_layers hidden_size = params.rnnlm_hidden_size num_steps = self.num_steps # initialize self # placeholders inputs = tf.placeholder('int32', shape=[None, num_steps], name='x') # [num_batch, num_steps] ground_truth = tf.placeholder('int32', shape=[None, num_steps], name='y') # [num_batch, num_steps] - right shift version of x is_training = tf.placeholder(tf.bool) keep_prob = tf.placeholder(tf.float32) batch_size = tf.shape(inputs)[0] normal_initializer = tf.random_normal_initializer(stddev=0.1) ones_initializer = tf.constant_initializer(1.0) with tf.variable_scope('RNNLM', initializer=normal_initializer): # Embeddings with tf.device('/cpu:0'): embedding_params = tf.get_variable('embedding_params', [vocab_size, hidden_size]) inputs_embedding = tf.nn.embedding_lookup(embedding_params, inputs) inputs_embedding = tf.cond(is_training, lambda: tf.nn.dropout(inputs_embedding, keep_prob), lambda: inputs_embedding) # Recurrence # Slightly better results can be obtained with forget gate biases # initialized to 1 but the hyperparameters of the model would need to be # different than reported in the paper. def lstm_cell(): # With the latest TensorFlow source code (as of Mar 27, 2017), # the BasicLSTMCell will need a reuse parameter which is unfortunately not # defined in TensorFlow 1.0. To maintain backwards compatibility, we add # an argument check here: if 'reuse' in inspect.getargspec( tf.contrib.rnn.BasicLSTMCell.__init__).args: return tf.contrib.rnn.BasicLSTMCell( hidden_size, forget_bias=0.0, state_is_tuple=True, reuse=tf.get_variable_scope().reuse) else: return tf.contrib.rnn.BasicLSTMCell( hidden_size, forget_bias=0.0, state_is_tuple=True) def attn_cell(): return tf.contrib.rnn.DropoutWrapper(lstm_cell(), output_keep_prob=keep_prob) cell = tf.contrib.rnn.MultiRNNCell( [attn_cell() for _ in range(num_layers)], state_is_tuple=True) self.initial_state = cell.zero_state(batch_size, 'float32') ### RNN ### outputs = [] state = self.initial_state with tf.variable_scope('core'): for time_step in range(num_steps): if time_step > 0:tf.get_variable_scope().reuse_variables() (cell_output, state) = cell(inputs_embedding[:, time_step, :], state) outputs.append(cell_output) output = tf.reshape(tf.stack(axis=1, values=outputs), [-1, hidden_size]) #[batch_size * num_steps, hidden_size] weight_o = tf.get_variable('output_weight', [hidden_size, vocab_size], 'float32') bias_o = tf.get_variable('output_bias', [vocab_size], 'float32') logits = tf.matmul(output, weight_o) + bias_o loss = tf.contrib.legacy_seq2seq.sequence_loss_by_example( [logits], [tf.reshape(ground_truth, [-1])], [tf.ones([batch_size * num_steps], dtype='float32')]) batch_size = tf.cast(batch_size, tf.float32) self.loss = tf.reduce_sum(loss) / batch_size log_perp = loss self.final_state = state # for recurrent purpose if not eval_flag: def learning_rate_decay_fn(lr, global_step): return tf.train.exponential_decay(lr, global_step, decay_steps=3000, decay_rate=0.8, staircase=True) OPTIMIZER_SUMMARIES = ["learning_rate", "loss", "gradients", "gradient_norm"] opt_op = tf.contrib.layers.optimize_loss(self.loss, self.global_step, learning_rate=params.learning_rate, optimizer=tf.train.AdamOptimizer, clip_gradients=10., learning_rate_decay_fn=learning_rate_decay_fn, summaries=OPTIMIZER_SUMMARIES) self.x = inputs self.y = ground_truth self.is_training = is_training self.keep_prob = keep_prob self.log_perp = log_perp #[batch_size, 1] self.num_steps = num_steps # Output Module if not eval_flag: self.opt_op = opt_op else: self.opt_op = None
def main(): if a.seed is None: a.seed = random.randint(0, 2**31 - 1) tf.set_random_seed(a.seed) np.random.seed(a.seed) random.seed(a.seed) if not os.path.exists(a.output_dir): os.makedirs(a.output_dir) if a.mode == "test" or a.mode == "export": if a.checkpoint is None: raise Exception("checkpoint required for test mode") # load some options from the checkpoint options = {"which_direction", "ngf", "ndf", "lab_colorization"} with open(os.path.join(a.checkpoint, "options.json")) as f: for key, val in json.loads(f.read()).items(): if key in options: print("loaded", key, "=", val) setattr(a, key, val) # disable these features in test mode a.scale_size = CROP_SIZE a.flip = False for k, v in a._get_kwargs(): print(k, "=", v) with open(os.path.join(a.output_dir, "options.json"), "w") as f: f.write(json.dumps(vars(a), sort_keys=True, indent=4)) if a.mode == "export": # export the generator to a meta graph that can be imported later for standalone generation if a.lab_colorization: raise Exception("export not supported for lab_colorization") ipt = tf.placeholder(tf.string, shape=[1]) input_data = tf.decode_base64(ipt[0]) input_image = tf.image.decode_png(input_data) # remove alpha channel if present input_image = tf.cond(tf.equal(tf.shape(input_image)[2], 4), lambda: input_image[:, :, :3], lambda: input_image) # convert grayscale to RGB input_image = tf.cond(tf.equal(tf.shape(input_image)[2], 1), lambda: tf.image.grayscale_to_rgb(input_image), lambda: input_image) input_image = tf.image.convert_image_dtype(input_image, dtype=tf.float32) input_image.set_shape([CROP_SIZE, CROP_SIZE, 3]) batch_input = tf.expand_dims(input_image, axis=0) with tf.variable_scope("generator"): batch_output = deprocess( create_generator(preprocess(batch_input), 3)) output_image = tf.image.convert_image_dtype(batch_output, dtype=tf.uint8)[0] if a.output_filetype == "png": output_data = tf.image.encode_png(output_image) elif a.output_filetype == "jpeg": output_data = tf.image.encode_jpeg(output_image, quality=80) else: raise Exception("invalid filetype") output = tf.convert_to_tensor([tf.encode_base64(output_data)]) key = tf.placeholder(tf.string, shape=[1]) inputs = {"key": key.name, "input": ipt.name} tf.add_to_collection("inputs", json.dumps(inputs)) outputs = { "key": tf.identity(key).name, "output": output.name, } tf.add_to_collection("outputs", json.dumps(outputs)) init_op = tf.global_variables_initializer() restore_saver = tf.train.Saver() export_saver = tf.train.Saver() with tf.Session() as sess: sess.run(init_op) print("loading model from checkpoint") checkpoint = tf.train.latest_checkpoint(a.checkpoint) restore_saver.restore(sess, checkpoint) print("exporting model") export_saver.export_meta_graph( filename=os.path.join(a.output_dir, "export.meta")) export_saver.save(sess, os.path.join(a.output_dir, "export"), write_meta_graph=False) return examples = load_examples() print("examples count = %d" % examples.count) # inputs and targets are [batch_size, height, width, channels] model = create_model(examples.inputs, examples.targets) # undo colorization splitting on images that we use for display/output if a.lab_colorization: if a.which_direction == "AtoB": # inputs is brightness, this will be handled fine as a grayscale image # need to augment targets and outputs with brightness targets = augment(examples.targets, examples.inputs) outputs = augment(model.outputs, examples.inputs) # inputs can be deprocessed normally and handled as if they are single channel # grayscale images inputs = deprocess(examples.inputs) elif a.which_direction == "BtoA": # inputs will be color channels only, get brightness from targets inputs = augment(examples.inputs, examples.targets) targets = deprocess(examples.targets) outputs = deprocess(model.outputs) else: raise Exception("invalid direction") else: inputs = deprocess(examples.inputs) targets = deprocess(examples.targets) outputs = deprocess(model.outputs) if a.img_masks: masks = examples.masks def convert(image): if a.aspect_ratio != 1.0: # upscale to correct aspect ratio size = [CROP_SIZE, int(round(CROP_SIZE * a.aspect_ratio))] image = tf.image.resize_images( image, size=size, method=tf.image.ResizeMethod.BICUBIC) return tf.image.convert_image_dtype(image, dtype=tf.uint8, saturate=True) # reverse any processing on images so they can be written to disk or displayed to user with tf.name_scope("convert_inputs"): converted_inputs = convert(inputs) with tf.name_scope("convert_targets"): converted_targets = convert(targets) with tf.name_scope("convert_outputs"): converted_outputs = convert(outputs) if a.img_masks and a.use_cue: with tf.name_scope("convert_masks"): converted_masks = convert(masks) # converted_masks = tf.expand_dims(converted_masks) with tf.name_scope("encode_images"): display_fetches = { "paths": examples.paths, "inputs": tf.map_fn(tf.image.encode_png, converted_inputs, dtype=tf.string, name="input_pngs"), "targets": tf.map_fn(tf.image.encode_png, converted_targets, dtype=tf.string, name="target_pngs"), "outputs": tf.map_fn(tf.image.encode_png, converted_outputs, dtype=tf.string, name="output_pngs"), } if a.img_masks and a.use_cue: # TODO add circle masks to this shit display_fetches['mask'] = tf.map_fn(tf.image.encode_png, converted_masks, dtype=tf.string, name="output_pngs"), # summaries with tf.name_scope("inputs_summary"): tf.summary.image("inputs", converted_inputs) with tf.name_scope("targets_summary"): tf.summary.image("targets", converted_targets) with tf.name_scope("outputs_summary"): tf.summary.image("outputs", converted_outputs) if a.img_masks and a.use_cue: with tf.name_scope("masks_summary"): tf.summary.image("masks", converted_masks) with tf.name_scope("predict_real_summary"): tf.summary.image( "predict_real", tf.image.convert_image_dtype(model.predict_real, dtype=tf.uint8)) with tf.name_scope("predict_fake_summary"): tf.summary.image( "predict_fake", tf.image.convert_image_dtype(model.predict_fake, dtype=tf.uint8)) tf.summary.scalar("discriminator_loss", model.discrim_loss) tf.summary.scalar("generator_loss_GAN", model.gen_loss_GAN) tf.summary.scalar("generator_loss_L1", model.gen_loss_L1) for var in tf.trainable_variables(): tf.summary.histogram(var.op.name + "/values", var) for grad, var in model.discrim_grads_and_vars + model.gen_grads_and_vars: tf.summary.histogram(var.op.name + "/gradients", grad) with tf.name_scope("parameter_count"): parameter_count = tf.reduce_sum( [tf.reduce_prod(tf.shape(v)) for v in tf.trainable_variables()]) saver = tf.train.Saver(max_to_keep=1) logdir = a.output_dir if (a.trace_freq > 0 or a.summary_freq > 0) else None sv = tf.train.Supervisor(logdir=logdir, save_summaries_secs=0, saver=None) with sv.managed_session() as sess: print("parameter_count =", sess.run(parameter_count)) if a.checkpoint is not None: print("loading model from checkpoint") checkpoint = tf.train.latest_checkpoint(a.checkpoint) saver.restore(sess, checkpoint) max_steps = 2**32 if a.max_epochs is not None: max_steps = examples.steps_per_epoch * a.max_epochs if a.max_steps is not None: max_steps = a.max_steps if a.mode == "test": # testing # at most, process the test data once start = time.time() max_steps = min(examples.steps_per_epoch, max_steps) for step in range(max_steps): results = sess.run(display_fetches) filesets = save_images(results) for i, f in enumerate(filesets): print("evaluated image", f["name"]) index_path = append_index(filesets) print("wrote index at", index_path) print("rate", (time.time() - start) / max_steps) else: # training start = time.time() for step in range(max_steps): def should(freq): return freq > 0 and ((step + 1) % freq == 0 or step == max_steps - 1) options = None run_metadata = None if should(a.trace_freq): options = tf.RunOptions( trace_level=tf.RunOptions.FULL_TRACE) run_metadata = tf.RunMetadata() fetches = { "train": model.train, "global_step": sv.global_step, } if should(a.progress_freq): fetches["discrim_loss"] = model.discrim_loss fetches["gen_loss_GAN"] = model.gen_loss_GAN fetches["gen_loss_L1"] = model.gen_loss_L1 if should(a.summary_freq): fetches["summary"] = sv.summary_op if should(a.display_freq): fetches["display"] = display_fetches results = sess.run(fetches, options=options, run_metadata=run_metadata) if should(a.summary_freq): print("recording summary") sv.summary_writer.add_summary(results["summary"], results["global_step"]) if should(a.display_freq): filesets = save_images(results["display"], step=results["global_step"]) append_index(filesets, step=True) if should(a.trace_freq): print("recording trace") sv.summary_writer.add_run_metadata( run_metadata, "step_%d" % results["global_step"]) if should(a.progress_freq): # global_step will have the correct step count if we resume from a checkpoint train_epoch = math.ceil(results["global_step"] / examples.steps_per_epoch) train_step = (results["global_step"] - 1) % examples.steps_per_epoch + 1 rate = (step + 1) * a.batch_size / (time.time() - start) remaining = (max_steps - step) * a.batch_size / rate print( "progress epoch %d step %d image/sec %0.1f remaining %dm" % (train_epoch, train_step, rate, remaining / 60)) print("discrim_loss", results["discrim_loss"]) print("gen_loss_GAN", results["gen_loss_GAN"]) print("gen_loss_L1", results["gen_loss_L1"]) if should(a.save_freq): print("saving model") saver.save(sess, os.path.join(a.output_dir, "model"), global_step=sv.global_step) if sv.should_stop(): break
def true_segments_1d(segments, mode=SegmentsMode.CENTERS, max_gap=0, min_length=0, name=None): """Labels contiguous True runs in segments. Args: segments: 1D boolean tensor. mode: The SegmentsMode. Returns the start of each segment (STARTS), or the rounded center of each segment (CENTERS). max_gap: Fill gaps of length at most `max_gap` between true segments. int. min_length: Minimum length of a returned segment. int. name: Optional name for the op. Returns: run_centers: int32 tensor. Depending on `mode`, either the start of each True run, or the (rounded) center of each True run. run_lengths: int32; the lengths of each True run. """ with tf.name_scope(name, "true_segments", [segments]): segments = tf.convert_to_tensor(segments, tf.bool) run_starts, run_lengths = _segments_1d(segments, mode=SegmentsMode.STARTS) # Take only the True runs. After whichever run is True first, the True runs # are every other run. first_run = tf.cond( # First value is False, or all values are False. Handles empty segments # correctly. tf.logical_or(tf.reduce_any(segments[0:1]), ~tf.reduce_any(segments)), lambda: tf.constant(0), lambda: tf.constant(1)) num_runs = tf.shape(run_starts)[0] run_nums = tf.range(num_runs) is_true_run = tf.equal(run_nums % 2, first_run % 2) # Find gaps between True runs that can be merged. is_gap = tf.logical_and( tf.not_equal(run_nums % 2, first_run % 2), tf.logical_and(tf.greater(run_nums, first_run), tf.less(run_nums, num_runs - 1))) fill_gap = tf.logical_and(is_gap, tf.less_equal(run_lengths, max_gap)) # Segment the consecutive runs of True or False values based on whether they # are True, or are a gap of False values that can be bridged. Then, flatten # the runs of runs. runs_to_merge = tf.logical_or(is_true_run, fill_gap) run_of_run_starts, _ = _segments_1d(runs_to_merge, mode=SegmentsMode.STARTS) # Get the start of every new run from the original run starts. merged_run_starts = tf.gather(run_starts, run_of_run_starts) # Make an array mapping the original runs to their run of runs. Increment # the number for every run of run start except for the first one, so that # the array has values from 0 to num_run_of_runs. merged_run_inds = tf.cumsum( tf.sparse_to_dense( sparse_indices=tf.cast(run_of_run_starts[1:, None], tf.int64), output_shape=tf.cast(num_runs[None], tf.int64), sparse_values=tf.ones_like(run_of_run_starts[1:]))) # Sum the lengths of the original runs that were merged. merged_run_lengths = tf.segment_sum(run_lengths, merged_run_inds) if mode is SegmentsMode.CENTERS: merged_starts_or_centers = (merged_run_starts + tf.floordiv(merged_run_lengths - 1, 2)) else: merged_starts_or_centers = merged_run_starts # If there are no true values, increment first_run to 1, so we will skip # the single (false) run. first_run += tf.to_int32(tf.logical_not(tf.reduce_any(segments))) merged_starts_or_centers = merged_starts_or_centers[first_run::2] merged_run_lengths = merged_run_lengths[first_run::2] # Only take segments at least min_length long. is_long_enough = tf.greater_equal(merged_run_lengths, min_length) is_long_enough.set_shape([None]) merged_starts_or_centers = tf.boolean_mask(merged_starts_or_centers, is_long_enough) merged_run_lengths = tf.boolean_mask(merged_run_lengths, is_long_enough) return merged_starts_or_centers, merged_run_lengths
def conv_embed(label, it_same, it_diff): next_same = it_same.get_next() next_diff = it_diff.get_next() input_layer = preprocess( tf.cond(label, lambda: next_diff, lambda: next_same)) conv1 = tf.layers.conv2d(inputs=input_layer, filters=64, kernel_size=[10, 10], padding="valid", activation=tf.nn.relu) pool1 = tf.layers.max_pooling2d(inputs=conv1, pool_size=[2, 2], strides=2) conv2 = tf.layers.conv2d(inputs=pool1, filters=128, kernel_size=[7, 7], padding="valid", activation=tf.nn.relu) pool2 = tf.layers.max_pooling2d(inputs=conv2, pool_size=[2, 2], strides=2) conv3 = tf.layers.conv2d(inputs=pool2, filters=128, kernel_size=[4, 4], padding="valid", activation=tf.nn.relu) pool3 = tf.layers.max_pooling2d(inputs=conv3, pool_size=[2, 2], strides=2) conv4 = tf.layers.conv2d(inputs=pool3, filters=256, kernel_size=[4, 4], padding="valid", activation=tf.nn.relu) pool4 = tf.layers.max_pooling2d(inputs=conv4, pool_size=[2, 2], strides=2) conv5 = tf.layers.conv2d(inputs=pool4, filters=256, kernel_size=[3, 3], padding="valid", activation=tf.nn.relu) pool5 = tf.layers.max_pooling2d(inputs=conv5, pool_size=[2, 2], strides=2) flat = tf.contrib.layers.flatten(inputs=pool5) dense = tf.layers.dense(inputs=flat, units=4096, activation=tf.nn.relu) dropout = tf.layers.dropout(inputs=dense, rate=0.5) sphere = tf.nn.l2_normalize( dropout, axis=1) #project outputs of previous layer onto surface of hypersphere loss = pairwise_quadratic_loss(sphere, label) optimizer = tf.train.AdamOptimizer() train = optimizer.minimize(loss) return {"embedding": sphere, "loss": loss, "train": train}
def _compute_loss(self,input_rnn,seq_len,ner_ids,scoreMatrix,is_train,dropout_embedding_keep,dropout_lstm_keep, dropout_lstm_output_keep,dropout_fcl_ner_keep,dropout_fcl_rel_keep,mask,reuse=False): """计算损失""" with tf.variable_scope("loss_conputation",reuse=reuse): if self.config.use_dropout: input_rnn=tf.nn.dropout(input_rnn,keep_prob=dropout_embedding_keep) #BiLSTM底层编码 # for i in range(self.config.num_lstm_layers): # if self.config.use_dropout and i>0: # input_rnn=tf.nn.dropout(input_rnn,keep_prob=dropout_lstm_keep) # lstm_fw_cell=tf.nn.rnn_cell.LSTMCell(self.config.hidden_size_lstm) # lstm_bw_cell=tf.nn.rnn_cell.LSTMCell(self.config.hidden_size_lstm) # # outputs,states=tf.nn.bidirectional_dynamic_rnn(lstm_fw_cell, # lstm_bw_cell, # input_rnn, # sequence_length=seq_len, # dtype=tf.float32, # scope="BiLSTM"+str(i)) # input_rnn = tf.concat(outputs, axis=-1) # lstm_output = input_rnn input_rnn=tf.keras.layers.Bidirectional(CuDNNLSTM(units=self.config.hidden_size_lstm,return_sequences=True))(inputs=input_rnn) input_rnn=tf.keras.layers.Bidirectional(CuDNNLSTM(units=self.config.hidden_size_lstm,return_sequences=True))(inputs=input_rnn) lstm_output=input_rnn lstm_output = self._attention(lstm_output,mask) if self.config.use_dropout: lstm_output=tf.nn.dropout(lstm_output,keep_prob=dropout_lstm_output_keep) #计算LSTM发射分数 nerScores=self._get_ner_score(lstm_output,self.config.num_ner_classes,dropout_fcl_ner_keep) #CRF计算NER损失 log_likelihood,trans_params=tf.contrib.crf.crf_log_likelihood(nerScores,ner_ids,seq_len) lossNER=tf.reduce_mean(-log_likelihood) #viterbi解码预测NER标签 predNERS,viterbi_score=tf.contrib.crf.crf_decode(nerScores,trans_params,seq_len) #RC输入 if self.config.label_embedding_size>0: labels=tf.cond(is_train,lambda:ner_ids,lambda:predNERS) label_matrix=tf.get_variable(name="label_embedding",shape=[self.config.num_ner_classes, self.config.label_embedding_size],dtype=tf.float32) label_embedding=tf.nn.embedding_lookup(label_matrix,labels) rel_input=tf.concat([lstm_output,label_embedding],axis=2) else: rel_input=lstm_output #关系抽取分数 relScore=self._get_head_selection_score(rel_input,dropout_keep_in_prob=dropout_fcl_rel_keep) #交叉熵计算损失函数 lossRel=tf.reduce_mean(tf.nn.sigmoid_cross_entropy_with_logits(logits=relScore,labels=scoreMatrix)) #预测关系 probas=tf.nn.sigmoid(relScore) predRel=tf.round(probas) #四舍五入 return lossNER,lossRel,predNERS,predRel,relScore
def _build_model(self): self.X = tf.placeholder(tf.float32, [None, self.input_dim]) self.y = tf.placeholder(tf.float32, [None, self.label_count]) self.domain = tf.placeholder(tf.float32, [None, 2]) self.l = tf.placeholder(tf.float32, []) self.train = tf.placeholder(tf.bool, []) batch_size = self.batch_size X_input = self.X # X_input = (tf.cast(self.X, tf.float32) - pixel_mean) / 255. # CNN model for feature extraction with tf.variable_scope('feature_extractor'): input = X_input for layer in self.encoder_h_layers: input = self._add_layer(input, layer) # The domain-invariant feature self.feature = input # MLP for class prediction with tf.variable_scope('label_predictor'): # Switches to route target examples (second half of batch) differently # depending on train or test mode. all_features = lambda: self.feature source_features = lambda: tf.slice(self.feature, [0, 0], [batch_size // 2, -1]) classify_feats = tf.cond(self.train, source_features, all_features) all_labels = lambda: self.y source_labels = lambda: tf.slice(self.y, [0, 0], [batch_size // 2, -1]) self.classify_labels = tf.cond(self.train, source_labels, all_labels) print("tf.shape(all_labels)=", tf.shape(all_labels())) print("tf.shape(source_labels)=", tf.shape(source_labels())) #tf.shape(all_labels)= Tensor("label_predictor/Shape:0", shape=(2,), dtype=int32) #tf.shape(source_labels)= Tensor("label_predictor/Shape_1:0", shape=(2,), dtype=int32) input = classify_feats for layer in self.clf_h_layers: input = self._add_layer(input, layer) #h_fc0 = self._add_layer(classify_feats, 128) h_fc1 = self._add_layer(input, self.label_count, None) logits = h_fc1 self.pred = tf.nn.softmax(logits) self.pred_loss = tf.nn.softmax_cross_entropy_with_logits( logits=logits, labels=self.classify_labels) # Small MLP for domain prediction with adversarial loss with tf.variable_scope('domain_predictor'): # Flip the gradient when backpropagating through this operation feat = flip_gradient(self.feature, self.l) #feat = self.feature input = feat for layer in self.da_h_layers: input = self._add_layer(input, layer) #d_h_fc0 = self._add_layer(feat, 32) d_b_fc1 = self._add_layer(input, 2, None) d_logits = d_b_fc1 self.domain_pred = tf.nn.softmax(d_logits) self.domain_loss = tf.nn.softmax_cross_entropy_with_logits( logits=d_logits, labels=self.domain)
def add_model(self): #nn_in = self.input_x with tf.name_scope("embedding"): self.ent_embeddings = tf.get_variable(name = "ent_embedding", shape = [self.max_ent, self.p.inp_dim], initializer = tf.contrib.layers.xavier_initializer(uniform = False), regularizer=self.regularizer) self.rel_embeddings = tf.get_variable(name = "rel_embedding", shape = [self.max_rel, self.p.inp_dim], initializer = tf.contrib.layers.xavier_initializer(uniform = False), regularizer=self.regularizer) self.time_embeddings = tf.get_variable(name = "time_embedding",shape = [self.max_time, self.p.inp_dim], initializer = tf.contrib.layers.xavier_initializer(uniform =False)) transE_in_dim = self.p.inp_dim transE_in = self.ent_embeddings ####################------------------------ time aware GCN MODEL ---------------------------############## ## Some transE style model #### neutral = tf.constant(0) ## mode = 1 for train mode = -1 test test_type = tf.constant(0) ## pred_mode = 1 for head -1 for tail query_type = tf.constant(0) ## query mode =1 for head tail , -1 for rel def f_train(): pos_h_e = tf.squeeze(tf.nn.embedding_lookup(transE_in, self.pos_head)) pos_t_e = tf.squeeze(tf.nn.embedding_lookup(transE_in, self.pos_tail)) pos_r_e = tf.squeeze(tf.nn.embedding_lookup(self.rel_embeddings, self.rel)) return pos_h_e, pos_t_e, pos_r_e def f_test(): def head_tail_query(): def f_head(): e2 = tf.squeeze(tf.nn.embedding_lookup(transE_in, self.pos_tail)) pos_h_e = transE_in pos_t_e = tf.reshape(tf.tile(e2,[self.max_ent]),(self.max_ent, transE_in_dim)) return pos_h_e, pos_t_e def f_tail(): e1 = tf.squeeze(tf.nn.embedding_lookup(transE_in, self.pos_head)) pos_h_e = tf.reshape(tf.tile(e1,[self.max_ent]),(self.max_ent, transE_in_dim)) pos_t_e = transE_in return pos_h_e, pos_t_e pos_h_e, pos_t_e = tf.cond(self.pred_mode > test_type, f_head, f_tail) r = tf.squeeze(tf.nn.embedding_lookup(self.rel_embeddings,self.rel)) pos_r_e = tf.reshape(tf.tile(r,[self.max_ent]),(self.max_ent,transE_in_dim)) return pos_h_e, pos_t_e, pos_r_e def rel_query(): e1 = tf.squeeze(tf.nn.embedding_lookup(transE_in, self.pos_head)) e2 = tf.squeeze(tf.nn.embedding_lookup(transE_in, self.pos_tail)) pos_h_e = tf.reshape(tf.tile(e1,[self.max_rel]),(self.max_rel, transE_in_dim)) pos_t_e = tf.reshape(tf.tile(e2,[self.max_rel]),(self.max_rel, transE_in_dim)) pos_r_e = self.rel_embeddings return pos_h_e, pos_t_e, pos_r_e pos_h_e, pos_t_e, pos_r_e = tf.cond(self.query_mode > query_type, head_tail_query, rel_query) return pos_h_e, pos_t_e, pos_r_e pos_h_e, pos_t_e, pos_r_e = tf.cond(self.mode > neutral, f_train, f_test) neg_h_e = tf.squeeze(tf.nn.embedding_lookup(transE_in, self.neg_head)) neg_t_e = tf.squeeze(tf.nn.embedding_lookup(transE_in, self.neg_tail)) #### ----- time -----### t_1 = tf.squeeze(tf.nn.embedding_lookup(self.time_embeddings, self.start_year)) pos_h_e_t_1 = self.time_projection(pos_h_e,t_1) neg_h_e_t_1 = self.time_projection(neg_h_e,t_1) pos_t_e_t_1 = self.time_projection(pos_t_e,t_1) neg_t_e_t_1 = self.time_projection(neg_t_e,t_1) pos_r_e_t_1 = self.time_projection(pos_r_e,t_1) # pos_r_e_t_1 = pos_r_e if self.p.L1_flag: pos = tf.reduce_sum(abs(pos_h_e_t_1 + pos_r_e_t_1 - pos_t_e_t_1), 1, keep_dims = True) neg = tf.reduce_sum(abs(neg_h_e_t_1 + pos_r_e_t_1 - neg_t_e_t_1), 1, keep_dims = True) #self.predict = pos else: pos = tf.reduce_sum((pos_h_e_t_1 + pos_r_e_t_1 - pos_t_e_t_1) ** 2, 1, keep_dims = True) neg = tf.reduce_sum((neg_h_e_t_1 + pos_r_e_t_1 - neg_t_e_t_1) ** 2, 1, keep_dims = True) #self.predict = pos ''' debug_nn([self.pred_mode,self.mode], feed_dict = self.create_feed_dict(self.data[0:self.p.batch_size],dtype='test')) ''' return pos, neg
def testNeuralLinUCBUpdateNumTrainSteps0(self, batch_size=1, context_dim=10): """Check NeuralLinUCBAgent updates when behaving like LinUCB.""" # Construct a `Trajectory` for the given action, observation, reward. num_actions = 5 initial_step, final_step = _get_initial_and_final_steps( batch_size, context_dim) action = np.random.randint(num_actions, size=batch_size, dtype=np.int32) action_step = _get_action_step(action) experience = _get_experience(initial_step, action_step, final_step) # Construct an agent and perform the update. observation_spec = tensor_spec.TensorSpec([context_dim], tf.float32) time_step_spec = time_step.time_step_spec(observation_spec) action_spec = tensor_spec.BoundedTensorSpec(dtype=tf.int32, shape=(), minimum=0, maximum=num_actions - 1) encoder = DummyNet(obs_dim=context_dim) encoding_dim = 10 agent = neural_linucb_agent.NeuralLinUCBAgent( time_step_spec=time_step_spec, action_spec=action_spec, encoding_network=encoder, encoding_network_num_train_steps=0, encoding_dim=encoding_dim, optimizer=tf.compat.v1.train.AdamOptimizer(learning_rate=1e-2)) loss_info = agent.train(experience) self.evaluate(agent.initialize()) self.evaluate(tf.compat.v1.global_variables_initializer()) self.evaluate(loss_info) final_a = self.evaluate(agent.cov_matrix) final_b = self.evaluate(agent.data_vector) # Compute the expected updated estimates. observations_list = tf.dynamic_partition( data=tf.reshape(tf.cast(experience.observation, tf.float64), [batch_size, context_dim]), partitions=tf.convert_to_tensor(action), num_partitions=num_actions) rewards_list = tf.dynamic_partition( data=tf.reshape(tf.cast(experience.reward, tf.float64), [batch_size]), partitions=tf.convert_to_tensor(action), num_partitions=num_actions) expected_a_updated_list = [] expected_b_updated_list = [] for _, (observations_for_arm, rewards_for_arm) in enumerate( zip(observations_list, rewards_list)): encoded_observations_for_arm, _ = encoder(observations_for_arm) encoded_observations_for_arm = tf.cast( encoded_observations_for_arm, dtype=tf.float64) num_samples_for_arm_current = tf.cast( tf.shape(rewards_for_arm)[0], tf.float64) num_samples_for_arm_total = num_samples_for_arm_current # pylint: disable=cell-var-from-loop def true_fn(): a_new = tf.matmul(encoded_observations_for_arm, encoded_observations_for_arm, transpose_a=True) b_new = bandit_utils.sum_reward_weighted_observations( rewards_for_arm, encoded_observations_for_arm) return a_new, b_new def false_fn(): return (tf.zeros([encoding_dim, encoding_dim], dtype=tf.float64), tf.zeros([encoding_dim], dtype=tf.float64)) a_new, b_new = tf.cond( tf.squeeze(num_samples_for_arm_total) > 0, true_fn, false_fn) expected_a_updated_list.append(self.evaluate(a_new)) expected_b_updated_list.append(self.evaluate(b_new)) # Check that the actual updated estimates match the expectations. self.assertAllClose(expected_a_updated_list, final_a) self.assertAllClose(expected_b_updated_list, final_b)
def _parse_line(line): input_data = dict.fromkeys(input_keys) line_split = tf.strings.split([line], '|').values # process uid input_data['uid'] = tf.strings.to_number(line_split[data_map['uid']], out_type=tf.int64) def _string_to_sparse(string, dense_shape): number = tf.strings.to_number(tf.strings.split([string], sep=',').values, out_type=tf.int64) return tf.sparse.SparseTensor(indices=tf.expand_dims(number, axis=1), values=tf.ones_like(number, dtype=tf.int8), dense_shape=dense_shape) # process image input_data['image'] = tf.sparse.to_dense( tf.cond( tf.strings.length(line_split[data_map['image']]) > 0, true_fn=lambda: _string_to_sparse( line_split[data_map['image']], dense_shape=[num_softid]), false_fn=lambda: tf.sparse.SparseTensor( indices=np.empty((0, 1), dtype=np.int64), values=tf.constant([], dtype=tf.int8), dense_shape=[num_softid]))) # process label label = tf.sparse.to_dense( tf.cond( tf.strings.length(line_split[data_map['label']]) > 0, true_fn=lambda: _string_to_sparse( line_split[data_map['label']], dense_shape=[num_label]), false_fn=lambda: tf.sparse.SparseTensor( indices=np.empty((0, 1), dtype=np.int64), values=tf.constant([], dtype=tf.int8), dense_shape=[num_label]))) # process lists def _string_to_list(string, padding): number = tf.strings.to_number(tf.strings.split([string], sep=',').values, out_type=tf.int32) return tf.concat([ tf.tile(tf.constant([padding], dtype=tf.int32), tf.math.maximum(length_his - tf.shape(number), 0)), number[-length_his:] ], axis=0) input_data['new_time_list'] = tf.cond( tf.strings.length(line_split[data_map['new_time_list']]) > 0, true_fn=lambda: _string_to_list( line_split[data_map['new_time_list']], padding=-7) // 7, false_fn=lambda: tf.tile(tf.constant([-1], dtype=tf.int32), [length_his])) input_data['new_soft_list'] = tf.cond( tf.strings.length(line_split[data_map['new_soft_list']]) > 0, true_fn=lambda: _string_to_list( line_split[data_map['new_soft_list']], padding=-1), false_fn=lambda: tf.tile(tf.constant([-1], dtype=tf.int32), [length_his])) input_data['new_mask_list'] = tf.to_int32( tf.greater_equal( input_data['new_time_list'], tf.tile(tf.constant([0], dtype=tf.int32), [length_his]))) input_data['new_bert_mask_index'] = tf.reverse(tf.math.top_k( tf.random.shuffle( tf.constant(np.asarray(range(length_his)), dtype=tf.int8))[:bert_mask_num], k=bert_mask_num).values, axis=[0]) input_data['new_bert_mask'] = tf.sparse.to_dense( tf.sparse.SparseTensor(indices=tf.expand_dims(tf.to_int64( input_data['new_bert_mask_index']), axis=1), values=tf.ones_like( input_data['new_bert_mask_index'], dtype=tf.int8), dense_shape=[length_his])) input_data['loss_time_list'] = tf.cond( tf.strings.length(line_split[data_map['loss_time_list']]) > 0, true_fn=lambda: _string_to_list( line_split[data_map['loss_time_list']], padding=-7) // 7, false_fn=lambda: tf.tile(tf.constant([-1], dtype=tf.int32), [length_his])) input_data['loss_soft_list'] = tf.cond( tf.strings.length(line_split[data_map['loss_soft_list']]) > 0, true_fn=lambda: _string_to_list( line_split[data_map['loss_soft_list']], padding=-1), false_fn=lambda: tf.tile(tf.constant([-1], dtype=tf.int32), [length_his])) input_data['loss_mask_list'] = tf.to_int32( tf.greater_equal( input_data['loss_time_list'], tf.tile(tf.constant([0], dtype=tf.int32), [length_his]))) input_data['loss_bert_mask_index'] = tf.reverse(tf.math.top_k( tf.random.shuffle( tf.constant(np.asarray(range(length_his)), dtype=tf.int8))[:bert_mask_num], k=bert_mask_num).values, axis=[0]) input_data['loss_bert_mask'] = tf.sparse.to_dense( tf.sparse.SparseTensor(indices=tf.expand_dims(tf.to_int64( input_data['loss_bert_mask_index']), axis=1), values=tf.ones_like( input_data['loss_bert_mask_index'], dtype=tf.int8), dense_shape=[length_his])) return input_data, label
def augment_rotate(self, image_a, image_b): r = tf.unstack(tf.random_uniform([1], minval=0, maxval=2, dtype=tf.int32, seed=None, name=None)) rotate_boolean = tf.equal(0, r, name="check-rotate-boolean") [image_a, image_b] = tf.cond(rotate_boolean[0], lambda: self.rotate_data(image_a, image_b), lambda: [image_a, image_b]) return image_a, image_b
def build_model(self): with tf.variable_scope('Model', reuse=tf.AUTO_REUSE): # Model Feeds self.ratings = tf.placeholder(dtype=tf.float32, shape=[None, self.num_item], name='ratings') self.output_mask = tf.placeholder(dtype=tf.bool, shape=[None, self.num_item], name='output_mask') self.istraining = tf.placeholder(dtype=tf.bool, shape=[], name='training_flag') self.isnegsample = tf.placeholder(dtype=tf.bool, shape=[], name='negative_sample_flag') self.add_hidden_noise = tf.placeholder(dtype=tf.bool, shape=[], name='add_hidden_noise') self.add_weight_noise = tf.placeholder(dtype=tf.bool, shape=[], name='add_weight_noise') self.layer1_dropout_rate = tf.placeholder( dtype=tf.float32, shape=[], name='layer1_dropout_rate') input = self.ratings # Encoder Variables layer1_w = tf.get_variable( name='encoder_weights', shape=[self.num_item, self.num_factors], initializer=tf.truncated_normal_initializer(mean=0.0, stddev=0.03)) layer1_b = tf.get_variable(name='encoder_bias', shape=[self.num_factors], initializer=tf.zeros_initializer()) # user_node = tf.get_variable(name='user_nodes', # shape=[self.num_factors], # initializer=tf.truncated_normal_initializer(mean=0.0, stddev=0.03) # ) w2_noise = tf.get_variable(name='w2_noise', shape=[self.num_factors, self.num_item], initializer=tf.zeros_initializer(), dtype=tf.float32, trainable=False) hidden_noise_tr = tf.get_variable( name='hidden_noise_tr', shape=[self.batch_size, self.num_factors], initializer=tf.zeros_initializer(), dtype=tf.float32, trainable=False) # Decoder Variables layer2_w = tf.get_variable( name='decoder_weights', shape=[self.num_factors, self.num_item], initializer=tf.truncated_normal_initializer(mean=0.0, stddev=0.03)) layer2_b = tf.get_variable(name='decoder_bias', shape=[self.num_item], initializer=tf.zeros_initializer()) # Original AE Model org_encoder = tf.sigmoid(tf.matmul(input, layer1_w) + layer1_b) org_encoder = tf.cond( self.istraining, lambda: tf.layers.dropout(org_encoder, rate=self.layer1_dropout_rate, name='layer1_dropout'), lambda: org_encoder) if self.robust_test: # Robustness Testing on W2 org_decoder = tf.identity( tf.matmul(org_encoder, layer2_w + w2_noise) + layer2_b) self.w2 = layer2_w self.w2_org = layer2_w - w2_noise else: org_decoder = tf.identity( tf.matmul(org_encoder, layer2_w) + layer2_b) mask = tf.cond( self.isnegsample, lambda: tf.cast(self.output_mask, dtype=org_decoder.dtype), lambda: tf.sign(self.ratings)) self.org_output = tf.cond(self.istraining, lambda: tf.multiply(org_decoder, mask), lambda: org_decoder) org_base_loss = tf.reduce_sum( tf.nn.sigmoid_cross_entropy_with_logits( labels=input, logits=self.org_output)) org_base_loss = org_base_loss / tf.cast(tf.shape(input)[0], dtype=org_base_loss.dtype) # org_reg_loss = self.ae_regs[0] * tf.nn.l2_loss(layer1_w) + self.ae_regs[1] * tf.nn.l2_loss(layer1_b) + \ # self.ae_regs[2] * tf.nn.l2_loss(layer2_w2) + self.ae_regs[3] * tf.nn.l2_loss(layer2_b) # org_loss = org_base_loss + org_reg_loss # Noisy Model if not self.robust_test: noisy_encoder = tf.sigmoid( tf.matmul(input, layer1_w) + layer1_b) noisy_encoder = tf.cond( self.istraining, lambda: noisy_encoder + hidden_noise_tr, lambda: noisy_encoder) # layer1 = tf.cond(self.istraining, # lambda : tf.sigmoid(tf.matmul(input, layer1_w + layer1_delta) + layer1_b), # lambda : tf.sigmoid(tf.matmul(input, layer1_w) + layer1_b)) noisy_encoder = tf.cond( self.istraining, lambda: tf.layers.dropout(noisy_encoder, rate=self.layer1_dropout_rate, name='layer1_dropout'), lambda: noisy_encoder) self.w2 = layer2_w self.w2_org = tf.cond(self.add_weight_noise, lambda: layer2_w - w2_noise, lambda: layer2_w) noisy_decoder = tf.identity( tf.matmul(noisy_encoder, layer2_w) + layer2_b) # Output # Determine whether negative samples should be considered mask = tf.cond( self.isnegsample, lambda: tf.cast( self.output_mask, dtype=noisy_decoder.dtype), lambda: tf.sign(self.ratings)) self.noisy_output = tf.cond( self.istraining, lambda: tf.multiply(noisy_decoder, mask), lambda: noisy_decoder) # self.pred_y = tf.sigmoid(self.output) # self.pred_y = self.output # Noisy Model Loss # base_loss = tf.nn.l2_loss(input - self.pred_y) noisy_base_loss = tf.reduce_sum( tf.nn.sigmoid_cross_entropy_with_logits( labels=input, logits=self.noisy_output)) noisy_base_loss = noisy_base_loss / tf.cast( tf.shape(input)[0], dtype=noisy_base_loss.dtype) # Average over the batches ################## Mix the outputs # self.mixed_output = (1-self.loss_ratio) * self.org_output + self.loss_ratio * self.output self.pred_y = tf.sigmoid(self.org_output) # self.pred_y = self.mixed_output # Mix the losses # base_loss = tf.cond(tf.logical_or(self.add_concat_noise, self.add_weight_noise), # lambda : org_base_loss + self.loss_ratio * base_loss, # lambda : base_loss) if not self.robust_test: base_loss = org_base_loss + self.loss_ratio * noisy_base_loss else: base_loss = org_base_loss reg_loss = self.ae_regs[0] * tf.nn.l2_loss(layer1_w) + self.ae_regs[1] * tf.nn.l2_loss(layer1_b) + \ self.ae_regs[2] * tf.nn.l2_loss(layer2_w) + self.ae_regs[3] * tf.nn.l2_loss(layer2_b) self.loss = base_loss + reg_loss # Optimizer self.opt = tf.train.AdagradOptimizer(self.lr).minimize(self.loss) # Add Noise (Random or Adversial) if self.robust_test: # Robust Testing in Evaluation if self.random_noise: random_noise = tf.random_normal( shape=tf.shape(layer2_w), mean=tf.reduce_mean(layer2_w), stddev=0.01) self.update_delta = w2_noise.assign( self.eps * random_noise / tf.norm(random_noise)) if self.adv_noise: self.grad_delta = tf.gradients(ys=org_base_loss, xs=w2_noise)[0] self.grad_delta_dense = tf.stop_gradient(self.grad_delta) self.update_delta = w2_noise.assign( self.eps * self.grad_delta_dense / tf.norm(self.grad_delta_dense)) else: # Gradients Computation of the Noise in Training self.grad_delta = tf.gradients(ys=base_loss, xs=hidden_noise_tr)[0] # convert the IndexedSlice Data to Dense Tensor self.grad_delta_dense = tf.stop_gradient(self.grad_delta) # self.grad_shape = tf.shape(self.grad_delta_dense) # normalization: new_grad = (grad / |grad|) * eps self.update_delta = hidden_noise_tr.assign( self.eps * self.grad_delta_dense / tf.norm(self.grad_delta_dense)) print('Model Building Completed.')
def parse(self, serialized): """See `_RankingDataParser`.""" list_size = self._list_size context_feature_spec = self._context_feature_spec example_feature_spec = self._example_feature_spec # Convert `FixedLenFeature` in `example_feature_spec` to # `FixedLenSequenceFeature` to parse the `feature_lists` in SequenceExample. # In addition, we collect non-trivial `default_value`s (neither "" nor 0) # for post-processing. This is because no `default_value` except None is # allowed for `FixedLenSequenceFeature`. Also, we set allow_missing=True and # handle the missing feature_list later. fixed_len_sequence_features = {} padding_values = {} non_trivial_padding_values = {} for k, s in six.iteritems(example_feature_spec): if not isinstance(s, tf.io.FixedLenFeature): continue fixed_len_sequence_features[k] = tf.io.FixedLenSequenceFeature( s.shape, s.dtype, allow_missing=True) scalar = _get_scalar_default_value(s.dtype, s.default_value) padding_values[k] = scalar if scalar: non_trivial_padding_values[k] = scalar sequence_features = example_feature_spec.copy() sequence_features.update(fixed_len_sequence_features) context, examples, sizes = tf.io.parse_sequence_example( serialized, context_features=context_feature_spec, sequence_features=sequence_features) # Reset to no trivial padding values for example features. for k, v in six.iteritems(non_trivial_padding_values): tensor = examples[k] # [batch_size, num_frames, feature_size] tensor.get_shape().assert_has_rank(3) size = tf.reshape(sizes[k], [-1, 1, 1]) # [batch_size, 1, 1] rank = tf.reshape( tf.tile(tf.range(tf.shape(input=tensor)[1]), [tf.shape(input=tensor)[0]]), tf.shape(input=tensor)) tensor = tf.compat.v1.where( tf.less(rank, tf.cast(size, tf.int32)), tensor, tf.fill(tf.shape(input=tensor), tf.cast(v, tensor.dtype))) examples[k] = tensor list_size_arg = list_size if list_size is None: # Use dynamic list_size. This is needed to pad missing feature_list. list_size_dynamic = tf.reduce_max(input_tensor=tf.stack( [tf.shape(input=t)[1] for t in six.itervalues(examples)])) list_size = list_size_dynamic # Collect features. Truncate or pad example features to normalize the tensor # shape: [batch_size, num_frames, ...] --> [batch_size, list_size, ...] features = {} features.update(context) for k, t in six.iteritems(examples): # Old shape: [batch_size, num_frames, ...] shape = tf.shape(input=t) ndims = t.get_shape().rank num_frames = shape[1] # New shape: [batch_size, list_size, ...] new_shape = tf.concat([[shape[0], list_size], shape[2:]], 0) def truncate_fn(t=t, ndims=ndims, new_shape=new_shape): """Truncates the tensor.""" if isinstance(t, tf.sparse.SparseTensor): return tf.sparse.slice(t, [0] * ndims, tf.cast(new_shape, dtype=tf.int64)) else: return tf.slice(t, [0] * ndims, new_shape) def pad_fn(k=k, t=t, ndims=ndims, num_frames=num_frames, new_shape=new_shape): """Pads the tensor.""" if isinstance(t, tf.sparse.SparseTensor): return tf.sparse.reset_shape(t, new_shape) else: # Paddings has shape [n, 2] where n is the rank of the tensor. paddings = tf.stack([[0, 0], [0, list_size - num_frames]] + [[0, 0]] * (ndims - 2)) return tf.pad(tensor=t, paddings=paddings, constant_values=padding_values[k]) tensor = tf.cond(pred=num_frames > list_size, true_fn=truncate_fn, false_fn=pad_fn) # Infer static shape for Tensor. Set the 2nd dim to None and set_shape # merges `static_shape` with the existing static shape of the thensor. if not isinstance(tensor, tf.sparse.SparseTensor): static_shape = t.get_shape().as_list() static_shape[1] = list_size_arg tensor.set_shape(static_shape) features[k] = tensor return features
def slim_get_batch(num_classes, batch_size, split_name, file_pattern, num_readers, num_preprocessing_threads, image_preprocessing_fn, anchor_encoder, num_epochs=None, is_training=True): """Gets a dataset tuple with instructions for reading Pascal VOC dataset. Args: num_classes: total class numbers in dataset. batch_size: the size of each batch. split_name: 'train' of 'val'. file_pattern: The file pattern to use when matching the dataset sources (full path). num_readers: the max number of reader used for reading tfrecords. num_preprocessing_threads: the max number of threads used to run preprocessing function. image_preprocessing_fn: the function used to dataset augumentation. anchor_encoder: the function used to encoder all anchors. num_epochs: total epoches for iterate this dataset. is_training: whether we are in traing phase. Returns: A batch of [image, shape, loc_targets, cls_targets, match_scores]. """ if split_name not in data_splits_num: raise ValueError('split name %s was not recognized.' % split_name) # Features in Pascal VOC TFRecords. keys_to_features = { 'image/encoded': tf.FixedLenFeature((), tf.string, default_value=''), 'image/format': tf.FixedLenFeature((), tf.string, default_value='jpeg'), 'image/filename': tf.FixedLenFeature((), tf.string, default_value=''), 'image/height': tf.FixedLenFeature([1], tf.int64), 'image/width': tf.FixedLenFeature([1], tf.int64), 'image/channels': tf.FixedLenFeature([1], tf.int64), 'image/shape': tf.FixedLenFeature([3], tf.int64), 'image/object/bbox/xmin': tf.VarLenFeature(dtype=tf.float32), 'image/object/bbox/ymin': tf.VarLenFeature(dtype=tf.float32), 'image/object/bbox/xmax': tf.VarLenFeature(dtype=tf.float32), 'image/object/bbox/ymax': tf.VarLenFeature(dtype=tf.float32), 'image/object/bbox/label': tf.VarLenFeature(dtype=tf.int64), 'image/object/bbox/difficult': tf.VarLenFeature(dtype=tf.int64), 'image/object/bbox/truncated': tf.VarLenFeature(dtype=tf.int64), } items_to_handlers = { 'image': slim.tfexample_decoder.Image('image/encoded', 'image/format'), 'filename': slim.tfexample_decoder.Tensor('image/filename'), 'shape': slim.tfexample_decoder.Tensor('image/shape'), 'object/bbox': slim.tfexample_decoder.BoundingBox( ['ymin', 'xmin', 'ymax', 'xmax'], 'image/object/bbox/'), 'object/label': slim.tfexample_decoder.Tensor('image/object/bbox/label'), 'object/difficult': slim.tfexample_decoder.Tensor('image/object/bbox/difficult'), 'object/truncated': slim.tfexample_decoder.Tensor('image/object/bbox/truncated'), } decoder = slim.tfexample_decoder.TFExampleDecoder(keys_to_features, items_to_handlers) labels_to_names = {} for name, pair in VOC_LABELS.items(): labels_to_names[pair[0]] = name dataset = slim.dataset.Dataset( data_sources=file_pattern, reader=tf.TFRecordReader, decoder=decoder, num_samples=data_splits_num[split_name], items_to_descriptions=None, num_classes=num_classes, labels_to_names=labels_to_names) with tf.name_scope('dataset_data_provider'): provider = slim.dataset_data_provider.DatasetDataProvider( dataset, num_readers=num_readers, common_queue_capacity=32 * batch_size, common_queue_min=8 * batch_size, shuffle=is_training, num_epochs=num_epochs) [org_image, filename, shape, glabels_raw, gbboxes_raw, isdifficult] = provider.get(['image', 'filename', 'shape', 'object/label', 'object/bbox', 'object/difficult']) if is_training: # if all is difficult, then keep the first one isdifficult_mask =tf.cond(tf.count_nonzero(isdifficult, dtype=tf.int32) < tf.shape(isdifficult)[0], lambda : isdifficult < tf.ones_like(isdifficult), lambda : tf.one_hot(0, tf.shape(isdifficult)[0], on_value=True, off_value=False, dtype=tf.bool)) glabels_raw = tf.boolean_mask(glabels_raw, isdifficult_mask) gbboxes_raw = tf.boolean_mask(gbboxes_raw, isdifficult_mask) # Pre-processing image, labels and bboxes. tensors_to_batch = [] if is_training: image, glabels, gbboxes = image_preprocessing_fn(org_image, glabels_raw, gbboxes_raw) gt_targets, gt_labels, gt_scores = anchor_encoder(glabels, gbboxes) tensors_to_batch = [image, filename, shape, gt_targets, gt_labels, gt_scores] else: image, output_shape = image_preprocessing_fn(org_image, glabels_raw, gbboxes_raw) tensors_to_batch = [image, filename, shape, output_shape] return tf.train.batch(tensors_to_batch, dynamic_pad=(not is_training), batch_size=batch_size, allow_smaller_final_batch=(not is_training), num_threads=num_preprocessing_threads, capacity=64 * batch_size)
def cond(*args, **kwargs): return tf.cond(*args, **kwargs)
def _inference(self): with tf.variable_scope(self._name): # Input module # Ax_ij shape is (batch_size, sentence_size ,embedding_size) query_word_emb = tf.nn.embedding_lookup(self._que_emb, self._queries) # Global question embedding. When the languages ques_emb = tf.reduce_sum( query_word_emb, 1) # shape is (batch_size, embedding_size) loss = tf.reshape(self._zeros, [-1, 1], name='loss') # (batch_size, 1) # The first state (entity) is given. init_state_index = tf.reshape(self._paths[:, 0], [-1, 1]) # (batch_size, 1) # KG1 is the first KG to be used in the path, KG2 the other. # Only in answer module do we cross language for state embedding. state_emb = tf.nn.embedding_lookup( self._kg1_ent_emb, init_state_index) # (b,1)->(b,1,e) state_emb = tf.squeeze(state_emb, [1]) # (batch_size, embedding_size) dir_align = tf.constant(self._is_direct_align, dtype=tf.bool) # e_M21 = tf.matrix_inverse(self._e_M12) # Reasoning module path = pre_t_idx = init_state_index for hop in range(self._hops): step = self._steps[hop] py_is_1 = self._lan_labels[hop] == self._lan_labels[0] is_1 = tf.constant(py_is_1, dtype=tf.bool) py_is_cross = hop != 0 and (self._lan_labels[hop] != self._lan_labels[hop - 1]) is_cross = tf.constant(py_is_cross, dtype=tf.bool) # Cross-lingual processing # head_emb = tf.cond(is_1, # lambda: tf.matmul(tf.nn.embedding_lookup(self._kg2_ent_emb, # tf.cast(pre_t_idx, tf.int64)), # tf.cond(tf.constant(self._is_dual_matrices), # lambda: self._e_M21, # lambda: e_M21)), # lambda: tf.matmul(tf.nn.embedding_lookup(self._kg1_ent_emb, # tf.cast(pre_t_idx, tf.int64)), self._e_M12)) # head_logits = tf.cond(is_1, # lambda: tf.matmul(head_emb, self._kg1_ent_emb, transpose_b=True), # lambda: tf.matmul(head_emb, self._kg2_ent_emb, transpose_b=True)) if system() == "Windows": # Windows compatible if self._is_direct_align: head_index = tf.cond( is_1, lambda: self._cor_a_table_21.lookup( tf.cast(pre_t_idx, tf.int32)), lambda: self._cor_a_table_12.lookup( tf.cast(pre_t_idx, tf.int32))) else: head_index = tf.cond( is_1, lambda: self._pred_a_table_21.lookup( tf.cast(pre_t_idx, tf.int32)), lambda: self._pred_a_table_12.lookup( tf.cast(pre_t_idx, tf.int32))) else: # Linux compatible if self._is_direct_align: head_index = tf.cond( is_1, lambda: self._cor_a_table_21.lookup( tf.cast(pre_t_idx, tf.int64)), lambda: self._cor_a_table_12.lookup( tf.cast(pre_t_idx, tf.int64))) else: head_index = tf.cond( is_1, lambda: self._pred_a_table_21.lookup( tf.cast(pre_t_idx, tf.int64)), lambda: self._pred_a_table_12.lookup( tf.cast(pre_t_idx, tf.int64))) path = tf.cond( is_cross, lambda: tf.concat( axis=1, values=[ path, tf.reshape(tf.cast(head_index, tf.int32), [-1, 1]) ]), lambda: path) state_emb = tf.cond( is_cross, lambda: tf.cond( is_1, lambda: tf.nn.embedding_lookup( self._kg1_ent_emb, head_index), lambda: tf.nn. embedding_lookup(self._kg2_ent_emb, head_index)), lambda: state_emb) gate = tf.matmul(ques_emb, tf.cond(is_1, lambda: tf.matmul(self._kg1_rel_emb, self._kg1_Mrq), lambda: tf.matmul(self._kg2_rel_emb, self._kg2_Mrq)), transpose_b=True) \ + tf.matmul(state_emb, tf.cond(is_1, lambda: tf.matmul(self._kg1_rel_emb, self._kg1_Mrs), lambda: tf.matmul(self._kg2_rel_emb, self._kg2_Mrs)), transpose_b=True) relation_logits = gate relation_index = tf.argmax(relation_logits, 1) gate = tf.nn.softmax(gate) real_rel_one_hot = tf.cond( is_cross, lambda: tf.one_hot( self._paths[:, step + 2], tf.cond(is_1, lambda: tf.constant(self._rel_size_1), lambda: tf.constant(self._rel_size_2)), on_value=1.0, off_value=0.0, axis=-1), lambda: tf.one_hot( self._paths[:, step + 1], tf.cond(is_1, lambda: tf.constant(self._rel_size_1), lambda: tf.constant(self._rel_size_2)), on_value=1.0, off_value=0.0, axis=-1)) state_emb = state_emb + tf.matmul( gate, tf.cond( is_1, lambda: tf.matmul(self._kg1_rel_emb, self._kg1_Mrs), lambda: tf.matmul(self._kg2_rel_emb, self._kg2_Mrs))) loss += tf.reshape( tf.nn.softmax_cross_entropy_with_logits( logits=relation_logits, labels=real_rel_one_hot), [-1, 1]) # (b,1) ques_emb = ques_emb - tf.matmul( gate, tf.cond( is_1, lambda: tf.matmul(self._kg1_rel_emb, self._kg1_Mrq), lambda: tf.matmul(self._kg2_rel_emb, self._kg2_Mrq))) # Answer module tail_logits = tf.cond( is_1, lambda: tf.matmul(tf.matmul(state_emb, self._kg1_Mse), self._kg1_ent_emb, transpose_b=True), lambda: tf.matmul(tf.matmul(state_emb, self._kg2_Mse), self._kg2_ent_emb, transpose_b=True)) tail_index = tf.argmax(tail_logits, 1) # # (batch_size, embedding_size) # tail_index = tf.cast(tail_index, tf.float32) # relation_index = tf.cast(relation_index, tf.float32) # # if relation_index == 0, stop inference, tail_index = previous tail; if not, tail won't change # tail_index = relation_index / (relation_index + 1e-15) * tail_index \ # + (1 - relation_index / (relation_index + 1e-15)) * tf.cast(path[:, -1], tf.float32) path = tf.concat(axis=1, values=[ path, tf.reshape( tf.cast(relation_index, tf.int32), [-1, 1]) ]) path = tf.concat(axis=1, values=[ path, tf.reshape(tf.cast(tail_index, tf.int32), [-1, 1]) ]) # (b,self._rel_size_1) real_tail_one_hot = tf.cond( is_cross, lambda: tf.one_hot( self._paths[:, step + 3], tf.cond(is_1, lambda: tf.constant(self._ent_size_1), lambda: tf.constant(self._ent_size_2)), on_value=1.0, off_value=0.0, axis=-1), lambda: tf.one_hot( self._paths[:, step + 2], tf.cond(is_1, lambda: tf.constant(self._ent_size_1), lambda: tf.constant(self._ent_size_2)), on_value=1.0, off_value=0.0, axis=-1)) loss += tf.reshape( tf.nn.softmax_cross_entropy_with_logits( logits=tail_logits, labels=real_tail_one_hot), [-1, 1]) # (b,1) pre_t_idx = tail_index return loss, path
def setup_to_run(m, args, is_training, batch_norm_is_training, summary_mode): # Set up the model. tf.set_random_seed(args.solver.seed) task_params = args.navtask.task_params num_steps = task_params.num_steps num_goals = task_params.num_goals num_actions = task_params.num_actions num_actions_ = num_actions n_views = task_params.n_views batch_norm_is_training_op = \ tf.placeholder_with_default(batch_norm_is_training, shape=[], name='batch_norm_is_training_op') # Setup the inputs m.input_tensors = {} lstm_states = []; lstm_state_dims = []; state_names = []; updated_state_ops = []; init_state_ops = []; if args.arch.lstm_output: lstm_states += ['lstm_output'] lstm_state_dims += [args.arch.lstm_output_dim+task_params.num_actions] if args.arch.lstm_ego: lstm_states += ['lstm_ego'] lstm_state_dims += [args.arch.lstm_ego_dim + args.arch.lstm_ego_out] lstm_states += ['lstm_img'] lstm_state_dims += [args.arch.lstm_img_dim + args.arch.lstm_img_out] elif args.arch.lstm_img: # An LSTM only on the image lstm_states += ['lstm_img'] lstm_state_dims += [args.arch.lstm_img_dim + args.arch.lstm_img_out] else: # No LSTMs involved here. None m.input_tensors['common'], m.input_tensors['step'], m.input_tensors['train'] = \ _inputs(task_params, lstm_states, lstm_state_dims) with tf.name_scope('check_size'): is_single_step = tf.equal(tf.unstack(tf.shape(m.input_tensors['step']['imgs']), num=6)[1], 1) images_reshaped = tf.reshape(m.input_tensors['step']['imgs'], shape=[-1, task_params.img_height, task_params.img_width, task_params.img_channels], name='re_image') rel_goal_loc_reshaped = tf.reshape(m.input_tensors['step']['rel_goal_loc'], shape=[-1, task_params.rel_goal_loc_dim], name='re_rel_goal_loc') x, vars_ = get_repr_from_image( images_reshaped, task_params.modalities, task_params.data_augment, args.arch.encoder, args.solver.freeze_conv, args.solver.wt_decay, is_training) # Reshape into nice things so that these can be accumulated over time steps # for faster backprop. sh_before = x.get_shape().as_list() m.encoder_output = tf.reshape( x, shape=[task_params.batch_size, -1, n_views] + sh_before[1:]) x = tf.reshape(m.encoder_output, shape=[-1] + sh_before[1:]) # Add a layer to reduce dimensions for a fc layer. if args.arch.dim_reduce_neurons > 0: ks = 1; neurons = args.arch.dim_reduce_neurons; init_var = np.sqrt(2.0/(ks**2)/neurons) batch_norm_param = args.arch.batch_norm_param batch_norm_param['is_training'] = batch_norm_is_training_op m.conv_feat = slim.conv2d( x, neurons, kernel_size=ks, stride=1, normalizer_fn=slim.batch_norm, normalizer_params=batch_norm_param, padding='SAME', scope='dim_reduce', weights_regularizer=slim.l2_regularizer(args.solver.wt_decay), weights_initializer=tf.random_normal_initializer(stddev=init_var)) reshape_conv_feat = slim.flatten(m.conv_feat) sh = reshape_conv_feat.get_shape().as_list() m.reshape_conv_feat = tf.reshape(reshape_conv_feat, shape=[-1, sh[1]*n_views]) # Restore these from a checkpoint. if args.solver.pretrained_path is not None: m.init_fn = slim.assign_from_checkpoint_fn(args.solver.pretrained_path, vars_) else: m.init_fn = None # Hit the goal_location with a bunch of fully connected layers, to embed it # into some space. with tf.variable_scope('embed_goal'): batch_norm_param = args.arch.batch_norm_param batch_norm_param['is_training'] = batch_norm_is_training_op m.embed_goal, _ = tf_utils.fc_network( rel_goal_loc_reshaped, neurons=args.arch.goal_embed_neurons, wt_decay=args.solver.wt_decay, name='goal_embed', offset=0, batch_norm_param=batch_norm_param, dropout_ratio=args.arch.fc_dropout, is_training=is_training) if args.arch.embed_goal_for_state: with tf.variable_scope('embed_goal_for_state'): batch_norm_param = args.arch.batch_norm_param batch_norm_param['is_training'] = batch_norm_is_training_op m.embed_goal_for_state, _ = tf_utils.fc_network( m.input_tensors['common']['rel_goal_loc_at_start'][:,0,:], neurons=args.arch.goal_embed_neurons, wt_decay=args.solver.wt_decay, name='goal_embed', offset=0, batch_norm_param=batch_norm_param, dropout_ratio=args.arch.fc_dropout, is_training=is_training) # Hit the goal_location with a bunch of fully connected layers, to embed it # into some space. with tf.variable_scope('embed_img'): batch_norm_param = args.arch.batch_norm_param batch_norm_param['is_training'] = batch_norm_is_training_op m.embed_img, _ = tf_utils.fc_network( m.reshape_conv_feat, neurons=args.arch.img_embed_neurons, wt_decay=args.solver.wt_decay, name='img_embed', offset=0, batch_norm_param=batch_norm_param, dropout_ratio=args.arch.fc_dropout, is_training=is_training) # For lstm_ego, and lstm_image, embed the ego motion, accumulate it into an # LSTM, combine with image features and accumulate those in an LSTM. Finally # combine what you get from the image LSTM with the goal to output an action. if args.arch.lstm_ego: ego_reshaped = preprocess_egomotion(m.input_tensors['step']['incremental_locs'], m.input_tensors['step']['incremental_thetas']) with tf.variable_scope('embed_ego'): batch_norm_param = args.arch.batch_norm_param batch_norm_param['is_training'] = batch_norm_is_training_op m.embed_ego, _ = tf_utils.fc_network( ego_reshaped, neurons=args.arch.ego_embed_neurons, wt_decay=args.solver.wt_decay, name='ego_embed', offset=0, batch_norm_param=batch_norm_param, dropout_ratio=args.arch.fc_dropout, is_training=is_training) state_name, state_init_op, updated_state_op, out_op = lstm_setup( 'lstm_ego', m.embed_ego, task_params.batch_size, is_single_step, args.arch.lstm_ego_dim, args.arch.lstm_ego_out, num_steps*num_goals, m.input_tensors['step']['lstm_ego']) state_names += [state_name] init_state_ops += [state_init_op] updated_state_ops += [updated_state_op] # Combine the output with the vision features. m.img_ego_op = combine_setup('img_ego', args.arch.combine_type_ego, m.embed_img, out_op, args.arch.img_embed_neurons[-1], args.arch.lstm_ego_out) # LSTM on these vision features. state_name, state_init_op, updated_state_op, out_op = lstm_setup( 'lstm_img', m.img_ego_op, task_params.batch_size, is_single_step, args.arch.lstm_img_dim, args.arch.lstm_img_out, num_steps*num_goals, m.input_tensors['step']['lstm_img']) state_names += [state_name] init_state_ops += [state_init_op] updated_state_ops += [updated_state_op] m.img_for_goal = out_op num_img_for_goal_neurons = args.arch.lstm_img_out elif args.arch.lstm_img: # LSTM on just the image features. state_name, state_init_op, updated_state_op, out_op = lstm_setup( 'lstm_img', m.embed_img, task_params.batch_size, is_single_step, args.arch.lstm_img_dim, args.arch.lstm_img_out, num_steps*num_goals, m.input_tensors['step']['lstm_img']) state_names += [state_name] init_state_ops += [state_init_op] updated_state_ops += [updated_state_op] m.img_for_goal = out_op num_img_for_goal_neurons = args.arch.lstm_img_out else: m.img_for_goal = m.embed_img num_img_for_goal_neurons = args.arch.img_embed_neurons[-1] if args.arch.use_visit_count: m.embed_visit_count = visit_count_fc( m.input_tensors['step']['visit_count'], m.input_tensors['step']['last_visit'], args.arch.goal_embed_neurons, args.solver.wt_decay, args.arch.fc_dropout, is_training=is_training) m.embed_goal = m.embed_goal + m.embed_visit_count m.combined_f = combine_setup('img_goal', args.arch.combine_type, m.img_for_goal, m.embed_goal, num_img_for_goal_neurons, args.arch.goal_embed_neurons[-1]) # LSTM on the combined representation. if args.arch.lstm_output: name = 'lstm_output' # A few fully connected layers here. with tf.variable_scope('action_pred'): batch_norm_param = args.arch.batch_norm_param batch_norm_param['is_training'] = batch_norm_is_training_op x, _ = tf_utils.fc_network( m.combined_f, neurons=args.arch.pred_neurons, wt_decay=args.solver.wt_decay, name='pred', offset=0, batch_norm_param=batch_norm_param, dropout_ratio=args.arch.fc_dropout) if args.arch.lstm_output_init_state_from_goal: # Use the goal embedding to initialize the LSTM state. # UGLY CLUGGY HACK: if this is doing computation for a single time step # then this will not involve back prop, so we can use the state input from # the feed dict, otherwise we compute the state representation from the # goal and feed that in. Necessary for using goal location to generate the # state representation. m.embed_goal_for_state = tf.expand_dims(m.embed_goal_for_state, dim=1) state_op = tf.cond(is_single_step, lambda: m.input_tensors['step'][name], lambda: m.embed_goal_for_state) state_name, state_init_op, updated_state_op, out_op = lstm_setup( name, x, task_params.batch_size, is_single_step, args.arch.lstm_output_dim, num_actions_, num_steps*num_goals, state_op) init_state_ops += [m.embed_goal_for_state] else: state_op = m.input_tensors['step'][name] state_name, state_init_op, updated_state_op, out_op = lstm_setup( name, x, task_params.batch_size, is_single_step, args.arch.lstm_output_dim, num_actions_, num_steps*num_goals, state_op) init_state_ops += [state_init_op] state_names += [state_name] updated_state_ops += [updated_state_op] out_op = tf.reshape(out_op, shape=[-1, num_actions_]) if num_actions_ > num_actions: m.action_logits_op = out_op[:,:num_actions] m.baseline_op = out_op[:,num_actions:] else: m.action_logits_op = out_op m.baseline_op = None m.action_prob_op = tf.nn.softmax(m.action_logits_op) else: # A few fully connected layers here. with tf.variable_scope('action_pred'): batch_norm_param = args.arch.batch_norm_param batch_norm_param['is_training'] = batch_norm_is_training_op out_op, _ = tf_utils.fc_network( m.combined_f, neurons=args.arch.pred_neurons, wt_decay=args.solver.wt_decay, name='pred', offset=0, num_pred=num_actions_, batch_norm_param=batch_norm_param, dropout_ratio=args.arch.fc_dropout, is_training=is_training) if num_actions_ > num_actions: m.action_logits_op = out_op[:,:num_actions] m.baseline_op = out_op[:,num_actions:] else: m.action_logits_op = out_op m.baseline_op = None m.action_prob_op = tf.nn.softmax(m.action_logits_op) m.train_ops = {} m.train_ops['step'] = m.action_prob_op m.train_ops['common'] = [m.input_tensors['common']['orig_maps'], m.input_tensors['common']['goal_loc'], m.input_tensors['common']['rel_goal_loc_at_start']] m.train_ops['state_names'] = state_names m.train_ops['init_state'] = init_state_ops m.train_ops['updated_state'] = updated_state_ops m.train_ops['batch_norm_is_training_op'] = batch_norm_is_training_op # Flat list of ops which cache the step data. m.train_ops['step_data_cache'] = [tf.no_op()] if args.solver.freeze_conv: m.train_ops['step_data_cache'] = [m.encoder_output] else: m.train_ops['step_data_cache'] = [] ewma_decay = 0.99 if is_training else 0.0 weight = tf.ones_like(m.input_tensors['train']['action'], dtype=tf.float32, name='weight') m.reg_loss_op, m.data_loss_op, m.total_loss_op, m.acc_ops = \ compute_losses_multi_or( m.action_logits_op, m.input_tensors['train']['action'], weights=weight, num_actions=num_actions, data_loss_wt=args.solver.data_loss_wt, reg_loss_wt=args.solver.reg_loss_wt, ewma_decay=ewma_decay) if args.solver.freeze_conv: vars_to_optimize = list(set(tf.trainable_variables()) - set(vars_)) else: vars_to_optimize = None m.lr_op, m.global_step_op, m.train_op, m.should_stop_op, m.optimizer, \ m.sync_optimizer = tf_utils.setup_training( m.total_loss_op, args.solver.initial_learning_rate, args.solver.steps_per_decay, args.solver.learning_rate_decay, args.solver.momentum, args.solver.max_steps, args.solver.sync, args.solver.adjust_lr_sync, args.solver.num_workers, args.solver.task, vars_to_optimize=vars_to_optimize, clip_gradient_norm=args.solver.clip_gradient_norm, typ=args.solver.typ, momentum2=args.solver.momentum2, adam_eps=args.solver.adam_eps) if args.arch.sample_gt_prob_type == 'inverse_sigmoid_decay': m.sample_gt_prob_op = tf_utils.inverse_sigmoid_decay(args.arch.isd_k, m.global_step_op) elif args.arch.sample_gt_prob_type == 'zero': m.sample_gt_prob_op = tf.constant(-1.0, dtype=tf.float32) elif args.arch.sample_gt_prob_type.split('_')[0] == 'step': step = int(args.arch.sample_gt_prob_type.split('_')[1]) m.sample_gt_prob_op = tf_utils.step_gt_prob( step, m.input_tensors['step']['step_number'][0,0,0]) m.sample_action_type = args.arch.action_sample_type m.sample_action_combine_type = args.arch.action_sample_combine_type _add_summaries(m, summary_mode, args.summary.arop_full_summary_iters) m.init_op = tf.group(tf.global_variables_initializer(), tf.local_variables_initializer()) m.saver_op = tf.train.Saver(keep_checkpoint_every_n_hours=4, write_version=tf.train.SaverDef.V2) return m
def _randomly_negate_tensor(tensor): """With 50% prob turn the tensor negative.""" should_flip = tf.cast(tf.floor(tf.random_uniform([]) + 0.5), tf.bool) final_tensor = tf.cond(should_flip, lambda: tensor, lambda: -tensor) return final_tensor
def init_step(self): # q : action for RL # value_matrix : state for RL self.q = tf.placeholder(tf.int32, [self.args.batch_size, self.args.seq_len], name='step_q') self.a = tf.placeholder(tf.int32, [self.args.batch_size, self.args.seq_len], name='step_a') self.value_matrix = tf.placeholder( tf.float32, [self.args.memory_size, self.args.memory_value_state_dim], name='step_value_matrix') slice_a = tf.split(self.a, self.args.seq_len, 1) a = tf.squeeze(slice_a[0], 1) slice_q = tf.split(self.q, self.args.seq_len, 1) q = tf.squeeze(slice_q[0], 1) q_embed = self.embedding_q(q) correlation_weight = self.memory.attention(q_embed) stacked_value_matrix = tf.tile(tf.expand_dims(self.value_matrix, 0), tf.stack([self.args.batch_size, 1, 1])) # -1 for sampling # 0, 1 for given answer self.qa = tf.cond( tf.squeeze(a) < 0, lambda: self.sampling_a_given_q(q, stacked_value_matrix), lambda: q + tf.multiply(a, self.args.n_questions)) a = (self.qa - 1) // self.args.n_questions qa_embed = self.embedding_qa(self.qa) ######### Before Step ########## prev_read_content, prev_summary, prev_pred_logits, prev_pred_prob = self.inference( q_embed, correlation_weight, stacked_value_matrix, reuse_flag=True) ######### STEP ##################### knowledge_growth = self.calculate_knowledge_growth( stacked_value_matrix, correlation_weight, qa_embed, prev_read_content, prev_summary, prev_pred_prob) # TODO : refactor sampling_a_given_q to return a only for below function call self.stepped_value_matrix = tf.squeeze(self.memory.value.write_given_a( stacked_value_matrix, correlation_weight, knowledge_growth, a, True), axis=0) #self.stepped_value_matrix = tf.squeeze(self.memory.value.write(stacked_value_matrix, correlation_weight, qa_embed, knowledge_growth, True), axis=0) self.stepped_read_content, self.stepped_summary, self.stepped_pred_logits, self.stepped_pred_prob = self.inference( q_embed, correlation_weight, self.stepped_value_matrix, reuse_flag=True) ######### After Step ######### self.value_matrix_difference = tf.squeeze( tf.reduce_sum(self.stepped_value_matrix - stacked_value_matrix)) self.read_content_difference = tf.squeeze( tf.reduce_sum(self.stepped_read_content - prev_read_content)) self.summary_difference = tf.squeeze( tf.reduce_sum(self.stepped_summary - prev_summary)) self.pred_logit_difference = tf.squeeze( tf.reduce_sum(self.stepped_pred_logits - prev_pred_logits)) self.pred_prob_difference = tf.squeeze( tf.reduce_sum(self.stepped_pred_prob - prev_pred_prob))
def generate_temporal_curves(self, seed=None): # Set kernel parameters # Either choose a set of random parameters for the mini-batch l1 = tf.random_uniform([self._batch_size, self._y_size, self._x_size], self._l1_min, self._l1_max, seed=seed) sigma_f = tf.random_uniform([self._batch_size, self._y_size], self._sigma_min, self._sigma_max, seed=seed) l1_vel = tf.random_uniform([self._batch_size, self._y_size, self._x_size], -1*self._l1_vel, self._l1_vel, seed=seed) sigma_f_vel = tf.random_uniform([self._batch_size, self._y_size], -1*self._sigma_vel, self._sigma_vel, seed=seed) if self._testing: num_total_points = 400 else: num_total_points = 100 y_value_base = tf.random_normal([self._batch_size, self._y_size, num_total_points, 1],seed=seed) curve_list = [] if (self._case==2) or (self._case==3): # sparse time or long term tracking idx = tf.random_shuffle(tf.range(self._len_seq), seed=seed)[:(self._len_given)] for t in range(self._len_seq): if seed is not None: _seed = seed * t else: _seed = seed if self._case==1: # using len_given if t < self._len_given: num_context = tf.random_uniform(shape=[], minval=5, maxval=self._max_num_context, dtype=tf.int32, seed=_seed) else: num_context = tf.constant(0) #num_context = tf.constant(1) if self._case==2: # sparse time nc_cond = tf.where(tf.equal(idx,t)) nc_cond = tf.reshape(nc_cond, [-1]) num_context = tf.cond(tf.equal(tf.size(nc_cond),0), lambda:tf.constant(0), lambda:tf.random_uniform(shape=[], minval=5, maxval=self._max_num_context, dtype=tf.int32, seed=_seed)) if self._case==3: # long term tracking nc_cond = tf.where(tf.equal(idx,t)) nc_cond = tf.reshape(nc_cond, [-1]) num_context = tf.cond(tf.equal(tf.size(nc_cond),0), lambda:tf.constant(0), lambda:tf.constant(1)) if self._temporal: encoded_t = None else: encoded_t = 0.25 + 0.5*t/self._len_seq curve_list.append(self.generate_curves(l1, sigma_f, num_context, y_value_base, _seed, encoded_t)) vel_noise = l1_vel * self._noise_factor * tf. random_normal([ self._batch_size,self._y_size, self._x_size], seed=_seed) l1 += l1_vel + vel_noise vel_noise = sigma_f_vel * self._noise_factor * tf. random_normal( [self._batch_size, self._x_size], seed=_seed) sigma_f += sigma_f_vel + vel_noise if self._testing: for t in range(self._len_seq,self._len_seq+self._len_gen): if seed is not None: _seed = seed * t else: _seed = seed num_context = tf.constant(0) if self._temporal: encoded_t = None else: encoded_t = 0.25 + 0.5*t/self._len_seq curve_list.append(self.generate_curves(l1, sigma_f, num_context, y_value_base, _seed, encoded_t)) vel_noise = l1_vel * self._noise_factor * tf. random_normal([ self._batch_size,self._y_size, self._x_size], seed=_seed) l1 += l1_vel + vel_noise vel_noise = sigma_f_vel*self._noise_factor*tf.random_normal( [self._batch_size, self._x_size], seed=_seed) sigma_f += sigma_f_vel + vel_noise context_x_list, context_y_list = [], [] target_x_list, target_y_list = [], [] num_total_points_list = [] num_context_points_list = [] for t in range(len(curve_list)): (context_x, context_y), target_x = curve_list[t].query target_y = curve_list[t].target_y num_total_points_list.append(curve_list[t].num_total_points) num_context_points_list.append(curve_list[t].num_context_points) context_x_list.append(context_x) context_y_list.append(context_y) target_x_list.append(target_x) target_y_list.append(target_y) query = ((context_x_list, context_y_list), target_x_list) return NPRegressionDescription( query=query, target_y=target_y_list, num_total_points=num_total_points_list, num_context_points=num_context_points_list, hyperparams=[tf.constant(0)])