def compute_loss(self,emb_batch,curr_batch_size=None): outloss=[] prediction=[] for idx_batch in range(self.config.batch_size): tree_states=self.compute_states(emb_batch,idx_batch) logits = self.create_output(tree_states) labels1=tf.gather(self.labels,idx_batch) labels2=tf.reduce_sum(tf.to_int32(tf.not_equal(labels1,-1))) labels=tf.gather(labels1,tf.range(labels2)) loss = self.calc_loss(logits,labels) pred = tf.nn.softmax(logits) pred_root=tf.gather(pred,labels2-1) prediction.append(pred_root) outloss.append(loss) batch_loss=tf.pack(outloss) self.pred = tf.pack(prediction) return batch_loss
def center_loss(features, label, alpha, nrof_classes): """Center loss based on the paper "A Discriminative Feature Learning Approach for Deep Face Recognition" (http://ydwen.github.io/papers/WenECCV16.pdf) """ # 获取特征向量长度 nrof_features = features.get_shape()[1] # 生成可以共享的变量centers with tf.variable_scope('center', reuse=True): centers = tf.get_variable('centers') label = tf.reshape(label, [-1]) # 取出对应label下对应的center值,注意label里面的值可能会重复,因为一个标签下有可能会出现多个人 centers_batch = tf.gather(centers, label) # 求特征点到中心的距离并乘以一定的系数,alfa是center的更新速度,越大代表更新的越慢 diff = centers_batch - features # 获取一个batch中同一样本出现的次数,这里需要理解论文中的更新公式 unique_label, unique_idx, unique_count = tf.unique_with_counts(label) appear_times = tf.gather(unique_count, unique_idx) appear_times = tf.reshape(appear_times, [-1, 1]) diff = diff / tf.cast((1 + appear_times), tf.float32) diff = alpha * diff # 更新center,输出是将对应于label的centers减去对应的diff,如果同一个标签出现多次,那么就减去多次 centers = tf.scatter_sub(centers, label, diff) # 求center loss,这里是将l2_loss里面的值进行平方相加,再除以2,并没有进行开方 loss = tf.nn.l2_loss(features - centers_batch) return loss, centers
def fast_rcnn_minibatch(self, reference_boxes): with tf.variable_scope('fast_rcnn_minibatch'): reference_boxes_mattached_gtboxes, object_mask, label = \ self.fast_rcnn_find_positive_negative_samples(reference_boxes) positive_indices = tf.reshape(tf.where(tf.not_equal(object_mask, 0.)), [-1]) num_of_positives = tf.minimum(tf.shape(positive_indices)[0], tf.cast(self.fast_rcnn_minibatch_size*self.fast_rcnn_positives_ratio, tf.int32)) positive_indices = tf.random_shuffle(positive_indices) positive_indices = tf.slice(positive_indices, begin=[0], size=[num_of_positives]) negative_indices = tf.reshape(tf.where(tf.equal(object_mask, 0.)), [-1]) num_of_negatives = tf.minimum(tf.shape(negative_indices)[0], self.fast_rcnn_minibatch_size - num_of_positives) negative_indices = tf.random_shuffle(negative_indices) negative_indices = tf.slice(negative_indices, begin=[0], size=[num_of_negatives]) minibatch_indices = tf.concat([positive_indices, negative_indices], axis=0) minibatch_indices = tf.random_shuffle(minibatch_indices) minibatch_reference_boxes_mattached_gtboxes = tf.gather(reference_boxes_mattached_gtboxes, minibatch_indices) object_mask = tf.gather(object_mask, minibatch_indices) label = tf.gather(label, minibatch_indices) label_one_hot = tf.one_hot(label, self.num_classes + 1) return minibatch_indices, minibatch_reference_boxes_mattached_gtboxes, object_mask, label_one_hot
def _recurrence(node_h,node_c,idx_var): node_info=tf.gather(treestr,idx_var) child_h=tf.gather(node_h,node_info) child_c=tf.gather(node_c,node_info) flat_ = tf.reshape(child_h,[-1]) tmp=tf.matmul(tf.expand_dims(flat_,0),cW) u,o,i,fl,fr=tf.split(1,5,tmp) i=tf.nn.sigmoid(i+bi) o=tf.nn.sigmoid(o+bo) u=tf.nn.tanh(u+bu) fl=tf.nn.sigmoid(fl+bf) fr=tf.nn.sigmoid(fr+bf) f=tf.concat(0,[fl,fr]) c = i * u + tf.reduce_sum(f*child_c,[0]) h = o * tf.nn.tanh(c) node_h = tf.concat(0,[node_h,h]) node_c = tf.concat(0,[node_c,c]) idx_var=tf.add(idx_var,1) return node_h,node_c,idx_var
def modular_layer(inputs, modules: ModulePool, parallel_count: int, context: ModularContext): with tf.variable_scope(None, 'modular_layer'): inputs = context.begin_modular(inputs) flat_inputs = tf.layers.flatten(inputs) logits = tf.layers.dense(flat_inputs, modules.module_count * parallel_count) logits = tf.reshape(logits, [-1, parallel_count, modules.module_count]) ctrl = tfd.Categorical(logits) initializer = tf.random_uniform_initializer(maxval=modules.module_count, dtype=tf.int32) shape = [context.dataset_size, parallel_count] best_selection_persistent = tf.get_variable('best_selection', shape, tf.int32, initializer) if context.mode == ModularMode.E_STEP: # 1 x batch_size x 1 best_selection = tf.gather(best_selection_persistent, context.data_indices)[tf.newaxis] # sample_size x batch_size x 1 sampled_selection = tf.reshape(ctrl.sample(), [context.sample_size, -1, parallel_count]) selection = tf.concat([best_selection, sampled_selection[1:]], axis=0) selection = tf.reshape(selection, [-1, parallel_count]) elif context.mode == ModularMode.M_STEP: selection = tf.gather(best_selection_persistent, context.data_indices) elif context.mode == ModularMode.EVALUATION: selection = ctrl.mode() else: raise ValueError('Invalid modular mode') attrs = ModularLayerAttributes(selection, best_selection_persistent, ctrl) context.layers.append(attrs) return run_modules(inputs, selection, modules.module_fnc, modules.output_shape)
def proposal_top_layer(rpn_cls_prob, rpn_bbox_pred, im_info, _feat_stride, anchors, num_anchors): """A layer that just selects the top region proposals without using non-maximal suppression, For details please see the technical report """ rpn_top_n = cfg.TEST.RPN_TOP_N scores = rpn_cls_prob[:, :, :, num_anchors:] rpn_bbox_pred = tf.reshape(rpn_bbox_pred, shape=(-1, 4)) scores = tf.reshape(scores, shape=(-1,)) # Do the selection here top_scores, top_inds = tf.nn.top_k(scores, k=rpn_top_n) top_scores = tf.reshape(top_scores, shape=(-1, 1)) top_anchors = tf.gather(anchors, top_inds) top_rpn_bbox = tf.gather(rpn_bbox_pred, top_inds) proposals = bbox_transform_inv_tf(top_anchors, top_rpn_bbox) # Clip predicted boxes to image proposals = clip_boxes_tf(proposals, im_info[:2]) # Output rois blob # Our RPN implementation only supports a single input image, so all # batch inds are 0 proposals = tf.to_float(proposals) batch_inds = tf.zeros((rpn_top_n, 1)) blob = tf.concat([batch_inds, proposals], 1) return blob, top_scores
def f(X): """ prob: n probabilities box: nx4 boxes Returns: n boolean, the selection """ prob, box = X output_shape = tf.shape(prob) # filter by score threshold ids = tf.reshape(tf.where(prob > cfg.TEST.RESULT_SCORE_THRESH), [-1]) prob = tf.gather(prob, ids) box = tf.gather(box, ids) # NMS within each class selection = tf.image.non_max_suppression( box, prob, cfg.TEST.RESULTS_PER_IM, cfg.TEST.FRCNN_NMS_THRESH) selection = tf.to_int32(tf.gather(ids, selection)) # sort available in TF>1.4.0 # sorted_selection = tf.contrib.framework.sort(selection, direction='ASCENDING') sorted_selection = -tf.nn.top_k(-selection, k=tf.size(selection))[0] mask = tf.sparse_to_dense( sparse_indices=sorted_selection, output_shape=output_shape, sparse_values=True, default_value=False) return mask
def _define_experience(self, agent_indices, observ, action, reward): """Implement the branch of experience() entered during training.""" update_filters = tf.summary.merge( [self._observ_filter.update(observ), self._reward_filter.update(reward)]) with tf.control_dependencies([update_filters]): if self._config.train_on_agent_action: # NOTE: Doesn't seem to change much. action = self._last_action batch = (observ, action, tf.gather(self._last_mean, agent_indices), tf.gather(self._last_logstd, agent_indices), reward) append = self._episodes.append(batch, agent_indices) with tf.control_dependencies([append]): norm_observ = self._observ_filter.transform(observ) norm_reward = tf.reduce_mean(self._reward_filter.transform(reward)) # pylint: disable=g-long-lambda summary = tf.cond( self._should_log, lambda: tf.summary.merge([ update_filters, self._observ_filter.summary(), self._reward_filter.summary(), tf.summary.scalar('memory_size', self._memory_index), tf.summary.histogram('normalized_observ', norm_observ), tf.summary.histogram('action', self._last_action), tf.summary.scalar('normalized_reward', norm_reward) ]), str) return summary
def build_predict(self,Xnew,task_ind): """ We need to assume the task_ind starts from 0 """ Fmean,Fvar = 0,0 for i in np.arange(self.rank): for j in np.arange(self.num_latent_list[i]): lat_id = np.sum(self.num_latent_list[:i],dtype = np.int64) + j if self.whiten_list[lat_id]: # need to compute fmean and fvar by the weights fmean, fvar = conditionals.gaussian_gp_predict_whitened(Xnew, self.Z[lat_id], self.kern_list[i], self.q_mu_list[lat_id], self.q_sqrt_list[lat_id], 1) else: fmean, fvar = conditionals.gaussian_gp_predict(Xnew, self.Z[lat_id], self.kern_list[i], self.q_mu_list[lat_id], self.q_sqrt_list[lat_id],1) W_ij = tf.gather(self.W,task_ind)[lat_id] Fmean += (fmean + self.mean_function_list[lat_id](Xnew))*W_ij Fvar += fvar * tf.square(W_ij) if self.tsk: for i in np.arange(self.num_tasks): lat_id = np.sum(self.num_latent_list,dtype = np.int64) + i if self.whiten_list[lat_id]: # need to compute fmean and fvar by the weights fmean, fvar = conditionals.gaussian_gp_predict_whitened(Xnew, self.Z[lat_id], self.tskern_list[i], self.q_mu_list[lat_id], self.q_sqrt_list[lat_id], 1) else: fmean, fvar = conditionals.gaussian_gp_predict(Xnew, self.Z[lat_id], self.tskern_list[i], self.q_mu_list[lat_id], self.q_sqrt_list[lat_id], 1) switch = tf.cast(tf.equal(tf.to_int64(i), task_ind),tf.float64) W_ij = tf.gather(self.Kappa,i)[0]*switch Fmean += (fmean + self.mean_function_list[lat_id](Xnew))*W_ij Fvar += fvar * tf.square(W_ij) return Fmean, Fvar
def rpn_proposals(self): with tf.variable_scope('rpn_proposals'): rpn_decode_boxes = encode_and_decode.decode_boxes(encode_boxes=self.rpn_encode_boxes, reference_boxes=self.anchors, scale_factors=self.scale_factors) rpn_softmax_scores = slim.softmax(self.rpn_scores) rpn_object_score = rpn_softmax_scores[:, 1] # second column represent object if self.top_k_nms: rpn_object_score, top_k_indices = tf.nn.top_k(rpn_object_score, k=self.top_k_nms) rpn_decode_boxes = tf.gather(rpn_decode_boxes, top_k_indices) # NMS valid_indices = tf_wrapper.nms_rotate_tf(boxes_list=rpn_decode_boxes, scores=rpn_object_score, iou_threshold=self.rpn_nms_iou_threshold, max_output_size=self.max_proposals_num, use_gpu=cfgs.NMS_USE_GPU) valid_boxes = tf.gather(rpn_decode_boxes, valid_indices) valid_scores = tf.gather(rpn_object_score, valid_indices) # print_tensors(valid_scores, 'rpn_score') rpn_proposals_boxes, rpn_proposals_scores = tf.cond( tf.less(tf.shape(valid_boxes)[0], self.max_proposals_num), lambda: boxes_utils.padd_boxes_with_zeros(valid_boxes, valid_scores, self.max_proposals_num), lambda: (valid_boxes, valid_scores)) return rpn_proposals_boxes, rpn_proposals_scores
def scheduled_sample_count(ground_truth_x, generated_x, batch_size, scheduled_sample_var): """Sample batch with specified mix of groundtruth and generated data points. Args: ground_truth_x: tensor of ground-truth data points. generated_x: tensor of generated data points. batch_size: batch size scheduled_sample_var: number of ground-truth examples to include in batch. Returns: New batch with num_ground_truth sampled from ground_truth_x and the rest from generated_x. """ num_ground_truth = scheduled_sample_var idx = tf.random_shuffle(tf.range(batch_size)) ground_truth_idx = tf.gather(idx, tf.range(num_ground_truth)) generated_idx = tf.gather(idx, tf.range(num_ground_truth, batch_size)) ground_truth_examps = tf.gather(ground_truth_x, ground_truth_idx) generated_examps = tf.gather(generated_x, generated_idx) output = tf.dynamic_stitch([ground_truth_idx, generated_idx], [ground_truth_examps, generated_examps]) # if batch size is known set it. if isinstance(batch_size, int): output.set_shape([batch_size] + common_layers.shape_list(output)[1:]) return output
def _tf_linear_interp1d(x_to_interpolate, fn_x, fn_y): """Tensorflow implementation of 1d linear interpolation. Args: x_to_interpolate: tf.float32 Tensor of shape (num_examples,) over which 1d linear interpolation is performed. fn_x: Monotonically-increasing, non-repeating tf.float32 Tensor of shape (length,) used as the domain to approximate a function. fn_y: tf.float32 Tensor of shape (length,) used as the range to approximate a function. Returns: tf.float32 Tensor of shape (num_examples,) """ x_pad = tf.concat([fn_x[:1] - 1, fn_x, fn_x[-1:] + 1], axis=0) y_pad = tf.concat([fn_y[:1], fn_y, fn_y[-1:]], axis=0) interval_idx = _find_interval_containing_new_value(x_pad, x_to_interpolate) # Interpolate alpha = ( (x_to_interpolate - tf.gather(x_pad, interval_idx)) / (tf.gather(x_pad, interval_idx + 1) - tf.gather(x_pad, interval_idx))) interpolation = ((1 - alpha) * tf.gather(y_pad, interval_idx) + alpha * tf.gather(y_pad, interval_idx + 1)) return interpolation
def make_minibatch(self, valid_anchors): with tf.variable_scope('rpn_minibatch'): # in labels(shape is [N, ]): 1 is positive, 0 is negative, -1 is ignored labels, anchor_matched_gtboxes, object_mask = \ self.rpn_find_positive_negative_samples(valid_anchors) # [num_of_valid_anchors, ] positive_indices = tf.reshape(tf.where(tf.equal(labels, 1.0)), [-1]) # use labels is same as object_mask num_of_positives = tf.minimum(tf.shape(positive_indices)[0], tf.cast(self.rpn_mini_batch_size * self.rpn_positives_ratio, tf.int32)) # num of positives <= minibatch_size * 0.5 positive_indices = tf.random_shuffle(positive_indices) positive_indices = tf.slice(positive_indices, begin=[0], size=[num_of_positives]) # positive_anchors = tf.gather(self.anchors, positive_indices) negative_indices = tf.reshape(tf.where(tf.equal(labels, 0.0)), [-1]) num_of_negatives = tf.minimum(self.rpn_mini_batch_size - num_of_positives, tf.shape(negative_indices)[0]) negative_indices = tf.random_shuffle(negative_indices) negative_indices = tf.slice(negative_indices, begin=[0], size=[num_of_negatives]) # negative_anchors = tf.gather(self.anchors, negative_indices) minibatch_indices = tf.concat([positive_indices, negative_indices], axis=0) minibatch_indices = tf.random_shuffle(minibatch_indices) minibatch_anchor_matched_gtboxes = tf.gather(anchor_matched_gtboxes, minibatch_indices) object_mask = tf.gather(object_mask, minibatch_indices) labels = tf.cast(tf.gather(labels, minibatch_indices), tf.int32) labels_one_hot = tf.one_hot(labels, depth=2) return minibatch_indices, minibatch_anchor_matched_gtboxes, object_mask, labels_one_hot
def _log_unnormalized_prob(self, x): mean = tf.squeeze(tf.gather(x, [0], axis=-1), axis=-1) precision = self._maybe_assert_valid_sample( tf.squeeze(tf.gather(x, [1], axis=-1), axis=-1)) return (tf.math.xlogy(self.concentration - 0.5, precision) - self.rate * precision - 0.5 * self._lambda * precision * tf.square(mean - self.loc))
def append(self, transitions, rows=None): """Append a batch of transitions to rows of the memory. Args: transitions: Tuple of transition quantities with batch dimension. rows: Episodes to append to, defaults to all. Returns: Operation. """ rows = tf.range(self._capacity) if rows is None else rows assert rows.shape.ndims == 1 assert_capacity = tf.assert_less( rows, self._capacity, message='capacity exceeded') with tf.control_dependencies([assert_capacity]): assert_max_length = tf.assert_less( tf.gather(self._length, rows), self._max_length, message='max length exceeded') append_ops = [] with tf.control_dependencies([assert_max_length]): for buffer_, elements in zip(self._buffers, transitions): timestep = tf.gather(self._length, rows) indices = tf.stack([rows, timestep], 1) append_ops.append(tf.scatter_nd_update(buffer_, indices, elements)) with tf.control_dependencies(append_ops): episode_mask = tf.reduce_sum(tf.one_hot( rows, self._capacity, dtype=tf.int32), 0) return self._length.assign_add(episode_mask)
def _map_to_tfidf(x): """Calculates the inverse document frequency of terms in the corpus. Args: x : a SparseTensor of int64 representing string indices in vocab. Returns: The tf*idf values """ # Add one to the reduced term freqnencies to avoid dividing by zero. idf = tf.log(tf.to_double(corpus_size) / ( 1.0 + tf.to_double(reduced_term_freq))) dense_doc_sizes = tf.to_double(tf.sparse_reduce_sum(tf.SparseTensor( indices=x.indices, values=tf.ones_like(x.values), dense_shape=x.dense_shape), 1)) # For every term in x, divide the idf by the doc size. # The two gathers both result in shape <sum_doc_sizes> idf_over_doc_size = (tf.gather(idf, x.values) / tf.gather(dense_doc_sizes, x.indices[:, 0])) return tf.SparseTensor( indices=x.indices, values=idf_over_doc_size, dense_shape=x.dense_shape)
def full_loss_op(self, logits, labels): """Adds loss ops to the computational graph. Hint: Use sparse_softmax_cross_entropy_with_logits Hint: Remember to add l2_loss (see tf.nn.l2_loss) Args: logits: tensor(num_nodes, output_size) labels: python list, len = num_nodes Returns: loss: tensor 0-D """ if self.full_loss is None: loss = None # YOUR CODE HERE l2_loss = self.config.l2 * tf.add_n(tf.get_collection("l2_loss")) idx = tf.where(tf.less(self.labelholder,2)) logits = tf.gather(logits, idx) labels = tf.gather(labels, idx) objective_loss = tf.reduce_sum(tf.nn.sparse_softmax_cross_entropy_with_logits(logits=logits, labels=labels)) loss = objective_loss + l2_loss tf.summary.scalar("loss_l2", l2_loss) tf.summary.scalar("loss_objective", tf.reduce_sum(objective_loss)) tf.summary.scalar("loss_total", loss) self.full_loss = loss # END YOUR CODE return self.full_loss
def _partition_and_stitch(self, args, func_name): """ args is a list of tensors, to be passed to self.likelihoods.<func_name> args[-1] is the 'Y' argument, which contains the indexes to self.likelihoods. This function splits up the args using dynamic_partition, calls the relevant function on the likelihoods, and re-combines the result. """ # get the index from Y Y = args[-1] ind = tf.gather(tf.transpose(Y), tf.shape(Y)[1] - 1) # ind = Y[:,-1] ind = tf.cast(ind, tf.int32) Y = tf.transpose(tf.gather(tf.transpose(Y), tf.range(0, tf.shape(Y)[1] - 1))) # Y = Y[:,:-1] args[-1] = Y # split up the arguments into chunks corresponding to the relevant likelihoods args = zip(*[tf.dynamic_partition(X, ind, self.num_likelihoods) for X in args]) # apply the likelihood-function to each section of the data funcs = [getattr(lik, func_name) for lik in self.likelihood_list] results = [f(*args_i) for f, args_i in zip(funcs, args)] # stitch the results back together partitions = tf.dynamic_partition(tf.range(0, tf.size(ind)), ind, self.num_likelihoods) results = tf.dynamic_stitch(partitions, results) return results
def reorder_beam(beam_size, batch_size, beam_val, output, is_first, tensors_to_reorder): """Reorder to minimize beam costs.""" # beam_val is [batch_size x beam_size]; let b = batch_size * beam_size # decided is len x b x a x b # output is b x out_size; step is b x len x a x b; outputs = tf.split(axis=0, num_or_size_splits=beam_size, value=tf.nn.log_softmax(output)) all_beam_vals, all_beam_idx = [], [] beam_range = 1 if is_first else beam_size for i in xrange(beam_range): top_out, top_out_idx = tf.nn.top_k(outputs[i], k=beam_size) cur_beam_val = beam_val[:, i] top_out = tf.Print(top_out, [top_out, top_out_idx, beam_val, i, cur_beam_val], "GREPO", summarize=8) all_beam_vals.append(top_out + tf.expand_dims(cur_beam_val, 1)) all_beam_idx.append(top_out_idx) all_beam_idx = tf.reshape(tf.transpose(tf.concat(axis=1, values=all_beam_idx), [1, 0]), [-1]) top_beam, top_beam_idx = tf.nn.top_k(tf.concat(axis=1, values=all_beam_vals), k=beam_size) top_beam_idx = tf.Print(top_beam_idx, [top_beam, top_beam_idx], "GREP", summarize=8) reordered = [[] for _ in xrange(len(tensors_to_reorder) + 1)] top_out_idx = [] for i in xrange(beam_size): which_idx = top_beam_idx[:, i] * batch_size + tf.range(batch_size) top_out_idx.append(tf.gather(all_beam_idx, which_idx)) which_beam = top_beam_idx[:, i] / beam_size # [batch] which_beam = which_beam * batch_size + tf.range(batch_size) reordered[0].append(tf.gather(output, which_beam)) for i, t in enumerate(tensors_to_reorder): reordered[i + 1].append(tf.gather(t, which_beam)) new_tensors = [tf.concat(axis=0, values=t) for t in reordered] top_out_idx = tf.concat(axis=0, values=top_out_idx) return (top_beam, new_tensors[0], top_out_idx, new_tensors[1:])
def sparse_dot_product0(emb, tuples, use_matmul=True, output_type='real'): """ Compute the dot product of complex vectors. It uses complex vectors but tensorflow does not optimize in the complex space (or there is a bug in the gradient propagation with complex numbers...) :param emb: embeddings :param tuples: indices at which we compute dot products :return: scores (dot products) """ n_t = tuples.get_shape()[0].value rk = emb.get_shape()[1].value emb_sel_a = tf.gather(emb, tuples[:, 0]) emb_sel_b = tf.gather(emb, tuples[:, 1]) if use_matmul: pred_cplx = tf.squeeze(tf.batch_matmul( tf.reshape(emb_sel_a, [n_t, rk, 1]), tf.reshape(emb_sel_b, [n_t, rk, 1]), adj_x=True)) else: pred_cplx = tf.reduce_sum(tf.mul(tf.conj(emb_sel_a), emb_sel_b), 1) if output_type == 'complex': return pred_cplx elif output_type == 'real': return tf.real(pred_cplx) + tf.imag(pred_cplx) elif output_type == 'real': return tf.abs(pred_cplx) elif output_type == 'angle': raise NotImplementedError('No argument or inverse-tanh function for complex number in Tensorflow') else: raise NotImplementedError()
def sparse_hermitian_product(emb, tuples): """ Compute the Hermitian inner product between selected complex embeddings This corresponds to the usual dot product applied on the conjugate of the first vector: <conj(x), y> where conj is the complex conjugate (obtained by inverting the imaginary part) We consider that the embedding dimension is twice the rank, where the first part is in emb[:,:rk] and the imaginary part is in emb[:,rk:]. It computes S[i] = <conj(E[I[i,1]], E[I[i,2]]> Usage: S = sparse_hermitian_product(E, I): :param emb: embedding matrix of size [n_emb, 2 * r] containing float numbers where r is the complex rank :param tuples: tuple matrix of size [n_t, 2] containing integers that correspond to the indices of the embeddings :return: a pair containing the real and imaginary parts of the Hermitian dot products """ rk = emb.get_shape()[1].value // 2 emb_re = emb[:, :rk] emb_im = emb[:, rk:] emb_sel_a_re = tf.gather(emb_re, tuples[:, 0]) emb_sel_a_im = tf.gather(emb_im, tuples[:, 0]) emb_sel_b_re = tf.gather(emb_re, tuples[:, 1]) emb_sel_b_im = tf.gather(emb_im, tuples[:, 1]) pred_re = tf.reduce_sum(tf.mul(emb_sel_a_re, emb_sel_b_re) + tf.mul(emb_sel_a_im, emb_sel_b_im), 1) pred_im = tf.reduce_sum(tf.mul(emb_sel_a_re, emb_sel_b_im) - tf.mul(emb_sel_a_im, emb_sel_b_re), 1) return pred_re, pred_im
def get_mention_emb(self, text_emb, text_outputs, mention_starts, mention_ends): mention_emb_list = [] mention_start_emb = tf.gather(text_outputs, mention_starts) # [num_mentions, emb] mention_emb_list.append(mention_start_emb) mention_end_emb = tf.gather(text_outputs, mention_ends) # [num_mentions, emb] mention_emb_list.append(mention_end_emb) mention_width = 1 + mention_ends - mention_starts # [num_mentions] if self.config["use_features"]: mention_width_index = mention_width - 1 # [num_mentions] mention_width_emb = tf.gather(tf.get_variable("mention_width_embeddings", [self.config["max_mention_width"], self.config["feature_size"]]), mention_width_index) # [num_mentions, emb] mention_width_emb = tf.nn.dropout(mention_width_emb, self.dropout) mention_emb_list.append(mention_width_emb) if self.config["model_heads"]: mention_indices = tf.expand_dims(tf.range(self.config["max_mention_width"]), 0) + tf.expand_dims(mention_starts, 1) # [num_mentions, max_mention_width] mention_indices = tf.minimum(util.shape(text_outputs, 0) - 1, mention_indices) # [num_mentions, max_mention_width] mention_text_emb = tf.gather(text_emb, mention_indices) # [num_mentions, max_mention_width, emb] self.head_scores = util.projection(text_outputs, 1) # [num_words, 1] mention_head_scores = tf.gather(self.head_scores, mention_indices) # [num_mentions, max_mention_width, 1] mention_mask = tf.expand_dims(tf.sequence_mask(mention_width, self.config["max_mention_width"], dtype=tf.float32), 2) # [num_mentions, max_mention_width, 1] mention_attention = tf.nn.softmax(mention_head_scores + tf.log(mention_mask), dim=1) # [num_mentions, max_mention_width, 1] mention_head_emb = tf.reduce_sum(mention_attention * mention_text_emb, 1) # [num_mentions, emb] mention_emb_list.append(mention_head_emb) mention_emb = tf.concat(mention_emb_list, 1) # [num_mentions, emb] return mention_emb
def c_body(c, pa): # Zeroing predictions below threshold with tf.variable_scope('bboxes_c_select', reuse=True): c_scores = b_scores[:, c] c_fmask = tf.cast(tf.greater(c_scores, confidence_threshold), scores.dtype) c_scores = c_scores * c_fmask c_bboxes = b_bboxes * tf.expand_dims(c_fmask, axis=-1) # Apply NMS with tf.variable_scope('bboxes_c_nms', reuse=True): c_indices = tf.image.non_max_suppression(c_bboxes, c_scores, top_k, nms_threshold) size = tf.size(c_indices) c_batch_ = tf.to_float(b) * tf.ones(shape=[top_k, 1], dtype=tf.float32) # len(indices) x 1 c_labels = tf.to_float(c) * tf.ones(shape=[top_k, 1], dtype=tf.float32) # len(indices) x 1 extra_size = top_k - size c_scores = tf.expand_dims(tf.gather(c_scores, c_indices), axis=-1) # len(indices) x 1 empty_c_scores = tf.zeros([extra_size, 1], dtype=tf.float32) c_scores = tf.concat([c_scores, empty_c_scores], axis=0) c_bboxes = tf.gather(c_bboxes, c_indices) # len(indices) x 4 empty_c_bboxes = tf.zeros([extra_size, 4], dtype=tf.float32) c_bboxes = tf.concat([c_bboxes, empty_c_bboxes], axis=0) c_predictions = tf.concat([c_batch_, c_labels, c_scores, c_bboxes], axis=1) # len(indices) x 7 return c + 1, pa.write(index=c - 1, value=c_predictions)
def build_loss(self, logits, labels, lambs): # put a sigfunction on logits and then transpose logits = tf.transpose(framwork.sig_func(logits)) # according to the labels, erase rows which is not in labels labels_unique = tf.constant(range(self.image_classes), dtype=tf.int32) labels_num = self.image_classes logits = tf.gather(logits, indices=labels_unique) lambs = tf.gather(lambs, indices=labels_unique) # set the value of each row to True when it occurs in labels template = tf.tile(tf.expand_dims(labels_unique, dim=1), [1, self.batch_size]) labels_expand = tf.tile(tf.expand_dims(labels, dim=0), [labels_num, 1]) indict_logic = tf.equal(labels_expand, template) # split the tensor along rows logit_list = tf.split(0, labels_num, logits) indict_logic_list = tf.split(0, labels_num, indict_logic) lambda_list = tf.split(0, self.image_classes, lambs) # loss_list = list() # for i in range(self.image_classes): # loss_list.append(framwork.loss_func(logit_list[i], indict_logic_list[i], lambda_list[i])) loss_list = map(framwork.loss_func, logit_list, indict_logic_list, lambda_list) loss = tf.add_n(loss_list) tensors_dict = {'labels_unique': labels_unique, 'template': template, 'logits_sig_trans': logits, 'loss': loss, 'indict_logic': indict_logic} self.tensors_names.extend(tensors_dict.keys()) self.net_tensors.update(tensors_dict)
def mf_binary_likelihood(U, V, nzr, nzc, nzz, noise_prec, alpha, n, m, k, fix_entries=FIX_TRIANGLE): with tf.name_scope("priors"): U_prior = tf.reduce_sum(bf.dists.gaussian_log_density(U, stddev=alpha), name="U_prior") V_prior = tf.reduce_sum(bf.dists.gaussian_log_density(V, stddev=alpha), name="V_prior") if fix_entries == FIX_IDENTITY: mask = np.float32(np.vstack((np.eye(k), np.ones((m-k, k))))) V = V * mask elif fix_entries == FIX_TRIANGLE: mask = np.float32(np.tril(np.ones((m, k)))) V = V * mask else: pass with tf.name_scope("model"): Us = tf.gather(U, nzr, name="Us") #tf.histogram_summary("Us", Us) Vs = tf.gather(V, nzc, name="Vs") #tf.histogram_summary("Vs", Vs) Rs = tf.reduce_sum(tf.mul(Us, Vs), reduction_indices=1, name="Rs") #tf.histogram_summary("rs", Rs) probs, _ = bf.transforms.logit(Rs * noise_prec) #tf.histogram_summary("probs", probs) ll = tf.reduce_sum(bf.dists.bernoulli_log_density(nzz, probs), name="ll") joint_logprob = U_prior + V_prior + ll return joint_logprob
def testIndexedSlices(self): for v1_first in [True, False]: with self.test_session(): v1 = tf.Variable(np.array([[0.0, 1.0], [10.0, 11.0], [20.0, 21.0]]).astype(np.float32)) v1_at_1 = tf.IndexedSlices( control_flow_ops.with_dependencies([v1.initializer], v1.ref()), tf.constant([1]) ) v2 = tf.Variable(np.array([[0.1, 1.1], [10.1, 11.1], [20.1, 21.1]]).astype(np.float32)) v2_at_1 = tf.IndexedSlices( control_flow_ops.with_dependencies([v2.initializer], v2.ref()), tf.constant([1]) ) st1, st2 = control_flow_ops.tuple([v1_at_1, v2_at_1]) g1 = tf.gather(st1.values, st1.indices) g2 = tf.gather(st2.values, st2.indices) # v1 is not initialized. with self.assertRaisesOpError("Attempting to use uninitialized value"): v1.eval() # v2 is not initialized. with self.assertRaisesOpError("Attempting to use uninitialized value"): v2.eval() if v1_first: # Getting g1 initializes v2. self.assertAllClose([[10.0, 11.0]], g1.eval()) self.assertAllClose([[0.1, 1.1], [10.1, 11.1], [20.1, 21.1]], v2.eval()) else: # Getting g2 initializes v1. self.assertAllClose([[10.1, 11.1]], g2.eval()) self.assertAllClose([[0.0, 1.0], [10.0, 11.0], [20.0, 21.0]], v1.eval())
def build_network(self): net_tensors = self.net_tensors with self.net_graph.as_default(), tf.device(self.net_device): logits = tf.placeholder(dtype=tf.float32, shape=(self.batch_size, self.image_classes)) labels = tf.placeholder(dtype=tf.int32, shape=(self.batch_size,)) lambs = tf.placeholder(dtype=tf.float32, shape=(self.image_classes,)) # put a sigfunction on logits and then transpose logits = tf.transpose(framwork.sig_func(logits)) # according to the labels, erase rows which is not in labels labels_unique = tf.constant(range(self.image_classes), dtype=tf.int32) labels_num = self.image_classes logits = tf.gather(logits, indices=labels_unique) lambs = tf.gather(lambs, indices=labels_unique) # set the value of each row to True when it occurs in labels templete = tf.tile(tf.expand_dims(labels_unique, dim=1), [1, self.batch_size]) labels_expand = tf.tile(tf.expand_dims(labels, dim=0), [labels_num, 1]) indict_logic = tf.equal(labels_expand, templete) # split the tensor along rows logit_list = tf.split(0, labels_num, logits) indict_logic_list = tf.split(0, labels_num, indict_logic) lamb_list = tf.split(0, self.image_classes, lambs) logit_list = [tf.squeeze(item) for item in logit_list] indict_logic_list = [tf.squeeze(item) for item in indict_logic_list] left_right_tuples = list() for i in range(self.image_classes): left_right_tuples.append(framwork.lamb_func(logit_list[i], indict_logic_list[i], lamb=lamb_list[i])) # func = framwork.lamb_func() # left_right_tuples = map(func, logit_list, indict_logic_list, lamb_list) net_tensors.update({'left_right_tuples': left_right_tuples, 'logits': logits, 'labels': labels, 'lambs': lambs})
def _ProcessSingleScale(scale_index, boxes, features, scales, scores, reuse=True): """Resize the image and run feature extraction and keypoint selection. This function will be passed into tf.while_loop() and be called repeatedly. The input boxes are collected from the previous iteration [0: scale_index -1]. We get the current scale by image_scales[scale_index], and run image resizing, feature extraction and keypoint selection. Then we will get a new set of selected_boxes for current scale. In the end, we concat the previous boxes with current selected_boxes as the output. Args: scale_index: A valid index in the image_scales. boxes: Box tensor with the shape of [N, 4]. features: Feature tensor with the shape of [N, depth]. scales: Scale tensor with the shape of [N]. scores: Attention score tensor with the shape of [N]. reuse: Whether or not the layer and its variables should be reused. Returns: scale_index: The next scale index for processing. boxes: Concatenated box tensor with the shape of [K, 4]. K >= N. features: Concatenated feature tensor with the shape of [K, depth]. scales: Concatenated scale tensor with the shape of [K]. scores: Concatenated attention score tensor with the shape of [K]. """ scale = tf.gather(image_scales, scale_index) new_image_size = tf.to_int32(tf.round(original_image_shape_float * scale)) resized_image = tf.image.resize_bilinear(image_tensor, new_image_size) attention, feature_map = model_fn( resized_image, normalized_image=True, reuse=reuse) rf_boxes = CalculateReceptiveBoxes( tf.shape(feature_map)[1], tf.shape(feature_map)[2], rf, stride, padding) # Re-project back to the original image space. rf_boxes = tf.divide(rf_boxes, scale) attention = tf.reshape(attention, [-1]) feature_map = tf.reshape(feature_map, [-1, feature_depth]) # Use attention score to select feature vectors. indices = tf.reshape(tf.where(attention >= abs_thres), [-1]) selected_boxes = tf.gather(rf_boxes, indices) selected_features = tf.gather(feature_map, indices) selected_scores = tf.gather(attention, indices) selected_scales = tf.ones_like(selected_scores, tf.float32) / scale # Concat with the previous result from different scales. boxes = tf.concat([boxes, selected_boxes], 0) features = tf.concat([features, selected_features], 0) scales = tf.concat([scales, selected_scales], 0) scores = tf.concat([scores, selected_scores], 0) return scale_index + 1, boxes, features, scales, scores
def proposal_layer(rpn_cls_prob, rpn_bbox_pred, im_info, cfg_key, _feat_stride, anchors, num_anchors): if type(cfg_key) == bytes: cfg_key = cfg_key.decode('utf-8') pre_nms_topN = cfg[cfg_key].RPN_PRE_NMS_TOP_N post_nms_topN = cfg[cfg_key].RPN_POST_NMS_TOP_N nms_thresh = cfg[cfg_key].RPN_NMS_THRESH # Get the scores and bounding boxes scores = rpn_cls_prob[:, :, :, num_anchors:] scores = tf.reshape(scores, shape=(-1,)) rpn_bbox_pred = tf.reshape(rpn_bbox_pred, shape=(-1, 4)) proposals = bbox_transform_inv_tf(anchors, rpn_bbox_pred) proposals = clip_boxes_tf(proposals, im_info[:2]) # Non-maximal suppression indices = tf.image.non_max_suppression(proposals, scores, max_output_size=post_nms_topN, iou_threshold=nms_thresh) boxes = tf.gather(proposals, indices) boxes = tf.to_float(boxes) scores = tf.gather(scores, indices) scores = tf.reshape(scores, shape=(-1, 1)) # Only support single image as input batch_inds = tf.zeros((tf.shape(indices)[0], 1), dtype=tf.float32) blob = tf.concat([batch_inds, boxes], 1) return blob, scores
def build_eval_graph(self): analogy_a = tf.placeholder(dtype=tf.int32) analogy_b = tf.placeholder(dtype=tf.int32) analogy_c = tf.placeholder(dtype=tf.int32) norm_w_embed = tf.nn.l2_normalize(self._w_embed_in, 1) a_embed = tf.gather(norm_w_embed, analogy_a) b_embed = tf.gather(norm_w_embed, analogy_b) c_embed = tf.gather(norm_w_embed, analogy_c) target = c_embed + (b_embed - a_embed) cosine_analogy_dist = tf.matmul(target, norm_w_embed, transpose_b=True) _, analogy_indices = tf.nn.top_k(cosine_analogy_dist, word_config.top_k_analogy) near_word = tf.placeholder(dtype=tf.int32) near_embed = tf.gather(norm_w_embed, near_word) cosine_near_dist = tf.matmul(near_embed, norm_w_embed, transpose_b=True) near_val, near_ind = tf.nn.top_k(cosine_near_dist, min(1000, self._vocab_size)) self._analogy_a = analogy_a self._analogy_b = analogy_b self._analogy_c = analogy_c self._analogy_indices = analogy_indices self._near_word = near_word self._near_val = near_val self._near_ind = near_ind
def mil_prediction(pred, n=1): i = tf.argsort(pred[:, 1], axis=0) i = i[len(pred) - n:len(pred)] return (tf.gather(pred, i), i)
def __init__(self, field_sizes=None, embed_size=10, layer_sizes=None, layer_acts=None, drop_out=None, embed_l2=None, layer_l2=None, init_path=None, opt_algo='gd', learning_rate=1e-2, random_seed=None, layer_norm=True): Model.__init__(self) init_vars = [] num_inputs = len(field_sizes) for i in range(num_inputs): init_vars.append(('embed_%d' % i, [field_sizes[i], embed_size], 'xavier', dtype)) num_pairs = int(num_inputs * (num_inputs - 1) / 2) node_in = num_inputs * embed_size + num_pairs init_vars.append(('kernel', [embed_size, num_pairs, embed_size], 'xavier', dtype)) for i in range(len(layer_sizes)): init_vars.append(('w%d' % i, [node_in, layer_sizes[i]], 'xavier', dtype)) init_vars.append(('b%d' % i, [layer_sizes[i]], 'zero', dtype)) node_in = layer_sizes[i] self.graph = tf.Graph() with self.graph.as_default(): if random_seed is not None: tf.set_random_seed(random_seed) self.X = [tf.sparse_placeholder(dtype) for i in range(num_inputs)] self.y = tf.placeholder(dtype) self.keep_prob_train = 1 - np.array(drop_out) self.keep_prob_test = np.ones_like(drop_out) self.layer_keeps = tf.placeholder(dtype) self.vars = utils.init_var_map(init_vars, init_path) w0 = [self.vars['embed_%d' % i] for i in range(num_inputs)] xw = tf.concat([ tf.sparse_tensor_dense_matmul(self.X[i], w0[i]) for i in range(num_inputs) ], 1) xw3d = tf.reshape(xw, [-1, num_inputs, embed_size]) row = [] col = [] for i in range(num_inputs - 1): for j in range(i + 1, num_inputs): row.append(i) col.append(j) # batch * pair * k p = tf.transpose( # pair * batch * k tf.gather( # field * batch * k tf.transpose(xw3d, [1, 0, 2]), row), [1, 0, 2]) # batch * pair * k q = tf.transpose(tf.gather(tf.transpose(xw3d, [1, 0, 2]), col), [1, 0, 2]) # batch * pair * k p = tf.reshape(p, [-1, num_pairs, embed_size]) # batch * pair * k q = tf.reshape(q, [-1, num_pairs, embed_size]) # k * pair * k k = self.vars[ 'kernel'] # 外积生成二维矩阵; kernel就是用来和二维矩阵进行"卷积"(对应位置相乘相加)的。 # batch * 1 * pair * k p = tf.expand_dims(p, 1) # 1表示在原来第一维度后面加一维 # batch * pair kp = tf.reduce_sum( # batch * pair * k tf.multiply( # batch * pair * k tf.transpose( # batch * k * pair tf.reduce_sum( # batch * k * pair * k tf.multiply(p, k), -1), [0, 2, 1]), q), -1) # # if layer_norm: # # x_mean, x_var = tf.nn.moments(xw, [1], keep_dims=True) # # xw = (xw - x_mean) / tf.sqrt(x_var) # # x_g = tf.Variable(tf.ones([num_inputs * embed_size]), name='x_g') # # x_b = tf.Variable(tf.zeros([num_inputs * embed_size]), name='x_b') # # x_g = tf.Print(x_g, [x_g[:10], x_b]) # # xw = xw * x_g + x_b # p_mean, p_var = tf.nn.moments(op, [1], keep_dims=True) # op = (op - p_mean) / tf.sqrt(p_var) # p_g = tf.Variable(tf.ones([embed_size**2]), name='p_g') # p_b = tf.Variable(tf.zeros([embed_size**2]), name='p_b') # # p_g = tf.Print(p_g, [p_g[:10], p_b]) # op = op * p_g + p_b l = tf.concat([xw, kp], 1) for i in range(len(layer_sizes)): wi = self.vars['w%d' % i] bi = self.vars['b%d' % i] l = tf.nn.dropout( utils.activate(tf.matmul(l, wi) + bi, layer_acts[i]), self.layer_keeps[i]) l = tf.squeeze(l) self.y_prob = tf.sigmoid(l) self.loss = tf.reduce_mean( tf.nn.sigmoid_cross_entropy_with_logits(logits=l, labels=self.y)) if layer_l2 is not None: self.loss += embed_l2 * tf.nn.l2_loss(xw) #tf.concat(w0, 0)) for i in range(len(layer_sizes)): wi = self.vars['w%d' % i] self.loss += layer_l2[i] * tf.nn.l2_loss(wi) self.optimizer = utils.get_optimizer(opt_algo, learning_rate, self.loss) config = tf.ConfigProto() config.gpu_options.allow_growth = True self.sess = tf.Session(config=config) tf.global_variables_initializer().run(session=self.sess)
def __init__(self, field_sizes=None, embed_size=10, layer_sizes=None, layer_acts=None, drop_out=None, embed_l2=None, layer_l2=None, init_path=None, opt_algo='gd', learning_rate=1e-2, random_seed=None): Model.__init__(self) init_vars = [] num_inputs = len(field_sizes) # 26 for i in range(num_inputs): # 一个field就对应一个embedding的参数 init_vars.append(('embed_%d' % i, [field_sizes[i], embed_size], 'xavier', dtype)) num_pairs = int(num_inputs * (num_inputs - 1) / 2) node_in = num_inputs * embed_size + num_pairs # 第一个隐藏层的输入维度,lz大小k * pairs, lp只是pairs,也就是lp一个pair生成一个值,lz一个pair生成一个embedding大小 # node_in = num_inputs * (embed_size + num_inputs) for i in range(len(layer_sizes)): init_vars.append(('w%d' % i, [node_in, layer_sizes[i]], 'xavier', dtype)) init_vars.append(('b%d' % i, [layer_sizes[i]], 'zero', dtype)) node_in = layer_sizes[i] self.graph = tf.Graph() with self.graph.as_default(): if random_seed is not None: tf.set_random_seed(random_seed) self.X = [tf.sparse_placeholder(dtype) for i in range(num_inputs) ] # num_input就是field的个数N,也就是说原始输入不用做one-hot self.y = tf.placeholder(dtype) self.keep_prob_train = 1 - np.array(drop_out) self.keep_prob_test = np.ones_like(drop_out) self.layer_keeps = tf.placeholder(dtype) self.vars = utils.init_var_map(init_vars, init_path) w0 = [self.vars['embed_%d' % i] for i in range(num_inputs)] xw = tf.concat([ tf.sparse_tensor_dense_matmul(self.X[i], w0[i]) for i in range(num_inputs) ], 1) # 相乘就是在做embedding,concat就是把结果拼接起来 xw3d = tf.reshape(xw, [-1, num_inputs, embed_size ]) # [num_samples, num_field, embed_sz] row = [] col = [] for i in range(num_inputs - 1): for j in range(i + 1, num_inputs): row.append(i) col.append(j) # batch * pair * k p = tf.transpose( # pair * batch * k tf.gather( # num * batch * k tf.transpose(xw3d, [1, 0, 2]), row), [1, 0, 2]) # batch * pair * k q = tf.transpose(tf.gather(tf.transpose(xw3d, [1, 0, 2]), col), [1, 0, 2]) p = tf.reshape(p, [-1, num_pairs, embed_size]) q = tf.reshape(q, [-1, num_pairs, embed_size]) ip = tf.reshape(tf.reduce_sum(p * q, [-1]), [-1, num_pairs]) # simple but redundant # batch * n * 1 * k, batch * 1 * n * k # ip = tf.reshape( # tf.reduce_sum( # tf.expand_dims(xw3d, 2) * # tf.expand_dims(xw3d, 1), # 3), # [-1, num_inputs**2]) l = tf.concat([xw, ip], 1) for i in range(len(layer_sizes)): wi = self.vars['w%d' % i] bi = self.vars['b%d' % i] l = tf.nn.dropout( utils.activate(tf.matmul(l, wi) + bi, layer_acts[i]), self.layer_keeps[i]) l = tf.squeeze(l) self.y_prob = tf.sigmoid(l) self.loss = tf.reduce_mean( tf.nn.sigmoid_cross_entropy_with_logits(logits=l, labels=self.y)) if layer_l2 is not None: self.loss += embed_l2 * tf.nn.l2_loss(xw) for i in range(len(layer_sizes)): wi = self.vars['w%d' % i] self.loss += layer_l2[i] * tf.nn.l2_loss(wi) self.optimizer = utils.get_optimizer(opt_algo, learning_rate, self.loss) config = tf.ConfigProto() config.gpu_options.allow_growth = True self.sess = tf.Session(config=config) tf.global_variables_initializer().run(session=self.sess)
def train_batch(self, source_charseq_ids, source_charseqs, target_charseq_ids, target_charseqs): #print('train batch') # TODO(lemmatizer_noattn): Modify target_charseqs by appending EOW; only the version with appended EOW is used from now on. target_charseqs = self._append_eow(target_charseqs) with tf.GradientTape() as tape: # TODO(lemmatizer_noattn): Embed source charseqs embedded = self._model.source_embeddings(source_charseqs) # TODO: Run self._model.source_rnn on the embedded sequences, returning outputs in `source_encoded`. source_encoded = self._model.source_rnn(embedded) # Copy the source_encoded to corresponding batch places, and then flatten it source_mask = tf.not_equal(source_charseq_ids, 0) source_encoded = tf.boolean_mask(tf.gather(source_encoded, source_charseq_ids), source_mask) targets = tf.boolean_mask(tf.gather(target_charseqs, target_charseq_ids), source_mask) class DecoderTraining(decoder.BaseDecoder): @property def batch_size(self): return tf.shape(self._source_encoded)[0] # TODO: Return the batch size of self._source_encoded, using tf.shape @property def output_size(self): return tf.shape(self._targets[1]) # TODO(lemmatizer_noattn): Return the number logits per each output @property def output_dtype(self): return tf.float32 # TODO(lemmatizer_noattn): Return the type of the logits def _with_attention(self, inputs, states): # TODO: Compute the attention. # - Take self._source_encoded and pass it through the self._model.attention_source_layer. att = self._model.attention_source_layer(self._source_encoded) # Because self._source_encoded does not change, you should in fact do it in `initialize`. # TODO: WHAT THE?? # - Pass `states` though self._model.attention_state_layer. states_att = self._model.attention_state_layer(states) # - Sum the two outputs. However, the first has shape [a, b, c] and the second [a, c]. Therefore, # somehow expand the second to [a, b, c] first. (Hint: use broadcasting rules.) states_att = tf.tile(tf.expand_dims(states_att, 1), [1, tf.shape(att)[1], 1]) #print('=======') #print(att.shape) #print(states_att.shape) #print('=======') sum_att = tf.add(att, states_att) #print(sum_att.shape) # - Pass the sum through `tf.tanh`, then self._model.attention_weight_layer. sum_att = self._model.attention_weight_layer(tf.tanh(sum_att)) # - Then, run softmax on a suitable axis (the one corresponding to characters), generating `weights`. weights = tf.math.softmax(sum_att, axis=1) # - Multiply `self._source_encoded` with `weights` and sum the result in the axis # corresponding to characters, generating `attention`. Therefore, `attention` is a a fixed-size # representation for every batch element, independently on how many characters had # the corresponding input forms. attention = tf.reduce_sum(self._source_encoded * weights, axis=1) # - Finally concatenate `inputs` and `attention` and return the result. #print('-------') #print(inputs.shape) #print(attention.shape) #print('-------') return tf.concat([inputs, attention], axis=1) def initialize(self, layer_inputs, initial_state=None): self._model, self._source_encoded, self._targets = layer_inputs # TODO(lemmatozer_noattn): Define `finished` as a vector of self.batch_size of `False` [see tf.fill]. finished = tf.fill([self.batch_size], False) # TODO(lemmatizer_noattn): Define `inputs` as a vector of self.batch_size of MorphoDataset.Factor.BOW [see tf.fill], # embedded using self._model.target_embedding inputs = self._model.target_embedding(tf.fill([self.batch_size], MorphoDataset.Factor.BOW)) # TODO: Define `states` as the last words from self._source_encoded #TODO: WHAT THE? #print('states') #print(self._source_encoded.shape) states = self._source_encoded[:, -1] #print(states.shape) # TODO: Pass `inputs` through `self._with_attention(inputs, states)`. #print('shapes') #print(inputs.shape) #print(states.shape) inputs = self._with_attention(inputs, states) #print(inputs.shape) return finished, inputs, states def step(self, time, inputs, states): # TODO(lemmatizer_noattn): Pass `inputs` and `[states]` through self._model.target_rnn_cell, generating # `outputs, [states]`. outputs, [states] = self._model.target_rnn_cell(inputs, [states]) # TODO(lemmatizer_noattn): Overwrite `outputs` by passing them through self._model.target_output_layer, outputs = self._model.target_output_layer(outputs) # TODO(lemmatizer_noattn): Define `next_inputs` by embedding `time`-th words from `self._targets`. next_inputs = self._model.target_embedding(self._targets[:,time]) # TODO(lemmatizer_noattn): Define `finished` as True if `time`-th word from `self._targets` is EOW, False otherwise. finished = tf.equal(self._targets[:, time], MorphoDataset.Factor.EOW) # Again, no == or !=. # TODO: Pass `next_inputs` through `self._with_attention(inputs, states)`. #print('stepping') #print(next_inputs.shape) #print(states.shape) next_inputs = self._with_attention(next_inputs, states) return outputs, states, next_inputs, finished #print('decode') output_layer, _, _ = DecoderTraining()([self._model, source_encoded, targets]) # TODO(lemmatizer_noattn): Compute loss. Use only nonzero `targets` as a mask. #print('loss') loss = self._loss(targets, output_layer, tf.not_equal(targets, 0)) #print('gradient') gradients = tape.gradient(loss, self._model.variables) #print('optimizer') self._optimizer.apply_gradients(zip(gradients, self._model.variables)) #print('metrics') tf.summary.experimental.set_step(self._optimizer.iterations) with self._writer.as_default(): for name, metric in self._metrics_training.items(): metric.reset_states() if name == "loss": metric(loss) else: metric(targets, output_layer, tf.not_equal(targets, 0)) tf.summary.scalar("train/{}".format(name), metric.result()) #print('end train batch') #print(output_layer.shape) return tf.math.argmax(output_layer, axis=2)
def predict_batch(self, source_charseq_ids, source_charseqs): # TODO(lemmatizer_noattn)(train_batch): Embed source charseqs embedded = self._model.source_embeddings(source_charseqs) # TODO(train_batch): Run self._model.source_rnn on the embedded sequences, returning outputs in `source_encoded`. source_encoded = self._model.source_rnn(embedded) # Copy the source_encoded to corresponding batch places, and then flatten it source_mask = tf.not_equal(source_charseq_ids, 0) source_encoded = tf.boolean_mask(tf.gather(source_encoded, source_charseq_ids), source_mask) class DecoderPrediction(decoder.BaseDecoder): @property def batch_size(self): return tf.shape(self._source_encoded)[0] # TODO(lemmatizer_noattn)(train_batch): Return the batch size of self._source_encoded, using tf.shape @property def output_size(self): return 1 # TODO(lemmatizer_noattn): Return 1 because we are returning directly the predictions @property def output_dtype(self): return tf.int32 # TODO(lemmatizer_noattn): Return tf.int32 because the predictions are integral def _with_attention(self, inputs, states): # TODO: A copy of _with_attention from train_batch; you can of course # move the definition to a place where it can be reused in both places. # TODO: Compute the attention. # - Take self._source_encoded and pass it through the self._model.attention_source_layer. att = self._model.attention_source_layer(self._source_encoded) # Because self._source_encoded does not change, you should in fact do it in `initialize`. # TODO: WHAT THE?? # - Pass `states` though self._model.attention_state_layer. states_att = self._model.attention_state_layer(states) # - Sum the two outputs. However, the first has shape [a, b, c] and the second [a, c]. Therefore, # somehow expand the second to [a, b, c] first. (Hint: use broadcasting rules.) states_att = tf.tile(tf.expand_dims(states_att, 1), [1, tf.shape(att)[1], 1]) sum_att = tf.add(att, states_att) # - Pass the sum through `tf.tanh`, then self._model.attention_weight_layer. sum_att = self._model.attention_weight_layer(tf.tanh(sum_att)) # - Then, run softmax on a suitable axis (the one corresponding to characters), generating `weights`. weights = tf.math.softmax(sum_att,axis=1) # - Multiply `self._source_encoded` with `weights` and sum the result in the axis # corresponding to characters, generating `attention`. Therefore, `attention` is a a fixed-size # representation for every batch element, independently on how many characters had # the corresponding input forms. attention = tf.reduce_sum(self._source_encoded * weights, axis=1) # - Finally concatenate `inputs` and `attention` and return the result. #print(inputs.shape) #print(attention.shape) return tf.concat([inputs, attention], axis=1) def initialize(self, layer_inputs, initial_state=None): self._model, self._source_encoded = layer_inputs # TODO(lemmatizer_noattn)(train_batch): Define `finished` as a vector of self.batch_size of `False` [see tf.fill]. finished = tf.fill([self.batch_size], False) # TODO(lemmatizer_noattn)(train_batch): Define `inputs` as a vector of self.batch_size of MorphoDataset.Factor.BOW [see tf.fill], # embedded using self._model.target_embedding inputs = self._model.target_embedding(tf.fill([self.batch_size], MorphoDataset.Factor.BOW)) # TODO(train_batch): Define `states` as the last words from self._source_encoded # TODO: WHAT THE?? states = self._source_encoded[:, -1] # TODO(train_batch): Pass `inputs` through `self._with_attention(inputs, states)`. inputs = self._with_attention(inputs, states) return finished, inputs, states def step(self, time, inputs, states): # TODO(lemmatizer_noattn)(train_batch): Pass `inputs` and `[states]` through self._model.target_rnn_cell, generating # `outputs, [states]`. outputs, [states] = self._model.target_rnn_cell(inputs, [states]) # TODO(lemmatizer_noattn)(train_batch): Overwrite `outputs` by passing them through self._model.target_output_layer, outputs = self._model.target_output_layer(outputs) # TODO(lemmatizer_noattn): Overwirte `outputs` by passing them through `tf.argmax` on suitable axis and with # `output_type=tf.int32` parameter. outputs = tf.argmax(outputs, axis=1, output_type=tf.int32) # TODO(lemmatizer_noattn): Define `next_inputs` by embedding the `outputs` next_inputs = self._model.target_embedding(outputs) # TODO(lemmatizer_noattn): Define `finished` as True if `outputs` are EOW, False otherwise. [No == or !=]. finished = tf.equal(outputs, MorphoDataset.Factor.EOW) # TODO(train_batch): Pass `next_inputs` through `self._with_attention(inputs, states)`. next_inputs = self._with_attention(next_inputs, states) return outputs, states, next_inputs, finished #print('predictions babyyyy') predictions, _, _ = DecoderPrediction(maximum_iterations=tf.shape(source_charseqs)[1] + 10)([self._model, source_encoded]) return predictions
def _interpolate(im, x, y, out_size): with tf.variable_scope('_interpolate'): # constants num_batch = tf.shape(im)[0] height = tf.shape(im)[1] width = tf.shape(im)[2] channels = tf.shape(im)[3] x = tf.cast(x, 'float32') y = tf.cast(y, 'float32') height_f = tf.cast(height, 'float32') width_f = tf.cast(width, 'float32') out_height = out_size[0] out_width = out_size[1] zero = tf.zeros([], dtype='int32') max_y = tf.cast(tf.shape(im)[1] - 1, 'int32') max_x = tf.cast(tf.shape(im)[2] - 1, 'int32') # scale indices from [-1, 1] to [0, width/height] x = (x + 1.0)*(width_f) / 2.0 y = (y + 1.0)*(height_f) / 2.0 # do sampling x0 = tf.cast(tf.floor(x), 'int32') x1 = x0 + 1 y0 = tf.cast(tf.floor(y), 'int32') y1 = y0 + 1 x0 = tf.clip_by_value(x0, zero, max_x) x1 = tf.clip_by_value(x1, zero, max_x) y0 = tf.clip_by_value(y0, zero, max_y) y1 = tf.clip_by_value(y1, zero, max_y) dim2 = width dim1 = width*height base = _repeat(tf.range(num_batch)*dim1, out_height*out_width) base_y0 = base + y0*dim2 base_y1 = base + y1*dim2 idx_a = base_y0 + x0 idx_b = base_y1 + x0 idx_c = base_y0 + x1 idx_d = base_y1 + x1 # use indices to lookup pixels in the flat image and restore # channels dim im_flat = tf.reshape(im, tf.pack([-1, channels])) im_flat = tf.cast(im_flat, 'float32') Ia = tf.gather(im_flat, idx_a) Ib = tf.gather(im_flat, idx_b) Ic = tf.gather(im_flat, idx_c) Id = tf.gather(im_flat, idx_d) # and finally calculate interpolated values x0_f = tf.cast(x0, 'float32') x1_f = tf.cast(x1, 'float32') y0_f = tf.cast(y0, 'float32') y1_f = tf.cast(y1, 'float32') wa = tf.expand_dims(((x1_f-x) * (y1_f-y)), 1) wb = tf.expand_dims(((x1_f-x) * (y-y0_f)), 1) wc = tf.expand_dims(((x-x0_f) * (y1_f-y)), 1) wd = tf.expand_dims(((x-x0_f) * (y-y0_f)), 1) output = tf.add_n([wa*Ia, wb*Ib, wc*Ic, wd*Id]) return output
def task_metalearn(inp, reuse=True): inputa, inputb, labela, labelb = inp if FLAGS.datasource in ['2D']: input_task_emb = tf.concat((inputa, labela), axis=-1) with tf.variable_scope('first_embedding_sync', reuse=tf.AUTO_REUSE): input_task_emb = tf.layers.dense(input_task_emb, units=FLAGS.sync_filters, name='first_embedding_sync_dense') if FLAGS.num_classes < FLAGS.update_batch_size: with tf.variable_scope('reg_clustering', reuse=tf.AUTO_REUSE): assign_mat = tf.nn.softmax(tf.layers.dense(input_task_emb, units=FLAGS.num_classes), dim=1) input_task_emb_cat = tf.matmul(tf.transpose(assign_mat, perm=[1, 0]), input_task_emb) elif FLAGS.datasource in ['plainmulti', 'artmulti']: input_task_emb = self.image_embed.model(tf.reshape(inputa, [-1, self.img_size, self.img_size, self.channels])) proto_emb = [] labela2idx = tf.argmax(labela, axis=1) for class_idx in range(FLAGS.num_classes): tmp_gs = tf.equal(labela2idx, class_idx) gs = tf.where(tmp_gs) new_vec = tf.reduce_mean(tf.gather(input_task_emb, gs), axis=0) proto_emb.append(new_vec) proto_emb = tf.squeeze(tf.stack(proto_emb)) label_cat = tf.eye(5) input_task_emb_cat = tf.concat((proto_emb, label_cat), axis=-1) if FLAGS.datasource in ['2D']: task_embed_vec, task_emb_loss = self.lstmae.model(input_task_emb) propagate_knowledge = self.metagraph.model(input_task_emb_cat) elif FLAGS.datasource in ['plainmulti', 'artmulti']: task_embed_vec, task_emb_loss = self.lstmae.model(input_task_emb_cat) propagate_knowledge = self.metagraph.model(proto_emb) task_embed_vec_graph, task_emb_loss_graph = self.lstmae_graph.model(propagate_knowledge) task_enhanced_emb_vec = tf.concat([task_embed_vec, task_embed_vec_graph], axis=1) with tf.variable_scope('task_specific_mapping', reuse=tf.AUTO_REUSE): eta = [] for key in weights.keys(): weight_size = np.prod(weights[key].get_shape().as_list()) eta.append(tf.reshape( tf.layers.dense(task_enhanced_emb_vec, weight_size, activation=tf.nn.sigmoid, name='eta_{}'.format(key)), tf.shape(weights[key]))) eta = dict(zip(weights.keys(), eta)) task_weights = dict(zip(weights.keys(), [weights[key] * eta[key] for key in weights.keys()])) task_outputbs, task_lossesb = [], [] if self.classification: task_accuraciesb = [] task_outputa = self.forward(inputa, task_weights, reuse=reuse) task_lossa = self.loss_func(task_outputa, labela) grads = tf.gradients(task_lossa, list(task_weights.values())) if FLAGS.stop_grad: grads = [tf.stop_gradient(grad) for grad in grads] gradients = dict(zip(task_weights.keys(), grads)) fast_weights = dict( zip(task_weights.keys(), [task_weights[key] - self.update_lr * gradients[key] for key in task_weights.keys()])) output = self.forward(inputb, fast_weights, reuse=True) task_outputbs.append(output) task_lossesb.append(self.loss_func(output, labelb)) for j in range(num_updates - 1): loss = self.loss_func(self.forward(inputa, fast_weights, reuse=True), labela) grads = tf.gradients(loss, list(fast_weights.values())) if FLAGS.stop_grad: grads = [tf.stop_gradient(grad) for grad in grads] gradients = dict(zip(fast_weights.keys(), grads)) fast_weights = dict(zip(fast_weights.keys(), [fast_weights[key] - self.update_lr * gradients[key] for key in fast_weights.keys()])) output = self.forward(inputb, fast_weights, reuse=True) task_outputbs.append(output) task_lossesb.append(self.loss_func(output, labelb)) task_output = [task_emb_loss, task_emb_loss_graph, task_outputa, task_outputbs, task_lossa, task_lossesb] if self.classification: task_accuracya = tf.contrib.metrics.accuracy(tf.argmax(tf.nn.softmax(task_outputa), 1), tf.argmax(labela, 1)) for j in range(num_updates): task_accuraciesb.append( tf.contrib.metrics.accuracy(tf.argmax(tf.nn.softmax(task_outputbs[j]), 1), tf.argmax(labelb, 1))) task_output.extend([task_accuracya, task_accuraciesb]) return task_output
def _compute_logits_impl(self, context_features, example_features, labels, mode, params, config): # Scatter/Gather per-example scores through groupwise comparison. Each # instance in a mini-batch will form a number of groups. Each group of # examples are scored by `_score_fn` and scores for individual examples are # accumulated into logits. with tf.compat.v1.name_scope('groupwise_dnn_v2'): batch_size, list_size, is_valid = _infer_sizes(example_features, labels) # For each example feature, assuming the shape is [batch_size, list_size, # feature_size], the groups are formed along the 2nd dim. Each group has a # 'group_size' number of indices in [0, list_size). Based on these # indices, we can gather the example feature into a sub-tensor for each # group. The total number of groups we have for a mini-batch is batch_size # * num_groups. Inside each group, we have a 'group_size' number of # examples. self._update_scatter_gather_indices(is_valid, mode) num_groups = tf.shape(input=self._indices_mask)[1] with tf.compat.v1.name_scope('group_features'): # For context features, We have shape [batch_size * num_groups, ...]. large_batch_context_features = {} for name, value in six.iteritems(context_features): # [batch_size, 1, ...]. value = tf.expand_dims(value, axis=1) # [batch_size, num_groups, ...]. value = tf.gather(value, tf.zeros([num_groups], tf.int32), axis=1) # [batch_size * num_groups, ...] large_batch_context_features[name] = utils.reshape_first_ndims( value, 2, [batch_size * num_groups]) # For example feature, we have shape [batch_size * num_groups, # group_size, ...]. large_batch_group_features = {} for name, value in six.iteritems(example_features): # [batch_size, num_groups, group_size, ...]. value = tf.gather_nd(value, self._feature_gather_indices) # [batch_size * num_groups, group_size, ...]. large_batch_group_features[name] = utils.reshape_first_ndims( value, 3, [batch_size * num_groups, self._group_size]) # Do the inference and get scores for the large batch of [batch_size * # num_groups, logits_size] and reshape them to [batch_size, num_groups, # logits_size]. with tf.compat.v1.variable_scope('group_score'): scores = self._score_fn(large_batch_context_features, large_batch_group_features, mode, params, config) scores = tf.reshape(scores, tf.shape(self._score_scatter_indices)[0:3]) with tf.compat.v1.name_scope('accumulate_scores'): # Reset invalid scores to 0 based on mask. scores = tf.where( tf.gather( tf.expand_dims(self._indices_mask, 2), tf.zeros([tf.shape(scores)[2]], tf.int32), axis=2), scores, tf.zeros_like(scores)) # Scatter scores from [batch_size, num_groups, logits_size] to # [batch_size, list_size]. logits = tf.scatter_nd(self._score_scatter_indices, scores, [batch_size, list_size]) # Use average. logits /= tf.cast(tf.shape(scores)[2], dtype=tf.float32) return logits
def adaptive_affinity_loss(labels, one_hot_lab, probs, size, num_classes, kld_margin, w_edge, w_not_edge): """Adaptive affinity field (AAF) loss. This function computes AAF loss. There are three components in the function: 1) extracts edges from the ground-truth labels. 2) extracts ignored pixels and their paired pixels (usually the eight corner pixels). 3) extracts eight corner pixels/predictions from the center in a (2*size+1)x(2*size+1) patch 4) computes KL-Divergence between center pixels and their paired pixels (the eight corner). 5) imposes adaptive weightings on the loss. Args: labels: A tensor of size [batch_size, height_in, width_in], indicating semantic segmentation ground-truth labels. one_hot_lab: A tensor of size [batch_size, height_in, width_in, num_classes] which is the ground-truth labels in the form of one-hot vector. probs: A tensor of size [batch_size, height_in, width_in, num_classes], indicating segmentation predictions. size: A number indicating the half size of a patch. num_classes: A number indicating the total number of valid classes. The kld_margin: A number indicating the margin for KL-Divergence at edge. w_edge: A number indicating the weighting for KL-Divergence at edge. w_not_edge: A number indicating the weighting for KL-Divergence at non-edge. Returns: Two 1-D tensors value indicating the loss at edge and non-edge. """ # Compute ignore map (e.g, label of 255 and their paired pixels). labels = tf.squeeze(labels, axis=-1) # NxHxW ignore = nnx.ignores_from_label(labels, num_classes, size) # NxHxWx8 not_ignore = tf.logical_not(ignore) not_ignore = tf.expand_dims(not_ignore, axis=3) # NxHxWx1x8 # Compute edge map. edge = nnx.edges_from_label(one_hot_lab, size, 255) # NxHxWxCx8 # Remove ignored pixels from the edge/non-edge. edge = tf.logical_and(edge, not_ignore) not_edge = tf.logical_and(tf.logical_not(edge), not_ignore) edge_indices = tf.where(tf.reshape(edge, [-1])) not_edge_indices = tf.where(tf.reshape(not_edge, [-1])) # Extract eight corner from the center in a patch as paired pixels. probs_paired = nnx.eightcorner_activation(probs, size) # NxHxWxCx8 probs = tf.expand_dims(probs, axis=-1) # NxHxWxCx1 bot_epsilon = tf.constant(1e-4, name='bot_epsilon') top_epsilon = tf.constant(1.0, name='top_epsilon') neg_probs = tf.clip_by_value( 1-probs, bot_epsilon, top_epsilon) neg_probs_paired = tf.clip_by_value( 1-probs_paired, bot_epsilon, top_epsilon) probs = tf.clip_by_value( probs, bot_epsilon, top_epsilon) probs_paired = tf.clip_by_value( probs_paired, bot_epsilon, top_epsilon) # Compute KL-Divergence. kldiv = probs_paired*tf.log(probs_paired/probs) kldiv += neg_probs_paired*tf.log(neg_probs_paired/neg_probs) edge_loss = tf.maximum(0.0, kld_margin-kldiv) not_edge_loss = kldiv # Impose weights on edge/non-edge losses. one_hot_lab = tf.expand_dims(one_hot_lab, axis=-1) w_edge = tf.reduce_sum(w_edge*one_hot_lab, axis=3, keep_dims=True) # NxHxWx1x1 w_not_edge = tf.reduce_sum(w_not_edge*one_hot_lab, axis=3, keep_dims=True) # NxHxWx1x1 edge_loss *= w_edge not_edge_loss *= w_not_edge not_edge_loss = tf.reshape(not_edge_loss, [-1]) not_edge_loss = tf.gather(not_edge_loss, not_edge_indices) edge_loss = tf.reshape(edge_loss, [-1]) edge_loss = tf.gather(edge_loss, edge_indices) return edge_loss, not_edge_loss
def cost(self): if self.locate is not False: self.W0 = tf.constant(self.w0, dtype=tf.float32) self.W1 = tf.constant(self.w1, dtype=tf.float32) self.qtrainPH = tf.placeholder(tf.float32, shape=(None, self.n_steps, self.n_obs), name="qtrain") self.qtrain_unflat = tf.reshape(self.qtrainPH, [-1, self.n_obs]) #b*nxp with tf.name_scope('cost'): if ((self.rnn is False) and (self.cnn is not 'fft')): B = tf.argmax(self.qtrain_unflat, 1) qtrain_OH = tf.one_hot( B, self.n_out, on_value=1, off_value=0, axis=-1) #need one-hot for CE calculation self.cross = tf.add( tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits( logits=self.logits, labels=qtrain_OH), name="cross"), self.reg) self.accuracy = tf.reduce_mean(tf.cast(tf.equal(self.A, B), tf.float32), name="accuracy") if self.locate is not False: with tf.name_scope('rep_outputs'): qhat_rep = tf.matmul(self.qhat, self.W0) #b*nx3x3x3l SE = tf.matmul( tf.square(tf.subtract(qhat_rep, self.qtrain_unflat)), self.W1) #b*nx3lx3lxl SEbn = tf.reduce_min(SE, 1) #b*n SSE = tf.reduce_mean(SEbn) #1 self.rmse = tf.add(tf.sqrt(SSE, name="rmse"), self.reg) else: self.rmse = tf.add( tf.sqrt(tf.reduce_mean( tf.square( tf.subtract(self.qhat, self.qtrain_unflat))), name="rmse"), self.reg) if self.locate is not False: self.cost = self.rmse else: self.cost = self.cross else: #yes rnn or cnn is fft B = tf.argmax(self.qtrain_unflat, 1) #b*nx1 qtrain_OH = tf.one_hot( B, self.n_out, on_value=1, off_value=0, axis=-1) #need one-hot for CE calculation qtrain_tran = tf.transpose(self.qtrainPH, [1, 0, 2]) #nxbxp self.qtrain_last = tf.gather(qtrain_tran, int(qtrain_tran.get_shape()[0]) - 1) #bxp BB = tf.argmax(self.qtrain_last, 1) #bx1 self.qtrain_last_OH = tf.one_hot( BB, self.n_out, on_value=1, off_value=0, axis=-1) #need one-hot for CE calculation self.cross_last = tf.add( tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits( logits=self.logits_last, labels=self.qtrain_last_OH), name="cross_last"), self.reg) self.accuracy_last = tf.reduce_mean(tf.cast( tf.equal(self.AA, BB), tf.float32), name="accuracy_last") if self.locate is not False: with tf.name_scope('rep_outputs'): qhat_rep = tf.matmul(self.qhat_last, self.W0) #bx3x3x3l SE = tf.matmul( tf.square(tf.subtract(qhat_rep, self.qtrain_last)), self.W1) #bx3lx3lxl SEnb = tf.reduce_min(SE, axis=1) #b SSE = tf.reduce_mean(SEnb) #1 self.rmse_last = tf.add(tf.sqrt(SSE, name="rmse"), self.reg) else: self.rmse_last = tf.add( tf.sqrt(tf.reduce_mean( tf.square( tf.subtract(self.qtrain_last, self.qhat_last))), name="rmse_last"), self.reg) if self.locate is not False: self.cost = self.rmse_last else: self.cost = self.cross_last with tf.name_scope('summaries'): self.train_summary = tf.summary.scalar('mean/train_cost', self.cost) self.valid_summary = tf.summary.scalar('mean/valid_cost', self.cost) if self.locate is False: if self.rnn is True or self.cnn is 'fft': self.train_acc_summary = tf.summary.scalar( 'mean/train_accuracy', self.accuracy_last) self.valid_acc_summary = tf.summary.scalar( 'mean/valid_accuracy', self.accuracy_last) else: self.train_acc_summary = tf.summary.scalar( 'mean/train_accuracy', self.accuracy) self.valid_acc_summary = tf.summary.scalar( 'mean/valid_accuracy', self.accuracy)
def main(ckpt_weights, image_size, output_size, model_def, class_num, depth_multiplier, obj_thresh, iou_thresh, train_set, test_image): h = Helper(None, class_num, f'data/{train_set}_anchor.npy', np.reshape(np.array(image_size), (-1, 2)), np.reshape(np.array(output_size), (-1, 2))) network = eval(model_def) # type :yolo_mobilev2 yolo_model, yolo_model_warpper = network([image_size[0], image_size[1], 3], len(h.anchors[0]), class_num, alpha=depth_multiplier) yolo_model_warpper.load_weights(str(ckpt_weights)) print(INFO, f' Load CKPT {str(ckpt_weights)}') orig_img = h._read_img(str(test_image)) image_shape = orig_img.shape[0:2] img, _ = h._process_img(orig_img, true_box=None, is_training=False, is_resize=True) """ load images """ img = tf.expand_dims(img, 0) y_pred = yolo_model_warpper.predict(img) """ box list """ _yxyx_box = [] _yxyx_box_scores = [] """ preprocess label """ for l, pred_label in enumerate(y_pred): """ split the label """ pred_xy = pred_label[..., 0:2] pred_wh = pred_label[..., 2:4] pred_confidence = pred_label[..., 4:5] pred_cls = pred_label[..., 5:] # box_scores = obj_score * class_score box_scores = tf.sigmoid(pred_cls) * tf.sigmoid(pred_confidence) # obj_mask = pred_confidence_score[..., 0] > obj_thresh """ reshape box """ # NOTE tf_xywh_to_all will auto use sigmoid function pred_xy_A, pred_wh_A = tf_xywh_to_all(pred_xy, pred_wh, l, h) boxes = correct_box(pred_xy_A, pred_wh_A, image_size, image_shape) boxes = tf.reshape(boxes, (-1, 4)) box_scores = tf.reshape(box_scores, (-1, class_num)) """ append box and scores to global list """ _yxyx_box.append(boxes) _yxyx_box_scores.append(box_scores) yxyx_box = tf.concat(_yxyx_box, axis=0) yxyx_box_scores = tf.concat(_yxyx_box_scores, axis=0) mask = yxyx_box_scores >= obj_thresh """ do nms for every classes""" _boxes = [] _scores = [] _classes = [] for c in range(class_num): class_boxes = tf.boolean_mask(yxyx_box, mask[:, c]) class_box_scores = tf.boolean_mask(yxyx_box_scores[:, c], mask[:, c]) select = tf.image.non_max_suppression( class_boxes, scores=class_box_scores, max_output_size=30, iou_threshold=iou_thresh) class_boxes = tf.gather(class_boxes, select) class_box_scores = tf.gather(class_box_scores, select) _boxes.append(class_boxes) _scores.append(class_box_scores) _classes.append(tf.ones_like(class_box_scores) * c) boxes = tf.concat(_boxes, axis=0) classes = tf.concat(_classes, axis=0) scores = tf.concat(_scores, axis=0) """ draw box """ font = ImageFont.truetype(font='asset/FiraMono-Medium.otf', size=tf.cast(tf.floor(3e-2 * image_shape[0] + 0.5), tf.int32).numpy()) thickness = (image_shape[0] + image_shape[1]) // 300 """ show result """ if len(classes) > 0: pil_img = Image.fromarray(orig_img) print(f'[top\tleft\tbottom\tright\tscore\tclass]') for i, c in enumerate(classes): box = boxes[i] score = scores[i] label = '{:2d} {:.2f}'.format(int(c.numpy()), score.numpy()) draw = ImageDraw.Draw(pil_img) label_size = draw.textsize(label, font) top, left, bottom, right = box print(f'[{top:.1f}\t{left:.1f}\t{bottom:.1f}\t{right:.1f}\t{score:.2f}\t{int(c):2d}]') top = max(0, tf.cast(tf.floor(top + 0.5), tf.int32)) left = max(0, tf.cast(tf.floor(left + 0.5), tf.int32)) bottom = min(image_shape[0], tf.cast(tf.floor(bottom + 0.5), tf.int32)) right = min(image_shape[1], tf.cast(tf.floor(right + 0.5), tf.int32)) if top - image_shape[0] >= 0: text_origin = tf.convert_to_tensor([left, top - label_size[1]]) else: text_origin = tf.convert_to_tensor([left, top + 1]) for j in range(thickness): draw.rectangle( [left + j, top + j, right - j, bottom - j], outline=h.colormap[c]) draw.rectangle( [tuple(text_origin), tuple(text_origin + label_size)], fill=h.colormap[c]) draw.text(text_origin, label, fill=(0, 0, 0), font=font) del draw pil_img.show() else: print(NOTE, ' no boxes detected')
def main(): """Create the model and start the training.""" args = get_arguments() logs_dir = os.path.join(LOGS_ROOT, args.exper_name, LOGS_DIR_SUFFIX) snap_dir = os.path.join(LOGS_ROOT, args.exper_name, SNAPSHOT_DIR_SUFFIX) h, w = map(int, args.input_size.split(',')) input_size = (h, w) c_h, c_w = map(int, args.crop_size.split(',')) crop_size = (c_h, c_w) tf.set_random_seed(args.random_seed) # Coordinator for threads coord = tf.train.Coordinator() with tf.name_scope("create_inputs"): reader = DataLoader(args.data_dir, input_size, crop_size, True, False, False, coord) image_batch, label_batch = reader.dequeue(args.batch_size) with tf.name_scope("validating_input"): validate_reader = DataLoader(args.data_dir, input_size, crop_size, False, True, False, coord) image_validate_batch, label_validate_batch = validate_reader.dequeue(args.batch_size) # for training with tf.variable_scope('', reuse=False): net = UNetResidual_Ext2({'data': image_batch}, is_training=True, num_classes=args.num_classes) for layer in net.layers: print(layer) print(net.layers[layer].shape) total_params = 0 for variable in tf.trainable_variables(): shape = variable.get_shape() variable_parameters = 1 for dim in shape: variable_parameters *= dim.value total_params += variable_parameters print("Number of trainable parameters: %d" % (total_params)) # for validation with tf.variable_scope(tf.get_variable_scope(), reuse=True): net_val = UNetResidual_Ext2({'data': image_validate_batch}, is_training=False, num_classes=args.num_classes) with tf.variable_scope('training_output'): # output training logits = net.getOutput() with tf.variable_scope('validation_output'): # output validation logits_validation = net_val.getOutput() restore_var = [v for v in tf.global_variables()] with tf.variable_scope('training_loss'): train_weights = tf.gather(CLASS_WEIGHTS, label_batch) # loss for training ce_loss = tf.losses.sparse_softmax_cross_entropy(label_batch, logits, train_weights) ce_reduced_loss = tf.reduce_mean(ce_loss) dice_loss = generalised_dice_loss(tf.nn.softmax(logits), label_batch) train_loss = (ce_reduced_loss + dice_loss) with tf.variable_scope('validation_loss'): valid_weights = tf.gather(CLASS_WEIGHTS, label_validate_batch) # loss for validation ce_loss_validation = tf.losses.sparse_softmax_cross_entropy(label_validate_batch, logits_validation, valid_weights) ce_reduced_loss_validation = tf.reduce_mean(ce_loss_validation) dice_loss_validation = generalised_dice_loss(tf.nn.softmax(logits_validation),label_validate_batch) valid_loss = (ce_reduced_loss_validation + dice_loss_validation) with tf.variable_scope('accuracy'): # accuracy preds = tf.argmax(logits_validation, axis=-1) acc = tf.reduce_mean(tf.cast(tf.equal(preds, label_validate_batch), tf.double)) with tf.variable_scope('train'): update_ops = tf.get_collection(tf.GraphKeys.UPDATE_OPS) with tf.control_dependencies(update_ops): # training optimizer optimizer = tf.train.AdamOptimizer(learning_rate = args.learning_rate) train_op = optimizer.minimize(train_loss, global_step=tf.train.get_global_step()) init = tf.global_variables_initializer() sess = tf.Session() sess.run(init) train_logger = Logger(logs_dir + '/train') valid_logger = Logger(logs_dir + '/valid') saver = tf.train.Saver(var_list=tf.global_variables(), max_to_keep=1) # initialize weight ckpt = tf.train.get_checkpoint_state(snap_dir) if ckpt and ckpt.model_checkpoint_path: loader = tf.train.Saver(var_list=restore_var) load_step = int(os.path.basename(ckpt.model_checkpoint_path).split('-')[1]) load(loader, sess, ckpt.model_checkpoint_path) else: print('No checkpoint file found.') load_step = 0 threads = tf.train.start_queue_runners(coord=coord, sess=sess) min_val_loss = 1e9 best_acc = 0 for step in range(args.num_steps): realStep = step + load_step start_time = time.time() loss_value, _ = sess.run([train_loss, train_op]) train_logger.log_scalar('loss', loss_value, realStep) if step > 0 and realStep % args.save_pred_every == 0: #save(saver, sess, snap_dir, realStep) print('Validation') numValidateImg = len(os.listdir(os.path.join(args.data_dir, 'valid', 'img'))) numStep = int(numValidateImg / args.batch_size) loss_validation_test = 0 accuracy = 0 for step_val in range(numStep): predicted, gt, l, ac = sess.run([logits_validation, label_validate_batch, valid_loss, acc]) loss_validation_test = loss_validation_test + l / numStep # accuracy = accuracy + ac / numStep accuracy = accuracy + cal_AJI_batch(predicted, gt, args.batch_size) / numStep print('Validation result: loss = %.5f, acc = %.5f' % (loss_validation_test, accuracy)) if accuracy > best_acc: print('Update model: previous loss: %.4lf, new loss: %.4lf, step: %d' % (min_val_loss, loss_validation_test, realStep)) min_val_loss = loss_validation_test best_acc = accuracy save(saver, sess, snap_dir, realStep) valid_logger.log_scalar('loss', loss_validation_test, realStep) valid_logger.log_scalar('acc', accuracy, realStep) duration = time.time() - start_time print('step %d \t loss = %.5lf, (%.3lf sec/step)' % (realStep, loss_value, duration)) coord.request_stop() coord.join(threads)
def propagate_one_time( iter_idx, h_e, h_v, h_u, h_e_history, h_v_history, h_u_history, atom_in_mol=atom_in_mol, # (n_atoms, n_mols) bond_in_mol=bond_in_mol # (n_bonds, n_mols) ): # update $ e'_k $ # $$ # e'_k = \phi^e (e_k, v_{rk}, v_{sk}, u) # $$ h_left = tf.gather( h_v, left_idxs) h_right = tf.gather( h_v, right_idxs) # (n_bonds, d_e) h_e = self.phi_e(h_e, h_e_0, h_left + h_right, tf.reduce_sum( tf.boolean_mask( tf.tile( tf.expand_dims( h_u, # (n_mols, d_u) 0), # (1, n_mols, d_u) [tf.shape(h_e)[0], 1, 1]), bond_in_mol), axis=1, keepdims=True)) h_e_history = tf.concat( [ h_e_history, tf.expand_dims( h_e, 1) ], axis=1) # aggregate $ \bar{e_i'} $ # $$ # \bar{e_i'} = \rho^{e \rightarrow v} (E'_i) # $$ # (n_atoms, d_e) h_e_bar_i = self.rho_e_v(h_e, atom_is_connected_to_bonds) # update $ v'_i $ # $$ # v'_i = phi^v (\bar{e_i}, v_i, u) # $$ # (n_atoms, d_v) h_v = self.phi_v( h_v, # (n_atoms, d_v) h_v_0, # (n_atoms, d_v) h_e_bar_i, # (n_atoms, d_v) tf.reduce_sum( tf.where( tf.tile( tf.expand_dims( atom_in_mol, 2), [1, 1, tf.shape(h_u)[1]]), tf.tile( tf.expand_dims( h_u, 0), [n_atoms, 1, 1]), tf.zeros_like( tf.tile( tf.expand_dims( h_u, 0), [n_atoms, 1, 1]))), axis=1)) h_v_history = tf.concat( [ h_v_history, tf.expand_dims( h_v, 1) ], axis=1) # aggregate $ \bar{e'} $ # $$ # \bar{e'} = \rhp^{e \rightarrow u} (E') # $$ # (n_mols, d_e) h_e_bar = self.rho_e_u(h_e, bond_in_mol) # aggregate $ \bar{v'} $ # $$ # \bar{v'} = \rho^{v \rightarrow u} (V') # $$ # (n_mols, d_v) h_v_bar = self.rho_v_u(h_v, atom_in_mol) # update $ u' $ # $$ # u' = \phi^u (\bar{e'}, \bar{v'}, u) # $$ # (n_mols, d_u) h_u = self.phi_u( h_u, h_u_0, h_e_bar, h_v_bar) h_u_history = tf.concat( [ h_u_history, tf.expand_dims( h_u, 1) ], axis=1) return ( iter_idx + 1, h_e, h_v, h_u, h_e_history, h_v_history, h_u_history)
def affinity_loss(labels, probs, num_classes, kld_margin): """Affinity Field (AFF) loss. This function computes AFF loss. There are several components in the function: 1) extracts edges from the ground-truth labels. 2) extracts ignored pixels and their paired pixels (the neighboring pixels on the eight corners). 3) extracts neighboring pixels on the eight corners from a 3x3 patch. 4) computes KL-Divergence between center pixels and their neighboring pixels from the eight corners. Args: labels: A tensor of size [batch_size, height_in, width_in], indicating semantic segmentation ground-truth labels. probs: A tensor of size [batch_size, height_in, width_in, num_classes], indicating segmentation predictions. num_classes: A number indicating the total number of valid classes. kld_margin: A number indicating the margin for KL-Divergence at edge. Returns: Two 1-D tensors value indicating the loss at edge and non-edge. """ # Compute ignore map (e.g, label of 255 and their paired pixels). labels = tf.squeeze(labels, axis=-1) # NxHxW ignore = nnx.ignores_from_label(labels, num_classes, 1) # NxHxWx8 not_ignore = tf.logical_not(ignore) not_ignore = tf.expand_dims(not_ignore, axis=3) # NxHxWx1x8 # Compute edge map. one_hot_lab = tf.one_hot(labels, depth=num_classes) edge = nnx.edges_from_label(one_hot_lab, 1, 255) # NxHxWxCx8 # Remove ignored pixels from the edge/non-edge. edge = tf.logical_and(edge, not_ignore) not_edge = tf.logical_and(tf.logical_not(edge), not_ignore) edge_indices = tf.where(tf.reshape(edge, [-1])) not_edge_indices = tf.where(tf.reshape(not_edge, [-1])) # Extract eight corner from the center in a patch as paired pixels. probs_paired = nnx.eightcorner_activation(probs, 1) # NxHxWxCx8 probs = tf.expand_dims(probs, axis=-1) # NxHxWxCx1 bot_epsilon = tf.constant(1e-4, name='bot_epsilon') top_epsilon = tf.constant(1.0, name='top_epsilon') neg_probs = tf.clip_by_value( 1-probs, bot_epsilon, top_epsilon) probs = tf.clip_by_value( probs, bot_epsilon, top_epsilon) neg_probs_paired= tf.clip_by_value( 1-probs_paired, bot_epsilon, top_epsilon) probs_paired = tf.clip_by_value( probs_paired, bot_epsilon, top_epsilon) # Compute KL-Divergence. kldiv = probs_paired*tf.log(probs_paired/probs) kldiv += neg_probs_paired*tf.log(neg_probs_paired/neg_probs) not_edge_loss = kldiv edge_loss = tf.maximum(0.0, kld_margin-kldiv) not_edge_loss = tf.reshape(not_edge_loss, [-1]) not_edge_loss = tf.gather(not_edge_loss, not_edge_indices) edge_loss = tf.reshape(edge_loss, [-1]) edge_loss = tf.gather(edge_loss, edge_indices) return edge_loss, not_edge_loss
initial_state = lstmCell.zero_state(batchSize, tf.float32) value, _ = tf.nn.dynamic_rnn(lstmCell, data, initial_state=initial_state, dtype=tf.float32) weight = tf.Variable(tf.truncated_normal([lstmUnits, numClasses])) tf.summary.scalar('weights1', weight[1, 1]) tf.summary.scalar('weights2', weight[3, 1]) tf.summary.histogram("weights", weight) bias = tf.Variable(tf.constant(0.1, shape=[numClasses])) tf.summary.histogram("biases", bias) value = tf.transpose(value, [1, 0, 2]) last = tf.gather(value, int(value.get_shape()[0]) - 1) prediction = (tf.matmul(last, weight) + bias) correctPred = tf.equal(tf.argmax(prediction, 1), tf.argmax(labels, 1)) accuracy = tf.reduce_mean(tf.cast(correctPred, tf.float32)) loss = tf.reduce_mean( tf.nn.softmax_cross_entropy_with_logits(logits=prediction, labels=labels)) optimizer = tf.train.GradientDescentOptimizer(0.0005).minimize(loss) import datetime sess = tf.InteractiveSession() tf.summary.scalar('Loss', loss) tf.summary.scalar('Accuracy', accuracy) merged = tf.summary.merge_all() # logdir = "tensorboard/" + datetime.datetime.now().strftime("%Y%m%d-%H%M%S") + "/"
def build_model(self): """Builds the TensorFlow Computational Graph""" lr = 1e-3 # Learning rate lambda_reg = 1e-1 lambda_sl = 1 num_layers = 1 regularizer = tf.contrib.layers.l2_regularizer(scale=lambda_reg) self.X = tf.placeholder(name='X', dtype=tf.float32, shape=[None, self.d_in]) self.Y = tf.placeholder(name='Y', dtype=tf.float32, shape=[None]) # self.A = tf.placeholder(name='A', dtype=tf.int32, shape=[None]) self.sl_actions = tf.placeholder(name='SL_A', dtype=tf.int32, shape=[None]) self.margin_loss = tf.placeholder(name='L', dtype=tf.float32, shape=[None, self.d_out]) self.w = tf.placeholder(name='W', dtype=tf.float32, shape=[None]) self.lr = tf.placeholder(name='lr', dtype=tf.float32, shape=[]) batch_size = tf.shape(self.X)[0] demo_size = tf.shape(self.sl_actions)[0] hidden = self.X for _ in range(num_layers): hidden = tf.layers.dense(hidden, self.h, activation=tf.nn.relu, kernel_regularizer=regularizer) self.predictions = tf.layers.dense(hidden, self.d_out, name='output', kernel_regularizer=regularizer) q_rl = self.predictions[:batch_size, :] q_sl = self.predictions[batch_size:, :] gather_indices = tf.range(0, batch_size) * self.d_out + self.A Y_pred = tf.gather(tf.reshape(q_rl, [-1]), gather_indices) q_pred_sl = tf.reduce_max(q_sl + self.margin_loss, axis=1) gather_indices = tf.range(0, demo_size) * self.d_out + self.sl_actions q_opt_sl = tf.gather(tf.reshape(q_sl, [-1]), gather_indices) self.td_error = tf.abs(self.Y - Y_pred) self.loss_rl = tf.reduce_mean(tf.multiply(self.td_error**2, self.w)) self.loss_sl = tf.reduce_mean(q_pred_sl - q_opt_sl) self.loss = self.loss_rl + lambda_sl * self.loss_sl self.optimizer = tf.train.AdamOptimizer(learning_rate=self.lr) self.train_op = self.optimizer.minimize( self.loss, global_step=self.global_step_tensor) self.summaries = tf.summary.merge([ tf.summary.scalar("loss", self.loss), tf.summary.histogram("q_values_hist", self.predictions), tf.summary.scalar("max_q_value", tf.reduce_max(self.predictions)) ])
def sample_fast_rcnn_targets(boxes, gt_boxes, gt_labels): """ Sample some boxes from all proposals for training. #fg is guaranteed to be > 0, because ground truth boxes will be added as proposals. Args: boxes: nx4 region proposals, floatbox gt_boxes: mx4, floatbox gt_labels: m, int32 Returns: A BoxProposals instance. sampled_boxes: tx4 floatbox, the rois sampled_labels: t int64 labels, in [0, #class). Positive means foreground. fg_inds_wrt_gt: #fg indices, each in range [0, m-1]. It contains the matching GT of each foreground roi. """ iou = pairwise_iou(boxes, gt_boxes) # nxm proposal_metrics(iou) # add ground truth as proposals as well boxes = tf.concat([boxes, gt_boxes], axis=0) # (n+m) x 4 iou = tf.concat([iou, tf.eye(tf.shape(gt_boxes)[0])], axis=0) # (n+m) x m # #proposal=n+m from now on def sample_fg_bg(iou): fg_mask = tf.reduce_max(iou, axis=1) >= cfg.FRCNN.FG_THRESH fg_inds = tf.reshape(tf.where(fg_mask), [-1]) num_fg = tf.minimum(int(cfg.FRCNN.BATCH_PER_IM * cfg.FRCNN.FG_RATIO), tf.size(fg_inds), name='num_fg') fg_inds = tf.random_shuffle(fg_inds)[:num_fg] bg_inds = tf.reshape(tf.where(tf.logical_not(fg_mask)), [-1]) num_bg = tf.minimum(cfg.FRCNN.BATCH_PER_IM - num_fg, tf.size(bg_inds), name='num_bg') bg_inds = tf.random_shuffle(bg_inds)[:num_bg] add_moving_summary(num_fg, num_bg) return fg_inds, bg_inds fg_inds, bg_inds = sample_fg_bg(iou) # fg,bg indices w.r.t proposals best_iou_ind = tf.argmax(iou, axis=1) # #proposal, each in 0~m-1 fg_inds_wrt_gt = tf.gather(best_iou_ind, fg_inds) # num_fg all_indices = tf.concat([fg_inds, bg_inds], axis=0) # indices w.r.t all n+m proposal boxes ret_boxes = tf.gather(boxes, all_indices) ret_labels = tf.concat([ tf.gather(gt_labels, fg_inds_wrt_gt), tf.zeros_like(bg_inds, dtype=tf.int64) ], axis=0) # stop the gradient -- they are meant to be training targets return BoxProposals( tf.stop_gradient(ret_boxes, name='sampled_proposal_boxes'), tf.stop_gradient(ret_labels, name='sampled_labels'), tf.stop_gradient(fg_inds_wrt_gt))
def add_image_summaries(images: tf.Tensor, labels: tf.Tensor, preds: tf.Tensor, locs: tf.Tensor, k: int = 1) -> tf.Tensor: '''Adds image summaries for the k best and k worst images in each batch. Each image is overlayed with (lat, lon), label, and prediction. Args - images: tf.Tensor, shape [batch_size, H, W, C], type float32 - C must be either 3 (RGB order), or 1 (grayscale) - already standardized (relative to entire dataset) with mean 0, std 1 - labels: tf.Tensor, shape [batch_size] - preds: tf.Tensor, shape [batch_size] - locs: tf.Tensor, shape [batch_size, 2], each row is [lat, lon] - k: int, number of best and worst images to show per batch Returns: tf.summary, merged summaries ''' # For float tensors, tf.summary.image automatically scales min/max to 0/255. # Set +/- 3 std. dev. to 0/255. # We want to display images with our own scaling -> cast to tf.uint8 images = tf.clip_by_value((images / 6.0 + 0.5) * 255, clip_value_min=0, clip_value_max=255) images = tf.cast(images, tf.uint8) def write_on_imgs(imgs: np.ndarray, locs: np.ndarray, labels: np.ndarray, preds: np.ndarray) -> np.ndarray: '''Writes white text w/ black background onto images. Args - imgs: np.array, shape [num_imgs, H, W, C], type uint8 C must be either 1 or 3 - locs: np.array, shape [num_imgs, 2] - labels: np.array, shape [num_imgs] - preds: np.array, shape [num_imgs] Returns - new_imgs: np.array, shape [num_imgs, H, W, C] ''' C = imgs.shape[3] new_imgs = np.empty_like(imgs) for i, img in enumerate(imgs): if C == 1: img = img[:, :, 0] # remove C dim. new shape: [H, W] img = PIL.Image.fromarray(img) # write white text on black background draw = PIL.ImageDraw.Draw(img) text = 'loc: ({:.6f}, {:.6f})\nlabel: {:.4f}, pred: {:.4f}'.format( locs[i][0], locs[i][1], labels[i], preds[i]) size = draw.textsize(text) # (w, h) of text draw.rectangle(xy=[(0, 0), size], fill='black') draw.text(xy=(0, 0), text=text, fill='white') if C == 1: new_imgs[i, :, :, 0] = np.asarray(img) else: new_imgs[i] = np.asarray(img) return new_imgs diff = tf.abs(preds - labels) _, worst_indices = tf.nn.top_k(diff, k=k) _, best_indices = tf.nn.top_k(-1 * diff, k=k) worst_inputs = [ tf.gather(x, worst_indices) for x in [images, locs, labels, preds] ] worst_img_sum = tf.summary.image('worst_images_in_batch', tf.py_func(func=write_on_imgs, inp=worst_inputs, Tout=tf.uint8, stateful=False, name='write_on_worst_imgs'), max_outputs=k) best_inputs = [ tf.gather(x, best_indices) for x in [images, locs, labels, preds] ] best_img_sum = tf.summary.image('best_images_in_batch', tf.py_func(func=write_on_imgs, inp=best_inputs, Tout=tf.uint8, stateful=False, name='write_on_best_imgs'), max_outputs=k) return tf.summary.merge([worst_img_sum, best_img_sum])
def fg_labels(self): """ Returns: #fg""" return tf.gather(self.labels, self.fg_inds(), name='fg_labels')
def losses(self): encoded_fg_gt_boxes = encode_bbox_target( tf.gather(self.gt_boxes, self.proposals.fg_inds_wrt_gt), self.proposals.fg_boxes()) * self.bbox_regression_weights return fastrcnn_losses(self.proposals.labels, self.label_logits, encoded_fg_gt_boxes, self.fg_box_logits())
def fastrcnn_predictions(boxes, scores): """ Generate final results from predictions of all proposals. Args: boxes: n#classx4 floatbox in float32 scores: nx#class Returns: boxes: Kx4 scores: K labels: K """ assert boxes.shape[1] == cfg.DATA.NUM_CLASS assert scores.shape[1] == cfg.DATA.NUM_CLASS boxes = tf.transpose(boxes, [1, 0, 2])[1:, :, :] # #catxnx4 boxes.set_shape([None, cfg.DATA.NUM_CATEGORY, None]) scores = tf.transpose(scores[:, 1:], [1, 0]) # #catxn def f(X): """ prob: n probabilities box: nx4 boxes Returns: n boolean, the selection """ prob, box = X output_shape = tf.shape(prob, out_type=tf.int64) # filter by score threshold ids = tf.reshape(tf.where(prob > cfg.TEST.RESULT_SCORE_THRESH), [-1]) prob = tf.gather(prob, ids) box = tf.gather(box, ids) # NMS within each class selection = tf.image.non_max_suppression(box, prob, cfg.TEST.RESULTS_PER_IM, cfg.TEST.FRCNN_NMS_THRESH) selection = tf.gather(ids, selection) # sort available in TF>1.4.0 # sorted_selection = tf.contrib.framework.sort(selection, direction='ASCENDING') sorted_selection = -tf.nn.top_k(-selection, k=tf.size(selection))[0] if get_tf_version_tuple() >= (1, 13): mask = tf.sparse.SparseTensor(indices=tf.expand_dims( sorted_selection, 1), values=tf.ones_like(sorted_selection, dtype=tf.bool), dense_shape=output_shape) mask = tf.sparse.to_dense(mask, default_value=False) else: # this function is deprecated by TF mask = tf.sparse_to_dense(sparse_indices=sorted_selection, output_shape=output_shape, sparse_values=True, default_value=False) return mask # TF bug in version 1.11, 1.12: https://github.com/tensorflow/tensorflow/issues/22750 buggy_tf = get_tf_version_tuple() in [(1, 11), (1, 12)] masks = tf.map_fn(f, (scores, boxes), dtype=tf.bool, parallel_iterations=1 if buggy_tf else 10) # #cat x N selected_indices = tf.where( masks) # #selection x 2, each is (cat_id, box_id) scores = tf.boolean_mask(scores, masks) # filter again by sorting scores topk_scores, topk_indices = tf.nn.top_k(scores, tf.minimum(cfg.TEST.RESULTS_PER_IM, tf.size(scores)), sorted=False) filtered_selection = tf.gather(selected_indices, topk_indices) cat_ids, box_ids = tf.unstack(filtered_selection, axis=1) final_scores = tf.identity(topk_scores, name='scores') final_labels = tf.add(cat_ids, 1, name='labels') final_ids = tf.stack([cat_ids, box_ids], axis=1, name='all_ids') final_boxes = tf.gather_nd(boxes, final_ids, name='boxes') return final_boxes, final_scores, final_labels
def fg_box_logits(self): """ Returns: #fg x ? x 4 """ return tf.gather(self.box_logits, self.proposals.fg_inds(), name='fg_box_logits')
print(vectors.get_shape()) # (2000, 2) # D0 차원의 크기가 2000개이고 D1 차원은 크기가 2(각 점의 x, y 좌표) print(centroids.get_shape()) # (4, 2) # 센트로이드는 D0 차원의 크기가 4개이고 D1 차원은 vectors 와 동일한 2인 행렬 expanded_vectors = tf.expand_dims(vectors, 0) # 두 텐서에 차원을 추가 expanded_centroids = tf.expand_dims(centroids, 1) # 두 텐서를 2차원에서 3차원으로 만들어 뺄셈 가능하도록 # 크기를 맞추는 작업 # 각 점에 대한 유클리드 제곱거리 알고리즘 무한 반복 assignments = tf.argmin( tf.reduce_sum(tf.square(tf.subtract(expanded_vectors, expanded_centroids)), 2), 0) # 새로운 중심 계산하기 # 매 반복마다 새롭게 그룹화를 하면서 각 그룹에 해당하는 새로운 중심을 다시 계산 means = tf.concat([ tf.reduce_mean(tf.gather( vectors, tf.reshape(tf.where(tf.equal(assignments, c)), [1, -1])), reduction_indices=[1]) for c in range(k) ], 0) update_centroids = tf.assign(centroids, means) init_op = tf.global_variables_initializer() sess = tf.Session() sess.run(init_op) for i in range(100): _, centroid_values, assignment_values = sess.run( [update_centroids, centroids, assignments]) # assignment_values 텐서의 결과를 확인하기 위한 결과 그림 그리기 data = {"x": [], "y": [], "cluster": []}
def fg_boxes(self): """ Returns: #fg x4""" return tf.gather(self.boxes, self.fg_inds(), name='fg_boxes')
# Get the indices of elements in x whose values are greater than 30. # Hint: Use tf.where(). # Then extract elements whose values are greater than 30. # Hint: Use tf.gather(). ############################################################################### # YOUR CODE x = tf.constant([ 29.05088806, 27.61298943, 31.19073486, 29.35532951, 30.97266006, 26.67541885, 38.08450317, 20.74983215, 34.94445419, 34.45999146, 29.06485367, 36.01657104, 27.88236427, 20.56035233, 30.20379066, 29.51215172, 33.71149445, 28.59134293, 36.05556488, 28.66994858 ]) indicies = tf.where(tf.less(30.0, x)) over_30 = tf.gather(x, indicies) with tf.Session() as sess: print "Problem 1d:" print "Should only print elements from tensor over 30" sess.run(over_30) print over_30.eval() print "*" * 10 ############################################################################### # 1e: Create a diagnoal 2-d tensor of size 6 x 6 with the diagonal values of 1, # 2, ..., 6 # Hint: Use tf.range() and tf.diag(). ############################################################################### # YOUR CODE
def main(): net = ResNet depth = 50 loader = VOCLoader('07', 'test') net = net(config=net_config, depth=depth, training=False) num_classes = 21 batch_size = args.batch_size img_size = args.image_size image_ph = tf.placeholder(shape=[1, img_size, img_size, 3], dtype=tf.float32, name='img_ph') net.create_trunk(image_ph) bboxer = PriorBoxGrid(net_config) net.create_multibox_head(num_classes) confidence = tf.nn.softmax(tf.squeeze(net.outputs['confidence'])) location = tf.squeeze(net.outputs['location']) good_bboxes = decode_bboxes(location, bboxer.tiling) detection_list = [] score_list = [] for i in range(1, num_classes): class_mask = tf.greater(confidence[:, i], args.conf_thresh) class_scores = tf.boolean_mask(confidence[:, i], class_mask) class_bboxes = tf.boolean_mask(good_bboxes, class_mask) K = tf.minimum(tf.size(class_scores), args.top_k_nms) _, top_k_inds = tf.nn.top_k(class_scores, K) top_class_scores = tf.gather(class_scores, top_k_inds) top_class_bboxes = tf.gather(class_bboxes, top_k_inds) final_inds = tf.image.non_max_suppression( top_class_bboxes, top_class_scores, max_output_size=50, iou_threshold=args.nms_thresh) final_class_bboxes = tf.gather(top_class_bboxes, final_inds) final_scores = tf.gather(top_class_scores, final_inds) detection_list.append(final_class_bboxes) score_list.append(final_scores) net.create_segmentation_head(num_classes) segmentation = tf.cast( tf.argmax(tf.squeeze(net.outputs['segmentation']), axis=-1), tf.int32) times = [] # im = cv2.imread("/home/lear/kshmelko/testImage.jpg") # im = cv2.cvtColor(im, cv2.COLOR_BGR2RGB)/255.0 # im = im.astype(np.float32) with tf.Session(config=tf.ConfigProto(allow_soft_placement=True, log_device_placement=False)) as sess: sess.run(tf.global_variables_initializer()) ckpt_path = train_dir + '/model.ckpt-%i000' % args.ckpt log.debug("Restoring checkpoint %s" % ckpt_path) saver = tf.train.Saver(tf.global_variables()) saver.restore(sess, ckpt_path) for i in range(200): im = loader.load_image(loader.get_filenames()[i]) im = cv2.resize(im, (img_size, img_size)) im = im.reshape((1, img_size, img_size, 3)) st = time.time() sess.run([detection_list, score_list, segmentation], feed_dict={image_ph: im}) et = time.time() if i > 10: times.append(et - st) m = np.mean(times) s = np.std(times) fps = 1 / m log.info("Mean={0:.2f}ms; Std={1:.2f}ms; FPS={2:.1f}".format( m * 1000, s * 1000, fps))
'index': tf.VarLenFeature(dtype=tf.int64), 'value': tf.VarLenFeature(dtype=tf.float32), }) test_label = test_features['label'] test_index = test_features['index'] test_value = test_features['value'] test_label = tf.to_float(test_label, name="test_label") test_ind_vals = test_index.values test_ind_inds = test_index.indices test_val_vals = test_value.values test_val_vals_2d = tf.expand_dims(test_val_vals, 1) test_w_sparse = tf.gather(w, test_index.values, name="test_w_sparse") test_w_transp = tf.transpose(test_w_sparse) test_wtranspx = tf.matmul(test_w_transp, test_val_vals_2d, name="test_wTransX") test_pred = tf.sign(test_wtranspx) test_correctness = tf.equal(test_label, test_pred, name="test_correctness") # Test End # Train start with tf.device("/job:worker/task:%d" % FLAGS.task_index): filename_queue = createFileNameQueues(FLAGS.task_index)
def _build_graph(self): config = self.config x_size = config.dim_input_ctrl h_size = config.dim_hidden_ctrl a_size = config.dim_output_ctrl lr = self.lr_plh with self.graph.as_default(): model_name = config.controller_model_name initializer = tf.contrib.layers.xavier_initializer(uniform=True) if model_name == '2layer': hidden = slim.fully_connected(self.state_plh, h_size, weights_initializer=initializer, activation_fn=tf.nn.leaky_relu) self.logits = slim.fully_connected(hidden, a_size, weights_initializer=initializer, activation_fn=None) self.output = tf.nn.softmax(self.logits) elif model_name == '2layer_logits_clipping': hidden = slim.fully_connected(self.state_plh, h_size, weights_initializer=initializer, activation_fn=tf.nn.leaky_relu) self.logits = slim.fully_connected(hidden, a_size, weights_initializer=initializer, activation_fn=None) self.output = tf.nn.softmax(self.logits / config.logit_clipping_c) elif model_name == 'linear': self.logits = slim.fully_connected(self.state_plh, a_size, weights_initializer=initializer, activation_fn=None) self.output = tf.nn.softmax(self.logits) elif model_name == 'linear_logits_clipping': #self.logits = slim.fully_connected(self.state_plh, a_size, # weights_initializer=initializer, # activation_fn=None) # ----Old version---- w = tf.get_variable('w', shape=[x_size, a_size], dtype=tf.float32, initializer=initializer) b = tf.get_variable('b', shape=[a_size], dtype=tf.float32, initializer=tf.zeros_initializer()) self.logits = tf.matmul(self.state_plh, w) + b self.output = tf.nn.softmax(self.logits / config.logit_clipping_c) else: raise Exception('Invalid controller_model_name') self.chosen_action = tf.argmax(self.output, 1) self.action = tf.cast(tf.argmax(self.action_plh, 1), tf.int32) self.indexes = tf.range(0, tf.shape(self.output)[0])\ * tf.shape(self.output)[1] + self.action self.responsible_outputs = tf.gather(tf.reshape(self.output, [-1]), self.indexes) self.loss = -tf.reduce_mean(tf.log(self.responsible_outputs) * self.reward_plh) # ----Restore gradients and update them after several iterals.---- optimizer = tf.train.AdamOptimizer(learning_rate=lr) self.tvars = tf.trainable_variables() tvars = self.tvars self.gradient_plhs = [] for idx, var in enumerate(tvars): placeholder = tf.placeholder(tf.float32, name=str(idx) + '_plh') self.gradient_plhs.append(placeholder) gvs = optimizer.compute_gradients(self.loss, tvars) self.grads = [grad for grad, _ in gvs] self.train_op = optimizer.apply_gradients(zip(self.gradient_plhs, tvars)) #self.train_op = optimizer.apply_gradients(gvs) self.init = tf.global_variables_initializer() self.saver = tf.train.Saver()
def call(self, inputs): inputs = tf.cast(inputs, tf.int32) outputs = tf.gather(self.embeddings, inputs) return outputs
def main(argv): del argv # unused arg tf.io.gfile.makedirs(FLAGS.output_dir) logging.info('Saving checkpoints at %s', FLAGS.output_dir) tf.random.set_seed(FLAGS.seed) if FLAGS.use_gpu: logging.info('Use GPU') strategy = tf.distribute.MirroredStrategy() else: logging.info('Use TPU at %s', FLAGS.tpu if FLAGS.tpu is not None else 'local') resolver = tf.distribute.cluster_resolver.TPUClusterResolver( tpu=FLAGS.tpu) tf.config.experimental_connect_to_cluster(resolver) tf.tpu.experimental.initialize_tpu_system(resolver) strategy = tf.distribute.TPUStrategy(resolver) aug_params = { 'augmix': FLAGS.augmix, 'aug_count': FLAGS.aug_count, 'augmix_depth': FLAGS.augmix_depth, 'augmix_prob_coeff': FLAGS.augmix_prob_coeff, 'augmix_width': FLAGS.augmix_width, 'ensemble_size': 1, 'mixup_alpha': FLAGS.mixup_alpha, 'adaptive_mixup': FLAGS.adaptive_mixup, 'random_augment': FLAGS.random_augment, 'forget_mixup': FLAGS.forget_mixup, 'num_cores': FLAGS.num_cores, 'threshold': FLAGS.forget_threshold, } batch_size = (FLAGS.per_core_batch_size * FLAGS.num_cores // FLAGS.num_dropout_samples_training) train_input_fn = data_utils.load_input_fn( split=tfds.Split.TRAIN, name=FLAGS.dataset, batch_size=batch_size, use_bfloat16=FLAGS.use_bfloat16, proportion=FLAGS.train_proportion, validation_set=FLAGS.validation, aug_params=aug_params) if FLAGS.validation: validation_input_fn = data_utils.load_input_fn( split=tfds.Split.VALIDATION, name=FLAGS.dataset, batch_size=FLAGS.per_core_batch_size, use_bfloat16=FLAGS.use_bfloat16, validation_set=True) val_dataset = strategy.experimental_distribute_datasets_from_function( validation_input_fn) clean_test_input_fn = data_utils.load_input_fn( split=tfds.Split.TEST, name=FLAGS.dataset, batch_size=FLAGS.per_core_batch_size, use_bfloat16=FLAGS.use_bfloat16) train_dataset = strategy.experimental_distribute_dataset(train_input_fn()) test_datasets = { 'clean': strategy.experimental_distribute_datasets_from_function( clean_test_input_fn), } if FLAGS.corruptions_interval > 0: if FLAGS.dataset == 'cifar10': load_c_dataset = utils.load_cifar10_c else: load_c_dataset = functools.partial(utils.load_cifar100_c, path=FLAGS.cifar100_c_path) corruption_types, max_intensity = utils.load_corrupted_test_info( FLAGS.dataset) for corruption in corruption_types: for intensity in range(1, max_intensity + 1): dataset = load_c_dataset(corruption_name=corruption, corruption_intensity=intensity, batch_size=FLAGS.per_core_batch_size * FLAGS.num_cores, use_bfloat16=FLAGS.use_bfloat16) test_datasets['{0}_{1}'.format(corruption, intensity)] = ( strategy.experimental_distribute_dataset(dataset)) ds_info = tfds.builder(FLAGS.dataset).info num_train_examples = ds_info.splits['train'].num_examples # Train_proportion is a float so need to convert steps_per_epoch to int. if FLAGS.validation: # TODO(ywenxu): Remove hard-coding validation images. steps_per_epoch = int( (num_train_examples * FLAGS.train_proportion - 2500) // batch_size) steps_per_val = 2500 // (FLAGS.per_core_batch_size * FLAGS.num_cores) else: steps_per_epoch = int( num_train_examples * FLAGS.train_proportion) // batch_size steps_per_eval = ds_info.splits['test'].num_examples // batch_size num_classes = ds_info.features['label'].num_classes if FLAGS.use_bfloat16: tf.keras.mixed_precision.set_global_policy('mixed_bfloat16') summary_writer = tf.summary.create_file_writer( os.path.join(FLAGS.output_dir, 'summaries')) with strategy.scope(): logging.info('Building ResNet model') model = ub.models.wide_resnet_dropout( input_shape=ds_info.features['image'].shape, depth=28, width_multiplier=10, num_classes=num_classes, l2=FLAGS.l2, dropout_rate=FLAGS.dropout_rate, residual_dropout=FLAGS.residual_dropout, filterwise_dropout=FLAGS.filterwise_dropout) logging.info('Model input shape: %s', model.input_shape) logging.info('Model output shape: %s', model.output_shape) logging.info('Model number of weights: %s', model.count_params()) # Linearly scale learning rate and the decay epochs by vanilla settings. base_lr = FLAGS.base_learning_rate * batch_size / 128 lr_decay_epochs = [(int(start_epoch_str) * FLAGS.train_epochs) // 200 for start_epoch_str in FLAGS.lr_decay_epochs] lr_schedule = schedules.WarmUpPiecewiseConstantSchedule( steps_per_epoch, base_lr, decay_ratio=FLAGS.lr_decay_ratio, decay_epochs=lr_decay_epochs, warmup_epochs=FLAGS.lr_warmup_epochs) optimizer = tf.keras.optimizers.SGD(lr_schedule, momentum=0.9, nesterov=True) metrics = { 'train/negative_log_likelihood': tf.keras.metrics.Mean(), 'train/accuracy': tf.keras.metrics.SparseCategoricalAccuracy(), 'train/loss': tf.keras.metrics.Mean(), 'train/ece': um.ExpectedCalibrationError(num_bins=FLAGS.num_bins), 'test/negative_log_likelihood': tf.keras.metrics.Mean(), 'test/accuracy': tf.keras.metrics.SparseCategoricalAccuracy(), 'test/ece': um.ExpectedCalibrationError(num_bins=FLAGS.num_bins), } if FLAGS.corruptions_interval > 0: corrupt_metrics = {} for intensity in range(1, max_intensity + 1): for corruption in corruption_types: dataset_name = '{0}_{1}'.format(corruption, intensity) corrupt_metrics['test/nll_{}'.format(dataset_name)] = ( tf.keras.metrics.Mean()) corrupt_metrics['test/accuracy_{}'.format( dataset_name)] = ( tf.keras.metrics.SparseCategoricalAccuracy()) corrupt_metrics['test/ece_{}'.format(dataset_name)] = ( um.ExpectedCalibrationError(num_bins=FLAGS.num_bins)) checkpoint = tf.train.Checkpoint(model=model, optimizer=optimizer) latest_checkpoint = tf.train.latest_checkpoint(FLAGS.output_dir) initial_epoch = 0 if latest_checkpoint: # checkpoint.restore must be within a strategy.scope() so that optimizer # slot variables are mirrored. checkpoint.restore(latest_checkpoint) logging.info('Loaded checkpoint %s', latest_checkpoint) initial_epoch = optimizer.iterations.numpy() // steps_per_epoch @tf.function def train_step(iterator): """Training StepFn.""" def step_fn(inputs): """Per-Replica StepFn.""" if FLAGS.forget_mixup: images, labels, idx = inputs else: images, labels = inputs if FLAGS.augmix and FLAGS.aug_count >= 1: # Index 0 at augmix preprocessing is the unperturbed image. images = images[:, 1, ...] # This is for the case of combining AugMix and Mixup. if FLAGS.mixup_alpha > 0: labels = tf.split(labels, FLAGS.aug_count + 1, axis=0)[1] images = tf.tile(images, [FLAGS.num_dropout_samples_training, 1, 1, 1]) if FLAGS.mixup_alpha > 0: labels = tf.tile(labels, [FLAGS.num_dropout_samples_training, 1]) else: labels = tf.tile(labels, [FLAGS.num_dropout_samples_training]) with tf.GradientTape() as tape: logits = model(images, training=True) if FLAGS.use_bfloat16: logits = tf.cast(logits, tf.float32) if FLAGS.mixup_alpha > 0: negative_log_likelihood = tf.reduce_mean( tf.keras.losses.categorical_crossentropy( labels, logits, from_logits=True)) else: negative_log_likelihood = tf.reduce_mean( tf.keras.losses.sparse_categorical_crossentropy( labels, logits, from_logits=True)) l2_loss = sum(model.losses) loss = negative_log_likelihood + l2_loss # Scale the loss given the TPUStrategy will reduce sum all gradients. scaled_loss = loss / strategy.num_replicas_in_sync grads = tape.gradient(scaled_loss, model.trainable_variables) optimizer.apply_gradients(zip(grads, model.trainable_variables)) probs = tf.nn.softmax(logits) if FLAGS.mixup_alpha > 0: labels = tf.argmax(labels, axis=-1) metrics['train/ece'].update_state(labels, probs) metrics['train/loss'].update_state(loss) metrics['train/negative_log_likelihood'].update_state( negative_log_likelihood) metrics['train/accuracy'].update_state(labels, logits) if FLAGS.forget_mixup: train_predictions = tf.argmax(probs, -1) labels = tf.cast(labels, train_predictions.dtype) # For each ensemble member (1 here), we accumulate the accuracy counts. accuracy_counts = tf.cast( tf.reshape((train_predictions == labels), [1, -1]), tf.float32) return accuracy_counts, idx if FLAGS.forget_mixup: return strategy.run(step_fn, args=(next(iterator), )) else: strategy.run(step_fn, args=(next(iterator), )) @tf.function def test_step(iterator, dataset_name): """Evaluation StepFn.""" def step_fn(inputs): """Per-Replica StepFn.""" images, labels = inputs logits_list = [] for _ in range(FLAGS.num_dropout_samples): logits = model(images, training=False) if FLAGS.use_bfloat16: logits = tf.cast(logits, tf.float32) logits_list.append(logits) # Logits dimension is (num_samples, batch_size, num_classes). logits_list = tf.stack(logits_list, axis=0) probs_list = tf.nn.softmax(logits_list) probs = tf.reduce_mean(probs_list, axis=0) labels_broadcasted = tf.broadcast_to( labels, [FLAGS.num_dropout_samples, labels.shape[0]]) log_likelihoods = -tf.keras.losses.sparse_categorical_crossentropy( labels_broadcasted, logits_list, from_logits=True) negative_log_likelihood = tf.reduce_mean( -tf.reduce_logsumexp(log_likelihoods, axis=[0]) + tf.math.log(float(FLAGS.num_dropout_samples))) if dataset_name == 'clean': metrics['test/negative_log_likelihood'].update_state( negative_log_likelihood) metrics['test/accuracy'].update_state(labels, probs) metrics['test/ece'].update_state(labels, probs) elif dataset_name != 'validation': corrupt_metrics['test/nll_{}'.format( dataset_name)].update_state(negative_log_likelihood) corrupt_metrics['test/accuracy_{}'.format( dataset_name)].update_state(labels, probs) corrupt_metrics['test/ece_{}'.format( dataset_name)].update_state(labels, probs) if dataset_name == 'validation': return tf.reshape(probs, [1, -1, num_classes]), labels if dataset_name == 'validation': return strategy.run(step_fn, args=(next(iterator), )) else: strategy.run(step_fn, args=(next(iterator), )) metrics.update({'test/ms_per_example': tf.keras.metrics.Mean()}) train_iterator = iter(train_dataset) forget_counts_history = [] start_time = time.time() for epoch in range(initial_epoch, FLAGS.train_epochs): logging.info('Starting to run epoch: %s', epoch) acc_counts_list = [] idx_list = [] for step in range(steps_per_epoch): if FLAGS.forget_mixup: temp_accuracy_counts, temp_idx = train_step(train_iterator) acc_counts_list.append(temp_accuracy_counts) idx_list.append(temp_idx) else: train_step(train_iterator) current_step = epoch * steps_per_epoch + (step + 1) max_steps = steps_per_epoch * FLAGS.train_epochs time_elapsed = time.time() - start_time steps_per_sec = float(current_step) / time_elapsed eta_seconds = (max_steps - current_step) / steps_per_sec message = ('{:.1%} completion: epoch {:d}/{:d}. {:.1f} steps/s. ' 'ETA: {:.0f} min. Time elapsed: {:.0f} min'.format( current_step / max_steps, epoch + 1, FLAGS.train_epochs, steps_per_sec, eta_seconds / 60, time_elapsed / 60)) if step % 20 == 0: logging.info(message) # Only one of the forget_mixup and adaptive_mixup can be true. if FLAGS.forget_mixup: current_acc = [ tf.concat(list(acc_counts_list[i].values), axis=1) for i in range(len(acc_counts_list)) ] total_idx = [ tf.concat(list(idx_list[i].values), axis=0) for i in range(len(idx_list)) ] current_acc = tf.cast(tf.concat(current_acc, axis=1), tf.int32) total_idx = tf.concat(total_idx, axis=0) current_forget_path = os.path.join(FLAGS.output_dir, 'forget_counts.npy') last_acc_path = os.path.join(FLAGS.output_dir, 'last_acc.npy') if epoch == 0: forget_counts = tf.zeros([1, num_train_examples], dtype=tf.int32) last_acc = tf.zeros([1, num_train_examples], dtype=tf.int32) else: if 'last_acc' not in locals(): with tf.io.gfile.GFile(last_acc_path, 'rb') as f: last_acc = np.load(f) last_acc = tf.cast(tf.convert_to_tensor(last_acc), tf.int32) if 'forget_counts' not in locals(): with tf.io.gfile.GFile(current_forget_path, 'rb') as f: forget_counts = np.load(f) forget_counts = tf.cast( tf.convert_to_tensor(forget_counts), tf.int32) selected_last_acc = tf.gather(last_acc, total_idx, axis=1) forget_this_epoch = tf.cast(current_acc < selected_last_acc, tf.int32) forget_this_epoch = tf.transpose(forget_this_epoch) target_shape = tf.constant([num_train_examples, 1]) current_forget_counts = tf.scatter_nd( tf.reshape(total_idx, [-1, 1]), forget_this_epoch, target_shape) current_forget_counts = tf.transpose(current_forget_counts) acc_this_epoch = tf.transpose(current_acc) last_acc = tf.scatter_nd(tf.reshape(total_idx, [-1, 1]), acc_this_epoch, target_shape) # This is lower bound of true acc. last_acc = tf.transpose(last_acc) # TODO(ywenxu): We count the dropped examples as forget. Fix this later. forget_counts += current_forget_counts forget_counts_history.append(forget_counts) logging.info('forgetting counts') logging.info(tf.stack(forget_counts_history, 0)) with tf.io.gfile.GFile( os.path.join(FLAGS.output_dir, 'forget_counts_history.npy'), 'wb') as f: np.save(f, tf.stack(forget_counts_history, 0).numpy()) with tf.io.gfile.GFile(current_forget_path, 'wb') as f: np.save(f, forget_counts.numpy()) with tf.io.gfile.GFile(last_acc_path, 'wb') as f: np.save(f, last_acc.numpy()) aug_params['forget_counts_dir'] = current_forget_path train_input_fn = data_utils.load_input_fn( split=tfds.Split.TRAIN, name=FLAGS.dataset, batch_size=batch_size, use_bfloat16=FLAGS.use_bfloat16, validation_set=FLAGS.validation, aug_params=aug_params) train_dataset = strategy.experimental_distribute_dataset( train_input_fn()) train_iterator = iter(train_dataset) elif FLAGS.adaptive_mixup: val_iterator = iter(val_dataset) logging.info('Testing on validation dataset') predictions_list = [] labels_list = [] for step in range(steps_per_val): temp_predictions, temp_labels = test_step( val_iterator, 'validation') predictions_list.append(temp_predictions) labels_list.append(temp_labels) predictions = [ tf.concat(list(predictions_list[i].values), axis=1) for i in range(len(predictions_list)) ] labels = [ tf.concat(list(labels_list[i].values), axis=0) for i in range(len(labels_list)) ] predictions = tf.concat(predictions, axis=1) labels = tf.cast(tf.concat(labels, axis=0), tf.int64) def compute_acc_conf(preds, label, focus_class): class_preds = tf.boolean_mask(preds, label == focus_class, axis=1) class_pred_labels = tf.argmax(class_preds, axis=-1) confidence = tf.reduce_mean( tf.reduce_max(class_preds, axis=-1), -1) accuracy = tf.reduce_mean(tf.cast( class_pred_labels == focus_class, tf.float32), axis=-1) return accuracy - confidence calibration_per_class = [ compute_acc_conf(predictions, labels, i) for i in range(num_classes) ] calibration_per_class = tf.stack(calibration_per_class, axis=1) logging.info('calibration per class') logging.info(calibration_per_class) mixup_coeff = tf.where(calibration_per_class > 0, 1.0, FLAGS.mixup_alpha) mixup_coeff = tf.clip_by_value(mixup_coeff, 0, 1) logging.info('mixup coeff') logging.info(mixup_coeff) aug_params['mixup_coeff'] = mixup_coeff train_input_fn = data_utils.load_input_fn( split=tfds.Split.TRAIN, name=FLAGS.dataset, batch_size=batch_size, use_bfloat16=FLAGS.use_bfloat16, validation_set=True, aug_params=aug_params) train_dataset = strategy.experimental_distribute_dataset( train_input_fn()) train_iterator = iter(train_dataset) datasets_to_evaluate = {'clean': test_datasets['clean']} if (FLAGS.corruptions_interval > 0 and (epoch + 1) % FLAGS.corruptions_interval == 0): datasets_to_evaluate = test_datasets for dataset_name, test_dataset in datasets_to_evaluate.items(): test_iterator = iter(test_dataset) logging.info('Testing on dataset %s', dataset_name) for step in range(steps_per_eval): if step % 20 == 0: logging.info('Starting to run eval step %s of epoch: %s', step, epoch) test_start_time = time.time() test_step(test_iterator, dataset_name) ms_per_example = (time.time() - test_start_time) * 1e6 / batch_size metrics['test/ms_per_example'].update_state(ms_per_example) logging.info('Done with testing on %s', dataset_name) corrupt_results = {} if (FLAGS.corruptions_interval > 0 and (epoch + 1) % FLAGS.corruptions_interval == 0): corrupt_results = utils.aggregate_corrupt_metrics( corrupt_metrics, corruption_types, max_intensity) logging.info('Train Loss: %.4f, Accuracy: %.2f%%', metrics['train/loss'].result(), metrics['train/accuracy'].result() * 100) logging.info('Test NLL: %.4f, Accuracy: %.2f%%', metrics['test/negative_log_likelihood'].result(), metrics['test/accuracy'].result() * 100) total_results = { name: metric.result() for name, metric in metrics.items() } total_results.update(corrupt_results) with summary_writer.as_default(): for name, result in total_results.items(): tf.summary.scalar(name, result, step=epoch + 1) for metric in metrics.values(): metric.reset_states() if (FLAGS.checkpoint_interval > 0 and (epoch + 1) % FLAGS.checkpoint_interval == 0): checkpoint_name = checkpoint.save( os.path.join(FLAGS.output_dir, 'checkpoint')) logging.info('Saved checkpoint to %s', checkpoint_name) final_checkpoint_name = checkpoint.save( os.path.join(FLAGS.output_dir, 'checkpoint')) logging.info('Saved last checkpoint to %s', final_checkpoint_name)