def compute_top_k_and_ndcg(logits, # type: tf.Tensor duplicate_mask, # type: tf.Tensor match_mlperf=False # type: bool ): """Compute inputs of metric calculation. Args: logits: A tensor containing the predicted logits for each user. The shape of logits is (num_users_per_batch * (1 + NUM_EVAL_NEGATIVES),) Logits for a user are grouped, and the first element of the group is the true element. duplicate_mask: A vector with the same shape as logits, with a value of 1 if the item corresponding to the logit at that position has already appeared for that user. match_mlperf: Use the MLPerf reference convention for computing rank. Returns: is_top_k, ndcg and weights, all of which has size (num_users_in_batch,), and logits_by_user which has size (num_users_in_batch, (rconst.NUM_EVAL_NEGATIVES + 1)). """ logits_by_user = tf.reshape(logits, (-1, rconst.NUM_EVAL_NEGATIVES + 1)) duplicate_mask_by_user = tf.reshape(duplicate_mask, (-1, rconst.NUM_EVAL_NEGATIVES + 1)) if match_mlperf: # Set duplicate logits to the min value for that dtype. The MLPerf # reference dedupes during evaluation. logits_by_user *= (1 - duplicate_mask_by_user) logits_by_user += duplicate_mask_by_user * logits_by_user.dtype.min # Determine the location of the first element in each row after the elements # are sorted. sort_indices = tf.argsort( logits_by_user, axis=1, direction="DESCENDING") # Use matrix multiplication to extract the position of the true item from the # tensor of sorted indices. This approach is chosen because both GPUs and TPUs # perform matrix multiplications very quickly. This is similar to np.argwhere. # However this is a special case because the target will only appear in # sort_indices once. one_hot_position = tf.cast(tf.equal(sort_indices, rconst.NUM_EVAL_NEGATIVES), tf.int32) sparse_positions = tf.multiply( one_hot_position, tf.range(logits_by_user.shape[1])[tf.newaxis, :]) position_vector = tf.reduce_sum(sparse_positions, axis=1) in_top_k = tf.cast(tf.less(position_vector, rconst.TOP_K), tf.float32) ndcg = tf.math.log(2.) / tf.math.log( tf.cast(position_vector, tf.float32) + 2) ndcg *= in_top_k # If a row is a padded row, all but the first element will be a duplicate. metric_weights = tf.not_equal(tf.reduce_sum(duplicate_mask_by_user, axis=1), rconst.NUM_EVAL_NEGATIVES) return in_top_k, ndcg, metric_weights, logits_by_user
def _BatchNearestVisualWords(ind, selected_visual_words): """Compute nearest neighbor visual words for a batch of features. Args: ind: Integer index denoting feature. selected_visual_words: Partial set of visual words. Returns: output_ind: Next index. output_selected_visual_words: Updated set of visual words, including the visual words for the new batch. """ # Handle case of last batch, where there may be fewer than # `actual_batch_size` features. batch_size_to_use = tf.cond( tf.greater(ind + actual_batch_size, num_features), true_fn=lambda: num_features - ind, false_fn=lambda: actual_batch_size) # Denote B = batch_size_to_use. # K*B x D. tiled_features = tf.reshape( tf.tile( tf.slice(features, [ind, 0], [batch_size_to_use, self._feature_dimensionality]), [1, self._codebook_size]), [-1, self._feature_dimensionality]) # K*B x D. tiled_codebook = tf.reshape( tf.tile(tf.reshape(codebook, [1, -1]), [batch_size_to_use, 1]), [-1, self._feature_dimensionality]) # B x K. squared_distances = tf.reshape( tf.reduce_sum( tf.math.squared_difference(tiled_features, tiled_codebook), axis=1), [batch_size_to_use, self._codebook_size]) # B x K. nearest_visual_words = tf.argsort(squared_distances) # B x num_assignments. batch_selected_visual_words = tf.slice( nearest_visual_words, [0, 0], [batch_size_to_use, num_assignments]) selected_visual_words = tf.concat( [selected_visual_words, batch_selected_visual_words], axis=0) return ind + batch_size_to_use, selected_visual_words
def _update_metrics(self, metrics, distances, y, just_acc=False): metrics["total"] += 1 if just_acc: best = tf.argmin(distances) else: d_argsort = tf.argsort(distances) best = d_argsort[0] if "map100" not in metrics: metrics["map100"] = 0. for i in range(100): if self.all_doc_unique[d_argsort[i]] == y: metrics["map100"] += 1. / (i + 1.) break if self.all_doc_unique[best] == y: metrics["hits"] += 1
def sel_acc(y_true, y_pred, n_positions, n_exp, do_ratio, return_num=False): pred_sorted = tf.argsort(y_pred, axis=1, direction='DESCENDING') n_evt = tf.shape(y_true)[0] evt_id = tf.range(n_evt) matches_vec = [] for n in range(n_positions): index = tf.transpose( tf.stack([evt_id, tf.reshape(pred_sorted[:, n], shape=(n_evt, ))])) matches_vec.append(tf.gather_nd(y_true, index)) matches_sum = tf.add_n(matches_vec) valid = tf.cast(tf.equal(matches_sum, n_exp), tf.float32) if do_ratio: n_valid = tf.reduce_sum(valid) ratio = n_valid / tf.cast(n_evt, tf.float32) if return_num: ratio = ratio.numpy() return ratio return valid
def nucleus_sampling(logits, p=0.9): sorted_logits = tf.sort(logits, direction='DESCENDING') sorted_indices = tf.argsort(logits, direction='DESCENDING') cumulative_probs = tf.cumsum(tf.nn.softmax(sorted_logits)) t_sorted_indices_to_remove = cumulative_probs > p ''' Shift the indices to the right to keep also the first token above the threshold ''' indices = tf.range(1, tf.shape(logits)[0], 1) sorted_indices_to_remove = tf.scatter_nd(tf.expand_dims(indices, 1), t_sorted_indices_to_remove[:-1], logits.shape) indices_to_remove = tf.boolean_mask(sorted_indices, sorted_indices_to_remove) t = tf.ones(tf.shape(indices_to_remove)[0], dtype=tf.bool) to_remove = tf.scatter_nd(tf.expand_dims(indices_to_remove, 1), t, logits.shape) logits = tf.where( to_remove, tf.ones_like(logits, dtype=logits.dtype) * -1e10, logits ) sample = tf.multinomial(tf.expand_dims(logits, 0), num_samples=1, output_dtype=tf.int32) return tf.reduce_sum(sample)
def compute_patches(image, label, ps=Config.input.patch_size, ss=Config.input.sample_size): """Computes patches to be used as input to the Neural Network. If number of patches exceeds the sample_size (ss), the patches with most information are selected""" image_shape = tf.shape(image) h, w = tf.gather(image_shape, 0), tf.gather(image_shape, 1) pad_h = (ps - h % ps) pad_w = (ps - w % ps) pu = tf.random.uniform(shape=(), minval=0, maxval=pad_h, dtype=tf.int32) pd = pad_h - pu pl = tf.random.uniform(shape=(), minval=0, maxval=pad_w, dtype=tf.int32) pr = pad_w - pl image = tf.pad(image, ((pu, pd), (pl, pr), (0, 0)), constant_values=255) image_shape = tf.shape(image) h, w = tf.gather(image_shape, 0), tf.gather(image_shape, 1) image = tf.reshape(image, (h // ps, ps, w // ps, ps, 3)) image = tf.transpose(image, (0, 2, 1, 3, 4)) image = tf.reshape(image, (-1, ps, ps, 3)) num_images = tf.gather(tf.shape(image), 0) info = tf.reduce_sum( tf.cast(tf.reshape(image, (num_images, -1)), tf.int32), 1) if tf.gather(tf.shape(info), 0) < ss: image = tf.pad(image, ((0, ss - num_images), (0, 0), (0, 0), (0, 0)), constant_values=255) paddings = tf.zeros(ss - num_images, dtype=tf.int32) + (ps**2 * 3 * 255) info = tf.concat((info, paddings), axis=0) indices = tf.gather(tf.argsort(info), tf.range(ss)) image = tf.gather(image, indices) return image, label
def evaluation_step(x, y): predictions = model(x, training=False) loss = compiled_loss(y, predictions) for metric in metrics: metric.update_state(y, predictions) predictions = tf.reshape(predictions, [-1]) predictions = tf.cast(predictions, tf.float64) display_ids = x[DISPLAY_ID_COLUMN] display_ids = tf.reshape(display_ids, [-1]) labels = tf.reshape(y, [-1]) sorted_ids = tf.argsort(display_ids) display_ids = tf.gather(display_ids, indices=sorted_ids) predictions = tf.gather(predictions, indices=sorted_ids) labels = tf.gather(labels, indices=sorted_ids) _, display_ids_idx, display_ids_ads_count = tf.unique_with_counts( display_ids, out_idx=tf.int64) pad_length = 30 - tf.reduce_max(display_ids_ads_count) preds = tf.RaggedTensor.from_value_rowids(predictions, display_ids_idx).to_tensor() labels = tf.RaggedTensor.from_value_rowids( labels, display_ids_idx).to_tensor() labels_mask = tf.math.reduce_max(labels, 1) preds_masked = tf.boolean_mask(preds, labels_mask) labels_masked = tf.boolean_mask(labels, labels_mask) labels_masked = tf.argmax(labels_masked, axis=1, output_type=tf.int32) labels_masked = tf.reshape(labels_masked, [-1, 1]) preds_masked = tf.pad(preds_masked, [(0, 0), (0, pad_length)]) _, predictions_idx = tf.math.top_k(preds_masked, 12) indices = tf.math.equal(predictions_idx, labels_masked) indices_mask = tf.math.reduce_any(indices, 1) masked_indices = tf.boolean_mask(indices, indices_mask) res = tf.argmax(masked_indices, axis=1) ap_matrix = tf.divide(1, tf.add(res, 1)) ap_sum = tf.reduce_sum(ap_matrix) shape = tf.cast(tf.shape(indices)[0], tf.float64) display_id_counter.assign_add(shape) streaming_map.assign_add(ap_sum) return loss
def __init__(self, front: TensorType, threshold: TensorType | float = 0): """ :param front: non-dominated pareto front. :param threshold: a threshold used to screen out cells in partition : when its volume is below this threshold, its rejected directly in order to be more computationally efficient, if setting above 0, this partition strategy tends to return an approximated partition. """ tf.debugging.assert_equal( tf.cast(tf.reduce_sum(tf.abs(non_dominated(front)[1])), dtype=front.dtype), tf.zeros(shape=1, dtype=front.dtype), message=f"\ninput {front} " f"contains dominated points", ) self.front = tf.gather_nd(front, tf.argsort(front[:, :1], axis=0)) # sort self.front = front self._bounds = self._get_bound_index(threshold)
def search(self, query_vector, top_k=None): dot_query_vector = tf.matmul(query_vector, query_vector, transpose_b=True) squared_norms_query_vector = tf.expand_dims( tf.linalg.diag_part(dot_query_vector), 0) dot_product = tf.reduce_sum(self.embeddings * query_vector, axis=1) distances = tf.maximum( self.squared_norms_embeddings + squared_norms_query_vector - 2 * dot_product, 0) sorted_indices = tf.argsort(distances) if top_k: sorted_indices = sorted_indices[..., :top_k] nearest_labels = tf.reshape(tf.gather(self.labels, sorted_indices), shape=[-1, 1]) nearest_distances = tf.reshape(tf.gather(distances[0], sorted_indices), shape=[-1, 1]) return nearest_distances[..., 0], nearest_labels[..., 0], sorted_indices[0]
def sample_top_p(logits, top_p): """Chooses most probable logits with cumulative probabilities upto top_p. Sets the remaining logits to negative infinity. Args: logits: Input logits for next token. top_p: Float tensor with a value >=0 and < 1.0 Returns: Logits with top_p filtering applied. """ sorted_indices = tf.argsort(logits, direction="DESCENDING") # Flatten logits as tf.gather on TPU needs axis to be compile time constant. logits_shape = decoding_module.shape_list(logits) range_for_gather = tf.expand_dims(tf.range(0, logits_shape[0]), axis=1) range_for_gather = tf.tile(range_for_gather * logits_shape[1], [1, logits_shape[1]]) + sorted_indices flattened_logits = tf.reshape(logits, [-1]) flattened_sorted_indices = tf.reshape(range_for_gather, [-1]) sorted_logits = tf.reshape( tf.gather(flattened_logits, flattened_sorted_indices), [logits_shape[0], logits_shape[1]]) cumulative_probs = tf.cumsum(tf.nn.softmax(sorted_logits, axis=-1), axis=-1) # Remove tokens with cumulative probability above the threshold. sorted_indices_to_remove = cumulative_probs > top_p # Shift the indices to the right to keep the first token above threshold. sorted_indices_to_remove = tf.roll(sorted_indices_to_remove, 1, axis=-1) sorted_indices_to_remove = tf.concat([ tf.zeros_like(sorted_indices_to_remove[:, :1]), sorted_indices_to_remove[:, 1:] ], -1) # Scatter sorted indices to original indexes. indices_to_remove = scatter_values_on_batch_indices( sorted_indices_to_remove, sorted_indices) top_p_logits = set_tensor_by_indices_to_value(logits, indices_to_remove, np.NINF) return top_p_logits
def train_step(self, imgs, d_steps, g_steps, pruning): batch_size = len(imgs) # Adversarial ground truths valid = np.ones((batch_size, 1)) fake = np.zeros((batch_size, 1)) # Sample noise and generate a batch of new images for _ in range(d_steps): noise = self.prior(batch_size) gen_imgs = self.generator.predict(noise) # fakes_val = gen_imgs[:10] # ------------------------------- PRUNING -------------------------------- if pruning != 1: gen_scores = self.discriminator.predict(gen_imgs) gen_imgs_sorted_ids = tf.argsort(gen_scores, axis=0, direction='DESCENDING') leftover = int(gen_imgs_sorted_ids.shape[0] // (1 / pruning)) fake_output_best_ids = gen_imgs_sorted_ids[:leftover, :] gen_imgs = tf.gather_nd(gen_imgs, fake_output_best_ids) fake = np.zeros((gen_imgs.shape[0], 1)) # ------------------------------------------------------------------------ # Train the discriminator (real classified as ones and generated as zeros) d_loss_real = self.discriminator.train_on_batch(imgs, valid) d_loss_fake = self.discriminator.train_on_batch(gen_imgs, fake) d_loss = 0.5 * np.add(d_loss_real, d_loss_fake) #assert np.allclose(self.generator.predict(noise)[:10], fakes_val), "D update effects G" # Train Generator g_loss = -1 for _ in range(g_steps): # Train the generator (wants discriminator to mistake images as real) y_val = self.discriminator.predict(imgs[:10]) g_loss = self.combined.train_on_batch(noise, valid)
def randomSearch(encoder, decoder, final_layer, test_inp, k): """Helper method of inferenceRandomSearch """ score = 0 enc_inp, dec_inp, padding_mask, combined_mask = test_inp enc_output = encoder(inputs=enc_inp, mask=padding_mask, training=False) for slen in range(100): dec_output, attention_weights = decoder(inputs=dec_inp, enc_output=enc_output, look_ahead_mask=combined_mask, padding_mask=padding_mask, training=False) final_output = final_layer(inputs=dec_output) # select the last word from the seq_len dimension predictions = final_output[:, -1:, :] # Sort id and scores bestId = tf.argsort(predictions) bestScore = tf.sort(predictions) # Take one of the k best scores predicted_id = [] for idRow, scoreRow in zip(bestId, bestScore): for idR, scoreR in zip(idRow, scoreRow): # At beggining of sequence, allow for randomness, but after, always pick best score if slen > 15: k = 1 # Get one of the k best ID and its corresponding score randomIndex = random.randint(1, k) predicted_id.append([idR[-randomIndex]]) score += scoreR[-randomIndex] # Convert to tensor predicted_id = tf.convert_to_tensor(predicted_id) dec_inp = tf.concat([dec_inp, predicted_id], axis=-1) return dec_inp, score
def azimAvg_tensor(image, center=None): """ Calculate the azimuthally averaged power spectrum (1D), for a batch of image tensors. image - The image tensor, [N,C,H,W] format center - The [x,y] pixel coordinates used as the center. The default is None, which then uses the center of the image (including fractional pixels). """ batch, channel, height, width = image.shape.as_list() # Calculate the indices from the image y, x = np.indices([height, width]) y = np.tile(y, (batch, channel, 1, 1)) x = np.tile(x, (batch, channel, 1, 1)) if not center: center = np.array([(x.max()-x.min())/2.0, (x.max()-x.min())/2.0]) r = np.hypot(x - center[0], y - center[1]) # Get sorted radii ind = tf.argsort(tf.reshape(r, (batch, channel, -1,))) r_sorted = tf.gather(tf.reshape(r, (batch, channel, -1,)), ind, batch_dims=2) i_sorted = tf.gather(tf.reshape(image, (batch, channel, -1,)), ind, batch_dims=2) # Get the integer part of the radii (bin size = 1) r_int = tf.cast(r_sorted, tf.int32) # Find all pixels that fall within each radial bin. deltar = r_int[:,:,1:] - r_int[:,:,:-1] # Assumes all radii represented rind = tf.reshape(tf.where(deltar)[:,2], (batch, -1)) # location of changes in radius rind = tf.expand_dims(rind, axis=1) nr = tf.cast(rind[:,:,1:] - rind[:,:,:-1], tf.float32) # number of radius bin # Cumulative sum to figure out sums for each radius bin csum = tf.cumsum(i_sorted, axis=-1) tbin = tf.gather(csum, rind[:,:,1:], batch_dims=2) - tf.gather(csum, rind[:,:,:-1], batch_dims=2) radial_prof = tbin / nr return radial_prof
def inverse(x, w, b, v): tf.debugging.assert_equal(b[0],tf.zeros([1,],dtype=tf.float64)) w = tf.cast(w, dtype=tf.float64) b = tf.cast(b, dtype=tf.float64) v = tf.cast(v, dtype=tf.float64) # Calculate v0 if w.get_shape() != v.get_shape(): v0 = -(tf.reshape(b,[1,H]) @ v)[0] x = tf.dtypes.cast(x, tf.float64) # num_layer = w.shape[1] b_over_w = - b / w # the negative sign is important b_over_w = tf.reshape(b_over_w, [-1])# change it to row vector # Then we need to compute range for the inverse function r_x = tf.reshape( [tf.matmul( ( tf.nn.relu(w * b_over_w[i] + b) ), v) + v0 for i in range(1)], [-1] ) # pad r_x with 0 and inf # right side gives us the right index zero = tf.constant([0.], dtype=tf.float64) pos = tf.searchsorted(tf.sort(tf.concat([zero, r_x, [np.inf]],0)), x,side='right') # get the interval index # tf.print('pos',pos) pos = tf.reshape(pos,[]) # get only numerical value # change it to row vector v = tf.reshape(v, [-1]) w = tf.reshape(w, [-1]) index = tf.argsort(r_x) # sort according to the index v_b = tf.gather(v * b, index) v_w = tf.gather(v * w, index) num = x - v0 - tf.reduce_sum(v_b[:(pos - 1)]) deo = tf.reduce_sum(v_w[:(pos - 1)]) with tf.control_dependencies([ tf.debugging.assert_positive(num, message='inverse num'), tf.debugging.assert_positive(deo, message='inverse deo'), ]): return tf.math.divide_no_nan(num, deo)[0]
def segment_top_k(x, I, ratio): """ Returns indices to get the top K values in x segment-wise, according to the segments defined in I. K is not fixed, but it is defined as a ratio of the number of elements in each segment. :param x: a rank 1 Tensor; :param I: a rank 1 Tensor with segment IDs for x; :param ratio: float, ratio of elements to keep for each segment; :return: a rank 1 Tensor containing the indices to get the top K values of each segment in x. """ rt = tf.RaggedTensor.from_value_rowids(x, I) row_lengths = rt.row_lengths() dense = rt.to_tensor(default_value=-np.inf) indices = tf.cast(tf.argsort(dense, direction="DESCENDING"), tf.int64) row_starts = tf.cast(rt.row_starts(), tf.int64) indices = indices + tf.expand_dims(row_starts, 1) row_lengths = tf.cast( tf.math.ceil(ratio * tf.cast(row_lengths, tf.float32)), tf.int32) return tf.RaggedTensor.from_tensor(indices, row_lengths).values
def get_metrics(data, model, args, k: int): dataset = SessionDataset(data) loader = SessionDataLoader(dataset, batch_size=args.batch_size) recall_list, mrr_list = [], [] total_step = len(data) - data['SessionId'].nunique() for inputs, label, mask in tqdm(loader, total=total_step // args.batch_size, desc='Evaluation', mininterval=1): reset_hidden_states(model, mask) input_ohe = to_categorical(inputs, num_classes=args.num_items) input_ohe = np.expand_dims(input_ohe, axis=1) pred = model.predict(input_ohe, batch_size=args.batch_size) pred_arg = tf.argsort(pred, direction='DESCENDING') length = len(inputs) recall_list.extend([recall_k(pred_arg[i], label[i], k) for i in range(length)]) mrr_list.extend([mrr_k(pred_arg[i], label[i], k) for i in range(length)]) recall, mrr = np.mean(recall_list), np.mean(mrr_list) return recall, mrr
def calc_top_k(self, class_index, true_movement): # get the top companies probabilities top_indices = tf.where(tf.equal(self.model.prediction, class_index), name='top_indices') top_prob = tf.math.reduce_sum(tf.gather(self.model.prob, top_indices), axis=1)[:, class_index] # calc top k by model k = self.model.params.num_companies * self.top_k_percent # in case k passes the total companies of the movement, summary all of them corrected_k = k if top_indices.shape[0] >= k else tf.shape( top_indices, out_type=tf.int32)[0] top_k_indices = tf.argsort(top_prob, direction="DESCENDING")[:corrected_k] # finally, calculate success ratio of top k by true movement of the companies return tf.math.divide( tf.reduce_sum( tf.cast(tf.gather(true_movement, top_k_indices), dtype=tf.float32)), tf.cast(corrected_k, tf.float32))
def nucleus(logits, p): indices = tf.argsort(logits, direction='DESCENDING', axis=1) vals = tf.sort(tf.nn.softmax(logits, axis=1), direction='DESCENDING', axis=1) tail_ids = tf.cast( tf.argmax(tf.cast(tf.cumsum(vals, axis=1) > p, tf.int8), axis=1), tf.int32) logit_inds = tf.stack([tf.range(0, logits.shape[0].value), tail_ids], axis=1) tail_min_vals = tf.expand_dims(tf.gather_nd(logits, logit_inds), 1) return tf.where( logits < tail_min_vals, # if it is worse than this lower bound. I can do this for my ones too! does it for each batch simultaneously. tf.ones_like(logits, dtype=logits.dtype) * -1e10, logits, ) '''while_condition = lambda i, logits_to_return: tf.less(i, logits.shape[0].value)
def basic_ev(rand, genotypes, fitness, n_parents=8, cohort_factor=.8, mu=0.08): ''' {n_parents} parents are preserved as an elitist cohort. The children's mutation rates will be calculated as follows: The ith cohort of children has a mutation rate of ((i + 1)**{cohort_factor} - 1)*{mu}''' n_cohorts = (genotypes.shape[0] // n_parents) + 1 elite = tf.argsort(-fitness)[:n_parents] tf.print(elite, summarize=-1) indices = tf.concat([elite] * n_cohorts, axis=0)[:genotypes.shape[0]] parents_gene = tf.gather(genotypes, indices) mutation_factor = tf.reshape( tf.constant([((i // n_parents + 1)**cohort_factor - 1) for i in range(genotypes.shape[0])]), (genotypes.shape[0], 1)) # each cohort mutates a little more than the last children_gene = parents_gene + rand.normal( parents_gene.shape) * mutation_factor * mu return children_gene
def _bounds_2d(front: TensorType) -> BoundedVolumes: # this assumes the Pareto set has been sorted in ascending order on the first # objective, which implies the second objective is sorted in descending order len_front, number_of_objectives = front.shape pf_ext_idx = tf.concat( [ tf.zeros([1, number_of_objectives], dtype=tf.int32), tf.argsort(front, axis=0) + 1, tf.ones([1, number_of_objectives], dtype=tf.int32) * len_front + 1, ], axis=0, ) range_ = tf.range(len_front + 1)[:, None] lower = tf.concat([range_, tf.zeros_like(range_)], axis=-1) upper = tf.concat([range_ + 1, pf_ext_idx[::-1, 1:][: pf_ext_idx[-1, 0]]], axis=-1) return BoundedVolumes(lower, upper)
def partial_log_likelihood(prediction, t, y): """ calculate cox loss :param prediction: prediction of model :param t: event happen at the 't'th day :param y: true label :return: """ risk = tf.reshape(prediction, [-1]) time = tf.reshape(t, [-1]) E = tf.reshape(y, [-1]) sort_idx = tf.argsort(time, direction='DESCENDING') E = tf.gather(E, sort_idx) risk = tf.gather(risk, sort_idx) hazard_ratio = tf.exp(risk) log_risk = tf.math.log(tf.cumsum(hazard_ratio)) uncensored_likelihood = risk - log_risk censored_likelihood = tf.multiply(uncensored_likelihood, E) neg_likelihood = -tf.reduce_sum(censored_likelihood) * 0.01 return neg_likelihood
def quantization_topology(input, output, lmb): N = input.shape[0] adjacency_matrix = np.zeros((N, N)) D = squared_dist(input, tf.transpose(output)) d_min = tf.math.reduce_min(D, axis=1) s = tf.argsort(D.numpy(), axis=1)[:, :2].numpy() for i in range(N): idx = s[:, 0] == i si = s[idx] if len(si) > 0: for j in set(si[:, 1]): k = sum(si[:, 1] == j) adjacency_matrix[i, j] = 1 adjacency_matrix[j, i] = 1 E = tf.convert_to_tensor(adjacency_matrix, np.float32) Q = tf.norm(d_min) E = tf.norm(E, 2) cost = Q + lmb * E return cost
def nucleus_sampling(logits_BxN, top_p): sort_indices_BxN = tf.argsort(logits_BxN, axis=-1, direction='DESCENDING') probs_BxN = tf.gather(tf.nn.softmax(logits_BxN), sort_indices_BxN, batch_dims=1) cumprobs_BxN = tf.cumsum(probs_BxN, axis=-1, exclusive=True) # The top 1 candidate always will not be masked. # This way ensures at least 1 indices will be selected. sort_mask_BxN = tf.cast(tf.greater(cumprobs_BxN, top_p), logits_BxN.dtype) batch_indices_BxN = tf.tile( tf.expand_dims(tf.range(tf.shape(logits_BxN)[0]), axis=-1), [1, tf.shape(logits_BxN)[1]], ) top_p_mask_BxN = tf.scatter_nd( tf.stack([batch_indices_BxN, sort_indices_BxN], axis=-1), sort_mask_BxN, tf.shape(logits_BxN), ) logits_BxN -= top_p_mask_BxN * logits_BxN.dtype.max return logits_BxN
def calibration_plot(model, x, y, ece): py_x = tf.nn.softmax(model(x)) p = tf.max(py_x, axis=1) hat_y = tf.argmax(py_x, axis=1) y = tf.argmax(y, axis=1) acc = tf.cast(hat_y == y, tf.float32) idx = tf.argsort(p) p = p[idx] acc = acc[idx] plt.title(f'Calibration {model.name}: {ece}') plt.ylabel('Frequency') plt.xlabel('ACC/Conf') plt.xlim([0, 1]) plt.hist(acc, bins=20, color='blue', label='accuracy', alpha=.5) plt.hist(p, bins=20, color='red', label='confidence', alpha=.5) plt.legend() return 0
def score_matrix_to_binary_ranking(similarity_matrix, query_labels, candidate_labels, remove_top1=False): query_labels = tf.reshape(query_labels, [-1, 1]) candidate_labels = tf.reshape(candidate_labels, [-1, 1]) pair_signs = tf.cast( tf.equal(query_labels, tf.transpose(candidate_labels)), tf.float32) index_ranking = tf.argsort(similarity_matrix, axis=1, direction="DESCENDING") if remove_top1: index_ranking = index_ranking[:, 1:] gather_idx = arg_to_gather_nd(index_ranking) binary_ranking = tf.reshape(tf.gather_nd(pair_signs, gather_idx), index_ranking.shape) return binary_ranking
def NMS(self, rectangles, threshold, type): if rectangles.shape[0] == 0: return rectangles wh = rectangles[..., 2:4] - rectangles[..., 0:2] + tf.constant( [1, 1], dtype=tf.float32) area = wh[..., 0] * wh[..., 1] conf = rectangles[..., 4] I = tf.argsort(conf, direction='DESCENDING') i = 0 while i < I.shape[0]: idx = I[i] cur_upper_left = rectangles[idx, 0:2] cur_down_right = rectangles[idx, 2:4] cur_area = area[idx] following_idx = I[i + 1:] following_upper_left = tf.gather(rectangles[..., 0:2], following_idx) following_down_right = tf.gather(rectangles[..., 2:4], following_idx) following_area = tf.gather(area, following_idx) max_upper_left = tf.math.maximum(cur_upper_left, following_upper_left) min_down_right = tf.math.minimum(cur_down_right, following_down_right) intersect_wh = min_down_right - max_upper_left intersect_wh = tf.where(tf.math.greater(intersect_wh, 0), intersect_wh, tf.zeros_like(intersect_wh)) intersect_area = intersect_wh[..., 0] * intersect_wh[..., 1] if type == 'iom': overlap = intersect_area / tf.math.minimum( cur_area, following_area) else: overlap = intersect_area / (cur_area + following_area - intersect_area) indices = tf.where(tf.less(overlap, threshold)) following_idx = tf.gather_nd(following_idx, indices) I = tf.concat([I[:i + 1], following_idx], axis=0) i += 1 # result_rectangles.shape = (picked target num, 5) result_rectangles = tf.gather(rectangles, I) return result_rectangles
def combine_results(name2results, keys): """Combines network outputs. Args: name2results: Dict. For each object results, `z_vals` is required. keys: [str]. A list of keys to combine results over. Returns: results: Dict. Combined results. """ # Collect z values across all objects. z_vals_list = [] for _, results in name2results.items(): z_vals_list.append(results['z_vals']) # Concatenate lists of object results into a single tensor. z_vals = tf.concat(z_vals_list, axis=-1) # [R, S*O] # Compute the argsort indices. z_argsort_indices = tf.argsort(z_vals, -1) # [R, S*O] n_rays, n_samples = tf.shape(z_vals)[0], tf.shape(z_vals)[1] gather_indices = tf.range(n_rays)[:, None] # [R, 1] gather_indices = tf.tile(gather_indices, [1, n_samples]) # [R, S] gather_indices = tf.concat( [gather_indices[Ellipsis, None], z_argsort_indices[Ellipsis, None]], axis=-1) results = {} for k in keys: if k == 'z_vals': v_combined = z_vals else: v_list = [r[k] for r in name2results.values()] v_combined = tf.concat(v_list, axis=1) # [R, S*O, K] # Sort the tensors. v_sorted = tf.gather_nd( # [R, S, K] params=v_combined, # [R, S, K] indices=gather_indices) # [R, S, 2] results[k] = v_sorted return results
def __init__(self, size): self.graph = tf.Graph() self.session = tf.InteractiveSession(graph=self.graph) self.x = tf.placeholder(tf.float32, [None, size * size]) self.y = tf.placeholder(tf.float32, [None, size * size]) self.w_conv1 = self.weight_variable([5, 5, 1, 32]) self.b_conv1 = self.bias_variable([32]) self.x_image = tf.reshape(self.x, [-1, size, size, 1]) self.conv1 = tf.nn.relu( self.conv2d(self.x_image, self.w_conv1) + self.b_conv1) self.pool1 = self.max_pool(self.conv1) self.w_conv2 = self.weight_variable([5, 5, 32, 64]) self.b_conv2 = self.bias_variable([64]) self.conv2 = tf.nn.relu( self.conv2d(self.pool1, self.w_conv2) + self.b_conv2) self.pool2 = self.max_pool(self.conv2) max_size = self.pool2.shape[1].value self.w_fc1 = self.weight_variable([max_size * max_size * 64, 1024]) self.b_fc1 = self.bias_variable([1024]) self.pool2_flat = tf.reshape(self.pool2, [-1, max_size * max_size * 64]) self.fc1 = tf.nn.relu( tf.matmul(self.pool2_flat, self.w_fc1) + self.b_fc1) self.w_fc2 = self.weight_variable([1024, size * size]) self.b_fc2 = self.bias_variable([size * size]) self.y_conv = tf.nn.softmax( tf.matmul(self.fc1, self.w_fc2) + self.b_fc2) self.sorted_pred = tf.argsort(self.y_conv, direction="DESCENDING") self.cross_entropy = -tf.reduce_sum(self.y * tf.log(self.y_conv)) self.train_step = tf.train.AdamOptimizer(1e-4).minimize( self.cross_entropy) tf.summary.scalar("loss", self.cross_entropy) self.merged_summary = tf.summary.merge_all() self.saver = tf.train.Saver() self.session.run(tf.global_variables_initializer()) correction_prediction = tf.equal(tf.argmax(self.y_conv, 1), tf.argmax(self.y, 1))
def get_batch(self, model, dist, self_check=False): self.batch_iter += 1 if self.batch_iter % self.hard_every: return super(DssmTrainerWithCustomTokenizerAndHards, self).get_batch(model, self_check=self_check) print("HardBatch!") sub_set = sorted(np.random.choice(self.X_train.shape[0], size=self.batch_size // self.hard_neg_k, replace=False)) # print(sub_set) # self.doc_tokenizer.from_raw([self.train_doc_tokens[i] for i in range(self.X_train.shape[0])]) q_t = self.query_tokenizer.from_raw([self.train_query_tokens[i] for i in sub_set]) train_d_emb = model.call_doc(self.train_doc_tokens_tf) all_q_emb = model.call_query(q_t) for q_emb in all_q_emb: q_emb_r = tf.reshape(q_emb, (1, -1)) distances = dist(q_emb_r, train_d_emb) # print(distances) top = tf.argsort(distances[0])[:self.hard_neg_k - 1] # print(top) for cand in top.numpy(): sub_set.append(cand) sub_set = np.unique(sub_set) # print(len(sub_set)) d_t = self.doc_tokenizer.from_raw([self.train_doc_tokens[i] for i in sub_set]) q_t = self.query_tokenizer.from_raw([self.train_query_tokens[i] for i in sub_set]) if self_check: sub_set = self.X_train[sub_set] q_t_ = self.query_tokenizer(sub_set[:,0]) d_t_ = self.doc_tokenizer(sub_set[:,1]) d_t__, q_t__, d_t_, q_t_ = map(tf.sparse.to_dense, [d_t, q_t, d_t_, q_t_]) assert np.sum((q_t__ - q_t_) ** 2) < 1e-9 assert np.sum((d_t__ - d_t_) ** 2) < 1e-9 return q_t, d_t
def GetIOUNMS( boxes, scores, max_output_size, iou_threshold=0.5, iou_type='iou'): '''通过NMS筛选框''' # 分数倒序下标 scores_sort_indexes = tf.argsort(scores, direction='DESCENDING') # 排序后的框 boxes_sort = tf.gather(boxes, scores_sort_indexes) # NMS后的下标 result_indexes = tf.TensorArray(tf.int32, 0, dynamic_size=True) def boxes_foreach(idx, result_indexes, boxes_sort, scores_sort_indexes): # 已达到最大输出框数量 if idx >= max_output_size: return -1, result_indexes, boxes_sort, scores_sort_indexes boxes_num = tf.shape(boxes_sort)[0] # 已遍历所有框 if boxes_num == 0: return -1, result_indexes, boxes_sort, scores_sort_indexes # 取最高分的box boxes_top = boxes_sort[0:1, :] indexes_top = scores_sort_indexes[0] if boxes_num > 1: # 计算IOU boxes_other = boxes_sort[1:, :] indexes_other = scores_sort_indexes[1:] iou = GetIOU(boxes_top, boxes_other, iou_type) # 过滤掉小于阈值的数据 iou_mask = iou < iou_threshold boxes_sort = tf.boolean_mask(boxes_other, iou_mask) scores_sort_indexes = tf.boolean_mask(indexes_other, iou_mask) result_indexes = result_indexes.write(idx, indexes_top) return idx + 1, result_indexes, boxes_sort, scores_sort_indexes else: result_indexes = result_indexes.write(idx, indexes_top) return -1, result_indexes, boxes_sort, scores_sort_indexes _, result_indexes, _, _ = tf.while_loop(lambda i, *args: tf.math.not_equal(i, -1), boxes_foreach, [0, result_indexes, boxes_sort, scores_sort_indexes]) result_indexes = result_indexes.stack() return result_indexes
def predict(mode): PREDS = [] CONFS = [] NUM_SELECT = 10 batch_size = 320 for batch_id, X in enumerate(batch_generator(mode,batch_size=batch_size,loop=False)): x = X[0] print("Predicting {} - Batch {}".format(mode,batch_id)) pred = model.predict_on_batch(x) if batch_id == 0: print(pred) PREDS.append(tf.argsort(pred,axis=-1)[:,-NUM_SELECT:]) CONFS.append(tf.sort(pred,axis=-1)[:,-NUM_SELECT:]) PREDS = np.concatenate(PREDS,axis=0) CONFS = np.concatenate(CONFS,axis=0) PREDS = np.concatenate([PREDS,CONFS],axis=1) cols = ['pred_{}'.format(k) for k in range(NUM_SELECT)] + \ ['conf_{}'.format(k) for k in range(NUM_SELECT)] fname = os.path.join(DATA_DIR,'dom_pred_{}.csv'.format(mode)) pd.DataFrame(PREDS,index=range(PREDS.shape[0]),columns=cols).to_csv(fname)
def update_state(self, y_true, y_pred, sample_weight=None): """Update state of the metric. Args: y_true: the true labels with shape [batch_size, 1]. y_pred: model output, which is the similarity matrix with shape [batch_size, label_embedding_vocab_size] between context and full vocab label embeddings. sample_weight: Optional weighting of each example. Defaults to 1. """ similarities = y_pred label_indices = tf.expand_dims(tf.cast(y_true, dtype=tf.int32), -1) # Get the indices of similar labels in sorted order for each context. sorted_similarities = tf.argsort(similarities, axis=-1, direction='DESCENDING') ranks = tf.where(tf.equal(sorted_similarities, label_indices))[:, -1] ranks = tf.keras.backend.cast(ranks, 'float32') super().update_state(ranks, sample_weight=sample_weight)