def compute_top_k_and_ndcg(logits, # type: tf.Tensor
duplicate_mask, # type: tf.Tensor
match_mlperf=False # type: bool
):
"""Compute inputs of metric calculation.
Args:
logits: A tensor containing the predicted logits for each user. The shape
of logits is (num_users_per_batch * (1 + NUM_EVAL_NEGATIVES),) Logits
for a user are grouped, and the first element of the group is the true
element.
duplicate_mask: A vector with the same shape as logits, with a value of 1
if the item corresponding to the logit at that position has already
appeared for that user.
match_mlperf: Use the MLPerf reference convention for computing rank.
Returns:
is_top_k, ndcg and weights, all of which has size (num_users_in_batch,), and
logits_by_user which has size
(num_users_in_batch, (rconst.NUM_EVAL_NEGATIVES + 1)).
"""
logits_by_user = tf.reshape(logits, (-1, rconst.NUM_EVAL_NEGATIVES + 1))
duplicate_mask_by_user = tf.reshape(duplicate_mask,
(-1, rconst.NUM_EVAL_NEGATIVES + 1))
if match_mlperf:
# Set duplicate logits to the min value for that dtype. The MLPerf
# reference dedupes during evaluation.
logits_by_user *= (1 - duplicate_mask_by_user)
logits_by_user += duplicate_mask_by_user * logits_by_user.dtype.min
# Determine the location of the first element in each row after the elements
# are sorted.
sort_indices = tf.argsort(
logits_by_user, axis=1, direction="DESCENDING")
# Use matrix multiplication to extract the position of the true item from the
# tensor of sorted indices. This approach is chosen because both GPUs and TPUs
# perform matrix multiplications very quickly. This is similar to np.argwhere.
# However this is a special case because the target will only appear in
# sort_indices once.
one_hot_position = tf.cast(tf.equal(sort_indices, rconst.NUM_EVAL_NEGATIVES),
tf.int32)
sparse_positions = tf.multiply(
one_hot_position, tf.range(logits_by_user.shape[1])[tf.newaxis, :])
position_vector = tf.reduce_sum(sparse_positions, axis=1)
in_top_k = tf.cast(tf.less(position_vector, rconst.TOP_K), tf.float32)
ndcg = tf.math.log(2.) / tf.math.log(
tf.cast(position_vector, tf.float32) + 2)
ndcg *= in_top_k
# If a row is a padded row, all but the first element will be a duplicate.
metric_weights = tf.not_equal(tf.reduce_sum(duplicate_mask_by_user, axis=1),
rconst.NUM_EVAL_NEGATIVES)
return in_top_k, ndcg, metric_weights, logits_by_user
def _BatchNearestVisualWords(ind, selected_visual_words):
"""Compute nearest neighbor visual words for a batch of features.
Args:
ind: Integer index denoting feature.
selected_visual_words: Partial set of visual words.
Returns:
output_ind: Next index.
output_selected_visual_words: Updated set of visual words, including
the visual words for the new batch.
"""
# Handle case of last batch, where there may be fewer than
# `actual_batch_size` features.
batch_size_to_use = tf.cond(
tf.greater(ind + actual_batch_size, num_features),
true_fn=lambda: num_features - ind,
false_fn=lambda: actual_batch_size)
# Denote B = batch_size_to_use.
# K*B x D.
tiled_features = tf.reshape(
tf.tile(
tf.slice(features, [ind, 0],
[batch_size_to_use, self._feature_dimensionality]),
[1, self._codebook_size]), [-1, self._feature_dimensionality])
# K*B x D.
tiled_codebook = tf.reshape(
tf.tile(tf.reshape(codebook, [1, -1]), [batch_size_to_use, 1]),
[-1, self._feature_dimensionality])
# B x K.
squared_distances = tf.reshape(
tf.reduce_sum(
tf.math.squared_difference(tiled_features, tiled_codebook),
axis=1), [batch_size_to_use, self._codebook_size])
# B x K.
nearest_visual_words = tf.argsort(squared_distances)
# B x num_assignments.
batch_selected_visual_words = tf.slice(
nearest_visual_words, [0, 0], [batch_size_to_use, num_assignments])
selected_visual_words = tf.concat(
[selected_visual_words, batch_selected_visual_words], axis=0)
return ind + batch_size_to_use, selected_visual_words
def _update_metrics(self, metrics, distances, y, just_acc=False):
metrics["total"] += 1
if just_acc:
best = tf.argmin(distances)
else:
d_argsort = tf.argsort(distances)
best = d_argsort[0]
if "map100" not in metrics:
metrics["map100"] = 0.
for i in range(100):
if self.all_doc_unique[d_argsort[i]] == y:
metrics["map100"] += 1. / (i + 1.)
break
if self.all_doc_unique[best] == y:
metrics["hits"] += 1
def sel_acc(y_true, y_pred, n_positions, n_exp, do_ratio, return_num=False):
pred_sorted = tf.argsort(y_pred, axis=1, direction='DESCENDING')
n_evt = tf.shape(y_true)[0]
evt_id = tf.range(n_evt)
matches_vec = []
for n in range(n_positions):
index = tf.transpose(
tf.stack([evt_id,
tf.reshape(pred_sorted[:, n], shape=(n_evt, ))]))
matches_vec.append(tf.gather_nd(y_true, index))
matches_sum = tf.add_n(matches_vec)
valid = tf.cast(tf.equal(matches_sum, n_exp), tf.float32)
if do_ratio:
n_valid = tf.reduce_sum(valid)
ratio = n_valid / tf.cast(n_evt, tf.float32)
if return_num:
ratio = ratio.numpy()
return ratio
return valid
def nucleus_sampling(logits, p=0.9):
sorted_logits = tf.sort(logits, direction='DESCENDING')
sorted_indices = tf.argsort(logits, direction='DESCENDING')
cumulative_probs = tf.cumsum(tf.nn.softmax(sorted_logits))
t_sorted_indices_to_remove = cumulative_probs > p
''' Shift the indices to the right to keep also the first token above the threshold '''
indices = tf.range(1, tf.shape(logits)[0], 1)
sorted_indices_to_remove = tf.scatter_nd(tf.expand_dims(indices, 1), t_sorted_indices_to_remove[:-1], logits.shape)
indices_to_remove = tf.boolean_mask(sorted_indices, sorted_indices_to_remove)
t = tf.ones(tf.shape(indices_to_remove)[0], dtype=tf.bool)
to_remove = tf.scatter_nd(tf.expand_dims(indices_to_remove, 1), t, logits.shape)
logits = tf.where(
to_remove,
tf.ones_like(logits, dtype=logits.dtype) * -1e10,
logits
)
sample = tf.multinomial(tf.expand_dims(logits, 0), num_samples=1, output_dtype=tf.int32)
return tf.reduce_sum(sample)
def compute_patches(image,
label,
ps=Config.input.patch_size,
ss=Config.input.sample_size):
"""Computes patches to be used as input to the Neural Network.
If number of patches exceeds the sample_size (ss), the
patches with most information are selected"""
image_shape = tf.shape(image)
h, w = tf.gather(image_shape, 0), tf.gather(image_shape, 1)
pad_h = (ps - h % ps)
pad_w = (ps - w % ps)
pu = tf.random.uniform(shape=(), minval=0, maxval=pad_h, dtype=tf.int32)
pd = pad_h - pu
pl = tf.random.uniform(shape=(), minval=0, maxval=pad_w, dtype=tf.int32)
pr = pad_w - pl
image = tf.pad(image, ((pu, pd), (pl, pr), (0, 0)), constant_values=255)
image_shape = tf.shape(image)
h, w = tf.gather(image_shape, 0), tf.gather(image_shape, 1)
image = tf.reshape(image, (h // ps, ps, w // ps, ps, 3))
image = tf.transpose(image, (0, 2, 1, 3, 4))
image = tf.reshape(image, (-1, ps, ps, 3))
num_images = tf.gather(tf.shape(image), 0)
info = tf.reduce_sum(
tf.cast(tf.reshape(image, (num_images, -1)), tf.int32), 1)
if tf.gather(tf.shape(info), 0) < ss:
image = tf.pad(image, ((0, ss - num_images), (0, 0), (0, 0), (0, 0)),
constant_values=255)
paddings = tf.zeros(ss - num_images,
dtype=tf.int32) + (ps**2 * 3 * 255)
info = tf.concat((info, paddings), axis=0)
indices = tf.gather(tf.argsort(info), tf.range(ss))
image = tf.gather(image, indices)
return image, label
def evaluation_step(x, y):
predictions = model(x, training=False)
loss = compiled_loss(y, predictions)
for metric in metrics:
metric.update_state(y, predictions)
predictions = tf.reshape(predictions, [-1])
predictions = tf.cast(predictions, tf.float64)
display_ids = x[DISPLAY_ID_COLUMN]
display_ids = tf.reshape(display_ids, [-1])
labels = tf.reshape(y, [-1])
sorted_ids = tf.argsort(display_ids)
display_ids = tf.gather(display_ids, indices=sorted_ids)
predictions = tf.gather(predictions, indices=sorted_ids)
labels = tf.gather(labels, indices=sorted_ids)
_, display_ids_idx, display_ids_ads_count = tf.unique_with_counts(
display_ids, out_idx=tf.int64)
pad_length = 30 - tf.reduce_max(display_ids_ads_count)
preds = tf.RaggedTensor.from_value_rowids(predictions,
display_ids_idx).to_tensor()
labels = tf.RaggedTensor.from_value_rowids(
labels, display_ids_idx).to_tensor()
labels_mask = tf.math.reduce_max(labels, 1)
preds_masked = tf.boolean_mask(preds, labels_mask)
labels_masked = tf.boolean_mask(labels, labels_mask)
labels_masked = tf.argmax(labels_masked, axis=1, output_type=tf.int32)
labels_masked = tf.reshape(labels_masked, [-1, 1])
preds_masked = tf.pad(preds_masked, [(0, 0), (0, pad_length)])
_, predictions_idx = tf.math.top_k(preds_masked, 12)
indices = tf.math.equal(predictions_idx, labels_masked)
indices_mask = tf.math.reduce_any(indices, 1)
masked_indices = tf.boolean_mask(indices, indices_mask)
res = tf.argmax(masked_indices, axis=1)
ap_matrix = tf.divide(1, tf.add(res, 1))
ap_sum = tf.reduce_sum(ap_matrix)
shape = tf.cast(tf.shape(indices)[0], tf.float64)
display_id_counter.assign_add(shape)
streaming_map.assign_add(ap_sum)
return loss
def __init__(self, front: TensorType, threshold: TensorType | float = 0):
"""
:param front: non-dominated pareto front.
:param threshold: a threshold used to screen out cells in partition : when its volume is
below this threshold, its rejected directly in order to be more computationally
efficient, if setting above 0, this partition strategy tends to return an
approximated partition.
"""
tf.debugging.assert_equal(
tf.cast(tf.reduce_sum(tf.abs(non_dominated(front)[1])),
dtype=front.dtype),
tf.zeros(shape=1, dtype=front.dtype),
message=f"\ninput {front} "
f"contains dominated points",
)
self.front = tf.gather_nd(front, tf.argsort(front[:, :1],
axis=0)) # sort
self.front = front
self._bounds = self._get_bound_index(threshold)
def search(self, query_vector, top_k=None):
dot_query_vector = tf.matmul(query_vector,
query_vector,
transpose_b=True)
squared_norms_query_vector = tf.expand_dims(
tf.linalg.diag_part(dot_query_vector), 0)
dot_product = tf.reduce_sum(self.embeddings * query_vector, axis=1)
distances = tf.maximum(
self.squared_norms_embeddings + squared_norms_query_vector -
2 * dot_product, 0)
sorted_indices = tf.argsort(distances)
if top_k:
sorted_indices = sorted_indices[..., :top_k]
nearest_labels = tf.reshape(tf.gather(self.labels, sorted_indices),
shape=[-1, 1])
nearest_distances = tf.reshape(tf.gather(distances[0], sorted_indices),
shape=[-1, 1])
return nearest_distances[..., 0], nearest_labels[...,
0], sorted_indices[0]
def sample_top_p(logits, top_p):
"""Chooses most probable logits with cumulative probabilities upto top_p.
Sets the remaining logits to negative infinity.
Args:
logits: Input logits for next token.
top_p: Float tensor with a value >=0 and < 1.0
Returns:
Logits with top_p filtering applied.
"""
sorted_indices = tf.argsort(logits, direction="DESCENDING")
# Flatten logits as tf.gather on TPU needs axis to be compile time constant.
logits_shape = decoding_module.shape_list(logits)
range_for_gather = tf.expand_dims(tf.range(0, logits_shape[0]), axis=1)
range_for_gather = tf.tile(range_for_gather * logits_shape[1],
[1, logits_shape[1]]) + sorted_indices
flattened_logits = tf.reshape(logits, [-1])
flattened_sorted_indices = tf.reshape(range_for_gather, [-1])
sorted_logits = tf.reshape(
tf.gather(flattened_logits, flattened_sorted_indices),
[logits_shape[0], logits_shape[1]])
cumulative_probs = tf.cumsum(tf.nn.softmax(sorted_logits, axis=-1),
axis=-1)
# Remove tokens with cumulative probability above the threshold.
sorted_indices_to_remove = cumulative_probs > top_p
# Shift the indices to the right to keep the first token above threshold.
sorted_indices_to_remove = tf.roll(sorted_indices_to_remove, 1, axis=-1)
sorted_indices_to_remove = tf.concat([
tf.zeros_like(sorted_indices_to_remove[:, :1]),
sorted_indices_to_remove[:, 1:]
], -1)
# Scatter sorted indices to original indexes.
indices_to_remove = scatter_values_on_batch_indices(
sorted_indices_to_remove, sorted_indices)
top_p_logits = set_tensor_by_indices_to_value(logits, indices_to_remove,
np.NINF)
return top_p_logits
def train_step(self, imgs, d_steps, g_steps, pruning):
batch_size = len(imgs)
# Adversarial ground truths
valid = np.ones((batch_size, 1))
fake = np.zeros((batch_size, 1))
# Sample noise and generate a batch of new images
for _ in range(d_steps):
noise = self.prior(batch_size)
gen_imgs = self.generator.predict(noise)
# fakes_val = gen_imgs[:10]
# ------------------------------- PRUNING --------------------------------
if pruning != 1:
gen_scores = self.discriminator.predict(gen_imgs)
gen_imgs_sorted_ids = tf.argsort(gen_scores,
axis=0,
direction='DESCENDING')
leftover = int(gen_imgs_sorted_ids.shape[0] // (1 / pruning))
fake_output_best_ids = gen_imgs_sorted_ids[:leftover, :]
gen_imgs = tf.gather_nd(gen_imgs, fake_output_best_ids)
fake = np.zeros((gen_imgs.shape[0], 1))
# ------------------------------------------------------------------------
# Train the discriminator (real classified as ones and generated as zeros)
d_loss_real = self.discriminator.train_on_batch(imgs, valid)
d_loss_fake = self.discriminator.train_on_batch(gen_imgs, fake)
d_loss = 0.5 * np.add(d_loss_real, d_loss_fake)
#assert np.allclose(self.generator.predict(noise)[:10], fakes_val), "D update effects G"
# Train Generator
g_loss = -1
for _ in range(g_steps):
# Train the generator (wants discriminator to mistake images as real)
y_val = self.discriminator.predict(imgs[:10])
g_loss = self.combined.train_on_batch(noise, valid)
def randomSearch(encoder, decoder, final_layer, test_inp, k):
"""Helper method of inferenceRandomSearch """
score = 0
enc_inp, dec_inp, padding_mask, combined_mask = test_inp
enc_output = encoder(inputs=enc_inp, mask=padding_mask, training=False)
for slen in range(100):
dec_output, attention_weights = decoder(inputs=dec_inp,
enc_output=enc_output,
look_ahead_mask=combined_mask,
padding_mask=padding_mask,
training=False)
final_output = final_layer(inputs=dec_output)
# select the last word from the seq_len dimension
predictions = final_output[:, -1:, :]
# Sort id and scores
bestId = tf.argsort(predictions)
bestScore = tf.sort(predictions)
# Take one of the k best scores
predicted_id = []
for idRow, scoreRow in zip(bestId, bestScore):
for idR, scoreR in zip(idRow, scoreRow):
# At beggining of sequence, allow for randomness, but after, always pick best score
if slen > 15:
k = 1
# Get one of the k best ID and its corresponding score
randomIndex = random.randint(1, k)
predicted_id.append([idR[-randomIndex]])
score += scoreR[-randomIndex]
# Convert to tensor
predicted_id = tf.convert_to_tensor(predicted_id)
dec_inp = tf.concat([dec_inp, predicted_id], axis=-1)
return dec_inp, score
def azimAvg_tensor(image, center=None):
"""
Calculate the azimuthally averaged power spectrum (1D), for a batch of image tensors.
image - The image tensor, [N,C,H,W] format
center - The [x,y] pixel coordinates used as the center. The default is
None, which then uses the center of the image (including
fractional pixels).
"""
batch, channel, height, width = image.shape.as_list()
# Calculate the indices from the image
y, x = np.indices([height, width])
y = np.tile(y, (batch, channel, 1, 1))
x = np.tile(x, (batch, channel, 1, 1))
if not center:
center = np.array([(x.max()-x.min())/2.0, (x.max()-x.min())/2.0])
r = np.hypot(x - center[0], y - center[1])
# Get sorted radii
ind = tf.argsort(tf.reshape(r, (batch, channel, -1,)))
r_sorted = tf.gather(tf.reshape(r, (batch, channel, -1,)), ind, batch_dims=2)
i_sorted = tf.gather(tf.reshape(image, (batch, channel, -1,)), ind, batch_dims=2)
# Get the integer part of the radii (bin size = 1)
r_int = tf.cast(r_sorted, tf.int32)
# Find all pixels that fall within each radial bin.
deltar = r_int[:,:,1:] - r_int[:,:,:-1] # Assumes all radii represented
rind = tf.reshape(tf.where(deltar)[:,2], (batch, -1)) # location of changes in radius
rind = tf.expand_dims(rind, axis=1)
nr = tf.cast(rind[:,:,1:] - rind[:,:,:-1], tf.float32) # number of radius bin
# Cumulative sum to figure out sums for each radius bin
csum = tf.cumsum(i_sorted, axis=-1)
tbin = tf.gather(csum, rind[:,:,1:], batch_dims=2) - tf.gather(csum, rind[:,:,:-1], batch_dims=2)
radial_prof = tbin / nr
return radial_prof
def inverse(x, w, b, v):
tf.debugging.assert_equal(b[0],tf.zeros([1,],dtype=tf.float64))
w = tf.cast(w, dtype=tf.float64)
b = tf.cast(b, dtype=tf.float64)
v = tf.cast(v, dtype=tf.float64)
# Calculate v0
if w.get_shape() != v.get_shape():
v0 = -(tf.reshape(b,[1,H]) @ v)[0]
x = tf.dtypes.cast(x, tf.float64)
# num_layer = w.shape[1]
b_over_w = - b / w # the negative sign is important
b_over_w = tf.reshape(b_over_w, [-1])# change it to row vector
# Then we need to compute range for the inverse function
r_x = tf.reshape(
[tf.matmul( ( tf.nn.relu(w * b_over_w[i] + b) ), v) + v0 for i in range(1)],
[-1]
)
# pad r_x with 0 and inf
# right side gives us the right index
zero = tf.constant([0.], dtype=tf.float64)
pos = tf.searchsorted(tf.sort(tf.concat([zero, r_x, [np.inf]],0)), x,side='right') # get the interval index
# tf.print('pos',pos)
pos = tf.reshape(pos,[]) # get only numerical value
# change it to row vector
v = tf.reshape(v, [-1])
w = tf.reshape(w, [-1])
index = tf.argsort(r_x)
# sort according to the index
v_b = tf.gather(v * b, index)
v_w = tf.gather(v * w, index)
num = x - v0 - tf.reduce_sum(v_b[:(pos - 1)])
deo = tf.reduce_sum(v_w[:(pos - 1)])
with tf.control_dependencies([
tf.debugging.assert_positive(num, message='inverse num'),
tf.debugging.assert_positive(deo, message='inverse deo'),
]):
return tf.math.divide_no_nan(num, deo)[0]
def segment_top_k(x, I, ratio):
"""
Returns indices to get the top K values in x segment-wise, according to
the segments defined in I. K is not fixed, but it is defined as a ratio of
the number of elements in each segment.
:param x: a rank 1 Tensor;
:param I: a rank 1 Tensor with segment IDs for x;
:param ratio: float, ratio of elements to keep for each segment;
:return: a rank 1 Tensor containing the indices to get the top K values of
each segment in x.
"""
rt = tf.RaggedTensor.from_value_rowids(x, I)
row_lengths = rt.row_lengths()
dense = rt.to_tensor(default_value=-np.inf)
indices = tf.cast(tf.argsort(dense, direction="DESCENDING"), tf.int64)
row_starts = tf.cast(rt.row_starts(), tf.int64)
indices = indices + tf.expand_dims(row_starts, 1)
row_lengths = tf.cast(
tf.math.ceil(ratio * tf.cast(row_lengths, tf.float32)), tf.int32)
return tf.RaggedTensor.from_tensor(indices, row_lengths).values
def get_metrics(data, model, args, k: int):
dataset = SessionDataset(data)
loader = SessionDataLoader(dataset, batch_size=args.batch_size)
recall_list, mrr_list = [], []
total_step = len(data) - data['SessionId'].nunique()
for inputs, label, mask in tqdm(loader, total=total_step // args.batch_size, desc='Evaluation', mininterval=1):
reset_hidden_states(model, mask)
input_ohe = to_categorical(inputs, num_classes=args.num_items)
input_ohe = np.expand_dims(input_ohe, axis=1)
pred = model.predict(input_ohe, batch_size=args.batch_size)
pred_arg = tf.argsort(pred, direction='DESCENDING')
length = len(inputs)
recall_list.extend([recall_k(pred_arg[i], label[i], k) for i in range(length)])
mrr_list.extend([mrr_k(pred_arg[i], label[i], k) for i in range(length)])
recall, mrr = np.mean(recall_list), np.mean(mrr_list)
return recall, mrr
def calc_top_k(self, class_index, true_movement):
# get the top companies probabilities
top_indices = tf.where(tf.equal(self.model.prediction, class_index),
name='top_indices')
top_prob = tf.math.reduce_sum(tf.gather(self.model.prob, top_indices),
axis=1)[:, class_index]
# calc top k by model
k = self.model.params.num_companies * self.top_k_percent
# in case k passes the total companies of the movement, summary all of them
corrected_k = k if top_indices.shape[0] >= k else tf.shape(
top_indices, out_type=tf.int32)[0]
top_k_indices = tf.argsort(top_prob,
direction="DESCENDING")[:corrected_k]
# finally, calculate success ratio of top k by true movement of the companies
return tf.math.divide(
tf.reduce_sum(
tf.cast(tf.gather(true_movement, top_k_indices),
dtype=tf.float32)), tf.cast(corrected_k, tf.float32))
def nucleus(logits, p):
indices = tf.argsort(logits, direction='DESCENDING', axis=1)
vals = tf.sort(tf.nn.softmax(logits, axis=1),
direction='DESCENDING',
axis=1)
tail_ids = tf.cast(
tf.argmax(tf.cast(tf.cumsum(vals, axis=1) > p, tf.int8), axis=1),
tf.int32)
logit_inds = tf.stack([tf.range(0, logits.shape[0].value), tail_ids],
axis=1)
tail_min_vals = tf.expand_dims(tf.gather_nd(logits, logit_inds), 1)
return tf.where(
logits <
tail_min_vals, # if it is worse than this lower bound. I can do this for my ones too! does it for each batch simultaneously.
tf.ones_like(logits, dtype=logits.dtype) * -1e10,
logits,
)
'''while_condition = lambda i, logits_to_return: tf.less(i, logits.shape[0].value)
def basic_ev(rand, genotypes, fitness, n_parents=8, cohort_factor=.8, mu=0.08):
''' {n_parents} parents are preserved as an elitist cohort.
The children's mutation rates will be calculated as follows:
The ith cohort of children has a mutation rate of ((i + 1)**{cohort_factor} - 1)*{mu}'''
n_cohorts = (genotypes.shape[0] // n_parents) + 1
elite = tf.argsort(-fitness)[:n_parents]
tf.print(elite, summarize=-1)
indices = tf.concat([elite] * n_cohorts, axis=0)[:genotypes.shape[0]]
parents_gene = tf.gather(genotypes, indices)
mutation_factor = tf.reshape(
tf.constant([((i // n_parents + 1)**cohort_factor - 1)
for i in range(genotypes.shape[0])]),
(genotypes.shape[0],
1)) # each cohort mutates a little more than the last
children_gene = parents_gene + rand.normal(
parents_gene.shape) * mutation_factor * mu
return children_gene
def _bounds_2d(front: TensorType) -> BoundedVolumes:
# this assumes the Pareto set has been sorted in ascending order on the first
# objective, which implies the second objective is sorted in descending order
len_front, number_of_objectives = front.shape
pf_ext_idx = tf.concat(
[
tf.zeros([1, number_of_objectives], dtype=tf.int32),
tf.argsort(front, axis=0) + 1,
tf.ones([1, number_of_objectives], dtype=tf.int32) * len_front + 1,
],
axis=0,
)
range_ = tf.range(len_front + 1)[:, None]
lower = tf.concat([range_, tf.zeros_like(range_)], axis=-1)
upper = tf.concat([range_ + 1, pf_ext_idx[::-1, 1:][: pf_ext_idx[-1, 0]]], axis=-1)
return BoundedVolumes(lower, upper)
def partial_log_likelihood(prediction, t, y):
"""
calculate cox loss
:param prediction: prediction of model
:param t: event happen at the 't'th day
:param y: true label
:return:
"""
risk = tf.reshape(prediction, [-1])
time = tf.reshape(t, [-1])
E = tf.reshape(y, [-1])
sort_idx = tf.argsort(time, direction='DESCENDING')
E = tf.gather(E, sort_idx)
risk = tf.gather(risk, sort_idx)
hazard_ratio = tf.exp(risk)
log_risk = tf.math.log(tf.cumsum(hazard_ratio))
uncensored_likelihood = risk - log_risk
censored_likelihood = tf.multiply(uncensored_likelihood, E)
neg_likelihood = -tf.reduce_sum(censored_likelihood) * 0.01
return neg_likelihood
def quantization_topology(input, output, lmb):
N = input.shape[0]
adjacency_matrix = np.zeros((N, N))
D = squared_dist(input, tf.transpose(output))
d_min = tf.math.reduce_min(D, axis=1)
s = tf.argsort(D.numpy(), axis=1)[:, :2].numpy()
for i in range(N):
idx = s[:, 0] == i
si = s[idx]
if len(si) > 0:
for j in set(si[:, 1]):
k = sum(si[:, 1] == j)
adjacency_matrix[i, j] = 1
adjacency_matrix[j, i] = 1
E = tf.convert_to_tensor(adjacency_matrix, np.float32)
Q = tf.norm(d_min)
E = tf.norm(E, 2)
cost = Q + lmb * E
return cost
def nucleus_sampling(logits_BxN, top_p):
sort_indices_BxN = tf.argsort(logits_BxN, axis=-1, direction='DESCENDING')
probs_BxN = tf.gather(tf.nn.softmax(logits_BxN),
sort_indices_BxN,
batch_dims=1)
cumprobs_BxN = tf.cumsum(probs_BxN, axis=-1, exclusive=True)
# The top 1 candidate always will not be masked.
# This way ensures at least 1 indices will be selected.
sort_mask_BxN = tf.cast(tf.greater(cumprobs_BxN, top_p), logits_BxN.dtype)
batch_indices_BxN = tf.tile(
tf.expand_dims(tf.range(tf.shape(logits_BxN)[0]), axis=-1),
[1, tf.shape(logits_BxN)[1]],
)
top_p_mask_BxN = tf.scatter_nd(
tf.stack([batch_indices_BxN, sort_indices_BxN], axis=-1),
sort_mask_BxN,
tf.shape(logits_BxN),
)
logits_BxN -= top_p_mask_BxN * logits_BxN.dtype.max
return logits_BxN
def calibration_plot(model, x, y, ece):
py_x = tf.nn.softmax(model(x))
p = tf.max(py_x, axis=1)
hat_y = tf.argmax(py_x, axis=1)
y = tf.argmax(y, axis=1)
acc = tf.cast(hat_y == y, tf.float32)
idx = tf.argsort(p)
p = p[idx]
acc = acc[idx]
plt.title(f'Calibration {model.name}: {ece}')
plt.ylabel('Frequency')
plt.xlabel('ACC/Conf')
plt.xlim([0, 1])
plt.hist(acc, bins=20, color='blue', label='accuracy', alpha=.5)
plt.hist(p, bins=20, color='red', label='confidence', alpha=.5)
plt.legend()
return 0
def score_matrix_to_binary_ranking(similarity_matrix,
query_labels,
candidate_labels,
remove_top1=False):
query_labels = tf.reshape(query_labels, [-1, 1])
candidate_labels = tf.reshape(candidate_labels, [-1, 1])
pair_signs = tf.cast(
tf.equal(query_labels, tf.transpose(candidate_labels)), tf.float32)
index_ranking = tf.argsort(similarity_matrix,
axis=1,
direction="DESCENDING")
if remove_top1:
index_ranking = index_ranking[:, 1:]
gather_idx = arg_to_gather_nd(index_ranking)
binary_ranking = tf.reshape(tf.gather_nd(pair_signs, gather_idx),
index_ranking.shape)
return binary_ranking
def NMS(self, rectangles, threshold, type):
if rectangles.shape[0] == 0: return rectangles
wh = rectangles[..., 2:4] - rectangles[..., 0:2] + tf.constant(
[1, 1], dtype=tf.float32)
area = wh[..., 0] * wh[..., 1]
conf = rectangles[..., 4]
I = tf.argsort(conf, direction='DESCENDING')
i = 0
while i < I.shape[0]:
idx = I[i]
cur_upper_left = rectangles[idx, 0:2]
cur_down_right = rectangles[idx, 2:4]
cur_area = area[idx]
following_idx = I[i + 1:]
following_upper_left = tf.gather(rectangles[..., 0:2],
following_idx)
following_down_right = tf.gather(rectangles[..., 2:4],
following_idx)
following_area = tf.gather(area, following_idx)
max_upper_left = tf.math.maximum(cur_upper_left,
following_upper_left)
min_down_right = tf.math.minimum(cur_down_right,
following_down_right)
intersect_wh = min_down_right - max_upper_left
intersect_wh = tf.where(tf.math.greater(intersect_wh, 0),
intersect_wh, tf.zeros_like(intersect_wh))
intersect_area = intersect_wh[..., 0] * intersect_wh[..., 1]
if type == 'iom':
overlap = intersect_area / tf.math.minimum(
cur_area, following_area)
else:
overlap = intersect_area / (cur_area + following_area -
intersect_area)
indices = tf.where(tf.less(overlap, threshold))
following_idx = tf.gather_nd(following_idx, indices)
I = tf.concat([I[:i + 1], following_idx], axis=0)
i += 1
# result_rectangles.shape = (picked target num, 5)
result_rectangles = tf.gather(rectangles, I)
return result_rectangles
def combine_results(name2results, keys):
"""Combines network outputs.
Args:
name2results: Dict. For each object results, `z_vals` is required.
keys: [str]. A list of keys to combine results over.
Returns:
results: Dict. Combined results.
"""
# Collect z values across all objects.
z_vals_list = []
for _, results in name2results.items():
z_vals_list.append(results['z_vals'])
# Concatenate lists of object results into a single tensor.
z_vals = tf.concat(z_vals_list, axis=-1) # [R, S*O]
# Compute the argsort indices.
z_argsort_indices = tf.argsort(z_vals, -1) # [R, S*O]
n_rays, n_samples = tf.shape(z_vals)[0], tf.shape(z_vals)[1]
gather_indices = tf.range(n_rays)[:, None] # [R, 1]
gather_indices = tf.tile(gather_indices, [1, n_samples]) # [R, S]
gather_indices = tf.concat(
[gather_indices[Ellipsis, None], z_argsort_indices[Ellipsis, None]],
axis=-1)
results = {}
for k in keys:
if k == 'z_vals':
v_combined = z_vals
else:
v_list = [r[k] for r in name2results.values()]
v_combined = tf.concat(v_list, axis=1) # [R, S*O, K]
# Sort the tensors.
v_sorted = tf.gather_nd( # [R, S, K]
params=v_combined, # [R, S, K]
indices=gather_indices) # [R, S, 2]
results[k] = v_sorted
return results
def __init__(self, size):
self.graph = tf.Graph()
self.session = tf.InteractiveSession(graph=self.graph)
self.x = tf.placeholder(tf.float32, [None, size * size])
self.y = tf.placeholder(tf.float32, [None, size * size])
self.w_conv1 = self.weight_variable([5, 5, 1, 32])
self.b_conv1 = self.bias_variable([32])
self.x_image = tf.reshape(self.x, [-1, size, size, 1])
self.conv1 = tf.nn.relu(
self.conv2d(self.x_image, self.w_conv1) + self.b_conv1)
self.pool1 = self.max_pool(self.conv1)
self.w_conv2 = self.weight_variable([5, 5, 32, 64])
self.b_conv2 = self.bias_variable([64])
self.conv2 = tf.nn.relu(
self.conv2d(self.pool1, self.w_conv2) + self.b_conv2)
self.pool2 = self.max_pool(self.conv2)
max_size = self.pool2.shape[1].value
self.w_fc1 = self.weight_variable([max_size * max_size * 64, 1024])
self.b_fc1 = self.bias_variable([1024])
self.pool2_flat = tf.reshape(self.pool2,
[-1, max_size * max_size * 64])
self.fc1 = tf.nn.relu(
tf.matmul(self.pool2_flat, self.w_fc1) + self.b_fc1)
self.w_fc2 = self.weight_variable([1024, size * size])
self.b_fc2 = self.bias_variable([size * size])
self.y_conv = tf.nn.softmax(
tf.matmul(self.fc1, self.w_fc2) + self.b_fc2)
self.sorted_pred = tf.argsort(self.y_conv, direction="DESCENDING")
self.cross_entropy = -tf.reduce_sum(self.y * tf.log(self.y_conv))
self.train_step = tf.train.AdamOptimizer(1e-4).minimize(
self.cross_entropy)
tf.summary.scalar("loss", self.cross_entropy)
self.merged_summary = tf.summary.merge_all()
self.saver = tf.train.Saver()
self.session.run(tf.global_variables_initializer())
correction_prediction = tf.equal(tf.argmax(self.y_conv, 1),
tf.argmax(self.y, 1))
def get_batch(self, model, dist, self_check=False):
self.batch_iter += 1
if self.batch_iter % self.hard_every:
return super(DssmTrainerWithCustomTokenizerAndHards, self).get_batch(model, self_check=self_check)
print("HardBatch!")
sub_set = sorted(np.random.choice(self.X_train.shape[0], size=self.batch_size // self.hard_neg_k,
replace=False))
# print(sub_set)
# self.doc_tokenizer.from_raw([self.train_doc_tokens[i] for i in range(self.X_train.shape[0])])
q_t = self.query_tokenizer.from_raw([self.train_query_tokens[i] for i in sub_set])
train_d_emb = model.call_doc(self.train_doc_tokens_tf)
all_q_emb = model.call_query(q_t)
for q_emb in all_q_emb:
q_emb_r = tf.reshape(q_emb, (1, -1))
distances = dist(q_emb_r, train_d_emb)
# print(distances)
top = tf.argsort(distances[0])[:self.hard_neg_k - 1]
# print(top)
for cand in top.numpy():
sub_set.append(cand)
sub_set = np.unique(sub_set)
# print(len(sub_set))
d_t = self.doc_tokenizer.from_raw([self.train_doc_tokens[i] for i in sub_set])
q_t = self.query_tokenizer.from_raw([self.train_query_tokens[i] for i in sub_set])
if self_check:
sub_set = self.X_train[sub_set]
q_t_ = self.query_tokenizer(sub_set[:,0])
d_t_ = self.doc_tokenizer(sub_set[:,1])
d_t__, q_t__, d_t_, q_t_ = map(tf.sparse.to_dense, [d_t, q_t, d_t_, q_t_])
assert np.sum((q_t__ - q_t_) ** 2) < 1e-9
assert np.sum((d_t__ - d_t_) ** 2) < 1e-9
return q_t, d_t
def GetIOUNMS(
boxes,
scores,
max_output_size,
iou_threshold=0.5,
iou_type='iou'):
'''通过NMS筛选框'''
# 分数倒序下标
scores_sort_indexes = tf.argsort(scores, direction='DESCENDING')
# 排序后的框
boxes_sort = tf.gather(boxes, scores_sort_indexes)
# NMS后的下标
result_indexes = tf.TensorArray(tf.int32, 0, dynamic_size=True)
def boxes_foreach(idx, result_indexes, boxes_sort, scores_sort_indexes):
# 已达到最大输出框数量
if idx >= max_output_size:
return -1, result_indexes, boxes_sort, scores_sort_indexes
boxes_num = tf.shape(boxes_sort)[0]
# 已遍历所有框
if boxes_num == 0:
return -1, result_indexes, boxes_sort, scores_sort_indexes
# 取最高分的box
boxes_top = boxes_sort[0:1, :]
indexes_top = scores_sort_indexes[0]
if boxes_num > 1:
# 计算IOU
boxes_other = boxes_sort[1:, :]
indexes_other = scores_sort_indexes[1:]
iou = GetIOU(boxes_top, boxes_other, iou_type)
# 过滤掉小于阈值的数据
iou_mask = iou < iou_threshold
boxes_sort = tf.boolean_mask(boxes_other, iou_mask)
scores_sort_indexes = tf.boolean_mask(indexes_other, iou_mask)
result_indexes = result_indexes.write(idx, indexes_top)
return idx + 1, result_indexes, boxes_sort, scores_sort_indexes
else:
result_indexes = result_indexes.write(idx, indexes_top)
return -1, result_indexes, boxes_sort, scores_sort_indexes
_, result_indexes, _, _ = tf.while_loop(lambda i, *args: tf.math.not_equal(i, -1), boxes_foreach, [0, result_indexes, boxes_sort, scores_sort_indexes])
result_indexes = result_indexes.stack()
return result_indexes
def predict(mode):
PREDS = []
CONFS = []
NUM_SELECT = 10
batch_size = 320
for batch_id, X in enumerate(batch_generator(mode,batch_size=batch_size,loop=False)):
x = X[0]
print("Predicting {} - Batch {}".format(mode,batch_id))
pred = model.predict_on_batch(x)
if batch_id == 0:
print(pred)
PREDS.append(tf.argsort(pred,axis=-1)[:,-NUM_SELECT:])
CONFS.append(tf.sort(pred,axis=-1)[:,-NUM_SELECT:])
PREDS = np.concatenate(PREDS,axis=0)
CONFS = np.concatenate(CONFS,axis=0)
PREDS = np.concatenate([PREDS,CONFS],axis=1)
cols = ['pred_{}'.format(k) for k in range(NUM_SELECT)] + \
['conf_{}'.format(k) for k in range(NUM_SELECT)]
fname = os.path.join(DATA_DIR,'dom_pred_{}.csv'.format(mode))
pd.DataFrame(PREDS,index=range(PREDS.shape[0]),columns=cols).to_csv(fname)
def update_state(self, y_true, y_pred, sample_weight=None):
"""Update state of the metric.
Args:
y_true: the true labels with shape [batch_size, 1].
y_pred: model output, which is the similarity matrix with shape
[batch_size, label_embedding_vocab_size] between context and full vocab
label embeddings.
sample_weight: Optional weighting of each example. Defaults to 1.
"""
similarities = y_pred
label_indices = tf.expand_dims(tf.cast(y_true, dtype=tf.int32), -1)
# Get the indices of similar labels in sorted order for each context.
sorted_similarities = tf.argsort(similarities,
axis=-1,
direction='DESCENDING')
ranks = tf.where(tf.equal(sorted_similarities, label_indices))[:, -1]
ranks = tf.keras.backend.cast(ranks, 'float32')
super().update_state(ranks, sample_weight=sample_weight)
