def compute_loss(retriever_logits, retriever_correct, reader_logits,
reader_correct):
"""Compute loss."""
# []
retriever_loss = marginal_log_loss(retriever_logits, retriever_correct)
# []
reader_loss = marginal_log_loss(tf.reshape(reader_logits, [-1]),
tf.reshape(reader_correct, [-1]))
# []
any_retrieved_correct = tf.reduce_any(retriever_correct)
any_reader_correct = tf.reduce_any(reader_correct)
retriever_loss *= tf.cast(any_retrieved_correct, tf.float32)
reader_loss *= tf.cast(any_reader_correct, tf.float32)
loss = retriever_loss + reader_loss
tf.summary.scalar("num_read_correct",
tf.reduce_sum(tf.cast(reader_correct, tf.int32)))
tf.summary.scalar("reader_loss", tf.reduce_mean(reader_loss))
tf.summary.scalar("retrieval_loss", tf.reduce_mean(retriever_loss))
# []
loss = tf.reduce_mean(loss)
return loss
def search(self, initial_ids, initial_cache):
"""Beam search for sequences with highest scores."""
state, state_shapes = self._create_initial_state(
initial_ids, initial_cache)
finished_state = tf.while_loop(self._continue_search,
self._search_step,
loop_vars=[state],
shape_invariants=[state_shapes],
parallel_iterations=1,
back_prop=False)
finished_state = finished_state[0]
alive_seq = finished_state[_StateKeys.ALIVE_SEQ]
alive_log_probs = finished_state[_StateKeys.ALIVE_LOG_PROBS]
finished_seq = finished_state[_StateKeys.FINISHED_SEQ]
finished_scores = finished_state[_StateKeys.FINISHED_SCORES]
finished_flags = finished_state[_StateKeys.FINISHED_FLAGS]
# Account for corner case where there are no finished sequences for a
# particular batch item. In that case, return alive sequences for that batch
# item.
finished_seq = tf.where(tf.reduce_any(finished_flags, 1), finished_seq,
alive_seq)
finished_scores = tf.where(tf.reduce_any(finished_flags, 1),
finished_scores, alive_log_probs)
return finished_seq, finished_scores
def compute_match_stats(predictions, labels):
"""Compute statistics that are used to compute evaluation metrics.
Args:
predictions: <int32> [batch_size]
labels: <int32> [batch_size, num_annotators]
Returns:
numerator: <float> [batch_size]
recall_weights: <float> [batch_size]
precision_weights: <float> [batch_size]
"""
# <int32> [batch_size, num_labels]
thresholded_labels = compute_thresholded_labels(labels)
non_null_mask = tf.greater(thresholded_labels, 0)
# <bool> [batch_size, num_labels]
exact_match = tf.equal(tf.expand_dims(predictions, -1), thresholded_labels)
non_null_match = tf.logical_and(exact_match, non_null_mask)
# <float> [batch_size]
non_null_match = tf.to_float(tf.reduce_any(non_null_match, axis=1))
non_null_gold = tf.to_float(tf.reduce_any(non_null_mask, 1))
non_null_predictions = tf.to_float(tf.greater(predictions, 0))
return non_null_match, non_null_gold, non_null_predictions
def compute_eval_metrics(labels, predictions, retriever_correct,
reader_correct):
"""Compute eval metrics."""
# []
exact_match = tf.gather(
tf.gather(reader_correct, predictions["block_index"]),
predictions["candidate"])
def _official_exact_match(predicted_answer, references):
is_correct = eval_utils.is_correct(
answers=[six.ensure_text(r, errors="ignore") for r in references],
prediction=six.ensure_text(predicted_answer, errors="ignore"),
is_regex=False)
return is_correct
official_exact_match = tf.py_func(func=_official_exact_match,
inp=[predictions["answer"], labels],
Tout=tf.bool)
eval_metric_ops = dict(
exact_match=tf.metrics.mean(exact_match),
official_exact_match=tf.metrics.mean(official_exact_match),
reader_oracle=tf.metrics.mean(tf.reduce_any(reader_correct)))
for k in (5, 10, 50, 100, 500, 1000, 5000):
eval_metric_ops["top_{}_match".format(k)] = tf.metrics.mean(
tf.reduce_any(retriever_correct[:k]))
return eval_metric_ops
def _batch_stitch(features, mean_length=4.0, stddev=2.0):
"""Stitches a batch of single-step data to a batch of multi-step data."""
batch_size = common_layers.shape_list(features['task'])[0]
num_sequences = tf.maximum(
tf.to_int32(tf.to_float(batch_size) / mean_length), 1)
lengths = tf.random.truncated_normal(shape=[num_sequences],
mean=mean_length,
stddev=stddev)
max_length = tf.reduce_max(lengths) * (tf.to_float(batch_size) /
tf.reduce_sum(lengths))
max_length = tf.to_int32(tf.ceil(max_length))
total_items = max_length * num_sequences
num_paddings = total_items - batch_size
indices = tf.random.shuffle(tf.range(total_items))
for key in features:
shape_list = common_layers.shape_list(features[key])
assert len(shape_list) >= 1
with tf.control_dependencies([
tf.assert_greater_equal(num_paddings,
0,
name='num_paddings_positive')
]):
paddings = [[0, num_paddings]] + [[0, 0]] * (len(shape_list) - 1)
features[key] = tf.pad(features[key],
paddings,
constant_values=-1 if key == 'obj_type' else 0)
features[key] = tf.gather(features[key], indices)
shape = [num_sequences, max_length]
if len(shape_list) >= 2:
shape += shape_list[1:]
features[key] = tf.reshape(features[key], shape)
# Remove all-padding seqs
step_mask = tf.reduce_any(tf.greater(features['task'], 1), axis=-1)
mask = tf.reduce_any(step_mask, axis=-1)
step_mask = tf.boolean_mask(step_mask, mask)
for key in features:
features[key] = tf.boolean_mask(features[key], mask=mask)
num_sequences = tf.shape(features['task'])[0]
# Sort steps within each seq
_, step_indices = tf.math.top_k(tf.to_int32(step_mask), k=max_length)
step_indices = step_indices + tf.expand_dims(
tf.range(num_sequences) * max_length, 1)
step_indices = tf.reshape(step_indices, [-1])
for key in features:
shape_list = common_layers.shape_list(features[key])
features[key] = tf.gather(
tf.reshape(features[key], [-1] + shape_list[2:]), step_indices)
features[key] = tf.reshape(features[key], shape_list)
features = _stitch(features)
return features
def crop_proposal():
def rand_vec(minval, maxval): return tf.random_uniform(
shape=(constants.NUM_CROP_PASSES, 1), minval=minval, maxval=maxval,
dtype=tf.float32)
width, height = rand_vec(0.3, 1), rand_vec(0.3, 1)
left, top = rand_vec(0, 1-width), rand_vec(0, 1-height)
right = left + width
bottom = top + height
ltrb = tf.concat([left, top, right, bottom], axis=1)
min_iou = tf.random_shuffle(constants.CROP_MIN_IOU_CHOICES)[0]
ious = calc_iou_tensor(ltrb, boxes)
# discard any bboxes whose center not in the cropped image
xc, yc = [tf.tile(0.5 * (boxes[:, i + 0] + boxes[:, i + 2])[tf.newaxis, :],
(constants.NUM_CROP_PASSES, 1)) for i in range(2)]
masks = tf.reduce_all(tf.stack([
tf.greater(xc, tf.tile(left, (1, num_boxes))),
tf.less(xc, tf.tile(right, (1, num_boxes))),
tf.greater(yc, tf.tile(top, (1, num_boxes))),
tf.less(yc, tf.tile(bottom, (1, num_boxes))),
], axis=2), axis=2)
# Checks of whether a crop is valid.
valid_aspect = tf.logical_and(tf.less(height/width, 2),
tf.less(width/height, 2))
valid_ious = tf.reduce_all(tf.greater(
ious, min_iou), axis=1, keepdims=True)
valid_masks = tf.reduce_any(masks, axis=1, keepdims=True)
valid_all = tf.cast(tf.reduce_all(tf.concat(
[valid_aspect, valid_ious, valid_masks], axis=1), axis=1), tf.int32)
# One indexed, as zero is needed for the case of no matches.
index = tf.range(1, 1 + constants.NUM_CROP_PASSES, dtype=tf.int32)
# Either one-hot, or zeros if there is no valid crop.
selection = tf.equal(tf.reduce_max(index * valid_all), index)
use_crop = tf.reduce_any(selection)
output_ltrb = tf.reduce_sum(tf.multiply(ltrb, tf.tile(tf.cast(
selection, tf.float32)[:, tf.newaxis], (1, 4))), axis=0)
output_masks = tf.reduce_any(tf.logical_and(masks, tf.tile(
selection[:, tf.newaxis], (1, num_boxes))), axis=0)
return use_crop, output_ltrb, output_masks
def _process_finished_state(self, finished_state):
alive_seq = finished_state[_StateKeys.ALIVE_SEQ]
alive_log_probs = finished_state[_StateKeys.ALIVE_LOG_PROBS]
finished_seq = finished_state[_StateKeys.FINISHED_SEQ]
finished_scores = finished_state[_StateKeys.FINISHED_SCORES]
finished_flags = finished_state[_StateKeys.FINISHED_FLAGS]
# Account for corner case where there are no finished sequences for a
# particular batch item. In that case, return alive sequences for that batch
# item.
finished_seq = tf.where(tf.reduce_any(finished_flags, 1), finished_seq,
alive_seq)
finished_scores = tf.where(tf.reduce_any(finished_flags, 1),
finished_scores, alive_log_probs)
return finished_seq, finished_scores
def _define_step(self, done, score, summary):
"""Combine operations of a phase.
Keeps track of the mean score and when to report it.
Args:
done: Tensor indicating whether current score can be used.
score: Tensor holding the current, possibly intermediate, score.
summary: Tensor holding summary string to write if not an empty string.
Returns:
Tuple of summary tensor, mean score, and new global step. The mean score
is zero for non reporting steps.
"""
if done.shape.ndims == 0:
done = done[None]
if score.shape.ndims == 0:
score = score[None]
score_mean = streaming_mean.StreamingMean((), tf.float32)
with tf.control_dependencies([done, score, summary]):
done_score = tf.gather(score, tf.where(done)[:, 0])
submit_score = tf.cond(tf.reduce_any(done),
lambda: score_mean.submit(done_score),
tf.no_op)
with tf.control_dependencies([submit_score]):
mean_score = tf.cond(self._report, score_mean.clear, float)
steps_made = tf.shape(score)[0]
next_step = self._step.assign_add(steps_made)
with tf.control_dependencies([mean_score, next_step]):
return tf.identity(summary), mean_score, next_step, steps_made
def aggregate_gradients_using_copy(tower_grads, use_mean, check_inf_nan):
"""Calculate the average gradient for each shared variable across all towers.
Note that this function provides a synchronization point across all towers.
Args:
tower_grads: List of lists of (gradient, variable) tuples. The outer list
is over towers. The inner list is over individual gradients.
use_mean: if True, mean is taken, else sum of gradients is taken.
check_inf_nan: check grads for nans and infs.
Returns:
The tuple ([(average_gradient, variable),], has_nan_or_inf) where the
gradient has been averaged across all towers. The variable is chosen from
the first tower. The has_nan_or_inf indicates the grads has nan or inf.
"""
agg_grads = []
has_nan_or_inf_list = []
for single_grads in zip(*tower_grads):
grad_and_var, has_nan_or_inf = aggregate_single_gradient_using_copy(
single_grads, use_mean, check_inf_nan)
agg_grads.append(grad_and_var)
has_nan_or_inf_list.append(has_nan_or_inf)
if check_inf_nan:
return agg_grads, tf.reduce_any(has_nan_or_inf_list)
else:
return agg_grads, None
def prune_outside_window(boxlist, window, scope=None):
"""Prunes bounding boxes that fall outside a given window.
This function prunes bounding boxes that even partially fall outside the given
window. See also clip_to_window which only prunes bounding boxes that fall
completely outside the window, and clips any bounding boxes that partially
overflow.
Args:
boxlist: a BoxList holding M_in boxes.
window: a float tensor of shape [4] representing [ymin, xmin, ymax, xmax]
of the window
scope: name scope.
Returns:
pruned_corners: a tensor with shape [M_out, 4] where M_out <= M_in
valid_indices: a tensor with shape [M_out] indexing the valid bounding boxes
in the input tensor.
"""
with tf.name_scope(scope, 'PruneOutsideWindow'):
y_min, x_min, y_max, x_max = tf.split(value=boxlist.get(),
num_or_size_splits=4,
axis=1)
win_y_min, win_x_min, win_y_max, win_x_max = tf.unstack(window)
coordinate_violations = tf.concat([
tf.less(y_min, win_y_min),
tf.less(x_min, win_x_min),
tf.greater(y_max, win_y_max),
tf.greater(x_max, win_x_max)
], 1)
valid_indices = tf.reshape(
tf.where(tf.logical_not(tf.reduce_any(coordinate_violations, 1))),
[-1])
return gather(boxlist, valid_indices), valid_indices
def correct_keypoints(image_shape, keypoints):
"""
Arguments:
image_shape: an int tensor with shape [3].
keypoints: an int tensor with shape [num_persons, 17, 3].
Returns:
an int tensor with shape [num_persons, 17, 3].
"""
y, x, v = tf.split(keypoints, 3, axis=2)
height = image_shape[0]
width = image_shape[1]
coordinate_violations = tf.concat([
tf.less(y, 0),
tf.less(x, 0),
tf.greater_equal(y, height),
tf.greater_equal(x, width)
],
axis=2) # shape [num_persons, 17, 4]
valid_indicator = tf.logical_not(
tf.reduce_any(coordinate_violations, axis=2))
valid_indicator = tf.expand_dims(valid_indicator, 2)
# it has shape [num_persons, 17, 1]
v *= tf.to_int32(valid_indicator)
keypoints = tf.concat([y, x, v], axis=2)
return keypoints
def _input_booster(example):
with tf.control_dependencies([tf.rank(example['input_refs']), 2]):
has_input = tf.reduce_any(
tf.greater(example['input_refs'][:, 1],
example['input_refs'][:, 0]))
return tf.logical_or(has_input,
tf.less(tf.random_uniform([]), 0.1))
def prune_completely_outside_window(boxes, window):
"""
Prunes bounding boxes that fall completely outside of the given window.
This function does not clip partially overflowing boxes.
Arguments:
boxes: a float tensor with shape [M_in, 4].
window: a float tensor with shape [4] representing [ymin, xmin, ymax, xmax]
of the window.
Returns:
boxes: a float tensor with shape [M_out, 4] where 0 <= M_out <= M_in.
valid_indices: a long tensor with shape [M_out] indexing the valid bounding boxes
in the input 'boxes' tensor.
"""
y_min, x_min, y_max, x_max = tf.split(boxes, num_or_size_splits=4, axis=1)
# they have shape [None, 1]
win_y_min, win_x_min, win_y_max, win_x_max = tf.unstack(window)
# they have shape []
coordinate_violations = tf.concat([
tf.greater_equal(y_min, win_y_max),
tf.greater_equal(x_min, win_x_max),
tf.less_equal(y_max, win_y_min),
tf.less_equal(x_max, win_x_min)
],
axis=1)
valid_indices = tf.squeeze(tf.where(
tf.logical_not(tf.reduce_any(coordinate_violations, 1))),
axis=1)
boxes = tf.gather(boxes, valid_indices)
return boxes, valid_indices
def prune_completely_outside_window(boxlist, window, scope=None):
"""Prunes bounding boxes that fall completely outside of the given window.
The function clip_to_window prunes bounding boxes that fall
completely outside the window, but also clips any bounding boxes that
partially overflow. This function does not clip partially overflowing boxes.
Args:
boxlist: a BoxList holding M_in boxes.
window: a float tensor of shape [4] representing [ymin, xmin, ymax, xmax]
of the window
scope: name scope.
Returns:
pruned_boxlist: a new BoxList with all bounding boxes partially or fully in
the window.
valid_indices: a tensor with shape [M_out] indexing the valid bounding boxes
in the input tensor.
"""
with tf.name_scope(scope, 'PruneCompleteleyOutsideWindow'):
y_min, x_min, y_max, x_max = tf.split(value=boxlist.get(),
num_or_size_splits=4,
axis=1)
win_y_min, win_x_min, win_y_max, win_x_max = tf.unstack(window)
coordinate_violations = tf.concat([
tf.greater_equal(y_min, win_y_max),
tf.greater_equal(x_min, win_x_max),
tf.less_equal(y_max, win_y_min),
tf.less_equal(x_max, win_x_min)
], 1)
valid_indices = tf.reshape(
tf.where(tf.logical_not(tf.reduce_any(coordinate_violations, 1))),
[-1])
return gather(boxlist, valid_indices), valid_indices
def random_substr(str_tensor, max_words):
"""Select random substring if the input has more than max_words."""
word_batch_r = tf.strings.split(str_tensor, result_type="RaggedTensor")
row_splits = word_batch_r.row_splits
words = word_batch_r.values
start_idx = row_splits[:-1]
end_idx = row_splits[1:]
words_per_example = end_idx - start_idx
ones = tf.ones_like(end_idx)
max_val = tf.maximum(ones, words_per_example - max_words)
max_words_batch = tf.reduce_max(words_per_example)
rnd = tf.random.uniform(tf.shape(start_idx),
minval=0,
maxval=max_words_batch,
dtype=tf.int64)
off_start_idx = tf.math.floormod(rnd, max_val)
new_words_per_example = tf.where(tf.equal(max_val, 1), words_per_example,
ones * max_words)
new_start_idx = start_idx + off_start_idx
new_end_idx = new_start_idx + new_words_per_example
indices = tf.expand_dims(tf.range(tf.size(words), dtype=tf.int64), axis=0)
within_limit = tf.logical_and(
tf.greater_equal(indices, tf.expand_dims(new_start_idx, axis=1)),
tf.less(indices, tf.expand_dims(new_end_idx, axis=1)))
keep_indices = tf.reduce_any(within_limit, axis=0)
keep_indices = tf.cast(keep_indices, dtype=tf.int32)
_, selected_words = tf.dynamic_partition(words, keep_indices, 2)
row_splits = tf.math.cumsum(new_words_per_example)
row_splits = tf.concat([[0], row_splits], axis=0)
new_tensor = tf.RaggedTensor.from_row_splits(values=selected_words,
row_splits=row_splits)
return tf.strings.reduce_join(new_tensor, axis=1, separator=" ")
def _self_suppression(iou, _, iou_sum, iou_threshold):
"""Suppress boxes in the same tile.
Compute boxes that cannot be suppressed by others (i.e.,
can_suppress_others), and then use them to suppress boxes in the same tile.
Args:
iou: a tensor of shape [batch_size, num_boxes_with_padding].
iou_sum: a scalar tensor.
iou_threshold: a scalar tensor.
Returns:
iou_suppressed: a tensor of shape [batch_size, num_boxes_with_padding].
iou_diff: a scalar tensor representing whether any box is supressed in
this step.
iou_sum_new: a scalar tensor of shape [batch_size] that represents
the iou sum after suppression.
iou_threshold: a scalar tensor.
"""
batch_size = tf.shape(iou)[0]
can_suppress_others = tf.cast(
tf.reshape(tf.reduce_max(iou, 1) <= iou_threshold, [batch_size, -1, 1]),
iou.dtype)
iou_after_suppression = tf.reshape(
tf.cast(
tf.reduce_max(can_suppress_others * iou, 1) <= iou_threshold,
iou.dtype), [batch_size, -1, 1]) * iou
iou_sum_new = tf.reduce_sum(iou_after_suppression, [1, 2])
return [
iou_after_suppression,
tf.reduce_any(iou_sum - iou_sum_new > iou_threshold), iou_sum_new,
iou_threshold
]
def __init__(self, regularizers_to_group):
"""Creates an instance.
Args:
regularizers_to_group: A list of generic_regularizers.OpRegularizer
objects. Their regularization_vector (alive_vector) are expected to be
of the same length. These are expected to be probabilistic regularizers.
Currently, the only supported OpRegularizer is ProbGatingRegularizer.
Raises:
ValueError: regularizers_to_group is not of length at least 2.
"""
if len(regularizers_to_group) < 2:
raise ValueError('Groups must be of at least size 2.')
regularization_vectors = []
alive_vectors = []
for reg in regularizers_to_group:
if not hasattr(reg,
'is_probabilistic') or not reg.is_probabilistic:
raise ValueError('Regularizer is not probabilistic.')
regularization_vectors.append(
tf.reshape(reg.regularization_vector, [1, -1]))
alive_vectors.append(tf.reshape(reg.alive_vector, [1, -1]))
regularization_vectors = tf.concat(regularization_vectors, axis=0)
alive_vectors = tf.concat(alive_vectors, axis=0)
# The probability that at least one of the channels is alive:
# 1 - \prod_i{1 - p_i}.
self._regularization_vector = 1.0 - tf.reduce_prod(
1.0 - regularization_vectors, axis=0)
self._alive_vector = tf.reduce_any(alive_vectors, axis=0)
def aggregate_gradients_using_copy_with_device_selection(
benchmark_cnn, tower_grads, use_mean, check_inf_nan):
"""Aggregate gradients, controlling device for the aggregation.
Args:
benchmark_cnn: benchmark_cnn class.
tower_grads: List of lists of (gradient, variable) tuples. The outer list
is over towers. The inner list is over individual gradients.
use_mean: if True, mean is taken, else sum of gradients is taken.
check_inf_nan: If true, check grads for nans and infs.
Returns:
The tuple ([(average_gradient, variable),], has_nan_or_inf) where the
gradient has been averaged across all towers. The variable is chosen from
the first tower. The has_nan_or_inf indicates the grads has nan or inf.
"""
if benchmark_cnn.local_parameter_device_flag == 'gpu':
avail_devices = benchmark_cnn.raw_devices
else:
avail_devices = [benchmark_cnn.param_server_device]
agg_grads = []
has_nan_or_inf_list = []
for i, single_grads in enumerate(zip(*tower_grads)):
with tf.device(avail_devices[i % len(avail_devices)]):
grad_and_var, has_nan_or_inf = aggregate_single_gradient_using_copy(
single_grads, use_mean, check_inf_nan)
agg_grads.append(grad_and_var)
has_nan_or_inf_list.append(has_nan_or_inf)
if check_inf_nan:
return agg_grads, tf.reduce_any(has_nan_or_inf_list)
else:
return agg_grads, None
def get_eval_metric_ops(targets, predictions):
"""Get a dictionary of eval metrics for `Experiment` object.
Args:
targets: `targets` that go into `model_fn` of `Experiment`.
predictions: Dictionary of predictions, output of `get_preds`.
Returns:
A dictionary of eval metrics.
"""
# TODO(seominjoon): yp should also consider no answer case.
yp1 = tf.expand_dims(predictions['yp1'], -1)
yp2 = tf.expand_dims(predictions['yp2'], -1)
answer_mask = tf.sequence_mask(targets['num_answers'])
start_correct = tf.reduce_any(
tf.equal(targets['word_answer_starts'], yp1) & answer_mask, 1)
end_correct = tf.reduce_any(
tf.equal(targets['word_answer_ends'], yp2) & answer_mask, 1)
correct = start_correct & end_correct
em = tf.py_func(
_enum_fn(partial(evaluator_util.compute_exact,
is_byte=True), dtype='float32'),
[predictions['a'], targets['answers'], answer_mask],
'float32')
f1 = tf.py_func(
_enum_fn(partial(evaluator_util.compute_f1,
is_byte=True), dtype='float32'),
[predictions['a'], targets['answers'], answer_mask],
'float32')
eval_metric_ops = {
'acc1': tf.metrics.mean(tf.cast(start_correct, 'float')),
'acc2': tf.metrics.mean(tf.cast(end_correct, 'float')),
'acc': tf.metrics.mean(tf.cast(correct, 'float')),
'em': tf.metrics.mean(em),
'f1': tf.metrics.mean(f1),
}
if 'plausible' in targets: # SQuAD2
# targets['plausible'] has values 1.0 for plausible (no-answer) examples
# and 0.0 otherwise.
eval_metric_ops['f1_no_answer'] = tf.metrics.mean(
f1 * targets['plausible'])
eval_metric_ops['em_no_answer'] = tf.metrics.mean(
em * targets['plausible'])
return eval_metric_ops
def cond(ctx, cache, probs):
# ctx = tf.Print(ctx,[tf.shape(ctx)])
is_eos = tf.reduce_all(
tf.reduce_any(tf.equal(ctx[:, -1:], eos_token), axis=1))
is_max_len = tf.greater_equal(get_shape_list(probs)[1], max_len)
is_min_len = tf.greater_equal(get_shape_list(probs)[1], min_len)
first_cond = tf.logical_and(is_eos, is_min_len)
return tf.logical_not(first_cond)
def next_inputs(self, time, outputs, state, sample_ids, name=None):
with tf.name_scope(name, "ScheduledOutputTrainingHelperNextInputs",
[time, outputs, state, sample_ids]):
(finished, base_next_inputs, state) = (
super(ScheduledOutputTrainingHelper, self).next_inputs(
time=time,
outputs=outputs,
state=state,
sample_ids=sample_ids,
name=name))
sample_ids = tf.cast(sample_ids, tf.bool)
def maybe_sample():
"""Perform scheduled sampling."""
def maybe_concatenate_auxiliary_inputs(outputs_, indices=None):
"""Concatenate outputs with auxiliary inputs, if they exist."""
if self._auxiliary_input_tas is None:
return outputs_
next_time = time + 1
auxiliary_inputs = tf.nest.map_structure(
lambda ta: ta.read(next_time), self._auxiliary_input_tas)
if indices is not None:
auxiliary_inputs = tf.gather_nd(auxiliary_inputs, indices)
return tf.nest.map_structure(
lambda x, y: tf.concat((x, y), -1),
outputs_, auxiliary_inputs)
if self._next_inputs_fn is None:
return tf.where(
sample_ids, maybe_concatenate_auxiliary_inputs(outputs),
base_next_inputs)
where_sampling = tf.cast(
tf.where(sample_ids), tf.int32)
where_not_sampling = tf.cast(
tf.where(tf.logical_not(sample_ids)), tf.int32)
outputs_sampling = tf.gather_nd(outputs, where_sampling)
inputs_not_sampling = tf.gather_nd(base_next_inputs,
where_not_sampling)
sampled_next_inputs = maybe_concatenate_auxiliary_inputs(
self._next_inputs_fn(outputs_sampling), where_sampling)
base_shape = tf.shape(base_next_inputs)
return (tf.scatter_nd(indices=where_sampling,
updates=sampled_next_inputs,
shape=base_shape)
+ tf.scatter_nd(indices=where_not_sampling,
updates=inputs_not_sampling,
shape=base_shape))
all_finished = tf.reduce_all(finished)
no_samples = tf.logical_not(tf.reduce_any(sample_ids))
next_inputs = tf.cond(
tf.logical_or(all_finished, no_samples),
lambda: base_next_inputs, maybe_sample)
return (finished, next_inputs, state)
def infer_step(result, length):
"""Inference step."""
def print_info(result, length, new_length):
vocab = self.problem_hparams.vocabulary["targets"]
tf.logging.info(
"length=%s new_length=%s length_diff=%s new_suffix=%s",
length,
new_length,
new_length - length,
str([
vocab._subtoken_id_to_subtoken_string(index) # pylint: disable=protected-access
for index in result[0, -block_size:, 0,
0][:new_length - length]
]).decode("unicode-escape"),
)
features["targets"] = tf.pad(result,
[[0, 0], [0, 1], [0, 0], [0, 0]])
samples, logits, losses = self.sample(features) # pylint: disable=unused-variable
_, top_k_indices = tf.nn.top_k(
logits[:, :-1, :1, :, :],
k=self._decode_hparams.guess_and_check_top_k)
in_top_k = tf.reduce_any(tf.equal(tf.to_int64(top_k_indices),
tf.expand_dims(result, 4)),
axis=4)
eos_cumsum = tf.cumsum(tf.to_int32(
tf.equal(result, text_encoder.EOS_ID)),
axis=1)
after_eos = tf.greater(common_layers.shift_right(eos_cumsum), 0)
correct = tf.logical_and(in_top_k, tf.logical_not(after_eos))
correct_cumsum = tf.cumsum(tf.to_int32(correct), axis=1)
perfect_cumsum = 1 + tf.range(tf.shape(correct)[1])
for axis in [0, 2, 3]:
perfect_cumsum = tf.expand_dims(perfect_cumsum, axis=axis)
new_length = tf.reduce_sum(tf.to_int32(
tf.equal(correct_cumsum, perfect_cumsum)),
axis=1)
new_length = tf.squeeze(new_length, axis=[0, 1, 2])
new_length = tf.minimum(new_length, decode_length)
new_result = tf.concat([
result[:, :new_length, :, :],
tf.reshape(samples[:, new_length, :block_size, :],
[1, block_size, 1, 1])
],
axis=1)
with tf.control_dependencies(
[tf.py_func(print_info, [result, length, new_length], [])]):
new_result = tf.identity(new_result)
return new_result, new_length
def _suppression_loop_body(boxes, iou_threshold, output_size, idx):
"""Process boxes in the range [idx*_NMS_TILE_SIZE, (idx+1)*_NMS_TILE_SIZE).
Args:
boxes: a tensor with a shape of [batch_size, anchors, 4].
iou_threshold: a float representing the threshold for deciding whether boxes
overlap too much with respect to IOU.
output_size: an int32 tensor of size [batch_size]. Representing the number
of selected boxes for each batch.
idx: an integer scalar representing induction variable.
Returns:
boxes: updated boxes.
iou_threshold: pass down iou_threshold to the next iteration.
output_size: the updated output_size.
idx: the updated induction variable.
"""
num_tiles = tf.shape(boxes)[1] // _NMS_TILE_SIZE
batch_size = tf.shape(boxes)[0]
# Iterates over tiles that can possibly suppress the current tile.
box_slice = tf.slice(boxes, [0, idx * _NMS_TILE_SIZE, 0],
[batch_size, _NMS_TILE_SIZE, 4])
_, box_slice, _, _ = tf.while_loop(
lambda _boxes, _box_slice, _threshold, inner_idx: inner_idx < idx,
_cross_suppression, [boxes, box_slice, iou_threshold,
tf.constant(0)])
# Iterates over the current tile to compute self-suppression.
iou = box_utils.bbox_overlap(box_slice, box_slice)
mask = tf.expand_dims(
tf.reshape(tf.range(_NMS_TILE_SIZE), [1, -1]) > tf.reshape(
tf.range(_NMS_TILE_SIZE), [-1, 1]), 0)
iou *= tf.cast(tf.logical_and(mask, iou >= iou_threshold), iou.dtype)
suppressed_iou, _, _, _ = tf.while_loop(
lambda _iou, loop_condition, _iou_sum, _: loop_condition,
_self_suppression,
[iou,
tf.constant(True),
tf.reduce_sum(iou, [1, 2]), iou_threshold])
suppressed_box = tf.reduce_sum(suppressed_iou, 1) > 0
box_slice *= tf.expand_dims(1.0 - tf.cast(suppressed_box, box_slice.dtype),
2)
# Uses box_slice to update the input boxes.
mask = tf.reshape(tf.cast(tf.equal(tf.range(num_tiles), idx), boxes.dtype),
[1, -1, 1, 1])
boxes = tf.tile(tf.expand_dims(
box_slice, [1]), [1, num_tiles, 1, 1]) * mask + tf.reshape(
boxes, [batch_size, num_tiles, _NMS_TILE_SIZE, 4]) * (1 - mask)
boxes = tf.reshape(boxes, [batch_size, -1, 4])
# Updates output_size.
output_size += tf.reduce_sum(
tf.cast(tf.reduce_any(box_slice > 0, [2]), tf.int32), [1])
return boxes, iou_threshold, output_size, idx + 1
def process_obj_mask(obj_id):
"""Create a mask for obj_id, skipping the background mask."""
mask = tf.logical_and(
tf.equal(current_seg, obj_id), # pylint: disable=cell-var-from-loop
tf.not_equal(tf.cast(0, tf.uint8), obj_id))
# Leave out vert small masks, that are most often errors.
size = tf.reduce_sum(tf.to_int32(mask))
mask = tf.logical_and(mask,
tf.greater(size, MIN_OBJECT_AREA))
if not self.boxify:
return mask
# Complete the mask to its bounding box.
binary_obj_masks_y = tf.reduce_any(mask,
axis=1,
keepdims=True)
binary_obj_masks_x = tf.reduce_any(mask,
axis=0,
keepdims=True)
return tf.logical_and(binary_obj_masks_y,
binary_obj_masks_x)
def _loop_cond(i, unused_alive_seq, alive_log_probs, unused_finished_seq,
finished_scores, finished_in_finished):
"""Checking termination condition.
We terminate when we decoded up to decode_length or the lowest scoring item
in finished has a greater score that the higest prob item in alive divided
by the max length penalty. Optionally also terminate if all alive scores
are below lower bound.
Args:
i: loop index
alive_log_probs: probabilities of the beams. [batch_size, beam_size]
finished_scores: scores for each of these sequences.
[batch_size, beam_size]
finished_in_finished: finished bools for each of these sequences.
[batch_size, beam_size]
Returns:
True to continue the loop, False to stop.
"""
max_length_penalty = tf.pow(((5. + tf.to_float(decode_length)) / 6.),
alpha)
# The best possible score of the most likley alive sequence
lower_bound_alive_scores = alive_log_probs[:, 0] / max_length_penalty
# Now to compute the lowest score of a finished sequence in finished
# If the sequence isn't finished, we multiply it's score by 0. since
# scores are all -ve, taking the min will give us the score of the lowest
# finished item.
lowest_score_of_finished_in_finished = tf.reduce_min(
finished_scores * tf.to_float(finished_in_finished), axis=1)
# If none of the sequences have finished, then the min will be 0 and
# we have to replace it by -ve INF if it is. The score of any seq in alive
# will be much higher than -ve INF and the termination condition will not
# be met.
lowest_score_of_finished_in_finished = _apply_negative_infinity_mask(
lowest_score_of_finished_in_finished,
tf.logical_not(tf.reduce_any(finished_in_finished, 1)))
# Will terminate beam search early if bound_is_met is True.
bound_is_met = tf.reduce_all(
tf.greater(lowest_score_of_finished_in_finished,
lower_bound_alive_scores))
# Check if all alive scores are below minimum.
if minimum_score:
minimum_score_log = tf.log(minimum_score)
bound_is_met = tf.logical_or(
bound_is_met,
tf.reduce_all(
tf.less(lower_bound_alive_scores, minimum_score_log)))
return tf.logical_and(tf.less(i, decode_length),
tf.logical_not(bound_is_met))
def reject(start, end):
"""Reject span sample."""
span = tokens[start:end + 1]
wordpiece_boundary = tf.logical_or(
tf.strings.regex_full_match(span[0], r"^##.*"),
tf.strings.regex_full_match(span[-1], r"^##.*"))
span = tokens[start:end]
stopwords = list(nltk_utils.get_stopwords() | set(string.punctuation))
non_stopword = tf.setdiff1d(span, stopwords)
all_stopword = tf.equal(tf.size(non_stopword.out), 0)
length = tf.equal(tf.size(span), 0)
return tf.reduce_any([wordpiece_boundary, all_stopword, length])
def preprocessing_fn(inputs):
"""Preprocessing function used in TF Transform.
Args:
inputs: the input dataset of tf.Examples
Returns:
preprocessed outputs
"""
vocab_table = tf.lookup.StaticHashTable(
tf.lookup.TextFileInitializer(vocab_file, tf.string,
tf.lookup.TextFileIndex.WHOLE_LINE,
tf.int64,
tf.lookup.TextFileIndex.LINE_NUMBER),
-1)
tokenizer = BertTokenizer()
tokens = tokenizer.tokenize(inputs[text_key])
wordpiece_tokenizer = WordpieceTokenizer(vocab_table,
token_out_type=tf.string)
wordpieces = wordpiece_tokenizer.tokenize(tokens)
wordpieces_flat = wordpieces.flat_values
wordpieces_flat.set_shape([None])
wordpieces = tf.RaggedTensor.from_nested_row_splits(
wordpieces_flat, wordpieces.nested_row_splits)
known_mask = tf.cast(tf.not_equal(wordpieces, '[UNK]'), tf.int32)
num_non_unk_wordpieces = tf.reduce_sum(known_mask, axis=[1, 2])
wordpiece_is_unknown = tf.equal(wordpieces, '[UNK]')
token_has_unknown = tf.reduce_any(wordpiece_is_unknown, axis=-1)
unknown_tokens = tf.ragged.boolean_mask(tokens, token_has_unknown)
unknown_lengths = tf.strings.length(unknown_tokens)
num_dropped_chars = tf.math.reduce_sum(unknown_lengths, axis=1)
token_lengths = tf.strings.length(tokens)
total_chars = tf.reduce_sum(token_lengths, axis=-1)
num_preserved_chars = total_chars - num_dropped_chars
flattened = tf.RaggedTensor.from_row_splits(
wordpieces.flat_values,
tf.gather(wordpieces.values.row_splits, wordpieces.row_splits))
outputs = {}
outputs['num_non_unk_wordpieces'] = tf.cast(num_non_unk_wordpieces,
tf.int64)
outputs['num_dropped_chars'] = tf.cast(num_dropped_chars, tf.int64)
outputs['num_preserved_chars'] = tf.cast(num_preserved_chars, tf.int64)
outputs['wordpieces'] = flattened.to_sparse()
outputs['lang'] = tf.convert_to_tensor(inputs[language_code_key])
return outputs
def apply_motion_filter(self, data):
ims = tf.cast(data['images'], tf.float32) / 255.
intensities = tf.reduce_mean(ims, axis=-1) # [T,H,W]
delta_ims = tf.abs(intensities[1:] - intensities[:-1]) # [T-1,H,W]
if self.motion_area_thresh is not None:
moving = tf.cast(delta_ims > self.motion_thresh, tf.float32)
moving_fraction = tf.reduce_mean(moving, axis=[1,2])
motion_filter = tf.reduce_any(moving_fraction > self.motion_area_thresh)
else:
motion_filter = tf.reduce_max(delta_ims) > self.motion_thresh
return motion_filter
def _self_suppression(iou, _, iou_sum):
batch_size = tf.shape(iou)[0]
can_suppress_others = tf.cast(
tf.reshape(tf.reduce_max(iou, 1) <= 0.5, [batch_size, -1, 1]),
iou.dtype)
iou_suppressed = tf.reshape(
tf.cast(tf.reduce_max(can_suppress_others * iou, 1) <= 0.5, iou.dtype),
[batch_size, -1, 1]) * iou
iou_sum_new = tf.reduce_sum(iou_suppressed, [1, 2])
return [
iou_suppressed,
tf.reduce_any(iou_sum - iou_sum_new > 0.5), iou_sum_new
]
def compute_correct_candidates(candidate_starts, candidate_ends, gold_starts,
gold_ends):
"""Compute correct span."""
# [reader_beam_size, num_answers, num_candidates]
is_gold_start = tf.equal(
tf.expand_dims(tf.expand_dims(candidate_starts, 0), 0),
tf.expand_dims(gold_starts, -1))
is_gold_end = tf.equal(
tf.expand_dims(tf.expand_dims(candidate_ends, 0), 0),
tf.expand_dims(gold_ends, -1))
# [reader_beam_size, num_candidates]
return tf.reduce_any(tf.logical_and(is_gold_start, is_gold_end), 1)