def _compute_sparse_average_correct(input_, labels, per_example_weights, topk=1):
"""Returns the numerator and denominator of classifier accuracy."""
labels = tf.to_int64(labels)
labels.get_shape().assert_is_compatible_with([input_.get_shape()[0], None])
if topk == 1:
predictions = tf.reshape(tf.argmax(input_, 1), [-1, 1])
in_topk = tf.reduce_any(tf.equal(labels, predictions), reduction_indices=[1])
else:
# Use broadcasting to check if ANY of the predictions are in the top k.
# TODO(eiderman): For a multi-label top k, what does accuracy mean?
predictions = tf.reshape(tf.nn.top_k(input_, topk)[1], [-1, 1, topk])
labels = tf.expand_dims(labels, [-1])
in_topk = tf.reduce_any(tf.equal(tf.cast(labels, predictions.dtype), predictions), reduction_indices=[1, 2])
correct_predictions = tf.to_float(in_topk)
# If individual examples are weighted, then we want to normalize by that.
if per_example_weights is not None:
per_example_weights = _convert_and_assert_per_example_weights_compatible(
input_, per_example_weights, dtype=None
)
float_weights = tf.to_float(per_example_weights)
# TODO(eiderman): This should use an op that doesn't support broadcasting.
correct_predictions *= float_weights
num_examples = tf.reduce_sum(float_weights)
else:
# shape only holds ints, but we want to always return the same type
# for num_examples to make everything compatible.
num_examples = tf.to_float(tf.gather(tf.shape(input_), 0))
return tf.reduce_sum(correct_predictions), num_examples
def _define_step(self, done, score, summary):
"""Combine operations of a phase.
Keeps track of the mean score and when to report it.
Args:
done: Tensor indicating whether current score can be used.
score: Tensor holding the current, possibly intermediate, score.
summary: Tensor holding summary string to write if not an empty string.
Returns:
Tuple of summary tensor, mean score, and new global step. The mean score
is zero for non reporting steps.
"""
if done.shape.ndims == 0:
done = done[None]
if score.shape.ndims == 0:
score = score[None]
score_mean = streaming_mean.StreamingMean((), tf.float32)
with tf.control_dependencies([done, score, summary]):
done_score = tf.gather(score, tf.where(done)[:, 0])
submit_score = tf.cond(tf.reduce_any(done), lambda: score_mean.submit(done_score), tf.no_op)
with tf.control_dependencies([submit_score]):
mean_score = tf.cond(self._report, score_mean.clear, float)
steps_made = tf.shape(score)[0]
next_step = self._step.assign_add(steps_made)
with tf.control_dependencies([mean_score, next_step]):
return tf.identity(summary), mean_score, next_step, steps_made
def prune_completely_outside_window(boxlist, window, scope=None):
"""Prunes bounding boxes that fall completely outside of the given window.
The function clip_to_window prunes bounding boxes that fall
completely outside the window, but also clips any bounding boxes that
partially overflow. This function does not clip partially overflowing boxes.
Args:
boxlist: a BoxList holding M_in boxes.
window: a float tensor of shape [4] representing [ymin, xmin, ymax, xmax]
of the window
scope: name scope.
Returns:
pruned_boxlist: a new BoxList with all bounding boxes partially or fully in
the window.
valid_indices: a tensor with shape [M_out] indexing the valid bounding boxes
in the input tensor.
"""
with tf.name_scope(scope, 'PruneCompleteleyOutsideWindow'):
y_min, x_min, y_max, x_max = tf.split(
value=boxlist.get(), num_or_size_splits=4, axis=1)
win_y_min, win_x_min, win_y_max, win_x_max = tf.unstack(window)
coordinate_violations = tf.concat([
tf.greater_equal(y_min, win_y_max), tf.greater_equal(x_min, win_x_max),
tf.less_equal(y_max, win_y_min), tf.less_equal(x_max, win_x_min)
], 1)
valid_indices = tf.reshape(
tf.where(tf.logical_not(tf.reduce_any(coordinate_violations, 1))), [-1])
return gather(boxlist, valid_indices), valid_indices
def prune_outside_window(boxlist, window, scope=None):
"""Prunes bounding boxes that fall outside a given window.
This function prunes bounding boxes that even partially fall outside the given
window. See also clip_to_window which only prunes bounding boxes that fall
completely outside the window, and clips any bounding boxes that partially
overflow.
Args:
boxlist: a BoxList holding M_in boxes.
window: a float tensor of shape [4] representing [ymin, xmin, ymax, xmax]
of the window
scope: name scope.
Returns:
pruned_corners: a tensor with shape [M_out, 4] where M_out <= M_in
valid_indices: a tensor with shape [M_out] indexing the valid bounding boxes
in the input tensor.
"""
with tf.name_scope(scope, 'PruneOutsideWindow'):
y_min, x_min, y_max, x_max = tf.split(
value=boxlist.get(), num_or_size_splits=4, axis=1)
win_y_min, win_x_min, win_y_max, win_x_max = tf.unstack(window)
coordinate_violations = tf.concat([
tf.less(y_min, win_y_min), tf.less(x_min, win_x_min),
tf.greater(y_max, win_y_max), tf.greater(x_max, win_x_max)
], 1)
valid_indices = tf.reshape(
tf.where(tf.logical_not(tf.reduce_any(coordinate_violations, 1))), [-1])
return gather(boxlist, valid_indices), valid_indices
def compute_module(accum, module):
mask = tf.equal(module, selection)
reduced_mask = tf.reduce_any(mask, axis=-1)
indices = tf.where(reduced_mask)
affected_inp = tf.boolean_mask(inputs, reduced_mask)
output = module_fnc(affected_inp, module)
return accum + tf.scatter_nd(indices, output, tf.cast(output_shape, tf.int64))
def testArgRenames(self):
with self.test_session():
a = [[1.0, 2.0, 3.0], [4.0, 5.0, 6.0]]
b = [[True, False, False], [False, True, True]]
dim0 = [1]
dim1 = [1]
self.assertAllEqual(tf.reduce_any(b, reduction_indices=dim0).eval(), [True, True])
self.assertAllEqual(tf.reduce_all(b, reduction_indices=[0]).eval(), [False, False, False])
self.assertAllEqual(tf.reduce_all(b, reduction_indices=dim1).eval(), [False, False])
self.assertAllEqual(tf.reduce_sum(a, reduction_indices=[1]).eval(), [6.0, 15.0])
self.assertAllEqual(tf.reduce_sum(a, reduction_indices=[0, 1]).eval(), 21.0)
self.assertAllEqual(tf.reduce_sum(a, [0, 1]).eval(), 21.0)
self.assertAllEqual(tf.reduce_prod(a, reduction_indices=[1]).eval(), [6.0, 120.0])
self.assertAllEqual(tf.reduce_prod(a, reduction_indices=[0, 1]).eval(), 720.0)
self.assertAllEqual(tf.reduce_prod(a, [0, 1]).eval(), 720.0)
self.assertAllEqual(tf.reduce_mean(a, reduction_indices=[1]).eval(), [2.0, 5.0])
self.assertAllEqual(tf.reduce_mean(a, reduction_indices=[0, 1]).eval(), 3.5)
self.assertAllEqual(tf.reduce_mean(a, [0, 1]).eval(), 3.5)
self.assertAllEqual(tf.reduce_min(a, reduction_indices=[1]).eval(), [1.0, 4.0])
self.assertAllEqual(tf.reduce_min(a, reduction_indices=[0, 1]).eval(), 1.0)
self.assertAllEqual(tf.reduce_min(a, [0, 1]).eval(), 1.0)
self.assertAllEqual(tf.reduce_max(a, reduction_indices=[1]).eval(), [3.0, 6.0])
self.assertAllEqual(tf.reduce_max(a, reduction_indices=[0, 1]).eval(), 6.0)
self.assertAllEqual(tf.reduce_max(a, [0, 1]).eval(), 6.0)
self.assertAllClose(tf.reduce_logsumexp(a, reduction_indices=[1]).eval(), [3.40760589, 6.40760612])
self.assertAllClose(tf.reduce_logsumexp(a, reduction_indices=[0, 1]).eval(), 6.45619344711)
self.assertAllClose(tf.reduce_logsumexp(a, [0, 1]).eval(), 6.45619344711)
self.assertAllEqual(tf.expand_dims([[1, 2], [3, 4]], dim=1).eval(), [[[1, 2]], [[3, 4]]])
def kl_divergence(distribution_a, distribution_b,
allow_nan_stats=True, name=None):
"""Get the KL-divergence KL(distribution_a || distribution_b).
If there is no KL method registered specifically for `type(distribution_a)`
and `type(distribution_b)`, then the class hierarchies of these types are
searched.
If one KL method is registered between any pairs of classes in these two
parent hierarchies, it is used.
If more than one such registered method exists, the method whose registered
classes have the shortest sum MRO paths to the input types is used.
If more than one such shortest path exists, the first method
identified in the search is used (favoring a shorter MRO distance to
`type(distribution_a)`).
Args:
distribution_a: The first distribution.
distribution_b: The second distribution.
allow_nan_stats: Python `bool`, default `True`. When `True`,
statistics (e.g., mean, mode, variance) use the value "`NaN`" to
indicate the result is undefined. When `False`, an exception is raised
if one or more of the statistic's batch members are undefined.
name: Python `str` name prefixed to Ops created by this class.
Returns:
A Tensor with the batchwise KL-divergence between `distribution_a`
and `distribution_b`.
Raises:
NotImplementedError: If no KL method is defined for distribution types
of `distribution_a` and `distribution_b`.
"""
kl_fn = _registered_kl(type(distribution_a), type(distribution_b))
if kl_fn is None:
# TODO(b/117098119): For backwards compatibility, we check TF's registry as
# well. This typically happens when this function is called on a pair of
# TF's distributions.
with deprecation.silence():
return tf.distributions.kl_divergence(distribution_a, distribution_b)
with tf.name_scope("KullbackLeibler"):
kl_t = kl_fn(distribution_a, distribution_b, name=name)
if allow_nan_stats:
return kl_t
# Check KL for NaNs
kl_t = tf.identity(kl_t, name="kl")
with tf.control_dependencies([
tf.Assert(
tf.logical_not(
tf.reduce_any(tf.is_nan(kl_t))),
["KL calculation between %s and %s returned NaN values "
"(and was called with allow_nan_stats=False). Values:"
% (distribution_a.name, distribution_b.name), kl_t])]):
return tf.identity(kl_t, name="checked_kl")
def is_last_day_of_season(t):
t_ = dist_util.maybe_get_static_value(t)
if t_ is not None: # static case
step_in_cycle = t_ % num_steps_per_cycle
return any(step_in_cycle == changepoints)
else:
step_in_cycle = tf.floormod(t, num_steps_per_cycle)
return tf.reduce_any(tf.equal(step_in_cycle, changepoints))
def check_grads(grads_and_vars):
has_nan_ops = []
amax_ops = []
for grad, _ in grads_and_vars:
if grad is not None:
if isinstance(grad, tf.IndexedSlices):
x = grad.values
else:
x = grad
has_nan_ops.append(tf.reduce_any(tf.is_nan(x)))
amax_ops.append(tf.reduce_max(tf.abs(x)))
has_nan = tf.reduce_any(has_nan_ops)
amax = tf.reduce_max(amax_ops)
return has_nan, amax
def any(x, axis=None, keepdims=False):
"""Bitwise reduction (logical OR).
Return array of uint8 (0s and 1s).
"""
axis = normalize_axis(axis, ndim(x))
x = tf.cast(x, tf.bool)
x = tf.reduce_any(x, reduction_indices=axis, keep_dims=keepdims)
return tf.cast(x, tf.uint8)
def any(x, axis=None, keepdims=False):
'''Bitwise reduction (logical OR).
Returns an uint8 tensor (0s and 1s).
'''
axis = _normalize_axis(axis, ndim(x))
x = tf.cast(x, tf.bool)
x = tf.reduce_any(x, reduction_indices=axis, keep_dims=keepdims)
return tf.cast(x, tf.uint8)
def retrieve_seq_length_op3(data, pad_val=0):
"""An op to compute the length of a sequence, the data shape can be [batch_size, n_step(max)] or
[batch_size, n_step(max), n_features].
If the data has type of tf.string and pad_val is assigned as empty string (''), this op will compute the
length of the string sequence.
Parameters:
-----------
data : tensor
[batch_size, n_step(max)] or [batch_size, n_step(max), n_features] with zero padding on the right hand side.
pad_val:
By default 0. If the data is tf.string, please assign this as empty string ('')
Examples
-----------
>>> data = [[[1],[2],[0],[0],[0]],
>>> [[1],[2],[3],[0],[0]],
>>> [[1],[2],[6],[1],[0]]]
>>> data = tf.convert_to_tensor(data, dtype=tf.float32)
>>> length = tl.layers.retrieve_seq_length_op3(data)
[2, 3, 4]
>>> data = [[[1,2],[2,2],[1,2],[1,2],[0,0]],
>>> [[2,3],[2,4],[3,2],[0,0],[0,0]],
>>> [[3,3],[2,2],[5,3],[1,2],[0,0]]]
>>> data = tf.convert_to_tensor(data, dtype=tf.float32)
>>> length = tl.layers.retrieve_seq_length_op3(data)
[4, 3, 4]
>>> data = [[1,2,0,0,0],
>>> [1,2,3,0,0],
>>> [1,2,6,1,0]]
>>> data = tf.convert_to_tensor(data, dtype=tf.float32)
>>> length = tl.layers.retrieve_seq_length_op3(data)
[2, 3, 4]
>>> data = [['hello','world','','',''],
>>> ['hello','world','tensorlayer','',''],
>>> ['hello','world','tensorlayer','2.0','']]
>>> data = tf.convert_to_tensor(data, dtype=tf.string)
>>> length = tl.layers.retrieve_seq_length_op3(data, pad_val='')
[2, 3, 4]
"""
data_shape_size = data.get_shape().ndims
if data_shape_size == 3:
return tf.reduce_sum(
input_tensor=tf.cast(tf.reduce_any(input_tensor=tf.not_equal(data, pad_val), axis=2), dtype=tf.int32),
axis=1
)
elif data_shape_size == 2:
return tf.reduce_sum(input_tensor=tf.cast(tf.not_equal(data, pad_val), dtype=tf.int32), axis=1)
elif data_shape_size == 1:
raise ValueError("retrieve_seq_length_op3: data has wrong shape! Shape got ", data.get_shape().as_list())
else:
raise ValueError(
"retrieve_seq_length_op3: handling data with num of dims %s hasn't been implemented!" % (data_shape_size)
)
def remap_keys(sparse_tensor):
# Current indices of our SparseTensor that we need to fix
bad_indices = sparse_tensor.indices
# Current values of our SparseTensor that we need to fix
bad_values = sparse_tensor.values
# Group by the batch_indices and get the count for each
size = tf.segment_sum(data = tf.ones_like(bad_indices[:,0], dtype = tf.int64), segment_ids = bad_indices[:,0]) - 1
# The number of batch_indices (this should be batch_size unless it is a partially full batch)
length = tf.shape(size, out_type = tf.int64)[0]
# Finds the cumulative sum which we can use for indexing later
cum = tf.cumsum(size)
# The offsets between each example in the batch due to our concatentation of the keys in the decode_example method
length_range = tf.range(start = 0, limit = length, delta = 1, dtype = tf.int64)
# Indices of the SparseTensor's indices member of the rows we added by the concatentation of our keys in the decode_example method
cum_range = cum + length_range
# The keys that we have extracted back out of our concatentated SparseTensor
gathered_indices = tf.squeeze(tf.gather(bad_indices, cum_range)[:,1])
# The enumerated row indices of the SparseTensor's indices member
sparse_indices_range = tf.range(tf.shape(bad_indices, out_type = tf.int64)[0], dtype = tf.int64)
# We want to find here the row indices of the SparseTensor's indices member that are of our actual data and not the concatentated rows
# So we want to find the intersection of the two sets and then take the opposite of that
x = sparse_indices_range
s = cum_range
# Number of multiples we are going to tile x, which is our sparse_indices_range
tile_multiples = tf.concat([tf.ones(tf.shape(tf.shape(x)), dtype=tf.int64), tf.shape(s, out_type = tf.int64)], axis = 0)
# Expands x, our sparse_indices_range, into a rank 2 tensor and then multiplies the rows by 1 (no copying) and the columns by the number of examples in the batch
x_tile = tf.tile(tf.expand_dims(x, -1), tile_multiples)
# Essentially a vectorized logical or, that we then negate
x_not_in_s = ~tf.reduce_any(tf.equal(x_tile, s), -1)
# The SparseTensor's indices that are our actual data by using the boolean_mask we just made above applied to the entire indices member of our SparseTensor
selected_indices = tf.boolean_mask(tensor = bad_indices, mask = x_not_in_s, axis = 0)
# Apply the same boolean_mask to the entire values member of our SparseTensor to get the actual values data
selected_values = tf.boolean_mask(tensor = bad_values, mask = x_not_in_s, axis = 0)
# Need to replace the first column of our selected_indices with keys, so we first need to tile our gathered_indices
tiling = tf.tile(input = tf.expand_dims(gathered_indices[0], -1), multiples = tf.expand_dims(size[0] , -1))
# We have to repeatedly apply the tiling to each example in the batch
# Since it is jagged we cannot use tf.map_fn due to the stacking of the TensorArray, so we have to create our own custom version
def loop_body(i, tensor_grow):
return i + 1, tf.concat(values = [tensor_grow, tf.tile(input = tf.expand_dims(gathered_indices[i], -1), multiples = tf.expand_dims(size[i] , -1))], axis = 0)
_, result = tf.while_loop(lambda i, tensor_grow: i < length, loop_body, [tf.constant(1, dtype = tf.int64), tiling])
# Concatenate tiled keys with the 2nd column of selected_indices
selected_indices_fixed = tf.concat([tf.expand_dims(result, -1), tf.expand_dims(selected_indices[:, 1], -1)], axis = 1)
# Combine everything together back into a SparseTensor
remapped_sparse_tensor = tf.SparseTensor(indices = selected_indices_fixed, values = selected_values, dense_shape = sparse_tensor.dense_shape)
return remapped_sparse_tensor
def any(x, axis=None, keepdims=False):
'''Bitwise reduction (logical OR).
Return array of int8 (0s and 1s).
'''
if axis is not None and axis < 0:
axis = axis % len(x.get_shape())
x = tf.cast(x, tf.bool)
x = tf.reduce_any(x, reduction_indices=axis, keep_dims=keepdims)
return tf.cast(x, tf.int8)
def cond(loop_cnt, prev_out, _, __):
less = tf.less(loop_cnt, output_len_threshold)
is_regular_word = tf.reduce_any(
tf.not_equal(
prev_out,
tf.one_hot([0], FEATURE_SIZE) # <eos>
)
)
return tf.logical_and(less, is_regular_word)
def has_no_question_marks(line):
"""Returns True if the line of text has no question marks."""
# split the line into an array of characters
chars = tf.string_split(line[tf.newaxis], "").values
# for each character check if it is a question mark
is_question = tf.equal(chars, "?")
any_question = tf.reduce_any(is_question)
no_question = ~any_question
return no_question
def cond(output_one_hot, _, results):
# limit the number of output length to avoid infinite generation
less = tf.less(results.size(), 10)
# output_one_hot is a regular word if any element is unequal to <eos>
is_regular_word = tf.reduce_any(
tf.not_equal(
output_one_hot,
tf.one_hot([[0]], vocabulary_size) # <eos>
)
)
return tf.logical_and(less, is_regular_word)
def target_mask_op(data, pad_val=0): # HangSheng: return tensor for mask,if input is tf.string
"""Return tensor for mask, if input is ``tf.string``."""
data_shape_size = data.get_shape().ndims
if data_shape_size == 3:
return tf.cast(tf.reduce_any(input_tensor=tf.not_equal(data, pad_val), axis=2), dtype=tf.int32)
elif data_shape_size == 2:
return tf.cast(tf.not_equal(data, pad_val), dtype=tf.int32)
elif data_shape_size == 1:
raise ValueError("target_mask_op: data has wrong shape!")
else:
raise ValueError("target_mask_op: handling data_shape_size %s hasn't been implemented!" % (data_shape_size))
def rnn_model(full_image):
with tf.variable_scope('main_recurrence') as scope:
low_res = tf.image.resize_images(full_image, FLAGS.context_image_size, FLAGS.context_image_size)
context = context_network(low_res)
classifications_list = []
#provide 0 initialization to lstm1
lstm1 = rnn_cell.BasicLSTMCell(FLAGS.lstm_size)
lstm1_state = tf.zeros([FLAGS.batch_size, lstm1.state_size])
lstm1_outputs = []
lstm1_states = []
with tf.variable_scope('lstm2') as scope:
#provide context initialization to lstm2
lstm2 = rnn_cell.BasicLSTMCell(FLAGS.lstm_size)
lstm2_initial_input = tf.zeros([FLAGS.batch_size, FLAGS.lstm_size])
lstm2_output, lstm2_state = lstm2(lstm2_initial_input, context)
emission = emission_network(lstm2_output)
location, keep_going = _parse_emission_output(emission)
scope.reuse_variables()
valid_classification_list = []
for step in xrange(FLAGS.max_recurrent_steps):
if step > 0:
tf.get_variable_scope().reuse_variables()
keep_going_threshold = tf.constant(FLAGS.keep_going_threshold, dtype=tf.float32)
glimpse_out, glimpse_vars = glimpse_network(full_image, location)
lstm1_output, lstm1_state = lstm1(glimpse_out, lstm1_state)
classifications_list.append(classification_network(lstm1_state))
valids = tf.squeeze(tf.greater(keep_going, keep_going_threshold))
valid_classification_list.append(tf.to_int32(valids))
if not tf.reduce_any(valids):
break
with tf.variable_scope('lstm2') as scope:
scope.reuse_variables()
lstm2_output, lstm2_state = lstm2(lstm1_output, lstm2_state)
location, keep_going = _parse_emission_output(emission_network(lstm2_output))
valid_classifications = tf.pad(tf.pack(valid_classification_list), tf.convert_to_tensor([[0,FLAGS.max_recurrent_steps-step-1],[0,0]]))
classifications = tf.pad(tf.pack(classifications_list), tf.convert_to_tensor([[0,FLAGS.max_recurrent_steps-step-1],[0,0],[0,0]]))
classifications = attention(classifications, valid_classifications)
classifications.get_shape()
return tf.squeeze(classifications), glimpse_vars
def clusterize(batch):
start = time.time()
n = len(batch)
dim = len(batch[0])
print batch
points = tf.placeholder(tf.int32, [n,dim])
cluster_assignments = tf.Variable(tf.zeros([n], dtype=tf.int64))
# Use K random points as the starting centroids
centroids = tf.Variable(tf.slice(tf.random_shuffle(points), [0,0], [K,dim]))
# Replicate to n copies of each centroid and K copies of each
# point, then subtract and compute the sum of squared distances.
rep_centroids = tf.reshape(tf.tile(centroids, [n, 1]), [n, K, dim])
rep_points = tf.reshape(tf.tile(points, [1, K]), [n, K, dim])
sum_squares = tf.reduce_sum(tf.square(rep_points - rep_centroids), reduction_indices=2)
# Use argmin to select the lowest-distance point
best_centroids = tf.argmin(sum_squares, 1)
did_assignments_change = tf.reduce_any(tf.not_equal(best_centroids,
cluster_assignments))
def bucket_mean(data, bucket_ids, num_buckets):
total = tf.unsorted_segment_sum(data, bucket_ids, num_buckets)
count = tf.unsorted_segment_sum(tf.ones_like(data), bucket_ids, num_buckets)
return total / count
means = bucket_mean(points, best_centroids, K)
# Do not write to the assigned clusters variable until after
# computing whether the assignments have changed - hence with_dependencies
with tf.control_dependencies([did_assignments_change]):
do_updates = tf.group(
centroids.assign(means),
cluster_assignments.assign(best_centroids))
changed = True
iters = 0
sess = tf.Session()
sess.run(tf.initialize_all_variables(), feed_dict={points: batch})
while changed and iters < MAX_ITERS:
iters += 1
[changed, _] = sess.run([did_assignments_change, do_updates], feed_dict={points: batch})
[centers, assignments] = sess.run([centroids, cluster_assignments], feed_dict={points: batch})
end = time.time()
print ("Found in %.2f seconds" % (end-start)), iters, "iterations"
return [centers, assignments]
def drop_some(columns,
drop_prob=.15):
"""Zeros out columns with probability `drop_prob`.
Used for rounds of local drop path.
"""
num_columns = tensor_shape(columns)[0]
mask = tf.random_uniform([num_columns])>drop_prob
scale = num_columns/tf.reduce_sum(tf.cast(mask, tf.float32))
return tf.cond(tf.reduce_any(mask),
lambda : apply_mask(mask, columns) * scale,
lambda : random_column(columns))
def _compare(self, x, reduction_axes, keep_dims, use_gpu=False):
np_ans = x
if reduction_axes is None:
np_ans = np.any(np_ans, keepdims=keep_dims)
else:
for ra in reduction_axes[::-1]:
np_ans = np.any(np_ans, axis=ra, keepdims=keep_dims)
with self.test_session(use_gpu=use_gpu):
if reduction_axes is not None:
reduction_axes = np.array(reduction_axes).astype(np.int32)
tf_ans = tf.reduce_any(x, reduction_axes, keep_dims)
out = tf_ans.eval()
self.assertAllEqual(np_ans, out)
self.assertShapeEqual(np_ans, tf_ans)
def search(self, initial_ids, initial_cache):
"""Beam search for sequences with highest scores."""
state, state_shapes = self._create_initial_state(initial_ids, initial_cache)
finished_state = tf.while_loop(
self._continue_search, self._search_step, loop_vars=[state],
shape_invariants=[state_shapes], parallel_iterations=1, back_prop=False)
finished_state = finished_state[0]
alive_seq = finished_state[_StateKeys.ALIVE_SEQ]
alive_log_probs = finished_state[_StateKeys.ALIVE_LOG_PROBS]
finished_seq = finished_state[_StateKeys.FINISHED_SEQ]
finished_scores = finished_state[_StateKeys.FINISHED_SCORES]
finished_flags = finished_state[_StateKeys.FINISHED_FLAGS]
# Account for corner case where there are no finished sequences for a
# particular batch item. In that case, return alive sequences for that batch
# item.
finished_seq = tf.where(
tf.reduce_any(finished_flags, 1), finished_seq, alive_seq)
finished_scores = tf.where(
tf.reduce_any(finished_flags, 1), finished_scores, alive_log_probs)
return finished_seq, finished_scores
def downscale(self, inp, mask):
with vs.variable_scope("Downscale"):
inp2d = tf.reshape(tf.transpose(inp, perm=[1, 0, 2]), [-1, 2 * self.size])
out2d = rnn_cell.linear(inp2d, self.size, True, 1.0)
out3d = tf.reshape(out2d, [self.batch_size, -1, self.size])
out3d = tf.transpose(out3d, perm=[1, 0, 2])
out = tanh(out3d)
mask = tf.transpose(mask)
mask = tf.reshape(mask, [-1, 2])
mask = tf.cast(mask, tf.bool)
mask = tf.reduce_any(mask, reduction_indices=1)
mask = tf.to_int32(mask)
mask = tf.reshape(mask, [self.batch_size, -1])
mask = tf.transpose(mask)
return out, mask
def _continue_search(self, state):
"""Return whether to continue the search loop.
The loops should terminate when
1) when decode length has been reached, or
2) when the worst score in the finished sequences is better than the best
score in the alive sequences (i.e. the finished sequences are provably
unchanging)
Args:
state: A dictionary with the current loop state.
Returns:
Bool tensor with value True if loop should continue, False if loop should
terminate.
"""
i = state[_StateKeys.CUR_INDEX]
alive_log_probs = state[_StateKeys.ALIVE_LOG_PROBS]
finished_scores = state[_StateKeys.FINISHED_SCORES]
finished_flags = state[_StateKeys.FINISHED_FLAGS]
not_at_max_decode_length = tf.less(i, self.max_decode_length)
# Calculate largest length penalty (the larger penalty, the better score).
max_length_norm = _length_normalization(self.alpha, self.max_decode_length)
# Get the best possible scores from alive sequences.
best_alive_scores = alive_log_probs[:, 0] / max_length_norm
# Compute worst score in finished sequences for each batch element
finished_scores *= tf.to_float(finished_flags) # set filler scores to zero
lowest_finished_scores = tf.reduce_min(finished_scores, axis=1)
# If there are no finished sequences in a batch element, then set the lowest
# finished score to -INF for that element.
finished_batches = tf.reduce_any(finished_flags, 1)
lowest_finished_scores += (1. - tf.to_float(finished_batches)) * -INF
worst_finished_score_better_than_best_alive_score = tf.reduce_all(
tf.greater(lowest_finished_scores, best_alive_scores)
)
return tf.logical_and(
not_at_max_decode_length,
tf.logical_not(worst_finished_score_better_than_best_alive_score)
)
def _is_finished(i, unused_alive_seq, alive_log_probs, unused_finished_seq,
finished_scores, finished_in_finished, unused_states):
"""Checking termination condition.
We terminate when we decoded up to decode_length or the lowest scoring item
in finished has a greater score that the higest prob item in alive divided
by the max length penalty
Args:
i: loop index
alive_log_probs: probabilities of the beams. [batch_size, beam_size]
finished_scores: scores for each of these sequences.
[batch_size, beam_size]
finished_in_finished: finished bools for each of these sequences.
[batch_size, beam_size]
Returns:
Bool.
"""
if not stop_early:
return tf.less(i, decode_length)
max_length_penalty = tf.pow(((5. + tf.to_float(decode_length)) / 6.), alpha)
# The best possible score of the most likley alive sequence
lower_bound_alive_scores = alive_log_probs[:, 0] / max_length_penalty
# Now to compute the lowest score of a finished sequence in finished
# If the sequence isn't finished, we multiply it's score by 0. since
# scores are all -ve, taking the min will give us the score of the lowest
# finished item.
lowest_score_of_fininshed_in_finished = tf.reduce_min(
finished_scores * tf.to_float(finished_in_finished), axis=1)
# If none of the sequences have finished, then the min will be 0 and
# we have to replace it by -ve INF if it is. The score of any seq in alive
# will be much higher than -ve INF and the termination condition will not
# be met.
lowest_score_of_fininshed_in_finished += (
(1. - tf.to_float(tf.reduce_any(finished_in_finished, 1))) * -INF)
bound_is_met = tf.reduce_all(
tf.greater(lowest_score_of_fininshed_in_finished,
lower_bound_alive_scores))
return tf.logical_and(
tf.less(i, decode_length), tf.logical_not(bound_is_met))
def downscale(self, inp, mask):
with vs.variable_scope("Downscale"):
inshape = tf.shape(inp)
T, batch_size, dim = inshape[0], inshape[1], inshape[2]
inp2d = tf.reshape(tf.transpose(inp, perm=[1, 0, 2]), [-1, 2 * self.size])
out2d = rnn_cell.linear(inp2d, self.size, True, 1.0)
out3d = tf.reshape(out2d, tf.pack((batch_size, tf.to_int32(T/2), dim)))
out3d = tf.transpose(out3d, perm=[1, 0, 2])
out3d.set_shape([None, None, self.size])
out = tanh(out3d)
mask = tf.transpose(mask)
mask = tf.reshape(mask, [-1, 2])
mask = tf.cast(mask, tf.bool)
mask = tf.reduce_any(mask, reduction_indices=1)
mask = tf.to_int32(mask)
mask = tf.reshape(mask, tf.pack([batch_size, -1]))
mask = tf.transpose(mask)
return out, mask
def insert(self, ids, scores):
"""Insert the ids and scores into the TopN."""
with tf.control_dependencies(self.last_ops):
scatter_op = tf.scatter_update(self.id_to_score, ids, scores)
larger_scores = tf.greater(scores, self.sl_scores[0])
def shortlist_insert():
larger_ids = tf.boolean_mask(tf.to_int64(ids), larger_scores)
larger_score_values = tf.boolean_mask(scores, larger_scores)
shortlist_ids, new_ids, new_scores = tensor_forest_ops.top_n_insert(
self.sl_ids, self.sl_scores, larger_ids, larger_score_values
)
u1 = tf.scatter_update(self.sl_ids, shortlist_ids, new_ids)
u2 = tf.scatter_update(self.sl_scores, shortlist_ids, new_scores)
return tf.group(u1, u2)
# We only need to insert into the shortlist if there are any
# scores larger than the threshold.
cond_op = tf.cond(tf.reduce_any(larger_scores), shortlist_insert, tf.no_op)
with tf.control_dependencies([cond_op]):
self.last_ops = [scatter_op, cond_op]
def __call__(self, inputs, state, timestep = 0, scope=None):
with vs.variable_scope(scope or type(self).__name__):
# define within cell constants/ counters used to control while loop for ACTStep
prob = tf.constant(0.0,tf.float32,[self.batch_size], name="prob")
prob_compare = tf.constant(0.0,tf.float32,[self.batch_size], name="prob_compare")
counter = tf.constant(0.0, tf.float32,[self.batch_size], name="counter")
acc_outputs = tf.zeros_like(state, tf.float32, name="output_accumulator")
acc_states = tf.zeros_like(state, tf.float32, name="state_accumulator")
batch_mask = tf.constant(True, tf.bool,[self.batch_size])
# While loop stops when this predicate is FALSE.
# Ie all (probability < 1-eps AND counter < N) are false.
#x = self.ACTStep(batch_mask,prob_compare,prob,counter,state,inputs,acc_outputs,acc_states)
pred = lambda batch_mask,prob_compare,prob,\
counter,state,input,acc_output,acc_state:\
tf.reduce_any(
tf.logical_and(
tf.less(prob_compare,self.one_minus_eps),
tf.less(counter,self.N)))
# only stop if all of the batch have passed either threshold
# Do while loop iterations until predicate above is false.
_,_,remainders,iterations,_,_,output,next_state = \
control_flow_ops.while_loop(pred,self.ACTStep,
[batch_mask,prob_compare,prob,
counter,state,inputs, acc_outputs, acc_states])
#accumulate remainder and N values
self.ACT_remainder.append(tf.reduce_mean(1 - remainders))
self.ACT_iterations.append(tf.reduce_mean(iterations))
if self.sigmoid_output:
output = tf.sigmoid(tf.nn.rnn_cell._linear(output,self.batch_size,0.0))
return output, next_state
def _build(self, incoming, *args, **kwargs):
"""
Args:
incoming: List of Tensors.
Returns:
Merged Tensors.
"""
self._built_modules = [module(incoming, *args, **kwargs) for module in self._modules]
if self.merge_mode == self.MergeMode.CONCAT:
x = tf.concat(axis=self.axis, values=self._built_modules)
elif self.merge_mode == self.MergeMode.ELEMENTWISE_SUM:
x = self._built_modules[0]
for i in xrange(1, len(self._built_modules)):
x = tf.add(x, self._built_modules[i])
elif self.merge_mode == self.MergeMode.ELEMENTWISE_MUL:
x = self._built_modules[0]
for i in xrange(1, len(self._built_modules)):
x = tf.multiply(x, self._built_modules[i])
elif self.merge_mode == self.MergeMode.SUM:
x = tf.reduce_sum(tf.concat(axis=self.axis, values=self._built_modules), axis=self.axis)
elif self.merge_mode == self.MergeMode.MEAN:
x = tf.reduce_mean(tf.concat(axis=self.axis, values=self._built_modules), axis=self.axis)
elif self.merge_mode == self.MergeMode.PROD:
x = tf.reduce_prod(tf.concat(axis=self.axis, values=self._built_modules), axis=self.axis)
elif self.merge_mode == self.MergeMode.MAX:
x = tf.reduce_max(tf.concat(axis=self.axis, values=self._built_modules), axis=self.axis)
elif self.merge_mode == self.MergeMode.MIN:
x = tf.reduce_min(tf.concat(axis=self.axis, values=self._built_modules), axis=self.axis)
elif self.merge_mode == self.MergeMode.AND:
x = tf.reduce_all(tf.concat(axis=self.axis, values=self._built_modules), axis=self.axis)
elif self.merge_mode == self.MergeMode.OR:
x = tf.reduce_any(tf.concat(axis=self.axis, values=self._built_modules), axis=self.axis)
else:
raise Exception('Unknown merge mode', str(self.merge_mode))
track(x, tf.GraphKeys.LAYER_TENSOR, self.module_name)
return x
def KeepGoing(*args): return tf.logical_and(args[1] < max_steps, tf.reduce_any(args[3]))
def cond_fn(w, should_continue):
del w
return tf.reduce_any(input_tensor=should_continue)
def interpolate(grid, samples, world2grid):
"""Returns the trilinearly interpolated function values on the grid.
Args:
grid: Tensor with shape [batch_size, depth, height, width]. The function to
interpolate.
samples: Tensor with shape [batch_size, sample_count, 3]. The xyz triplets.
world2grid: Tensor with shape [batch_size, 4, 4]. A rigid body transform
mapping from sample coordinates to grid coordinates.
Returns:
sdf: Tensor with shape [batch_size, sample_count, 1]. The ground truth
sdf at the sample locations. Differentiable w.r.t. samples.
invalid: Tensor with shape [batch_size, sample_count, 1] and type tf.bool.
True where the input samples map outside the supplied grid. The sdf
tensor will be zero at these locations and there will be no gradient.
"""
xyzw_samples = tf.pad(samples,
paddings=tf.constant([[0, 0], [0, 0], [0, 1]]),
mode='CONSTANT',
constant_values=1)
ensure_shape(samples, [-1, -1, 3])
batch_size, sample_count = samples.get_shape().as_list()[:2]
ensure_shape(grid, [batch_size, -1, -1, -1])
xyzw_samples = tf.ensure_shape(xyzw_samples, [batch_size, sample_count, 4])
grid_frame_samples = tf.matmul(xyzw_samples, world2grid,
transpose_b=True)[..., :3]
lower_coords = tf.floor(grid_frame_samples)
alpha = grid_frame_samples - lower_coords
min_alpha = 1e-05
max_alpha = 1 - 1e-05
alpha = tf.clip_by_value(alpha, min_alpha, max_alpha)
lower_coords = tf.cast(lower_coords, tf.int32)
upper_coords = tf.cast(tf.ceil(grid_frame_samples), tf.int32)
depth, height, width = grid.get_shape().as_list()[1:]
max_vals = np.array([[[width, height, depth]]], dtype=np.int32) - 1
max_vals = tf.constant(max_vals)
is_invalid = tf.logical_or(
tf.reduce_any(lower_coords < 0, axis=-1, keep_dims=True),
tf.reduce_any(upper_coords > max_vals, axis=-1, keep_dims=True))
log.info('is_invalid vs lower_coords: %s vs %s' %
(repr(is_invalid.get_shape().as_list()),
repr(lower_coords.get_shape().as_list())))
lower_coords = tf.where_v2(is_invalid, 0, lower_coords)
log.info('Post-where lower_coords: %s' %
repr(lower_coords.get_shape().as_list()))
upper_coords = tf.where_v2(is_invalid, 0, upper_coords)
lca = tf.split(lower_coords, 3, axis=-1)[::-1]
uca = tf.split(upper_coords, 3, axis=-1)[::-1]
aca = tf.unstack(alpha, axis=-1)[::-1]
lca[0] = tf.ensure_shape(lca[0], [batch_size, sample_count, 1])
lca[1] = tf.ensure_shape(lca[1], [batch_size, sample_count, 1])
lca[2] = tf.ensure_shape(lca[2], [batch_size, sample_count, 1])
batch_indices = np.arange(batch_size, dtype=np.int32)
batch_indices = np.reshape(batch_indices, [batch_size, 1, 1])
batch_indices = np.tile(batch_indices, [1, sample_count, 1])
batch_indices = tf.constant(batch_indices, dtype=tf.int32)
def batch_gather_nd(source, index_list):
return tf.gather_nd(source,
tf.concat([batch_indices] + index_list, axis=-1))
def lerp(lval, uval, alpha):
return lval * (1 - alpha) + uval * (alpha)
def lookup_and_lerp(lidx, uidx, alpha):
return lerp(batch_gather_nd(grid, lidx), batch_gather_nd(grid, uidx),
alpha)
c00 = lookup_and_lerp([lca[0], lca[1], lca[2]], [uca[0], lca[1], lca[2]],
aca[0])
c01 = lookup_and_lerp([lca[0], lca[1], uca[2]], [uca[0], lca[1], uca[2]],
aca[0])
c10 = lookup_and_lerp([lca[0], uca[1], lca[2]], [uca[0], uca[1], lca[2]],
aca[0])
c11 = lookup_and_lerp([lca[0], uca[1], uca[2]], [uca[0], uca[1], uca[2]],
aca[0])
c0 = lerp(c00, c10, aca[1])
c1 = lerp(c01, c11, aca[1])
sdf = tf.expand_dims(lerp(c0, c1, aca[2]), axis=-1)
log.info('is_invalid vs sdf coords: %s vs %s' % (repr(
is_invalid.get_shape().as_list()), repr(sdf.get_shape().as_list())))
sdf = tf.where_v2(is_invalid, 1e-5, sdf)
sdf = tf.ensure_shape(sdf, [batch_size, sample_count, 1])
return sdf, is_invalid
def _build_lstms(self):
# now the LSTMs
# these will collect the initial states for the forward
# (and reverse LSTMs if we are doing bidirectional)
# parse the options
lstm_dim = self.options['lstm']['dim']
projection_dim = self.options['lstm']['projection_dim']
n_lstm_layers = self.options['lstm'].get('n_layers', 1)
cell_clip = self.options['lstm'].get('cell_clip')
proj_clip = self.options['lstm'].get('proj_clip')
use_skip_connections = self.options['lstm']['use_skip_connections']
"""
if use_skip_connections:
print("USING SKIP CONNECTIONS")
else:
print("NOT USING SKIP CONNECTIONS")
"""
# the sequence lengths from input mask
if self.use_character_inputs:
mask = tf.reduce_any(self.ids_placeholder > 0, axis=2)
else:
mask = self.ids_placeholder > 0
sequence_lengths = tf.reduce_sum(tf.cast(mask, tf.int32), axis=1)
batch_size = tf.shape(sequence_lengths)[0]
# for each direction, we'll store tensors for each layer
self.lstm_outputs = {'forward': [], 'backward': []}
self.lstm_state_sizes = {'forward': [], 'backward': []}
self.lstm_init_states = {'forward': [], 'backward': []}
self.lstm_final_states = {'forward': [], 'backward': []}
update_ops = []
for direction in ['forward', 'backward']:
if direction == 'forward':
layer_input = self.embedding
else:
layer_input = tf.reverse_sequence(self.embedding,
sequence_lengths,
seq_axis=1,
batch_axis=0)
for i in range(n_lstm_layers):
if projection_dim < lstm_dim:
# are projecting down output
lstm_cell = tf.nn.rnn_cell.LSTMCell(
lstm_dim,
num_proj=projection_dim,
cell_clip=cell_clip,
proj_clip=proj_clip)
else:
lstm_cell = tf.nn.rnn_cell.LSTMCell(lstm_dim,
cell_clip=cell_clip,
proj_clip=proj_clip)
if use_skip_connections:
# ResidualWrapper adds inputs to outputs
if i == 0:
# don't add skip connection from token embedding to
# 1st layer output
pass
else:
# add a skip connection
lstm_cell = tf.nn.rnn_cell.ResidualWrapper(lstm_cell)
# collect the input state, run the dynamic rnn, collect
# the output
state_size = lstm_cell.state_size
# the LSTMs are stateful. To support multiple batch sizes,
# we'll allocate size for states up to max_batch_size,
# then use the first batch_size entries for each batch
init_states = [
tf.Variable(tf.zeros([self._max_batch_size, dim]),
trainable=False)
for dim in lstm_cell.state_size
]
batch_init_states = [
state[:batch_size, :] for state in init_states
]
if direction == 'forward':
i_direction = 0
else:
i_direction = 1
variable_scope_name = 'RNN_{0}/RNN/MultiRNNCell/Cell{1}'.format(
i_direction, i)
with tf.variable_scope(variable_scope_name):
layer_output, final_state = tf.nn.dynamic_rnn(
lstm_cell,
layer_input,
sequence_length=sequence_lengths,
initial_state=tf.nn.rnn_cell.LSTMStateTuple(
*batch_init_states),
)
self.lstm_state_sizes[direction].append(lstm_cell.state_size)
self.lstm_init_states[direction].append(init_states)
self.lstm_final_states[direction].append(final_state)
if direction == 'forward':
self.lstm_outputs[direction].append(layer_output)
else:
self.lstm_outputs[direction].append(
tf.reverse_sequence(layer_output,
sequence_lengths,
seq_axis=1,
batch_axis=0))
with tf.control_dependencies([layer_output]):
# update the initial states
for i in range(2):
new_state = tf.concat([
final_state[i][:batch_size, :],
init_states[i][batch_size:, :]
],
axis=0)
state_update_op = tf.assign(init_states[i], new_state)
update_ops.append(state_update_op)
layer_input = layer_output
self.mask = mask
self.sequence_lengths = sequence_lengths
self.update_state_op = tf.group(*update_ops)
def beam_search(symbols_to_logits_fn,
initial_ids,
beam_size,
decode_length,
vocab_size,
alpha,
states=None,
eos_id=EOS_ID):
"""Beam search with length penalties.
Requires a function that can take the currently decoded sybmols and return
the logits for the next symbol. The implementation is inspired by
https://arxiv.org/abs/1609.08144.
When running, the beam search steps can be visualized by using tfdbg to watch
the operations generating the output ids for each beam step. These operations
have the pattern:
(alive|finished)_topk_(seq,scores)
Operations marked `alive` represent the new beam sequences that will be
processed in the next step. Operations marked `finished` represent the
completed beam sequences, which may be padded with 0s if no beams finished.
Operations marked `seq` store the full beam sequence for the time step.
Operations marked `scores` store the sequence's final log scores.
The beam search steps will be processed sequentially in order, so when
capturing observed from these operations, tensors, clients can make
assumptions about which step is being recorded.
Args:
symbols_to_logits_fn: Interface to the model, to provide logits.
Shoud take [batch_size, decoded_ids] and return [batch_size, vocab_size]
initial_ids: Ids to start off the decoding, this will be the first thing
handed to symbols_to_logits_fn (after expanding to beam size)
[batch_size]
beam_size: Size of the beam.
decode_length: Number of steps to decode for.
vocab_size: Size of the vocab, must equal the size of the logits returned by
symbols_to_logits_fn
alpha: alpha for length penalty.
eos_id: ID for end of sentence.
states: dict (possibly nested) of decoding states.
Returns:
Tuple of
(decoded beams [batch_size, beam_size, decode_length]
decoding probablities [batch_size, beam_size])
"""
batch_size = tf.shape(initial_ids)[0]
# Assume initial_ids are prob 1.0
initial_log_probs = tf.constant([[0.] + [-float("inf")] * (beam_size - 1)])
# Expand to beam_size (batch_size, beam_size)
alive_log_probs = tf.tile(initial_log_probs, [batch_size, 1])
# Expand each batch and state to beam_size
alive_seq = expand_to_beam_size(initial_ids, beam_size)
alive_seq = tf.expand_dims(alive_seq, axis=2) # (batch_size, beam_size, 1)
if states:
states = nest.map_structure(
lambda state: expand_to_beam_size(state, beam_size), states)
else:
states = {}
# Finished will keep track of all the sequences that have finished so far
# Finished log probs will be negative infinity in the beginning
# finished_flags will keep track of booleans
finished_seq = tf.zeros(tf.shape(alive_seq), tf.int32)
# Setting the scores of the initial to negative infinity.
finished_scores = tf.ones([batch_size, beam_size]) * -INF
finished_flags = tf.zeros([batch_size, beam_size], tf.bool)
def grow_finished(finished_seq, finished_scores, finished_flags, curr_seq,
curr_scores, curr_finished):
"""Given sequences and scores, will gather the top k=beam size sequences.
Args:
finished_seq: Current finished sequences.
[batch_size, beam_size, current_decoded_length]
finished_scores: scores for each of these sequences.
[batch_size, beam_size]
finished_flags: finished bools for each of these sequences.
[batch_size, beam_size]
curr_seq: current topk sequence that has been grown by one position.
[batch_size, beam_size, current_decoded_length]
curr_scores: scores for each of these sequences. [batch_size, beam_size]
curr_finished: Finished flags for each of these sequences.
[batch_size, beam_size]
Returns:
Tuple of
(Topk sequences based on scores,
log probs of these sequences,
Finished flags of these sequences)
"""
# First append a column of 0'ids to finished to make the same length with
# finished scores
finished_seq = tf.concat(
[finished_seq,
tf.zeros([batch_size, beam_size, 1], tf.int32)],
axis=2)
# Set the scores of the unfinished seq in curr_seq to large negative
# values
curr_scores += (1. - tf.to_float(curr_finished)) * -INF
# concatenating the sequences and scores along beam axis
curr_finished_seq = tf.concat([finished_seq, curr_seq], axis=1)
curr_finished_scores = tf.concat([finished_scores, curr_scores],
axis=1)
curr_finished_flags = tf.concat([finished_flags, curr_finished],
axis=1)
return compute_topk_scores_and_seq(curr_finished_seq,
curr_finished_scores,
curr_finished_scores,
curr_finished_flags, beam_size,
batch_size, "grow_finished")
def grow_alive(curr_seq, curr_scores, curr_log_probs, curr_finished,
states):
"""Given sequences and scores, will gather the top k=beam size sequences.
Args:
curr_seq: current topk sequence that has been grown by one position.
[batch_size, beam_size, i+1]
curr_scores: scores for each of these sequences. [batch_size, beam_size]
curr_log_probs: log probs for each of these sequences.
[batch_size, beam_size]
curr_finished: Finished flags for each of these sequences.
[batch_size, beam_size]
states: dict (possibly nested) of decoding states.
Returns:
Tuple of
(Topk sequences based on scores,
log probs of these sequences,
Finished flags of these sequences)
"""
# Set the scores of the finished seq in curr_seq to large negative
# values
curr_scores += tf.to_float(curr_finished) * -INF
return compute_topk_scores_and_seq(curr_seq, curr_scores,
curr_log_probs, curr_finished,
beam_size, batch_size, "grow_alive",
states)
def grow_topk(i, alive_seq, alive_log_probs, states):
r"""Inner beam seach loop.
This function takes the current alive sequences, and grows them to topk
sequences where k = 2*beam. We use 2*beam because, we could have beam_size
number of sequences that might hit <EOS> and there will be no alive
sequences to continue. With 2*beam_size, this will not happen. This relies
on the assumption the vocab size is > beam size. If this is true, we'll
have at least beam_size non <EOS> extensions if we extract the next top
2*beam words.
Length penalty is given by = (5+len(decode)/6) ^ -\alpha. Pls refer to
https://arxiv.org/abs/1609.08144.
Args:
i: loop index
alive_seq: Topk sequences decoded so far [batch_size, beam_size, i+1]
alive_log_probs: probabilities of these sequences. [batch_size, beam_size]
states: dict (possibly nested) of decoding states.
Returns:
Tuple of
(Topk sequences extended by the next word,
The log probs of these sequences,
The scores with length penalty of these sequences,
Flags indicating which of these sequences have finished decoding,
dict of transformed decoding states)
"""
# Get the logits for all the possible next symbols
flat_ids = tf.reshape(alive_seq, [batch_size * beam_size, -1])
# (batch_size * beam_size, decoded_length)
if states:
flat_states = nest.map_structure(
lambda state: tf.reshape(state, [batch_size * beam_size, -1]),
states)
flat_logits, flat_states = symbols_to_logits_fn(
flat_ids, flat_states)
states = nest.map_structure(
lambda state: tf.reshape(state, [batch_size, beam_size, -1]),
flat_states)
else:
flat_logits = symbols_to_logits_fn(flat_ids)
logits = tf.reshape(flat_logits, (batch_size, beam_size, -1))
# Convert logits to normalized log probs
candidate_log_probs = log_prob_from_logits(logits)
# Multiply the probabilites by the current probabilites of the beam.
# (batch_size, beam_size, vocab_size) + (batch_size, beam_size, 1)
log_probs = candidate_log_probs + tf.expand_dims(alive_log_probs,
axis=2)
length_penalty = tf.pow(((5. + tf.to_float(i + 1)) / 6.), alpha)
curr_scores = log_probs / length_penalty
# Flatten out (beam_size, vocab_size) probs in to a list of possibilites
flat_curr_scores = tf.reshape(curr_scores,
[-1, beam_size * vocab_size])
topk_scores, topk_ids = tf.nn.top_k(flat_curr_scores, k=beam_size * 2)
# Recovering the log probs because we will need to send them back
topk_log_probs = topk_scores * length_penalty
# Work out what beam the top probs are in.
topk_beam_index = topk_ids // vocab_size
topk_ids %= vocab_size # Unflatten the ids
# The next three steps are to create coordinates for tf.gather_nd to pull
# out the correct seqences from id's that we need to grow.
# We will also use the coordinates to gather the booleans of the beam items
# that survived.
batch_pos = compute_batch_indices(batch_size, beam_size * 2)
# top beams will give us the actual coordinates to do the gather.
# stacking will create a tensor of dimension batch * beam * 2, where the
# last dimension contains the i,j gathering coordinates.
topk_coordinates = tf.stack([batch_pos, topk_beam_index], axis=2)
# Gather up the most probable 2*beams both for the ids and finished_in_alive
# bools
topk_seq = tf.gather_nd(alive_seq, topk_coordinates)
if states:
states = nest.map_structure(
lambda state: tf.gather_nd(state, topk_coordinates), states)
# Append the most probable alive
topk_seq = tf.concat(
[topk_seq, tf.expand_dims(topk_ids, axis=2)], axis=2)
topk_finished = tf.equal(topk_ids, eos_id)
return topk_seq, topk_log_probs, topk_scores, topk_finished, states
def inner_loop(i, alive_seq, alive_log_probs, finished_seq,
finished_scores, finished_flags, states):
"""Inner beam seach loop.
There are three groups of tensors, alive, finished, and topk.
The alive group contains information about the current alive sequences
The topk group contains information about alive + topk current decoded words
the finished group contains information about finished sentences, that is,
the ones that have decoded to <EOS>. These are what we return.
The general beam search algorithm is as follows:
While we haven't terminated (pls look at termination condition)
1. Grow the current alive to get beam*2 topk sequences
2. Among the topk, keep the top beam_size ones that haven't reached EOS
into alive
3. Among the topk, keep the top beam_size ones have reached EOS into
finished
Repeat
To make things simple with using fixed size tensors, we will end
up inserting unfinished sequences into finished in the beginning. To stop
that we add -ve INF to the score of the unfinished sequence so that when a
true finished sequence does appear, it will have a higher score than all the
unfinished ones.
Args:
i: loop index
alive_seq: Topk sequences decoded so far [batch_size, beam_size, i+1]
alive_log_probs: probabilities of the beams. [batch_size, beam_size]
finished_seq: Current finished sequences.
[batch_size, beam_size, i+1]
finished_scores: scores for each of these sequences.
[batch_size, beam_size]
finished_flags: finished bools for each of these sequences.
[batch_size, beam_size]
states: dict (possibly nested) of decoding states.
Returns:
Tuple of
(Incremented loop index
New alive sequences,
Log probs of the alive sequences,
New finished sequences,
Scores of the new finished sequences,
Flags inidicating which sequence in finished as reached EOS,
dict of final decoding states)
"""
# Each inner loop, we carry out three steps:
# 1. Get the current topk items.
# 2. Extract the ones that have finished and haven't finished
# 3. Recompute the contents of finished based on scores.
topk_seq, topk_log_probs, topk_scores, topk_finished, states = grow_topk(
i, alive_seq, alive_log_probs, states)
alive_seq, alive_log_probs, _, states = grow_alive(
topk_seq, topk_scores, topk_log_probs, topk_finished, states)
finished_seq, finished_scores, finished_flags, _ = grow_finished(
finished_seq, finished_scores, finished_flags, topk_seq,
topk_scores, topk_finished)
return (i + 1, alive_seq, alive_log_probs, finished_seq,
finished_scores, finished_flags, states)
def _is_finished(i, unused_alive_seq, alive_log_probs, unused_finished_seq,
finished_scores, finished_in_finished, unused_states):
"""Checking termination condition.
We terminate when we decoded up to decode_length or the lowest scoring item
in finished has a greater score that the higest prob item in alive divided
by the max length penalty
Args:
i: loop index
alive_log_probs: probabilities of the beams. [batch_size, beam_size]
finished_scores: scores for each of these sequences.
[batch_size, beam_size]
finished_in_finished: finished bools for each of these sequences.
[batch_size, beam_size]
Returns:
Bool.
"""
max_length_penalty = tf.pow(((5. + tf.to_float(decode_length)) / 6.),
alpha)
# The best possible score of the most likley alive sequence
lower_bound_alive_scores = alive_log_probs[:, 0] / max_length_penalty
# Now to compute the lowest score of a finished sequence in finished
# If the sequence isn't finished, we multiply it's score by 0. since
# scores are all -ve, taking the min will give us the score of the lowest
# finished item.
lowest_score_of_fininshed_in_finished = tf.reduce_min(
finished_scores * tf.to_float(finished_in_finished), axis=1)
# If none of the sequences have finished, then the min will be 0 and
# we have to replace it by -ve INF if it is. The score of any seq in alive
# will be much higher than -ve INF and the termination condition will not
# be met.
lowest_score_of_fininshed_in_finished += (
(1. - tf.to_float(tf.reduce_any(finished_in_finished, 1))) * -INF)
bound_is_met = tf.reduce_all(
tf.greater(lowest_score_of_fininshed_in_finished,
lower_bound_alive_scores))
return tf.logical_and(tf.less(i, decode_length),
tf.logical_not(bound_is_met))
(_, alive_seq, alive_log_probs, finished_seq, finished_scores,
finished_flags, _) = tf.while_loop(
_is_finished,
inner_loop, [
tf.constant(0), alive_seq, alive_log_probs, finished_seq,
finished_scores, finished_flags, states
],
shape_invariants=[
tf.TensorShape([]),
tf.TensorShape([None, None, None]),
alive_log_probs.get_shape(),
tf.TensorShape([None, None, None]),
finished_scores.get_shape(),
finished_flags.get_shape(),
nest.map_structure(
lambda tensor: tf.TensorShape([None] * tensor.shape.ndims),
states),
],
parallel_iterations=1,
back_prop=False)
alive_seq.set_shape((None, beam_size, None))
finished_seq.set_shape((None, beam_size, None))
# Accounting for corner case: It's possible that no sequence in alive for a
# particular batch item ever reached EOS. In that case, we should just copy
# the contents of alive for that batch item. tf.reduce_any(finished_flags, 1)
# if 0, means that no sequence for that batch index had reached EOS. We need
# to do the same for the scores as well.
finished_seq = tf.where(tf.reduce_any(finished_flags, 1), finished_seq,
alive_seq)
finished_scores = tf.where(tf.reduce_any(finished_flags, 1),
finished_scores, alive_log_probs)
return finished_seq, finished_scores
def compute_from_emb(candidate_span_emb):
with tf.variable_scope("prediction_scope", reuse=tf.AUTO_REUSE):
candidate_mention_scores = self.get_mention_scores(
candidate_span_emb) # [k, 1]
candidate_mention_scores = tf.squeeze(candidate_mention_scores,
1) # [k]
k = tf.to_int32(
tf.floor(
tf.to_float(tf.shape(context_outputs)[0]) *
self.config["top_span_ratio"]))
top_span_indices = coref_ops.extract_spans(
tf.expand_dims(candidate_mention_scores, 0),
tf.expand_dims(candidate_starts, 0),
tf.expand_dims(candidate_ends, 0), tf.expand_dims(k, 0),
util.shape(context_outputs, 0), True) # [1, k]
top_span_indices.set_shape([1, None])
top_span_indices = tf.squeeze(top_span_indices, 0) # [k]
top_span_starts = tf.gather(candidate_starts,
top_span_indices) # [k]
top_span_ends = tf.gather(candidate_ends,
top_span_indices) # [k]
top_span_emb = tf.gather(candidate_span_emb,
top_span_indices) # [k, emb]
top_span_cluster_ids = tf.gather(candidate_cluster_ids,
top_span_indices) # [k]
top_span_mention_scores = tf.gather(candidate_mention_scores,
top_span_indices) # [k]
top_span_sentence_indices = tf.gather(
candidate_sentence_indices, top_span_indices) # [k]
top_span_speaker_ids = tf.gather(speaker_ids,
top_span_starts) # [k]
self.head_scores = tf.gather(
candidate_head_scores,
top_span_indices) # [k, max_span_width]
c = tf.minimum(self.config["max_top_antecedents"], k)
if self.config["coarse_to_fine"]:
top_antecedents, top_antecedents_mask, top_fast_antecedent_scores, top_antecedent_offsets = self.coarse_to_fine_pruning(
top_span_emb, top_span_mention_scores, c)
else:
top_antecedents, top_antecedents_mask, top_fast_antecedent_scores, top_antecedent_offsets = self.distance_pruning(
top_span_emb, top_span_mention_scores, c)
dummy_scores = tf.zeros([k, 1]) # [k, 1]
for i in range(self.config["coref_depth"]):
with tf.variable_scope("coref_layer", reuse=tf.AUTO_REUSE):
top_antecedent_emb = tf.gather(
top_span_emb, top_antecedents) # [k, c, emb]
top_antecedent_scores = top_fast_antecedent_scores + self.get_slow_antecedent_scores(
top_span_emb, top_antecedents, top_antecedent_emb,
top_antecedent_offsets, top_span_speaker_ids,
genre_emb) # [k, c]
top_antecedent_weights = tf.nn.softmax(
tf.concat([dummy_scores, top_antecedent_scores],
1)) # [k, c + 1]
top_antecedent_emb = tf.concat([
tf.expand_dims(top_span_emb, 1), top_antecedent_emb
], 1) # [k, c + 1, emb]
attended_span_emb = tf.reduce_sum(
tf.expand_dims(top_antecedent_weights, 2) *
top_antecedent_emb, 1) # [k, emb]
with tf.variable_scope("f"):
f = tf.sigmoid(
util.projection(
tf.concat(
[top_span_emb, attended_span_emb], 1),
util.shape(top_span_emb, -1))) # [k, emb]
top_span_emb = f * attended_span_emb + (
1 - f) * top_span_emb # [k, emb]
top_antecedent_scores = tf.concat(
[dummy_scores, top_antecedent_scores], 1) # [k, c + 1]
top_antecedent_cluster_ids = tf.gather(
top_span_cluster_ids, top_antecedents) # [k, c]
top_antecedent_cluster_ids += tf.to_int32(
tf.log(tf.to_float(top_antecedents_mask))) # [k, c]
same_cluster_indicator = tf.equal(top_antecedent_cluster_ids,
tf.expand_dims(
top_span_cluster_ids,
1)) # [k, c]
non_dummy_indicator = tf.expand_dims(top_span_cluster_ids > 0,
1) # [k, 1]
pairwise_labels = tf.logical_and(same_cluster_indicator,
non_dummy_indicator) # [k, c]
dummy_labels = tf.logical_not(
tf.reduce_any(pairwise_labels, 1, keepdims=True)) # [k, 1]
top_antecedent_labels = tf.concat(
[dummy_labels, pairwise_labels], 1) # [k, c + 1]
loss = self.softmax_loss(top_antecedent_scores,
top_antecedent_labels) # [k]
loss = tf.reduce_sum(loss) # []
return [
candidate_starts, candidate_ends, candidate_mention_scores,
top_span_starts, top_span_ends, top_antecedents,
top_antecedent_scores
], loss
# gold_standard: The real answers.
gold_standard = tf.placeholder(tf.float32, [None, 1, D], "answer")
with tf.variable_scope('accuracy'):
eq = tf.equal(context, gold_standard)
corrbool = tf.reduce_all(eq, -1)
logloc = tf.reduce_max(logits, -1, keep_dims=True)
# locs: A boolean tensor that indicates where the score
# matches the minimum score. This happens on multiple dimensions,
# so in the off chance there's one or two indexes that match
# we make sure it matches in all indexes.
locs = tf.equal(logits, logloc)
# correctsbool: A boolean tensor that indicates for which
# words in the context the score always matches the minimum score.
correctsbool = tf.reduce_any(tf.logical_and(locs, corrbool), -1)
# corrects: A tensor that is simply correctsbool cast to floats.
corrects = tf.where(correctsbool,
tf.ones_like(correctsbool, dtype=tf.float32),
tf.zeros_like(correctsbool, dtype=tf.float32))
# corr: corrects, but for the right answer instead of our selected answer.
corr = tf.where(corrbool, tf.ones_like(corrbool, dtype=tf.float32),
tf.zeros_like(corrbool, dtype=tf.float32))
with tf.variable_scope("loss"):
# Use sigmoid cross entropy as the base loss,
# with our distances as the relative probabilities. There are
# multiple correct labels, for each location of the answer word within the context.
loss = tf.nn.sigmoid_cross_entropy_with_logits(logits=tf.nn.l2_normalize(
logits, -1),
labels=corr)
def tf_enqueue(self, baseline=None, **values):
# Constants and parameters
zero = tf.constant(value=0, dtype=util.tf_dtype(dtype='long'))
one = tf.constant(value=1, dtype=util.tf_dtype(dtype='long'))
capacity = tf.constant(value=self.capacity,
dtype=util.tf_dtype(dtype='long'))
horizon = self.horizon.value()
discount = self.discount.value()
assertions = list()
# Check whether horizon at most capacity
assertions.append(tf.debugging.assert_less_equal(x=horizon,
y=capacity))
# Check whether at most one terminal
assertions.append(
tf.debugging.assert_less_equal(x=tf.count_nonzero(
input_tensor=values['terminal'],
dtype=util.tf_dtype(dtype='long')),
y=one))
# Check whether, if any, last value is terminal
assertions.append(
tf.debugging.assert_equal(x=tf.reduce_any(
input_tensor=tf.math.greater(x=values['terminal'], y=zero)),
y=tf.math.greater(
x=values['terminal'][-1], y=zero)))
# Get number of overwritten values
with tf.control_dependencies(control_inputs=assertions):
value = values['terminal']
if util.tf_dtype(dtype='long') in (tf.int32, tf.int64):
num_values = tf.shape(input=value,
out_type=util.tf_dtype(dtype='long'))[0]
else:
num_values = tf.dtypes.cast(x=tf.shape(input=value)[0],
dtype=util.tf_dtype(dtype='long'))
start = tf.maximum(x=self.buffer_index, y=capacity)
limit = tf.maximum(x=(self.buffer_index + num_values), y=capacity)
num_overwritten = limit - start
def update_overwritten_rewards():
# Get relevant buffer rewards
buffer_limit = self.buffer_index + tf.minimum(
x=(num_overwritten + horizon), y=capacity)
buffer_indices = tf.range(start=self.buffer_index,
limit=buffer_limit)
buffer_indices = tf.mod(x=buffer_indices, y=capacity)
rewards = tf.gather(params=self.buffers['reward'],
indices=buffer_indices)
# Get relevant values rewards
values_limit = tf.maximum(x=(num_overwritten + horizon - capacity),
y=zero)
rewards = tf.concat(values=(rewards,
values['reward'][:values_limit]),
axis=0)
# Horizon baseline value
if self.estimate_horizon == 'early' and baseline is not None:
assert baseline is not None
# Baseline estimate
buffer_indices = buffer_indices[horizon + one:]
states = OrderedDict()
for name, buffer in self.buffers['states'].items():
state = tf.gather(params=buffer, indices=buffer_indices)
states[name] = tf.concat(
values=(state, values['states'][name][:values_limit]),
axis=0)
internals = OrderedDict()
for name, buffer in self.buffers['internals'].items():
internal = tf.gather(params=buffer, indices=buffer_indices)
internals[name] = tf.concat(
values=(internal,
values['internals'][name][:values_limit]),
axis=0)
auxiliaries = OrderedDict()
for name, buffer in self.buffers['auxiliaries'].items():
auxiliary = tf.gather(params=buffer,
indices=buffer_indices)
auxiliaries[name] = tf.concat(
values=(auxiliary,
values['auxiliaries'][name][:values_limit]),
axis=0)
# Dependency horizon
# TODO: handle arbitrary non-optimization horizons!
dependency_horizon = baseline.dependency_horizon(
is_optimization=False)
assertion = tf.debugging.assert_equal(x=dependency_horizon,
y=zero)
with tf.control_dependencies(control_inputs=(assertion, )):
some_state = next(iter(states.values()))
if util.tf_dtype(dtype='long') in (tf.int32, tf.int64):
batch_size = tf.shape(
input=some_state,
out_type=util.tf_dtype(dtype='long'))[0]
else:
batch_size = tf.dtypes.cast(
x=tf.shape(input=some_state)[0],
dtype=util.tf_dtype(dtype='long'))
starts = tf.range(start=batch_size,
dtype=util.tf_dtype(dtype='long'))
lengths = tf.ones(shape=(batch_size, ),
dtype=util.tf_dtype(dtype='long'))
Module.update_tensors(dependency_starts=starts,
dependency_lengths=lengths)
if self.estimate_actions:
actions = OrderedDict()
for name, buffer in self.buffers['actions'].items():
action = tf.gather(params=buffer,
indices=buffer_indices)
actions[name] = tf.concat(
values=(action,
values['actions'][name][:values_limit]),
axis=0)
horizon_estimate = baseline.actions_value(
states=states,
internals=internals,
auxiliaries=auxiliaries,
actions=actions)
else:
horizon_estimate = baseline.states_value(
states=states,
internals=internals,
auxiliaries=auxiliaries)
else:
# Zero estimate
horizon_estimate = tf.zeros(shape=(num_overwritten, ),
dtype=util.tf_dtype(dtype='float'))
# Calculate discounted sum
def cond(discounted_sum, horizon):
return tf.math.greater_equal(x=horizon, y=zero)
def body(discounted_sum, horizon):
# discounted_sum = tf.Print(discounted_sum, (horizon, discounted_sum, rewards[horizon:]), summarize=10)
discounted_sum = discount * discounted_sum
discounted_sum = discounted_sum + rewards[horizon:horizon +
num_overwritten]
return discounted_sum, horizon - one
discounted_sum = self.while_loop(cond=cond,
body=body,
loop_vars=(horizon_estimate,
horizon),
back_prop=False)[0]
assertions = list()
assertions.append(
tf.debugging.assert_equal(x=tf.shape(input=horizon_estimate),
y=tf.shape(input=discounted_sum)))
assertions.append(
tf.debugging.assert_equal(x=tf.shape(
input=rewards, out_type=util.tf_dtype(dtype='long'))[0],
y=(horizon + num_overwritten)))
# Overwrite buffer rewards
with tf.control_dependencies(control_inputs=assertions):
indices = tf.range(start=self.buffer_index,
limit=(self.buffer_index + num_overwritten))
indices = tf.mod(x=indices, y=capacity)
indices = tf.expand_dims(input=indices, axis=1)
assignment = self.buffers['reward'].scatter_nd_update(
indices=indices, updates=discounted_sum)
with tf.control_dependencies(control_inputs=(assignment, )):
return util.no_operation()
any_overwritten = tf.math.greater(x=num_overwritten, y=zero)
updated_rewards = self.cond(pred=any_overwritten,
true_fn=update_overwritten_rewards,
false_fn=util.no_operation)
with tf.control_dependencies(control_inputs=(updated_rewards, )):
return super().tf_enqueue(**values)
def mesh_renderer(vertices,
triangles,
normals,
diffuse_colors,
camera_position,
camera_lookat,
camera_up,
light_positions,
light_intensities,
image_width,
image_height,
specular_colors=None,
shininess_coefficients=None,
ambient_color=None,
fov_y=40.0,
near_clip=0.01,
far_clip=10.0):
"""Renders an input scene using phong shading, and returns an output image.
Args:
vertices: 3-D float32 tensor with shape [batch_size, vertex_count, 3]. Each
triplet is an xyz position in world space.
triangles: 2-D int32 tensor with shape [triangle_count, 3]. Each triplet
should contain vertex indices describing a triangle such that the
triangle's normal points toward the viewer if the forward order of the
triplet defines a clockwise winding of the vertices. Gradients with
respect to this tensor are not available.
normals: 3-D float32 tensor with shape [batch_size, vertex_count, 3]. Each
triplet is the xyz vertex normal for its corresponding vertex. Each
vector is assumed to be already normalized.
diffuse_colors: 3-D float32 tensor with shape [batch_size,
vertex_count, 3]. The RGB diffuse reflection in the range [0,1] for
each vertex.
camera_position: 2-D tensor with shape [batch_size, 3] or 1-D tensor with
shape [3] specifying the XYZ world space camera position.
camera_lookat: 2-D tensor with shape [batch_size, 3] or 1-D tensor with
shape [3] containing an XYZ point along the center of the camera's gaze.
camera_up: 2-D tensor with shape [batch_size, 3] or 1-D tensor with shape
[3] containing the up direction for the camera. The camera will have no
tilt with respect to this direction.
light_positions: a 3-D tensor with shape [batch_size, light_count, 3]. The
XYZ position of each light in the scene. In the same coordinate space as
pixel_positions.
light_intensities: a 3-D tensor with shape [batch_size, light_count, 3]. The
RGB intensity values for each light. Intensities may be above one.
image_width: int specifying desired output image width in pixels.
image_height: int specifying desired output image height in pixels.
specular_colors: 3-D float32 tensor with shape [batch_size,
vertex_count, 3]. The RGB specular reflection in the range [0, 1] for
each vertex. If supplied, specular reflections will be computed, and
both specular_colors and shininess_coefficients are expected.
shininess_coefficients: a 0D-2D float32 tensor with maximum shape
[batch_size, vertex_count]. The phong shininess coefficient of each
vertex. A 0D tensor or float gives a constant shininess coefficient
across all batches and images. A 1D tensor must have shape [batch_size],
and a single shininess coefficient per image is used.
ambient_color: a 2D tensor with shape [batch_size, 3]. The RGB ambient
color, which is added to each pixel in the scene. If None, it is
assumed to be black.
fov_y: float, 0D tensor, or 1D tensor with shape [batch_size] specifying
desired output image y field of view in degrees.
near_clip: float, 0D tensor, or 1D tensor with shape [batch_size] specifying
near clipping plane distance.
far_clip: float, 0D tensor, or 1D tensor with shape [batch_size] specifying
far clipping plane distance.
Returns:
A 4-D float32 tensor of shape [batch_size, image_height, image_width, 4]
containing the lit RGBA color values for each image at each pixel. RGB
colors are the intensity values before tonemapping and can be in the range
[0, infinity]. Clipping to the range [0,1] with tf.clip_by_value is likely
reasonable for both viewing and training most scenes. More complex scenes
with multiple lights should tone map color values for display only. One
simple tonemapping approach is to rescale color values as x/(1+x); gamma
compression is another common techinque. Alpha values are zero for
background pixels and near one for mesh pixels.
Raises:
ValueError: An invalid argument to the method is detected.
"""
if len(vertices.shape) != 3:
raise ValueError(
'Vertices must have shape [batch_size, vertex_count, 3].')
batch_size = vertices.shape[0].value
if len(normals.shape) != 3:
raise ValueError(
'Normals must have shape [batch_size, vertex_count, 3].')
if len(light_positions.shape) != 3:
raise ValueError(
'Light_positions must have shape [batch_size, light_count, 3].')
if len(light_intensities.shape) != 3:
raise ValueError(
'Light_intensities must have shape [batch_size, light_count, 3].')
if len(diffuse_colors.shape) != 3:
raise ValueError(
'vertex_diffuse_colors must have shape [batch_size, vertex_count, 3].'
)
if (ambient_color is not None
and ambient_color.get_shape().as_list() != [batch_size, 3]):
raise ValueError('Ambient_color must have shape [batch_size, 3].')
if camera_position.get_shape().as_list() == [3]:
camera_position = tf.tile(tf.expand_dims(camera_position, axis=0),
[batch_size, 1])
elif camera_position.get_shape().as_list() != [batch_size, 3]:
raise ValueError('Camera_position must have shape [batch_size, 3]')
if camera_lookat.get_shape().as_list() == [3]:
camera_lookat = tf.tile(tf.expand_dims(camera_lookat, axis=0),
[batch_size, 1])
elif camera_lookat.get_shape().as_list() != [batch_size, 3]:
raise ValueError('Camera_lookat must have shape [batch_size, 3]')
if camera_up.get_shape().as_list() == [3]:
camera_up = tf.tile(tf.expand_dims(camera_up, axis=0), [batch_size, 1])
elif camera_up.get_shape().as_list() != [batch_size, 3]:
raise ValueError('Camera_up must have shape [batch_size, 3]')
if isinstance(fov_y, float):
fov_y = tf.constant(batch_size * [fov_y], dtype=tf.float32)
elif not fov_y.get_shape().as_list():
fov_y = tf.tile(tf.expand_dims(fov_y, 0), [batch_size])
elif fov_y.get_shape().as_list() != [batch_size]:
raise ValueError(
'Fov_y must be a float, a 0D tensor, or a 1D tensor with'
'shape [batch_size]')
if isinstance(near_clip, float):
near_clip = tf.constant(batch_size * [near_clip], dtype=tf.float32)
elif not near_clip.get_shape().as_list():
near_clip = tf.tile(tf.expand_dims(near_clip, 0), [batch_size])
elif near_clip.get_shape().as_list() != [batch_size]:
raise ValueError(
'Near_clip must be a float, a 0D tensor, or a 1D tensor'
'with shape [batch_size]')
if isinstance(far_clip, float):
far_clip = tf.constant(batch_size * [far_clip], dtype=tf.float32)
elif not far_clip.get_shape().as_list():
far_clip = tf.tile(tf.expand_dims(far_clip, 0), [batch_size])
elif far_clip.get_shape().as_list() != [batch_size]:
raise ValueError(
'Far_clip must be a float, a 0D tensor, or a 1D tensor'
'with shape [batch_size]')
if specular_colors is not None and shininess_coefficients is None:
raise ValueError(
'Specular colors were supplied without shininess coefficients.')
if shininess_coefficients is not None and specular_colors is None:
raise ValueError(
'Shininess coefficients were supplied without specular colors.')
if specular_colors is not None:
# Since a 0-D float32 tensor is accepted, also accept a float.
if isinstance(shininess_coefficients, float):
shininess_coefficients = tf.constant(shininess_coefficients,
dtype=tf.float32)
if len(specular_colors.shape) != 3:
raise ValueError(
'The specular colors must have shape [batch_size, '
'vertex_count, 3].')
if len(shininess_coefficients.shape) > 2:
raise ValueError(
'The shininess coefficients must have shape at most'
'[batch_size, vertex_count].')
# If we don't have per-vertex coefficients, we can just reshape the
# input shininess to broadcast later, rather than interpolating an
# additional vertex attribute:
if len(shininess_coefficients.shape) < 2:
vertex_attributes = tf.concat(
[normals, vertices, diffuse_colors, specular_colors], axis=2)
else:
vertex_attributes = tf.concat([
normals, vertices, diffuse_colors, specular_colors,
tf.expand_dims(shininess_coefficients, axis=2)
],
axis=2)
else:
vertex_attributes = tf.concat([normals, vertices, diffuse_colors],
axis=2)
camera_matrices = camera_utils.look_at(camera_position, camera_lookat,
camera_up)
perspective_transforms = camera_utils.perspective(
image_width / image_height, fov_y, near_clip, far_clip)
clip_space_transforms = tf.matmul(perspective_transforms, camera_matrices)
pixel_attributes = rasterize_triangles.rasterize_triangles(
vertices, vertex_attributes, triangles, clip_space_transforms,
image_width, image_height, [-1] * vertex_attributes.shape[2].value)
# Extract the interpolated vertex attributes from the pixel buffer and
# supply them to the shader:
pixel_normals = tf.nn.l2_normalize(pixel_attributes[:, :, :, 0:3], axis=3)
pixel_positions = pixel_attributes[:, :, :, 3:6]
diffuse_colors = pixel_attributes[:, :, :, 6:9]
if specular_colors is not None:
specular_colors = pixel_attributes[:, :, :, 9:12]
# Retrieve the interpolated shininess coefficients if necessary, or just
# reshape our input for broadcasting:
if len(shininess_coefficients.shape) == 2:
shininess_coefficients = pixel_attributes[:, :, :, 12]
else:
shininess_coefficients = tf.reshape(shininess_coefficients,
[-1, 1, 1])
pixel_mask = tf.cast(tf.reduce_any(diffuse_colors >= 0, axis=3),
tf.float32)
renders = phong_shader(normals=pixel_normals,
alphas=pixel_mask,
pixel_positions=pixel_positions,
light_positions=light_positions,
light_intensities=light_intensities,
diffuse_colors=diffuse_colors,
camera_position=camera_position
if specular_colors is not None else None,
specular_colors=specular_colors,
shininess_coefficients=shininess_coefficients,
ambient_color=ambient_color)
return renders
def add_triplet_loss_for_each_scale(batch_size, num_frames_per_video,
embedding_dim, scales_to_embeddings,
labels, scope):
"""Adds triplet loss for logits of each scale.
Args:
batch_size: Int, the number of video chunks sampled per batch
num_frames_per_video: Int, the number of frames per video.
embedding_dim: Int, the dimension of the learned embedding
scales_to_embeddings: A map from embedding names for different scales to
embeddings. The embeddings have shape [batch, embeddings_height,
embeddings_width, embedding_dim].
labels: Groundtruth labels with shape [batch, image_height, image_width, 1].
scope: String, the scope for the loss.
Raises:
ValueError: labels is None.
"""
if labels is None:
raise ValueError('No label for triplet loss.')
for scale, embeddings in scales_to_embeddings.iteritems():
loss_scope = None
if scope:
loss_scope = '%s_%s' % (scope, scale)
# Label is downsampled to the same size as logits.
scaled_labels = tf.image.resize_nearest_neighbor(labels,
resolve_shape(
embeddings,
4)[1:3],
align_corners=True)
# Reshape from [batch * num_frames, ...] to [batch, num_frames, ...].
h = tf.shape(embeddings)[1]
w = tf.shape(embeddings)[2]
new_labels_shape = tf.stack(
[batch_size, num_frames_per_video, h, w, 1])
reshaped_labels = tf.reshape(scaled_labels, new_labels_shape)
new_embeddings_shape = tf.stack(
[batch_size, num_frames_per_video, h, w, -1])
reshaped_embeddings = tf.reshape(embeddings, new_embeddings_shape)
with tf.name_scope(loss_scope):
total_loss = tf.constant(0, dtype=tf.float32)
for n in range(batch_size):
embedding = reshaped_embeddings[n]
label = reshaped_labels[n]
n_pixels = h * w
n_anchors_used = 256
sampled_anchor_indices = tf.random_shuffle(
tf.range(n_pixels))[:n_anchors_used]
anchors_pool = tf.reshape(embedding[0], [-1, embedding_dim])
anchors_pool_classes = tf.reshape(label[0], [-1])
anchors = tf.gather(anchors_pool, sampled_anchor_indices)
anchor_classes = tf.gather(anchors_pool_classes,
sampled_anchor_indices)
pos_neg_pool = tf.reshape(embedding[1:], [-1, embedding_dim])
pos_neg_pool_classes = tf.reshape(label[1:], [-1])
dists = embedding_utils.pairwise_distances(
anchors, pos_neg_pool)
pos_mask = tf.equal(anchor_classes[:, tf.newaxis],
pos_neg_pool_classes[tf.newaxis, :])
neg_mask = tf.logical_not(pos_mask)
pos_mask_f = tf.cast(pos_mask, tf.float32)
neg_mask_f = tf.cast(neg_mask, tf.float32)
pos_dists = pos_mask_f * dists + 1e20 * neg_mask_f
neg_dists = neg_mask_f * dists + 1e20 * pos_mask_f
pos_dists_min = tf.reduce_min(pos_dists, axis=1)
neg_dists_min = tf.reduce_min(neg_dists, axis=1)
margin = 1.0
loss = tf.nn.relu(pos_dists_min - neg_dists_min + margin)
# Handle case that no positive is present (per anchor).
any_pos = tf.reduce_any(pos_mask, axis=1)
loss *= tf.cast(any_pos, tf.float32)
# Average over anchors
loss = tf.reduce_mean(loss, axis=0)
total_loss += loss
total_loss /= batch_size
# Scale the loss up a bit.
total_loss *= 3.0
tf.add_to_collection(tf.GraphKeys.LOSSES, total_loss)
def beam_search(
symbols_to_logits_fn,
init_seq_BxT,
initial_cache_BxU,
vocab_size,
beam_size,
length_norm_fn,
eos_id=1,
):
"""Beam search.
Args:
symbols_to_logits_fn: fn(seq_BxT, cache_BxU, i) -> (logits_BxV, cache_BxU)
init_seq_BxT: initial sequence ids.
initial_cache_BxU: dictionary of tensors with shape BxU.
vocab_size: vocabulary size.
beam_size: beam size.
length_norm_fn: length normalization function.
eos_id: end of sequence.
Returns:
Tuple of (beams_BxMxT, scores_BxM). Beam searched sequences and scores.
"""
B, T = init_seq_BxT.shape
B = tf.shape(init_seq_BxT)[0]
M, V = beam_size, vocab_size
dtype = tf.float32
int_dtype = init_seq_BxT.dtype
def _loop_body(
i,
alive_seq_BxMxT,
alive_log_probs_BxM,
alive_cache_BxMxU,
finished_seq_BxMxT,
finished_scores_BxM,
):
"""Beam search loop body."""
# Decode one step with beam
logits_BMxV, cache_BMxU = symbols_to_logits_fn(
_flatten_beam_dim(alive_seq_BxMxT),
tf.nest.map_structure(_flatten_beam_dim, alive_cache_BxMxU),
i,
)
logits_BxMxV = _unflatten_beam_dim(logits_BMxV, M)
new_cache_BxMxU = tf.nest.map_structure(
lambda t: _unflatten_beam_dim(t, M), cache_BMxU)
# select top 2 * beam_size and fill alive and finished.
log_probs_BxMxV = logits_BxMxV - tf.reduce_logsumexp(
logits_BxMxV, axis=2, keepdims=True)
log_probs_BxMxV += tf.expand_dims(alive_log_probs_BxM, axis=2)
log_probs_BxMV = tf.reshape(log_probs_BxMxV, [B, -1])
new_log_probs_Bx2M, topk_indices_Bx2M = tf.nn.top_k(log_probs_BxMV,
k=2 * M)
topk_beam_Bx2M = topk_indices_Bx2M // V
topk_seq_Bx2MxT, new_cache_Bx2MxU = _gather_nested(
[alive_seq_BxMxT, new_cache_BxMxU], topk_beam_Bx2M)
topk_ids_Bx2M = topk_indices_Bx2M % V
new_seq_Bx2MxT = _update_i(topk_seq_Bx2MxT, topk_ids_Bx2M, i)
new_finished_flags_Bx2M = tf.cast(
tf.reduce_any(tf.equal(new_seq_Bx2MxT, eos_id), axis=-1), dtype)
# get new alive
_, topk_alive_indices_BxM = tf.nn.top_k(
new_log_probs_Bx2M + new_finished_flags_Bx2M * dtype.min, k=M)
(
alive_seq_BxMxT,
alive_log_probs_BxM,
alive_cache_BxMxU,
) = _gather_nested(
[new_seq_Bx2MxT, new_log_probs_Bx2M, new_cache_Bx2MxU],
topk_alive_indices_BxM,
)
# get new finished
new_scores_Bx2M = length_norm_fn(new_log_probs_Bx2M, i + 1)
new_scores_Bx2M += (1 - new_finished_flags_Bx2M) * dtype.min
finished_seq_Bx3MxT = tf.concat([finished_seq_BxMxT, new_seq_Bx2MxT],
axis=1)
finished_scores_Bx3M = tf.concat(
[finished_scores_BxM, new_scores_Bx2M], axis=1)
_, topk_finished_indices_BxM = tf.nn.top_k(finished_scores_Bx3M, k=M)
(finished_seq_BxMxT, finished_scores_BxM) = _gather_nested(
[finished_seq_Bx3MxT, finished_scores_Bx3M],
topk_finished_indices_BxM,
)
return [
i + 1,
alive_seq_BxMxT,
alive_log_probs_BxM,
alive_cache_BxMxU,
finished_seq_BxMxT,
finished_scores_BxM,
]
# initialize.
init_i = tf.constant(0, dtype=int_dtype)
init_alive_seq_BxMxT = _expand_to_beam_size(init_seq_BxT, M)
log_probs_1xM = tf.constant([[0.0] + [dtype.min] * (M - 1)], dtype=dtype)
init_alive_log_probs_BxM = tf.tile(log_probs_1xM, [B, 1])
init_alive_cache_BxMxU = tf.nest.map_structure(
lambda t: _expand_to_beam_size(t, M), initial_cache_BxU)
init_finished_seq_BxMxT = tf.zeros(tf.shape(init_alive_seq_BxMxT),
int_dtype)
init_finished_scores_BxM = tf.zeros([B, M], dtype=dtype) + dtype.min
tf.nest.map_structure(_flatten_beam_dim, init_alive_seq_BxMxT)
# run loop.
(
_,
final_alive_seq_BxMxT,
final_alive_scores_BxM,
_,
final_finished_seq_BxMxT,
final_finished_scores_BxM,
) = tf.while_loop(
lambda *args: True, # Always do T iterations
_loop_body,
loop_vars=[
init_i,
init_alive_seq_BxMxT,
init_alive_log_probs_BxM,
init_alive_cache_BxMxU,
init_finished_seq_BxMxT,
init_finished_scores_BxM,
],
parallel_iterations=1,
back_prop=False,
maximum_iterations=T,
)
# process finished.
final_finished_flag_BxMx1 = tf.reduce_any(tf.equal(
final_finished_seq_BxMxT, eos_id),
axis=-1,
keepdims=True)
final_seq_BxMxT = tf.where(
tf.tile(final_finished_flag_BxMx1, [1, 1, T]),
final_finished_seq_BxMxT,
final_alive_seq_BxMxT,
)
final_scores_BxM = tf.where(
tf.squeeze(final_finished_flag_BxMx1, axis=-1),
final_finished_scores_BxM,
final_alive_scores_BxM,
)
return final_seq_BxMxT, final_scores_BxM
def _loop_body(
i,
alive_seq_BxMxT,
alive_log_probs_BxM,
alive_cache_BxMxU,
finished_seq_BxMxT,
finished_scores_BxM,
):
"""Beam search loop body."""
# Decode one step with beam
logits_BMxV, cache_BMxU = symbols_to_logits_fn(
_flatten_beam_dim(alive_seq_BxMxT),
tf.nest.map_structure(_flatten_beam_dim, alive_cache_BxMxU),
i,
)
logits_BxMxV = _unflatten_beam_dim(logits_BMxV, M)
new_cache_BxMxU = tf.nest.map_structure(
lambda t: _unflatten_beam_dim(t, M), cache_BMxU)
# select top 2 * beam_size and fill alive and finished.
log_probs_BxMxV = logits_BxMxV - tf.reduce_logsumexp(
logits_BxMxV, axis=2, keepdims=True)
log_probs_BxMxV += tf.expand_dims(alive_log_probs_BxM, axis=2)
log_probs_BxMV = tf.reshape(log_probs_BxMxV, [B, -1])
new_log_probs_Bx2M, topk_indices_Bx2M = tf.nn.top_k(log_probs_BxMV,
k=2 * M)
topk_beam_Bx2M = topk_indices_Bx2M // V
topk_seq_Bx2MxT, new_cache_Bx2MxU = _gather_nested(
[alive_seq_BxMxT, new_cache_BxMxU], topk_beam_Bx2M)
topk_ids_Bx2M = topk_indices_Bx2M % V
new_seq_Bx2MxT = _update_i(topk_seq_Bx2MxT, topk_ids_Bx2M, i)
new_finished_flags_Bx2M = tf.cast(
tf.reduce_any(tf.equal(new_seq_Bx2MxT, eos_id), axis=-1), dtype)
# get new alive
_, topk_alive_indices_BxM = tf.nn.top_k(
new_log_probs_Bx2M + new_finished_flags_Bx2M * dtype.min, k=M)
(
alive_seq_BxMxT,
alive_log_probs_BxM,
alive_cache_BxMxU,
) = _gather_nested(
[new_seq_Bx2MxT, new_log_probs_Bx2M, new_cache_Bx2MxU],
topk_alive_indices_BxM,
)
# get new finished
new_scores_Bx2M = length_norm_fn(new_log_probs_Bx2M, i + 1)
new_scores_Bx2M += (1 - new_finished_flags_Bx2M) * dtype.min
finished_seq_Bx3MxT = tf.concat([finished_seq_BxMxT, new_seq_Bx2MxT],
axis=1)
finished_scores_Bx3M = tf.concat(
[finished_scores_BxM, new_scores_Bx2M], axis=1)
_, topk_finished_indices_BxM = tf.nn.top_k(finished_scores_Bx3M, k=M)
(finished_seq_BxMxT, finished_scores_BxM) = _gather_nested(
[finished_seq_Bx3MxT, finished_scores_Bx3M],
topk_finished_indices_BxM,
)
return [
i + 1,
alive_seq_BxMxT,
alive_log_probs_BxM,
alive_cache_BxMxU,
finished_seq_BxMxT,
finished_scores_BxM,
]
def test_Any(self):
t = tf.reduce_any(self.random(3, 4, 5),
reduction_indices=[0, 1],
keep_dims=True)
self.check(t)
def new_cond_tf_func(pred_b: TfVal, *carry: TfVal) -> TfVal:
pred = tf.reduce_any(pred_b, axis=list(range(len(pred_b.shape))))
return pred
def _post_process(self,
predict_classes,
predict_confidence,
predict_boxes,
threshold=0.05):
"""Predict boxes with probabilty threshold.
Args:
predict_classes: [batch_size, cell_size, cell_size, num_classes]
predict_confidence: [batch_size, cell_size, cell_size, boxes_per_cell]
predict_boxes: [batch_size, cell_size, cell_size, boxes_per_cell, 4(center_x, center_y, w, h)]
threshold: threshold of predict score.
Return: python list of predict_boxes Tensor.
predict_boxes shape is [num_predicted_boxes, 6(x, y, w, h, class_id, probability)].
python list lenght is batch size.
"""
with tf.name_scope("post_process"):
predict_boxes = self.convert_boxes_space_from_yolo_to_real(
predict_boxes)
if self.is_debug:
predict_classes = tf.Print(
predict_classes,
[tf.shape(predict_classes), predict_classes],
message="predict_classes:",
summarize=2000)
# [batch_size, cell_size, cell_size, boxes_per_cell, num_classes]
predict_probability = tf.expand_dims(
predict_confidence, 4) * tf.expand_dims(predict_classes, 3)
mask = predict_probability > threshold
if self.is_debug:
mask = tf.Print(mask, [tf.shape(mask), mask],
message="mask:",
summarize=2000)
# box_mask: shape is [batch_size, cell_size, cell_size, boxes_per_cell]
box_mask = tf.reduce_any(mask, axis=4)
max_predict_classes = tf.reduce_max(predict_classes, axis=3)
max_predict_classes = tf.reshape(
max_predict_classes,
[self.batch_size, self.cell_size, self.cell_size, 1])
max_predict_classes = tf.tile(max_predict_classes,
[1, 1, 1, self.boxes_per_cell])
# argmax_predict_classes be: [batch_size, cell_size, cell_size, self.boxes_per_cell]
argmax_predict_classes = tf.argmax(predict_classes, axis=3)
argmax_predict_classes = tf.reshape(
argmax_predict_classes,
[self.batch_size, self.cell_size, self.cell_size, 1])
argmax_predict_classes = tf.tile(argmax_predict_classes,
[1, 1, 1, self.boxes_per_cell])
if self.is_debug:
argmax_predict_classes = tf.Print(
argmax_predict_classes,
[tf.shape(argmax_predict_classes), argmax_predict_classes],
message="argmax_predict_classes:",
summarize=2000)
boxes_list = []
for i in range(self.batch_size):
box_mask_by_batch = box_mask[i, :, :, :]
predict_boxes_by_batch = predict_boxes[i, :, :, :, :]
# masked_boxes: [?, 4]
masked_boxes = tf.boolean_mask(predict_boxes_by_batch,
box_mask_by_batch)
if self.is_debug:
masked_boxes = tf.Print(
masked_boxes,
[i, tf.shape(masked_boxes), masked_boxes],
message="predicted_masked_boxes:",
summarize=2000)
predict_classes_by_batch = argmax_predict_classes[i, :, :, :]
masked_classes = tf.boolean_mask(predict_classes_by_batch,
box_mask_by_batch)
if self.is_debug:
masked_classes = tf.Print(
masked_classes,
[i, tf.shape(masked_classes), masked_classes],
message="masked_classes:",
summarize=2000)
predict_classes_probability_by_batch = max_predict_classes[
i, :, :, :]
masked_class_probability = tf.boolean_mask(
predict_classes_probability_by_batch, box_mask_by_batch)
if self.is_debug:
masked_class_probability = tf.Print(
masked_class_probability, [
i,
tf.shape(masked_class_probability),
masked_class_probability
],
message="masked_class_probability:",
summarize=2000)
masked_boxes = format_CXCYWH_to_XYWH(masked_boxes, axis=1)
# boxes: shape is [num_predicted_boxes, 6(x, y, w, h, class_id, probability)]
boxes = tf.stack([
masked_boxes[:, 0],
masked_boxes[:, 1],
masked_boxes[:, 2],
masked_boxes[:, 3],
tf.cast(masked_classes, tf.float32),
masked_class_probability,
],
axis=1)
boxes_list.append(boxes)
return boxes_list
def false_fn():
# b = tf.constant(np.array([0], dtype=np.int32), dtype=tf.int32)
# while_loop
c = lambda y: tf.reduce_any(tf.less(y, 10))
b = lambda i: tf.add(y, 1)
return tf.while_loop(c, b, [y])
def empty_bar_rate(tensor):
"""Return the ratio of empty bars to the total number of bars."""
if tensor.get_shape().ndims != 5:
raise ValueError("Input tensor must have 5 dimensions.")
return tf.reduce_mean(
tf.cast(tf.reduce_any(tensor > 0.5, (2, 3)), tf.float32), (0, 1))
def compute_loss(self, y_true, y_predicted):
"""Compute mutlibox loss.
# Arguments
y_true: Ground truth targets,
tensor of shape (?, num_boxes, 4 + num_classes + 8),
priors in ground truth are fictitious,
y_true[:, :, -8] has 1 if prior should be penalized
or in other words is assigned to some ground truth box,
y_true[:, :, -7:] are all 0.
y_pred: Predicted logits,
tensor of shape (?, num_boxes, 4 + num_classes + 8).
# Returns
loss: Loss for prediction, tensor of shape (?,).
# Questions
Q: What is 4 + num_classes + 8?
A: 4 is for box coordinates, but +8 is full of zeros in the y_true
It is probably related to the box coordinates and the... ?
"""
batch_size = tf.shape(y_true)[0]
num_boxes = tf.to_float(tf.shape(y_true)[1])
# loss for all prior boxes
confidence_loss = self._softmax_loss(y_true[:, :, 4:-8],
y_predicted[:, :, 4:-8])
localization_loss = self._l1_smooth_loss(y_true[:, :, :4],
y_predicted[:, :, :4])
# get positive loss
num_positives = tf.reduce_sum(y_true[:, :, -8], reduction_indices=-1)
positives_mask = y_true[:, :, -8]
positive_localization_loss = tf.reduce_sum(localization_loss *
positives_mask,
reduction_indices=1)
positive_confidence_loss = tf.reduce_sum(confidence_loss *
positives_mask,
reduction_indices=1)
# get negative loss, we penalize only confidence here
num_negatives_1 = self.negative_positive_ratio * num_positives
num_negatives_2 = num_boxes - num_positives
num_negatives = tf.minimum(num_negatives_1, num_negatives_2)
positive_num_negatives_bool = tf.greater(num_negatives, 0)
reduce_any_flag = tf.to_float(
tf.reduce_any(positive_num_negatives_bool))
weird = (1 - reduce_any_flag) * self.negatives_for_hard
num_negatives = tf.concat(0, [num_negatives, weird])
positive_num_negatives = tf.boolean_mask(num_negatives,
positive_num_negatives_bool)
num_negatives_batch = tf.reduce_min(positive_num_negatives)
# why is this not working?
#num_negatives_batch = self.negative_positive_ratio * num_positives
#num_negatives_batch = tf.to_int32(num_negatives_batch)
#num_negatives_batch = tf.concat(0, [num_negatives_batch, 0.0])
#--------------------------------------------------------------------------
confidence_start = 4 + self.background_label_id + 1 #5 this is correct
confidence_end = confidence_start + self.num_classes - 1
# does this contain all positives and negatives? yes
# this takes best the confidences for every batch for every num_boxes
max_confidences = tf.reduce_max(
y_predicted[:, :, confidence_start:confidence_end],
reduction_indices=2)
# this filters all the positives
negatives_mask = 1 - y_true[:, :, -8]
indices = tf.nn.top_k((max_confidences * negatives_mask),
k=num_negatives_batch)[1]
batch_indices = tf.expand_dims(tf.range(0, batch_size), 1)
batch_indices = tf.tile(batch_indices, (1, num_negatives_batch))
# remember the broadcasting from the + (possibly)
full_indices = (
tf.reshape(batch_indices, [-1]) * tf.to_int32(num_boxes) +
tf.reshape(indices, [-1]))
negative_confidence_loss = tf.gather(tf.reshape(confidence_loss, [-1]),
full_indices)
negative_confidence_loss = tf.reshape(
negative_confidence_loss, [batch_size, num_negatives_batch])
negative_confidence_loss = tf.reduce_sum(negative_confidence_loss,
reduction_indices=1)
# loss is sum of positives and negatives
total_loss = positive_confidence_loss + negative_confidence_loss
total_loss /= (num_positives + tf.to_float(num_negatives_batch))
num_positives = tf.select(tf.not_equal(num_positives, 0),
num_positives, tf.ones_like(num_positives))
total_loss += (self.alpha * positive_localization_loss) / num_positives
return total_loss
def cond(i, prev_base_states, prev_high_states, prev_ys, prev_embs, cost, ys_array, p_array):
return tf.logical_and(
tf.less(i, translation_maxlen),
tf.reduce_any(tf.not_equal(prev_ys, 0)))
def cond_batch(e, n, phi, tree):
# do parser if some sentence have not get the root node
return tf.reduce_any(tf.greater(n, tree_order-1))
def tf_enqueue(self,
states,
internals,
auxiliaries,
actions,
terminal,
reward,
baseline=None):
# Constants and parameters
zero = tf.constant(value=0, dtype=util.tf_dtype(dtype='long'))
one = tf.constant(value=1, dtype=util.tf_dtype(dtype='long'))
capacity = tf.constant(value=self.capacity,
dtype=util.tf_dtype(dtype='long'))
horizon = self.horizon.value()
discount = self.discount.value()
assertions = list()
# Check whether horizon at most capacity
assertions.append(
tf.debugging.assert_less_equal(
x=horizon,
y=capacity,
message=
"Estimator capacity has to be at least the same as the estimation horizon."
))
# Check whether at most one terminal
assertions.append(
tf.debugging.assert_less_equal(
x=tf.math.count_nonzero(input=terminal,
dtype=util.tf_dtype(dtype='long')),
y=one,
message="Timesteps contain more than one terminal."))
# Check whether, if any, last value is terminal
assertions.append(
tf.debugging.assert_equal(
x=tf.reduce_any(
input_tensor=tf.math.greater(x=terminal, y=zero)),
y=tf.math.greater(x=terminal[-1], y=zero),
message="Terminal is not the last timestep."))
# Get number of overwritten values
with tf.control_dependencies(control_inputs=assertions):
if util.tf_dtype(dtype='long') in (tf.int32, tf.int64):
num_values = tf.shape(input=terminal,
out_type=util.tf_dtype(dtype='long'))[0]
else:
num_values = tf.dtypes.cast(x=tf.shape(input=terminal)[0],
dtype=util.tf_dtype(dtype='long'))
overwritten_start = tf.maximum(x=self.buffer_index, y=capacity)
overwritten_limit = tf.maximum(x=(self.buffer_index + num_values),
y=capacity)
num_overwritten = overwritten_limit - overwritten_start
def update_overwritten_rewards():
# Get relevant buffer rewards
buffer_limit = self.buffer_index + tf.minimum(
x=(num_overwritten + horizon), y=capacity)
buffer_indices = tf.range(start=self.buffer_index,
limit=buffer_limit)
buffer_indices = tf.math.mod(x=buffer_indices, y=capacity)
rewards = tf.gather(params=self.buffers['reward'],
indices=buffer_indices)
# Get relevant values rewards
values_limit = tf.maximum(x=(num_overwritten + horizon - capacity),
y=zero)
rewards = tf.concat(values=(rewards, reward[:values_limit]),
axis=0)
# Horizon baseline value
if self.estimate_horizon == 'early':
assert baseline is not None
# Baseline estimate
buffer_indices = buffer_indices[horizon + one:]
_states = OrderedDict()
for name, buffer in self.buffers['states'].items():
state = tf.gather(params=buffer, indices=buffer_indices)
_states[name] = tf.concat(
values=(state, states[name][:values_limit + one]),
axis=0)
_internals = OrderedDict()
for name, buffer in self.buffers['internals'].items():
internal = tf.gather(params=buffer, indices=buffer_indices)
_internals[name] = tf.concat(
values=(internal,
internals[name][:values_limit + one]),
axis=0)
_auxiliaries = OrderedDict()
for name, buffer in self.buffers['auxiliaries'].items():
auxiliary = tf.gather(params=buffer,
indices=buffer_indices)
_auxiliaries[name] = tf.concat(
values=(auxiliary,
auxiliaries[name][:values_limit + one]),
axis=0)
# Dependency horizon
# TODO: handle arbitrary non-optimization horizons!
past_horizon = baseline.past_horizon(is_optimization=False)
assertion = tf.debugging.assert_equal(
x=past_horizon,
y=zero,
message=
"Temporary: baseline cannot depend on previous states.")
with tf.control_dependencies(control_inputs=(assertion, )):
some_state = next(iter(_states.values()))
if util.tf_dtype(dtype='long') in (tf.int32, tf.int64):
batch_size = tf.shape(
input=some_state,
out_type=util.tf_dtype(dtype='long'))[0]
else:
batch_size = tf.dtypes.cast(
x=tf.shape(input=some_state)[0],
dtype=util.tf_dtype(dtype='long'))
starts = tf.range(start=batch_size,
dtype=util.tf_dtype(dtype='long'))
lengths = tf.ones(shape=(batch_size, ),
dtype=util.tf_dtype(dtype='long'))
Module.update_tensors(dependency_starts=starts,
dependency_lengths=lengths)
if self.estimate_actions:
_actions = OrderedDict()
for name, buffer in self.buffers['actions'].items():
action = tf.gather(params=buffer,
indices=buffer_indices)
_actions[name] = tf.concat(
values=(action, actions[name][:values_limit]),
axis=0)
horizon_estimate = baseline.actions_value(
states=_states,
internals=_internals,
auxiliaries=_auxiliaries,
actions=_actions)
else:
horizon_estimate = baseline.states_value(
states=_states,
internals=_internals,
auxiliaries=_auxiliaries)
else:
# Zero estimate
horizon_estimate = tf.zeros(shape=(num_overwritten, ),
dtype=util.tf_dtype(dtype='float'))
# Calculate discounted sum
def cond(discounted_sum, horizon):
return tf.math.greater_equal(x=horizon, y=zero)
def body(discounted_sum, horizon):
# discounted_sum = tf.compat.v1.Print(
# discounted_sum, (horizon, discounted_sum, rewards[horizon:]), summarize=10
# )
discounted_sum = discount * discounted_sum
discounted_sum = discounted_sum + rewards[horizon:horizon +
num_overwritten]
return discounted_sum, horizon - one
discounted_sum, _ = self.while_loop(cond=cond,
body=body,
loop_vars=(horizon_estimate,
horizon),
back_prop=False)
assertions = [
tf.debugging.assert_equal(x=tf.shape(input=horizon_estimate),
y=tf.shape(input=discounted_sum),
message="Estimation check."),
tf.debugging.assert_equal(x=tf.shape(
input=rewards, out_type=util.tf_dtype(dtype='long'))[0],
y=(horizon + num_overwritten),
message="Estimation check.")
]
# Overwrite buffer rewards
with tf.control_dependencies(control_inputs=assertions):
indices = tf.range(start=self.buffer_index,
limit=(self.buffer_index + num_overwritten))
indices = tf.math.mod(x=indices, y=capacity)
indices = tf.expand_dims(input=indices, axis=1)
assignment = self.buffers['reward'].scatter_nd_update(
indices=indices, updates=discounted_sum)
with tf.control_dependencies(control_inputs=(assignment, )):
return util.no_operation()
any_overwritten = tf.math.greater(x=num_overwritten, y=zero)
updated_rewards = self.cond(pred=any_overwritten,
true_fn=update_overwritten_rewards,
false_fn=util.no_operation)
# Overwritten buffer indices
with tf.control_dependencies(control_inputs=(updated_rewards, )):
indices = tf.range(start=overwritten_start,
limit=overwritten_limit)
indices = tf.math.mod(x=indices, y=capacity)
# Get overwritten values
with tf.control_dependencies(control_inputs=(indices, )):
overwritten_values = OrderedDict()
for name, buffer in self.buffers.items():
if util.is_nested(name=name):
overwritten_values[name] = OrderedDict()
for inner_name, buffer in buffer.items():
overwritten_values[name][inner_name] = tf.gather(
params=buffer, indices=indices)
else:
overwritten_values[name] = tf.gather(params=buffer,
indices=indices)
# Buffer indices to (over)write
with tf.control_dependencies(control_inputs=util.flatten(
xs=overwritten_values)):
indices = tf.range(start=self.buffer_index,
limit=(self.buffer_index + num_values))
indices = tf.math.mod(x=indices, y=capacity)
indices = tf.expand_dims(input=indices, axis=1)
# Write new values
with tf.control_dependencies(control_inputs=(indices, )):
values = dict(states=states,
internals=internals,
auxiliaries=auxiliaries,
actions=actions,
terminal=terminal,
reward=reward)
assignments = list()
for name, buffer in self.buffers.items():
if util.is_nested(name=name):
for inner_name, buffer in buffer.items():
assignment = buffer.scatter_nd_update(
indices=indices, updates=values[name][inner_name])
assignments.append(assignment)
else:
assignment = buffer.scatter_nd_update(indices=indices,
updates=values[name])
assignments.append(assignment)
# Increment buffer index
with tf.control_dependencies(control_inputs=assignments):
assignment = self.buffer_index.assign_add(delta=num_values,
read_value=False)
# Return overwritten values or no-op
with tf.control_dependencies(control_inputs=(assignment, )):
any_overwritten = tf.math.greater(x=num_overwritten, y=zero)
overwritten_values = util.fmap(function=util.identity_operation,
xs=overwritten_values)
return any_overwritten, overwritten_values
def _scan_step_fn(state, example, packed_length, queue_size, spacing,
num_sequences, token_dtype): # pylint: disable=g-doc-args
"""Transform function used by tf.data.experimental.scan to process an example.
This is written as a stateless function rather than a class method because we
trace it with AutoGraph (in order to simplify the conditional), and this way
we don't have to worry about handling re-tracing semantics.
Args:
See the SequenceDatasetPacker class.
Returns:
The updated queue state, and either a packed example or a dummy sequence
which will be filtered out downstream.
"""
# Convert TensorArray tuples to lists since we'll need to replace them.
availability, contents, top_index = state
lengths = tf.concat([tf.shape(i) for i in example], axis=0)
start_availability = availability.stack()
can_fit = tf.reduce_all(tf.greater_equal(start_availability, lengths),
axis=1)
any_can_fit = tf.reduce_any(can_fit, axis=0)
# AutoGraph will convert this block to a tf.cond
if any_can_fit:
# This indicates where in the FFD queue rotation a given index sits
shifted_range = (tf.range(queue_size, dtype=INDEX_DTYPE) -
top_index) % queue_size
# Mark any indices which cannot accommodate the current example.
exclusion_mask = tf.cast(tf.logical_not(can_fit),
INDEX_DTYPE) * queue_size
# Index in [0, queue_size) in which to place the sample. Note, this index
# is the position in the actual TensorArray, not the index of the FFD queue.
queue_index = (tf.reduce_min(shifted_range + exclusion_mask) +
top_index) % queue_size
# NOTE(taylorrobie): We emit a non-empty Tensor for downstream checks.
output_contents = -tf.ones((1, num_sequences), dtype=token_dtype)
else:
index_range = top_index * packed_length + tf.range(packed_length)
output_contents = contents.gather(index_range)
# Reset the queue state.
availability = availability.write(
top_index,
packed_length * tf.ones((num_sequences, ), dtype=INDEX_DTYPE))
empty_contents = tf.zeros((packed_length, num_sequences * 2),
dtype=token_dtype)
contents = contents.scatter(index_range, empty_contents)
queue_index = top_index
top_index = (top_index + 1) % queue_size
pre_assign_availability = availability.read(queue_index)
space_left = pre_assign_availability - lengths - spacing
availability = availability.write(queue_index, space_left)
# ============================================================================
# == Update contents =========================================================
# ============================================================================
# Consider the following case for a seq-to-seq packing:
# (padding is represented as underscores)
#
# Queue starting state:
# [1, 3, 2, 4, 6, 1, _, _, _, _, _, ...]
# [5, 9, _, _, _, _, _, _, _, _, _, ...]
#
# Examples:
# [4, 2, 4], [3]
#
# Desired new queue state:
# [1, 3, 2, 4, 6, 1, _, _, 4, 2, 4, _, _, ...]
# [5, 9, _, _, 3, _, _, _, _, _, _, _, _, ...]
#
# This could be acomplished by creating a TensorArray for each of the two
# sequences, and scattering into the respective arrays. However TensorArray
# writes are extremely expensive relative to other operations. So instead we
# store the contents in a single TensorArray of shape (packed_length, 2), and
# we pad and concatenate the examples such that they can be added in a single
# assign:
#
# [_, _, _, _, 4, 2, 4]
# [3, _, _, _, _, _, _]
# +
# [1, 3, 2, 4, 6, 1, _, _, _, _, _, ...]
# [5, 9, _, _, _, _, _, _, _, _, _, ...]
#
# And in practice, the extra work of padding is neglidgable compared to
# the gain from vectorizing the TensorArray assign. We also store a bit mask
# denoting where sequences start which is used to compute segment and
# position metadata:
#
# [_, _, _, _, 1, _, _]
# [1, _, _, _, _, _, _]
# +
# [1, _, _, _, _, _, _, _, _, _, _, ...]
# [1, _, _, _, _, _, _, _, _, _, _, ...]
#
# Both the contents and the mask are concatenated in the same TensorArray
# for performance.
start_index = packed_length - pre_assign_availability
end_index = start_index + lengths
leftmost = tf.reduce_min(start_index, axis=0)
rightmost = tf.reduce_max(end_index, axis=0)
delta = rightmost - leftmost
pad_indices = [
tf.stack((start_index[i] - leftmost, rightmost - end_index[i]))
for i in range(num_sequences)
]
padded_examples = [
tf.pad(ex, padding[tf.newaxis, :])
for ex, padding in zip(example, pad_indices)
]
padded_examples = tf.transpose(tf.stack(padded_examples))
mask_update = tf.one_hot(start_index - leftmost,
delta,
dtype=contents.dtype,
axis=0)
content_update = tf.concat([padded_examples, mask_update], axis=1)
index_range = (
queue_index * packed_length + # Offset into the right section.
tf.range(delta, dtype=INDEX_DTYPE) + leftmost)
contents = contents.scatter(index_range,
contents.gather(index_range) + content_update)
state = (availability, contents, top_index)
return state, (tf.logical_not(any_can_fit), output_contents)
def acc(y_true, y_pred):
index = tf.reduce_any(y_true > 0.5, axis=-1)
res = tf.equal(tf.argmax(y_true, axis=-1), tf.argmax(y_pred, axis=-1))
index = tf.cast(index, tf.float32)
res = tf.cast(res, tf.float32)
return tf.reduce_sum(res * index) / (tf.reduce_sum(index) + 1e-7)
def _zeros_if_any_small(inputs, outputs, eps=1e-12):
return tf.where(tf.reduce_any(tf.less(tf.abs(inputs), eps), axis=-1),
tf.zeros_like(outputs), outputs)
def dcn_decode(encoding,
document_length,
state_size=100,
pool_size=4,
max_iter=4,
keep_prob=1.0):
""" DCN+ Dynamic Decoder.
Builds decoder graph that iterates over possible solutions to problem
until it returns same answer in two consecutive iterations or reaches `max_iter` iterations.
Args:
encoding: Tensor of rank 3, shape [N, D, xH]. Query-document encoding.
state_size: Scalar integer. Size of state and highway network.
pool_size: Scalar integer. Number of units that are max pooled in maxout network.
max_iter: Scalar integer. Maximum number of attempts for answer span start and end to settle.
keep_prob: Scalar float. Probability of keeping units during dropout.
Returns:
A tuple containing
TensorArray of answer span logits for each iteration.
TensorArray of logit masks for each iteration.
"""
with tf.variable_scope('decoder_loop', reuse=tf.AUTO_REUSE):
batch_size = tf.shape(encoding)[0]
lstm_dec = tf.contrib.rnn.LSTMCell(num_units=state_size)
lstm_dec = tf.contrib.rnn.DropoutWrapper(lstm_dec,
input_keep_prob=keep_prob)
# initialise loop variables
start = tf.zeros((batch_size, ), dtype=tf.int32)
end = document_length - 1
answer = tf.stack([start, end], axis=1)
state = lstm_dec.zero_state(batch_size, dtype=tf.float32)
not_settled = tf.tile([True], (batch_size, ))
logits = tf.TensorArray(tf.float32,
size=max_iter,
clear_after_read=False)
def calculate_not_settled_logits(not_settled, answer, output,
prev_logit):
enc_masked = tf.boolean_mask(encoding, not_settled)
output_masked = tf.boolean_mask(output, not_settled)
answer_masked = tf.boolean_mask(answer, not_settled)
document_length_masked = tf.boolean_mask(document_length,
not_settled)
new_logit = decoder_body(enc_masked, output_masked, answer_masked,
state_size, pool_size,
document_length_masked, keep_prob)
new_idx = tf.boolean_mask(tf.range(batch_size), not_settled)
logit = tf.dynamic_stitch([tf.range(batch_size), new_idx],
[prev_logit, new_logit])
return logit
for i in range(max_iter):
if i > 1:
tf.summary.scalar(
f'not_settled_iter_{i+1}',
tf.reduce_sum(tf.cast(not_settled, tf.float32)))
output, state = lstm_dec(start_and_end_encoding(encoding, answer),
state)
if i == 0:
logit = decoder_body(encoding, output, answer, state_size,
pool_size, document_length, keep_prob)
else:
prev_logit = logits.read(i - 1)
logit = tf.cond(
tf.reduce_any(not_settled),
lambda: calculate_not_settled_logits(
not_settled, answer, output, prev_logit),
lambda: prev_logit)
start_logit, end_logit = logit[:, :, 0], logit[:, :, 1]
start = tf.argmax(start_logit, axis=1, output_type=tf.int32)
end = tf.argmax(end_logit, axis=1, output_type=tf.int32)
new_answer = tf.stack([start, end], axis=1)
if i == 0:
not_settled = tf.tile([True], (batch_size, ))
else:
not_settled = tf.reduce_any(tf.not_equal(answer, new_answer),
axis=1)
not_settled = tf.reshape(not_settled, (batch_size, ))
answer = new_answer
logits = logits.write(i, logit)
return logits
print(sess.run(reduce_max1)) # tf.argmax(input, dimension, name=None)归约最大值的索引 index_max = tf.argmax(x1, 1) print(sess.run(index_max)) # reduce min 归约最小值 reduce_min1 = tf.reduce_min(x1, 0) print(sess.run(reduce_min1)) # tf.argmin(input, dimension, name=None)归约最小值的索引 index_min = tf.argmin(x1) print(sess.run(index_min)) # reduce all/any 归约与运算/或运算 x2 = [[True, False], [False, True]] reduce_all1 = tf.reduce_all(x2, 1, keep_dims=True) print(sess.run(reduce_all1)) reduce_any1 = tf.reduce_any(x2, 1, keep_dims=True) print(sess.run(reduce_any1)) # accumulate n 张量元素级(标量级)求和 # 叠加 x3 = [[1, 2, 3], [4, 5, 6]] x4 = [[7, 8, 9], [10, 11, 12]] a_n = tf.accumulate_n([x3, x3, x4, x3, x3]) print(sess.run(tf.shape(a_n))) print(sess.run(a_n)) # [2 3] # [[11 16 21] # [26 31 36]] ##tf.unique(x) 去重。返回去重后的tuple与原数组索引 x5 = [1, 2, 1, 1, 3, 5, 5, 3, 6, 6, 7, 7, 7, 7, 8]
def batch_assign_confidences(target_assigner,
anchors_batch,
gt_box_batch,
gt_box_3d_batch, # todo sep24
gt_class_confidences_batch,
gt_weights_batch=None,
unmatched_class_label=None,
include_background_class=True,
implicit_class_weight=1.0):
"""Batched assignment of classification and regression targets.
This differences between batch_assign_confidences and batch_assign_targets:
- 'batch_assign_targets' supports scalar (agnostic), vector (multiclass) and
tensor (high-dimensional) targets. 'batch_assign_confidences' only support
scalar (agnostic) and vector (multiclass) targets.
- 'batch_assign_targets' assumes the input class tensor using the binary
one/K-hot encoding. 'batch_assign_confidences' takes the class confidence
scores as the input, where 1 means positive classes, 0 means implicit
negative classes, and -1 means explicit negative classes.
- 'batch_assign_confidences' assigns the targets in the similar way as
'batch_assign_targets' except that it gives different weights for implicit
and explicit classes. This allows user to control the negative gradients
pushed differently for implicit and explicit examples during the training.
Args:
target_assigner: a target assigner.
anchors_batch: BoxList representing N box anchors or list of BoxList objects
with length batch_size representing anchor sets.
gt_box_batch: a list of BoxList objects with length batch_size
representing groundtruth boxes for each image in the batch
gt_class_confidences_batch: a list of tensors with length batch_size, where
each tensor has shape [num_gt_boxes_i, classification_target_size] and
num_gt_boxes_i is the number of boxes in the ith boxlist of
gt_box_batch. Note that in this tensor, 1 means explicit positive class,
-1 means explicit negative class, and 0 means implicit negative class.
gt_weights_batch: A list of 1-D tf.float32 tensors of shape
[num_gt_boxes_i] containing weights for groundtruth boxes.
unmatched_class_label: a float32 tensor with shape [d_1, d_2, ..., d_k]
which is consistent with the classification target for each
anchor (and can be empty for scalar targets). This shape must thus be
compatible with the groundtruth labels that are passed to the "assign"
function (which have shape [num_gt_boxes, d_1, d_2, ..., d_k]).
include_background_class: whether or not gt_class_confidences_batch includes
the background class.
implicit_class_weight: the weight assigned to implicit examples.
Returns:
batch_cls_targets: a tensor with shape [batch_size, num_anchors,
num_classes],
batch_cls_weights: a tensor with shape [batch_size, num_anchors,
num_classes],
batch_reg_targets: a tensor with shape [batch_size, num_anchors,
box_code_dimension]
batch_reg_weights: a tensor with shape [batch_size, num_anchors],
match: an int32 tensor of shape [batch_size, num_anchors] containing result
of anchor groundtruth matching. Each position in the tensor indicates an
anchor and holds the following meaning:
(1) if match[x, i] >= 0, anchor i is matched with groundtruth match[x, i].
(2) if match[x, i]=-1, anchor i is marked to be background .
(3) if match[x, i]=-2, anchor i is ignored since it is not background and
does not have sufficient overlap to call it a foreground.
Raises:
ValueError: if input list lengths are inconsistent, i.e.,
batch_size == len(gt_box_batch) == len(gt_class_targets_batch)
and batch_size == len(anchors_batch) unless anchors_batch is a single
BoxList, or if any element in gt_class_confidences_batch has rank > 2.
"""
if not isinstance(anchors_batch, list):
anchors_batch = len(gt_box_batch) * [anchors_batch]
if not all(
isinstance(anchors, box_list.BoxList) for anchors in anchors_batch):
raise ValueError('anchors_batch must be a BoxList or list of BoxLists.')
if not (len(anchors_batch)
== len(gt_box_batch)
== len(gt_class_confidences_batch)):
raise ValueError('batch size incompatible with lengths of anchors_batch, '
'gt_box_batch and gt_class_confidences_batch.')
cls_targets_list = []
cls_weights_list = []
reg_targets_3d_list = []
reg_weights_list = []
match_list = []
if gt_weights_batch is None:
gt_weights_batch = [None] * len(gt_class_confidences_batch)
for anchors, gt_boxes_3d, gt_class_confidences, gt_weights in zip( # _3d todo sep24
anchors_batch, gt_box_batch, gt_box_3d_batch, gt_class_confidences_batch,
gt_weights_batch):
if (gt_class_confidences is not None and
len(gt_class_confidences.get_shape().as_list()) > 2):
raise ValueError('The shape of the class target is not supported. ',
gt_class_confidences.get_shape())
cls_targets, _, reg_targets_3d, _, match = target_assigner.assign(
anchors, gt_boxes, gt_boxes_3d, gt_class_confidences, unmatched_class_label,
groundtruth_weights=gt_weights)
if include_background_class:
cls_targets_without_background = tf.slice(
cls_targets, [0, 1], [-1, -1])
else:
cls_targets_without_background = cls_targets
positive_mask = tf.greater(cls_targets_without_background, 0.0)
negative_mask = tf.less(cls_targets_without_background, 0.0)
explicit_example_mask = tf.logical_or(positive_mask, negative_mask)
positive_anchors = tf.reduce_any(positive_mask, axis=-1)
regression_weights = tf.cast(positive_anchors, dtype=tf.float32)
regression_targets_3d = (
reg_targets_3d * tf.expand_dims(regression_weights, axis=-1))
regression_weights_expanded = tf.expand_dims(regression_weights, axis=-1)
cls_targets_without_background = (
cls_targets_without_background *
(1 - tf.cast(negative_mask, dtype=tf.float32)))
cls_weights_without_background = ((1 - implicit_class_weight) * tf.cast(
explicit_example_mask, dtype=tf.float32) + implicit_class_weight)
if include_background_class:
cls_weights_background = (
(1 - implicit_class_weight) * regression_weights_expanded
+ implicit_class_weight)
classification_weights = tf.concat(
[cls_weights_background, cls_weights_without_background], axis=-1)
cls_targets_background = 1 - regression_weights_expanded
classification_targets = tf.concat(
[cls_targets_background, cls_targets_without_background], axis=-1)
else:
classification_targets = cls_targets_without_background
classification_weights = cls_weights_without_background
cls_targets_list.append(classification_targets)
cls_weights_list.append(classification_weights)
reg_targets_3d_list.append(regression_targets_3d)
reg_weights_list.append(regression_weights)
match_list.append(match)
batch_cls_targets = tf.stack(cls_targets_list)
batch_cls_weights = tf.stack(cls_weights_list)
batch_reg_targets_3d = tf.stack(reg_targets_3d_list)
batch_reg_weights = tf.stack(reg_weights_list)
batch_match = tf.stack(match_list)
return (batch_cls_targets, batch_cls_weights, batch_reg_targets_3d,
batch_reg_weights, batch_match)
def build_loss(self, predictions, examples, **kwargs):
"""Build tf graph to compute loss.
Args:
predictions: dict of prediction results keyed by name.
examples: dict of inputs keyed by name.
Returns:
loss_dict: dict of loss tensors keyed by name.
"""
options = self._model_proto
# Use inaccurate label to train.
logits = predictions[TextClassificationPredictions.logits]
mapping = {
# For coco mapping.
'traffic light': 'stoplight',
'fire hydrant': 'hydrant',
'stop sign': 'sign',
'parking meter': 'meter',
'sports ball': 'ball',
'baseball bat': 'bat',
'baseball glove': 'glove',
'tennis racket': 'racket',
'wine glass': 'wineglass',
'hot dog': 'hotdog',
'potted plant': 'plant',
'dining table': 'table',
'cell phone': 'cellphone',
'teddy bear': 'teddy',
'hair drier': 'hairdryer',
# For pascal mapping.
'aeroplane': 'airplane',
'diningtable': 'table',
'pottedplant': 'plant',
'tvmonitor': 'tv',
}
substituted_vocabulary_list = [
mapping.get(cls, cls) for cls in self._vocabulary_list
]
# Using ground-truth labels.
if options.label_option == text_classification_model_pb2.TextClassificationModel.GROUNDTRUTH:
labels = self._extract_class_label(
class_texts=examples[InputDataFields.object_texts],
vocabulary_list=self._vocabulary_list)
losses = tf.nn.sigmoid_cross_entropy_with_logits(labels=labels,
logits=logits)
loss = tf.reduce_mean(losses)
# Using exact-matching labels.
elif options.label_option == text_classification_model_pb2.TextClassificationModel.EXACT_MATCH:
labels = self._extract_class_label(
class_texts=slim.flatten(
examples[InputDataFields.caption_strings]),
vocabulary_list=substituted_vocabulary_list)
losses = tf.nn.sigmoid_cross_entropy_with_logits(labels=labels,
logits=logits)
loss = tf.reduce_mean(losses)
elif options.label_option == text_classification_model_pb2.TextClassificationModel.EXPAND_MATCH:
labels = self._extract_class_label(
class_texts=slim.flatten(
examples[InputDataFields.caption_strings]),
vocabulary_list=model_utils.expand_vocabulary(
self._vocabulary_list))
losses = tf.nn.sigmoid_cross_entropy_with_logits(labels=labels,
logits=logits)
loss = tf.reduce_mean(losses)
# Using exact-matching + w2v-matchiong.
elif options.label_option == text_classification_model_pb2.TextClassificationModel.EXACT_W2V_MATCH:
labels_gt = self._extract_class_label(
class_texts=slim.flatten(
examples[InputDataFields.caption_strings]),
vocabulary_list=substituted_vocabulary_list)
labels_ps = self._extract_pseudo_label(
texts=slim.flatten(examples[InputDataFields.caption_strings]),
vocabulary_list=self._vocabulary_list,
open_vocabulary_list=self._open_vocabulary_list,
embedding_dims=options.embedding_dims)
select_op = tf.reduce_any(labels_gt > 0, axis=-1)
labels = tf.where(select_op, labels_gt, labels_ps)
tf.summary.scalar('metrics/num_gt',
tf.reduce_sum(tf.to_float(select_op)))
losses = tf.nn.sigmoid_cross_entropy_with_logits(labels=labels,
logits=logits)
loss = tf.reduce_mean(losses)
# Using w2v-matchiong.
elif options.label_option == text_classification_model_pb2.TextClassificationModel.W2V_MATCH:
labels_ps = self._extract_pseudo_label(
texts=slim.flatten(examples[InputDataFields.caption_strings]),
vocabulary_list=self._vocabulary_list,
open_vocabulary_list=self._open_vocabulary_list,
embedding_dims=options.embedding_dims)
losses = tf.nn.sigmoid_cross_entropy_with_logits(labels=labels_ps,
logits=logits)
loss = tf.reduce_mean(losses)
else:
raise ValueError('not implemented')
return {'text_cross_entropy_loss': loss}
# 构建graph
points = tf.Variable(data)
cluster = tf.Variable(tf.zeros([N], dtype=tf.int64))
centers = tf.Variable(tf.slice(points.initialized_value(), [0, 0],
[K, 2])) # [第一步] 将原始数据前k个点当做初始中心
repCenters = tf.reshape(tf.tile(centers, [N, 1]), [N, K, 2]) # 复制操作，便于矩阵批量计算距离
repPoints = tf.reshape(tf.tile(points, [1, K]), [N, K, 2])
sumSqure = tf.reduce_sum(tf.square(repCenters - repPoints),
reduction_indices=2) # [第二步] 计算距离
bestCenter = tf.argmin(sumSqure, axis=1) # [第三步] 寻找最近的簇中心
change = tf.reduce_any(tf.not_equal(bestCenter, cluster)) # 检测簇中心是否还在变化
means = clusterMean(points, bestCenter, K) #[第四步] 计算簇内均值
# 将粗内均值变成新的簇中心，同时分类结果也要更新
with tf.control_dependencies([change]):
update = tf.group(centers.assign(means),
cluster.assign(bestCenter)) # 复制函数
with tf.Session() as sess:
sess.run(tf.initialize_all_variables())
changed = True
iterNum = 0
while changed and iterNum < MAX_ITERS: #这个例子里面循环了8次，changed就变成false了
iterNum += 1
# 运行graph
[changed, _] = sess.run([change, update])
def generate_episode(self, batch_mask, prob_compare, prob, counter,
episode, fact_vecs, acc_states, counter_int,
weight_container, bias_container):
"""Generate episode by applying attention to current fact vectors through a modified GRU"""
fact_vecs_t = tf.unpack(tf.transpose(fact_vecs, perm=[1, 0, 2]))
'''TRY REPLACING acc_states WITH episode AND SEE WHICH WORKS BETTER'''
attentions = [
tf.squeeze(self.get_attention(acc_states, fv), squeeze_dims=[1])
for fv in fact_vecs_t
]
attentions = tf.transpose(tf.pack(attentions))
softs = tf.nn.softmax(attentions)
softs = tf.split(1, self.max_input_len, softs)
gru_outputs = []
# set initial state to zero
h = tf.zeros((self.batch_size, self.hidden_size))
# use attention gru
for i, fv in enumerate(fact_vecs_t):
h = self._attention_GRU_step(fv, h, softs[i])
gru_outputs.append(h)
# extract gru outputs at proper index according to input_lens
gru_outputs = tf.pack(gru_outputs)
gru_outputs = tf.transpose(gru_outputs, perm=[1, 0, 2])
#analogous to output, new_state = self.inference_cell(input,state)
episode = _last_relevant(gru_outputs, self.input_len_placeholder)
''' # TARGET_SIDE ATTENTION
episode = self.generate_episode(prev_memory, fact_vecs, concat_all)
'''
p = tf.squeeze(tf.sigmoid(self.shared_linear_layer(episode, 1, True)))
new_batch_mask = tf.logical_and(tf.less(prob + p, self.one_minus_eps),
batch_mask)
new_float_mask = tf.cast(new_batch_mask, tf.float32)
prob += p * new_float_mask
prob_compare += p * tf.cast(batch_mask, tf.float32)
'''based on github.com/tensorflow/tensorflow/issues/5608#issuecomment-260549420'''
#untied
Wt = weight_container.read(counter_int)
bt = bias_container.read(counter_int)
#tied
#Wt = weight_container.read(0)
#bt = bias_container.read(0)
counter_int += 1
def use_remainder():
remainder = tf.constant(1.0, tf.float32, [self.batch_size]) - prob
remainder_expanded = tf.expand_dims(remainder, 1)
tiled_remainder = tf.tile(remainder_expanded,
[1, self.hidden_size])
acc_state = tf.nn.relu(
tf.matmul(
tf.concat(1, [acc_states, episode *
tiled_remainder]), Wt) + bt)
return acc_state
def normal():
p_expanded = tf.expand_dims(p * new_float_mask, 1)
tiled_p = tf.tile(p_expanded, [1, self.hidden_size])
acc_state = tf.nn.relu(
tf.matmul(tf.concat(1, [acc_states, episode * tiled_p]), Wt) +
bt)
return acc_state
counter += tf.constant(1.0, tf.float32,
[self.batch_size]) * new_float_mask
counter_condition = tf.less(counter, self.N)
condition = tf.reduce_any(
tf.logical_and(new_batch_mask, counter_condition))
acc_state = tf.cond(condition, normal, use_remainder)
'''ADD MECHANISM TO INCREASE HALT PROB IF MULTIPLE SIMILAR ATTENTION MASKS IN A ROW;
would be the difference between consecutive attention masks
based on this cooment: reddit.com/r/MachineLearning/comments/59sfz8/research_learning_to_reason_with_adaptive/d9bgqxw/'''
return (new_batch_mask, prob_compare, prob, counter, episode,
fact_vecs, acc_state, counter_int, weight_container,
bias_container)
