def testArgRenames(self):
with self.test_session():
a = [[1.0, 2.0, 3.0], [4.0, 5.0, 6.0]]
b = [[True, False, False], [False, True, True]]
dim0 = [1]
dim1 = [1]
self.assertAllEqual(tf.reduce_any(b, reduction_indices=dim0).eval(), [True, True])
self.assertAllEqual(tf.reduce_all(b, reduction_indices=[0]).eval(), [False, False, False])
self.assertAllEqual(tf.reduce_all(b, reduction_indices=dim1).eval(), [False, False])
self.assertAllEqual(tf.reduce_sum(a, reduction_indices=[1]).eval(), [6.0, 15.0])
self.assertAllEqual(tf.reduce_sum(a, reduction_indices=[0, 1]).eval(), 21.0)
self.assertAllEqual(tf.reduce_sum(a, [0, 1]).eval(), 21.0)
self.assertAllEqual(tf.reduce_prod(a, reduction_indices=[1]).eval(), [6.0, 120.0])
self.assertAllEqual(tf.reduce_prod(a, reduction_indices=[0, 1]).eval(), 720.0)
self.assertAllEqual(tf.reduce_prod(a, [0, 1]).eval(), 720.0)
self.assertAllEqual(tf.reduce_mean(a, reduction_indices=[1]).eval(), [2.0, 5.0])
self.assertAllEqual(tf.reduce_mean(a, reduction_indices=[0, 1]).eval(), 3.5)
self.assertAllEqual(tf.reduce_mean(a, [0, 1]).eval(), 3.5)
self.assertAllEqual(tf.reduce_min(a, reduction_indices=[1]).eval(), [1.0, 4.0])
self.assertAllEqual(tf.reduce_min(a, reduction_indices=[0, 1]).eval(), 1.0)
self.assertAllEqual(tf.reduce_min(a, [0, 1]).eval(), 1.0)
self.assertAllEqual(tf.reduce_max(a, reduction_indices=[1]).eval(), [3.0, 6.0])
self.assertAllEqual(tf.reduce_max(a, reduction_indices=[0, 1]).eval(), 6.0)
self.assertAllEqual(tf.reduce_max(a, [0, 1]).eval(), 6.0)
self.assertAllClose(tf.reduce_logsumexp(a, reduction_indices=[1]).eval(), [3.40760589, 6.40760612])
self.assertAllClose(tf.reduce_logsumexp(a, reduction_indices=[0, 1]).eval(), 6.45619344711)
self.assertAllClose(tf.reduce_logsumexp(a, [0, 1]).eval(), 6.45619344711)
self.assertAllEqual(tf.expand_dims([[1, 2], [3, 4]], dim=1).eval(), [[[1, 2]], [[3, 4]]])
def _is_finite(arg1, *args):
"""Checks if the supplied tensors are finite.
Args:
arg1: A numeric `Tensor`.
*args: (Optional) Other `Tensors` to check for finiteness.
Returns:
is_finite: Scalar boolean `Tensor` indicating whether all the supplied
tensors are finite.
"""
finite = tf.reduce_all(tf.is_finite(arg1))
for arg in args:
finite = finite & tf.reduce_all(tf.is_finite(arg))
return finite
def loop_fn(time, cell_output, cell_state, loop_state):
emit_output = cell_output # == None for time == 0
if cell_output is None: # time == 0
next_cell_state = initial_state
else:
next_cell_state = cell_state
elements_finished = (time >= sequence_length)
finished = tf.reduce_all(elements_finished)
next_input = tf.cond(
finished,
lambda: tf.zeros([batch_size, input_size], dtype=tf.float32),
lambda: inputs_ta.read(time))
next_target = tf.cond(
finished,
lambda: tf.zeros([batch_size, target_size], dtype=tf.float32),
lambda: targets_ta.read(time))
if loop_state is None:
next_input = tf.expand_dims(next_input, 1)
next_input = tf.pad(next_input, [[0,0], [window-1, 0], [0,0]])
else:
next_input = cat_hist(loop_state, next_input, 1)
next_loop_state = next_input
return (elements_finished, RnnHistInputTuple(next_input, next_target), next_cell_state, emit_output, next_loop_state)
def _has_enough_pixels_of_each_object_in_first_frame(
label, decoder_output_stride):
"""Checks if for each object (incl. background) enough pixels are visible.
During test time, we will usually not see a reference frame in which only
very few pixels of one object are visible. These cases can be problematic
during training, especially if more than the 1-nearest neighbor is used.
That's why this function can be used to detect and filter these cases.
Args:
label: Label tensor of shape [num_frames, height, width, 1].
decoder_output_stride: Integer, the stride of the decoder output.
Returns:
Boolean, whether the labels have enough pixels of each object in the first
frame.
"""
h, w = train_utils.resolve_shape(label)[1:3]
h_sub = model.scale_dimension(h, 1.0 / decoder_output_stride)
w_sub = model.scale_dimension(w, 1.0 / decoder_output_stride)
label_downscaled = tf.squeeze(
tf.image.resize_nearest_neighbor(label[0, tf.newaxis], [h_sub, w_sub],
align_corners=True), axis=0)
_, _, counts = tf.unique_with_counts(
tf.reshape(label_downscaled, [-1]))
has_enough_pixels_per_object = tf.reduce_all(
tf.greater_equal(counts, MIN_LABEL_COUNT))
return has_enough_pixels_per_object
def loss_fn(inputs, targets, input_sequence_length, output_sequence_length):
"""Creates the loss and the exports."""
logits = model(
inputs, targets, input_sequence_length, output_sequence_length)
targets = tf.cast(targets, tf.int32)
sq_sz_out_max = targets.shape[0].value
# Create a mask to ignore accuracy on buffer characters.
sequence_sizes = tf.cast(output_sequence_length, tf.float32)
lengths_transposed = tf.expand_dims(sequence_sizes, 1)
range_row = tf.expand_dims(
tf.range(0, sq_sz_out_max, 1, dtype=tf.float32), 0)
mask = tf.cast(tf.transpose(tf.less(range_row, lengths_transposed)),
tf.float32)
# Compute token accuracy and solved.
correct = tf.equal(tf.argmax(logits, 2), tf.argmax(targets, 2))
solved = tf.reduce_all(tf.boolean_mask(correct, tf.squeeze(mask)), axis=0)
token_acc = tf.reduce_sum(tf.cast(correct, tf.float32) * mask)
token_acc /= tf.reduce_sum(sequence_sizes)
# Compute Loss.
mask = tf.cast(tf.tile(tf.expand_dims(mask, 2), (1, 1, logits.shape[2])),
tf.float32)
masked_logits = logits * mask
masked_target = tf.cast(targets, tf.float32) * mask
logits_flat = tf.reshape(masked_logits,
[sq_sz_out_max * batch_size, -1])
target_flat = tf.reshape(masked_target,
[sq_sz_out_max * batch_size, -1])
xent = tf.nn.softmax_cross_entropy_with_logits(logits=logits_flat,
labels=target_flat)
loss = tf.reduce_mean(xent)
return loss, token_acc, solved
def test_mpi_allgather_variable_size(self):
"""Test that the allgather correctly gathers 1D, 2D, 3D tensors,
even if those tensors have different sizes along the first dim."""
with self.test_session() as session:
size = session.run(mpi.size())
rank = session.run(mpi.rank())
dtypes = tf.int32, tf.float32
dims = 1, 2, 3
for dtype, dim in itertools.product(dtypes, dims):
# Support tests up to MPI Size of 35
if size > 35:
break
tensor_sizes = [17, 32, 81, 12, 15, 23, 22] * 5
tensor_sizes = tensor_sizes[:size]
tensor = tf.ones([tensor_sizes[rank]] + [17] * (dim - 1),
dtype=dtype) * rank
gathered = mpi.allgather(tensor)
gathered_tensor = session.run(gathered)
expected_size = sum(tensor_sizes)
self.assertEqual(list(gathered_tensor.shape),
[expected_size] + [17] * (dim - 1))
for i in range(size):
rank_size = [tensor_sizes[i]] + [17] * (dim - 1)
rank_tensor = tf.slice(gathered,
[sum(tensor_sizes[:i])] + [0] * (dim - 1),
rank_size)
self.assertEqual(list(rank_tensor.shape), rank_size)
self.assertTrue(session.run(tf.reduce_all(tf.equal(rank_tensor, i))),
"mpi.allgather produces incorrect gathered tensor")
def test_horovod_allreduce_cpu_fused(self):
"""Test on CPU that the allreduce correctly sums 1D, 2D, 3D tensors
with Tensor Fusion."""
hvd.init()
size = hvd.size()
with self.test_session() as session:
dtypes = [tf.int32, tf.int64, tf.float32, tf.float64]
dims = [1, 2, 3]
tests = []
for dtype, dim in itertools.product(dtypes, dims):
with tf.device("/cpu:0"):
tf.set_random_seed(1234)
tensor = tf.random_uniform(
[17] * dim, -100, 100, dtype=dtype)
summed = hvd.allreduce(tensor, average=False)
multiplied = tensor * size
max_difference = tf.reduce_max(tf.abs(summed - multiplied))
# Threshold for floating point equality depends on number of
# ranks, since we're comparing against precise multiplication.
if size <= 3:
threshold = 0
elif size < 10:
threshold = 1e-4
elif size < 15:
threshold = 5e-4
else:
break
test = max_difference <= threshold
tests.append(test)
self.assertTrue(session.run(tf.reduce_all(tests)),
"hvd.allreduce produces incorrect results")
def test_horovod_broadcast(self):
"""Test that the broadcast correctly broadcasts 1D, 2D, 3D tensors."""
hvd.init()
rank = hvd.rank()
size = hvd.size()
# This test does not apply if there is only one worker.
if size == 1:
return
with self.test_session() as session:
dtypes = [tf.uint8, tf.int8, tf.uint16, tf.int16,
tf.int32, tf.int64, tf.float32, tf.float64,
tf.bool]
dims = [1, 2, 3]
root_ranks = list(range(size))
for dtype, dim, root_rank in itertools.product(dtypes, dims, root_ranks):
try:
tensor = tf.ones([17] * dim) * rank
root_tensor = tf.ones([17] * dim) * root_rank
if dtype == tf.bool:
tensor = tensor % 2
root_tensor = root_tensor % 2
tensor = tf.cast(tensor, dtype=dtype)
root_tensor = tf.cast(root_tensor, dtype=dtype)
broadcasted_tensor = hvd.broadcast(tensor, root_rank)
self.assertTrue(
session.run(tf.reduce_all(tf.equal(
tf.cast(root_tensor, tf.int32), tf.cast(broadcasted_tensor, tf.int32)))),
"hvd.broadcast produces incorrect broadcasted tensor")
except Exception:
import traceback
traceback.print_exc()
def new_mean_squared(grad_vec, decay, ms):
"""Calculates the new accumulated mean squared of the gradient.
Args:
grad_vec: the vector for the current gradient
decay: the decay term
ms: the previous mean_squared value
Returns:
the new mean_squared value
"""
decay_size = decay.get_shape().num_elements()
decay_check_ops = [
tf.assert_less_equal(decay, 1., summarize=decay_size),
tf.assert_greater_equal(decay, 0., summarize=decay_size)]
with tf.control_dependencies(decay_check_ops):
grad_squared = tf.square(grad_vec)
# If the previous mean_squared is the 0 vector, don't use the decay and just
# return the full grad_squared. This should only happen on the first timestep.
decay = tf.cond(tf.reduce_all(tf.equal(ms, 0.)),
lambda: tf.zeros_like(decay, dtype=tf.float32), lambda: decay)
# Update the running average of squared gradients.
epsilon = 1e-12
return (1. - decay) * (grad_squared + epsilon) + decay * ms
def aggregate_single_gradient(grad_and_vars, use_mean, check_inf_nan):
"""Calculate the average gradient for a shared variable across all towers.
Note that this function provides a synchronization point across all towers.
Args:
grad_and_vars: A list or tuple of (gradient, variable) tuples. Each
(gradient, variable) pair within the outer list represents the gradient
of the variable calculated for a single tower, and the number of pairs
equals the number of towers.
use_mean: if True, mean is taken, else sum of gradients is taken.
check_inf_nan: check grads for nans and infs.
Returns:
The tuple ([(average_gradient, variable),], has_nan_or_inf) where the
gradient has been averaged across all towers. The variable is chosen from
the first tower. The has_nan_or_inf indicates the grads has nan or inf.
"""
grads = [g for g, _ in grad_and_vars]
grad = tf.add_n(grads)
if use_mean and len(grads) > 1:
grad = tf.multiply(grad, 1.0 / len(grads))
v = grad_and_vars[0][1]
if check_inf_nan:
has_nan_or_inf = tf.logical_not(tf.reduce_all(tf.is_finite(grads)))
return (grad, v), has_nan_or_inf
else:
return (grad, v), None
def loop_fn_transition(time, previous_output, previous_state, previous_loop_state):
def get_next_input():
# dot product between previous ouput and weights, then + biases
output_logits = tf.add(tf.matmul(previous_output, W), b)
# Logits simply means that the function operates on the unscaled output of
# earlier layers and that the relative scale to understand the units is linear.
# It means, in particular, the sum of the inputs may not equal 1, that the values are not probabilities
# (you might have an input of 5).
# prediction value at current time step
# Returns the index with the largest value across axes of a tensor.
# Attention focusing
prediction = tf.argmax(output_logits, axis=1)
# embed prediction for the next input
next_input = tf.nn.embedding_lookup(embeddings, prediction)
return next_input
elements_finished = (time >= decoder_length) # this operation produces boolean tensor of [batch_size]
# defining if corresponding sequence has ended
# Computes the "logical and" of elements across dimensions of a tensor.
finished = tf.reduce_all(elements_finished) # -> boolean scalar
# Return either fn1() or fn2() based on the boolean predicate pred.
input = tf.cond(finished, lambda: pad_step_embedded, get_next_input)
# set previous to current
state = previous_state
output = previous_output
loop_state = None
return (elements_finished, input, state,
output, loop_state)
def testUniformSamplePdf(self):
with self.test_session():
a = 10.0
b = [11.0, 100.0]
uniform = tf.contrib.distributions.Uniform(a, b)
self.assertTrue(tf.reduce_all(uniform.pdf(uniform.sample_n(10)) > 0).eval(
))
def _verify_compatible_image_shapes(img1, img2):
"""
Checks if two image tensors are compatible for applying SSIM or PSNR.
This function checks if two sets of images have ranks at least 3, and if the
last three dimensions match.
Args:
img1: Tensor containing the first image batch.
img2: Tensor containing the second image batch.
Returns:
A tuple containing: the first tensor shape, the second tensor shape, and a
list of control_flow_ops.Assert() ops implementing the checks.
Raises:
ValueError: When static shape check fails.
"""
shape1 = img1.get_shape().with_rank_at_least(3)
shape2 = img2.get_shape().with_rank_at_least(3)
shape1[-3:].assert_is_compatible_with(shape2[-3:])
if shape1.ndims is not None and shape2.ndims is not None:
for dim1, dim2 in zip(reversed(shape1[:-3]), reversed(shape2[:-3])):
if not (dim1 == 1 or dim2 == 1 or dim1.is_compatible_with(dim2)):
raise ValueError('Two images are not compatible: %s and %s' % (shape1, shape2))
# Now assign shape tensors.
shape1, shape2 = tf.shape_n([img1, img2])
# TODO(sjhwang): Check if shape1[:-3] and shape2[:-3] are broadcastable.
checks = []
checks.append(tf.Assert(tf.greater_equal(tf.size(shape1), 3),
[shape1, shape2], summarize=10))
checks.append(tf.Assert(tf.reduce_all(tf.equal(shape1[-3:], shape2[-3:])),
[shape1, shape2], summarize=10))
return shape1, shape2, checks
def loop_continue_criterion(self, *args) -> tf.Tensor:
"""Decide whether to break out of the while loop.
The criterion for stopping the loop is that either all hypotheses are
finished or a maximum number of steps has been reached. Here the number
of steps is the number of steps of the underlying decoder minus one,
because this function is evaluated after the decoder step has been
called and its step has been incremented. This is caused by the fact
that we call the decoder body function at the end of the beam body
function. (And that, in turn, is to support ensembling.)
Arguments:
args: A ``BeamSearchLoopState`` instance.
Returns:
A scalar boolean ``Tensor``.
"""
loop_state = BeamSearchLoopState(*args)
beam_step = loop_state.decoder_loop_state.feedables.step - 1
finished = loop_state.search_state.finished
max_step_cond = tf.less(beam_step, self.max_steps)
unfinished_cond = tf.logical_not(tf.reduce_all(finished))
return tf.logical_and(max_step_cond, unfinished_cond)
def testUniformSamplePdf(self):
a = 10.0
b = [11.0, 100.0]
uniform = uniform_lib.Uniform(a, b)
self.assertTrue(
self.evaluate(
tf.reduce_all(uniform.prob(uniform.sample(10)) > 0)))
def ImageSample(inputs, borderMode='repeat'):
"""
Sample the images using the given coordinates, by bilinear interpolation.
This was described in the paper:
`Spatial Transformer Networks <http://arxiv.org/abs/1506.02025>`_.
This is equivalent to `torch.nn.functional.grid_sample`,
up to some non-trivial coordinate transformation.
This implementation returns pixel value at pixel (1, 1) for a floating point coordinate (1.0, 1.0).
Note that this may not be what you need.
Args:
inputs (list): [images, coords]. images has shape NHWC.
coords has shape (N, H', W', 2), where each pair of the last dimension is a (y, x) real-value
coordinate.
borderMode: either "repeat" or "constant" (zero-filled)
Returns:
tf.Tensor: a tensor named ``output`` of shape (N, H', W', C).
"""
log_deprecated("ImageSample", "Please implement it in your own code instead!", "2018-12-01")
image, mapping = inputs
assert image.get_shape().ndims == 4 and mapping.get_shape().ndims == 4
input_shape = image.get_shape().as_list()[1:]
assert None not in input_shape, \
"Images in ImageSample layer must have fully-defined shape"
assert borderMode in ['repeat', 'constant']
orig_mapping = mapping
mapping = tf.maximum(mapping, 0.0)
lcoor = tf.floor(mapping)
ucoor = lcoor + 1
diff = mapping - lcoor
neg_diff = 1.0 - diff # bxh2xw2x2
lcoory, lcoorx = tf.split(lcoor, 2, 3)
ucoory, ucoorx = tf.split(ucoor, 2, 3)
lyux = tf.concat([lcoory, ucoorx], 3)
uylx = tf.concat([ucoory, lcoorx], 3)
diffy, diffx = tf.split(diff, 2, 3)
neg_diffy, neg_diffx = tf.split(neg_diff, 2, 3)
ret = tf.add_n([sample(image, lcoor) * neg_diffx * neg_diffy,
sample(image, ucoor) * diffx * diffy,
sample(image, lyux) * neg_diffy * diffx,
sample(image, uylx) * diffy * neg_diffx], name='sampled')
if borderMode == 'constant':
max_coor = tf.constant([input_shape[0] - 1, input_shape[1] - 1], dtype=tf.float32)
mask = tf.greater_equal(orig_mapping, 0.0)
mask2 = tf.less_equal(orig_mapping, max_coor)
mask = tf.logical_and(mask, mask2) # bxh2xw2x2
mask = tf.reduce_all(mask, [3]) # bxh2xw2 boolean
mask = tf.expand_dims(mask, 3)
ret = ret * tf.cast(mask, tf.float32)
return tf.identity(ret, name='output')
def flip_img_coord(_img, _coord):
zeros = tf.constant([[0, 0, 0, 0, 0]]*30, tf.float32)
img_flipped = tf.image.flip_left_right(_img)
idx_invalid = tf.reduce_all(tf.equal(coord, 0), axis=-1)
coord_temp = tf.concat([tf.minimum(tf.maximum(1 - _coord[:, :1], 0), 1),
_coord[:, 1:]], axis=-1)
coord_flipped = tf.where(idx_invalid, zeros, coord_temp)
return img_flipped, coord_flipped
def custom_dynamic_rnn(cell, inputs, inputs_len, initial_state=None):
"""
Implements a dynamic rnn that can store scores in the pointer network,
the reason why we implements this is that the raw_rnn or dynamic_rnn function in Tensorflow
seem to require the hidden unit and memory unit has the same dimension, and we cannot
store the scores directly in the hidden unit.
Args:
cell: RNN cell
inputs: the input sequence to rnn
inputs_len: valid length
initial_state: initial_state of the cell
Returns:
outputs and state
"""
batch_size = tf.shape(inputs)[0]
max_time = tf.shape(inputs)[1]
inputs_ta = tf.TensorArray(dtype=tf.float32, size=max_time)
inputs_ta = inputs_ta.unstack(tf.transpose(inputs, [1, 0, 2]))
emit_ta = tf.TensorArray(dtype=tf.float32, dynamic_size=True, size=0)
t0 = tf.constant(0, dtype=tf.int32)
if initial_state is not None:
s0 = initial_state
else:
s0 = cell.zero_state(batch_size, dtype=tf.float32)
f0 = tf.zeros([batch_size], dtype=tf.bool)
def loop_fn(t, prev_s, emit_ta, finished):
"""
the loop function of rnn
"""
cur_x = inputs_ta.read(t)
scores, cur_state = cell(cur_x, prev_s)
# copy through
scores = tf.where(finished, tf.zeros_like(scores), scores)
if isinstance(cell, tc.rnn.LSTMCell):
cur_c, cur_h = cur_state
prev_c, prev_h = prev_s
cur_state = tc.rnn.LSTMStateTuple(tf.where(finished, prev_c, cur_c),
tf.where(finished, prev_h, cur_h))
else:
cur_state = tf.where(finished, prev_s, cur_state)
emit_ta = emit_ta.write(t, scores)
finished = tf.greater_equal(t + 1, inputs_len)
return [t + 1, cur_state, emit_ta, finished]
_, state, emit_ta, _ = tf.while_loop(
cond=lambda _1, _2, _3, finished: tf.logical_not(tf.reduce_all(finished)),
body=loop_fn,
loop_vars=(t0, s0, emit_ta, f0),
parallel_iterations=32,
swap_memory=False)
outputs = tf.transpose(emit_ta.stack(), [1, 0, 2])
return outputs, state
def safe_sum(x, alt_value=-np.inf, name=None):
"""Elementwise adds list members, replacing non-finite results with alt_value.
Args:
x: Python `list` of `Tensors` to elementwise add.
alt_value: Python scalar used to replace any elementwise sums which would
otherwise be non-finite.
name: Python `str` name prefixed to Ops created by this function.
Default value: `None` (i.e., "safe_sum").
Returns:
safe_sum: `Tensor` representing the elementwise sum of list of `Tensor`s
`x` or `alt_value` where sums are non-finite.
Raises:
TypeError: if `x` is not list-like.
ValueError: if `x` is empty.
"""
with tf.name_scope(name, 'safe_sum', [x, alt_value]):
if not is_list_like(x):
raise TypeError('Expected list input.')
if not x:
raise ValueError('Input should not be empty.')
n = np.int32(len(x))
in_shape = x[0].shape
x = tf.stack(x, axis=-1)
# The sum is NaN if any element is NaN or we see both +Inf and -Inf. Thus
# we will replace such rows with the `alt_value`. Typically the `alt_value`
# is chosen so the `MetropolisHastings` `TransitionKernel` always rejects
# the proposal. rejection.
# Regarding the following float-comparisons, recall comparing with NaN is
# always False, i.e., we're implicitly capturing NaN and explicitly
# capturing +/- Inf.
is_sum_determinate = (
tf.reduce_all(tf.is_finite(x) | (x >= 0.), axis=-1) &
tf.reduce_all(tf.is_finite(x) | (x <= 0.), axis=-1))
is_sum_determinate = tf.tile(
is_sum_determinate[..., tf.newaxis],
multiples=tf.concat([tf.ones(tf.rank(x) - 1, dtype=tf.int32), [n]],
axis=0))
alt_value = np.array(alt_value, x.dtype.as_numpy_dtype)
x = tf.where(is_sum_determinate, x, tf.fill(tf.shape(x), value=alt_value))
x = tf.reduce_sum(x, axis=-1)
x.set_shape(x.shape.merge_with(in_shape))
return x
def _resample_nearest(self, inputs, sample_coords):
# This is forward only as no gradient for tf.round
# read input shape
in_size = inputs.shape
partial_shape = not in_size.is_fully_defined()
in_spatial_size = None
try:
batch_size = int(in_size[0])
n_coords = int(sample_coords.shape[0])
except (TypeError, AssertionError, ValueError):
tf.logging.fatal('Unknown input shape, at least batch size '
'and rank of the inputs are required.')
raise
# quantise coordinates
spatial_coords = tf.round(sample_coords)
if not partial_shape:
in_spatial_size = in_size.as_list()[1:-1]
spatial_coords = self.boundary_func(
spatial_coords, in_spatial_size)
spatial_coords = tf.cast(spatial_coords, COORDINATES_TYPE)
if batch_size == n_coords:
batch_inputs = tf.unstack(inputs)
batch_coords = tf.unstack(spatial_coords)
gathered_image = [tf.gather_nd(img, coord) for (img, coord) in
zip(batch_inputs, batch_coords)]
elif n_coords == 1 and batch_size > 1:
gathered_image = [tf.gather_nd(img, spatial_coords[0])
for img in tf.unstack(inputs)]
else:
raise NotImplementedError
output = tf.stack(gathered_image, axis=0)
if self.boundary == 'ZERO' and in_spatial_size:
scale = 1. / (tf.constant(in_spatial_size, dtype=tf.float32) - 1.)
mask = tf.logical_and(
tf.reduce_all(sample_coords > 0,
axis=-1, keep_dims=True),
tf.reduce_all(scale * sample_coords < 1,
axis=-1, keep_dims=True))
return output * tf.to_float(mask)
return output
def all(x, axis=None, keepdims=False):
'''Bitwise reduction (logical AND).
Returns an uint8 tensor
'''
axis = _normalize_axis(axis, ndim(x))
x = tf.cast(x, tf.bool)
x = tf.reduce_all(x, reduction_indices=axis, keep_dims=keepdims)
return tf.cast(x, tf.uint8)
def smart_reduce_all(preds, name=None):
"""Identical to `tf.reduce_all` but operates statically if possible."""
with tf.name_scope(name, 'smart_reduce_all', [preds]):
pred_values = [tf.contrib.framework.smart_constant_value(p) for p in preds]
if any(p is False for p in pred_values):
return False
if any(p is None for p in pred_values):
return tf.reduce_all(preds)
return all(pred_values)
def _predicate(features, labels):
cond = []
features_length = features_length_fn(features) if features_length_fn is not None else None
labels_length = labels_length_fn(labels) if labels_length_fn is not None else None
if features_length is not None:
cond.extend(_length_constraints(features_length, maximum_features_length))
if labels_length is not None:
cond.extend(_length_constraints(labels_length, maximum_labels_length))
return tf.reduce_all(cond)
def multilayer_perceptron(x):
fc1 = layers.fully_connected(x, 256, activation_fn=tf.nn.relu, scope='fc1')
fc2 = layers.fully_connected(fc1, 256, activation_fn=tf.nn.relu, scope='fc2')
out = layers.fully_connected(fc2, 10, activation_fn=None, scope='out')
assert_op = tf.Assert(tf.reduce_all(out > 0), [out], name='assert_out_positive')
#out = tf.with_dependencies([assert_op], out)
with tf.control_dependencies([assert_op]):
out = tf.identity(out, name='out')
return out
def ImageSample(inputs, borderMode='repeat'):
"""
Sample the template image using the given coordinate, by bilinear interpolation.
This was described in the paper:
`Spatial Transformer Networks <http://arxiv.org/abs/1506.02025>`_.
Args:
inputs (list): [template, coords]. template has shape NHWC.
coords has shape (N,H',W',2), where each pair of the last dimension is a (y, x) real-value
coordinate.
borderMode: either "repeat" or "constant" (zero-filled)
Returns:
tf.Tensor: a tensor named ``output`` of shape (N,H',W',C).
"""
# TODO borderValue
template, mapping = inputs
assert template.get_shape().ndims == 4 and mapping.get_shape().ndims == 4
input_shape = template.get_shape().as_list()[1:]
assert None not in input_shape, \
"Images in ImageSample layer must have fully-defined shape"
assert borderMode in ['repeat', 'constant']
orig_mapping = mapping
mapping = tf.maximum(mapping, 0.0)
lcoor = tf.floor(mapping)
ucoor = lcoor + 1
diff = mapping - lcoor
neg_diff = 1.0 - diff # bxh2xw2x2
lcoory, lcoorx = tf.split(lcoor, 2, 3)
ucoory, ucoorx = tf.split(ucoor, 2, 3)
lyux = tf.concat([lcoory, ucoorx], 3)
uylx = tf.concat([ucoory, lcoorx], 3)
diffy, diffx = tf.split(diff, 2, 3)
neg_diffy, neg_diffx = tf.split(neg_diff, 2, 3)
# prod = tf.reduce_prod(diff, 3, keep_dims=True)
# diff = tf.Print(diff, [tf.is_finite(tf.reduce_sum(diff)), tf.shape(prod),
# tf.reduce_max(diff), diff], summarize=50)
ret = tf.add_n([sample(template, lcoor) * neg_diffx * neg_diffy,
sample(template, ucoor) * diffx * diffy,
sample(template, lyux) * neg_diffy * diffx,
sample(template, uylx) * diffy * neg_diffx], name='sampled')
if borderMode == 'constant':
max_coor = tf.constant([input_shape[0] - 1, input_shape[1] - 1], dtype=tf.float32)
mask = tf.greater_equal(orig_mapping, 0.0)
mask2 = tf.less_equal(orig_mapping, max_coor)
mask = tf.logical_and(mask, mask2) # bxh2xw2x2
mask = tf.reduce_all(mask, [3]) # bxh2xw2 boolean
mask = tf.expand_dims(mask, 3)
ret = ret * tf.cast(mask, tf.float32)
return tf.identity(ret, name='output')
def _logical_and(*args):
"""Convenience function which attempts to statically `reduce_all`."""
args_ = [_static_value(x) for x in args]
if any(x is not None and not bool(x) for x in args_):
return tf.constant(False)
if all(x is not None and bool(x) for x in args_):
return tf.constant(True)
if len(args) == 2:
return tf.logical_and(*args)
return tf.reduce_all(args)
def make_outer_masks(self, outer_masks, input_pianorolls):
"""Returns outer masks, if all zeros created by completion masking."""
outer_masks = tf.to_float(outer_masks)
# If outer_masks come in as all zeros, it means there's no masking,
# which also means nothing will be generated. In this case, use
# completion mask to make new outer masks.
outer_masks = tf.cond(
tf.reduce_all(tf.equal(outer_masks, 0)),
lambda: make_completion_masks(input_pianorolls),
lambda: outer_masks)
return outer_masks
def _resample_nearest(self, inputs, sample_coords):
in_size = inputs.shape.as_list()
in_spatial_size = in_size[1:-1]
# This is forward only as no gradient for tf.round
spatial_coords = self.boundary_func(
tf.round(sample_coords), in_spatial_size)
spatial_coords = tf.cast(spatial_coords, COORDINATES_TYPE)
output = tf.stack([
tf.gather_nd(img, coords) for (img, coords) in
zip(tf.unstack(inputs), tf.unstack(spatial_coords))])
if self.boundary == 'ZERO':
scale = 1. / (tf.constant(in_spatial_size, dtype=tf.float32) - 1)
mask = tf.logical_and(
tf.reduce_all(sample_coords > 0,
axis=-1, keep_dims=True),
tf.reduce_all(scale * sample_coords < 1,
axis=-1, keep_dims=True))
return output * tf.to_float(mask)
return output
def joinTest(self):
ts1 = tf.constant([[0,1], [1, 2], [2,5]])
ts2 = tf.constant([[1,3], [3, 8] ])
ts3 = tf.constant([ [0,1], [1,2], [2,3], [3,5], [5,8] ])
ts1 = tf.cast(ts1, tf.int64)
ts2 = tf.cast(ts2, tf.int64)
ts3 = tf.cast(ts3, tf.int64)
result = tf.user_ops.join_time_series([ts1, ts2])
eqs= tf.equal(result, ts3)
isEqual = tf.reduce_all(eqs)
with tf.Session() as sess:
print sess.run(isEqual)
def log_quaternion_loss_batch(predictions, labels, params):
"""A helper function to compute the error between quaternions.
Args:
predictions: A Tensor of size [batch_size, 4].
labels: A Tensor of size [batch_size, 4].
params: A dictionary of parameters. Expecting 'use_logging', 'batch_size'.
Returns:
A Tensor of size [batch_size], denoting the error between the quaternions.
"""
use_logging = params['use_logging']
assertions = []
if use_logging:
assertions.append(
tf.Assert(
tf.reduce_all(
tf.less(
tf.abs(tf.reduce_sum(tf.square(predictions), [1]) - 1),
1e-4)),
['The l2 norm of each prediction quaternion vector should be 1.']))
assertions.append(
tf.Assert(
tf.reduce_all(
tf.less(
tf.abs(tf.reduce_sum(tf.square(labels), [1]) - 1), 1e-4)),
['The l2 norm of each label quaternion vector should be 1.']))
with tf.control_dependencies(assertions):
product = tf.multiply(predictions, labels)
internal_dot_products = tf.reduce_sum(product, [1])
if use_logging:
internal_dot_products = tf.Print(
internal_dot_products,
[internal_dot_products, tf.shape(internal_dot_products)],
'internal_dot_products:')
logcost = tf.log(1e-4 + 1 - tf.abs(internal_dot_products))
return logcost
def make_completion_masks(pianorolls, outer_masks=1.): pianorolls = tf.to_float(pianorolls) masks = tf.reduce_all(tf.equal(pianorolls, 0), axis=2, keep_dims=True) inner_masks = tf.to_float(masks) + 0 * pianorolls return inner_masks * outer_masks
def project_distribution(supports,
weights,
target_support,
validate_args=False):
"""Projects a batch of (support, weights) onto target_support.
Based on equation (7) in (Bellemare et al., 2017):
https://arxiv.org/abs/1707.06887
In the rest of the comments we will refer to this equation simply as Eq7.
This code is not easy to digest, so we will use a running example to clarify
what is going on, with the following sample inputs:
* supports = [[0, 2, 4, 6, 8],
[1, 3, 4, 5, 6]]
* weights = [[0.1, 0.6, 0.1, 0.1, 0.1],
[0.1, 0.2, 0.5, 0.1, 0.1]]
* target_support = [4, 5, 6, 7, 8]
In the code below, comments preceded with 'Ex:' will be referencing the above
values.
Args:
supports: Tensor of shape (batch_size, num_dims) defining supports for the
distribution.
weights: Tensor of shape (batch_size, num_dims) defining weights on the
original support points. Although for the CategoricalDQN agent these
weights are probabilities, it is not required that they are.
target_support: Tensor of shape (num_dims) defining support of the projected
distribution. The values must be monotonically increasing. Vmin and Vmax
will be inferred from the first and last elements of this tensor,
respectively. The values in this tensor must be equally spaced.
validate_args: Whether we will verify the contents of the
target_support parameter.
Returns:
A Tensor of shape (batch_size, num_dims) with the projection of a batch of
(support, weights) onto target_support.
Raises:
ValueError: If target_support has no dimensions, or if shapes of supports,
weights, and target_support are incompatible.
"""
target_support_deltas = target_support[1:] - target_support[:-1]
# delta_z = `\Delta z` in Eq7.
delta_z = target_support_deltas[0]
validate_deps = []
supports.shape.assert_is_compatible_with(weights.shape)
supports[0].shape.assert_is_compatible_with(target_support.shape)
target_support.shape.assert_has_rank(1)
if validate_args:
# Assert that supports and weights have the same shapes.
validate_deps.append(
tf.Assert(
tf.reduce_all(tf.equal(tf.shape(supports), tf.shape(weights))),
[supports, weights]))
# Assert that elements of supports and target_support have the same shape.
validate_deps.append(
tf.Assert(
tf.reduce_all(
tf.equal(tf.shape(supports)[1], tf.shape(target_support))),
[supports, target_support]))
# Assert that target_support has a single dimension.
validate_deps.append(
tf.Assert(tf.equal(tf.size(tf.shape(target_support)), 1),
[target_support]))
# Assert that the target_support is monotonically increasing.
validate_deps.append(
tf.Assert(tf.reduce_all(target_support_deltas > 0),
[target_support]))
# Assert that the values in target_support are equally spaced.
validate_deps.append(
tf.Assert(tf.reduce_all(tf.equal(target_support_deltas, delta_z)),
[target_support]))
with tf.control_dependencies(validate_deps):
# Ex: `v_min, v_max = 4, 8`.
v_min, v_max = target_support[0], target_support[-1]
# Ex: `batch_size = 2`.
batch_size = tf.shape(supports)[0]
# `N` in Eq7.
# Ex: `num_dims = 5`.
num_dims = tf.shape(target_support)[0]
# clipped_support = `[\hat{T}_{z_j}]^{V_max}_{V_min}` in Eq7.
# Ex: `clipped_support = [[[ 4. 4. 4. 6. 8.]]
# [[ 4. 4. 4. 5. 6.]]]`.
clipped_support = tf.clip_by_value(supports, v_min, v_max)[:, None, :]
# Ex: `tiled_support = [[[[ 4. 4. 4. 6. 8.]
# [ 4. 4. 4. 6. 8.]
# [ 4. 4. 4. 6. 8.]
# [ 4. 4. 4. 6. 8.]
# [ 4. 4. 4. 6. 8.]]
# [[ 4. 4. 4. 5. 6.]
# [ 4. 4. 4. 5. 6.]
# [ 4. 4. 4. 5. 6.]
# [ 4. 4. 4. 5. 6.]
# [ 4. 4. 4. 5. 6.]]]]`.
tiled_support = tf.tile([clipped_support], [1, 1, num_dims, 1])
# Ex: `reshaped_target_support = [[[ 4.]
# [ 5.]
# [ 6.]
# [ 7.]
# [ 8.]]
# [[ 4.]
# [ 5.]
# [ 6.]
# [ 7.]
# [ 8.]]]`.
reshaped_target_support = tf.tile(target_support[:, None],
[batch_size, 1])
reshaped_target_support = tf.reshape(reshaped_target_support,
[batch_size, num_dims, 1])
# numerator = `|clipped_support - z_i|` in Eq7.
# Ex: `numerator = [[[[ 0. 0. 0. 2. 4.]
# [ 1. 1. 1. 1. 3.]
# [ 2. 2. 2. 0. 2.]
# [ 3. 3. 3. 1. 1.]
# [ 4. 4. 4. 2. 0.]]
# [[ 0. 0. 0. 1. 2.]
# [ 1. 1. 1. 0. 1.]
# [ 2. 2. 2. 1. 0.]
# [ 3. 3. 3. 2. 1.]
# [ 4. 4. 4. 3. 2.]]]]`.
numerator = tf.abs(tiled_support - reshaped_target_support)
quotient = 1 - (numerator / delta_z)
# clipped_quotient = `[1 - numerator / (\Delta z)]_0^1` in Eq7.
# Ex: `clipped_quotient = [[[[ 1. 1. 1. 0. 0.]
# [ 0. 0. 0. 0. 0.]
# [ 0. 0. 0. 1. 0.]
# [ 0. 0. 0. 0. 0.]
# [ 0. 0. 0. 0. 1.]]
# [[ 1. 1. 1. 0. 0.]
# [ 0. 0. 0. 1. 0.]
# [ 0. 0. 0. 0. 1.]
# [ 0. 0. 0. 0. 0.]
# [ 0. 0. 0. 0. 0.]]]]`.
clipped_quotient = tf.clip_by_value(quotient, 0, 1)
# Ex: `weights = [[ 0.1 0.6 0.1 0.1 0.1]
# [ 0.1 0.2 0.5 0.1 0.1]]`.
weights = weights[:, None, :]
# inner_prod = `\sum_{j=0}^{N-1} clipped_quotient * p_j(x', \pi(x'))`
# in Eq7.
# Ex: `inner_prod = [[[[ 0.1 0.6 0.1 0. 0. ]
# [ 0. 0. 0. 0. 0. ]
# [ 0. 0. 0. 0.1 0. ]
# [ 0. 0. 0. 0. 0. ]
# [ 0. 0. 0. 0. 0.1]]
# [[ 0.1 0.2 0.5 0. 0. ]
# [ 0. 0. 0. 0.1 0. ]
# [ 0. 0. 0. 0. 0.1]
# [ 0. 0. 0. 0. 0. ]
# [ 0. 0. 0. 0. 0. ]]]]`.
inner_prod = clipped_quotient * weights
# Ex: `projection = [[ 0.8 0.0 0.1 0.0 0.1]
# [ 0.8 0.1 0.1 0.0 0.0]]`.
projection = tf.reduce_sum(inner_prod, 3)
projection = tf.reshape(projection, [batch_size, num_dims])
return projection
def cond(loop_index, sample):
return tf.reduce_all(input_tensor=(sample <= 0.0))
def update_state(self,
gt_boxes,
gt_classes,
pred_boxes,
pred_scores,
pred_classes,
area_ranges=None,
gt_boxes_ignore=None,
sample_weights=None):
batch_size = tf.shape(gt_boxes)[0]
for i in range(self.num_classes):
for b in tf.range(batch_size):
valid_gt_mask = tf.logical_not(
tf.reduce_all(gt_boxes[b] <= 0, 1))
valid_pred_mask = tf.logical_not(
tf.reduce_all(pred_boxes[b] <= 0, 1))
valid_gt_boxes = tf.boolean_mask(gt_boxes[b], valid_gt_mask)
valid_gt_classes = tf.boolean_mask(gt_classes[b],
valid_gt_mask)
valid_pred_boxes = tf.boolean_mask(pred_boxes[b],
valid_pred_mask)
valid_pred_classes = tf.boolean_mask(pred_classes[b],
valid_pred_mask)
valid_pred_scores = tf.boolean_mask(pred_scores[b],
valid_pred_mask)
cls_gt_mask = valid_gt_classes == i + 1
cls_pred_mask = valid_pred_classes == i + 1
cls_gt_boxes = tf.boolean_mask(valid_gt_boxes, cls_gt_mask)
# cls_gt_classes = tf.boolean_mask(valid_gt_classes, cls_gt_mask)
cls_pred_boxes = tf.boolean_mask(valid_pred_boxes,
cls_pred_mask)
cls_pred_scores = tf.boolean_mask(valid_pred_scores,
cls_pred_mask)
# cls_pred_classes = tf.boolean_mask(valid_pred_classes, cls_pred_mask)
if tf.reduce_any(cls_pred_mask):
tp, fp = tf.numpy_function(func=self.tpfp_default,
inp=(cls_pred_boxes,
cls_pred_scores,
cls_gt_boxes),
Tout=(tf.float32, tf.float32))
# tp, fp = self.tpfp_default(cls_pred_boxes.numpy(), cls_pred_scores.numpy(), cls_gt_boxes.numpy())
if self.area_ranges == [(None, None)]:
self.num_gts[i][0] += tf.reduce_sum(
tf.cast(cls_gt_mask, tf.float32))
else:
gt_areas = (
(cls_gt_boxes[:, 2] - cls_gt_boxes[:, 0] + 1) *
(cls_gt_boxes[:, 3] - cls_gt_boxes[:, 1] + 1))
for k, (min_area,
max_area) in enumerate(self.area_ranges):
self.num_gts[i][k] += tf.reduce_sum(
tf.cast(
tf.logical_and(gt_areas >= min_area,
gt_areas < max_area),
tf.float32))
self.true_positives[i].append(tp)
self.false_positives[i].append(fp)
self.predicted_scores[i].append(cls_pred_scores)
def is_finished(self, done, time):
termination_criteria = tf.greater(tf.nn.sigmoid(done), 0.5)
minimum_requirement = tf.greater(time, self.min_iters)
termination = tf.logical_and(termination_criteria, minimum_requirement)
return tf.reduce_all(termination, axis=0)
def tf_check_increasing(t): # Ensure Time is Monotonically Increasing
assert_op = tf.Assert(tf.reduce_all(t[1:] > t[:-1]),
["Time must be monotonic"])
return tf.control_dependencies([assert_op])
def gradcheck(func,
inputs,
eps=1e-6,
atol=1e-5,
rtol=1e-3,
raise_exception=True):
r"""Check gradients computed via small finite differences against analytical
gradients w.r.t. tensors in :attr:`inputs` that are of floating point type
and with ``requires_grad=True``.
The check between numerical and analytical gradients uses :func:`~torch.allclose`.
.. note::
The default values are designed for :attr:`input` of double precision.
This check will likely fail if :attr:`input` is of less precision, e.g.,
``FloatTensor``.
.. warning::
If any checked tensor in :attr:`input` has overlapping memory, i.e.,
different indices pointing to the same memory address (e.g., from
:func:`torch.expand`), this check will likely fail because the numerical
gradients computed by point perturbation at such indices will change
values at all other indices that share the same memory address.
Args:
func (function): a Python function that takes Tensor inputs and returns
a Tensor or a tuple of Tensors
inputs (tuple of Tensor or Tensor): inputs to the function
eps (float, optional): perturbation for finite differences
atol (float, optional): absolute tolerance
rtol (float, optional): relative tolerance
raise_exception (bool, optional): indicating whether to raise an exception if
the check fails. The exception gives more information about the
exact nature of the failure. This is helpful when debugging gradchecks.
Returns:
True if all differences satisfy allclose condition
"""
tupled_inputs = _as_tuple(inputs)
# Make sure that gradients are saved for all inputs
any_input_requiring_grad = False
for inp in tupled_inputs:
if isinstance(inp, tf.Tensor):
if _requires_grad(inp):
if inp.dtype != tf.float64:
warnings.warn(
'At least one of the inputs that requires gradient '
'is not of double precision floating point. '
'This check will likely fail if all the inputs are '
'not of double precision floating point. ')
any_input_requiring_grad = True
# inp.retain_grad()
if not any_input_requiring_grad:
raise ValueError(
'gradcheck expects at least one input tensor to require gradient, '
'but none of the them have requires_grad=True.')
output = _differentiable_outputs(func(*tupled_inputs))
def fail_test(msg):
if raise_exception:
raise RuntimeError(msg)
return False
for i, o in enumerate(output):
if not _requires_grad(o):
continue
def fn(input):
return _as_tuple(func(*input))[i]
analytical, reentrant, correct_grad_sizes = get_analytical_jacobian(
tupled_inputs, o)
numerical = get_numerical_jacobian(fn, tupled_inputs, eps=eps)
if not correct_grad_sizes:
return fail_test('Analytical gradient has incorrect size')
for j, (a, n) in enumerate(zip(analytical, numerical)):
if _numel(a) != 0 or _numel(n) != 0:
if not allclose(a, n, rtol, atol):
return fail_test(
'Jacobian mismatch for output %d with respect to input %d,\n'
'numerical:%s\nanalytical:%s\n' % (i, j, n, a))
if not reentrant:
return fail_test(
'Backward is not reentrant, i.e., running backward with same '
'input and grad_output multiple times gives different values, '
'although analytical gradient matches numerical gradient')
# check if the backward multiplies by grad_output
with tf.GradientTape(persistent=True) as tape:
output = _differentiable_outputs(func(*tupled_inputs))
if any([_requires_grad(o) for o in output]):
diff_input_list = list(iter_tensors(tupled_inputs, True))
grads_input = tape.gradient(output, diff_input_list,
[tf.zeros_like(o) for o in output])
if not len(grads_input) == 0:
raise RuntimeError("no Tensors requiring grad found in input")
# grads_input = torch.autograd.grad(output, diff_input_list, [torch.zeros_like(o) for o in output],
# allow_unused=True)
for gi, i in zip(grads_input, diff_input_list):
if gi is None:
continue
if not tf.reduce_all(tf.equal(gi, 0)):
return fail_test('backward not multiplied by grad_output')
if gi.dtype != i.dtype:
return fail_test("grad is incorrect type")
if gi.shape != i.shape:
return fail_test('grad is incorrect size')
return True
def _all_close(x, y, rtol=1e-5, atol=1e-8):
return tf.reduce_all(tf.abs(x - y) <= tf.abs(y) * rtol + atol)
feed_previous=pred)
# loss
loss = tf.nn.seq2seq.sequence_loss(dec_outputs, labels, weights, vocab_size)
tf.scalar_summary("loss", loss)
summary_op = tf.merge_all_summaries()
# optimizer
optimizer = tf.train.AdamOptimizer(learning_rate=learning_rate)
train_op = optimizer.minimize(loss)
# evaluate
pred = tf.to_int32(tf.argmax(dec_outputs, 2)) # max_seq_length * batch_size
all_labels = tf.reshape(labels, [max_seq_length, -1])
matches = tf.equal(pred, all_labels)
reduce_column_matches = tf.cast(tf.cast(tf.reduce_all(matches, 0), tf.int32),
tf.float32) # boolean to int to float
accuracy = tf.reduce_mean(
reduce_column_matches) # correct only if perfectly correct
logdir = tempfile.mkdtemp()
print logdir
summary_writer = tf.train.SummaryWriter(logdir, sess.graph_def)
pdb.set_trace()
sess.run(tf.initialize_all_variables())
# DATA
with open(formulas_file) as f:
formulas = f.read().splitlines()
with open(train_list_file) as f:
b1 = tf.Variable(tf.random.normal(shape=(300, )))
w2 = tf.Variable(tf.random.normal(shape=(300, 10)))
b2 = tf.Variable(tf.random.normal(shape=(10, )))
z = tf.matmul(x, w1) + b1
z = tf.nn.elu(z)
logits = tf.matmul(z, w2) + b2
y_hat = tf.nn.softmax(logits)
loss_op = tf.reduce_sum(
tf.nn.softmax_cross_entropy_with_logits_v2(labels=y, logits=logits))
optimizer = tf.train.AdamOptimizer(learning_rate=lr)
train_op = optimizer.minimize(loss_op)
accuracy_op = tf.reduce_mean(
tf.to_float(tf.reduce_all(tf.equal(y, tf.round(y_hat)), axis=1)))
with tf.Session() as sess:
sess.run(tf.global_variables_initializer())
for epoch in range(num_epochs):
print(f"Epoch: {epoch}: ")
loss = 0.0
training_acc = 0.0
for batch in range(num_batches):
idx1 = batch * batch_size
idx2 = min(len(X_train), idx1 + batch_size)
r = range(idx1, idx2)
X_batch = X_train[r, :]
y_batch = y_train[r, :]
current_loss, current_training_acc, _ = sess.run(
[loss_op, accuracy_op, train_op],
def goal_fn(experience,
buffer_info,
relabel_orig_prob=0.0,
relabel_next_prob=0.5,
relabel_future_prob=0.0,
relabel_last_prob=0.0,
batch_size=None,
obs_dim=None,
gamma=None):
"""Given experience, sample goals in three ways.
The three ways are using the next state, an arbitrary future state, or a
random state. For the future state relabeling, care must be taken to ensure
that we don't sample experience across the episode boundary. We automatically
set relabel_random_prob = (1 - relabel_next_prob - relabel_future_prob).
Args:
experience: The experience that we aim to relabel.
buffer_info: Information about the replay buffer. We will not change this.
relabel_orig_prob: (float) Fraction of experience to not relabel.
relabel_next_prob: (float) Fraction of experience to relabel with the next
state.
relabel_future_prob: (float) Fraction of experience to relabel with a future
state.
relabel_last_prob: (float) Fraction of experience to relabel with the
final state.
batch_size: (int) The size of the batch.
obs_dim: (int) The dimension of the observation.
gamma: (float) The discount factor. Future states are sampled according to
a Geom(1 - gamma) distribution.
Returns:
experience: A modified version of the input experience where the goals
have been changed and the rewards and terminal flags are recomputed.
buffer_info: Information about the replay buffer.
"""
assert batch_size is not None
assert obs_dim is not None
assert gamma is not None
relabel_orig_num = int(relabel_orig_prob * batch_size)
relabel_next_num = int(relabel_next_prob * batch_size)
relabel_future_num = int(relabel_future_prob * batch_size)
relabel_last_num = int(relabel_last_prob * batch_size)
relabel_random_num = batch_size - (relabel_orig_num + relabel_next_num +
relabel_future_num + relabel_last_num)
assert relabel_random_num >= 0
orig_goals = experience.observation[:relabel_orig_num, 0, obs_dim:]
index = relabel_orig_num
next_goals = experience.observation[index:index + relabel_next_num,
1, :obs_dim]
index = relabel_orig_num + relabel_next_num
future_goals = get_future_goals(
experience.observation[index:index + relabel_future_num, :, :obs_dim],
experience.discount[index:index + relabel_future_num], gamma)
index = relabel_orig_num + relabel_next_num + relabel_future_num
last_goals = get_last_goals(
experience.observation[index:index + relabel_last_num, :, :obs_dim],
experience.discount[index:index + relabel_last_num])
# For random goals we take other states from the same batch.
random_goals = tf.random.shuffle(
experience.observation[:relabel_random_num, 0, :obs_dim])
new_goals = obs_to_goal(
tf.concat([next_goals, future_goals, last_goals, random_goals],
axis=0))
goals = tf.concat([orig_goals, new_goals], axis=0)
obs = experience.observation[:, :2, :obs_dim]
reward = tf.reduce_all(obs_to_goal(obs[:, 1]) == goals, axis=-1)
reward = tf.cast(reward, tf.float32)
reward = tf.tile(reward[:, None], [1, 2])
new_obs = tf.concat([obs, tf.tile(goals[:, None, :], [1, 2, 1])], axis=2)
experience = experience.replace(
observation=new_obs, # [B x 2 x 2 * obs_dim]
action=experience.action[:, :2],
step_type=experience.step_type[:, :2],
next_step_type=experience.next_step_type[:, :2],
discount=experience.discount[:, :2],
reward=reward,
)
return experience, buffer_info
def get_deepof_decoder(
input_shape,
latent_dim,
conv_filters=64,
dense_activation="relu",
gru_units_1=32,
gru_unroll=False,
bidirectional_merge="concat",
):
"""
Returns a deep neural network capable of decoding the structured latent space generated by one of the compatible
classes into a sequence of motion tracking instances, either reconstructing the original
input, or generating new data from given clusters.
Args:
input_shape (tuple): shape of the input data
latent_dim (int): dimensionality of the latent space
conv_filters (int): number of filters in the first convolutional layer
dense_activation (str): activation function for the dense layers. Defaults to "relu".
gru_units_1 (int): number of units in the first GRU layer. Defaults to 128.
gru_unroll (bool): whether to unroll the GRU layers. Defaults to False.
bidirectional_merge (str): how to merge the forward and backward GRU layers. Defaults to "concat".
Returns:
keras.Model: a keras model that can be trained to decode the latent space into a series of motion tracking
sequences.
"""
# Define and instantiate generator
g = Input(shape=latent_dim) # Decoder input, shaped as the latent space
x = Input(
shape=input_shape) # Encoder input, used to generate an output mask
validity_mask = tf.math.logical_not(tf.reduce_all(x == 0.0, axis=2))
generator = RepeatVector(input_shape[0])(g)
generator = Bidirectional(
GRU(
gru_units_1 // 2,
activation="tanh",
recurrent_activation="sigmoid",
return_sequences=True,
unroll=gru_unroll,
use_bias=True,
),
merge_mode=bidirectional_merge,
)(generator, mask=validity_mask)
generator = LayerNormalization()(generator)
generator = Bidirectional(
GRU(
gru_units_1,
activation="tanh",
recurrent_activation="sigmoid",
return_sequences=True,
unroll=gru_unroll,
use_bias=True,
),
merge_mode=bidirectional_merge,
)(generator)
generator = LayerNormalization()(generator)
generator = tf.keras.layers.Conv1D(
filters=conv_filters,
kernel_size=5,
strides=1,
padding="same",
activation=dense_activation,
kernel_initializer=he_uniform(),
use_bias=False,
)(generator)
generator = LayerNormalization()(generator)
x_decoded_mean = TimeDistributed(
Dense(tfpl.IndependentNormal.params_size(input_shape[1:]) //
2))(generator)
# Add a skip connection, adding information directly from the latent space to propagate through the decoder
# early in training.
x_decoded_mean = tf.keras.layers.Add()([
x_decoded_mean,
Dense(input_shape[-1], activation=dense_activation)(g)
])
x_decoded = tfpl.DistributionLambda(
make_distribution_fn=lambda decoded: tfd.Masked(
tfd.Independent(
tfd.Normal(
loc=decoded[0],
scale=tf.ones_like(decoded[0]),
validate_args=False,
allow_nan_stats=False,
),
reinterpreted_batch_ndims=1,
),
validity_mask=decoded[1],
),
convert_to_tensor_fn="mean",
)([x_decoded_mean, validity_mask])
# Zero out values that are not in the initial mask
x_decoded = tfpl.DistributionLambda(
make_distribution_fn=lambda decoded: tfd.Masked(
tfd.TransformedDistribution(
decoded[0],
tfb.Scale(
tf.cast(tf.expand_dims(decoded[1], axis=2), tf.float32)),
name="vae_reconstruction",
),
validity_mask=decoded[1],
),
convert_to_tensor_fn="mean",
)([x_decoded, validity_mask])
return Model([g, x], x_decoded, name="deepof_decoder")
def construct(self, args, source_chars, target_chars, bow, eow):
with self.session.graph.as_default():
if args.recodex:
tf.get_variable_scope().set_initializer(tf.glorot_uniform_initializer(seed=42))
# Inputs
self.sentence_lens = tf.placeholder(tf.int32, [None], name="sentence_lens")
self.source_ids = tf.placeholder(tf.int32, [None, None], name="source_ids")
self.source_seqs = tf.placeholder(tf.int32, [None, None], name="source_seqs")
self.source_seq_lens = tf.placeholder(tf.int32, [None], name="source_seq_lens")
self.target_ids = tf.placeholder(tf.int32, [None, None], name="target_ids")
self.target_seqs = tf.placeholder(tf.int32, [None, None], name="target_seqs")
self.target_seq_lens = tf.placeholder(tf.int32, [None], name="target_seq_lens")
# Append EOW after target_seqs
target_seqs = tf.reverse_sequence(self.target_seqs, self.target_seq_lens, 1)
target_seqs = tf.pad(target_seqs, [[0, 0], [1, 0]], constant_values=eow)
target_seq_lens = self.target_seq_lens + 1
target_seqs = tf.reverse_sequence(target_seqs, target_seq_lens, 1)
# Encoder
# Generate source embeddings for source chars, of shape [source_chars, args.char_dim].
source_embedding = tf.get_variable('source_embedding', shape=[source_chars, args.char_dim])
# Embed the self.source_seqs using the source embeddings.
source_embedded = tf.nn.embedding_lookup(source_embedding, self.source_seqs)
# Using a GRU with dimension args.rnn_dim, process the embedded self.source_seqs
# using forward RNN and store the resulting states into `source_states`.
_, source_states = tf.nn.dynamic_rnn(tf.nn.rnn_cell.GRUCell(args.rnn_dim), source_embedded,
dtype=tf.float32, sequence_length=self.source_seq_lens)
# Index the unique words using self.source_ids and self.target_ids.
sentence_mask = tf.sequence_mask(self.sentence_lens)
source_states = tf.boolean_mask(tf.nn.embedding_lookup(source_states, self.source_ids), sentence_mask)
source_lens = tf.boolean_mask(tf.nn.embedding_lookup(self.source_seq_lens, self.source_ids), sentence_mask)
target_seqs = tf.boolean_mask(tf.nn.embedding_lookup(target_seqs, self.target_ids), sentence_mask)
target_lens = tf.boolean_mask(tf.nn.embedding_lookup(target_seq_lens, self.target_ids), sentence_mask)
# Decoder
# Generate target embeddings for target chars, of shape [target_chars, args.char_dim].
target_embedding = tf.get_variable('target_embedding', shape=[target_chars, args.char_dim])
# Embed the target_seqs using the target embeddings.
target_embedded = tf.nn.embedding_lookup(target_embedding, target_seqs)
# Generate a decoder GRU with wimension args.rnn_dim.
target_GRU = tf.nn.rnn_cell.GRUCell(args.rnn_dim)
# Create a `decoder_layer` -- a fully connected layer with
# target_chars neurons used in the decoder to classify into target characters.
decoder_layer = tf.layers.Dense(target_chars)
# The DecoderTraining will be used during training. It will output logits for each
# target character.
class DecoderTraining(tf.contrib.seq2seq.Decoder):
@property
def batch_size(self):
# Return size of the batch, using for example source_states size
return tf.shape(source_states)[0]
@property
def output_dtype(self): return tf.float32 # Type for logits of target characters
@property
def output_size(self): return target_chars # Length of logits for every output
def initialize(self, name=None):
finished = target_lens <= 0 # False if target_lens > 0, True otherwise
states = source_states # Initial decoder state to use
# embedded BOW characters of shape [self.batch_size] using target embeddings.
# You can use tf.fill to generate BOWs of appropriate size.
inputs = tf.nn.embedding_lookup(target_embedding, tf.fill([self.batch_size], bow))
return finished, inputs, states
def step(self, time, inputs, states, name=None):
# Run the decoder GRU cell using inputs and states.
outputs, states = target_GRU(inputs, states)
# Apply the decoder_layer on outputs.
outputs = decoder_layer(outputs)
# Next input are words with index `time` in target_embedded.
next_input = tf.gather(target_embedded, time, axis=1)
# False if target_lens > time + 1, True otherwise.
finished = target_lens <= time + 1
return outputs, states, next_input, finished
output_layer, _, _ = tf.contrib.seq2seq.dynamic_decode(DecoderTraining())
self.predictions_training = tf.argmax(output_layer, axis=2, output_type=tf.int32)
# The DecoderPrediction will be used during prediction. It will
# directly output the predicted target characters.
class DecoderPrediction(tf.contrib.seq2seq.Decoder):
@property
def batch_size(self):
# Return size of the batch, using for example source_states size
return tf.shape(source_states)[0]
@property
def output_dtype(self): return tf.int32 # Type for predicted target characters
@property
def output_size(self): return 1 # Will return just one output
def initialize(self, name=None):
finished = tf.fill([self.batch_size], False) # False of shape [self.batch_size].
states = source_states # Initial decoder state to use.
# embedded BOW characters of shape [self.batch_size] using target embeddings.
# You can use tf.fill to generate BOWs of appropriate size.
inputs = tf.nn.embedding_lookup(target_embedding, tf.fill([self.batch_size], bow))
return finished, inputs, states
def step(self, time, inputs, states, name=None):
# Run the decoder GRU cell using inputs and states.
outputs, states = target_GRU(inputs, states)
outputs = decoder_layer(outputs) # Apply the decoder_layer on outputs.
# Use tf.argmax to choose most probable class (supply parameter `output_type=tf.int32`).
outputs = tf.argmax(outputs, axis=1, output_type=tf.int32)
# Embed `outputs` using target_embeddings
next_input = tf.nn.embedding_lookup(target_embedding, outputs)
# True where outputs==eow, False otherwise
# Use tf.equal for the comparison, Python's '==' is not overloaded
finished = tf.equal(outputs, eow)
return outputs, states, next_input, finished
self.predictions, _, self.prediction_lens = tf.contrib.seq2seq.dynamic_decode(
DecoderPrediction(), maximum_iterations=tf.reduce_max(source_lens) + 10)
# Training
weights = tf.sequence_mask(target_lens, dtype=tf.float32)
loss = tf.losses.sparse_softmax_cross_entropy(target_seqs, output_layer, weights=weights)
global_step = tf.train.create_global_step()
self.training = tf.train.AdamOptimizer().minimize(loss, global_step=global_step, name="training")
# Summaries
accuracy_training = tf.reduce_all(tf.logical_or(
tf.equal(self.predictions_training, target_seqs),
tf.logical_not(tf.sequence_mask(target_lens))), axis=1)
self.current_accuracy_training, self.update_accuracy_training = tf.metrics.mean(accuracy_training)
minimum_length = tf.minimum(tf.shape(self.predictions)[1], tf.shape(target_seqs)[1])
accuracy = tf.logical_and(
tf.equal(self.prediction_lens, target_lens),
tf.reduce_all(tf.logical_or(
tf.equal(self.predictions[:, :minimum_length], target_seqs[:, :minimum_length]),
tf.logical_not(tf.sequence_mask(target_lens, maxlen=minimum_length))), axis=1))
self.current_accuracy, self.update_accuracy = tf.metrics.mean(accuracy)
self.current_loss, self.update_loss = tf.metrics.mean(loss, weights=tf.reduce_sum(weights))
self.reset_metrics = tf.variables_initializer(tf.get_collection(tf.GraphKeys.METRIC_VARIABLES))
summary_writer = tf.contrib.summary.create_file_writer(args.logdir, flush_millis=10 * 1000)
self.summaries = {}
with summary_writer.as_default(), tf.contrib.summary.record_summaries_every_n_global_steps(10):
self.summaries["train"] = [tf.contrib.summary.scalar("train/loss", self.update_loss),
tf.contrib.summary.scalar("train/accuracy", self.update_accuracy_training)]
with summary_writer.as_default(), tf.contrib.summary.always_record_summaries():
for dataset in ["dev", "test"]:
self.summaries[dataset] = [tf.contrib.summary.scalar(dataset + "/loss", self.current_loss),
tf.contrib.summary.scalar(dataset + "/accuracy", self.current_accuracy)]
# Initialize variables
self.session.run(tf.global_variables_initializer())
with summary_writer.as_default():
tf.contrib.summary.initialize(session=self.session, graph=self.session.graph)
def test_categorical_embedding_to_encoding_bilstm_oov_mapping_with_dropout(
self):
"""
Asserts "out of vocabulary" words are mapped to the same value when dropout_rate is used,
while word "in vocabulary" are mapped to their own embeddings.
"""
embedding_size = 128
encoding_size = 512
vocabulary_file = (
"ml4ir/applications/classification/tests/data/configs/vocabulary/entity_id.csv"
)
feature_info = {
"name": "categorical_variable",
"feature_layer_info": {
"type": "numeric",
"fn": "categorical_embedding_to_encoding_bilstm",
"args": {
"vocabulary_file": vocabulary_file,
"encoding_type": "bilstm",
"encoding_size": encoding_size,
"embedding_size": embedding_size,
"dropout_rate": 0.2,
},
},
"tfrecord_type": SequenceExampleTypeKey.CONTEXT,
}
df_vocabulary = self.file_io.read_df(vocabulary_file)
vocabulary = df_vocabulary.values.flatten().tolist()
# Define a batch where each row is a single word from the vocabulary (so that we can later compare each word
# encoding)
words = vocabulary + ["out_of_vocabulary", "out_of_vocabulary2"]
string_tensor = tf.constant([[[word]] for word in words])
# If the embedding lookup input dimension were computed incorrectly, the following call would fail with:
# tensorflow.python.framework.errors_impl.InvalidArgumentError: indices[0,0,7] = 8 is not in [0, 8)
sequence_encoding = categorical_fns.categorical_embedding_to_encoding_bilstm(
string_tensor, feature_info, self.file_io)
# Strings 1 and 2 should result in the same embedding because they are both OOV
# Check that each word, except the OOV ones are mapped to different encodings (actually different embeddings,
# but the encoding on top should be equivalent as each row is a single word).
for word1_position in range(0, len(vocabulary)):
# +1 to include one OOV word
for word2_position in range(word1_position + 1,
len(vocabulary) + 1):
if tf.reduce_all(
tf.equal(sequence_encoding[word1_position],
sequence_encoding[word2_position])):
word1, word2 = words[word1_position], words[word2_position]
self.assertFalse(
tf.reduce_all(
tf.equal(
sequence_encoding[word1_position],
sequence_encoding[word2_position],
)),
msg=
"Check that non-OOV words map to different encodings, and not also not to OOV:"
"{} vs. {}".format(word1, word2),
)
self.assertTrue(
tf.reduce_all(
tf.equal(sequence_encoding[-1], sequence_encoding[-2])),
msg="Check that OOV words map to the same encodings",
)
assert not tf.reduce_all(
tf.equal(sequence_encoding[0], sequence_encoding[1]))
def _condition(unused_step, finished, unused_inputs,
unused_lengths, unused_log_probs, unused_cache):
return tf.logical_not(tf.reduce_all(finished))
# define graph
red_sum = tf.reduce_sum(x)
# axis=1 if sum all columns, axis=0 if sum all rows
red_sum_column = tf.reduce_sum(x, axis=1)
red_sum_row = tf.reduce_sum(x, axis=0)
red_min = tf.reduce_min(x)
red_min_column = tf.reduce_min(x, axis=1)
red_max = tf.reduce_max(x)
red_max_column = tf.reduce_max(x, axis=1)
red_mean = tf.reduce_mean(x)
red_mean_column = tf.reduce_mean(x, axis=1)
red_bool_all_0 = tf.reduce_all(bool_tensor)
red_bool_all = tf.reduce_all(bool_tensor, axis=1)
red_bool_any_0 = tf.reduce_any(bool_tensor)
red_bool_any = tf.reduce_any(bool_tensor, axis=1)
# run with session
with tf.Session() as session:
merged = tf.summary.merge_all()
writer = tf.summary.FileWriter("logs", session.graph)
print "Reduce sum without passed axis parameter: ", session.run(red_sum)
print("Reduce sum with passed axis=1 (sum columns): ",
session.run(red_sum_column))
print("Reduce sum with passed axis=0 (sum rows): ",
session.run(red_sum_row))
def extract_non_ignore_indices(sequences):
return tf.compat.v1.where(
tf.reduce_all(input_tensor=tf.not_equal(
tf.expand_dims(sequences, 2), [[ignore_ids]]),
axis=2))
def sampling_tf(neighborhood, sample_mode, name=None):
""" Method to sample the points of a point cloud.
Args:
neighborhood: A `Neighborhood` instance, which contains a point cloud with
its neighbors.
sample_mode: An `int`specifiying the sample mode,
`0` for average, `1` for poisson.
Returns:
sampled_points: A `float` `Tensor` of shape [S, D], the sampled points.
sampled_batch_ids: An `int` `Tensor` of shape [S], the batch ids.
sampled_indices: An `int` `Tensor` of shape [S], the indices to the
unsampled points.
Following the order of neighborhood._grid._sorted_points.
"""
points = neighborhood._grid._sorted_points
batch_ids = neighborhood._grid._sorted_batch_ids
num_points = tf.shape(points)[0]
if sample_mode == 0:
# poisson sampling
nb_ranges = tf.concat(([0], neighborhood._samples_neigh_ranges),
axis=0)
neighbors = neighborhood._neighbors
num_points = tf.shape(neighborhood._grid._sorted_points)[0]
log_probabilities = tf.ones([num_points])
sampled_indices = tf.zeros([0], dtype=tf.int64)
# to set log prob to -inf <=> prob to zero
tf_neg_inf = tf.constant(-np.inf)
#sample points until all log probabilites are set to -inf
while not tf.reduce_all(tf.math.is_inf(log_probabilities)):
choice = tf.random.categorical(
tf.expand_dims(log_probabilities, axis=0), 1)[0]
# add choice to sampled indices
sampled_indices = tf.concat((sampled_indices, choice), axis=0)
# set log probability of neighbors to -inf
sample_neighbors = \
neighbors[nb_ranges[choice[0]]:nb_ranges[choice[0] + 1], 0]
num_neighbors = tf.shape(sample_neighbors)[0]
log_probabilities = tf.tensor_scatter_nd_update(
log_probabilities, tf.expand_dims(sample_neighbors, axis=1),
tf.repeat(tf_neg_inf, num_neighbors))
sampled_points = tf.gather(points, sampled_indices)
sampled_batch_ids = tf.gather(batch_ids, sampled_indices)
elif sample_mode == 1:
# cell average sampling
keys = neighborhood._grid._sorted_keys
# replace keys with numbers 0 to num_unique keys
unique, _, counts = tf.unique_with_counts(keys)
num_unique_keys = tf.shape(unique)[0]
new_keys = tf.repeat(tf.range(0, num_unique_keys), counts)
# average over points with same cell key
sampled_points = tf.math.segment_mean(points, new_keys)
# get batch of a point in the same cell
sampled_indices = tf.math.segment_min(tf.range(0, num_points),
new_keys)
sampled_batch_ids = tf.gather(batch_ids, sampled_indices)
return sampled_points, sampled_batch_ids, sampled_indices
def condition2(sigma, ak, am):
sigma = tf.matmul(sigma, testOp2)
return tf.logical_not(
tf.reduce_all(tf.equal(sigma, tf.zeros([4], dtype=tf.float64))))
def _loop_cond(curr):
# TODO(b/112524024): Also take into account max_iterations.
return ~tf.reduce_all(input_tensor=curr.stopped)
def _filter_invalid_transition(trajectories, unused_arg1):
return tf.reduce_all(~trajectories.is_boundary()[:-1])
def loop_fn(time, cell_output, cell_state, loop_state):
elements_finished = (time >= self.max_time)
finished = tf.reduce_all(elements_finished)
if cell_output is None:
'''
time == 0, used for initialization before first call to cell
This is just to defined the desired shape of the tensors
'''
next_cell_state = self.cell.zero_state(self.batch_size, tf.float32)
'''
the emit_output in this case tells TF how future emits look
For the first call to loop_fn the emit_output corresponds to
the emit_structure which is then used to determine the size of
the zero_tensor for the emit_ta (defaults to cell.output_size).
'''
emit_output = tf.tuple([tf.zeros([self.output_dim]), tf.zeros([self.output_dim]),
tf.zeros([self.output_dim])])
# tf.zeros([config.batch_size, output_dim], dtype=tf.float32) # tf.zeros([output_dim])
next_loop_state = output_ta
'''
this is the initial step, i.e. there is no output from a previous time step, what we feed here
can highly depend on the data. In this case we just assign the actual input in the first time step.
'''
if (self.is_sample):
next_in = self.input_
else:
next_in = inputs_ta.read(time)
else:
'''
t > 0, called right after call to cell, i.e. cell_output is the output from time t-1.
here you can do whatever ou want with cell_output before assigning it to emit_output.
In this case, we don't do anything pass the last state to the next
'''
next_cell_state = cell_state
next_loop_state = self.get_next_loop_state(loop_state, cell_state, time)
'''Next Output'''
# cell_output = tf.Print(cell_output,[cell_output], message="cell_output")
mean, var, current_z = self.get_output_step(cell_output)
# current_z = tf.Print(current_z,[current_z], message="current z")
emit_output = tf.tuple([mean, var, current_z])
# tf.tuple([mean, var]) tf.concat([mean, var],1) cell_output mean
next_in = current_z
if (self.is_sample):
next_input = tf.cond(finished,
lambda: tf.zeros([self.batch_size, self.rnn_input_dim], dtype=tf.float32),
lambda: next_in)
else:
next_input = tf.cond(finished,
lambda: tf.zeros([self.batch_size, self.rnn_input_dim], dtype=tf.float32),
lambda: inputs_ta.read(time))
next_input.set_shape([None, self.rnn_input_dim])
return (finished, next_input, next_cell_state, emit_output, next_loop_state)
def contains_nan(w):
for w_ in w:
if tf.reduce_all(input_tensor=tf.math.is_nan(w_)) is None:
return tf.reduce_all(input_tensor=tf.math.is_nan(w_))
return tf.reduce_all(input_tensor=tf.math.is_nan(w_))
def multiPred(y_true, y_pred):
y = K.greater(y_pred, 0.5)
y_p = K.cast(y, K.floatx())
score = tf.reduce_all(K.equal(y_true, y_p), 1)
score = K.cast(score, K.floatx())
return tf.reduce_mean(score)
rgb_model = load_i3d_model(num_classes=NUM_CLASSES) init_model(model=rgb_model,sess=sess, ckpt_path=ckpt_path,eval_type=eval_type) sess.run(eps_rgb.initializer) model_logits, _ = rgb_model(adversarial_inputs_rgb, is_training=False, dropout_keep_prob=1.0) softmax = tf.nn.softmax(logits = model_logits) labels = tf.placeholder(tf.int64, (_BATCH_SIZE,), name='labels') corret_cls_prob = tf.reduce_sum(tf.gather(softmax,labels,axis=-1)) # for untargeted attack set to -1.0 label_coeff_default = tf.constant(-1.0, dtype=tf.float32) labels_coeff = tf.placeholder_with_default(label_coeff_default, name='label_coeff', shape=label_coeff_default.shape) _is_adversarial = tf.cast(tf.reduce_all(tf.not_equal(tf.argmax(softmax,axis=-1),labels)), dtype=tf.float32) # is_adversarial = tf.cast(tf.argmax(softmax[0, ...]) == labels, dtype=tf.float32) inputs = rgb_input perturbation = eps_rgb # target label: for untargeted attack set original label # adversarial classification loss # scores, ind = tf.math.top_k(tf.squeeze(softmax,axis=0),k=2) # max_non_correct_cls = scores[tf.cast(tf.squeeze(tf.equal(tf.cast(ind[0], dtype=tf.int64), labels)),dtype=tf.int64)] max_non_correct_cls = tf.reduce_max(softmax-tf.one_hot(labels,NUM_CLASSES),axis=-1)
def _loop_cond(curr):
return (curr.iteration <
max_iterations) & ~tf.reduce_all(input_tensor=curr.stopped)
def single_loss_graph(self, pred_obj_coord_image, image_padding, segmentation_image,
target_obj_coord_image, seg_and_coord_padding, color):
""" Loss for one batch entry.
pred_obj_coords: [height, width, 3]. The predicted object coordinates.
image_padding: [(bottom, right)]. The widths of the paddings of the original image.
segmentation_image: height, width, 3]. The segmentation image.
target_obj_coords: [height, width, 3]. The ground-truth object coordinates.
seg_and_coord_padding: [(bottom, right)]. The widths of the paddings of the object
coord image and segmentation image. It differs from the paddings of
the original input image because both are not reszied.
color: [r, g, b]. The object's color in the segmentation image.
"""
# The mask that takes care of downsampling, i.e. that only every i-th pixel is computed
# in case that the output image is smaller than the input image
input_shape = tf.shape(target_obj_coord_image)
batch_size = input_shape[0]
# The shape of the image that the network output
output_shape = tf.shape(pred_obj_coord_image)
# With the downsampling ratio we know how much smaller the padding is in the output image
ratio_y = output_shape[0] / input_shape[0]
ratio_x = output_shape[1] / input_shape[1]
# We multiply top and bottom by the y ratio
image_padding = tf.cast(image_padding, tf.float64)
image_padding_y = image_padding[0] * ratio_y
image_padding_x = image_padding[1] * ratio_x
image_padding_y = tf.cast(image_padding_y, tf.int32)
image_padding_x = tf.cast(image_padding_x, tf.int32)
# Now we have the actual predicted area for each batch image individually
# [:2] because we only need (height, width) without the channels
actual_output_shape = output_shape[:2] - (image_padding_y, image_padding_x)
acutal_output_shape_with_channels = [actual_output_shape[0], actual_output_shape[1], 3]
# Crop the output prediction to the actual area without padding
cropped_pred_obj_coord_image = pred_obj_coord_image[:actual_output_shape[0],
:actual_output_shape[1],
:]
# Now we compute the original shape of the ground truth and segmentation
actual_input_shape = input_shape[:2] - tf.cast(seg_and_coord_padding, tf.int32)
# And also crop the segmentation and prediction to the actual area
cropped_segmentation_image = segmentation_image[:actual_input_shape[0],
:actual_input_shape[1],
:]
cropped_target_obj_coord_image = target_obj_coord_image[:actual_input_shape[0],
:actual_input_shape[1],
:]
# Here we compute the relevant indices, e.g. if the output image is 1/8th of the original
# input image, we use only every 8-th pixel horizontally and vertically. The code takes
# care of the output image's size not being a divisor of the input image's.
indices = self.compute_indices_graph(tf.shape(cropped_segmentation_image)[:2],
tf.shape(cropped_pred_obj_coord_image)[:2])
#indices = tf.Print(indices, [indices], "indices", summarize=(63 * 63))
# Now create and object coord and segmentation image of the size of the output shape
# that contains only the relevent indices
final_target_obj_coord_image = tf.gather_nd(cropped_target_obj_coord_image,
indices)
final_target_obj_coord_image = tf.reshape(final_target_obj_coord_image,
acutal_output_shape_with_channels)
final_segmentation_image = tf.gather_nd(cropped_segmentation_image, indices)
final_segmentation_image = tf.reshape(final_segmentation_image,
acutal_output_shape_with_channels)
segmentation_mask = tf.equal(final_segmentation_image, color)
# We have a matrix of bool values of which indices to use after this step
segmentation_mask = tf.reduce_all(segmentation_mask, axis=2)
segmentation_mask = tf.cast(segmentation_mask, tf.bool)
# L1 loss: sum of squared element-wise differences
diff = final_target_obj_coord_image - cropped_pred_obj_coord_image
squared_diff = tf.square(diff)
loss = tf.reduce_sum(squared_diff, axis=2)
loss = tf.sqrt(loss)
# This is the actual L1 loss, the computation above is the euclidean distance
# We commented it out, because it's unnecessary, sqrt implies positive values already
#loss = tf.abs(loss)
loss = tf.boolean_mask(loss, segmentation_mask)
return tf.reduce_mean(loss)
def box_voting(selected_boxes, pool_boxes, iou_thresh=0.5):
"""Performs box voting as described in S. Gidaris and N. Komodakis, ICCV 2015.
Performs box voting as described in 'Object detection via a multi-region &
semantic segmentation-aware CNN model', Gidaris and Komodakis, ICCV 2015. For
each box 'B' in selected_boxes, we find the set 'S' of boxes in pool_boxes
with iou overlap >= iou_thresh. The location of B is set to the weighted
average location of boxes in S (scores are used for weighting). And the score
of B is set to the average score of boxes in S.
Args:
selected_boxes: BoxList containing a subset of boxes in pool_boxes. These
boxes are usually selected from pool_boxes using non max suppression.
pool_boxes: BoxList containing a set of (possibly redundant) boxes.
iou_thresh: (float scalar) iou threshold for matching boxes in
selected_boxes and pool_boxes.
Returns:
BoxList containing averaged locations and scores for each box in
selected_boxes.
Raises:
ValueError: if
a) selected_boxes or pool_boxes is not a BoxList.
b) if iou_thresh is not in [0, 1].
c) pool_boxes does not have a scores field.
"""
if not 0.0 <= iou_thresh <= 1.0:
raise ValueError('iou_thresh must be between 0 and 1')
if not isinstance(selected_boxes, box_list.BoxList):
raise ValueError('selected_boxes must be a BoxList')
if not isinstance(pool_boxes, box_list.BoxList):
raise ValueError('pool_boxes must be a BoxList')
if not pool_boxes.has_field('scores'):
raise ValueError('pool_boxes must have a \'scores\' field')
iou_ = iou(selected_boxes, pool_boxes)
match_indicator = tf.to_float(tf.greater(iou_, iou_thresh))
num_matches = tf.reduce_sum(match_indicator, 1)
# TODO: Handle the case where some boxes in selected_boxes do not
# match to any boxes in pool_boxes. For such boxes without any matches, we
# should return the original boxes without voting.
match_assert = tf.Assert(tf.reduce_all(tf.greater(num_matches, 0)), [
'Each box in selected_boxes must match with at least one box '
'in pool_boxes.'
])
scores = tf.expand_dims(pool_boxes.get_field('scores'), 1)
scores_assert = tf.Assert(tf.reduce_all(tf.greater_equal(scores, 0)),
['Scores must be non negative.'])
with tf.control_dependencies([scores_assert, match_assert]):
sum_scores = tf.matmul(match_indicator, scores)
averaged_scores = tf.reshape(sum_scores, [-1]) / num_matches
box_locations = tf.matmul(match_indicator,
pool_boxes.get() * scores) / sum_scores
averaged_boxes = box_list.BoxList(box_locations)
_copy_extra_fields(averaged_boxes, selected_boxes)
averaged_boxes.add_field('scores', averaged_scores)
return averaged_boxes
def preprocess_true_boxes(true_boxes, anchors, feat_size, image_size):
"""
:param true_boxes: x, y, w, h, class
:param anchors:
:param feat_size:
:param image_size:
:return:
"""
num_anchors = cfg.num_anchors_per_layer
true_wh = tf.expand_dims(true_boxes[..., 2:4], 2) # [batch, 30, 1, 2]
true_wh_half = true_wh / 2.
true_mins = 0 - true_wh_half
true_maxes = true_wh_half
img_wh = tf.reshape(tf.stack([image_size[2], image_size[1]]), [1, -1])
anchors = anchors / tf.cast(img_wh, tf.float32) # normalize
anchors_shape = tf.shape(anchors) # [num_anchors, 2]
anchors = tf.reshape(
anchors,
[1, 1, anchors_shape[0], anchors_shape[1]]) # [1, 1, num_anchors, 2]
anchors_half = anchors / 2.
anchors_mins = 0 - anchors_half
anchors_maxes = anchors_half
intersect_mins = tf.maximum(true_mins, anchors_mins)
intersect_maxes = tf.minimum(true_maxes, anchors_maxes)
intersect_wh = tf.maximum(intersect_maxes - intersect_mins,
0.) # [batch, 30, num_anchors, 2]
intersect_areas = intersect_wh[..., 0] * intersect_wh[
..., 1] # [batch, 30, num_anchors]
true_areas = true_wh[..., 0] * true_wh[..., 1] # [batch, 30, 1]
anchors_areas = anchors[..., 0] * anchors[..., 1] # [1, 1, num_anchors]
union_areas = true_areas + anchors_areas - intersect_areas # [batch, 30, num_anchors]
iou_scores = intersect_areas / union_areas # [batch, 30, num_anchors]
valid = tf.logical_not(tf.reduce_all(tf.equal(iou_scores, 0),
axis=-1)) # [batch, 30]
iout_argmax = tf.cast(tf.argmax(iou_scores, axis=-1),
tf.int32) # [batch, 30], (0, 1, 2)
anchors = tf.reshape(anchors, [-1, 2]) # has been normalize by img shape
anchors_cf = tf.gather(anchors, iout_argmax) # [batch, 30, 2]
feat_wh = tf.reshape(tf.stack([feat_size[2], feat_size[1]]),
[1, -1]) # (1, 2)
cxy = tf.cast(tf.floor(true_boxes[..., :2] * tf.cast(feat_wh, tf.float32)),
tf.int32) # [batch, 30, 2] bx = cx + σ(tx)
sig_xy = tf.cast(true_boxes[..., :2] * tf.cast(feat_wh, tf.float32) -
tf.cast(cxy, tf.float32), tf.float32) # [batch, 30, 2]
idx = cxy[..., 1] * (num_anchors * feat_size[2]) + num_anchors * cxy[
..., 0] + iout_argmax # [batch, 30]
idx_one_hot = tf.one_hot(idx,
depth=feat_size[1] * feat_size[2] *
num_anchors) # [batch, 30, 13x13x3]
idx_one_hot = tf.reshape(
idx_one_hot,
[-1, cfg.train.max_truth, feat_size[1], feat_size[2], num_anchors, 1
]) # (batch, 30, 13, 13, 3, 1)
loc_scale = 2 - true_boxes[..., 2] * true_boxes[..., 3] # (batch, 30)
mask = []
loc_cls = []
scale = []
for i in range(cfg.batch_size):
idx_i = tf.where(valid[i])[:, 0] # (?, ) # false / true
mask_i = tf.gather(idx_one_hot[i], idx_i) # (?, 13, 13, 3, 1)
scale_i = tf.gather(loc_scale[i], idx_i) # (?, )
scale_i = tf.reshape(scale_i, [-1, 1, 1, 1, 1]) # (?, 1, 1, 1, 1)
scale_i = scale_i * mask_i # (?, 13, 13, 3, 1)
scale_i = tf.reduce_sum(scale_i, axis=0) # (13, 13, 3, 1)
scale_i = tf.maximum(tf.minimum(scale_i, 2), 1)
scale.append(scale_i)
true_boxes_i = tf.gather(true_boxes[i], idx_i) # (?, 5)
sig_xy_i = tf.gather(sig_xy[i], idx_i) # (?, 2)
anchors_cf_i = tf.gather(anchors_cf[i], idx_i) # (?, 2)
twh = tf.log(true_boxes_i[:, 2:4] / anchors_cf_i)
loc_cls_i = tf.concat([sig_xy_i, twh, true_boxes_i[:, -1:]],
axis=-1) # (?, 5)
loc_cls_i = tf.reshape(loc_cls_i, [-1, 1, 1, 1, 5]) # (?, 1, 1, 1, 5)
loc_cls_i = loc_cls_i * mask_i # (?, 13, 13, 3, 5)
loc_cls_i = tf.reduce_sum(loc_cls_i, axis=[0]) # (13, 13, 3, 5)
# exception, one anchor is responsible for 2 or more object
loc_cls_i = tf.concat(
[loc_cls_i[..., :4],
tf.minimum(loc_cls_i[..., -1:], 19)], axis=-1)
loc_cls.append(loc_cls_i)
mask_i = tf.reduce_sum(mask_i, axis=[0]) # (13, 13, 3, 1)
mask_i = tf.minimum(mask_i, 1)
mask.append(mask_i)
loc_cls = tf.stack(loc_cls, axis=0) # (σ(tx), σ(tx), tw, th, cls)
mask = tf.stack(mask, axis=0)
scale = tf.stack(scale, axis=0)
return loc_cls, mask, scale
def apply_updates(self):
assert not self._updates_applied
self._updates_applied = True
devices = list(self._dev_grads.keys())
total_grads = sum(len(grads) for grads in self._dev_grads.values())
assert len(devices) >= 1 and total_grads >= 1
ops = []
with absolute_name_scope(self.scope):
# Cast gradients to FP32 and calculate partial sum within each device.
dev_grads = OrderedDict() # device => [(grad, var), ...]
for dev_idx, dev in enumerate(devices):
with tf.name_scope('ProcessGrads%d' % dev_idx), tf.device(dev):
sums = []
for gv in zip(*self._dev_grads[dev]):
assert all(v is gv[0][1] for g, v in gv)
g = [tf.cast(g, tf.float32) for g, v in gv]
g = g[0] if len(g) == 1 else tf.add_n(g)
sums.append((g, gv[0][1]))
dev_grads[dev] = sums
# Sum gradients across devices.
if len(devices) > 1:
with tf.name_scope('SumAcrossGPUs'), tf.device(None):
for var_idx, grad_shape in enumerate(self._grad_shapes):
g = [dev_grads[dev][var_idx][0] for dev in devices]
if np.prod(
grad_shape
): # nccl does not support zero-sized tensors
g = all_sum(g)
for dev, gg in zip(devices, g):
dev_grads[dev][var_idx] = (
gg, dev_grads[dev][var_idx][1])
# Apply updates separately on each device.
for dev_idx, (dev, grads) in enumerate(dev_grads.items()):
with tf.name_scope('ApplyGrads%d' % dev_idx), tf.device(dev):
# Scale gradients as needed.
if self.use_loss_scaling or total_grads > 1:
with tf.name_scope('Scale'):
coef = tf.constant(np.float32(1.0 / total_grads),
name='coef')
coef = self.undo_loss_scaling(coef)
grads = [(g * coef, v) for g, v in grads]
# Check for overflows.
with tf.name_scope('CheckOverflow'):
grad_ok = tf.reduce_all(
tf.stack([
tf.reduce_all(tf.is_finite(g))
for g, v in grads
]))
# Update weights and adjust loss scaling.
with tf.name_scope('UpdateWeights'):
opt = self._dev_opt[dev]
ls_var = self.get_loss_scaling_var(dev)
if not self.use_loss_scaling:
ops.append(
tf.cond(grad_ok,
lambda: opt.apply_gradients(grads),
tf.no_op))
else:
ops.append(
tf.cond(
grad_ok, lambda: tf.group(
tf.assign_add(ls_var, self.
loss_scaling_inc),
opt.apply_gradients(grads)),
lambda: tf.group(
tf.assign_sub(ls_var, self.
loss_scaling_dec))))
# Report statistics on the last device.
if dev == devices[-1]:
with tf.name_scope('Statistics'):
ops.append(
autosummary(self.id + '/learning_rate',
self.learning_rate))
ops.append(
autosummary(self.id + '/overflow_frequency',
tf.where(grad_ok, 0, 1)))
if self.use_loss_scaling:
ops.append(
autosummary(self.id + '/loss_scaling_log2',
ls_var))
# Initialize variables and group everything into a single op.
self.reset_optimizer_state()
init_uninited_vars(list(self._dev_ls_var.values()))
return tf.group(*ops, name='TrainingOp')
