def _preprocess_fn(
self, features,
labels,
mode
):
"""Resize images and convert them from uint8 -> float32."""
if 'image' in features:
ndim = len(features.image.shape)
is_sequence = (ndim > 4)
input_size = self._src_img_res
target_size = self._crop_size
features.original_image = features.image
features.image = distortion.preprocess_image(features.image, mode,
is_sequence, input_size,
target_size)
features.image = tf.image.convert_image_dtype(features.image, tf.float32)
out_feature_spec = self.get_out_feature_specification(mode)
if out_feature_spec.image.shape != features.image.shape:
features.image = meta_tfdata.multi_batch_apply(
tf.image.resize_images, 2, features.image,
out_feature_spec.image.shape.as_list()[-3:-1])
if self._mixup_alpha > 0. and labels and mode == TRAIN:
lmbda = tfp.distributions.Beta(
self._mixup_alpha, self._mixup_alpha).sample()
for key, x in features.items():
if x.dtype in FLOAT_DTYPES:
features[key] = lmbda * x + (1-lmbda)*tf.reverse(x, axis=[0])
if labels is not None:
for key, x in labels.items():
if x.dtype in FLOAT_DTYPES:
labels[key] = lmbda * x + (1 - lmbda) * tf.reverse(x, axis=[0])
return features, labels
def reconstruct(self, inputs, samples=1, sample_static=False,
sample_dynamic=False, swap_static=False, swap_dynamic=False,
fix_static=False, fix_dynamic=False):
"""Reconstruct the given input sequences.
Args:
inputs: A batch of image sequences `x_{1:T}` of shape
`[batch_size, timesteps, height, width, channels]`.
samples: Number of samples to draw from the latent distributions.
sample_static: Boolean for whether or not to randomly sample the
static latent variable `f` from its prior distribution.
sample_dynamic: Boolean for whether or not to randomly sample the
dynamic latent variable `z_{1:T}` from its prior distribution.
swap_static: Boolean for whether or not to swap the encodings for
the static latent variable `f` between the examples.
swap_dynamic: Boolean for whether or not to swap the encodings for
the dynamic latent variable `z_{1:T}` between the examples.
fix_static: Boolean for whether or not to share the same random
sample of the static latent variable `f` from its prior across
all examples.
fix_dynamic: Boolean for whether or not to share the same random
sample of the dynamic latent variable `z_{1:T}` from its prior
across all examples.
Returns:
A batched Independent distribution wrapping a set of Normal
distributions over the pixels of the reconstruction of the input,
where the Independent distribution has event shape [height, width,
channels], batch shape [samples, batch_size, timesteps], and
sample shape [sample_shape, samples, batch_size, timesteps,
height, width, channels].
"""
batch_size = tf.shape(input=inputs)[-5]
length = len(tf.unstack(inputs, axis=-4)) # hack for graph mode
features = self.compressor(inputs) # (..., batch, timesteps, hidden)
if sample_static:
static_sample, _ = self.sample_static_prior(
samples, batch_size, fix_static)
else:
static_sample, _ = self.sample_static_posterior(features, samples)
if swap_static:
static_sample = tf.reverse(static_sample, axis=[1])
if sample_dynamic:
dynamic_sample, _ = self.sample_dynamic_prior(
samples, batch_size, length, fix_dynamic)
else:
dynamic_sample, _ = self.sample_dynamic_posterior(
features, samples, static_sample)
if swap_dynamic:
dynamic_sample = tf.reverse(dynamic_sample, axis=[1])
likelihood = self.decoder((dynamic_sample, static_sample))
return likelihood
def discounted_return(reward, length, discount):
"""Discounted Monte-Carlo returns."""
timestep = tf.range(reward.shape[1].value)
mask = tf.cast(timestep[None, :] < length[:, None], tf.float32)
return_ = tf.reverse(
tf.transpose(
tf.scan(lambda agg, cur: cur + discount * agg,
tf.transpose(tf.reverse(mask * reward, [1]), [1, 0]),
tf.zeros_like(reward[:, -1]), 1, False), [1, 0]), [1])
return tf.check_numerics(tf.stop_gradient(return_), 'return')
def lambda_advantage(reward, value, length, discount):
"""Generalized Advantage Estimation."""
timestep = tf.range(reward.shape[1].value)
mask = tf.cast(timestep[None, :] < length[:, None], tf.float32)
next_value = tf.concat([value[:, 1:], tf.zeros_like(value[:, -1:])], 1)
delta = reward + discount * next_value - value
advantage = tf.reverse(
tf.transpose(
tf.scan(lambda agg, cur: cur + discount * agg,
tf.transpose(tf.reverse(mask * delta, [1]), [1, 0]),
tf.zeros_like(delta[:, -1]), 1, False), [1, 0]), [1])
return tf.check_numerics(tf.stop_gradient(advantage), 'advantage')
def equirectangular_padding(images, num_paddings):
"""Pad equirectangular panorama images.
Args:
images: a 4-D tensor of shape `[BATCH, HEIGHT, WIDTH, CHANNELS]`.
num_paddings: a 2x2 integer list [[n_top, n_bottom], [n_left, n_right]]
representing the number of rows or columns to pad around the images.
Returns:
a 4-D tensor representing padded images with a shape of
`[BATCH, n_top+HEIGHT+n_bottom, n_left+WIDTH+n_right, CHANNELS]`.
Raises:
ValueError: 'images' has the wrong dimensions.
The number of paddings exceeds the height or width dimension.
"""
with tf.name_scope(None, 'equirectangular_padding',
[images, num_paddings]):
if len(images.shape) != 4:
raise ValueError("'images' has the wrong dimensions.")
shape = images.shape.as_list()
height, width = shape[1], shape[2]
top, down = num_paddings[0][0], num_paddings[0][1]
left, right = num_paddings[1][0], num_paddings[1][1]
if top > height or down > height:
raise ValueError(
'The number of paddings exceeds the height dimension.')
if left > width or right > width:
raise ValueError(
'The number of paddings exceeds the width dimension.')
semicircle = tf.cast(width / 2, tf.int32)
# The padded rows should be symmetric to 'images', but they should be
# shifted by 180 degrees. Copy the rightmost column (2*pi-w) as the padded
# colomn on the left and copy the leftmost column (0+w) as the padded colomn
# on the right.
top_padding = tf.reverse(tf.roll(images[:, :top, :, :],
axis=2,
shift=semicircle),
axis=[1])
bottom_padding = tf.roll(tf.reverse(images, axis=[1])[:, :down, :, :],
axis=2,
shift=semicircle)
padded_images = tf.concat([top_padding, images, bottom_padding], 1)
left_padding = tf.reverse(tf.reverse(padded_images,
axis=[2])[:, :, :left, :],
axis=[2])
right_padding = padded_images[:, :, :right, :]
padded_images = tf.concat([left_padding, padded_images, right_padding],
2)
return padded_images
def lambda_return(reward, value, length, discount, lambda_):
"""TD-lambda returns."""
timestep = tf.range(reward.shape[1].value)
mask = tf.cast(timestep[None, :] < length[:, None], tf.float32)
sequence = mask * reward + discount * value * (1 - lambda_)
discount = mask * discount * lambda_
sequence = tf.stack([sequence, discount], 2)
return_ = tf.reverse(
tf.transpose(
tf.scan(lambda agg, cur: cur[0] + cur[1] * agg,
tf.transpose(tf.reverse(sequence, [1]), [1, 2, 0]),
tf.zeros_like(value[:, -1]), 1, False), [1, 0]), [1])
return tf.check_numerics(tf.stop_gradient(return_), 'return')
def rotate(tensor):
# flipLeftRight
tensor = tf.where(
tf.bitwise.bitwise_and(sym, 1) > 0, tf.reverse(tensor, axis=[0]),
tensor)
# flipUpDown
tensor = tf.where(
tf.bitwise.bitwise_and(sym, 2) > 0, tf.reverse(tensor, axis=[1]),
tensor)
# flipDiagonal
tensor = tf.where(
tf.bitwise.bitwise_and(sym, 4) > 0,
tf.transpose(tensor, perm=[1, 0, 2]), tensor)
return tensor
def body(self, features):
"""Build the main body of the model.
Args:
features: A dict of "inputs" and "targets" which have already been passed
through an embedding layer. Inputs should have shape
[batch_size, max_seq_length, 1, embedding_size]. Targets should have
shape [batch_size, max_seq_length, 1, 1]
Returns:
The logits which get passed to the top of the model for inference.
A tensor of shape [batch_size, seq_length, 1, embedding_size]
"""
inputs = features.get("inputs")
targets = features["targets"]
if inputs is not None:
inputs = common_layers.flatten4d3d(inputs)
_, final_encoder_state = self._rnn(tf.reverse(inputs, axis=[1]),
"encoder")
else:
final_encoder_state = None
shifted_targets = common_layers.shift_right(targets)
decoder_outputs, _ = self._rnn(
common_layers.flatten4d3d(shifted_targets),
"decoder",
initial_state=final_encoder_state)
return decoder_outputs
def _conv_2d(X, h, strides=[1, 1, 1, 1]):
"""
Perform 2d convolution in tensorflow.
X will to be manipulated to be of shape [batch, height, width, ch],
and h to be of shape [height, width, ch, num]. This function does the
necessary reshaping before calling the conv2d function, and does the
reshaping on the output, returning Y of shape [batch, height, width]
"""
# Check the shape of X is what we expect
if len(X.shape) != 3:
raise ValueError('X needs to be of shape [batch, height, width] ' +
'for conv_2d')
# Check the shape of h is what we expect
if len(h.shape) != 2:
raise ValueError('Filter inputs must only have height and width ' +
'for conv_2d')
# Add in the unit dimensions for conv
X = tf.expand_dims(X, axis=-1)
h = tf.expand_dims(tf.expand_dims(h, axis=-1), axis=-1)
# Have to reverse h as tensorflow 2d conv is actually cross-correlation
h = tf.reverse(h, axis=[0, 1])
Y = tf.nn.conv2d(X, h, strides=strides, padding='VALID')
# Remove the final dimension, returning a result of shape
# [batch, height, width]
Y = tf.squeeze(Y, axis=-1)
return Y
def build_graph(parameters):
input_tensor = tf.compat.v1.placeholder(
dtype=parameters["dtype"],
name=("input"),
shape=parameters["base_shape"])
outs = tf.reverse(input_tensor, axis=[get_valid_axis(parameters)])
return [input_tensor], [outs]
def process_reals(x, lod, mirror_augment, drange_data, drange_net):
with tf.name_scope('ProcessReals'):
with tf.name_scope('DynamicRange'):
x = tf.cast(x, tf.float32)
x = misc.adjust_dynamic_range(x, drange_data, drange_net)
if mirror_augment:
with tf.name_scope('MirrorAugment'):
s = tf.shape(x)
mask = tf.random_uniform([s[0], 1, 1, 1], 0.0, 1.0)
mask = tf.tile(mask, [1, s[1], s[2], s[3]])
x = tf.where(mask < 0.5, x, tf.reverse(x, axis=[3]))
with tf.name_scope(
'FadeLOD'
): # Smooth crossfade between consecutive levels-of-detail.
s = tf.shape(x)
y = tf.reshape(x, [-1, s[1], s[2] // 2, 2, s[3] // 2, 2])
y = tf.reduce_mean(y, axis=[3, 5], keepdims=True)
y = tf.tile(y, [1, 1, 1, 2, 1, 2])
y = tf.reshape(y, [-1, s[1], s[2], s[3]])
x = tfutil.lerp(x, y, lod - tf.floor(lod))
with tf.name_scope(
'UpscaleLOD'
): # Upscale to match the expected input/output size of the networks.
s = tf.shape(x)
factor = tf.cast(2**tf.floor(lod), tf.int32)
x = tf.reshape(x, [-1, s[1], s[2], 1, s[3], 1])
x = tf.tile(x, [1, 1, 1, factor, 1, factor])
x = tf.reshape(x, [-1, s[1], s[2] * factor, s[3] * factor])
return x
def rectilinear_projection(images, resolution, fov, rotations):
"""Convert equirectangular panoramic images to perspective images.
First, the panorama images are rotated by the input parameter "rotations".
Then, the region with the field of view "fov" centered at camera's look-at -Z
axis is projected into perspective images. The -Z axis corresponds to the
spherical coordinates (pi/2, pi/2) which is (HEIGHT/2, WIDTH/4) on the pano.
Args:
images: a 4-D tensor of shape `[BATCH, HEIGHT, WIDTH, CHANNELS]`.
resolution: a 2-D tuple or list containing the resolution of desired output.
fov: (float) camera's horizontal field of view in degrees.
rotations: [BATCH, 3, 3] rotation matrices.
Returns:
4-D tensor of shape `[BATCH, HEIGHT, WIDTH, CHANNELS]`
Raises:
ValueError: 'images' has the wrong dimensions.
ValueError: 'images' is not a float tensor.
ValueError: 'rotations' has the wrong dimensions.
"""
with tf.name_scope(None, 'rectilinear_projection',
[images, resolution, fov, rotations]):
if len(images.shape) != 4:
raise ValueError("'images' has the wrong dimensions.")
if images.dtype != tf.float32 and images.dtype != tf.float64:
raise ValueError("'images' must be a float tensor.")
if rotations.shape[-2:] != [3, 3]:
raise ValueError("'rotations' has the wrong dimensions.")
shape = images.shape.as_list()
batch = shape[0]
cartesian_coordinates = geometry.generate_cartesian_grid(
resolution, fov)
# create batch -> [batch, height, width, 3]
cartesian_coordinates = tf.tile(
tf.expand_dims(cartesian_coordinates, axis=0), [batch, 1, 1, 1])
# The rotation matrices have to be [batch, height, width, 3, 3].
flip_x = tf.constant([[-1., 0., 0.], [0., 1., 0.], [0., 0., 1.]])
rotations = tf.matmul(flip_x,
tf.matmul(rotations, flip_x, transpose_a=True))
rotated_coordinates = tf.matmul(rotations[:, tf.newaxis, tf.newaxis],
tf.expand_dims(cartesian_coordinates,
-1),
transpose_a=True)
axis_convert = tf.constant([[0., 0., 1.], [1., 0., 0.], [0., 1., 0.]])
rotated_coordinates = tf.matmul(axis_convert, rotated_coordinates)
rotated_coordinates = tf.squeeze(rotated_coordinates, -1)
spherical_coordinates = geometry.cartesian_to_spherical(
rotated_coordinates)
# The azimuth of 'spherical_coordinates' decreases from left to right but
# the x should increase from left to right.
spherical_coordinates = tf.reverse(spherical_coordinates, [2])
return equirectangular_sampler(images, spherical_coordinates)
def reverse(self):
"""Returns the reverse of the sequence."""
return ViewSequence(self.scene_id, self.sequence_id,
tf.reverse(self.timestamp, [0]),
tf.reverse(self.rgb, [0]),
tf.reverse(self.pano, [0]),
tf.reverse(self.depth, [0]),
tf.reverse(self.normal, [0]),
tf.reverse(self.pose, [0]),
tf.reverse(self.intrinsics, [0]),
tf.reverse(self.resolution, [0]))
def tf_image_flip_ud(im1, im2, flo, mode):
distort_up_down_random = tf.random.uniform([1], 0, 1.0,
dtype=tf.float32)[0]
flip = tf.less(distort_up_down_random, 0.5)
flip_mask = tf.stack([flip, False, False])
flip_axis = tf.boolean_mask([0, 1, 2], flip_mask)
im1 = tf.reverse(im1, flip_axis)
im2 = tf.reverse(im2, flip_axis)
flo = tf.reverse(flo, flip_axis)
if flip:
v = flo[:, :, 1:] * -1
else:
v = flo[:, :, 1:]
u = flo[:, :, :1]
flo = tf.concat([u, v], axis=2)
return im1, im2, flo, mode
def _flip_masks_left_right(masks):
"""Left-right flip masks.
Args:
masks: rank 3 float32 tensor with shape
[num_instances, height, width] representing instance masks.
Returns:
flipped masks: rank 3 float32 tensor with shape
[num_instances, height, width] representing instance masks.
"""
return tf.reverse(masks, axis=[2])
def calculate_generalized_advantage_estimator(
reward, value, done, gae_gamma, gae_lambda):
# pylint: disable=g-doc-args
"""Generalized advantage estimator.
Returns:
GAE estimator. It will be one element shorter than the input; this is
because to compute GAE for [0, ..., N-1] one needs V for [1, ..., N].
"""
# pylint: enable=g-doc-args
next_value = value[1:, :]
next_not_done = 1 - tf.cast(done[1:, :], tf.float32)
delta = (reward[:-1, :] + gae_gamma * next_value * next_not_done
- value[:-1, :])
return_ = tf.reverse(tf.scan(
lambda agg, cur: cur[0] + cur[1] * gae_gamma * gae_lambda * agg,
[tf.reverse(delta, [0]), tf.reverse(next_not_done, [0])],
tf.zeros_like(delta[0, :]),
parallel_iterations=1), [0])
return tf.check_numerics(return_, "return")
def _compute_boxes_using_masks(self, masks, image_shape, image_info,
image_scale, offset):
"""Computes boundig boxes using masks."""
masks = tf.cast(masks, tf.int8)
x = tf.reduce_max(masks, axis=1)
xmin = tf.cast(tf.argmax(x, 1), tf.int16)
xmax = tf.cast(image_shape[1], tf.int16) - tf.cast(
tf.argmax(tf.reverse(x, [1]), 1), tf.int16)
y = tf.reduce_max(masks, axis=2)
ymin = tf.cast(tf.argmax(y, 1), tf.int16)
ymax = tf.cast(image_shape[0], tf.int16) - tf.cast(
tf.argmax(tf.reverse(y, [1]), 1), tf.int16)
bbox = tf.stack([ymin, xmin, ymax, xmax], -1)
# Clips boxes.
bbox = tf.cast(bbox, tf.float32)
bbox = input_utils.resize_and_crop_boxes(bbox, image_scale,
image_info[1, :], offset)
bbox += tf.tile(tf.expand_dims(offset, axis=0), [1, 2])
bbox /= tf.tile(tf.expand_dims(image_scale, axis=0), [1, 2])
return bbox
def mixup(batch_size, alpha, same_mix_weight_per_batch, use_truncated_beta,
use_random_shuffling, images, labels):
"""Applies Mixup regularization to a batch of images and labels.
[1] Hongyi Zhang, Moustapha Cisse, Yann N. Dauphin, David Lopez-Paz
Mixup: Beyond Empirical Risk Minimization.
ICLR'18, https://arxiv.org/abs/1710.09412
Arguments:
batch_size: The input batch size for images and labels.
alpha: Float that controls the strength of Mixup regularization.
same_mix_weight_per_batch: whether to use the same mix coef over the batch.
use_truncated_beta: whether to sample from Beta_[0,1](alpha, alpha) or from
the truncated distribution Beta_[1/2, 1](alpha, alpha).
use_random_shuffling: Whether to pair images by random shuffling
(default is a deterministic pairing by reversing the batch).
images: A batch of images of shape [batch_size, ...]
labels: A batch of labels of shape [batch_size, num_classes]
Returns:
A tuple of (images, labels) with the same dimensions as the input with
Mixup regularization applied.
"""
if same_mix_weight_per_batch:
mix_weight = ed.Beta(alpha, alpha, sample_shape=[1, 1])
mix_weight = tf.tile(mix_weight, [batch_size, 1])
else:
mix_weight = ed.Beta(alpha, alpha, sample_shape=[batch_size, 1])
if use_truncated_beta:
mix_weight = tf.maximum(mix_weight, 1. - mix_weight)
images_mix_weight = tf.reshape(mix_weight, [batch_size, 1, 1, 1])
images_mix_weight = tf.cast(images_mix_weight, images.dtype)
if not use_random_shuffling:
# Mixup on a single batch is implemented by taking a weighted sum with the
# same batch in reverse.
mixup_index = tf.reverse(tf.range(batch_size), axis=[0])
else:
mixup_index = tf.random.shuffle(tf.range(batch_size))
images_mix = (images * images_mix_weight + tf.gather(images, mixup_index) *
(1. - images_mix_weight))
mix_weight = tf.cast(mix_weight, labels.dtype)
labels_mix = labels * mix_weight + tf.gather(
labels, mixup_index) * (1. - mix_weight)
return images_mix, labels_mix
def calculate_ngram_weight(unstacked_tensor, batch_size, embed_size):
stacked_tensor = tf.stack(unstacked_tensor, axis=1)
stacked_tensor = tf.reverse(stacked_tensor, [1])
index = tf.constant(len(unstacked_tensor))
expected_result = tf.zeros([batch_size, embed_size])
def condition(index, summation):
return tf.greater(index, 0)
def body(index, summation):
precessed = tf.slice(stacked_tensor, [0, index - 1, 0], [-1, -1, -1])
summand = tf.reduce_mean(precessed, 1)
return tf.subtract(index, 1), tf.add(summation, summand)
result = tf.while_loop(condition, body, [index, expected_result])
return result[1]
def demosaic(bayer_images): # FIXME: tf.image.flip_... -> tf.reverse
"""Bilinearly demosaics a batch of RGGB Bayer images."""
bayer_images.shape.assert_is_compatible_with((None, None, None, 4))
# This implementation exploits how edges are aligned when upsampling with
# tf.image.resize_bilinear().
with tf.name_scope(None, 'demosaic'):
shape = tf.shape(bayer_images)
shape = [shape[1] * 2, shape[2] * 2]
red = bayer_images[Ellipsis, 0:1]
red = tf.image.resize_bilinear(red, shape)
green_red = bayer_images[Ellipsis, 1:2]
green_red = tf.reverse(green_red, axis=[2])
green_red = tf.image.resize_bilinear(green_red, shape)
green_red = tf.reverse(green_red, axis=[2])
green_red = tf.space_to_depth(green_red, 2)
green_blue = bayer_images[Ellipsis, 2:3]
green_blue = tf.reverse(green_blue, axis=[1])
green_blue = tf.image.resize_bilinear(green_blue, shape)
green_blue = tf.reverse(green_blue, axis=[1])
green_blue = tf.space_to_depth(green_blue, 2)
green_at_red = (green_red[Ellipsis, 0] + green_blue[Ellipsis, 0]) / 2
green_at_green_red = green_red[Ellipsis, 1]
green_at_green_blue = green_blue[Ellipsis, 2]
green_at_blue = (green_red[Ellipsis, 3] + green_blue[Ellipsis, 3]) / 2
green_planes = [
green_at_red, green_at_green_red, green_at_green_blue,
green_at_blue
]
green = tf.depth_to_space(tf.stack(green_planes, axis=-1), 2)
blue = bayer_images[Ellipsis, 3:4]
blue = tf.reverse(tf.reverse(blue, axis=[2]), axis=[1])
blue = tf.image.resize_bilinear(blue, shape)
blue = tf.reverse(tf.reverse(blue, axis=[2]), axis=[1])
rgb_images = tf.concat([red, green, blue], axis=-1)
return rgb_images
def _reverse(self, t, lengths):
"""Time reverse the provided tensor or list of tensors.
Assumes the top dimension is the time dimension.
Args:
t: 3D tensor or list of 2D tensors to be reversed
lengths: 1D tensor of lengths, or `None`
Returns:
A reversed tensor or list of tensors
"""
if isinstance(t, list):
return list(reversed(t))
else:
if lengths is None:
return tf.reverse(t, [0])
else:
return tf.reverse_sequence(t, lengths, 0, 1)
def fix_variables(self, sess, pretrained_model):
print('Fix VGG16 layers..')
with tf.variable_scope('Fix_VGG16'):
with tf.device("/cpu:0"):
# fix the vgg16 issue from conv weights to fc weights
# fix RGB to BGR
fc6_conv = tf.get_variable("fc6_conv", [7, 7, 512, 4096],
trainable=False)
fc7_conv = tf.get_variable("fc7_conv", [1, 1, 4096, 4096],
trainable=False)
conv1_rgb = tf.get_variable("conv1_rgb", [3, 3, 3, 64],
trainable=False)
restorer_fc = tf.train.Saver({
"vgg_16/fc6/weights":
fc6_conv,
"vgg_16/fc7/weights":
fc7_conv,
"vgg_16/conv1/conv1_1/weights":
conv1_rgb
})
restorer_fc.restore(sess, pretrained_model)
sess.run(
tf.assign(
self._variables_to_fix['vgg_16/fc6/weights:0'],
tf.reshape(
fc6_conv,
self._variables_to_fix['vgg_16/fc6/weights:0'].
get_shape())))
sess.run(
tf.assign(
self._variables_to_fix['vgg_16/fc7/weights:0'],
tf.reshape(
fc7_conv,
self._variables_to_fix['vgg_16/fc7/weights:0'].
get_shape())))
sess.run(
tf.assign(
self.
_variables_to_fix['vgg_16/conv1/conv1_1/weights:0'],
tf.reverse(conv1_rgb, [2])))
def _reverse_seq(sequence, sequence_lengths=None):
"""Reverse sequence along dim 0.
Args:
sequence: Tensor of shape [T, B, ...].
sequence_lengths: (optional) tensor of shape [B]. If `None`, only reverse
along dim 0.
Returns:
Tensor of same shape as sequence with dim 0 reversed up to sequence_lengths.
"""
if sequence_lengths is None:
return tf.reverse(sequence, [0])
sequence_lengths = tf.convert_to_tensor(sequence_lengths)
with tf.control_dependencies(
[tf.assert_equal(sequence.shape[1], sequence_lengths.shape[0])]):
return tf.reverse_sequence(sequence,
sequence_lengths,
seq_axis=0,
batch_axis=1)
def subsample_positive():
# Shuffle the foreground indices
disable_fg_inds = tf.random_shuffle(fg_inds, seed=self._seed)
# Select the indices that we have to ignore, this is
# `tf.shape(fg_inds)[0] - num_fg` because we want to get only
# `num_fg` foreground labels.
disable_place = (tf.shape(fg_inds)[0] - num_fg)
disable_fg_inds = disable_fg_inds[:disable_place]
# Order the indices for sparse_to_dense compatibility
disable_fg_inds, _ = tf.nn.top_k(disable_fg_inds,
k=tf.shape(disable_fg_inds)[-1])
disable_fg_inds = tf.reverse(disable_fg_inds, [0])
disable_fg_inds = tf.sparse_to_dense(disable_fg_inds,
tf.shape(labels,
out_type=tf.int64),
True,
default_value=False)
# Put -1 to ignore the anchors in the selected indices
return tf.where(condition=tf.squeeze(disable_fg_inds),
x=tf.to_float(tf.fill(tf.shape(labels), -1)),
y=labels)
def _propagate_solution(self, args):
"""Perform least squares to get A- and B-matrices and propagate forward
Args:
args: Various arguments and specifications
"""
# Define X- and Y-matrices
X = self.g_vals[:, :-1]
Y = self.g_vals[:, 1:]
# Solve for A and B using least-squares
self.K = tf.matrix_solve_ls(X, Y, l2_regularizer=args.l2_regularizer)
self.A = self.K[:, :args.latent_dim]
self.B = self.K[:, args.latent_dim:]
# Perform least squares to find A-inverse
self.A_inv = tf.matrix_solve_ls(Y,
self.g_vals[:, :-1],
l2_regularizer=args.l2_regularizer)
# Get predicted code at final time step
self.z_t = self.g_vals[:, -1]
# Create recursive predictions for z
z_t = tf.expand_dims(self.z_t, axis=1)
z_vals = [z_t]
for t in range(args.seq_length - 2, -1, -1):
z_t = tf.matmul(z_t, self.A_inv)
z_vals.append(z_t)
self.z_vals_reshape = tf.stack(z_vals, axis=1)
# Flip order
self.z_vals_reshape = tf.squeeze(tf.reverse(self.z_vals_reshape, [1]))
# Reshape predicted z-values
self.z_vals = tf.reshape(
self.z_vals_reshape,
[args.batch_size * args.seq_length, args.latent_dim])
def test_helium_atom(self):
x1 = np.linspace(-1, 1, 6, dtype=np.float32)
x2 = np.zeros(6, dtype=np.float32) + 0.25
x12 = np.array([x1] + [x2 for _ in range(5)]).T
atoms = [system.Atom(symbol='He', coords=(0, 0, 0))]
hpsi_test = np.array([
-0.25170374, -0.43359983, -0.61270618, -1.56245542, -0.39977553,
-0.2300467
])
log_helium = _wfn_to_log(_helium)
h = hamiltonian.exact_hamiltonian(atoms, 2)
xs = tf.constant(x12)
_, hpsi = h(log_helium, xs)
with tf.train.MonitoredSession() as session:
hpsi_ = session.run(hpsi)
xs = tf.reverse(xs, axis=[1])
_, hpsi2 = h(log_helium, xs)
with tf.train.MonitoredSession() as session:
hpsi2_ = session.run(hpsi2)
np.testing.assert_allclose(hpsi_[:, 0], hpsi_test, rtol=1.e-6)
np.testing.assert_allclose(hpsi_, hpsi2_, rtol=1.e-6)
def flip_image(image, kp, pose=None, gt3d=None):
"""
Flipping image and kp.
kp is 3 x N!
pose is 72D
gt3d is 14 x 3
"""
image = tf.reverse(image, [1])
new_kp = kp
new_x = tf.cast(tf.shape(image)[0], dtype=kp.dtype) - kp[0, :] - 1
new_kp = tf.concat([tf.expand_dims(new_x, 0), kp[1:, :]], 0)
# Swap left and right limbs by gathering them in the right order
# For COCO+
swap_inds = tf.constant(
[5, 4, 3, 2, 1, 0, 11, 10, 9, 8, 7, 6, 12, 13, 14, 16, 15, 18, 17])
new_kp = tf.transpose(tf.gather(tf.transpose(new_kp), swap_inds))
if pose is not None:
new_pose = reflect_pose(pose)
new_gt3d = reflect_joints3d(gt3d)
return image, new_kp, new_pose, new_gt3d
else:
return image, new_kp
def project(v, tau=1):
p = tf.reduce_sum(v, 2)
p = tf.ones_like(p) - tf.exp(-p*tau)
return tf.reverse(tf.transpose(p), [True, False])
def build_bert_inputs(example):
"""Convert example <Tensor [30, 70]> into bert inputs."""
k_size = FLAGS.k_size
CLS_ID = tf.constant([101], dtype=tf.int64) # pylint: disable=invalid-name
SEP_ID = tf.constant([102], dtype=tf.int64) # pylint: disable=invalid-name
max_len = tf.constant([FLAGS.max_para_length])
context_size = tf.constant([FLAGS.context_size])
intermediate_examples_tensor = tf.reduce_sum(tf.abs(example), 1)
examples_zero_vector = tf.zeros(shape=(1, 1), dtype=tf.int64)
examples_bool_mask = tf.squeeze(
tf.not_equal(intermediate_examples_tensor, examples_zero_vector))
paragraph_len = tf.reduce_sum(tf.cast(examples_bool_mask, tf.int32))
start = tf.random.uniform([1],
0,
tf.reshape(paragraph_len, []) -
tf.reshape(context_size, []) + 1,
dtype=tf.int32)
# Slice the document into the before, after and context.
# Discard the zero padding.
sizes = tf.squeeze(
tf.concat([[
start, context_size, paragraph_len - context_size - start,
max_len - paragraph_len
]], 0))
before, context, after, _ = tf.split(example, sizes, axis=0)
# Gather the context removing zero padding at end of sentences.
non_zeros = tf.where(tf.not_equal(context, tf.zeros_like(context)))
context_gathered = tf.gather_nd(context, non_zeros)
# Flip before so we select the 4 sentences closest to target
before = tf.reverse(before, axis=[0])
# pad both to longer than needed
paddings = tf.constant([[0, 8], [0, 0]])
before = tf.pad(before, paddings)
after = tf.pad(after, paddings)
# Extend targets to 3 sentences
# pad both
before_minus_one = before[1:][:k_size]
before_minus_two = before[2:][:k_size]
after_plus_one = after[1:][:k_size]
after_plus_two = after[2:][:k_size]
before = before[:k_size]
after = after[:k_size]
before = tf.concat([before_minus_two, before_minus_one, before], axis=1)
after = tf.concat([after, after_plus_one, after_plus_two], axis=1)
############################################################################
# These 8 sentences are the 8 surrounding targets. Some are padding.
targets = tf.concat([before, after], axis=0)
# Remove the padding from the sourrounding sentences
# Eg. if context starts at beginning of paragraph, before is all padding
intermediate_tensor = tf.reduce_sum(tf.abs(targets), 1)
zero_vector = tf.zeros(shape=(1, 1), dtype=tf.int64)
bool_mask = tf.squeeze(tf.not_equal(intermediate_tensor, zero_vector))
bool_mask.set_shape([None])
targets = tf.boolean_mask(targets, bool_mask)
# Randomly select 4 targets
# We will also select the label_types for each selected target
indices = tf.range(0, limit=tf.shape(targets)[0], dtype=tf.int32)
shuffled_indices = tf.random.shuffle(indices)[:k_size]
targets = tf.gather(targets, shuffled_indices)
if k_size == 4:
full_labels = tf.concat([tf.range(3, -1, -1), tf.range(4, 8)], axis=0)
elif k_size == 3:
full_labels = tf.concat([tf.range(2, -1, -1), tf.range(3, 6)], axis=0)
elif k_size == 2:
full_labels = tf.concat([tf.range(1, -1, -1), tf.range(2, 4)], axis=0)
elif k_size == 1:
full_labels = tf.concat([tf.range(0, -1, -1), tf.range(1, 2)], axis=0)
label_types = tf.boolean_mask(full_labels, bool_mask)
label_types = tf.gather(label_types, shuffled_indices)
# create inputs
bert_inputs = []
input_masks = []
segment_ids = []
# make context
ctx_segment_id = tf.concat([
tf.zeros_like(CLS_ID, dtype=tf.int64),
tf.zeros_like(context_gathered),
tf.zeros_like(SEP_ID, dtype=tf.int64)
],
axis=0)
ctx_segment_id = pad_and_cut(ctx_segment_id, FLAGS.max_seq_length)
segment_ids.append(ctx_segment_id)
new_ctx_input = tf.concat([CLS_ID, context_gathered, SEP_ID], axis=0)
ctx_input_mask = tf.ones_like(new_ctx_input)
ctx_input_mask = pad_and_cut(ctx_input_mask, FLAGS.max_seq_length)
input_masks.append(ctx_input_mask)
padded_new_ctx_input = pad_and_cut(new_ctx_input, FLAGS.max_seq_length)
bert_inputs.append(padded_new_ctx_input)
for i in range(k_size):
target_non_zero = tf.where(
tf.not_equal(targets[i], tf.zeros_like(targets[i])))
targets_stripped = tf.gather_nd(targets[i], target_non_zero)
if FLAGS.include_context:
segment_id = tf.concat([
tf.zeros_like(CLS_ID, dtype=tf.int64),
tf.zeros_like(context_gathered),
tf.zeros_like(SEP_ID, dtype=tf.int64),
tf.ones_like(targets_stripped),
tf.ones_like(SEP_ID, dtype=tf.int64)
],
axis=0)
else:
segment_id = tf.concat([
tf.zeros_like(CLS_ID, dtype=tf.int64),
tf.zeros_like(targets_stripped),
tf.zeros_like(SEP_ID, dtype=tf.int64)
],
axis=0)
segment_id = pad_and_cut(segment_id, FLAGS.max_seq_length)
segment_ids.append(segment_id)
if FLAGS.include_context:
new_input = tf.concat(
[CLS_ID, context_gathered, SEP_ID, targets_stripped, SEP_ID], axis=0)
else:
new_input = tf.concat([CLS_ID, targets_stripped, SEP_ID], axis=0)
input_mask = tf.ones_like(new_input)
input_mask = pad_and_cut(input_mask, FLAGS.max_seq_length)
input_masks.append(input_mask)
padded_new_input = pad_and_cut(new_input, FLAGS.max_seq_length)
bert_inputs.append(padded_new_input)
bert_inputs = tf.stack(bert_inputs, axis=0)
input_masks = tf.stack(input_masks, axis=0)
segment_ids = tf.stack(segment_ids, axis=0)
out = Outputs_And_Context(bert_inputs, input_masks, segment_ids, label_types,
context_gathered)
return out
#Setting the hyperparameters
epochs = 75
batch_size = 64
rnn_size = 512
num_layers = 3
encoding_embedding_size = 512 #512 columns
decoding_embedding_size = 512
learning_rate = 0.01
learning_rate_decay = 0.90
min_learning_rate = 0.0001
keep_probability = 0.5 #drop 50 % of the nodes each time in the hidden layer
#Defining a session
tf.reset_default_graph()
session = tf.InteractiveSession()
#Loading the model inputs
inputs, targets, lr, keep_prob = model_inputs()
#Setting the Sequence length
sequence_length = tf.placeholder_with_default(25, None, name="sequence_length")
#Getting the shape of the input tensor
input_shape = tf.shape(inputs)
#Getting the test and training predictions
training_predictions, test_predictions = seq2seq_model(
tf.reverse(inputs, [-1]), targets, keep_prob, batch_size, sequence_length,
len(answersint2word), len(questionswords2int), encoding_embedding_size,
decoding_embedding_size, rnn_size, num_layers, questionswords2int)