def testErrorDataAndIndicesSizeMismatch(self):
indices = [tf.constant([0, 4, 7]),
tf.constant([1, 6, 2, 3, 5])]
data = [tf.constant([0, 40, 70]),
tf.constant([10, 60, 20, 30])]
with self.assertRaises(ValueError):
tf.dynamic_stitch(indices, data)
def testErrorDataDimSizeMismatch(self):
indices = [tf.constant([0, 4, 5]),
tf.constant([1, 6, 2, 3])]
data = [tf.constant([[0], [40], [70]]),
tf.constant([[10, 11], [60, 61], [20, 21], [30, 31]])]
with self.assertRaises(ValueError):
tf.dynamic_stitch(indices, data)
def testErrorIndicesMultiDimensional(self):
indices = [tf.constant([0, 4, 7]),
tf.constant([[1, 6, 2, 3, 5]])]
data = [tf.constant([[0, 40, 70]]),
tf.constant([10, 60, 20, 30, 50])]
with self.assertRaises(ValueError):
tf.dynamic_stitch(indices, data)
def loop(q_, mask, mass_, found_):
q_list = tf.dynamic_partition(q_, mask, 2)
condition_indices = tf.dynamic_partition(tf.range(tf.shape(q_)[0]), mask, 2) # 0 element it False,
# 1 element if true
p = q_list[1] * (1.0 - mass_) / tf.reduce_sum(q_list[1])
p_new = tf.dynamic_stitch(condition_indices, [q_list[0], p])
# condition verification and mask modification
less_mask = tf.cast(tf.less(u, p_new), tf.int32) # 0 when u is bigger than p, 1 when u is less than p
condition_indices = tf.dynamic_partition(tf.range(tf.shape(p_new)[0]), less_mask,
2) # 0 when u is bigger than p, 1 when u is less than p
split_p_new = tf.dynamic_partition(p_new, less_mask, 2)
split_u = tf.dynamic_partition(u, less_mask, 2)
alpha = tf.dynamic_stitch(condition_indices, [split_p_new[0], split_u[1]])
mass_ += tf.reduce_sum(split_u[1])
mask = mask * (tf.ones_like(less_mask) - less_mask)
found_ = tf.cond(tf.equal(tf.reduce_sum(less_mask), 0),
lambda: False,
lambda: True)
alpha = tf.reshape(alpha, q_.shape)
return alpha, mask, mass_, found_
def circular_convolution(v, k):
"""Computes circular convolution.
Args:
v: a 1-D `Tensor` (vector)
k: a 1-D `Tensor` (kernel)
"""
size = int(v.get_shape()[0])
kernel_size = int(k.get_shape()[0])
kernel_shift = int(math.floor(kernel_size/2.0))
def loop(idx):
if idx < 0: return size + idx
if idx >= size : return idx - size
else: return idx
kernels = []
for i in xrange(size):
indices = [loop(i+j) for j in xrange(kernel_shift, -kernel_shift-1, -1)]
v_ = tf.gather(v, indices)
kernels.append(tf.reduce_sum(v_ * k, 0))
# # code with double loop
# for i in xrange(size):
# for j in xrange(kernel_size):
# idx = i + kernel_shift - j + 1
# if idx < 0: idx = idx + size
# if idx >= size: idx = idx - size
# w = tf.gather(v, int(idx)) * tf.gather(kernel, j)
# output = tf.scatter_add(output, [i], tf.reshape(w, [1, -1]))
return tf.dynamic_stitch([i for i in xrange(size)], kernels)
def scheduled_sample_count(ground_truth_x,
generated_x,
batch_size,
scheduled_sample_var):
"""Sample batch with specified mix of groundtruth and generated data points.
Args:
ground_truth_x: tensor of ground-truth data points.
generated_x: tensor of generated data points.
batch_size: batch size
scheduled_sample_var: number of ground-truth examples to include in batch.
Returns:
New batch with num_ground_truth sampled from ground_truth_x and the rest
from generated_x.
"""
num_ground_truth = scheduled_sample_var
idx = tf.random_shuffle(tf.range(batch_size))
ground_truth_idx = tf.gather(idx, tf.range(num_ground_truth))
generated_idx = tf.gather(idx, tf.range(num_ground_truth, batch_size))
ground_truth_examps = tf.gather(ground_truth_x, ground_truth_idx)
generated_examps = tf.gather(generated_x, generated_idx)
output = tf.dynamic_stitch([ground_truth_idx, generated_idx],
[ground_truth_examps, generated_examps])
# if batch size is known set it.
if isinstance(batch_size, int):
output.set_shape([batch_size] + common_layers.shape_list(output)[1:])
return output
def _partition_and_stitch(self, args, func_name):
"""
args is a list of tensors, to be passed to self.likelihoods.<func_name>
args[-1] is the 'Y' argument, which contains the indexes to self.likelihoods.
This function splits up the args using dynamic_partition, calls the
relevant function on the likelihoods, and re-combines the result.
"""
# get the index from Y
Y = args[-1]
ind = Y[:, -1]
ind = tf.cast(ind, tf.int32)
Y = Y[:, :-1]
args[-1] = Y
# split up the arguments into chunks corresponding to the relevant likelihoods
args = zip(*[tf.dynamic_partition(X, ind, self.num_likelihoods) for X in args])
# apply the likelihood-function to each section of the data
with params_as_tensors_for(self, convert=False):
funcs = [getattr(lik, func_name) for lik in self.likelihood_list]
results = [f(*args_i) for f, args_i in zip(funcs, args)]
# stitch the results back together
partitions = tf.dynamic_partition(tf.range(0, tf.size(ind)), ind, self.num_likelihoods)
results = tf.dynamic_stitch(partitions, results)
return results
def indices_to_dense_vector(indices,
size,
indices_value=1.,
default_value=0,
dtype=tf.float32):
"""Creates dense vector with indices set to specific value and rest to zeros.
This function exists because it is unclear if it is safe to use
tf.sparse_to_dense(indices, [size], 1, validate_indices=False)
with indices which are not ordered.
This function accepts a dynamic size (e.g. tf.shape(tensor)[0])
Args:
indices: 1d Tensor with integer indices which are to be set to
indices_values.
size: scalar with size (integer) of output Tensor.
indices_value: values of elements specified by indices in the output vector
default_value: values of other elements in the output vector.
dtype: data type.
Returns:
dense 1D Tensor of shape [size] with indices set to indices_values and the
rest set to default_value.
"""
size = tf.to_int32(size)
zeros = tf.ones([size], dtype=dtype) * default_value
values = tf.ones_like(indices, dtype=dtype) * indices_value
return tf.dynamic_stitch([tf.range(size), tf.to_int32(indices)],
[zeros, values])
def testSumGradArgs(self):
with self.test_session(use_gpu=False):
indices = [tf.convert_to_tensor([0, 1, 2, 3]),
tf.convert_to_tensor([2, 3])]
values = [tf.convert_to_tensor([2, 3, 5, 7]), tf.convert_to_tensor([1, 1])]
self.assertAllEqual(
tf.dynamic_stitch(indices, values).eval(), [2, 3, 1, 1])
def _create_regression_targets(self, anchors, groundtruth_boxes, match):
"""Returns a regression target for each anchor.
Args:
anchors: a BoxList representing N anchors
groundtruth_boxes: a BoxList representing M groundtruth_boxes
match: a matcher.Match object
Returns:
reg_targets: a float32 tensor with shape [N, box_code_dimension]
"""
matched_anchor_indices = match.matched_column_indices()
unmatched_ignored_anchor_indices = (match.
unmatched_or_ignored_column_indices())
matched_gt_indices = match.matched_row_indices()
matched_anchors = box_list_ops.gather(anchors,
matched_anchor_indices)
matched_gt_boxes = box_list_ops.gather(groundtruth_boxes,
matched_gt_indices)
matched_reg_targets = self._box_coder.encode(matched_gt_boxes,
matched_anchors)
unmatched_ignored_reg_targets = tf.tile(
self._default_regression_target(),
tf.stack([tf.size(unmatched_ignored_anchor_indices), 1]))
reg_targets = tf.dynamic_stitch(
[matched_anchor_indices, unmatched_ignored_anchor_indices],
[matched_reg_targets, unmatched_ignored_reg_targets])
# TODO: summarize the number of matches on average.
return reg_targets
def _create_classification_targets(self, groundtruth_labels, match):
"""Create classification targets for each anchor.
Assign a classification target of for each anchor to the matching
groundtruth label that is provided by match. Anchors that are not matched
to anything are given the target self._unmatched_cls_target
Args:
groundtruth_labels: a tensor of shape [num_gt_boxes, d_1, ... d_k]
with labels for each of the ground_truth boxes. The subshape
[d_1, ... d_k] can be empty (corresponding to scalar labels).
match: a matcher.Match object that provides a matching between anchors
and groundtruth boxes.
Returns:
cls_targets: a float32 tensor with shape [num_anchors, d_1, d_2 ... d_k],
where the subshape [d_1, ..., d_k] is compatible with groundtruth_labels
which has shape [num_gt_boxes, d_1, d_2, ... d_k].
"""
matched_anchor_indices = match.matched_column_indices()
unmatched_ignored_anchor_indices = (match.
unmatched_or_ignored_column_indices())
matched_gt_indices = match.matched_row_indices()
matched_cls_targets = tf.gather(groundtruth_labels, matched_gt_indices)
ones = self._unmatched_cls_target.shape.ndims * [1]
unmatched_ignored_cls_targets = tf.tile(
tf.expand_dims(self._unmatched_cls_target, 0),
tf.stack([tf.size(unmatched_ignored_anchor_indices)] + ones))
cls_targets = tf.dynamic_stitch(
[matched_anchor_indices, unmatched_ignored_anchor_indices],
[matched_cls_targets, unmatched_ignored_cls_targets])
return cls_targets
def CircularConvolution(vector, kernel):
size = int(vector.get_shape()[0])
kernel_size = int(kernel.get_shape()[0])
kernel_shift = int(math.floor(kernel_size/2.0))
output = tf.zeros_like(vector)
def loop(idx):
if idx < 0: return size + idx
if idx >= size : return idx - size
else: return idx
kernels = []
for i in xrange(size):
indices = [loop(i+j) for j in xrange(kernel_shift, -kernel_shift-1, -1)]
v = tf.gather(vector, indices)
kernels.append(tf.reduce_sum(v * kernel, 0, keep_dims=True))
output = tf.dynamic_stitch([[i] for i in xrange(size)], kernels)
# # code with double loop
# for i in xrange(size):
# for j in xrange(kernel_size):
# idx = i + kernel_shift - j + 1
# if idx < 0: idx = idx + size
# if idx >= size: idx = idx - size
# w = tf.gather(vector, int(idx)) * tf.gather(kernel, j)
# output = tf.scatter_add(output, [i], tf.reshape(w, [1, -1]))
return output
def test_dynamic_stitch(sess):
x = tf.zeros((1, 3))
y = tf.dynamic_stitch([[0], [0]], [x, tf.ones((1, 3))])
z = tf.gather(y, [0])
with sess.as_default():
analytic, numeric = tf.test.compute_gradient(x, (1, 3), z, (1, 3))
assert np.allclose(analytic, numeric)
def testOneListOneDimensional(self):
with self.test_session():
indices = [tf.constant([1, 6, 2, 3, 5, 0, 4, 7])]
data = [tf.constant([10, 60, 20, 30, 50, 0, 40, 70])]
stitched_t = tf.dynamic_stitch(indices, data)
stitched_val = stitched_t.eval()
self.assertAllEqual([0, 10, 20, 30, 40, 50, 60, 70], stitched_val)
# Dimension 0 is determined by the max index in indices, so we
# can only infer that the output is a vector of some unknown
# length.
self.assertEqual([None], stitched_t.get_shape().as_list())
def testStitchOrder(self):
with self.test_session():
indices = []
np_values = []
values = []
for _ in range(10):
indices.extend([tf.convert_to_tensor(np.arange(100).astype(np.int32))])
np_values.extend([np.random.uniform(size=100)])
values.extend([tf.convert_to_tensor(np_values[-1])])
stitched = tf.dynamic_stitch(indices, values).eval()
self.assertAllEqual(np_values[-1], stitched)
def testScalar(self):
with self.test_session():
indices = [tf.constant(0), tf.constant(1)]
data = [tf.constant(40), tf.constant(60)]
for step in -1, 1:
stitched_t = tf.dynamic_stitch(indices[::step], data)
stitched_val = stitched_t.eval()
self.assertAllEqual([40, 60][::step], stitched_val)
# Dimension 0 is determined by the max index in indices, so we
# can only infer that the output is a vector of some unknown
# length.
self.assertEqual([None], stitched_t.get_shape().as_list())
def __call__(self, X):
ind = tf.gather(tf.transpose(X), tf.shape(X)[1]-1) # ind = X[:,-1]
ind = tf.cast(ind, tf.int32)
X = tf.transpose(tf.gather(tf.transpose(X), tf.range(0, tf.shape(X)[1]-1))) # X = X[:,:-1]
# split up X into chunks corresponding to the relevant likelihoods
x_list = tf.dynamic_partition(X, ind, len(self.meanfunction_list))
# apply the likelihood-function to each section of the data
results = [m(x) for x, m in zip(x_list, self.meanfunction_list)]
# stitch the results back together
partitions = tf.dynamic_partition(tf.range(0, tf.size(ind)), ind, len(self.meanfunction_list))
return tf.dynamic_stitch(partitions, results)
def split_apply_merge(inp, partitions, fns):
"""Split input according to partitions. Pass results through fns and merge.
Args:
inp: the input vector
partitions: tensor of same length as input vector, having values 0, 1
fns: the two functions.
Returns:
the vector routed, where routed[i] = fns[partitions[i]](inp[i])
"""
new_inputs = tf.dynamic_partition(inp, partitions, len(fns))
new_outputs = [fns[i](x) for i, x in enumerate(new_inputs)]
new_indices = tf.dynamic_partition(tf.range(0, inp.get_shape()[0]), partitions, len(fns))
return tf.dynamic_stitch(new_indices, new_outputs)
def replace_features(coarse_features, fine_features, replace_idxs):
""" Replace fine features with the corresponding coarse features
Trick.
use tf.dynamic_stitch ops
"""
# TODO: simplify indexing
def _convert_to_1d_idxs(src_idxs):
""" Convert 2D idxs to 1D idxs
within 1D tensor whose shape is (b*h*w*c)
"""
batch_idx_len = map_channel.value * map_width.value * map_height.value
batch_idx_base = [i*batch_idx_len for i in xrange(batch_size.value)]
batch_1d = map_channel.value * map_width.value * src_idxs[:,0] + \
map_channel.value * src_idxs[:,1]
batch_1d = tf.add(batch_1d,batch_idx_base)
flat_idxs = [batch_1d+i for i in xrange(map_channel.value)]
flat_idxs = tf.reshape(tf.transpose(tf.pack(flat_idxs)), [-1])
return flat_idxs
batch_size, map_height, map_width, map_channel = coarse_features.get_shape()
# flatten coarse features
flat_coarse_features = tf.reshape(coarse_features, [batch_size.value,-1])
flat_coarse_features = tf.reshape(flat_coarse_features, [-1])
# flatten fine features
flat_fine_features = [tf.reshape(i,[-1]) for i in fine_features]
flat_fine_features = tf.concat(0,flat_fine_features)
flat_fine_idxs = [_convert_to_1d_idxs(i) for i in replace_idxs]
flat_fine_idxs = tf.concat(0,flat_fine_idxs)
# extract coarse features to be replaced
# this is required for hint-based training
flat_coarse_replaced = tf.gather(flat_coarse_features, flat_fine_idxs, validate_indices=False)
merged = tf.dynamic_stitch([tf.range(0,flat_coarse_features.get_shape()[0]),flat_fine_idxs],
[flat_coarse_features,flat_fine_features])
merged = tf.reshape(merged,coarse_features.get_shape())
return merged, flat_coarse_replaced, flat_fine_features
def _match_when_rows_are_non_empty():
"""Performs matching when the rows of similarity matrix are non empty.
Returns:
matches: int32 tensor indicating the row each column matches to.
"""
# Matches for each column
matches = tf.argmax(similarity_matrix, 0)
# Deal with matched and unmatched threshold
if self._matched_threshold is not None:
# Get logical indices of ignored and unmatched columns as tf.int64
matched_vals = tf.reduce_max(similarity_matrix, 0)
below_unmatched_threshold = tf.greater(self._unmatched_threshold,
matched_vals)
between_thresholds = tf.logical_and(
tf.greater_equal(matched_vals, self._unmatched_threshold),
tf.greater(self._matched_threshold, matched_vals))
if self._negatives_lower_than_unmatched:
matches = self._set_values_using_indicator(matches,
below_unmatched_threshold,
-1)
matches = self._set_values_using_indicator(matches,
between_thresholds,
-2)
else:
matches = self._set_values_using_indicator(matches,
below_unmatched_threshold,
-2)
matches = self._set_values_using_indicator(matches,
between_thresholds,
-1)
if self._force_match_for_each_row:
forced_matches_ids = tf.cast(tf.argmax(similarity_matrix, 1), tf.int32)
# Set matches[forced_matches_ids] = [0, ..., R], R is number of rows.
row_range = tf.range(tf.shape(similarity_matrix)[0])
col_range = tf.range(tf.shape(similarity_matrix)[1])
forced_matches_values = tf.cast(row_range, matches.dtype)
keep_matches_ids, _ = tf.setdiff1d(col_range, forced_matches_ids)
keep_matches_values = tf.gather(matches, keep_matches_ids)
matches = tf.dynamic_stitch(
[forced_matches_ids,
keep_matches_ids], [forced_matches_values, keep_matches_values])
return tf.cast(matches, tf.int32)
def circular_convolution(v, k):
size = int(v.get_shape()[0])
kernel_size = int(k.get_shape()[0])
kernel_shift = int(math.floor(kernel_size/2.0))
def loop(idx):
if idx < 0: return size + idx
if idx >= size : return idx - size
else: return idx
kernels = []
for i in xrange(size):
indices = [loop(i+j) for j in xrange(kernel_shift, -kernel_shift-1, -1)]
v_ = tf.gather(v, indices)
kernels.append(tf.reduce_sum(v_ * k, 0))
return tf.dynamic_stitch([i for i in xrange(size)], kernels)
def testHigherRank(self):
with self.test_session() as sess:
indices = [tf.constant(6), tf.constant([4, 1]),
tf.constant([[5, 2], [0, 3]])]
data = [tf.constant([61, 62]), tf.constant([[41, 42], [11, 12]]),
tf.constant([[[51, 52], [21, 22]], [[1, 2], [31, 32]]])]
stitched_t = tf.dynamic_stitch(indices, data)
stitched_val = stitched_t.eval()
correct = 10 * np.arange(7)[:, None] + [1, 2]
self.assertAllEqual(correct, stitched_val)
self.assertEqual([None, 2], stitched_t.get_shape().as_list())
# Test gradients
stitched_grad = 7 * stitched_val
grads = tf.gradients(stitched_t, indices + data, stitched_grad)
self.assertEqual(grads[:3], [None] * 3) # Indices have no gradients
for datum, grad in zip(data, sess.run(grads[3:])):
self.assertAllEqual(7 * datum.eval(), grad)
def testSimpleTwoDimensional(self):
with self.test_session():
indices = [tf.constant([0, 4, 7]),
tf.constant([1, 6]),
tf.constant([2, 3, 5])]
data = [tf.constant([[0, 1], [40, 41], [70, 71]]),
tf.constant([[10, 11], [60, 61]]),
tf.constant([[20, 21], [30, 31], [50, 51]])]
stitched_t = tf.dynamic_stitch(indices, data)
stitched_val = stitched_t.eval()
self.assertAllEqual(
[[0, 1], [10, 11], [20, 21], [30, 31],
[40, 41], [50, 51], [60, 61], [70, 71]], stitched_val)
# Dimension 0 is determined by the max index in indices, so we
# can only infer that the output is a matrix with 2 columns and
# some unknown number of rows.
self.assertEqual([None, 2], stitched_t.get_shape().as_list())
def scheduled_sample(ground_truth_x, generated_x, batch_size, num_ground_truth):
"""Sample batch with specified mix of ground truth and generated data points.
Args:
ground_truth_x: tensor of ground-truth data points.
generated_x: tensor of generated data points.
batch_size: batch size
num_ground_truth: number of ground-truth examples to include in batch.
Returns:
New batch with num_ground_truth sampled from ground_truth_x and the rest
from generated_x.
"""
idx = tf.random_shuffle(tf.range(int(batch_size)))
ground_truth_idx = tf.gather(idx, tf.range(num_ground_truth))
generated_idx = tf.gather(idx, tf.range(num_ground_truth, int(batch_size)))
ground_truth_examps = tf.gather(ground_truth_x, ground_truth_idx)
generated_examps = tf.gather(generated_x, generated_idx)
return tf.dynamic_stitch([ground_truth_idx, generated_idx],
[ground_truth_examps, generated_examps])
def scheduled_sample(ground_truth_x, generated_x, batch_size,
num_ground_truth):
"""Sample batch with specified mix of ground truth and generated data_files points.
Args:
ground_truth_x: tensor of ground-truth data_files points.
generated_x: tensor of generated data_files points.
batch_size: batch size
num_ground_truth: number of ground-truth examples to include in batch.
Returns:
New batch with num_ground_truth sampled from ground_truth_x and the rest
from generated_x.
"""
idx = tf.random_shuffle(tf.range(int(batch_size)))
ground_truth_idx = tf.gather(idx, tf.range(num_ground_truth))
generated_idx = tf.gather(idx, tf.range(num_ground_truth, int(batch_size)))
ground_truth_examps = tf.gather(ground_truth_x, ground_truth_idx)
generated_examps = tf.gather(generated_x, generated_idx)
return tf.dynamic_stitch([ground_truth_idx, generated_idx],
[ground_truth_examps, generated_examps])
def _build_output_graph(self, rep, t, dim_in, dropout_representation,
dropout_regression, num_regression_layers):
''' Construct output/regression layers '''
i0, i1 = tf.to_int32(tf.where(t < 1)[:, 0]), tf.to_int32(
tf.where(t > 0)[:, 0])
rep0, rep1 = tf.gather(rep, i0), tf.gather(rep, i1)
y0, weights_out0, weights_pred0 = self._build_output(
rep0, dim_in, dropout_representation, dropout_regression,
num_regression_layers)
y1, weights_out1, weights_pred1 = self._build_output(
rep1, dim_in, dropout_representation, dropout_regression,
num_regression_layers)
y = tf.dynamic_stitch([i0, i1], [y0, y1])
weights_out = weights_out0 + weights_out1
weights_pred = weights_pred0 + weights_pred1
y_concat = tf.concat([y0, y1], axis=0)
return y, y_concat, weights_out, weights_pred
def testSimpleTwoDimensional(self):
with self.test_session():
indices = [
tf.constant([0, 4, 7]),
tf.constant([1, 6]),
tf.constant([2, 3, 5])
]
data = [
tf.constant([[0, 1], [40, 41], [70, 71]]),
tf.constant([[10, 11], [60, 61]]),
tf.constant([[20, 21], [30, 31], [50, 51]])
]
stitched_t = tf.dynamic_stitch(indices, data)
stitched_val = stitched_t.eval()
self.assertAllEqual([[0, 1], [10, 11], [20, 21], [30, 31],
[40, 41], [50, 51], [60, 61], [70, 71]],
stitched_val)
# Dimension 0 is determined by the max index in indices, so we
# can only infer that the output is a matrix with 2 columns and
# some unknown number of rows.
self.assertEqual([None, 2], stitched_t.get_shape().as_list())
def _stitch_mat_from_vecs(vector_list):
""" Stitches a given list of vectors into a 3x3 matrix.
Input:
vector_list: list of 9 tensors, which will be stitched into a matrix. list contains matrix elements
in a row-first fashion (m11, m12, m13, m21, m22, m23, m31, m32, m33). Length of the vectors has
to be the same, because it is interpreted as batch dimension.
"""
assert len(vector_list
) == 9, "There have to be exactly 9 tensors in vector_list."
batch_size = vector_list[0].get_shape().as_list()[0]
vector_list = [tf.reshape(x, [1, batch_size]) for x in vector_list]
trafo_matrix = tf.dynamic_stitch(
[[0], [1], [2], [3], [4], [5], [6], [7], [8]], vector_list)
trafo_matrix = tf.reshape(trafo_matrix, [3, 3, batch_size])
trafo_matrix = tf.transpose(trafo_matrix, [2, 0, 1])
return trafo_matrix
def f(params_1d):
"""A function that can be used by tfp.optimizer.lbfgs_minimize.
This function is created by function_factory.
Args:
params_1d [in]: a 1D tf.Tensor.
Returns:
A scalar loss and the gradients w.r.t. the `params_1d`.
"""
assign_new_model_parameters(params_1d)
loss_value, grads = model.get_grad(*args)
grads = tf.dynamic_stitch(idx, grads)
# print out iteration & loss
f.iter.assign_add(1)
if f.iter % model.print_epoch == 0:
tf.print("Iter:", f.iter, "loss:", loss_value)
return loss_value, grads
def _assign_rbox_target(self, anchors, groundtruth_boxes, groundtruth_rotations,
match):
'''
Args:
anchors: a BoxList representing N anchors
groundtruth_boxes: a BoxList representing M groundtruth boxes
groundtruth_rotations:
Returns:
reg_targets: a float32 tensor with shape [num_anchors, 5]
reg_weights: a float32 tensor with shape [num_anchors]
'''
if not isinstance(anchors, box_list.BoxList):
raise ValueError('anchors must be an BoxList')
if not isinstance(groundtruth_boxes, box_list.BoxList):
raise ValueError('groundtruth_boxes must be an BoxList')
matched_anchor_indices = match.matched_column_indices()
unmatched_ignored_anchor_indices = (match.
unmatched_or_ignored_column_indices())
matched_gt_indices = match.matched_row_indices()
matched_anchors = box_list_ops.gather(anchors,
matched_anchor_indices)
matched_gt_boxes = box_list_ops.gather(groundtruth_boxes,
matched_gt_indices)
matched_gt_rotations = tf.gather(groundtruth_rotations,
matched_gt_indices)
matched_reg_targets = self._box_coder.encode(matched_gt_boxes,
matched_gt_rotations,
matched_anchors)
default_target = tf.constant([5*[0]], tf.float32)
unmatched_ignored_reg_targets = tf.tile(
default_target,
tf.stack([tf.size(unmatched_ignored_anchor_indices), 1]))
reg_targets = tf.dynamic_stitch(
[matched_anchor_indices, unmatched_ignored_anchor_indices],
[matched_reg_targets, unmatched_ignored_reg_targets])
reg_weights = tf.cast(match.matched_column_indicator(), tf.float32)
return reg_targets, reg_weights
def _transform_col(self, data, interpolators, inverse):
if not inverse:
lower_bound_x = interpolators.low_quantile
upper_bound_x = interpolators.high_quantile
lower_bound_y = self.dtype(0)
upper_bound_y = self.dtype(1)
else:
lower_bound_x = self.dtype(0)
upper_bound_x = self.dtype(1)
lower_bound_y = interpolators.low_quantile
upper_bound_y = interpolators.high_quantile
lower_bounds_mask = (data - self.BOUNDS_THRESHOLD < lower_bound_x)
upper_bounds_mask = (data + self.BOUNDS_THRESHOLD > upper_bound_x)
in_range_mask = tf.logical_not(tf.logical_or(lower_bounds_mask, upper_bounds_mask))
data_in_range = tf.boolean_mask(data, in_range_mask)
if not inverse:
interpolated = 0.5*(
interpolators.quantiles_to_references_forward.interp(data_in_range) -
interpolators.quantiles_to_references_backward.interp(-data_in_range))
else:
interpolated = interpolators.references_to_quantiles.interp(data_in_range)
res = tf.dynamic_stitch(
[nonzero(upper_bounds_mask),
nonzero(in_range_mask),
nonzero(lower_bounds_mask)],
[tf.fill(tf.count_nonzero(upper_bounds_mask, keepdims=True), upper_bound_y),
interpolated,
tf.fill(tf.count_nonzero(lower_bounds_mask, keepdims=True), lower_bound_y)])
if not inverse:
res = self.output_distribution.quantile(res)
clip_min = self.output_distribution.quantile(tf.constant(
self.BOUNDS_THRESHOLD - np.spacing(1), dtype=self.dtype))
clip_max = self.output_distribution.quantile(tf.constant(
1 - (self.BOUNDS_THRESHOLD - np.spacing(1)), dtype=self.dtype))
res = tf.clip_by_value(res, clip_min, clip_max)
return res
def dce_loss(features, labels, centers, T, flags):
# size = features.shape[0]
dist = distance(features, centers, flags) #50,10,5
mask = tf.one_hot(labels, flags.num_classes)
mask_rep = tf.tile(tf.expand_dims(mask, axis=2),
multiples=[1, 1, flags.num_protos])
dist_not_this_class, dist_this_class = tf.dynamic_partition(
dist, tf.cast(mask_rep, tf.int32), 2)
dist_this_class = tf.reshape(dist_this_class, (-1, 1, flags.num_protos))
dist_not_this_class = tf.reshape(
dist_not_this_class, (-1, flags.num_classes - 1, flags.num_protos))
dist_this_class_max = tf.reduce_min(dist_this_class, axis=2)
dist_not_this_class_min = tf.reduce_max(dist_not_this_class, axis=2)
dist_not_this_class_min = tf.reshape(dist_not_this_class_min, (-1, ))
dist_this_class_max = tf.reshape(dist_this_class_max, (-1, ))
# import ipdb
# ipdb.set_trace()
condition_indices= tf.dynamic_partition(
tf.range(flags.num_classes * tf.shape(dist)[0]),\
tf.cast(tf.reshape(mask, (-1,)), tf.int32), 2)
logits = tf.dynamic_stitch(condition_indices,
[dist_not_this_class_min, dist_this_class_max])
logits = tf.reshape(logits, (-1, flags.num_classes))
logits = -logits / T
mean_loss = softmax_loss(logits, labels)
# # for another loss calculation.
# dist_not_this_class_max = tf.reduce_min(dist_not_this_class, axis = 2)
# dist_not_this_class_max = tf.reshape(dist_not_this_class_max, (-1,))
# condition_indices= tf.dynamic_partition(
# tf.range(flags.num_classes * dist.shape[0]),\
# tf.cast(tf.reshape(mask, (-1,)), tf.int32), 2)
# logits = tf.dynamic_stitch(condition_indices, [dist_not_this_class_max, dist_this_class_max])
# logits = tf.reshape(logits, (-1,flags.num_classes))
# logits = -logits / T
# mean_loss_2 = softmax_loss(logits, labels)
return mean_loss #+ mean_loss_2
def f(params_1d):
"""A function that can be used by tfp.optimizer.lbfgs_minimize.
This function is created by function_factory.
Args:
params_1d [in]: a 1D tf.Tensor.
Returns:
A scalar loss and the gradients the `params_1d`.
"""
# use GradientTape so that we can calculate the gradient of loss
with tf.GradientTape(persistent=True) as tape1:
# update the parameters in the model
assign_new_model_parameters(params_1d)
with tf.GradientTape(persistent=True) as tape2:
with tf.GradientTape(persistent=True) as tape3:
Psi = model(tf.concat([x, y], 1))
Psi_x, Psi_y = tape3.gradient(Psi, [x, y])
Psi_xx = tape2.gradient(Psi_x, x)
Psi_yy = tape2.gradient(Psi_y, y)
loss_e = residual(Psi_xx=Psi_xx, Psi_yy=Psi_yy, x=x, y=y)
loss_b = loss_bc(model=model,
x=x_train_bc,
y=y_train_bc,
x_d=x_train_bc_d,
y_d=y_train_bc_d,
bc_train=bc_train)
# calculate the loss
loss_value = loss_e + loss_b
# calculate gradients and convert to 1D tf.Tensor
grads = tape1.gradient(loss_value, model.trainable_variables)
grads = tf.dynamic_stitch(idx, grads)
# print out iteration & loss
f.iter.assign_add(1)
if f.iter % 2000 == 0:
tf.print("Iter:", f.iter, "loss:", loss_value)
# store loss value so we can retrieve later
tf.py_function(f.history.append,
inp=[[loss_value, loss_e, loss_b]],
Tout=[])
return loss_value, grads
def _build_output_graph(self, rep, t, dim_in, dim_out, do_out, FLAGS):
''' Construct output/regression layers '''
if FLAGS.split_output:
i0 = tf.to_int32(tf.where(t < 1)[:,0])
i1 = tf.to_int32(tf.where(t > 0)[:,0])
rep0 = tf.gather(rep, i0)
rep1 = tf.gather(rep, i1)
y0, weights_out0, weights_pred0 = self._build_output(rep0, dim_in, dim_out, do_out, FLAGS)
y1, weights_out1, weights_pred1 = self._build_output(rep1, dim_in, dim_out, do_out, FLAGS)
y = tf.dynamic_stitch([i0, i1], [y0, y1])
weights_out = weights_out0 + weights_out1
weights_pred = weights_pred0 + weights_pred1
else:
h_input = tf.concat(1,[rep, t])
y, weights_out, weights_pred = self._build_output(h_input, dim_in+1, dim_out, do_out, FLAGS)
return y, weights_out, weights_pred
def f(params_1d):
"""A function that can be used by tfp.optimizer.lbfgs_minimize.
This function is created by function_factory.
Args:
params_1d [in]: a 1D tf.Tensor.
Returns:
A scalar loss and the gradients w.r.t. the `params_1d`.
"""
# use GradientTape so that we can calculate the gradient of loss w.r.t. parameters
with tf.GradientTape() as tape:
# update the parameters in the model
assign_new_model_parameters(params_1d)
# calculate the loss
loss_value = loss_func(model)
# calculate gradients and convert to 1D tf.Tensor
grads = tape.gradient(loss_value, model.trainable_variables)
grads = tf.dynamic_stitch(idx, grads)
# print out iteration & loss
f.iter.assign_add(1)
tf.print("Iter:", f.iter, "loss:", loss_value)
if self.callback_list is not None:
info_dict = {
'iter': f.iter,
'loss': loss_value,
'grad': grads,
}
for callback in self.callback_list:
callback(model, info_dict=info_dict)
return loss_value, grads
def lookup(self, keys, name=None):
"""Looks up `keys` in a table, outputs the corresponding values."""
if keys.dtype.base_dtype != self._key_dtype:
raise TypeError(
"Signature mismatch. Keys must be dtype %s, got %s." %
(self._key_dtype, keys.dtype))
self._check_keys(keys)
num_shards = self._num_shards
if num_shards == 1:
return self._table_shards[0].lookup(keys, name=name)
shard_indices = self._shard_indices(keys)
key_shards = tf.dynamic_partition(keys, shard_indices, num_shards)
value_shards = [
self._table_shards[i].lookup(key_shards[i], name=name)
for i in range(num_shards)
]
num_keys = tf.compat.v1.shape(keys)[0]
original_indices = tf.range(num_keys)
partitioned_indices = tf.dynamic_partition(original_indices,
shard_indices, num_shards)
return tf.dynamic_stitch(partitioned_indices, value_shards)
def minimize(self, loss_func, model):
optim_func = self._function_wrapper(loss_func, model)
# convert initial model parameters to a 1D tf.Tensor
init_params = tf.dynamic_stitch(optim_func.idx,
model.trainable_variables)
# train the model with L-BFGS solver
results = tfp.optimizer.lbfgs_minimize(
value_and_gradients_function=optim_func,
initial_position=init_params,
max_iterations=self.max_iterations,
tolerance=self.tolerance,
x_tolerance=self.tolerance,
f_relative_tolerance=self.tolerance,
**self.lbfgs_kwargs)
# after training, the final optimized parameters are still in results.position
# so we have to manually put them back to the model
optim_func.assign_new_model_parameters(results.position)
print("L-BFGS complete, and parameters updated !")
return model
def _create_targets(anchors, groundtruth_boxes, matches):
"""Returns regression targets for each anchor.
Arguments:
anchors: a float tensor with shape [num_anchors, 4].
groundtruth_boxes: a float tensor with shape [N, 4].
matches: a int tensor with shape [num_anchors].
Returns:
reg_targets: a float tensor with shape [num_anchors, 4].
"""
matched_anchor_indices = tf.where(tf.greater_equal(
matches, 0)) # shape [num_matches, 1]
matched_anchor_indices = tf.squeeze(matched_anchor_indices, axis=1)
matched_gt_indices = tf.gather(
matches, matched_anchor_indices) # shape [num_matches]
matched_anchors = tf.gather(
anchors, matched_anchor_indices) # shape [num_matches, 4]
matched_gt_boxes = tf.gather(groundtruth_boxes,
matched_gt_indices) # shape [num_matches, 4]
matched_reg_targets = encode(matched_gt_boxes,
matched_anchors) # shape [num_matches, 4]
unmatched_anchor_indices = tf.where(tf.equal(matches, -1))
unmatched_anchor_indices = tf.squeeze(unmatched_anchor_indices, axis=1)
# it has shape [num_anchors - num_matches]
unmatched_reg_targets = tf.zeros([tf.size(unmatched_anchor_indices), 4])
# it has shape [num_anchors - num_matches, 4]
matched_anchor_indices = tf.to_int32(matched_anchor_indices)
unmatched_anchor_indices = tf.to_int32(unmatched_anchor_indices)
reg_targets = tf.dynamic_stitch(
[matched_anchor_indices, unmatched_anchor_indices],
[matched_reg_targets, unmatched_reg_targets])
return reg_targets
def K(self, X, X2=None, presliced=False):
if X2 is None:
X2 = X
ind_X = X[:, -1]
ind_X = tf.cast(ind_X, tf.int32)
ind_X2 = X2[:, -1]
ind_X2 = tf.cast(ind_X2, tf.int32)
X = X[:, :-1]
X2 = X2[:, :-1]
if not presliced:
X, X2 = self._slice(X, X2)
ind_X_parts = tf.dynamic_partition(tf.range(0, tf.size(ind_X)), ind_X,
self.output_dim)
ind_X2_parts = tf.dynamic_partition(tf.range(0, tf.size(ind_X2)),
ind_X2, self.output_dim)
Ks = []
for k, p, p2 in zip(self.kernels, ind_X_parts, ind_X2_parts):
gram = k.K(tf.gather(X, p), tf.gather(X2, p2))
Ks.append(gram)
N = tf.shape(X)[0]
N2 = tf.shape(X2)[0]
Ks_scattered = []
for gram, p, p2 in zip(Ks, ind_X_parts, ind_X2_parts):
p2 = p2[:, None]
shape = tf.stack([N2, tf.shape(p)[0]])
scattered = tf.transpose(
tf.scatter_nd(p2, tf.transpose(gram), shape))
Ks_scattered.append(scattered)
return tf.dynamic_stitch(ind_X_parts, Ks_scattered)
def _update_alloc_and_usage_vectors(self, pre_write_weightings,
pre_read_weightings, pre_usage_vector,
free_gates):
retention_vector = tf.reduce_prod(1 - free_gates * pre_read_weightings,
axis=1,
keepdims=False,
name='retention_prod')
usage_vector = (
pre_usage_vector + pre_write_weightings -
pre_usage_vector * pre_write_weightings) * retention_vector
sorted_usage, free_list = tf.nn.top_k(-1 * usage_vector, self.h_N)
sorted_usage = -1 * sorted_usage
cumprod_sorted_usage = tf.cumprod(sorted_usage, axis=1, exclusive=True)
corrected_free_list = free_list + self.const_batch_memory_range
cumprod_sorted_usage_re = [
tf.reshape(cumprod_sorted_usage, [
-1,
]),
]
corrected_free_list_re = [
tf.reshape(corrected_free_list, [-1]),
]
stitched_usage = tf.dynamic_stitch(corrected_free_list_re,
cumprod_sorted_usage_re,
name=None)
stitched_usage = tf.reshape(stitched_usage, [self.h_B, self.h_N])
alloc_weighting = (1 - usage_vector) * stitched_usage
return alloc_weighting, usage_vector
def log_inv_probit(x):
'''
This produces the correct asymptotic behaviour for log(inv_probit(x)) for large negative x
'''
## All x's that are below a negative threshold are calculated using an asymptotic approximation.
## All x's above that threshold are calculated normally.
# -1.45026 is the zero-crossing of the error between the approximation and truth, minimising the discontinuous jump when moving between the two.
split_mask = tf.cast(tf.greater(x, -1.45026), tf.int32)
split_idx = tf.dynamic_partition(tf.range(tf.shape(x)[0]), split_mask, 2)
# Split the x values to the different piecewise regions
x_split = tf.dynamic_partition(x, split_mask, 2)
# Approximation of log (inv_probit (x)) for x < -1.45026
y_0 = tf.log(1 - tf.exp(1.4 * x_split[0])) - (x_split[0]**2) / 2 - tf.log(
-x_split[0]) - np.log(1.136 * np.sqrt(2 * np.pi))
# Use tf.log (inv_probit(x)) as it is accurate for x > -1.45026
y_1 = tf.log(inv_probit(x_split[1]))
# Stitch the results back together
return tf.dynamic_stitch(split_idx, [y_0, y_1])
def observed_data_layer(observed_data, missing_data, condition_indices,
output_dim, name, reuse):
#Train a layer with the observed data and reuse it for the missing data
obs_output = tf.layers.dense(
inputs=observed_data,
units=output_dim,
activation=None,
kernel_initializer=tf.random_normal_initializer(stddev=0.05),
name=name,
reuse=reuse,
trainable=True)
miss_output = tf.layers.dense(
inputs=missing_data,
units=output_dim,
activation=None,
kernel_initializer=tf.random_normal_initializer(stddev=0.05),
name=name,
reuse=True,
trainable=False)
#Join back the data
output = tf.dynamic_stitch(condition_indices, [miss_output, obs_output])
return output
def testHigherRank(self):
with self.test_session() as sess:
indices = [
tf.constant(6),
tf.constant([4, 1]),
tf.constant([[5, 2], [0, 3]])
]
data = [
tf.constant([61, 62]),
tf.constant([[41, 42], [11, 12]]),
tf.constant([[[51, 52], [21, 22]], [[1, 2], [31, 32]]])
]
stitched_t = tf.dynamic_stitch(indices, data)
stitched_val = stitched_t.eval()
correct = 10 * np.arange(7)[:, None] + [1, 2]
self.assertAllEqual(correct, stitched_val)
self.assertEqual([None, 2], stitched_t.get_shape().as_list())
# Test gradients
stitched_grad = 7 * stitched_val
grads = tf.gradients(stitched_t, indices + data, stitched_grad)
self.assertEqual(grads[:3],
[None] * 3) # Indices have no gradients
for datum, grad in zip(data, sess.run(grads[3:])):
self.assertAllEqual(7 * datum.eval(), grad)
def scheduled_sample(ground_truth_x, generated_x, batch_size, num_ground_truth):
"""Sample batch with specified mix of ground truth and generated data points.
Args:
ground_truth_x: tensor of ground-truth data points.
generated_x: tensor of generated data points.
batch_size: batch size
num_ground_truth: number of ground-truth examples to include in batch.
Returns:
New batch with num_ground_truth sampled from ground_truth_x and the rest
from generated_x.
"""
ground_truth_mask = tf.concat([tf.zeros([num_ground_truth], dtype=tf.int32),
tf.ones([batch_size - num_ground_truth], dtype=tf.int32)], axis=0)
ground_truth_mask = tf.reshape(ground_truth_mask, [batch_size])
ground_truth_mask = tf.random_shuffle(ground_truth_mask)
ground_truth_partitioned = tf.dynamic_partition(ground_truth_x, ground_truth_mask, 2)
generated_partitioned = tf.dynamic_partition(generated_x, ground_truth_mask, 2)
stitch_indices = tf.dynamic_partition(tf.range(batch_size), ground_truth_mask, 2)
stitch_data = [ground_truth_partitioned[0], generated_partitioned[1]]
outputs = tf.dynamic_stitch(stitch_indices, stitch_data)
outputs = tf.reshape(outputs, [int(batch_size)] + outputs.get_shape().as_list()[1:])
return outputs
def _partition_and_stitch(self, args, func_name):
"""
args is a list of tensors, to be passed to self.likelihoods.<func_name>
args[-1] is the 'Y' argument, which contains the indexes to self.likelihoods.
This function splits up the args using dynamic_partition, calls the
relevant function on the likelihoods, and re-combines the result.
"""
# get the index from Y
Y = args[-1]
ind = tf.gather(tf.transpose(Y), tf.shape(Y)[1] - 1) # ind = Y[:,-1]
ind = tf.cast(ind, tf.int32)
Y = tf.transpose(
tf.gather(tf.transpose(Y),
tf.range(0,
tf.shape(Y)[1] - 1))) # Y = Y[:,:-1]
args[-1] = Y
# split up the arguments into chunks corresponding to the relevant likelihoods
args = zip(
*
[tf.dynamic_partition(X, ind, self.num_likelihoods) for X in args])
# apply the likelihood-function to each section of the data
with params_as_tensors_for(self, convert=False):
funcs = [getattr(lik, func_name) for lik in self.likelihood_list]
results = [f(*args_i) for f, args_i in zip(funcs, args)]
# stitch the results back together
partitions = tf.dynamic_partition(tf.range(0, tf.size(ind)), ind,
self.num_likelihoods)
results = tf.dynamic_stitch(partitions, results)
return results
def _sample_n(self, n, seed=None):
if self._use_static_graph:
# This sampling approach is almost the same as the approach used by
# `MixtureSameFamily`. The differences are due to having a list of
# `Distribution` objects rather than a single object, and maintaining
# random seed management that is consistent with the non-static code path.
samples = []
cat_samples = self.cat.sample(n, seed=seed)
for c in range(self.num_components):
seed = distribution_util.gen_new_seed(seed, "mixture")
samples.append(self.components[c].sample(n, seed=seed))
x = tf.stack(samples, -self._static_event_shape.ndims - 1) # [n, B, k, E]
npdt = x.dtype.as_numpy_dtype
mask = tf.one_hot(
indices=cat_samples, # [n, B]
depth=self._num_components, # == k
on_value=np.ones([], dtype=npdt),
off_value=np.zeros([], dtype=npdt)) # [n, B, k]
mask = distribution_utils.pad_mixture_dimensions(
mask, self, self._cat,
self._static_event_shape.ndims) # [n, B, k, [1]*e]
return tf.reduce_sum(
x * mask, axis=-1 - self._static_event_shape.ndims) # [n, B, E]
with tf.control_dependencies(self._assertions):
n = tf.convert_to_tensor(n, name="n")
static_n = tensor_util.constant_value(n)
n = int(static_n) if static_n is not None else n
cat_samples = self.cat.sample(n, seed=seed)
static_samples_shape = cat_samples.get_shape()
if static_samples_shape.is_fully_defined():
samples_shape = static_samples_shape.as_list()
samples_size = static_samples_shape.num_elements()
else:
samples_shape = tf.shape(cat_samples)
samples_size = tf.size(cat_samples)
static_batch_shape = self.batch_shape
if static_batch_shape.is_fully_defined():
batch_shape = static_batch_shape.as_list()
batch_size = static_batch_shape.num_elements()
else:
batch_shape = self.batch_shape_tensor()
batch_size = tf.reduce_prod(batch_shape)
static_event_shape = self.event_shape
if static_event_shape.is_fully_defined():
event_shape = np.array(static_event_shape.as_list(), dtype=np.int32)
else:
event_shape = self.event_shape_tensor()
# Get indices into the raw cat sampling tensor. We will
# need these to stitch sample values back out after sampling
# within the component partitions.
samples_raw_indices = tf.reshape(tf.range(0, samples_size), samples_shape)
# Partition the raw indices so that we can use
# dynamic_stitch later to reconstruct the samples from the
# known partitions.
partitioned_samples_indices = tf.dynamic_partition(
data=samples_raw_indices,
partitions=cat_samples,
num_partitions=self.num_components)
# Copy the batch indices n times, as we will need to know
# these to pull out the appropriate rows within the
# component partitions.
batch_raw_indices = tf.reshape(
tf.tile(tf.range(0, batch_size), [n]), samples_shape)
# Explanation of the dynamic partitioning below:
# batch indices are i.e., [0, 1, 0, 1, 0, 1]
# Suppose partitions are:
# [1 1 0 0 1 1]
# After partitioning, batch indices are cut as:
# [batch_indices[x] for x in 2, 3]
# [batch_indices[x] for x in 0, 1, 4, 5]
# i.e.
# [1 1] and [0 0 0 0]
# Now we sample n=2 from part 0 and n=4 from part 1.
# For part 0 we want samples from batch entries 1, 1 (samples 0, 1),
# and for part 1 we want samples from batch entries 0, 0, 0, 0
# (samples 0, 1, 2, 3).
partitioned_batch_indices = tf.dynamic_partition(
data=batch_raw_indices,
partitions=cat_samples,
num_partitions=self.num_components)
samples_class = [None for _ in range(self.num_components)]
for c in range(self.num_components):
n_class = tf.size(partitioned_samples_indices[c])
seed = distribution_util.gen_new_seed(seed, "mixture")
samples_class_c = self.components[c].sample(n_class, seed=seed)
# Pull out the correct batch entries from each index.
# To do this, we may have to flatten the batch shape.
# For sample s, batch element b of component c, we get the
# partitioned batch indices from
# partitioned_batch_indices[c]; and shift each element by
# the sample index. The final lookup can be thought of as
# a matrix gather along locations (s, b) in
# samples_class_c where the n_class rows correspond to
# samples within this component and the batch_size columns
# correspond to batch elements within the component.
#
# Thus the lookup index is
# lookup[c, i] = batch_size * s[i] + b[c, i]
# for i = 0 ... n_class[c] - 1.
lookup_partitioned_batch_indices = (
batch_size * tf.range(n_class) + partitioned_batch_indices[c])
samples_class_c = tf.reshape(
samples_class_c, tf.concat([[n_class * batch_size], event_shape],
0))
samples_class_c = tf.gather(
samples_class_c,
lookup_partitioned_batch_indices,
name="samples_class_c_gather")
samples_class[c] = samples_class_c
# Stitch back together the samples across the components.
lhs_flat_ret = tf.dynamic_stitch(
indices=partitioned_samples_indices, data=samples_class)
# Reshape back to proper sample, batch, and event shape.
ret = tf.reshape(
lhs_flat_ret, tf.concat(
[samples_shape, self.event_shape_tensor()], 0))
ret.set_shape(
tf.TensorShape(static_samples_shape).concatenate(self.event_shape))
return ret
def layer_op(self, inputs, is_training, mask, use_local_stats=False):
"""
Parameters:
inputs: image to normalize. This typically represents a sparse subset of channels
from a sparse convolution.
is_training: boolean that is True during training. When True, the layer uses batch
statistics for normalization and records a moving average of means and variances. When False, the layer uses previously computed moving averages for normalization
mask: 1-Tensor with a binary mask identifying the sparse channels represented in inputs
"""
if mask is None:
mask=tf.ones([self.n_dense_channels])>0
else:
mask=mask
input_shape = inputs.shape
mask_shape = mask.shape
# operates on all dims except the last dim
params_shape = mask_shape[-1:]
assert params_shape[0]==self.n_dense_channels, \
'Mask size {} must match n_dense_channels {}.'.format(params_shape[0],self.n_dense_channels)
axes = list(range(input_shape.ndims - 1))
# create trainable variables and moving average variables
beta = tf.get_variable(
'beta',
shape=params_shape,
initializer=self.initializers['beta'],
regularizer=self.regularizers['beta'],
dtype=tf.float32, trainable=True)
gamma = tf.get_variable(
'gamma',
shape=params_shape,
initializer=self.initializers['gamma'],
regularizer=self.regularizers['gamma'],
dtype=tf.float32, trainable=True)
collections = [tf.GraphKeys.GLOBAL_VARIABLES]
moving_mean = tf.get_variable(
'moving_mean',
shape=params_shape,
initializer=self.initializers['moving_mean'],
dtype=tf.float32, trainable=False, collections=collections)
moving_variance = tf.get_variable(
'moving_variance',
shape=params_shape,
initializer=self.initializers['moving_variance'],
dtype=tf.float32, trainable=False, collections=collections)
# mean and var
mean, variance = tf.nn.moments(inputs, axes)
# only update masked moving averages
mean_update=tf.dynamic_stitch([tf.to_int32(tf.where(mask)[:,0]),tf.to_int32(tf.where(~mask)[:,0])],[mean,tf.boolean_mask(moving_mean,~mask)])
variance_update=tf.dynamic_stitch([tf.to_int32(tf.where(mask)[:,0]),tf.to_int32(tf.where(~mask)[:,0])],[variance,tf.boolean_mask(moving_variance,~mask)])
update_moving_mean = moving_averages.assign_moving_average(
moving_mean, mean_update, self.moving_decay).op
update_moving_variance = moving_averages.assign_moving_average(
moving_variance, variance_update, self.moving_decay).op
tf.add_to_collection(tf.GraphKeys.UPDATE_OPS, update_moving_mean)
tf.add_to_collection(tf.GraphKeys.UPDATE_OPS, update_moving_variance)
# call the normalisation function
if is_training or use_local_stats:
# with tf.control_dependencies(
# [update_moving_mean, update_moving_variance]):
outputs = tf.nn.batch_normalization(
inputs, mean, variance,
tf.boolean_mask(beta,mask), tf.boolean_mask(gamma,mask), self.eps, name='batch_norm')
else:
outputs = tf.nn.batch_normalization(
inputs, tf.boolean_mask(moving_mean,mask), tf.boolean_mask(moving_variance,mask),
tf.boolean_mask(beta,mask), tf.boolean_mask(gamma,mask), self.eps, name='batch_norm')
outputs.set_shape(inputs.get_shape())
return outputs
neigh_articles = tf.gather(rep_law_1, adj_list[i]) # size: n * deg * law_size
neigh_articles = tf.reshape(neigh_articles, [-1, i, lstm_size * 2])
article = law_representation[i] # size: n * law_size
article_1 = tf.transpose(tf.reshape(tf.tile(article, [i, 1]), [i, -1, lstm_size * 2]), [1, 0, 2])
interaction_vec = tf.concat([article_1, neigh_articles], axis=-1) # size: [n, deg, law_size *2]
neigh_articles = tf.reduce_mean(
tf.tensordot(interaction_vec, W_similar, [2, 0]) + tf.expand_dims(B_similar, axis=0),
1) # [n, law_size]
new_article = tf.nn.tanh(Full_inter_1(article - neigh_articles))
article_new_list.append(new_article) # [max_degree, N, law_size]
law_conv = tf.dynamic_stitch(indices, article_new_list) # size: [183, law_size]
article_new_list_1 = [tf.nn.tanh(article_new_list[0])]
for i in range(1, max_deg + 1):
if i not in deg_list:
article_new_list_1.append(tf.nn.tanh(article_new_list[i]))
continue
neigh_articles = tf.gather(law_conv, adj_list[i]) # size: n * deg * law_size
neigh_articles = tf.reshape(neigh_articles, [-1, i, lstm_size * 2])
article = article_new_list[i] # size: n * law_size
article_1 = tf.transpose(tf.reshape(tf.tile(article, [i, 1]), [i, -1, lstm_size * 2]), [1, 0, 2])
interaction_vec = tf.concat([article_1, neigh_articles], axis=-1) # size: [n, deg, law_size *2]
neigh_articles = tf.reduce_mean(
def _sample_n(self, n, seed=None):
if self._use_static_graph:
with tf.control_dependencies(self._assertions):
# This sampling approach is almost the same as the approach used by
# `MixtureSameFamily`. The differences are due to having a list of
# `Distribution` objects rather than a single object, and maintaining
# random seed management that is consistent with the non-static code
# path.
samples = []
cat_samples = self.cat.sample(n, seed=seed)
stream = seed_stream.SeedStream(seed, salt="Mixture")
for c in range(self.num_components):
samples.append(self.components[c].sample(n, seed=stream()))
x = tf.stack(samples, -self._static_event_shape.ndims -
1) # [n, B, k, E]
npdt = x.dtype.as_numpy_dtype
mask = tf.one_hot(
indices=cat_samples, # [n, B]
depth=self._num_components, # == k
on_value=np.ones([], dtype=npdt),
off_value=np.zeros([], dtype=npdt)) # [n, B, k]
mask = distribution_util.pad_mixture_dimensions(
mask, self, self._cat,
self._static_event_shape.ndims) # [n, B, k, [1]*e]
return tf.reduce_sum(
input_tensor=x * mask,
axis=-1 - self._static_event_shape.ndims) # [n, B, E]
with tf.control_dependencies(self._assertions):
n = tf.convert_to_tensor(value=n, name="n")
static_n = tf.get_static_value(n)
n = int(static_n) if static_n is not None else n
cat_samples = self.cat.sample(n, seed=seed)
static_samples_shape = cat_samples.shape
if static_samples_shape.is_fully_defined():
samples_shape = static_samples_shape.as_list()
samples_size = static_samples_shape.num_elements()
else:
samples_shape = tf.shape(input=cat_samples)
samples_size = tf.size(input=cat_samples)
static_batch_shape = self.batch_shape
if static_batch_shape.is_fully_defined():
batch_shape = static_batch_shape.as_list()
batch_size = static_batch_shape.num_elements()
else:
batch_shape = self.batch_shape_tensor()
batch_size = tf.reduce_prod(input_tensor=batch_shape)
static_event_shape = self.event_shape
if static_event_shape.is_fully_defined():
event_shape = np.array(static_event_shape.as_list(),
dtype=np.int32)
else:
event_shape = self.event_shape_tensor()
# Get indices into the raw cat sampling tensor. We will
# need these to stitch sample values back out after sampling
# within the component partitions.
samples_raw_indices = tf.reshape(tf.range(0, samples_size),
samples_shape)
# Partition the raw indices so that we can use
# dynamic_stitch later to reconstruct the samples from the
# known partitions.
partitioned_samples_indices = tf.dynamic_partition(
data=samples_raw_indices,
partitions=cat_samples,
num_partitions=self.num_components)
# Copy the batch indices n times, as we will need to know
# these to pull out the appropriate rows within the
# component partitions.
batch_raw_indices = tf.reshape(
tf.tile(tf.range(0, batch_size), [n]), samples_shape)
# Explanation of the dynamic partitioning below:
# batch indices are i.e., [0, 1, 0, 1, 0, 1]
# Suppose partitions are:
# [1 1 0 0 1 1]
# After partitioning, batch indices are cut as:
# [batch_indices[x] for x in 2, 3]
# [batch_indices[x] for x in 0, 1, 4, 5]
# i.e.
# [1 1] and [0 0 0 0]
# Now we sample n=2 from part 0 and n=4 from part 1.
# For part 0 we want samples from batch entries 1, 1 (samples 0, 1),
# and for part 1 we want samples from batch entries 0, 0, 0, 0
# (samples 0, 1, 2, 3).
partitioned_batch_indices = tf.dynamic_partition(
data=batch_raw_indices,
partitions=cat_samples,
num_partitions=self.num_components)
samples_class = [None for _ in range(self.num_components)]
stream = seed_stream.SeedStream(seed, salt="Mixture")
for c in range(self.num_components):
n_class = tf.size(input=partitioned_samples_indices[c])
samples_class_c = self.components[c].sample(n_class,
seed=stream())
# Pull out the correct batch entries from each index.
# To do this, we may have to flatten the batch shape.
# For sample s, batch element b of component c, we get the
# partitioned batch indices from
# partitioned_batch_indices[c]; and shift each element by
# the sample index. The final lookup can be thought of as
# a matrix gather along locations (s, b) in
# samples_class_c where the n_class rows correspond to
# samples within this component and the batch_size columns
# correspond to batch elements within the component.
#
# Thus the lookup index is
# lookup[c, i] = batch_size * s[i] + b[c, i]
# for i = 0 ... n_class[c] - 1.
lookup_partitioned_batch_indices = (
batch_size * tf.range(n_class) +
partitioned_batch_indices[c])
samples_class_c = tf.reshape(
samples_class_c,
tf.concat([[n_class * batch_size], event_shape], 0))
samples_class_c = tf.gather(samples_class_c,
lookup_partitioned_batch_indices,
name="samples_class_c_gather")
samples_class[c] = samples_class_c
# Stitch back together the samples across the components.
lhs_flat_ret = tf.dynamic_stitch(
indices=partitioned_samples_indices, data=samples_class)
# Reshape back to proper sample, batch, and event shape.
ret = tf.reshape(
lhs_flat_ret,
tf.concat(
[samples_shape, self.event_shape_tensor()], 0))
ret.set_shape(
tf.TensorShape(static_samples_shape).concatenate(
self.event_shape))
return ret
train_op = tf.keras.optimizers.Adam()
num_epoch = 10000
print_epoch = 100
pred_model = model([l1, l2, l3], train_op, num_epoch, print_epoch)
#convert the training data to tensors
Xint_tf = tf.convert_to_tensor(Xint)
Yint_tf = tf.convert_to_tensor(Yint)
#training
print("Training (ADAM)...")
pred_model.network_learn(Xint_tf, Yint_tf)
print("Training (LBFGS)...")
loss_func = tfp_function_factory(pred_model, Xint_tf, Yint_tf)
# convert initial model parameters to a 1D tf.Tensor
init_params = tf.dynamic_stitch(loss_func.idx,
pred_model.trainable_variables)
# train the model with L-BFGS solver
results = tfp.optimizer.lbfgs_minimize(
value_and_gradients_function=loss_func,
initial_position=init_params,
max_iterations=4000,
num_correction_pairs=50,
tolerance=1e-14)
# after training, the final optimized parameters are still in results.position
# so we have to manually put them back to the model
loss_func.assign_new_model_parameters(results.position)
print("Testing...")
numPtsTest = 2 * numPts
x_test = np.linspace(xmin, xmax, numPtsTest)
x_test = np.array(x_test)[np.newaxis].T
def layer_op(self, inputs, is_training, mask, use_local_stats=False):
"""
:param inputs: image to normalize. This typically represents a sparse
subset of channels from a sparse convolution.
:param is_training: boolean that is True during training.
When True, the layer uses batch statistics for normalization and
records a moving average of means and variances.
When False, the layer uses previously computed moving averages
for normalization.
:param mask: 1-Tensor with a binary mask identifying the sparse
channels represented in inputs
:param use_local_stats:
:return:
"""
if mask is None:
mask = tf.ones([self.n_dense_channels]) > 0
else:
mask = mask
input_shape = inputs.shape
mask_shape = mask.shape
# operates on all dims except the last dim
params_shape = mask_shape[-1:]
assert params_shape[0] == self.n_dense_channels, \
'Mask size {} must match n_dense_channels {}.'.format(
params_shape[0], self.n_dense_channels)
axes = list(range(input_shape.ndims - 1))
# create trainable variables and moving average variables
beta = tf.get_variable(
'beta',
shape=params_shape,
initializer=self.initializers['beta'],
regularizer=self.regularizers['beta'],
dtype=tf.float32, trainable=True)
gamma = tf.get_variable(
'gamma',
shape=params_shape,
initializer=self.initializers['gamma'],
regularizer=self.regularizers['gamma'],
dtype=tf.float32, trainable=True)
beta = tf.boolean_mask(beta, mask)
gamma = tf.boolean_mask(gamma, mask)
collections = [tf.GraphKeys.GLOBAL_VARIABLES]
moving_mean = tf.get_variable(
'moving_mean',
shape=params_shape,
initializer=self.initializers['moving_mean'],
dtype=tf.float32, trainable=False, collections=collections)
moving_variance = tf.get_variable(
'moving_variance',
shape=params_shape,
initializer=self.initializers['moving_variance'],
dtype=tf.float32, trainable=False, collections=collections)
# mean and var
mean, variance = tf.nn.moments(inputs, axes)
# only update masked moving averages
mean_update = tf.dynamic_stitch(
[tf.to_int32(tf.where(mask)[:, 0]),
tf.to_int32(tf.where(~mask)[:, 0])],
[mean,
tf.boolean_mask(moving_mean, ~mask)])
variance_update = tf.dynamic_stitch(
[tf.to_int32(tf.where(mask)[:, 0]),
tf.to_int32(tf.where(~mask)[:, 0])],
[variance,
tf.boolean_mask(moving_variance, ~mask)])
update_moving_mean = moving_averages.assign_moving_average(
moving_mean, mean_update, self.moving_decay).op
update_moving_variance = moving_averages.assign_moving_average(
moving_variance, variance_update, self.moving_decay).op
tf.add_to_collection(tf.GraphKeys.UPDATE_OPS, update_moving_mean)
tf.add_to_collection(tf.GraphKeys.UPDATE_OPS, update_moving_variance)
# call the normalisation function
if is_training or use_local_stats:
outputs = tf.nn.batch_normalization(
inputs, mean, variance,
beta, gamma, self.eps, name='batch_norm')
else:
outputs = tf.nn.batch_normalization(
inputs,
tf.boolean_mask(moving_mean, mask),
tf.boolean_mask(moving_variance, mask),
beta, gamma, self.eps, name='batch_norm')
outputs.set_shape(inputs.get_shape())
return outputs
def merge_data(self, data_list):
with tf.name_scope(None, "merge_data") as scope:
return tf.dynamic_stitch(self.stitch_indices, data_list)
# from https://github.com/tensorflow/tensorflow/issues/7251
import os
os.environ["CUDA_VISIBLE_DEVICES"]="0"
import tensorflow as tf
from tensorflow.python.client.timeline import Timeline
with tf.device("/gpu:0"):
x = tf.ones(100, name="x")
idxs = tf.range(100)
for i in range(10):
y = tf.identity(x, name="identity-"+str(i))
x = tf.dynamic_stitch([idxs, idxs], [x, y], name="stitch-"+str(i))
config = tf.ConfigProto(graph_options=tf.GraphOptions(optimizer_options=tf.OptimizerOptions(opt_level=tf.OptimizerOptions.L0)))
sess = tf.InteractiveSession(config=config)
metadata = tf.RunMetadata()
sess.run(x, options=tf.RunOptions(trace_level=tf.RunOptions.FULL_TRACE,
output_partition_graphs=True),
run_metadata=metadata)
timeline = Timeline(metadata.step_stats)
with open("dynamic_stitch_gpu_profile.json", "w") as f:
f.write(timeline.generate_chrome_trace_format())
with open("dynamic_stitch_gpu_profile.pbtxt", "w") as f:
f.write(str(metadata))
# coding: utf-8 import tensorflow as tf import numpy as np with tf.Session() as sess: indices1 = tf.constant([0, 3]) #对应数据元素应放的重组后所在位置 indices2 = tf.constant([1, 2]) data1 = tf.constant([-1, -2]) data2 = tf.constant([-3, -4]) #数据 result = tf.dynamic_stitch([indices1, indices2], [data1, data2]) print(result.eval())
def testErrorIndicesMultiDimensional(self):
indices = [tf.constant([0, 4, 7]), tf.constant([[1, 6, 2, 3, 5]])]
data = [tf.constant([[0, 40, 70]]), tf.constant([10, 60, 20, 30, 50])]
with self.assertRaises(ValueError):
tf.dynamic_stitch(indices, data)
def testErrorDataAndIndicesSizeMismatch(self):
indices = [tf.constant([0, 4, 7]), tf.constant([1, 6, 2, 3, 5])]
data = [tf.constant([0, 40, 70]), tf.constant([10, 60, 20, 30])]
with self.assertRaises(ValueError):
tf.dynamic_stitch(indices, data)
def rotate_image_tensor(image, angle, mode='black'):
"""
Rotates a 3D tensor (HWD), which represents an image by given radian angle.
New image has the same size as the input image.
mode controls what happens to border pixels.
mode = 'black' results in black bars (value 0 in unknown areas)
mode = 'white' results in value 255 in unknown areas
mode = 'ones' results in value 1 in unknown areas
mode = 'repeat' keeps repeating the closest pixel known
"""
s = tf.shape(image)
assert s.get_shape()[0] == 3, "Input needs to be 3D."
assert (mode == 'repeat') or (mode == 'black') or (mode == 'white') or (
mode == 'ones'), "Unknown boundary mode."
image_center = [
tf.floor(tf.cast(s[0] / 2, tf.float32)),
tf.floor(tf.cast(s[1] / 2, tf.float32))
]
# Coordinates of new image
coord1 = tf.range(s[0])
coord2 = tf.range(s[1])
# Create vectors of those coordinates in order to vectorize the image
coord1_vec = tf.tile(coord1, [s[1]])
coord2_vec_unordered = tf.tile(coord2, [s[0]])
coord2_vec_unordered = tf.reshape(coord2_vec_unordered, [s[0], s[1]])
coord2_vec = tf.reshape(tf.transpose(coord2_vec_unordered, [1, 0]), [-1])
# center coordinates since rotation center is supposed to be in the image center
coord1_vec_centered = coord1_vec - tf.to_int32(image_center[0])
coord2_vec_centered = coord2_vec - tf.to_int32(image_center[1])
coord_new_centered = tf.cast(
tf.stack([coord1_vec_centered, coord2_vec_centered]), tf.float32)
# Perform backward transformation of the image coordinates
rot_mat_inv = tf.dynamic_stitch(
[0, 1, 2, 3],
[tf.cos(angle),
tf.sin(angle), -tf.sin(angle),
tf.cos(angle)])
rot_mat_inv = tf.reshape(rot_mat_inv, shape=[2, 2])
coord_old_centered = tf.matmul(rot_mat_inv, coord_new_centered)
# Find nearest neighbor in old image
coord1_old_nn = tf.cast(
tf.round(coord_old_centered[0, :] + image_center[0]), tf.int32)
coord2_old_nn = tf.cast(
tf.round(coord_old_centered[1, :] + image_center[1]), tf.int32)
# Clip values to stay inside image coordinates
if mode == 'repeat':
coord_old1_clipped = tf.minimum(tf.maximum(coord1_old_nn, 0), s[0] - 1)
coord_old2_clipped = tf.minimum(tf.maximum(coord2_old_nn, 0), s[1] - 1)
else:
outside_ind1 = tf.logical_or(tf.greater(coord1_old_nn, s[0] - 1),
tf.less(coord1_old_nn, 0))
outside_ind2 = tf.logical_or(tf.greater(coord2_old_nn, s[1] - 1),
tf.less(coord2_old_nn, 0))
outside_ind = tf.logical_or(outside_ind1, outside_ind2)
coord_old1_clipped = tf.boolean_mask(coord1_old_nn,
tf.logical_not(outside_ind))
coord_old2_clipped = tf.boolean_mask(coord2_old_nn,
tf.logical_not(outside_ind))
coord1_vec = tf.boolean_mask(coord1_vec, tf.logical_not(outside_ind))
coord2_vec = tf.boolean_mask(coord2_vec, tf.logical_not(outside_ind))
coord_old_clipped = tf.cast(
tf.transpose(tf.stack([coord_old1_clipped, coord_old2_clipped]),
[1, 0]), tf.int32)
# Coordinates of the new image
coord_new = tf.transpose(
tf.cast(tf.stack([coord1_vec, coord2_vec]), tf.int32), [1, 0])
num_channels = image.get_shape().as_list()[2]
image_channel_list = tf.split(image, num_channels, axis=2)
image_rotated_channel_list = list()
for image_channel in image_channel_list:
image_chan_new_values = tf.gather_nd(tf.squeeze(image_channel),
coord_old_clipped)
if (mode == 'black') or (mode == 'repeat'):
background_color = 0
elif mode == 'ones':
background_color = 1
elif mode == 'white':
background_color = 255
else:
background_color = 0
image_rotated_channel_list.append(
tf.sparse_to_dense(coord_new, [s[0], s[1]],
image_chan_new_values,
background_color,
validate_indices=False))
image_rotated = tf.transpose(tf.stack(image_rotated_channel_list),
[1, 2, 0])
return image_rotated
def testInt32Gpu(self):
with self.test_session(use_gpu=True):
indices = [tf.convert_to_tensor([0, 1, 2]), tf.convert_to_tensor([2, 3])]
values = [tf.convert_to_tensor([12, 23, 34]), tf.convert_to_tensor([1, 2])]
self.assertAllEqual(
tf.dynamic_stitch(indices, values).eval(), [12, 23, 1, 2])
def get(self):
""" Provides input data to the graph. """
# calculate size of each record (this lists what is contained in the db and how many bytes are occupied)
record_bytes = 0
encoding_bytes = 4
kp_xyz_entries = 3 * self.num_kp
record_bytes += encoding_bytes * kp_xyz_entries
encoding_bytes = 4
kp_uv_entries = 2 * self.num_kp
record_bytes += encoding_bytes * kp_uv_entries
kp_vis_entries = self.num_kp
record_bytes += encoding_bytes * kp_vis_entries
image_bytes = self.image_size[0] * self.image_size[1] * 3
record_bytes += image_bytes
""" READ DATA ITEMS"""
# Start reader
reader = tf.FixedLengthRecordReader(header_bytes=0,
record_bytes=record_bytes)
_, value = reader.read(
tf.train.string_input_producer([self.path_to_db]))
# decode to floats
bytes_read = 0
data_dict = dict()
record_bytes_float32 = tf.decode_raw(value, tf.float32)
# 1. Read keypoint xyz
keypoint_xyz21 = tf.reshape(
tf.slice(record_bytes_float32, [bytes_read // 4],
[kp_xyz_entries]), [self.num_kp, 3])
bytes_read += encoding_bytes * kp_xyz_entries
keypoint_xyz21 /= 1000.0 # scale to meters
keypoint_xyz21 = self.convert_kp(keypoint_xyz21)
# calculate wrist coord
if self.use_wrist_coord:
wrist_xyz = keypoint_xyz21[16, :] + 2.0 * (keypoint_xyz21[0, :] -
keypoint_xyz21[16, :])
keypoint_xyz21 = tf.concat(
[tf.expand_dims(wrist_xyz, 0), keypoint_xyz21[1:, :]], 0)
data_dict['keypoint_xyz21'] = keypoint_xyz21
# 2. Read keypoint uv AND VIS
keypoint_uv_vis21 = tf.reshape(
tf.slice(record_bytes_float32, [bytes_read // 4],
[kp_uv_entries + kp_vis_entries]), [self.num_kp, 3])
bytes_read += encoding_bytes * (kp_uv_entries + kp_vis_entries)
keypoint_uv_vis21 = self.convert_kp(keypoint_uv_vis21)
keypoint_uv21 = keypoint_uv_vis21[:, :2]
keypoint_vis21 = tf.equal(keypoint_uv_vis21[:, 2], 1.0)
# calculate wrist vis
if self.use_wrist_coord:
wrist_vis = tf.logical_or(keypoint_vis21[16], keypoint_vis21[0])
keypoint_vis21 = tf.concat(
[tf.expand_dims(wrist_vis, 0), keypoint_vis21[1:]], 0)
wrist_uv = keypoint_uv21[16, :] + 2.0 * (keypoint_uv21[0, :] -
keypoint_uv21[16, :])
keypoint_uv21 = tf.concat(
[tf.expand_dims(wrist_uv, 0), keypoint_uv21[1:, :]], 0)
data_dict['keypoint_vis21'] = keypoint_vis21
if self.coord_uv_noise:
noise = tf.truncated_normal([42, 2],
mean=0.0,
stddev=self.coord_uv_noise_sigma)
keypoint_uv21 += noise
data_dict['keypoint_uv21'] = keypoint_uv21
# decode to uint8
record_bytes_uint8 = tf.decode_raw(value, tf.uint8)
# 4. Read image
image = tf.reshape(
tf.slice(record_bytes_uint8, [bytes_read], [image_bytes]),
[self.image_size[0], self.image_size[1], 3])
image = tf.cast(image, tf.float32)
bytes_read += image_bytes
# subtract mean
image = image / 255.0 - 0.5
if self.hue_aug:
image = tf.image.random_hue(image, self.hue_aug_max)
data_dict['image'] = image
""" CONSTANTS """
# Camera intrinsics
sx = 822.79041
sy = 822.79041
tx = 318.47345
ty = 250.31296
data_dict['cam_mat'] = tf.constant([[sx, 0.0, tx], [0.0, sy, ty],
[0.0, 0.0, 1.0]])
# Hand side: this dataset only contains left hands
data_dict['hand_side'] = tf.one_hot(tf.constant(0, dtype=tf.int32),
depth=2,
on_value=1.0,
off_value=0.0,
dtype=tf.float32)
assert bytes_read == record_bytes, "Doesnt add up."
""" DEPENDENT DATA ITEMS: XYZ represenations. """
# make coords relative to root joint
kp_coord_xyz_root = keypoint_xyz21[0, :] # this is the palm coord
kp_coord_xyz21_rel = keypoint_xyz21 - kp_coord_xyz_root # relative coords in metric coords
index_root_bone_length = tf.sqrt(
tf.reduce_sum(
tf.square(kp_coord_xyz21_rel[12, :] -
kp_coord_xyz21_rel[11, :])))
data_dict['keypoint_scale'] = index_root_bone_length
data_dict[
'keypoint_xyz21_normed'] = kp_coord_xyz21_rel / index_root_bone_length # normalized by length of 12->11
# calculate local coordinates
kp_coord_xyz21_local = bone_rel_trafo(
data_dict['keypoint_xyz21_normed'])
kp_coord_xyz21_local = tf.squeeze(kp_coord_xyz21_local)
data_dict['keypoint_xyz21_local'] = kp_coord_xyz21_local
# calculate viewpoint and coords in canonical coordinates
kp_coord_xyz21_rel_can, rot_mat = canonical_trafo(
data_dict['keypoint_xyz21_normed'])
kp_coord_xyz21_rel_can, rot_mat = tf.squeeze(
kp_coord_xyz21_rel_can), tf.squeeze(rot_mat)
data_dict['keypoint_xyz21_can'] = kp_coord_xyz21_rel_can
data_dict['rot_mat'] = tf.matrix_inverse(rot_mat)
""" DEPENDENT DATA ITEMS: HAND CROP """
if self.hand_crop:
crop_center = keypoint_uv21[12, ::-1]
# catch problem, when no valid kp available (happens almost never)
crop_center = tf.cond(tf.reduce_all(tf.is_finite(crop_center)),
lambda: crop_center,
lambda: tf.constant([0.0, 0.0]))
crop_center.set_shape([
2,
])
if self.crop_center_noise:
noise = tf.truncated_normal(
[2], mean=0.0, stddev=self.crop_center_noise_sigma)
crop_center += noise
crop_scale_noise = tf.constant(1.0)
if self.crop_scale_noise:
crop_scale_noise = tf.squeeze(
tf.random_uniform([1], minval=1.0, maxval=1.2))
if not self.use_wrist_coord:
wrist_uv = keypoint_uv21[16, :] + 2.0 * (keypoint_uv21[0, :] -
keypoint_uv21[16, :])
keypoint_uv21 = tf.concat(
[tf.expand_dims(wrist_uv, 0), keypoint_uv21[1:, :]], 0)
# select visible coords only
kp_coord_h = tf.boolean_mask(keypoint_uv21[:, 1], keypoint_vis21)
kp_coord_w = tf.boolean_mask(keypoint_uv21[:, 0], keypoint_vis21)
kp_coord_hw = tf.stack([kp_coord_h, kp_coord_w], 1)
# determine size of crop (measure spatial extend of hw coords first)
min_coord = tf.maximum(tf.reduce_min(kp_coord_hw, 0), 0.0)
max_coord = tf.minimum(tf.reduce_max(kp_coord_hw, 0),
self.image_size)
# find out larger distance wrt the center of crop
crop_size_best = 2 * tf.maximum(max_coord - crop_center,
crop_center - min_coord)
crop_size_best = tf.reduce_max(crop_size_best)
crop_size_best = tf.minimum(tf.maximum(crop_size_best, 50.0),
500.0)
# catch problem, when no valid kp available
crop_size_best = tf.cond(
tf.reduce_all(tf.is_finite(crop_size_best)),
lambda: crop_size_best, lambda: tf.constant(200.0))
crop_size_best.set_shape([])
# calculate necessary scaling
scale = tf.cast(self.crop_size, tf.float32) / crop_size_best
scale = tf.minimum(tf.maximum(scale, 1.0), 10.0)
scale *= crop_scale_noise
data_dict['crop_scale'] = scale
if self.crop_offset_noise:
noise = tf.truncated_normal(
[2], mean=0.0, stddev=self.crop_offset_noise_sigma)
crop_center += noise
# Crop image
img_crop = crop_image_from_xy(tf.expand_dims(image, 0),
crop_center, self.crop_size, scale)
data_dict['image_crop'] = tf.squeeze(img_crop)
# Modify uv21 coordinates
crop_center_float = tf.cast(crop_center, tf.float32)
keypoint_uv21_u = (
data_dict['keypoint_uv21'][:, 0] -
crop_center_float[1]) * scale + self.crop_size // 2
keypoint_uv21_v = (
data_dict['keypoint_uv21'][:, 1] -
crop_center_float[0]) * scale + self.crop_size // 2
keypoint_uv21 = tf.stack([keypoint_uv21_u, keypoint_uv21_v], 1)
data_dict['keypoint_uv21'] = keypoint_uv21
# Modify camera intrinsics
scale = tf.reshape(scale, [
1,
])
scale_matrix = tf.dynamic_stitch([
[0], [1], [2], [3], [4], [5], [6], [7], [8]
], [scale, [0.0], [0.0], [0.0], scale, [0.0], [0.0], [0.0], [1.0]])
scale_matrix = tf.reshape(scale_matrix, [3, 3])
crop_center_float = tf.cast(crop_center, tf.float32)
trans1 = crop_center_float[0] * scale - self.crop_size // 2
trans2 = crop_center_float[1] * scale - self.crop_size // 2
trans1 = tf.reshape(trans1, [
1,
])
trans2 = tf.reshape(trans2, [
1,
])
trans_matrix = tf.dynamic_stitch(
[[0], [1], [2], [3], [4], [5], [6], [7], [8]],
[[1.0], [0.0], -trans2, [0.0], [1.0], -trans1, [0.0], [0.0],
[1.0]])
trans_matrix = tf.reshape(trans_matrix, [3, 3])
data_dict['cam_mat'] = tf.matmul(
trans_matrix, tf.matmul(scale_matrix, data_dict['cam_mat']))
""" DEPENDENT DATA ITEMS: Scoremap from the SUBSET of 21 keypoints"""
# create scoremaps from the subset of 2D annoataion
keypoint_hw21 = tf.stack([keypoint_uv21[:, 1], keypoint_uv21[:, 0]],
-1)
scoremap_size = self.image_size
if self.hand_crop:
scoremap_size = (self.crop_size, self.crop_size)
scoremap = self.create_multiple_gaussian_map(keypoint_hw21,
scoremap_size,
self.sigma,
valid_vec=keypoint_vis21)
if self.scoremap_dropout:
scoremap = tf.nn.dropout(scoremap,
self.scoremap_dropout_prob,
noise_shape=[1, 1, 21])
scoremap *= self.scoremap_dropout_prob
data_dict['scoremap'] = scoremap
if self.random_crop_to_size:
tensor_stack = tf.concat([
data_dict['image'],
tf.expand_dims(tf.cast(data_dict['hand_parts'], tf.float32),
-1),
tf.cast(data_dict['hand_mask'], tf.float32)
], 2)
s = tensor_stack.get_shape().as_list()
tensor_stack_cropped = tf.random_crop(
tensor_stack,
[self.random_crop_size, self.random_crop_size, s[2]])
data_dict = dict(
) # delete everything else because the random cropping makes the data invalid anyway
data_dict['image'], data_dict['hand_parts'], data_dict['hand_mask'] = tensor_stack_cropped[:, :, :3],\
tf.cast(tensor_stack_cropped[:, :, 3], tf.int32),\
tf.cast(tensor_stack_cropped[:, :, 4:], tf.int32)
names, tensors = zip(*data_dict.items())
if self.shuffle:
tensors = tf.train.shuffle_batch_join([tensors],
batch_size=self.batch_size,
capacity=100,
min_after_dequeue=50,
enqueue_many=False)
else:
tensors = tf.train.batch_join([tensors],
batch_size=self.batch_size,
capacity=100,
enqueue_many=False)
return dict(zip(names, tensors))
