def testErrorDataAndIndicesSizeMismatch(self): indices = [tf.constant([0, 4, 7]), tf.constant([1, 6, 2, 3, 5])] data = [tf.constant([0, 40, 70]), tf.constant([10, 60, 20, 30])] with self.assertRaises(ValueError): tf.dynamic_stitch(indices, data)
def testErrorDataDimSizeMismatch(self): indices = [tf.constant([0, 4, 5]), tf.constant([1, 6, 2, 3])] data = [tf.constant([[0], [40], [70]]), tf.constant([[10, 11], [60, 61], [20, 21], [30, 31]])] with self.assertRaises(ValueError): tf.dynamic_stitch(indices, data)
def testErrorIndicesMultiDimensional(self): indices = [tf.constant([0, 4, 7]), tf.constant([[1, 6, 2, 3, 5]])] data = [tf.constant([[0, 40, 70]]), tf.constant([10, 60, 20, 30, 50])] with self.assertRaises(ValueError): tf.dynamic_stitch(indices, data)
def loop(q_, mask, mass_, found_): q_list = tf.dynamic_partition(q_, mask, 2) condition_indices = tf.dynamic_partition(tf.range(tf.shape(q_)[0]), mask, 2) # 0 element it False, # 1 element if true p = q_list[1] * (1.0 - mass_) / tf.reduce_sum(q_list[1]) p_new = tf.dynamic_stitch(condition_indices, [q_list[0], p]) # condition verification and mask modification less_mask = tf.cast(tf.less(u, p_new), tf.int32) # 0 when u is bigger than p, 1 when u is less than p condition_indices = tf.dynamic_partition(tf.range(tf.shape(p_new)[0]), less_mask, 2) # 0 when u is bigger than p, 1 when u is less than p split_p_new = tf.dynamic_partition(p_new, less_mask, 2) split_u = tf.dynamic_partition(u, less_mask, 2) alpha = tf.dynamic_stitch(condition_indices, [split_p_new[0], split_u[1]]) mass_ += tf.reduce_sum(split_u[1]) mask = mask * (tf.ones_like(less_mask) - less_mask) found_ = tf.cond(tf.equal(tf.reduce_sum(less_mask), 0), lambda: False, lambda: True) alpha = tf.reshape(alpha, q_.shape) return alpha, mask, mass_, found_
def circular_convolution(v, k): """Computes circular convolution. Args: v: a 1-D `Tensor` (vector) k: a 1-D `Tensor` (kernel) """ size = int(v.get_shape()[0]) kernel_size = int(k.get_shape()[0]) kernel_shift = int(math.floor(kernel_size/2.0)) def loop(idx): if idx < 0: return size + idx if idx >= size : return idx - size else: return idx kernels = [] for i in xrange(size): indices = [loop(i+j) for j in xrange(kernel_shift, -kernel_shift-1, -1)] v_ = tf.gather(v, indices) kernels.append(tf.reduce_sum(v_ * k, 0)) # # code with double loop # for i in xrange(size): # for j in xrange(kernel_size): # idx = i + kernel_shift - j + 1 # if idx < 0: idx = idx + size # if idx >= size: idx = idx - size # w = tf.gather(v, int(idx)) * tf.gather(kernel, j) # output = tf.scatter_add(output, [i], tf.reshape(w, [1, -1])) return tf.dynamic_stitch([i for i in xrange(size)], kernels)
def scheduled_sample_count(ground_truth_x, generated_x, batch_size, scheduled_sample_var): """Sample batch with specified mix of groundtruth and generated data points. Args: ground_truth_x: tensor of ground-truth data points. generated_x: tensor of generated data points. batch_size: batch size scheduled_sample_var: number of ground-truth examples to include in batch. Returns: New batch with num_ground_truth sampled from ground_truth_x and the rest from generated_x. """ num_ground_truth = scheduled_sample_var idx = tf.random_shuffle(tf.range(batch_size)) ground_truth_idx = tf.gather(idx, tf.range(num_ground_truth)) generated_idx = tf.gather(idx, tf.range(num_ground_truth, batch_size)) ground_truth_examps = tf.gather(ground_truth_x, ground_truth_idx) generated_examps = tf.gather(generated_x, generated_idx) output = tf.dynamic_stitch([ground_truth_idx, generated_idx], [ground_truth_examps, generated_examps]) # if batch size is known set it. if isinstance(batch_size, int): output.set_shape([batch_size] + common_layers.shape_list(output)[1:]) return output
def _partition_and_stitch(self, args, func_name): """ args is a list of tensors, to be passed to self.likelihoods.<func_name> args[-1] is the 'Y' argument, which contains the indexes to self.likelihoods. This function splits up the args using dynamic_partition, calls the relevant function on the likelihoods, and re-combines the result. """ # get the index from Y Y = args[-1] ind = Y[:, -1] ind = tf.cast(ind, tf.int32) Y = Y[:, :-1] args[-1] = Y # split up the arguments into chunks corresponding to the relevant likelihoods args = zip(*[tf.dynamic_partition(X, ind, self.num_likelihoods) for X in args]) # apply the likelihood-function to each section of the data with params_as_tensors_for(self, convert=False): funcs = [getattr(lik, func_name) for lik in self.likelihood_list] results = [f(*args_i) for f, args_i in zip(funcs, args)] # stitch the results back together partitions = tf.dynamic_partition(tf.range(0, tf.size(ind)), ind, self.num_likelihoods) results = tf.dynamic_stitch(partitions, results) return results
def indices_to_dense_vector(indices, size, indices_value=1., default_value=0, dtype=tf.float32): """Creates dense vector with indices set to specific value and rest to zeros. This function exists because it is unclear if it is safe to use tf.sparse_to_dense(indices, [size], 1, validate_indices=False) with indices which are not ordered. This function accepts a dynamic size (e.g. tf.shape(tensor)[0]) Args: indices: 1d Tensor with integer indices which are to be set to indices_values. size: scalar with size (integer) of output Tensor. indices_value: values of elements specified by indices in the output vector default_value: values of other elements in the output vector. dtype: data type. Returns: dense 1D Tensor of shape [size] with indices set to indices_values and the rest set to default_value. """ size = tf.to_int32(size) zeros = tf.ones([size], dtype=dtype) * default_value values = tf.ones_like(indices, dtype=dtype) * indices_value return tf.dynamic_stitch([tf.range(size), tf.to_int32(indices)], [zeros, values])
def testSumGradArgs(self): with self.test_session(use_gpu=False): indices = [tf.convert_to_tensor([0, 1, 2, 3]), tf.convert_to_tensor([2, 3])] values = [tf.convert_to_tensor([2, 3, 5, 7]), tf.convert_to_tensor([1, 1])] self.assertAllEqual( tf.dynamic_stitch(indices, values).eval(), [2, 3, 1, 1])
def _create_regression_targets(self, anchors, groundtruth_boxes, match): """Returns a regression target for each anchor. Args: anchors: a BoxList representing N anchors groundtruth_boxes: a BoxList representing M groundtruth_boxes match: a matcher.Match object Returns: reg_targets: a float32 tensor with shape [N, box_code_dimension] """ matched_anchor_indices = match.matched_column_indices() unmatched_ignored_anchor_indices = (match. unmatched_or_ignored_column_indices()) matched_gt_indices = match.matched_row_indices() matched_anchors = box_list_ops.gather(anchors, matched_anchor_indices) matched_gt_boxes = box_list_ops.gather(groundtruth_boxes, matched_gt_indices) matched_reg_targets = self._box_coder.encode(matched_gt_boxes, matched_anchors) unmatched_ignored_reg_targets = tf.tile( self._default_regression_target(), tf.stack([tf.size(unmatched_ignored_anchor_indices), 1])) reg_targets = tf.dynamic_stitch( [matched_anchor_indices, unmatched_ignored_anchor_indices], [matched_reg_targets, unmatched_ignored_reg_targets]) # TODO: summarize the number of matches on average. return reg_targets
def _create_classification_targets(self, groundtruth_labels, match): """Create classification targets for each anchor. Assign a classification target of for each anchor to the matching groundtruth label that is provided by match. Anchors that are not matched to anything are given the target self._unmatched_cls_target Args: groundtruth_labels: a tensor of shape [num_gt_boxes, d_1, ... d_k] with labels for each of the ground_truth boxes. The subshape [d_1, ... d_k] can be empty (corresponding to scalar labels). match: a matcher.Match object that provides a matching between anchors and groundtruth boxes. Returns: cls_targets: a float32 tensor with shape [num_anchors, d_1, d_2 ... d_k], where the subshape [d_1, ..., d_k] is compatible with groundtruth_labels which has shape [num_gt_boxes, d_1, d_2, ... d_k]. """ matched_anchor_indices = match.matched_column_indices() unmatched_ignored_anchor_indices = (match. unmatched_or_ignored_column_indices()) matched_gt_indices = match.matched_row_indices() matched_cls_targets = tf.gather(groundtruth_labels, matched_gt_indices) ones = self._unmatched_cls_target.shape.ndims * [1] unmatched_ignored_cls_targets = tf.tile( tf.expand_dims(self._unmatched_cls_target, 0), tf.stack([tf.size(unmatched_ignored_anchor_indices)] + ones)) cls_targets = tf.dynamic_stitch( [matched_anchor_indices, unmatched_ignored_anchor_indices], [matched_cls_targets, unmatched_ignored_cls_targets]) return cls_targets
def CircularConvolution(vector, kernel): size = int(vector.get_shape()[0]) kernel_size = int(kernel.get_shape()[0]) kernel_shift = int(math.floor(kernel_size/2.0)) output = tf.zeros_like(vector) def loop(idx): if idx < 0: return size + idx if idx >= size : return idx - size else: return idx kernels = [] for i in xrange(size): indices = [loop(i+j) for j in xrange(kernel_shift, -kernel_shift-1, -1)] v = tf.gather(vector, indices) kernels.append(tf.reduce_sum(v * kernel, 0, keep_dims=True)) output = tf.dynamic_stitch([[i] for i in xrange(size)], kernels) # # code with double loop # for i in xrange(size): # for j in xrange(kernel_size): # idx = i + kernel_shift - j + 1 # if idx < 0: idx = idx + size # if idx >= size: idx = idx - size # w = tf.gather(vector, int(idx)) * tf.gather(kernel, j) # output = tf.scatter_add(output, [i], tf.reshape(w, [1, -1])) return output
def test_dynamic_stitch(sess): x = tf.zeros((1, 3)) y = tf.dynamic_stitch([[0], [0]], [x, tf.ones((1, 3))]) z = tf.gather(y, [0]) with sess.as_default(): analytic, numeric = tf.test.compute_gradient(x, (1, 3), z, (1, 3)) assert np.allclose(analytic, numeric)
def testOneListOneDimensional(self): with self.test_session(): indices = [tf.constant([1, 6, 2, 3, 5, 0, 4, 7])] data = [tf.constant([10, 60, 20, 30, 50, 0, 40, 70])] stitched_t = tf.dynamic_stitch(indices, data) stitched_val = stitched_t.eval() self.assertAllEqual([0, 10, 20, 30, 40, 50, 60, 70], stitched_val) # Dimension 0 is determined by the max index in indices, so we # can only infer that the output is a vector of some unknown # length. self.assertEqual([None], stitched_t.get_shape().as_list())
def testStitchOrder(self): with self.test_session(): indices = [] np_values = [] values = [] for _ in range(10): indices.extend([tf.convert_to_tensor(np.arange(100).astype(np.int32))]) np_values.extend([np.random.uniform(size=100)]) values.extend([tf.convert_to_tensor(np_values[-1])]) stitched = tf.dynamic_stitch(indices, values).eval() self.assertAllEqual(np_values[-1], stitched)
def testScalar(self): with self.test_session(): indices = [tf.constant(0), tf.constant(1)] data = [tf.constant(40), tf.constant(60)] for step in -1, 1: stitched_t = tf.dynamic_stitch(indices[::step], data) stitched_val = stitched_t.eval() self.assertAllEqual([40, 60][::step], stitched_val) # Dimension 0 is determined by the max index in indices, so we # can only infer that the output is a vector of some unknown # length. self.assertEqual([None], stitched_t.get_shape().as_list())
def __call__(self, X): ind = tf.gather(tf.transpose(X), tf.shape(X)[1]-1) # ind = X[:,-1] ind = tf.cast(ind, tf.int32) X = tf.transpose(tf.gather(tf.transpose(X), tf.range(0, tf.shape(X)[1]-1))) # X = X[:,:-1] # split up X into chunks corresponding to the relevant likelihoods x_list = tf.dynamic_partition(X, ind, len(self.meanfunction_list)) # apply the likelihood-function to each section of the data results = [m(x) for x, m in zip(x_list, self.meanfunction_list)] # stitch the results back together partitions = tf.dynamic_partition(tf.range(0, tf.size(ind)), ind, len(self.meanfunction_list)) return tf.dynamic_stitch(partitions, results)
def split_apply_merge(inp, partitions, fns): """Split input according to partitions. Pass results through fns and merge. Args: inp: the input vector partitions: tensor of same length as input vector, having values 0, 1 fns: the two functions. Returns: the vector routed, where routed[i] = fns[partitions[i]](inp[i]) """ new_inputs = tf.dynamic_partition(inp, partitions, len(fns)) new_outputs = [fns[i](x) for i, x in enumerate(new_inputs)] new_indices = tf.dynamic_partition(tf.range(0, inp.get_shape()[0]), partitions, len(fns)) return tf.dynamic_stitch(new_indices, new_outputs)
def replace_features(coarse_features, fine_features, replace_idxs): """ Replace fine features with the corresponding coarse features Trick. use tf.dynamic_stitch ops """ # TODO: simplify indexing def _convert_to_1d_idxs(src_idxs): """ Convert 2D idxs to 1D idxs within 1D tensor whose shape is (b*h*w*c) """ batch_idx_len = map_channel.value * map_width.value * map_height.value batch_idx_base = [i*batch_idx_len for i in xrange(batch_size.value)] batch_1d = map_channel.value * map_width.value * src_idxs[:,0] + \ map_channel.value * src_idxs[:,1] batch_1d = tf.add(batch_1d,batch_idx_base) flat_idxs = [batch_1d+i for i in xrange(map_channel.value)] flat_idxs = tf.reshape(tf.transpose(tf.pack(flat_idxs)), [-1]) return flat_idxs batch_size, map_height, map_width, map_channel = coarse_features.get_shape() # flatten coarse features flat_coarse_features = tf.reshape(coarse_features, [batch_size.value,-1]) flat_coarse_features = tf.reshape(flat_coarse_features, [-1]) # flatten fine features flat_fine_features = [tf.reshape(i,[-1]) for i in fine_features] flat_fine_features = tf.concat(0,flat_fine_features) flat_fine_idxs = [_convert_to_1d_idxs(i) for i in replace_idxs] flat_fine_idxs = tf.concat(0,flat_fine_idxs) # extract coarse features to be replaced # this is required for hint-based training flat_coarse_replaced = tf.gather(flat_coarse_features, flat_fine_idxs, validate_indices=False) merged = tf.dynamic_stitch([tf.range(0,flat_coarse_features.get_shape()[0]),flat_fine_idxs], [flat_coarse_features,flat_fine_features]) merged = tf.reshape(merged,coarse_features.get_shape()) return merged, flat_coarse_replaced, flat_fine_features
def _match_when_rows_are_non_empty(): """Performs matching when the rows of similarity matrix are non empty. Returns: matches: int32 tensor indicating the row each column matches to. """ # Matches for each column matches = tf.argmax(similarity_matrix, 0) # Deal with matched and unmatched threshold if self._matched_threshold is not None: # Get logical indices of ignored and unmatched columns as tf.int64 matched_vals = tf.reduce_max(similarity_matrix, 0) below_unmatched_threshold = tf.greater(self._unmatched_threshold, matched_vals) between_thresholds = tf.logical_and( tf.greater_equal(matched_vals, self._unmatched_threshold), tf.greater(self._matched_threshold, matched_vals)) if self._negatives_lower_than_unmatched: matches = self._set_values_using_indicator(matches, below_unmatched_threshold, -1) matches = self._set_values_using_indicator(matches, between_thresholds, -2) else: matches = self._set_values_using_indicator(matches, below_unmatched_threshold, -2) matches = self._set_values_using_indicator(matches, between_thresholds, -1) if self._force_match_for_each_row: forced_matches_ids = tf.cast(tf.argmax(similarity_matrix, 1), tf.int32) # Set matches[forced_matches_ids] = [0, ..., R], R is number of rows. row_range = tf.range(tf.shape(similarity_matrix)[0]) col_range = tf.range(tf.shape(similarity_matrix)[1]) forced_matches_values = tf.cast(row_range, matches.dtype) keep_matches_ids, _ = tf.setdiff1d(col_range, forced_matches_ids) keep_matches_values = tf.gather(matches, keep_matches_ids) matches = tf.dynamic_stitch( [forced_matches_ids, keep_matches_ids], [forced_matches_values, keep_matches_values]) return tf.cast(matches, tf.int32)
def circular_convolution(v, k): size = int(v.get_shape()[0]) kernel_size = int(k.get_shape()[0]) kernel_shift = int(math.floor(kernel_size/2.0)) def loop(idx): if idx < 0: return size + idx if idx >= size : return idx - size else: return idx kernels = [] for i in xrange(size): indices = [loop(i+j) for j in xrange(kernel_shift, -kernel_shift-1, -1)] v_ = tf.gather(v, indices) kernels.append(tf.reduce_sum(v_ * k, 0)) return tf.dynamic_stitch([i for i in xrange(size)], kernels)
def testHigherRank(self): with self.test_session() as sess: indices = [tf.constant(6), tf.constant([4, 1]), tf.constant([[5, 2], [0, 3]])] data = [tf.constant([61, 62]), tf.constant([[41, 42], [11, 12]]), tf.constant([[[51, 52], [21, 22]], [[1, 2], [31, 32]]])] stitched_t = tf.dynamic_stitch(indices, data) stitched_val = stitched_t.eval() correct = 10 * np.arange(7)[:, None] + [1, 2] self.assertAllEqual(correct, stitched_val) self.assertEqual([None, 2], stitched_t.get_shape().as_list()) # Test gradients stitched_grad = 7 * stitched_val grads = tf.gradients(stitched_t, indices + data, stitched_grad) self.assertEqual(grads[:3], [None] * 3) # Indices have no gradients for datum, grad in zip(data, sess.run(grads[3:])): self.assertAllEqual(7 * datum.eval(), grad)
def testSimpleTwoDimensional(self): with self.test_session(): indices = [tf.constant([0, 4, 7]), tf.constant([1, 6]), tf.constant([2, 3, 5])] data = [tf.constant([[0, 1], [40, 41], [70, 71]]), tf.constant([[10, 11], [60, 61]]), tf.constant([[20, 21], [30, 31], [50, 51]])] stitched_t = tf.dynamic_stitch(indices, data) stitched_val = stitched_t.eval() self.assertAllEqual( [[0, 1], [10, 11], [20, 21], [30, 31], [40, 41], [50, 51], [60, 61], [70, 71]], stitched_val) # Dimension 0 is determined by the max index in indices, so we # can only infer that the output is a matrix with 2 columns and # some unknown number of rows. self.assertEqual([None, 2], stitched_t.get_shape().as_list())
def scheduled_sample(ground_truth_x, generated_x, batch_size, num_ground_truth): """Sample batch with specified mix of ground truth and generated data points. Args: ground_truth_x: tensor of ground-truth data points. generated_x: tensor of generated data points. batch_size: batch size num_ground_truth: number of ground-truth examples to include in batch. Returns: New batch with num_ground_truth sampled from ground_truth_x and the rest from generated_x. """ idx = tf.random_shuffle(tf.range(int(batch_size))) ground_truth_idx = tf.gather(idx, tf.range(num_ground_truth)) generated_idx = tf.gather(idx, tf.range(num_ground_truth, int(batch_size))) ground_truth_examps = tf.gather(ground_truth_x, ground_truth_idx) generated_examps = tf.gather(generated_x, generated_idx) return tf.dynamic_stitch([ground_truth_idx, generated_idx], [ground_truth_examps, generated_examps])
def scheduled_sample(ground_truth_x, generated_x, batch_size, num_ground_truth): """Sample batch with specified mix of ground truth and generated data_files points. Args: ground_truth_x: tensor of ground-truth data_files points. generated_x: tensor of generated data_files points. batch_size: batch size num_ground_truth: number of ground-truth examples to include in batch. Returns: New batch with num_ground_truth sampled from ground_truth_x and the rest from generated_x. """ idx = tf.random_shuffle(tf.range(int(batch_size))) ground_truth_idx = tf.gather(idx, tf.range(num_ground_truth)) generated_idx = tf.gather(idx, tf.range(num_ground_truth, int(batch_size))) ground_truth_examps = tf.gather(ground_truth_x, ground_truth_idx) generated_examps = tf.gather(generated_x, generated_idx) return tf.dynamic_stitch([ground_truth_idx, generated_idx], [ground_truth_examps, generated_examps])
def _build_output_graph(self, rep, t, dim_in, dropout_representation, dropout_regression, num_regression_layers): ''' Construct output/regression layers ''' i0, i1 = tf.to_int32(tf.where(t < 1)[:, 0]), tf.to_int32( tf.where(t > 0)[:, 0]) rep0, rep1 = tf.gather(rep, i0), tf.gather(rep, i1) y0, weights_out0, weights_pred0 = self._build_output( rep0, dim_in, dropout_representation, dropout_regression, num_regression_layers) y1, weights_out1, weights_pred1 = self._build_output( rep1, dim_in, dropout_representation, dropout_regression, num_regression_layers) y = tf.dynamic_stitch([i0, i1], [y0, y1]) weights_out = weights_out0 + weights_out1 weights_pred = weights_pred0 + weights_pred1 y_concat = tf.concat([y0, y1], axis=0) return y, y_concat, weights_out, weights_pred
def testSimpleTwoDimensional(self): with self.test_session(): indices = [ tf.constant([0, 4, 7]), tf.constant([1, 6]), tf.constant([2, 3, 5]) ] data = [ tf.constant([[0, 1], [40, 41], [70, 71]]), tf.constant([[10, 11], [60, 61]]), tf.constant([[20, 21], [30, 31], [50, 51]]) ] stitched_t = tf.dynamic_stitch(indices, data) stitched_val = stitched_t.eval() self.assertAllEqual([[0, 1], [10, 11], [20, 21], [30, 31], [40, 41], [50, 51], [60, 61], [70, 71]], stitched_val) # Dimension 0 is determined by the max index in indices, so we # can only infer that the output is a matrix with 2 columns and # some unknown number of rows. self.assertEqual([None, 2], stitched_t.get_shape().as_list())
def _stitch_mat_from_vecs(vector_list): """ Stitches a given list of vectors into a 3x3 matrix. Input: vector_list: list of 9 tensors, which will be stitched into a matrix. list contains matrix elements in a row-first fashion (m11, m12, m13, m21, m22, m23, m31, m32, m33). Length of the vectors has to be the same, because it is interpreted as batch dimension. """ assert len(vector_list ) == 9, "There have to be exactly 9 tensors in vector_list." batch_size = vector_list[0].get_shape().as_list()[0] vector_list = [tf.reshape(x, [1, batch_size]) for x in vector_list] trafo_matrix = tf.dynamic_stitch( [[0], [1], [2], [3], [4], [5], [6], [7], [8]], vector_list) trafo_matrix = tf.reshape(trafo_matrix, [3, 3, batch_size]) trafo_matrix = tf.transpose(trafo_matrix, [2, 0, 1]) return trafo_matrix
def f(params_1d): """A function that can be used by tfp.optimizer.lbfgs_minimize. This function is created by function_factory. Args: params_1d [in]: a 1D tf.Tensor. Returns: A scalar loss and the gradients w.r.t. the `params_1d`. """ assign_new_model_parameters(params_1d) loss_value, grads = model.get_grad(*args) grads = tf.dynamic_stitch(idx, grads) # print out iteration & loss f.iter.assign_add(1) if f.iter % model.print_epoch == 0: tf.print("Iter:", f.iter, "loss:", loss_value) return loss_value, grads
def _assign_rbox_target(self, anchors, groundtruth_boxes, groundtruth_rotations, match): ''' Args: anchors: a BoxList representing N anchors groundtruth_boxes: a BoxList representing M groundtruth boxes groundtruth_rotations: Returns: reg_targets: a float32 tensor with shape [num_anchors, 5] reg_weights: a float32 tensor with shape [num_anchors] ''' if not isinstance(anchors, box_list.BoxList): raise ValueError('anchors must be an BoxList') if not isinstance(groundtruth_boxes, box_list.BoxList): raise ValueError('groundtruth_boxes must be an BoxList') matched_anchor_indices = match.matched_column_indices() unmatched_ignored_anchor_indices = (match. unmatched_or_ignored_column_indices()) matched_gt_indices = match.matched_row_indices() matched_anchors = box_list_ops.gather(anchors, matched_anchor_indices) matched_gt_boxes = box_list_ops.gather(groundtruth_boxes, matched_gt_indices) matched_gt_rotations = tf.gather(groundtruth_rotations, matched_gt_indices) matched_reg_targets = self._box_coder.encode(matched_gt_boxes, matched_gt_rotations, matched_anchors) default_target = tf.constant([5*[0]], tf.float32) unmatched_ignored_reg_targets = tf.tile( default_target, tf.stack([tf.size(unmatched_ignored_anchor_indices), 1])) reg_targets = tf.dynamic_stitch( [matched_anchor_indices, unmatched_ignored_anchor_indices], [matched_reg_targets, unmatched_ignored_reg_targets]) reg_weights = tf.cast(match.matched_column_indicator(), tf.float32) return reg_targets, reg_weights
def _transform_col(self, data, interpolators, inverse): if not inverse: lower_bound_x = interpolators.low_quantile upper_bound_x = interpolators.high_quantile lower_bound_y = self.dtype(0) upper_bound_y = self.dtype(1) else: lower_bound_x = self.dtype(0) upper_bound_x = self.dtype(1) lower_bound_y = interpolators.low_quantile upper_bound_y = interpolators.high_quantile lower_bounds_mask = (data - self.BOUNDS_THRESHOLD < lower_bound_x) upper_bounds_mask = (data + self.BOUNDS_THRESHOLD > upper_bound_x) in_range_mask = tf.logical_not(tf.logical_or(lower_bounds_mask, upper_bounds_mask)) data_in_range = tf.boolean_mask(data, in_range_mask) if not inverse: interpolated = 0.5*( interpolators.quantiles_to_references_forward.interp(data_in_range) - interpolators.quantiles_to_references_backward.interp(-data_in_range)) else: interpolated = interpolators.references_to_quantiles.interp(data_in_range) res = tf.dynamic_stitch( [nonzero(upper_bounds_mask), nonzero(in_range_mask), nonzero(lower_bounds_mask)], [tf.fill(tf.count_nonzero(upper_bounds_mask, keepdims=True), upper_bound_y), interpolated, tf.fill(tf.count_nonzero(lower_bounds_mask, keepdims=True), lower_bound_y)]) if not inverse: res = self.output_distribution.quantile(res) clip_min = self.output_distribution.quantile(tf.constant( self.BOUNDS_THRESHOLD - np.spacing(1), dtype=self.dtype)) clip_max = self.output_distribution.quantile(tf.constant( 1 - (self.BOUNDS_THRESHOLD - np.spacing(1)), dtype=self.dtype)) res = tf.clip_by_value(res, clip_min, clip_max) return res
def dce_loss(features, labels, centers, T, flags): # size = features.shape[0] dist = distance(features, centers, flags) #50,10,5 mask = tf.one_hot(labels, flags.num_classes) mask_rep = tf.tile(tf.expand_dims(mask, axis=2), multiples=[1, 1, flags.num_protos]) dist_not_this_class, dist_this_class = tf.dynamic_partition( dist, tf.cast(mask_rep, tf.int32), 2) dist_this_class = tf.reshape(dist_this_class, (-1, 1, flags.num_protos)) dist_not_this_class = tf.reshape( dist_not_this_class, (-1, flags.num_classes - 1, flags.num_protos)) dist_this_class_max = tf.reduce_min(dist_this_class, axis=2) dist_not_this_class_min = tf.reduce_max(dist_not_this_class, axis=2) dist_not_this_class_min = tf.reshape(dist_not_this_class_min, (-1, )) dist_this_class_max = tf.reshape(dist_this_class_max, (-1, )) # import ipdb # ipdb.set_trace() condition_indices= tf.dynamic_partition( tf.range(flags.num_classes * tf.shape(dist)[0]),\ tf.cast(tf.reshape(mask, (-1,)), tf.int32), 2) logits = tf.dynamic_stitch(condition_indices, [dist_not_this_class_min, dist_this_class_max]) logits = tf.reshape(logits, (-1, flags.num_classes)) logits = -logits / T mean_loss = softmax_loss(logits, labels) # # for another loss calculation. # dist_not_this_class_max = tf.reduce_min(dist_not_this_class, axis = 2) # dist_not_this_class_max = tf.reshape(dist_not_this_class_max, (-1,)) # condition_indices= tf.dynamic_partition( # tf.range(flags.num_classes * dist.shape[0]),\ # tf.cast(tf.reshape(mask, (-1,)), tf.int32), 2) # logits = tf.dynamic_stitch(condition_indices, [dist_not_this_class_max, dist_this_class_max]) # logits = tf.reshape(logits, (-1,flags.num_classes)) # logits = -logits / T # mean_loss_2 = softmax_loss(logits, labels) return mean_loss #+ mean_loss_2
def f(params_1d): """A function that can be used by tfp.optimizer.lbfgs_minimize. This function is created by function_factory. Args: params_1d [in]: a 1D tf.Tensor. Returns: A scalar loss and the gradients the `params_1d`. """ # use GradientTape so that we can calculate the gradient of loss with tf.GradientTape(persistent=True) as tape1: # update the parameters in the model assign_new_model_parameters(params_1d) with tf.GradientTape(persistent=True) as tape2: with tf.GradientTape(persistent=True) as tape3: Psi = model(tf.concat([x, y], 1)) Psi_x, Psi_y = tape3.gradient(Psi, [x, y]) Psi_xx = tape2.gradient(Psi_x, x) Psi_yy = tape2.gradient(Psi_y, y) loss_e = residual(Psi_xx=Psi_xx, Psi_yy=Psi_yy, x=x, y=y) loss_b = loss_bc(model=model, x=x_train_bc, y=y_train_bc, x_d=x_train_bc_d, y_d=y_train_bc_d, bc_train=bc_train) # calculate the loss loss_value = loss_e + loss_b # calculate gradients and convert to 1D tf.Tensor grads = tape1.gradient(loss_value, model.trainable_variables) grads = tf.dynamic_stitch(idx, grads) # print out iteration & loss f.iter.assign_add(1) if f.iter % 2000 == 0: tf.print("Iter:", f.iter, "loss:", loss_value) # store loss value so we can retrieve later tf.py_function(f.history.append, inp=[[loss_value, loss_e, loss_b]], Tout=[]) return loss_value, grads
def _build_output_graph(self, rep, t, dim_in, dim_out, do_out, FLAGS): ''' Construct output/regression layers ''' if FLAGS.split_output: i0 = tf.to_int32(tf.where(t < 1)[:,0]) i1 = tf.to_int32(tf.where(t > 0)[:,0]) rep0 = tf.gather(rep, i0) rep1 = tf.gather(rep, i1) y0, weights_out0, weights_pred0 = self._build_output(rep0, dim_in, dim_out, do_out, FLAGS) y1, weights_out1, weights_pred1 = self._build_output(rep1, dim_in, dim_out, do_out, FLAGS) y = tf.dynamic_stitch([i0, i1], [y0, y1]) weights_out = weights_out0 + weights_out1 weights_pred = weights_pred0 + weights_pred1 else: h_input = tf.concat(1,[rep, t]) y, weights_out, weights_pred = self._build_output(h_input, dim_in+1, dim_out, do_out, FLAGS) return y, weights_out, weights_pred
def f(params_1d): """A function that can be used by tfp.optimizer.lbfgs_minimize. This function is created by function_factory. Args: params_1d [in]: a 1D tf.Tensor. Returns: A scalar loss and the gradients w.r.t. the `params_1d`. """ # use GradientTape so that we can calculate the gradient of loss w.r.t. parameters with tf.GradientTape() as tape: # update the parameters in the model assign_new_model_parameters(params_1d) # calculate the loss loss_value = loss_func(model) # calculate gradients and convert to 1D tf.Tensor grads = tape.gradient(loss_value, model.trainable_variables) grads = tf.dynamic_stitch(idx, grads) # print out iteration & loss f.iter.assign_add(1) tf.print("Iter:", f.iter, "loss:", loss_value) if self.callback_list is not None: info_dict = { 'iter': f.iter, 'loss': loss_value, 'grad': grads, } for callback in self.callback_list: callback(model, info_dict=info_dict) return loss_value, grads
def lookup(self, keys, name=None): """Looks up `keys` in a table, outputs the corresponding values.""" if keys.dtype.base_dtype != self._key_dtype: raise TypeError( "Signature mismatch. Keys must be dtype %s, got %s." % (self._key_dtype, keys.dtype)) self._check_keys(keys) num_shards = self._num_shards if num_shards == 1: return self._table_shards[0].lookup(keys, name=name) shard_indices = self._shard_indices(keys) key_shards = tf.dynamic_partition(keys, shard_indices, num_shards) value_shards = [ self._table_shards[i].lookup(key_shards[i], name=name) for i in range(num_shards) ] num_keys = tf.compat.v1.shape(keys)[0] original_indices = tf.range(num_keys) partitioned_indices = tf.dynamic_partition(original_indices, shard_indices, num_shards) return tf.dynamic_stitch(partitioned_indices, value_shards)
def minimize(self, loss_func, model): optim_func = self._function_wrapper(loss_func, model) # convert initial model parameters to a 1D tf.Tensor init_params = tf.dynamic_stitch(optim_func.idx, model.trainable_variables) # train the model with L-BFGS solver results = tfp.optimizer.lbfgs_minimize( value_and_gradients_function=optim_func, initial_position=init_params, max_iterations=self.max_iterations, tolerance=self.tolerance, x_tolerance=self.tolerance, f_relative_tolerance=self.tolerance, **self.lbfgs_kwargs) # after training, the final optimized parameters are still in results.position # so we have to manually put them back to the model optim_func.assign_new_model_parameters(results.position) print("L-BFGS complete, and parameters updated !") return model
def _create_targets(anchors, groundtruth_boxes, matches): """Returns regression targets for each anchor. Arguments: anchors: a float tensor with shape [num_anchors, 4]. groundtruth_boxes: a float tensor with shape [N, 4]. matches: a int tensor with shape [num_anchors]. Returns: reg_targets: a float tensor with shape [num_anchors, 4]. """ matched_anchor_indices = tf.where(tf.greater_equal( matches, 0)) # shape [num_matches, 1] matched_anchor_indices = tf.squeeze(matched_anchor_indices, axis=1) matched_gt_indices = tf.gather( matches, matched_anchor_indices) # shape [num_matches] matched_anchors = tf.gather( anchors, matched_anchor_indices) # shape [num_matches, 4] matched_gt_boxes = tf.gather(groundtruth_boxes, matched_gt_indices) # shape [num_matches, 4] matched_reg_targets = encode(matched_gt_boxes, matched_anchors) # shape [num_matches, 4] unmatched_anchor_indices = tf.where(tf.equal(matches, -1)) unmatched_anchor_indices = tf.squeeze(unmatched_anchor_indices, axis=1) # it has shape [num_anchors - num_matches] unmatched_reg_targets = tf.zeros([tf.size(unmatched_anchor_indices), 4]) # it has shape [num_anchors - num_matches, 4] matched_anchor_indices = tf.to_int32(matched_anchor_indices) unmatched_anchor_indices = tf.to_int32(unmatched_anchor_indices) reg_targets = tf.dynamic_stitch( [matched_anchor_indices, unmatched_anchor_indices], [matched_reg_targets, unmatched_reg_targets]) return reg_targets
def K(self, X, X2=None, presliced=False): if X2 is None: X2 = X ind_X = X[:, -1] ind_X = tf.cast(ind_X, tf.int32) ind_X2 = X2[:, -1] ind_X2 = tf.cast(ind_X2, tf.int32) X = X[:, :-1] X2 = X2[:, :-1] if not presliced: X, X2 = self._slice(X, X2) ind_X_parts = tf.dynamic_partition(tf.range(0, tf.size(ind_X)), ind_X, self.output_dim) ind_X2_parts = tf.dynamic_partition(tf.range(0, tf.size(ind_X2)), ind_X2, self.output_dim) Ks = [] for k, p, p2 in zip(self.kernels, ind_X_parts, ind_X2_parts): gram = k.K(tf.gather(X, p), tf.gather(X2, p2)) Ks.append(gram) N = tf.shape(X)[0] N2 = tf.shape(X2)[0] Ks_scattered = [] for gram, p, p2 in zip(Ks, ind_X_parts, ind_X2_parts): p2 = p2[:, None] shape = tf.stack([N2, tf.shape(p)[0]]) scattered = tf.transpose( tf.scatter_nd(p2, tf.transpose(gram), shape)) Ks_scattered.append(scattered) return tf.dynamic_stitch(ind_X_parts, Ks_scattered)
def _update_alloc_and_usage_vectors(self, pre_write_weightings, pre_read_weightings, pre_usage_vector, free_gates): retention_vector = tf.reduce_prod(1 - free_gates * pre_read_weightings, axis=1, keepdims=False, name='retention_prod') usage_vector = ( pre_usage_vector + pre_write_weightings - pre_usage_vector * pre_write_weightings) * retention_vector sorted_usage, free_list = tf.nn.top_k(-1 * usage_vector, self.h_N) sorted_usage = -1 * sorted_usage cumprod_sorted_usage = tf.cumprod(sorted_usage, axis=1, exclusive=True) corrected_free_list = free_list + self.const_batch_memory_range cumprod_sorted_usage_re = [ tf.reshape(cumprod_sorted_usage, [ -1, ]), ] corrected_free_list_re = [ tf.reshape(corrected_free_list, [-1]), ] stitched_usage = tf.dynamic_stitch(corrected_free_list_re, cumprod_sorted_usage_re, name=None) stitched_usage = tf.reshape(stitched_usage, [self.h_B, self.h_N]) alloc_weighting = (1 - usage_vector) * stitched_usage return alloc_weighting, usage_vector
def log_inv_probit(x): ''' This produces the correct asymptotic behaviour for log(inv_probit(x)) for large negative x ''' ## All x's that are below a negative threshold are calculated using an asymptotic approximation. ## All x's above that threshold are calculated normally. # -1.45026 is the zero-crossing of the error between the approximation and truth, minimising the discontinuous jump when moving between the two. split_mask = tf.cast(tf.greater(x, -1.45026), tf.int32) split_idx = tf.dynamic_partition(tf.range(tf.shape(x)[0]), split_mask, 2) # Split the x values to the different piecewise regions x_split = tf.dynamic_partition(x, split_mask, 2) # Approximation of log (inv_probit (x)) for x < -1.45026 y_0 = tf.log(1 - tf.exp(1.4 * x_split[0])) - (x_split[0]**2) / 2 - tf.log( -x_split[0]) - np.log(1.136 * np.sqrt(2 * np.pi)) # Use tf.log (inv_probit(x)) as it is accurate for x > -1.45026 y_1 = tf.log(inv_probit(x_split[1])) # Stitch the results back together return tf.dynamic_stitch(split_idx, [y_0, y_1])
def observed_data_layer(observed_data, missing_data, condition_indices, output_dim, name, reuse): #Train a layer with the observed data and reuse it for the missing data obs_output = tf.layers.dense( inputs=observed_data, units=output_dim, activation=None, kernel_initializer=tf.random_normal_initializer(stddev=0.05), name=name, reuse=reuse, trainable=True) miss_output = tf.layers.dense( inputs=missing_data, units=output_dim, activation=None, kernel_initializer=tf.random_normal_initializer(stddev=0.05), name=name, reuse=True, trainable=False) #Join back the data output = tf.dynamic_stitch(condition_indices, [miss_output, obs_output]) return output
def testHigherRank(self): with self.test_session() as sess: indices = [ tf.constant(6), tf.constant([4, 1]), tf.constant([[5, 2], [0, 3]]) ] data = [ tf.constant([61, 62]), tf.constant([[41, 42], [11, 12]]), tf.constant([[[51, 52], [21, 22]], [[1, 2], [31, 32]]]) ] stitched_t = tf.dynamic_stitch(indices, data) stitched_val = stitched_t.eval() correct = 10 * np.arange(7)[:, None] + [1, 2] self.assertAllEqual(correct, stitched_val) self.assertEqual([None, 2], stitched_t.get_shape().as_list()) # Test gradients stitched_grad = 7 * stitched_val grads = tf.gradients(stitched_t, indices + data, stitched_grad) self.assertEqual(grads[:3], [None] * 3) # Indices have no gradients for datum, grad in zip(data, sess.run(grads[3:])): self.assertAllEqual(7 * datum.eval(), grad)
def scheduled_sample(ground_truth_x, generated_x, batch_size, num_ground_truth): """Sample batch with specified mix of ground truth and generated data points. Args: ground_truth_x: tensor of ground-truth data points. generated_x: tensor of generated data points. batch_size: batch size num_ground_truth: number of ground-truth examples to include in batch. Returns: New batch with num_ground_truth sampled from ground_truth_x and the rest from generated_x. """ ground_truth_mask = tf.concat([tf.zeros([num_ground_truth], dtype=tf.int32), tf.ones([batch_size - num_ground_truth], dtype=tf.int32)], axis=0) ground_truth_mask = tf.reshape(ground_truth_mask, [batch_size]) ground_truth_mask = tf.random_shuffle(ground_truth_mask) ground_truth_partitioned = tf.dynamic_partition(ground_truth_x, ground_truth_mask, 2) generated_partitioned = tf.dynamic_partition(generated_x, ground_truth_mask, 2) stitch_indices = tf.dynamic_partition(tf.range(batch_size), ground_truth_mask, 2) stitch_data = [ground_truth_partitioned[0], generated_partitioned[1]] outputs = tf.dynamic_stitch(stitch_indices, stitch_data) outputs = tf.reshape(outputs, [int(batch_size)] + outputs.get_shape().as_list()[1:]) return outputs
def _partition_and_stitch(self, args, func_name): """ args is a list of tensors, to be passed to self.likelihoods.<func_name> args[-1] is the 'Y' argument, which contains the indexes to self.likelihoods. This function splits up the args using dynamic_partition, calls the relevant function on the likelihoods, and re-combines the result. """ # get the index from Y Y = args[-1] ind = tf.gather(tf.transpose(Y), tf.shape(Y)[1] - 1) # ind = Y[:,-1] ind = tf.cast(ind, tf.int32) Y = tf.transpose( tf.gather(tf.transpose(Y), tf.range(0, tf.shape(Y)[1] - 1))) # Y = Y[:,:-1] args[-1] = Y # split up the arguments into chunks corresponding to the relevant likelihoods args = zip( * [tf.dynamic_partition(X, ind, self.num_likelihoods) for X in args]) # apply the likelihood-function to each section of the data with params_as_tensors_for(self, convert=False): funcs = [getattr(lik, func_name) for lik in self.likelihood_list] results = [f(*args_i) for f, args_i in zip(funcs, args)] # stitch the results back together partitions = tf.dynamic_partition(tf.range(0, tf.size(ind)), ind, self.num_likelihoods) results = tf.dynamic_stitch(partitions, results) return results
def _sample_n(self, n, seed=None): if self._use_static_graph: # This sampling approach is almost the same as the approach used by # `MixtureSameFamily`. The differences are due to having a list of # `Distribution` objects rather than a single object, and maintaining # random seed management that is consistent with the non-static code path. samples = [] cat_samples = self.cat.sample(n, seed=seed) for c in range(self.num_components): seed = distribution_util.gen_new_seed(seed, "mixture") samples.append(self.components[c].sample(n, seed=seed)) x = tf.stack(samples, -self._static_event_shape.ndims - 1) # [n, B, k, E] npdt = x.dtype.as_numpy_dtype mask = tf.one_hot( indices=cat_samples, # [n, B] depth=self._num_components, # == k on_value=np.ones([], dtype=npdt), off_value=np.zeros([], dtype=npdt)) # [n, B, k] mask = distribution_utils.pad_mixture_dimensions( mask, self, self._cat, self._static_event_shape.ndims) # [n, B, k, [1]*e] return tf.reduce_sum( x * mask, axis=-1 - self._static_event_shape.ndims) # [n, B, E] with tf.control_dependencies(self._assertions): n = tf.convert_to_tensor(n, name="n") static_n = tensor_util.constant_value(n) n = int(static_n) if static_n is not None else n cat_samples = self.cat.sample(n, seed=seed) static_samples_shape = cat_samples.get_shape() if static_samples_shape.is_fully_defined(): samples_shape = static_samples_shape.as_list() samples_size = static_samples_shape.num_elements() else: samples_shape = tf.shape(cat_samples) samples_size = tf.size(cat_samples) static_batch_shape = self.batch_shape if static_batch_shape.is_fully_defined(): batch_shape = static_batch_shape.as_list() batch_size = static_batch_shape.num_elements() else: batch_shape = self.batch_shape_tensor() batch_size = tf.reduce_prod(batch_shape) static_event_shape = self.event_shape if static_event_shape.is_fully_defined(): event_shape = np.array(static_event_shape.as_list(), dtype=np.int32) else: event_shape = self.event_shape_tensor() # Get indices into the raw cat sampling tensor. We will # need these to stitch sample values back out after sampling # within the component partitions. samples_raw_indices = tf.reshape(tf.range(0, samples_size), samples_shape) # Partition the raw indices so that we can use # dynamic_stitch later to reconstruct the samples from the # known partitions. partitioned_samples_indices = tf.dynamic_partition( data=samples_raw_indices, partitions=cat_samples, num_partitions=self.num_components) # Copy the batch indices n times, as we will need to know # these to pull out the appropriate rows within the # component partitions. batch_raw_indices = tf.reshape( tf.tile(tf.range(0, batch_size), [n]), samples_shape) # Explanation of the dynamic partitioning below: # batch indices are i.e., [0, 1, 0, 1, 0, 1] # Suppose partitions are: # [1 1 0 0 1 1] # After partitioning, batch indices are cut as: # [batch_indices[x] for x in 2, 3] # [batch_indices[x] for x in 0, 1, 4, 5] # i.e. # [1 1] and [0 0 0 0] # Now we sample n=2 from part 0 and n=4 from part 1. # For part 0 we want samples from batch entries 1, 1 (samples 0, 1), # and for part 1 we want samples from batch entries 0, 0, 0, 0 # (samples 0, 1, 2, 3). partitioned_batch_indices = tf.dynamic_partition( data=batch_raw_indices, partitions=cat_samples, num_partitions=self.num_components) samples_class = [None for _ in range(self.num_components)] for c in range(self.num_components): n_class = tf.size(partitioned_samples_indices[c]) seed = distribution_util.gen_new_seed(seed, "mixture") samples_class_c = self.components[c].sample(n_class, seed=seed) # Pull out the correct batch entries from each index. # To do this, we may have to flatten the batch shape. # For sample s, batch element b of component c, we get the # partitioned batch indices from # partitioned_batch_indices[c]; and shift each element by # the sample index. The final lookup can be thought of as # a matrix gather along locations (s, b) in # samples_class_c where the n_class rows correspond to # samples within this component and the batch_size columns # correspond to batch elements within the component. # # Thus the lookup index is # lookup[c, i] = batch_size * s[i] + b[c, i] # for i = 0 ... n_class[c] - 1. lookup_partitioned_batch_indices = ( batch_size * tf.range(n_class) + partitioned_batch_indices[c]) samples_class_c = tf.reshape( samples_class_c, tf.concat([[n_class * batch_size], event_shape], 0)) samples_class_c = tf.gather( samples_class_c, lookup_partitioned_batch_indices, name="samples_class_c_gather") samples_class[c] = samples_class_c # Stitch back together the samples across the components. lhs_flat_ret = tf.dynamic_stitch( indices=partitioned_samples_indices, data=samples_class) # Reshape back to proper sample, batch, and event shape. ret = tf.reshape( lhs_flat_ret, tf.concat( [samples_shape, self.event_shape_tensor()], 0)) ret.set_shape( tf.TensorShape(static_samples_shape).concatenate(self.event_shape)) return ret
def layer_op(self, inputs, is_training, mask, use_local_stats=False): """ Parameters: inputs: image to normalize. This typically represents a sparse subset of channels from a sparse convolution. is_training: boolean that is True during training. When True, the layer uses batch statistics for normalization and records a moving average of means and variances. When False, the layer uses previously computed moving averages for normalization mask: 1-Tensor with a binary mask identifying the sparse channels represented in inputs """ if mask is None: mask=tf.ones([self.n_dense_channels])>0 else: mask=mask input_shape = inputs.shape mask_shape = mask.shape # operates on all dims except the last dim params_shape = mask_shape[-1:] assert params_shape[0]==self.n_dense_channels, \ 'Mask size {} must match n_dense_channels {}.'.format(params_shape[0],self.n_dense_channels) axes = list(range(input_shape.ndims - 1)) # create trainable variables and moving average variables beta = tf.get_variable( 'beta', shape=params_shape, initializer=self.initializers['beta'], regularizer=self.regularizers['beta'], dtype=tf.float32, trainable=True) gamma = tf.get_variable( 'gamma', shape=params_shape, initializer=self.initializers['gamma'], regularizer=self.regularizers['gamma'], dtype=tf.float32, trainable=True) collections = [tf.GraphKeys.GLOBAL_VARIABLES] moving_mean = tf.get_variable( 'moving_mean', shape=params_shape, initializer=self.initializers['moving_mean'], dtype=tf.float32, trainable=False, collections=collections) moving_variance = tf.get_variable( 'moving_variance', shape=params_shape, initializer=self.initializers['moving_variance'], dtype=tf.float32, trainable=False, collections=collections) # mean and var mean, variance = tf.nn.moments(inputs, axes) # only update masked moving averages mean_update=tf.dynamic_stitch([tf.to_int32(tf.where(mask)[:,0]),tf.to_int32(tf.where(~mask)[:,0])],[mean,tf.boolean_mask(moving_mean,~mask)]) variance_update=tf.dynamic_stitch([tf.to_int32(tf.where(mask)[:,0]),tf.to_int32(tf.where(~mask)[:,0])],[variance,tf.boolean_mask(moving_variance,~mask)]) update_moving_mean = moving_averages.assign_moving_average( moving_mean, mean_update, self.moving_decay).op update_moving_variance = moving_averages.assign_moving_average( moving_variance, variance_update, self.moving_decay).op tf.add_to_collection(tf.GraphKeys.UPDATE_OPS, update_moving_mean) tf.add_to_collection(tf.GraphKeys.UPDATE_OPS, update_moving_variance) # call the normalisation function if is_training or use_local_stats: # with tf.control_dependencies( # [update_moving_mean, update_moving_variance]): outputs = tf.nn.batch_normalization( inputs, mean, variance, tf.boolean_mask(beta,mask), tf.boolean_mask(gamma,mask), self.eps, name='batch_norm') else: outputs = tf.nn.batch_normalization( inputs, tf.boolean_mask(moving_mean,mask), tf.boolean_mask(moving_variance,mask), tf.boolean_mask(beta,mask), tf.boolean_mask(gamma,mask), self.eps, name='batch_norm') outputs.set_shape(inputs.get_shape()) return outputs
neigh_articles = tf.gather(rep_law_1, adj_list[i]) # size: n * deg * law_size neigh_articles = tf.reshape(neigh_articles, [-1, i, lstm_size * 2]) article = law_representation[i] # size: n * law_size article_1 = tf.transpose(tf.reshape(tf.tile(article, [i, 1]), [i, -1, lstm_size * 2]), [1, 0, 2]) interaction_vec = tf.concat([article_1, neigh_articles], axis=-1) # size: [n, deg, law_size *2] neigh_articles = tf.reduce_mean( tf.tensordot(interaction_vec, W_similar, [2, 0]) + tf.expand_dims(B_similar, axis=0), 1) # [n, law_size] new_article = tf.nn.tanh(Full_inter_1(article - neigh_articles)) article_new_list.append(new_article) # [max_degree, N, law_size] law_conv = tf.dynamic_stitch(indices, article_new_list) # size: [183, law_size] article_new_list_1 = [tf.nn.tanh(article_new_list[0])] for i in range(1, max_deg + 1): if i not in deg_list: article_new_list_1.append(tf.nn.tanh(article_new_list[i])) continue neigh_articles = tf.gather(law_conv, adj_list[i]) # size: n * deg * law_size neigh_articles = tf.reshape(neigh_articles, [-1, i, lstm_size * 2]) article = article_new_list[i] # size: n * law_size article_1 = tf.transpose(tf.reshape(tf.tile(article, [i, 1]), [i, -1, lstm_size * 2]), [1, 0, 2]) interaction_vec = tf.concat([article_1, neigh_articles], axis=-1) # size: [n, deg, law_size *2] neigh_articles = tf.reduce_mean(
def _sample_n(self, n, seed=None): if self._use_static_graph: with tf.control_dependencies(self._assertions): # This sampling approach is almost the same as the approach used by # `MixtureSameFamily`. The differences are due to having a list of # `Distribution` objects rather than a single object, and maintaining # random seed management that is consistent with the non-static code # path. samples = [] cat_samples = self.cat.sample(n, seed=seed) stream = seed_stream.SeedStream(seed, salt="Mixture") for c in range(self.num_components): samples.append(self.components[c].sample(n, seed=stream())) x = tf.stack(samples, -self._static_event_shape.ndims - 1) # [n, B, k, E] npdt = x.dtype.as_numpy_dtype mask = tf.one_hot( indices=cat_samples, # [n, B] depth=self._num_components, # == k on_value=np.ones([], dtype=npdt), off_value=np.zeros([], dtype=npdt)) # [n, B, k] mask = distribution_util.pad_mixture_dimensions( mask, self, self._cat, self._static_event_shape.ndims) # [n, B, k, [1]*e] return tf.reduce_sum( input_tensor=x * mask, axis=-1 - self._static_event_shape.ndims) # [n, B, E] with tf.control_dependencies(self._assertions): n = tf.convert_to_tensor(value=n, name="n") static_n = tf.get_static_value(n) n = int(static_n) if static_n is not None else n cat_samples = self.cat.sample(n, seed=seed) static_samples_shape = cat_samples.shape if static_samples_shape.is_fully_defined(): samples_shape = static_samples_shape.as_list() samples_size = static_samples_shape.num_elements() else: samples_shape = tf.shape(input=cat_samples) samples_size = tf.size(input=cat_samples) static_batch_shape = self.batch_shape if static_batch_shape.is_fully_defined(): batch_shape = static_batch_shape.as_list() batch_size = static_batch_shape.num_elements() else: batch_shape = self.batch_shape_tensor() batch_size = tf.reduce_prod(input_tensor=batch_shape) static_event_shape = self.event_shape if static_event_shape.is_fully_defined(): event_shape = np.array(static_event_shape.as_list(), dtype=np.int32) else: event_shape = self.event_shape_tensor() # Get indices into the raw cat sampling tensor. We will # need these to stitch sample values back out after sampling # within the component partitions. samples_raw_indices = tf.reshape(tf.range(0, samples_size), samples_shape) # Partition the raw indices so that we can use # dynamic_stitch later to reconstruct the samples from the # known partitions. partitioned_samples_indices = tf.dynamic_partition( data=samples_raw_indices, partitions=cat_samples, num_partitions=self.num_components) # Copy the batch indices n times, as we will need to know # these to pull out the appropriate rows within the # component partitions. batch_raw_indices = tf.reshape( tf.tile(tf.range(0, batch_size), [n]), samples_shape) # Explanation of the dynamic partitioning below: # batch indices are i.e., [0, 1, 0, 1, 0, 1] # Suppose partitions are: # [1 1 0 0 1 1] # After partitioning, batch indices are cut as: # [batch_indices[x] for x in 2, 3] # [batch_indices[x] for x in 0, 1, 4, 5] # i.e. # [1 1] and [0 0 0 0] # Now we sample n=2 from part 0 and n=4 from part 1. # For part 0 we want samples from batch entries 1, 1 (samples 0, 1), # and for part 1 we want samples from batch entries 0, 0, 0, 0 # (samples 0, 1, 2, 3). partitioned_batch_indices = tf.dynamic_partition( data=batch_raw_indices, partitions=cat_samples, num_partitions=self.num_components) samples_class = [None for _ in range(self.num_components)] stream = seed_stream.SeedStream(seed, salt="Mixture") for c in range(self.num_components): n_class = tf.size(input=partitioned_samples_indices[c]) samples_class_c = self.components[c].sample(n_class, seed=stream()) # Pull out the correct batch entries from each index. # To do this, we may have to flatten the batch shape. # For sample s, batch element b of component c, we get the # partitioned batch indices from # partitioned_batch_indices[c]; and shift each element by # the sample index. The final lookup can be thought of as # a matrix gather along locations (s, b) in # samples_class_c where the n_class rows correspond to # samples within this component and the batch_size columns # correspond to batch elements within the component. # # Thus the lookup index is # lookup[c, i] = batch_size * s[i] + b[c, i] # for i = 0 ... n_class[c] - 1. lookup_partitioned_batch_indices = ( batch_size * tf.range(n_class) + partitioned_batch_indices[c]) samples_class_c = tf.reshape( samples_class_c, tf.concat([[n_class * batch_size], event_shape], 0)) samples_class_c = tf.gather(samples_class_c, lookup_partitioned_batch_indices, name="samples_class_c_gather") samples_class[c] = samples_class_c # Stitch back together the samples across the components. lhs_flat_ret = tf.dynamic_stitch( indices=partitioned_samples_indices, data=samples_class) # Reshape back to proper sample, batch, and event shape. ret = tf.reshape( lhs_flat_ret, tf.concat( [samples_shape, self.event_shape_tensor()], 0)) ret.set_shape( tf.TensorShape(static_samples_shape).concatenate( self.event_shape)) return ret
train_op = tf.keras.optimizers.Adam() num_epoch = 10000 print_epoch = 100 pred_model = model([l1, l2, l3], train_op, num_epoch, print_epoch) #convert the training data to tensors Xint_tf = tf.convert_to_tensor(Xint) Yint_tf = tf.convert_to_tensor(Yint) #training print("Training (ADAM)...") pred_model.network_learn(Xint_tf, Yint_tf) print("Training (LBFGS)...") loss_func = tfp_function_factory(pred_model, Xint_tf, Yint_tf) # convert initial model parameters to a 1D tf.Tensor init_params = tf.dynamic_stitch(loss_func.idx, pred_model.trainable_variables) # train the model with L-BFGS solver results = tfp.optimizer.lbfgs_minimize( value_and_gradients_function=loss_func, initial_position=init_params, max_iterations=4000, num_correction_pairs=50, tolerance=1e-14) # after training, the final optimized parameters are still in results.position # so we have to manually put them back to the model loss_func.assign_new_model_parameters(results.position) print("Testing...") numPtsTest = 2 * numPts x_test = np.linspace(xmin, xmax, numPtsTest) x_test = np.array(x_test)[np.newaxis].T
def layer_op(self, inputs, is_training, mask, use_local_stats=False): """ :param inputs: image to normalize. This typically represents a sparse subset of channels from a sparse convolution. :param is_training: boolean that is True during training. When True, the layer uses batch statistics for normalization and records a moving average of means and variances. When False, the layer uses previously computed moving averages for normalization. :param mask: 1-Tensor with a binary mask identifying the sparse channels represented in inputs :param use_local_stats: :return: """ if mask is None: mask = tf.ones([self.n_dense_channels]) > 0 else: mask = mask input_shape = inputs.shape mask_shape = mask.shape # operates on all dims except the last dim params_shape = mask_shape[-1:] assert params_shape[0] == self.n_dense_channels, \ 'Mask size {} must match n_dense_channels {}.'.format( params_shape[0], self.n_dense_channels) axes = list(range(input_shape.ndims - 1)) # create trainable variables and moving average variables beta = tf.get_variable( 'beta', shape=params_shape, initializer=self.initializers['beta'], regularizer=self.regularizers['beta'], dtype=tf.float32, trainable=True) gamma = tf.get_variable( 'gamma', shape=params_shape, initializer=self.initializers['gamma'], regularizer=self.regularizers['gamma'], dtype=tf.float32, trainable=True) beta = tf.boolean_mask(beta, mask) gamma = tf.boolean_mask(gamma, mask) collections = [tf.GraphKeys.GLOBAL_VARIABLES] moving_mean = tf.get_variable( 'moving_mean', shape=params_shape, initializer=self.initializers['moving_mean'], dtype=tf.float32, trainable=False, collections=collections) moving_variance = tf.get_variable( 'moving_variance', shape=params_shape, initializer=self.initializers['moving_variance'], dtype=tf.float32, trainable=False, collections=collections) # mean and var mean, variance = tf.nn.moments(inputs, axes) # only update masked moving averages mean_update = tf.dynamic_stitch( [tf.to_int32(tf.where(mask)[:, 0]), tf.to_int32(tf.where(~mask)[:, 0])], [mean, tf.boolean_mask(moving_mean, ~mask)]) variance_update = tf.dynamic_stitch( [tf.to_int32(tf.where(mask)[:, 0]), tf.to_int32(tf.where(~mask)[:, 0])], [variance, tf.boolean_mask(moving_variance, ~mask)]) update_moving_mean = moving_averages.assign_moving_average( moving_mean, mean_update, self.moving_decay).op update_moving_variance = moving_averages.assign_moving_average( moving_variance, variance_update, self.moving_decay).op tf.add_to_collection(tf.GraphKeys.UPDATE_OPS, update_moving_mean) tf.add_to_collection(tf.GraphKeys.UPDATE_OPS, update_moving_variance) # call the normalisation function if is_training or use_local_stats: outputs = tf.nn.batch_normalization( inputs, mean, variance, beta, gamma, self.eps, name='batch_norm') else: outputs = tf.nn.batch_normalization( inputs, tf.boolean_mask(moving_mean, mask), tf.boolean_mask(moving_variance, mask), beta, gamma, self.eps, name='batch_norm') outputs.set_shape(inputs.get_shape()) return outputs
def merge_data(self, data_list): with tf.name_scope(None, "merge_data") as scope: return tf.dynamic_stitch(self.stitch_indices, data_list)
# from https://github.com/tensorflow/tensorflow/issues/7251 import os os.environ["CUDA_VISIBLE_DEVICES"]="0" import tensorflow as tf from tensorflow.python.client.timeline import Timeline with tf.device("/gpu:0"): x = tf.ones(100, name="x") idxs = tf.range(100) for i in range(10): y = tf.identity(x, name="identity-"+str(i)) x = tf.dynamic_stitch([idxs, idxs], [x, y], name="stitch-"+str(i)) config = tf.ConfigProto(graph_options=tf.GraphOptions(optimizer_options=tf.OptimizerOptions(opt_level=tf.OptimizerOptions.L0))) sess = tf.InteractiveSession(config=config) metadata = tf.RunMetadata() sess.run(x, options=tf.RunOptions(trace_level=tf.RunOptions.FULL_TRACE, output_partition_graphs=True), run_metadata=metadata) timeline = Timeline(metadata.step_stats) with open("dynamic_stitch_gpu_profile.json", "w") as f: f.write(timeline.generate_chrome_trace_format()) with open("dynamic_stitch_gpu_profile.pbtxt", "w") as f: f.write(str(metadata))
# coding: utf-8 import tensorflow as tf import numpy as np with tf.Session() as sess: indices1 = tf.constant([0, 3]) #对应数据元素应放的重组后所在位置 indices2 = tf.constant([1, 2]) data1 = tf.constant([-1, -2]) data2 = tf.constant([-3, -4]) #数据 result = tf.dynamic_stitch([indices1, indices2], [data1, data2]) print(result.eval())
def rotate_image_tensor(image, angle, mode='black'): """ Rotates a 3D tensor (HWD), which represents an image by given radian angle. New image has the same size as the input image. mode controls what happens to border pixels. mode = 'black' results in black bars (value 0 in unknown areas) mode = 'white' results in value 255 in unknown areas mode = 'ones' results in value 1 in unknown areas mode = 'repeat' keeps repeating the closest pixel known """ s = tf.shape(image) assert s.get_shape()[0] == 3, "Input needs to be 3D." assert (mode == 'repeat') or (mode == 'black') or (mode == 'white') or ( mode == 'ones'), "Unknown boundary mode." image_center = [ tf.floor(tf.cast(s[0] / 2, tf.float32)), tf.floor(tf.cast(s[1] / 2, tf.float32)) ] # Coordinates of new image coord1 = tf.range(s[0]) coord2 = tf.range(s[1]) # Create vectors of those coordinates in order to vectorize the image coord1_vec = tf.tile(coord1, [s[1]]) coord2_vec_unordered = tf.tile(coord2, [s[0]]) coord2_vec_unordered = tf.reshape(coord2_vec_unordered, [s[0], s[1]]) coord2_vec = tf.reshape(tf.transpose(coord2_vec_unordered, [1, 0]), [-1]) # center coordinates since rotation center is supposed to be in the image center coord1_vec_centered = coord1_vec - tf.to_int32(image_center[0]) coord2_vec_centered = coord2_vec - tf.to_int32(image_center[1]) coord_new_centered = tf.cast( tf.stack([coord1_vec_centered, coord2_vec_centered]), tf.float32) # Perform backward transformation of the image coordinates rot_mat_inv = tf.dynamic_stitch( [0, 1, 2, 3], [tf.cos(angle), tf.sin(angle), -tf.sin(angle), tf.cos(angle)]) rot_mat_inv = tf.reshape(rot_mat_inv, shape=[2, 2]) coord_old_centered = tf.matmul(rot_mat_inv, coord_new_centered) # Find nearest neighbor in old image coord1_old_nn = tf.cast( tf.round(coord_old_centered[0, :] + image_center[0]), tf.int32) coord2_old_nn = tf.cast( tf.round(coord_old_centered[1, :] + image_center[1]), tf.int32) # Clip values to stay inside image coordinates if mode == 'repeat': coord_old1_clipped = tf.minimum(tf.maximum(coord1_old_nn, 0), s[0] - 1) coord_old2_clipped = tf.minimum(tf.maximum(coord2_old_nn, 0), s[1] - 1) else: outside_ind1 = tf.logical_or(tf.greater(coord1_old_nn, s[0] - 1), tf.less(coord1_old_nn, 0)) outside_ind2 = tf.logical_or(tf.greater(coord2_old_nn, s[1] - 1), tf.less(coord2_old_nn, 0)) outside_ind = tf.logical_or(outside_ind1, outside_ind2) coord_old1_clipped = tf.boolean_mask(coord1_old_nn, tf.logical_not(outside_ind)) coord_old2_clipped = tf.boolean_mask(coord2_old_nn, tf.logical_not(outside_ind)) coord1_vec = tf.boolean_mask(coord1_vec, tf.logical_not(outside_ind)) coord2_vec = tf.boolean_mask(coord2_vec, tf.logical_not(outside_ind)) coord_old_clipped = tf.cast( tf.transpose(tf.stack([coord_old1_clipped, coord_old2_clipped]), [1, 0]), tf.int32) # Coordinates of the new image coord_new = tf.transpose( tf.cast(tf.stack([coord1_vec, coord2_vec]), tf.int32), [1, 0]) num_channels = image.get_shape().as_list()[2] image_channel_list = tf.split(image, num_channels, axis=2) image_rotated_channel_list = list() for image_channel in image_channel_list: image_chan_new_values = tf.gather_nd(tf.squeeze(image_channel), coord_old_clipped) if (mode == 'black') or (mode == 'repeat'): background_color = 0 elif mode == 'ones': background_color = 1 elif mode == 'white': background_color = 255 else: background_color = 0 image_rotated_channel_list.append( tf.sparse_to_dense(coord_new, [s[0], s[1]], image_chan_new_values, background_color, validate_indices=False)) image_rotated = tf.transpose(tf.stack(image_rotated_channel_list), [1, 2, 0]) return image_rotated
def testInt32Gpu(self): with self.test_session(use_gpu=True): indices = [tf.convert_to_tensor([0, 1, 2]), tf.convert_to_tensor([2, 3])] values = [tf.convert_to_tensor([12, 23, 34]), tf.convert_to_tensor([1, 2])] self.assertAllEqual( tf.dynamic_stitch(indices, values).eval(), [12, 23, 1, 2])
def get(self): """ Provides input data to the graph. """ # calculate size of each record (this lists what is contained in the db and how many bytes are occupied) record_bytes = 0 encoding_bytes = 4 kp_xyz_entries = 3 * self.num_kp record_bytes += encoding_bytes * kp_xyz_entries encoding_bytes = 4 kp_uv_entries = 2 * self.num_kp record_bytes += encoding_bytes * kp_uv_entries kp_vis_entries = self.num_kp record_bytes += encoding_bytes * kp_vis_entries image_bytes = self.image_size[0] * self.image_size[1] * 3 record_bytes += image_bytes """ READ DATA ITEMS""" # Start reader reader = tf.FixedLengthRecordReader(header_bytes=0, record_bytes=record_bytes) _, value = reader.read( tf.train.string_input_producer([self.path_to_db])) # decode to floats bytes_read = 0 data_dict = dict() record_bytes_float32 = tf.decode_raw(value, tf.float32) # 1. Read keypoint xyz keypoint_xyz21 = tf.reshape( tf.slice(record_bytes_float32, [bytes_read // 4], [kp_xyz_entries]), [self.num_kp, 3]) bytes_read += encoding_bytes * kp_xyz_entries keypoint_xyz21 /= 1000.0 # scale to meters keypoint_xyz21 = self.convert_kp(keypoint_xyz21) # calculate wrist coord if self.use_wrist_coord: wrist_xyz = keypoint_xyz21[16, :] + 2.0 * (keypoint_xyz21[0, :] - keypoint_xyz21[16, :]) keypoint_xyz21 = tf.concat( [tf.expand_dims(wrist_xyz, 0), keypoint_xyz21[1:, :]], 0) data_dict['keypoint_xyz21'] = keypoint_xyz21 # 2. Read keypoint uv AND VIS keypoint_uv_vis21 = tf.reshape( tf.slice(record_bytes_float32, [bytes_read // 4], [kp_uv_entries + kp_vis_entries]), [self.num_kp, 3]) bytes_read += encoding_bytes * (kp_uv_entries + kp_vis_entries) keypoint_uv_vis21 = self.convert_kp(keypoint_uv_vis21) keypoint_uv21 = keypoint_uv_vis21[:, :2] keypoint_vis21 = tf.equal(keypoint_uv_vis21[:, 2], 1.0) # calculate wrist vis if self.use_wrist_coord: wrist_vis = tf.logical_or(keypoint_vis21[16], keypoint_vis21[0]) keypoint_vis21 = tf.concat( [tf.expand_dims(wrist_vis, 0), keypoint_vis21[1:]], 0) wrist_uv = keypoint_uv21[16, :] + 2.0 * (keypoint_uv21[0, :] - keypoint_uv21[16, :]) keypoint_uv21 = tf.concat( [tf.expand_dims(wrist_uv, 0), keypoint_uv21[1:, :]], 0) data_dict['keypoint_vis21'] = keypoint_vis21 if self.coord_uv_noise: noise = tf.truncated_normal([42, 2], mean=0.0, stddev=self.coord_uv_noise_sigma) keypoint_uv21 += noise data_dict['keypoint_uv21'] = keypoint_uv21 # decode to uint8 record_bytes_uint8 = tf.decode_raw(value, tf.uint8) # 4. Read image image = tf.reshape( tf.slice(record_bytes_uint8, [bytes_read], [image_bytes]), [self.image_size[0], self.image_size[1], 3]) image = tf.cast(image, tf.float32) bytes_read += image_bytes # subtract mean image = image / 255.0 - 0.5 if self.hue_aug: image = tf.image.random_hue(image, self.hue_aug_max) data_dict['image'] = image """ CONSTANTS """ # Camera intrinsics sx = 822.79041 sy = 822.79041 tx = 318.47345 ty = 250.31296 data_dict['cam_mat'] = tf.constant([[sx, 0.0, tx], [0.0, sy, ty], [0.0, 0.0, 1.0]]) # Hand side: this dataset only contains left hands data_dict['hand_side'] = tf.one_hot(tf.constant(0, dtype=tf.int32), depth=2, on_value=1.0, off_value=0.0, dtype=tf.float32) assert bytes_read == record_bytes, "Doesnt add up." """ DEPENDENT DATA ITEMS: XYZ represenations. """ # make coords relative to root joint kp_coord_xyz_root = keypoint_xyz21[0, :] # this is the palm coord kp_coord_xyz21_rel = keypoint_xyz21 - kp_coord_xyz_root # relative coords in metric coords index_root_bone_length = tf.sqrt( tf.reduce_sum( tf.square(kp_coord_xyz21_rel[12, :] - kp_coord_xyz21_rel[11, :]))) data_dict['keypoint_scale'] = index_root_bone_length data_dict[ 'keypoint_xyz21_normed'] = kp_coord_xyz21_rel / index_root_bone_length # normalized by length of 12->11 # calculate local coordinates kp_coord_xyz21_local = bone_rel_trafo( data_dict['keypoint_xyz21_normed']) kp_coord_xyz21_local = tf.squeeze(kp_coord_xyz21_local) data_dict['keypoint_xyz21_local'] = kp_coord_xyz21_local # calculate viewpoint and coords in canonical coordinates kp_coord_xyz21_rel_can, rot_mat = canonical_trafo( data_dict['keypoint_xyz21_normed']) kp_coord_xyz21_rel_can, rot_mat = tf.squeeze( kp_coord_xyz21_rel_can), tf.squeeze(rot_mat) data_dict['keypoint_xyz21_can'] = kp_coord_xyz21_rel_can data_dict['rot_mat'] = tf.matrix_inverse(rot_mat) """ DEPENDENT DATA ITEMS: HAND CROP """ if self.hand_crop: crop_center = keypoint_uv21[12, ::-1] # catch problem, when no valid kp available (happens almost never) crop_center = tf.cond(tf.reduce_all(tf.is_finite(crop_center)), lambda: crop_center, lambda: tf.constant([0.0, 0.0])) crop_center.set_shape([ 2, ]) if self.crop_center_noise: noise = tf.truncated_normal( [2], mean=0.0, stddev=self.crop_center_noise_sigma) crop_center += noise crop_scale_noise = tf.constant(1.0) if self.crop_scale_noise: crop_scale_noise = tf.squeeze( tf.random_uniform([1], minval=1.0, maxval=1.2)) if not self.use_wrist_coord: wrist_uv = keypoint_uv21[16, :] + 2.0 * (keypoint_uv21[0, :] - keypoint_uv21[16, :]) keypoint_uv21 = tf.concat( [tf.expand_dims(wrist_uv, 0), keypoint_uv21[1:, :]], 0) # select visible coords only kp_coord_h = tf.boolean_mask(keypoint_uv21[:, 1], keypoint_vis21) kp_coord_w = tf.boolean_mask(keypoint_uv21[:, 0], keypoint_vis21) kp_coord_hw = tf.stack([kp_coord_h, kp_coord_w], 1) # determine size of crop (measure spatial extend of hw coords first) min_coord = tf.maximum(tf.reduce_min(kp_coord_hw, 0), 0.0) max_coord = tf.minimum(tf.reduce_max(kp_coord_hw, 0), self.image_size) # find out larger distance wrt the center of crop crop_size_best = 2 * tf.maximum(max_coord - crop_center, crop_center - min_coord) crop_size_best = tf.reduce_max(crop_size_best) crop_size_best = tf.minimum(tf.maximum(crop_size_best, 50.0), 500.0) # catch problem, when no valid kp available crop_size_best = tf.cond( tf.reduce_all(tf.is_finite(crop_size_best)), lambda: crop_size_best, lambda: tf.constant(200.0)) crop_size_best.set_shape([]) # calculate necessary scaling scale = tf.cast(self.crop_size, tf.float32) / crop_size_best scale = tf.minimum(tf.maximum(scale, 1.0), 10.0) scale *= crop_scale_noise data_dict['crop_scale'] = scale if self.crop_offset_noise: noise = tf.truncated_normal( [2], mean=0.0, stddev=self.crop_offset_noise_sigma) crop_center += noise # Crop image img_crop = crop_image_from_xy(tf.expand_dims(image, 0), crop_center, self.crop_size, scale) data_dict['image_crop'] = tf.squeeze(img_crop) # Modify uv21 coordinates crop_center_float = tf.cast(crop_center, tf.float32) keypoint_uv21_u = ( data_dict['keypoint_uv21'][:, 0] - crop_center_float[1]) * scale + self.crop_size // 2 keypoint_uv21_v = ( data_dict['keypoint_uv21'][:, 1] - crop_center_float[0]) * scale + self.crop_size // 2 keypoint_uv21 = tf.stack([keypoint_uv21_u, keypoint_uv21_v], 1) data_dict['keypoint_uv21'] = keypoint_uv21 # Modify camera intrinsics scale = tf.reshape(scale, [ 1, ]) scale_matrix = tf.dynamic_stitch([ [0], [1], [2], [3], [4], [5], [6], [7], [8] ], [scale, [0.0], [0.0], [0.0], scale, [0.0], [0.0], [0.0], [1.0]]) scale_matrix = tf.reshape(scale_matrix, [3, 3]) crop_center_float = tf.cast(crop_center, tf.float32) trans1 = crop_center_float[0] * scale - self.crop_size // 2 trans2 = crop_center_float[1] * scale - self.crop_size // 2 trans1 = tf.reshape(trans1, [ 1, ]) trans2 = tf.reshape(trans2, [ 1, ]) trans_matrix = tf.dynamic_stitch( [[0], [1], [2], [3], [4], [5], [6], [7], [8]], [[1.0], [0.0], -trans2, [0.0], [1.0], -trans1, [0.0], [0.0], [1.0]]) trans_matrix = tf.reshape(trans_matrix, [3, 3]) data_dict['cam_mat'] = tf.matmul( trans_matrix, tf.matmul(scale_matrix, data_dict['cam_mat'])) """ DEPENDENT DATA ITEMS: Scoremap from the SUBSET of 21 keypoints""" # create scoremaps from the subset of 2D annoataion keypoint_hw21 = tf.stack([keypoint_uv21[:, 1], keypoint_uv21[:, 0]], -1) scoremap_size = self.image_size if self.hand_crop: scoremap_size = (self.crop_size, self.crop_size) scoremap = self.create_multiple_gaussian_map(keypoint_hw21, scoremap_size, self.sigma, valid_vec=keypoint_vis21) if self.scoremap_dropout: scoremap = tf.nn.dropout(scoremap, self.scoremap_dropout_prob, noise_shape=[1, 1, 21]) scoremap *= self.scoremap_dropout_prob data_dict['scoremap'] = scoremap if self.random_crop_to_size: tensor_stack = tf.concat([ data_dict['image'], tf.expand_dims(tf.cast(data_dict['hand_parts'], tf.float32), -1), tf.cast(data_dict['hand_mask'], tf.float32) ], 2) s = tensor_stack.get_shape().as_list() tensor_stack_cropped = tf.random_crop( tensor_stack, [self.random_crop_size, self.random_crop_size, s[2]]) data_dict = dict( ) # delete everything else because the random cropping makes the data invalid anyway data_dict['image'], data_dict['hand_parts'], data_dict['hand_mask'] = tensor_stack_cropped[:, :, :3],\ tf.cast(tensor_stack_cropped[:, :, 3], tf.int32),\ tf.cast(tensor_stack_cropped[:, :, 4:], tf.int32) names, tensors = zip(*data_dict.items()) if self.shuffle: tensors = tf.train.shuffle_batch_join([tensors], batch_size=self.batch_size, capacity=100, min_after_dequeue=50, enqueue_many=False) else: tensors = tf.train.batch_join([tensors], batch_size=self.batch_size, capacity=100, enqueue_many=False) return dict(zip(names, tensors))