def bucket_fn(x): """Compute the element bucket and update the histogram.""" ix = len_fn(x) if ix.dtype == tf.int32: ix = tf.to_int64(ix) elif ix.dtype != tf.int64: raise ValueError("Len function returned a non-int") adds_to_bins = tf.to_int64(tf.greater(hist_bounds, ix)) # pad with a 1 for the "larger than all" bin adds_to_bins = tf.pad(adds_to_bins, [[0, 1]], constant_values=1) new_counts = tf.assign_add(hist_counts, adds_to_bins) bin_ix = n_hist_binds - tf.reduce_sum(adds_to_bins) # Computes the quantile based on the counts of the exammple's bucket bucket_ix = tf.floordiv(((n_buckets - 1) * new_counts[bin_ix]), new_counts[-1]) return bucket_ix
def _resample(): """Helper method for resampling inside cond.""" x = example ans_start = tf.to_int64(x['answers_start_token'][0]) ans_end = tf.to_int64(x['answers_end_token'][0]) min_start = tf.maximum(tf.to_int64(0), ans_end - max_length + 1) max_start = ans_start start_idx = tf.random_uniform([], min_start, max_start + 1, dtype=tf.int64) for k in ['answers_start_token', 'answers_end_token']: x[k] -= start_idx x['context_tokens'] = x['context_tokens'][start_idx:start_idx + max_length] x['context_length'] = tf.to_int64(tf.shape(x['context_tokens'])[0]) return x
def get_decode_steps(extended_indices, output_vocab_size, model_config): """Convert Tensor of indices in extended vocabulary to DecodeStep.""" extended_indices = tf.to_int64(extended_indices) action_types = _get_action_type(extended_indices, output_vocab_size, model_config) action_ids = _get_action_id(extended_indices, action_types, output_vocab_size, model_config) return DecodeSteps(action_types=action_types, action_ids=action_ids)
def preprocess_example(self, example, mode, _): # Resize from usual size ~1350x60 to 90x4 in this test. img = example["inputs"] img = tf.to_int64( tf.image.resize_images(img, [90, 4], tf.image.ResizeMethod.AREA)) img = tf.image.per_image_standardization(img) example["inputs"] = img return example
def preprocess_example(self, example, mode, unused_hparams): example["inputs"].set_shape([_CIFAR10_IMAGE_SIZE, _CIFAR10_IMAGE_SIZE, 3]) example["inputs"] = tf.to_int64(example["inputs"]) example["inputs"] = tf.reshape(example["inputs"], (-1,)) del example["targets"] # Ensure unconditional generation return example
def preprocess_example(self, example, mode, unused_hparams): example["inputs"].set_shape( [_CIFAR10_IMAGE_SIZE, _CIFAR10_IMAGE_SIZE, 3]) example["inputs"] = tf.to_int64(example["inputs"]) if mode == tf.estimator.ModeKeys.TRAIN: example["inputs"] = image_utils.random_shift(example["inputs"], wsr=0.1, hsr=0.1) return example
def conditional_bilinear_classifier(self, inputs1, inputs2, n_classes, probs, add_bias1=True, add_bias2=True): """""" input_shape = tf.shape(inputs1) batch_size = input_shape[0] bucket_size = input_shape[1] input_size = inputs1.get_shape().as_list()[-1] input_shape_to_set = [ tf.Dimension(None), tf.Dimension(None), input_size + 1 ] output_shape = tf.stack( [batch_size, bucket_size, n_classes, bucket_size]) if len(probs.get_shape().as_list()) == 2: probs = tf.to_float( tf.one_hot(tf.to_int64(probs), bucket_size, 1, 0)) else: probs = tf.stop_gradient(probs) if self.moving_params is None: keep_prob = self.mlp_keep_prob else: keep_prob = 1 if isinstance(keep_prob, tf.Tensor) or keep_prob < 1: noise_shape = tf.stack([batch_size, 1, input_size]) inputs1 = tf.nn.dropout(inputs1, keep_prob, noise_shape=noise_shape) inputs2 = tf.nn.dropout(inputs2, keep_prob, noise_shape=noise_shape) inputs1 = tf.concat( 2, [inputs1, tf.ones(tf.stack([batch_size, bucket_size, 1]))]) inputs1.set_shape(input_shape_to_set) inputs2 = tf.concat( 2, [inputs2, tf.ones(tf.stack([batch_size, bucket_size, 1]))]) inputs2.set_shape(input_shape_to_set) bilin = linalg.bilinear(inputs1, inputs2, n_classes, add_bias1=add_bias1, add_bias2=add_bias2, initializer=tf.zeros_initializer, moving_params=self.moving_params) weighted_bilin = tf.batch_matmul(bilin, tf.expand_dims(probs, 3)) return weighted_bilin, bilin
def infer_step(result, length): """Inference step.""" def print_info(result, length, new_length): vocab = self.problem_hparams.vocabulary["targets"] tf.logging.info( "length=%s new_length=%s length_diff=%s new_suffix=%s", length, new_length, new_length - length, str([ vocab._subtoken_id_to_subtoken_string(index) # pylint: disable=protected-access for index in result[0, -block_size:, 0, 0][:new_length - length] ]).decode("unicode-escape"), ) features["targets"] = tf.pad(result, [[0, 0], [0, 1], [0, 0], [0, 0]]) samples, logits, losses = self.sample(features) # pylint: disable=unused-variable _, top_k_indices = tf.nn.top_k( logits[:, :-1, :1, :, :], k=self._decode_hparams.guess_and_check_top_k) in_top_k = tf.reduce_any(tf.equal(tf.to_int64(top_k_indices), tf.expand_dims(result, 4)), axis=4) eos_cumsum = tf.cumsum(tf.to_int32( tf.equal(result, text_encoder.EOS_ID)), axis=1) after_eos = tf.greater(common_layers.shift_right(eos_cumsum), 0) correct = tf.logical_and(in_top_k, tf.logical_not(after_eos)) correct_cumsum = tf.cumsum(tf.to_int32(correct), axis=1) perfect_cumsum = 1 + tf.range(tf.shape(correct)[1]) for axis in [0, 2, 3]: perfect_cumsum = tf.expand_dims(perfect_cumsum, axis=axis) new_length = tf.reduce_sum(tf.to_int32( tf.equal(correct_cumsum, perfect_cumsum)), axis=1) new_length = tf.squeeze(new_length, axis=[0, 1, 2]) new_length = tf.minimum(new_length, decode_length) new_result = tf.concat([ result[:, :new_length, :, :], tf.reshape(samples[:, new_length, :block_size, :], [1, block_size, 1, 1]) ], axis=1) with tf.control_dependencies( [tf.py_func(print_info, [result, length, new_length], [])]): new_result = tf.identity(new_result) return new_result, new_length
def spherical_hashing( inputs, num_bits, pack_bits=True, reuse=False, scope='SphericalHashing' ): """Spherical hashing from https://dl.acm.org/citation.cfm?id=2882197 Heo, Jae-Pil, et al. "Spherical hashing." Computer Vision and Pattern Recognition (CVPR), 2012 IEEE Conference on. IEEE, 2012. Args: inputs: A float32 `Tensor` of input data. num_bits: Number of pivots (equivalently, number of hash bits). pack_bits: Whether to pack bits into int64s. reuse: Whether or not variables should be reused. To be able to reuse `scope` must be given. scope: Optional variable scope. Returns: A tuple of output bits and a dictionary of model variables. """ with tf.variable_scope(scope, 'SphericalHashing', [inputs], reuse=reuse): inputs_dims = inputs.shape[1] pivots = tf.Variable( name='Pivots', shape=[num_bits, inputs_dims], initial_value=tf.zeros([num_bits, inputs_dims]), collections=[tf.GraphKeys.MODEL_VARIABLES], ) radii = tf.Variable( name='Radii', shape=[num_bits, 1], initial_value=tf.zeros([num_bits, 1]), collections=[tf.GraphKeys.MODEL_VARIABLES], ) distances = pairwise_distances(inputs, pivots) thresholds = distances - tf.transpose(radii) bits = tf.cast(thresholds <= 0, tf.uint8, name='Bits') if pack_bits: pow_2 = tf.pow(tf.to_int64(2), tf.range(0, 64, dtype=tf.int64)) bits = tf.reshape(bits, [-1, 64]) bits = tf.reduce_sum(tf.to_int64(bits) * pow_2, axis=1) bits = tf.reshape(bits, [-1, num_bits // 64]) end_points = {'Pivots': pivots, 'Radii': radii} return bits, end_points
def normalize_example(self, example, hparams): """Assumes that example contains both inputs and targets.""" length = self.max_length(hparams) def _to_constant_shape(tensor): tensor = tensor[:length] tensor = tf.pad(tensor, [(0, length - tf.shape(tensor)[0])]) return tf.reshape(tensor, [length]) if self.has_inputs: example['inputs'] = _to_constant_shape(example['inputs']) example['targets'] = _to_constant_shape(example['targets']) elif 'inputs' in example: if self.packed_length: raise ValueError('cannot concatenate packed examples on the fly.') inputs = example.pop('inputs')[:-1] # Remove EOS token. targets = tf.concat([inputs, example['targets']], 0) example['targets'] = _to_constant_shape(targets) else: example['targets'] = _to_constant_shape(example['targets']) if self.packed_length: if self.has_inputs: if 'inputs_segmentation' in example: example['inputs_segmentation'] = _to_constant_shape( example['inputs_segmentation']) example['inputs_position'] = _to_constant_shape( example['inputs_position']) else: example['inputs_segmentation'] = tf.to_int64( tf.not_equal(example['inputs'], 0)) example['inputs_position'] = ( example['inputs_segmentation'] * tf.range(length, dtype=tf.int64)) if 'targets_segmentation' in example: example['targets_segmentation'] = _to_constant_shape( example['targets_segmentation']) example['targets_position'] = _to_constant_shape( example['targets_position']) else: example['targets_segmentation'] = tf.to_int64( tf.not_equal(example['targets'], 0)) example['targets_position'] = ( example['targets_segmentation'] * tf.range(length, dtype=tf.int64)) return example
def _convert_ids_to_strings(tgt_vocab_file, ids): """Convert prediction ids to words.""" with tf.Session() as sess: reverse_target_vocab_table = lookup_ops.index_to_string_table_from_file( tgt_vocab_file, default_value=vocab_utils.UNK) sess.run(tf.tables_initializer()) translations = sess.run( reverse_target_vocab_table.lookup( tf.to_int64(tf.convert_to_tensor(np.asarray(ids))))) return translations
def _dedup_tensor(sp_tensor: tf.SparseTensor) -> tf.SparseTensor: """Dedup values of a SparseTensor along each row. Args: sp_tensor: A 2D SparseTensor to be deduped. Returns: A deduped SparseTensor of shape [batch_size, max_len], where max_len is the maximum number of unique values for a row in the Tensor. """ string_batch_index = tf.as_string(sp_tensor.indices[:, 0]) # tf.unique only works on 1D tensors. To avoid deduping across examples, # prepend each feature value with the example index. This requires casting # to and from strings for non-string features. string_values = sp_tensor.values original_dtype = sp_tensor.values.dtype if original_dtype != tf.string: string_values = tf.as_string(sp_tensor.values) index_and_value = tf.strings.join([string_batch_index, string_values], separator='|') unique_index_and_value, _ = tf.unique(index_and_value) # split is a shape [tf.size(values), 2] tensor. The first column contains # indices and the second column contains the feature value (we assume no # feature contains | so we get exactly 2 values from the string split). split = tf.string_split(unique_index_and_value, delimiter='|') split = tf.reshape(split.values, [-1, 2]) string_indices = split[:, 0] values = split[:, 1] indices = tf.reshape( tf.string_to_number(string_indices, out_type=tf.int32), [-1]) if original_dtype != tf.string: values = tf.string_to_number(values, out_type=original_dtype) values = tf.reshape(values, [-1]) # Convert example indices into SparseTensor indices, e.g. # [0, 0, 0, 1, 3, 3] -> [[0,0], [0,1], [0,2], [1,0], [3,0], [3,1]] batch_size = tf.to_int32(sp_tensor.dense_shape[0]) new_indices, max_len = _example_index_to_sparse_index(indices, batch_size) return tf.SparseTensor( indices=tf.to_int64(new_indices), values=values, dense_shape=[tf.to_int64(batch_size), max_len])
def streaming_confusion_matrix(labels, predictions, num_classes, weights=None): """Calculate a streaming confusion matrix. Calculates a confusion matrix. For estimation over a stream of data, the function creates an `update_op` operation. Args: labels: A `Tensor` of ground truth labels with shape [batch size] and of type `int32` or `int64`. The tensor will be flattened if its rank > 1. predictions: A `Tensor` of prediction results for semantic labels, whose shape is [batch size] and type `int32` or `int64`. The tensor will be flattened if its rank > 1. num_classes: The possible number of labels the prediction task can have. This value must be provided, since a confusion matrix of dimension = [num_classes, num_classes] will be allocated. weights: Optional `Tensor` whose rank is either 0, or the same rank as `labels`, and must be broadcastable to `labels` (i.e., all dimensions must be either `1`, or the same as the corresponding `labels` dimension). Returns: total_cm: A `Tensor` representing the confusion matrix. update_op: An operation that increments the confusion matrix. """ with tf.variable_scope(None, "streaming_confusion_matrix", [predictions, labels]): # Local variable to accumulate the predictions in the confusion matrix. total_cm = slim.local_variable( tf.zeros([num_classes, num_classes], tf.float64), name="total_confusion_matrix") # Cast the type to int64 required by confusion_matrix_ops. predictions = math_ops.cast(predictions, tf.int64) labels = math_ops.cast(labels, tf.int64) num_classes = math_ops.cast(num_classes, tf.int64) # Flatten the input if its rank > 1. if predictions.get_shape().ndims > 1: predictions = array_ops.reshape(predictions, [-1]) if labels.get_shape().ndims > 1: labels = array_ops.reshape(labels, [-1]) if (weights is not None) and (weights.get_shape().ndims > 1): weights = array_ops.reshape(weights, [-1]) # Accumulate the prediction to current confusion matrix. current_cm = tf.math.confusion_matrix( labels, predictions, tf.to_int64(num_classes), weights=weights, dtype=tf.float64, name="current_confusion_matrix") update_op = state_ops.assign_add(total_cm, current_cm) return total_cm, update_op
def _build_target_distribution(self): """Builds the C51 target distribution as per Bellemare et al. (2017). First, we compute the support of the Bellman target, r + gamma Z'. Where Z' is the support of the next state distribution: * Evenly spaced in [-vmax, vmax] if the current state is nonterminal; * 0 otherwise (duplicated num_atoms times). Second, we compute the next-state probabilities, corresponding to the action with highest expected value. Finally we project the Bellman target (support + probabilities) onto the original support. Returns: target_distribution: tf.tensor, the target distribution from the replay. """ batch_size = self._replay.batch_size # size of rewards: batch_size x 1 rewards = self._replay.rewards[:, None] # size of tiled_support: batch_size x num_atoms tiled_support = tf.tile(self._support, [batch_size]) tiled_support = tf.reshape(tiled_support, [batch_size, self._num_atoms]) # size of target_support: batch_size x num_atoms is_terminal_multiplier = 1. - tf.cast(self._replay.terminals, tf.float32) # Incorporate terminal state to discount factor. # size of gamma_with_terminal: batch_size x 1 gamma_with_terminal = self.cumulative_gamma * is_terminal_multiplier gamma_with_terminal = gamma_with_terminal[:, None] target_support = rewards + gamma_with_terminal * tiled_support # size of next_qt_argmax: 1 x batch_size next_qt_argmax = tf.argmax( self._replay_next_target_net_outputs.q_values, axis=1)[:, None] batch_indices = tf.range(tf.to_int64(batch_size))[:, None] # size of next_qt_argmax: batch_size x 2 batch_indexed_next_qt_argmax = tf.concat( [batch_indices, next_qt_argmax], axis=1) # size of next_probabilities: batch_size x num_atoms next_probabilities = tf.gather_nd( self._replay_next_target_net_outputs.probabilities, batch_indexed_next_qt_argmax) return rainbow_agent.project_distribution(target_support, next_probabilities, self._support)
def hacked_tf_one_hot(indices, depth, on_value, off_value, name=None): '''Emulates new tf.one_hot in master. # Real signature: tf.one_hot(indices, depth, on_value, off_value, axis=None, name=None) # Assumed signature: tf.one_hot(indices, depth, on_value, off_value, axis=-1, name=None) Not needed if using newer versions of TensorFlow. ''' N = tf.shape(indices)[0] range_Nx1 = tf.expand_dims(tf.to_int64(tf.range(N)), 1) indices_Nx1 = tf.expand_dims(indices, 1) concat = tf.concat(1, [range_Nx1, indices_Nx1]) as_dense = tf.sparse_to_dense( concat, tf.to_int64(tf.pack([N, depth])), # Assumption: axis=-1 on_value, off_value) one_hot = tf.reshape(as_dense, (-1, depth), name=name) return one_hot
def loss(self, logits, labels, p_labels, regularization, state): """Adds to the inference model the layers required to generate loss.""" with tf.name_scope('loss'): with tf.name_scope('cross_entropy'): labels = tf.to_int64(labels) print("Shape of labels", labels.get_shape().as_list()) print("Shape of logits", logits.get_shape().as_list()) if labels.get_shape().as_list() == logits.get_shape().as_list(): cross_entropy = tf.nn.softmax_cross_entropy_with_logits(logits=logits, labels=labels) cross_entropy = tf.Print(cross_entropy, [cross_entropy], message="Cross entropy: ") else: cross_entropy = tf.nn.sparse_softmax_cross_entropy_with_logits(logits=logits, labels=labels) y = labels p = p_labels y_1 = tf.equal(y, 1) y_0 = tf.equal(y, 0) p_1 = tf.equal(p, 1) p_0 = tf.equal(p, 0) loss_t0_s0 = tf.reduce_mean(tf.gather(cross_entropy, tf.where(tf.math.logical_and(y_0, p_0)))) loss_t0_s1 = tf.reduce_mean(tf.gather(cross_entropy, tf.where(tf.math.logical_and(y_0, p_1)))) loss_t1_s0 = tf.reduce_mean(tf.gather(cross_entropy, tf.where(tf.math.logical_and(y_1, p_0)))) loss_t1_s1 = tf.reduce_mean(tf.gather(cross_entropy, tf.where(tf.math.logical_and(y_1, p_1)))) cross_entropy = tf.reduce_mean(cross_entropy) cross_entropy = tf.Print(cross_entropy, [cross_entropy], message="Mean cross entropy: ") # print('Cross entropy: ', cross_entropy) with tf.name_scope('regularization'): regularization *= tf.add_n(self.regularizers) loss = cross_entropy \ + (state['eta'] / 4 + (state['lam0'] - state['lam1']) / 2) * loss_t1_s0 \ + (state['eta'] / 4 + (state['lam1'] - state['lam0']) / 2) * loss_t1_s1 # loss = cross_entropy + regularization state['lam0'] = 0.5 * (state['lam0'] - state['lam1']) + 0.5 * state['eta'] * (loss_t1_s0 - loss_t1_s1) state['lam1'] = 0.5 * (state['lam1'] - state['lam0']) + 0.5 * state['eta'] * (loss_t1_s1 - loss_t1_s0) # print( state['lam0']) # Summaries for TensorBoard. tf.summary.scalar('loss/cross_entropy', cross_entropy) tf.summary.scalar('loss/regularization', regularization) tf.summary.scalar('loss/total', loss) with tf.name_scope('averages'): averages = tf.train.ExponentialMovingAverage(0.9) op_averages = averages.apply([cross_entropy, regularization, loss]) tf.summary.scalar('loss/avg/cross_entropy', averages.average(cross_entropy)) tf.summary.scalar('loss/avg/regularization', averages.average(regularization)) tf.summary.scalar('loss/avg/total', averages.average(loss)) with tf.control_dependencies([op_averages]): loss_average = tf.identity(averages.average(loss), name='control') return loss, loss_average, state
def inner_loop( i, hit_eos, next_id, next_id_tag, decoded_ids, decoded_ids_tag, cache, log_prob, ): """One step of greedy decoding.""" logits, logits_tag, cache = symbols_to_logits_fn( next_id, next_id_tag, i, cache) log_probs = common_layers.log_prob_from_logits(logits) temperature = sampling_temperature if hparams.sampling_method == 'random_per_example': next_id = common_layers.sample_temperature_per_example( logits, temperature, top_k) else: if hparams.sampling_method == 'argmax': temperature = 0.0 next_id = common_layers.sample_with_temperature( logits, temperature, top_k) if hparams.sampling_method == 'random_per_example': next_id_tag = common_layers.sample_temperature_per_example( logits_tag, temperature, top_k) else: if hparams.sampling_method == 'argmax': temperature = 0.0 next_id_tag = common_layers.sample_with_temperature( logits_tag, temperature, top_k) log_prob_indices = tf.stack( [tf.range(tf.to_int64(batch_size)), next_id], axis=1) log_prob += tf.gather_nd( log_probs, log_prob_indices) * (1 - tf.to_float(hit_eos)) hit_eos |= tf.equal(next_id, eos_id) next_id = tf.expand_dims(next_id, axis=1) decoded_ids = tf.concat([decoded_ids, next_id], axis=1) next_id_tag = tf.expand_dims(next_id_tag, axis=1) decoded_ids_tag = tf.concat([decoded_ids_tag, next_id_tag], axis=1) return ( i + 1, hit_eos, next_id, next_id_tag, decoded_ids, decoded_ids_tag, cache, log_prob, )
def unpool_layer2x2_batch(bottom, argmax): bottom_shape = tf.shape(bottom) top_shape = [ bottom_shape[0], bottom_shape[1] * 2, bottom_shape[2] * 2, bottom_shape[3] ] batch_size = top_shape[0] height = top_shape[1] width = top_shape[2] channels = top_shape[3] argmax_shape = tf.to_int64([batch_size, height, width, channels]) argmax = unravel_argmax(argmax, argmax_shape) t1 = tf.to_int64(tf.range(channels)) t1 = tf.tile(t1, [batch_size * (width // 2) * (height // 2)]) t1 = tf.reshape(t1, [-1, channels]) t1 = tf.transpose(t1, perm=[1, 0]) t1 = tf.reshape(t1, [channels, batch_size, height // 2, width // 2, 1]) t1 = tf.transpose(t1, perm=[1, 0, 2, 3, 4]) t2 = tf.to_int64(tf.range(batch_size)) t2 = tf.tile(t2, [channels * (width // 2) * (height // 2)]) t2 = tf.reshape(t2, [-1, batch_size]) t2 = tf.transpose(t2, perm=[1, 0]) t2 = tf.reshape(t2, [batch_size, channels, height // 2, width // 2, 1]) t3 = tf.transpose(argmax, perm=[1, 4, 2, 3, 0]) t = tf.concat(4, [t2, t3, t1]) indices = tf.reshape(t, [(height // 2) * (width // 2) * channels * batch_size, 4]) x1 = tf.transpose(bottom, perm=[0, 3, 1, 2]) values = tf.reshape(x1, [-1]) delta = tf.SparseTensor(indices, values, tf.to_int64(top_shape)) return tf.sparse_tensor_to_dense(tf.sparse_reorder(delta))
def _get_action_type(extended_indices, output_vocab_size, model_config): """Returns action_type tensor.""" action_type = tf.constant(0, dtype=tf.int64) for action_type_range in _get_action_types_to_range( output_vocab_size, model_config): index_in_range = tf.logical_and( tf.greater_equal(extended_indices, action_type_range.start_index), tf.less(extended_indices, action_type_range.end_index)) action_type += ( tf.to_int64(index_in_range) * tf.constant(action_type_range.action_type, dtype=tf.int64)) return action_type
def loop_function(prev, _): if output_projection is not None: prev = nn_ops.xw_plus_b(prev, output_projection[0], output_projection[1]) prev_symbol = math_ops.argmax(prev, 1) + tf.to_int64( self.batch_index_bias) # Note that gradients will not propagate through the second parameter of # embedding_lookup. emb_prev = embedding_ops.embedding_lookup(embedding, prev_symbol) if not update_embedding: emb_prev = tf.stop_gradient(emb_prev) return emb_prev
def convert_attribution(attribution, sequence_feature_map, seq_mask, delta_time, attribution_threshold, attribution_max_delta_time, prefix=''): """Constructs the attribution of what inputs result in a higher prediction. Attribution here refers to the timesteps in which the predictions (derived from the logits) increased. We are only interested in increases in the previous attribution_max_delta_time. Args: attribution: A Tensor of shape [batch, max_sequence_length, 1] computed using some attribution method. sequence_feature_map: A dictionary from name to (Sparse)Tensor. seq_mask: A Tensor of shape [batch_size, max_sequence_length, 1] indicating which timesteps are padded. delta_time: A Tensor of shape [batch_size, max_sequence_length] describing the time to prediction. attribution_threshold: Attribution values below this threshold will be dropped. attribution_max_delta_time: Attribution is limited to values that are no older than that many seconds at time of prediction. prefix: A string to prepend to the feature names for the attribution_dict. Returns: A dictionary from feature names to SparseTensors of dense_shape [batch_size, max_sequence_length, 1]. """ # We do not want attribution in the padding. attribution *= seq_mask # We focus on attribution in the past 12h. # [batch_size, max_sequence_length, 1] attribution *= tf.to_float(delta_time < attribution_max_delta_time) # We get rid of low attribution. attribution_indices = tf.where(attribution > attribution_threshold) attribution_values = tf.gather_nd(attribution, attribution_indices) # Now, attribution.indices indicate in the input timesteps which we should # attend to. attribution_dict = {} for feature, sp_feature in sequence_feature_map.items(): # Limitation: This is not going to work for sequence feature in which # the third (last/token) dimension is > 1. In that case only the first # token would be highlighted. attribution_dict[prefix + feature] = tf.sparse.expand_dims( tf.SparseTensor( indices=attribution_indices, values=attribution_values, dense_shape=tf.to_int64(tf.shape(sp_feature))), axis=1) return attribution_dict
def _get_target_embeddings(encoder_input, output_vocab_embeddings_table, decode_steps, model_config): """Get target embeddings from either output table or copied from input. Args: encoder_input: Tensor representing encoder output of shape (batch size, input length, encoder dims). output_vocab_embeddings_table: Embeddings for output vocabulary of shape (output_vocab_size, target embedding dims). decode_steps: DecodeSteps tuple with tensors of shape (batch size, # steps). model_config: ModelConfig proto. Returns: Tensor of shape (batch_size, # steps, target embedding dims) representing unnormalized logits for both copy and generate actions. """ input_length = tf.shape(encoder_input)[1] # Size of one_hot is (batch size, # steps, input length). one_hot = tf.one_hot(decode_steps.action_ids, input_length) # Size of encoder_dims is (batch size, input length, encoder dims). # Size of matrix multiplication is then (batch size, # steps, encoder_dims). copy_embeddings = tf.matmul(one_hot, encoder_input) # Need a linear transformation to ensure copy embeddings are right size. # Shape will then be (batch size, # steps, target_embedding_dims) copy_embeddings = common_layers.linear_transform( copy_embeddings, model_config.model_parameters.target_embedding_dims, "copy_embeddings_transform", ) # Simply get the generate embeddings from the output vocab table. generate_steps = tf.equal( decode_steps.action_types, tf.constant(constants.GENERATE_ACTION, dtype=tf.int64), ) generate_embeddings = common_layers.embedding_lookup( output_vocab_embeddings_table, decode_steps.action_ids * tf.to_int64(generate_steps), ) # For a given step, only use either copy OR generate embeddings. copy_steps = tf.equal(decode_steps.action_types, tf.constant(constants.COPY_ACTION, dtype=tf.int64)) copy_mask = tf.to_float(tf.expand_dims(copy_steps, axis=-1)) generate_mask = tf.to_float(tf.expand_dims(generate_steps, axis=-1)) target_embeddings = (copy_embeddings * copy_mask + generate_embeddings * generate_mask) return target_embeddings
def key_func(unused_1, unused_2, unused_3, src_len, tgt_len): """Calculate bucket_width by maximum source sequence length.""" # Pairs with length [0, bucket_width) go to bucket 0, length # [bucket_width, 2 * bucket_width) go to bucket 1, etc. Pairs with length # over ((num_bucket-1) * bucket_width) words all go into the last bucket. if src_max_len: bucket_width = (src_max_len + num_buckets - 1) // num_buckets else: bucket_width = 10 # Bucket sentence pairs by the length of their source sentence and target # sentence. bucket_id = tf.maximum(src_len // bucket_width, tgt_len // bucket_width) return tf.to_int64(tf.minimum(num_buckets, bucket_id))
def metric_fn(answers, prediction, start, end, yp1, yp2, num_answers): """Compute span accuracies and token F1/EM scores.""" yp1 = tf.expand_dims(yp1, -1) yp2 = tf.expand_dims(yp2, -1) answer_mask = tf.sequence_mask(num_answers) start = tf.to_int64(start) end = tf.to_int64(end) start_correct = tf.reduce_any(tf.equal(start, yp1) & answer_mask, 1) end_correct = tf.reduce_any(tf.equal(end, yp2) & answer_mask, 1) correct = start_correct & end_correct em = tf.py_func(enum_fn(_exact_match_score, dtype='float32'), [prediction, answers, answer_mask], 'float32') f1 = tf.py_func(enum_fn(_f1_score, dtype='float32'), [prediction, answers, answer_mask], 'float32') eval_metric_ops = { # TODO(ddohan): Add other useful metrics 'acc_start': tf.metrics.mean(tf.cast(start_correct, 'float')), 'acc_end': tf.metrics.mean(tf.cast(end_correct, 'float')), 'acc_span': tf.metrics.mean(tf.cast(correct, 'float')), 'em': tf.metrics.mean(em), 'f1': tf.metrics.mean(f1), # Number of questions processed 'num_question': tf.metrics.true_positives(tf.ones([tf.shape(prediction)][0]), tf.ones([tf.shape(prediction)][0])) } return eval_metric_ops
def get_extended_indices(decode_steps, output_vocab_size, model_config): """Convert DecodeSteps into a tensor of extended action ids.""" # This initial value will be broadcast to the length of decode_steps. extended_action_indices = tf.constant(0, dtype=tf.int64) for action_type_range in _get_action_types_to_range( output_vocab_size, model_config): is_type = tf.equal( tf.constant(action_type_range.action_type, dtype=tf.int64), decode_steps.action_types) # For each timestep, exactly one of the action_type_ranges will be added, # so this sum will populate each entry on exactly one iteration. extended_action_indices += ( tf.to_int64(is_type) * (decode_steps.action_ids + action_type_range.start_index)) return extended_action_indices
def _get_action_id(extended_indices, action_types, output_vocab_size: int, model_config: ModelConfig): """Returns action_id tensor.""" # This initial value will be broadcast to the length of decode_steps. action_ids = tf.constant(0, dtype=tf.int64) for action_type_range in _get_action_types_to_range( output_vocab_size, model_config): is_type = tf.equal( tf.constant(action_type_range.action_type, dtype=tf.int64), action_types) # For each timestep, exactly one of the action_type_ranges will be added, # so this sum will populate each entry on exactly one iteration. action_ids += tf.to_int64(is_type) * (extended_indices - action_type_range.start_index) return action_ids
def map_fn_3(src, tgt_in, tgt_out, src_len, tgt_len): # pylint: disable=missing-docstring # Pairs with length [0, bucket_width) go to bucket 0, length # [bucket_width, 2 * bucket_width) go to bucket 1, etc. Pairs with length # over ((num_bucket-1) * bucket_width) words all go into the last bucket. if src_max_len: bucket_width = (src_max_len + num_buckets - 1) // num_buckets else: bucket_width = 10 # Bucket sentence pairs by the length of their source sentence and target # sentence. bucket_id = tf.maximum(src_len // bucket_width, tgt_len // bucket_width) return tf.to_int64( tf.minimum(num_buckets, bucket_id)), src, tgt_in, tgt_out, src_len, tgt_len
def RNN(self, inputs): """""" input_size = inputs.get_shape().as_list()[-1] cell = self.recur_cell(self._config, input_size=input_size, moving_params=self.moving_params) lengths = tf.reshape(tf.to_int64(self.sequence_lengths), [-1]) if self.moving_params is None: ff_keep_prob = self.ff_keep_prob recur_keep_prob = self.recur_keep_prob else: ff_keep_prob = 1 recur_keep_prob = 1 if self.recur_bidir: top_recur, fw_recur, bw_recur = rnn.dynamic_bidirectional_rnn( cell, cell, inputs, lengths, ff_keep_prob=ff_keep_prob, recur_keep_prob=recur_keep_prob, dtype=tf.float32) fw_cell, fw_out = tf.split(1, 2, fw_recur) bw_cell, bw_out = tf.split(1, 2, bw_recur) end_recur = tf.concat(1, [fw_out, bw_out]) top_recur.set_shape([ tf.Dimension(None), tf.Dimension(None), tf.Dimension(2 * self.recur_size) ]) else: top_recur, end_recur = rnn.dynamic_rnn( cell, inputs, lengths, ff_keep_prob=ff_keep_prob, recur_keep_prob=recur_keep_prob, dtype=tf.float32) top_recur.set_shape([ tf.Dimension(None), tf.Dimension(None), tf.Dimension(self.recur_size) ]) return top_recur, end_recur
def fn(x): """ Arguments: keypoints: a float tensor with shape [17, 3]. box: a float tensor with shape [4]. Returns: a float tensor with shape [height, width, 17]. """ keypoints, box = x ymin, xmin, ymax, xmax = tf.unstack(box, axis=0) y, x, v = tf.unstack(keypoints, axis=1) keypoints = tf.stack([y, x], axis=1) part_id = tf.where(v > 0.0) # shape [num_visible, 1] part_id = tf.to_int32(part_id) num_visible = tf.shape(part_id)[0] keypoints = tf.gather(keypoints, tf.squeeze(part_id, 1)) # it has shape [num_visible, 2], they have absolute coordinates # transform keypoints coordinates # to be relative to the box h, w = ymax - ymin, xmax - xmin height, width = CROP_SIZE translation = tf.stack([ymin, xmin]) scaler = tf.to_float(tf.stack([height / h, width / w], axis=0)) keypoints -= translation keypoints *= scaler keypoints = tf.to_int32(tf.round(keypoints)) # it has shape [num_visible, 2] y, x = tf.unstack(keypoints, axis=1) y = tf.clip_by_value(y, 0, height - 1) x = tf.clip_by_value(x, 0, width - 1) keypoints = tf.stack([y, x], axis=1) indices = tf.to_int64(tf.concat([keypoints, part_id], axis=1)) values = tf.ones([num_visible], dtype=tf.float32) binary_map = tf.sparse.SparseTensor( indices, values, dense_shape=[height, width, 17]) binary_map = tf.sparse.to_dense(binary_map, default_value=0, validate_indices=False) return binary_map
def my_fn(features): """Encode all features that are strings and return a dictionary. Args: features: a dictionary Returns: a dictionary """ ret = {} for k, v in features.items(): if v.dtype == tf.string: v = vocabulary.encode_tf(v) v = tf.concat([tf.to_int64(v), [1]], 0) ret[k] = v else: tf.logging.info( "encode_all_features: skipping non-string feature %s:%s", k, v) return ret