def provide_groundtruth(self, groundtruth_text, scope=None): with tf.name_scope(scope, 'ProvideGroundtruth', [groundtruth_text]): batch_size = shape_utils.combined_static_and_dynamic_shape( groundtruth_text)[0] if self._reverse: groundtruth_text = ops.string_reverse(groundtruth_text) text_labels, text_lengths = self._label_map.text_to_labels( groundtruth_text, pad_value=self.end_label, return_lengths=True) start_labels = tf.fill([batch_size, 1], tf.constant(self.start_label, tf.int64)) end_labels = tf.fill([batch_size, 1], tf.constant(self.end_label, tf.int64)) if not self._sync: decoder_inputs = tf.concat( [start_labels, start_labels, text_labels], axis=1) decoder_targets = tf.concat( [start_labels, text_labels, end_labels], axis=1) decoder_lengths = text_lengths + 2 else: decoder_inputs = tf.concat([start_labels, text_labels], axis=1) decoder_targets = tf.concat([text_labels, end_labels], axis=1) decoder_lengths = text_lengths + 1 # set maximum lengths decoder_inputs = decoder_inputs[:, :self._max_num_steps] decoder_targets = decoder_targets[:, :self._max_num_steps] decoder_lengths = tf.minimum(decoder_lengths, self._max_num_steps) self._groundtruth_dict['decoder_inputs'] = decoder_inputs self._groundtruth_dict['decoder_targets'] = decoder_targets self._groundtruth_dict['decoder_lengths'] = decoder_lengths
def _aggregate_recognition_results(self, text_list, scores_list, scope=None): """Aggregate recognition results by picking up ones with highest scores. Args text_list: a list of tensors with shape [batch_size] scores_list: a list of tensors with shape [batch_size] """ with tf.variable_scope(scope, 'AggregateRecognitionResults', (text_list + scores_list)): stacked_text = tf.stack(text_list, axis=1) stacked_scores = tf.stack(scores_list, axis=1) argmax_scores = tf.argmax(stacked_scores, axis=1) batch_size = shape_utils.combined_static_and_dynamic_shape( stacked_text)[0] indices = tf.stack( [tf.range(batch_size, dtype=tf.int64), argmax_scores], axis=1) aggregated_text = tf.gather_nd(stacked_text, indices) aggregated_scores = tf.gather_nd(stacked_scores, indices) recognition_dict = { 'text': aggregated_text, 'scores': aggregated_scores } return recognition_dict
def __call__(self, logits, labels, lengths, scope=None): """ Args: logits: float32 tensor with shape [batch_size, max_time, num_classes] labels: int32 tensor with shape [batch_size, max_time] lengths: int32 tensor with shape [batch_size] """ with tf.name_scope(scope, 'SequenceCrossEntropyLoss', [logits, labels, lengths]): raw_losses = tf.nn.sparse_softmax_cross_entropy_with_logits( labels=labels, logits=logits) batch_size, max_time = shape_utils.combined_static_and_dynamic_shape( labels) mask = tf.less(tf.tile([tf.range(max_time)], [batch_size, 1]), tf.expand_dims(lengths, 1), name='mask') masked_losses = tf.multiply( raw_losses, tf.cast(mask, tf.float32), name='masked_losses') # => [batch_size, max_time] row_losses = tf.reduce_sum(masked_losses, 1, name='row_losses') if self._sequence_normalize: loss = tf.truediv(row_losses, tf.cast(tf.maximum(lengths, 1), tf.float32), name='seq_normed_losses') loss = tf.reduce_sum(row_losses) if self._sample_normalize: loss = tf.truediv( loss, tf.cast(tf.maximum(batch_size, 1), tf.float32)) if self._weight: loss = loss * self._weight return loss
def _localize(self, preprocessed_images): k = self._num_control_points conv_output = self._convnet.extract_features(preprocessed_images)[-1] batch_size = shape_utils.combined_static_and_dynamic_shape( conv_output)[0] conv_output = tf.reshape(conv_output, [batch_size, -1]) with arg_scope(self._fc_hyperparams): fc1 = fully_connected(conv_output, 512) fc2_weights_initializer = tf.zeros_initializer() fc2_biases_initializer = tf.constant_initializer(self._init_bias) fc2 = fully_connected(0.1 * fc1, 2 * k, weights_initializer=fc2_weights_initializer, biases_initializer=fc2_biases_initializer, activation_fn=None, normalizer_fn=None) if self._summarize_activations: tf.summary.histogram('fc1', fc1) tf.summary.histogram('fc2', fc2) if self._activation == 'sigmoid': ctrl_pts = tf.sigmoid(fc2) elif self._activation == 'none': ctrl_pts = fc2 else: raise ValueError('Unknown activation: {}'.format(self._activation)) ctrl_pts = tf.reshape(ctrl_pts, [batch_size, k, 2]) return ctrl_pts
def extract_features(self, preprocessed_inputs, scope=None): with tf.variable_scope(scope, 'FeatureExtractor', [preprocessed_inputs]): feature_maps = self._convnet.extract_features(preprocessed_inputs) if len(self._brnn_fn_list) > 0: feature_sequences_list = [] for i, feature_map in enumerate(feature_maps): shape_assert = tf.Assert(tf.equal( tf.shape(feature_map)[1], 1 ), [ 'Feature map height must be 1 if bidirectional RNN is going to be applied.' ]) batch_size, _, _, map_depth = shape_utils.combined_static_and_dynamic_shape( feature_map) with tf.control_dependencies([shape_assert]): feature_sequence = tf.reshape(feature_map, [batch_size, -1, map_depth]) for j, brnn_fn in enumerate(self._brnn_fn_list): brnn_object = brnn_fn() feature_sequence = brnn_object.predict( feature_sequence, scope='BidirectionalRnn_Branch_{}_{}'.format(i, j)) feature_sequences_list.append(feature_sequence) feature_maps = [ tf.expand_dims(fmap, axis=1) for fmap in feature_sequences_list ] return feature_maps
def postprocess(self, predictions_dict, scope=None): with tf.variable_scope(scope, 'Postprocess', list(predictions_dict.values())): logits = predictions_dict['logits'] batch_size, max_time, _ = shape_utils.combined_static_and_dynamic_shape(logits) logits_time_major = tf.transpose(logits, [1,0,2]) sparse_labels, log_prob = tf.nn.ctc_greedy_decoder( logits_time_major, tf.fill([batch_size], max_time), merge_repeated=True ) labels = tf.sparse_tensor_to_dense(sparse_labels[0], default_value=-1) text = self._label_map.labels_to_text(labels) recognitions_dict = {'text': text} return recognitions_dict
def loss(self, predictions_dict, scope=None): with tf.variable_scope(scope, 'Loss', list(predictions_dict.values())): logits = predictions_dict['logits'] batch_size, max_time, _ = shape_utils.combined_static_and_dynamic_shape(logits) losses = tf.nn.ctc_loss( tf.cast(self._groundtruth_dict['text_labels_sparse'], tf.int32), predictions_dict['logits'], tf.fill([batch_size], max_time), preprocess_collapse_repeated=False, ctc_merge_repeated=True, ignore_longer_outputs_than_inputs=True, time_major=False) loss = tf.reduce_mean(losses) return {'RecognitionLoss': loss}
def predict(self, feature_maps, scope=None): if not isinstance(feature_maps, (list, tuple)): raise ValueError('`feature_maps` must be list of tuple') with tf.variable_scope(scope, 'Predict', feature_maps): batch_size = shape_utils.combined_static_and_dynamic_shape( feature_maps[0])[0] decoder_cell = self._build_decoder_cell(feature_maps) decoder = self._build_decoder(decoder_cell, batch_size) outputs, _, output_lengths = seq2seq.dynamic_decode( decoder=decoder, output_time_major=False, impute_finished=False, maximum_iterations=self._max_num_steps) # apply regularizer filter_weights = lambda vars: [ x for x in vars if x.op.name.endswith('kernel') ] tf.contrib.layers.apply_regularization( self._rnn_regularizer, filter_weights(decoder_cell.trainable_weights)) outputs_dict = None if self._is_training: assert isinstance(outputs, seq2seq.BasicDecoderOutput) outputs_dict = { 'labels': outputs.sample_id, 'logits': outputs.rnn_output, } else: assert isinstance(outputs, seq2seq.FinalBeamSearchDecoderOutput) prediction_labels = outputs.predicted_ids[:, :, 0] prediction_lengths = output_lengths[:, 0] prediction_scores = tf.gather_nd( outputs.beam_search_decoder_output.scores[:, :, 0], tf.stack([tf.range(batch_size), prediction_lengths - 1], axis=1)) outputs_dict = { 'labels': prediction_labels, 'scores': prediction_scores, 'lengths': prediction_lengths } return outputs_dict
def tile_activation_maps_rows_cols(maps, num_rows, num_cols): """ Args: maps: [batch_size, map_height, map_width, map_depth] Return: tiled_map: [batch_size, tiled_height, tiled_width] """ batch_size, map_height, map_width, map_depth = \ shape_utils.combined_static_and_dynamic_shape(maps) # padding num_maps = num_rows * num_cols padded_map = tf.cond(tf.greater(num_maps, map_depth), true_fn=lambda: tf.pad( maps, [[0, 0], [0, 0], [0, 0], [0, tf.maximum(num_maps - map_depth, 0)]]), false_fn=lambda: maps[:, :, :, :num_maps]) # reshape to [batch_size, map_height, map_width, num_rows, num_cols] reshaped_map = tf.reshape( padded_map, [batch_size, map_height, map_width, num_rows, num_cols]) # unstack and concat along widths width_concated_maps = tf.concat( tf.unstack( reshaped_map, axis=4 ), # => list of [batch_size, map_height, map_width, num_rows] axis=2) # => [batch_size, map_height, map_width * num_cols, num_rows] tiled_map = tf.concat( tf.unstack( width_concated_maps, axis=3 ), # => list of [batch_size, map_height, map_width * num_cols] axis=1) # => [batch_size, map_height * num_rows, map_width * num_cols] tiled_map = tf.expand_dims(tiled_map, axis=3) return tiled_map
def tile_activation_maps_max_dimensions(maps, max_height, max_width): batch_size, map_height, map_width, map_depth = \ shape_utils.combined_static_and_dynamic_shape(maps) num_rows = max_height // map_height num_cols = max_width // map_width return tile_activation_maps_rows_cols(maps, num_rows, num_cols)
def _batch_sample(self, images, batch_sampling_grid): """ Args: images: tensor of any time with shape [batch_size, image_h, image_w, depth] batch_sampling_grid; float32 tensor with shape [batch_size, num_sampling_pts, 2] """ if images.dtype != tf.float32: raise ValueError('image must be of type tf.float32') batch_G = batch_sampling_grid batch_size, image_h, image_w, _ = shape_utils.combined_static_and_dynamic_shape( images) n = shape_utils.combined_static_and_dynamic_shape( batch_sampling_grid)[1] batch_Gx = image_w * batch_G[:, :, 0] batch_Gy = image_h * batch_G[:, :, 1] batch_Gx = tf.clip_by_value(batch_Gx, 0., image_w - 2) batch_Gy = tf.clip_by_value(batch_Gy, 0., image_h - 2) batch_Gx0 = tf.cast(tf.floor(batch_Gx), tf.int32) # G* => [batch_size, n, 2] batch_Gx1 = batch_Gx0 + 1 # G*x, G*y => [batch_size, n] batch_Gy0 = tf.cast(tf.floor(batch_Gy), tf.int32) batch_Gy1 = batch_Gy0 + 1 def _get_pixels(images, batch_x, batch_y, batch_indices): indices = tf.stack([batch_indices, batch_y, batch_x], axis=2) # => [B, n, 3] pixels = tf.gather_nd(images, indices) return pixels batch_indices = tf.tile(tf.expand_dims(tf.range(batch_size), 1), [1, n]) # => [B, n] batch_I00 = _get_pixels(images, batch_Gx0, batch_Gy0, batch_indices) batch_I01 = _get_pixels(images, batch_Gx0, batch_Gy1, batch_indices) batch_I10 = _get_pixels(images, batch_Gx1, batch_Gy0, batch_indices) batch_I11 = _get_pixels(images, batch_Gx1, batch_Gy1, batch_indices) # => [B, n, d] batch_Gx0 = tf.to_float(batch_Gx0) batch_Gx1 = tf.to_float(batch_Gx1) batch_Gy0 = tf.to_float(batch_Gy0) batch_Gy1 = tf.to_float(batch_Gy1) batch_w00 = (batch_Gx1 - batch_Gx) * (batch_Gy1 - batch_Gy) batch_w01 = (batch_Gx1 - batch_Gx) * (batch_Gy - batch_Gy0) batch_w10 = (batch_Gx - batch_Gx0) * (batch_Gy1 - batch_Gy) batch_w11 = (batch_Gx - batch_Gx0) * (batch_Gy - batch_Gy0 ) # => [B, n] batch_pixels = tf.add_n([ tf.expand_dims(batch_w00, axis=2) * batch_I00, tf.expand_dims(batch_w01, axis=2) * batch_I01, tf.expand_dims(batch_w10, axis=2) * batch_I10, tf.expand_dims(batch_w11, axis=2) * batch_I11, ]) output_h, output_w = self._output_image_size output_maps = tf.reshape(batch_pixels, [batch_size, output_h, output_w, -1]) output_maps = tf.cast(output_maps, dtype=images.dtype) if self._summarize_activations: tf.summary.image('InputImage1', images[:2], max_outputs=2) tf.summary.image('InputImage2', images[-2:], max_outputs=2) tf.summary.image('RectifiedImage1', output_maps[:2], max_outputs=2) tf.summary.image('RectifiedImage2', output_maps[-2:], max_outputs=2) return output_maps