def preprocess_example(self, example, mode, hparams): """Runtime preprocessing. Return a dict or a tf.Data.Datset.from_tensor_slices (if you want each example to turn into multiple). Args: example: dict, features mode: tf.estimator.ModeKeys hparams: HParams, model hyperparameters Returns: dict or Dataset """ global data_alpha, data_beta, desired_alpha, invert_ratio example = preprocess_example_common(example, hparams, mode) if mode == tf.estimator.ModeKeys.TRAIN: #example["inputs"] = usr_utils.print_shape(example["inputs"], "inputs", dtype=tf.int64) tf.logging.info("Filter by length!") if hparams.lt_data_alpha < hparams.lt_desired_alpha: # Normal case: make longer data_alpha = hparams.lt_data_alpha data_beta = hparams.lt_data_beta desired_alpha = hparams.lt_desired_alpha invert_ratio = False else: # Make translation shorter tf.logging.info("Invert ratios (make translations shorter)") data_alpha = 1.0 - hparams.lt_data_alpha data_beta = hparams.lt_data_beta desired_alpha = 1.0 - hparams.lt_desired_alpha invert_ratio = True dataset = tf.data.Dataset.from_tensors(example) filtered = dataset.filter(filter_rejection_sampling) return filtered return example
def preprocess_example(self, example, mode, hparams): # Create loss mask if necessary if hparams.use_loss_mask and mode == tf.estimator.ModeKeys.TRAIN: expanded_target_roots = usr_utils.expand_memory_by_pop_1d( tf.equal(example["targets"], hparams.pop_id), example["target_roots"]) expanded_target_roots = expanded_target_roots[: -1] # Compensate for offset example["loss_mask"] = tf.cast( tf.logical_or( tf.equal(expanded_target_roots, text_encoder.EOS_ID), tf.greater(expanded_target_roots, hparams.max_terminal_id), ), tf.int32) while len(example["loss_mask"].get_shape()) < 3: example["loss_mask"] = tf.expand_dims(example["loss_mask"], axis=-1) # Annoyingly this is done for 'inputs' in problem.serving_input_fn # but not for other input modalities while len(example["target_roots"].get_shape()) < 3: example["target_roots"] = tf.expand_dims(example["target_roots"], axis=-1) return preprocess_example_common(example, hparams, mode)
def preprocess_example(self, example, unused_mode, unused_hparams): example = problem.preprocess_example_common(example, unused_mode, unused_hparams) example["targets"] = tf.reshape(example["targets"], [1, 1, self.ntasks]) return example
def preprocess_example(self, example, mode, hparams): return problem.preprocess_example_common(example, mode, hparams)