def __call__(self, dataset: tf.data.Dataset = None, model_feature_lengths: Mapping[str, int] = None): if predict_output is None: return [] task = dataset_providers.get_mixture_or_task(task_name) return list( enumerate(task.output_features[target_feature_name].vocabulary. encode(s) for s in predict_output))
def test_preprocessing(task_name, raw_data): """Test the preprocessing functionality of a given task. This function injects `raw_data` into `task` and runs the preprocessing routines from `task`, returning the output of `next(task.get_dataset().as_numpy_iterator())`. Args: task_name: A SeqIO task name. raw_data: A string-keyed dict of string-keyed dicts. The top-level dict should be keyed by dataset splits, and the second-level dict should hold the dataset data. Returns: The result of running the tasks' preprocessing code on `raw_data`. """ with DataInjector(task_name, raw_data): task = dataset_providers.get_mixture_or_task(task_name) return next(task.get_dataset(sequence_length=None).as_numpy_iterator())
def __init__(self, mixture_or_task_name: str, feature_converter: FeatureConverter = EncDecFeatureConverter, eval_split: str = "validation", use_cached: bool = False, sequence_length: Mapping[str, int] = None, summary_dir: Optional[str] = None): """Evaluator constructor. Args: mixture_or_task_name: a registered task or mixture name. feature_converter: a feature converter object to use to convert the task features to model features. Must be a subclass of seqio.FeatureConverter. eval_split: evaluation split. Typically "validation" or "test". use_cached: whether to use the cached dataset instead of processing it on the fly. sequence_length: an optional length specification. If specified, these will be the hard-limit on the evaluation data used for prediction. If none of the preprocessors depend on the sequence length, it can be left unspecified and the maximum length for each feature will be used. These lengths are computed while caching the datasets. summary_dir: an optional directory to save the evaluation results in Event protocol buffer format. Raises: ValueError if `sequence_length` is None but a preprocessor depends on its value. """ logging.info("Initializing Evaluator for '%s'", mixture_or_task_name) eval_tasks = dataset_providers.get_subtasks( dataset_providers.get_mixture_or_task(mixture_or_task_name)) self._eval_tasks = get_valid_eval_tasks(eval_tasks, eval_split) if not self._eval_tasks: logging.warning( "No eval task with valid split and metric fn found. Skipping eval." ) return # Determine if sequence_length arg is required. This occurs when any of the # task preprocessors have a `sequence_length` arg with no default value. sequence_length_required = False for task in eval_tasks: for prep in task.preprocessors: prep_params = inspect.signature(prep).parameters if ("sequence_length" in prep_params and prep_params["sequence_length"].default == inspect.Parameter.empty): if sequence_length is None: raise ValueError( f"Preprocessor '{prep.__name__}' in task '{task.name}' has a " "`sequence_length` argument, making it incompatible with " "automatic sequence length detection. Pass a valid " "`sequence_length` to `Evaluator` and try again.") sequence_length_required = True break def dataset_fn(task: Task) -> tf.data.Dataset: return task.get_dataset(sequence_length=sequence_length, split=eval_split, shuffle=False, use_cached=use_cached) # `task_datasets` have the output features from seqio.Task.get_dataset. # These features will be converted to "model features" by the feature # converter before being cached. cached_targets, cached_task_datasets, max_lengths = ( get_targets_and_examples(tasks=self._eval_tasks, dataset_fn=dataset_fn)) if sequence_length is None: logging.info("Setting sequence lengths to %s", max_lengths) sequence_length = max_lengths elif (sequence_length["inputs"] > max_lengths["inputs"] or sequence_length["targets"] > max_lengths["targets"]): logging.warning( "Given sequence lengths are longer than necessary for some " "evaluation inputs or targets, resulting in wasted computation. " "Consider passing `None` for `sequence_length` to have them be " "automatically computed.\n Got: %s,\n Max Lengths: %s", sequence_length, max_lengths) elif not sequence_length_required and ( sequence_length["inputs"] == max_lengths["inputs"] or sequence_length["targets"] == max_lengths["targets"]): logging.warning( "Given sequence lengths *may be* insufficient for some evaluation " "inputs or targets. Such sequences will be truncated to fit, " "likely leading to sub-optimal results. Consider passing `None` " "for `sequence_length` to have them be automatically computed.\n" ) self._cached_model_datasets = {} # Convert the task features to model features for task in self._eval_tasks: eval_ds = feature_converter(cached_task_datasets[task.name], sequence_length) # The eval dataset is enumerated to ensure that the order is preserved # throughout the entire evaluation process. self._cached_model_datasets[task.name] = eval_ds.enumerate() self._cached_targets = cached_targets self._cached_task_datasets = cached_task_datasets self._model_feature_lengths = feature_converter.get_model_feature_lengths( sequence_length) if summary_dir: with tf.compat.v1.Graph().as_default(): self._summary_writer = tf.compat.v1.summary.FileWriter( summary_dir) else: self._summary_writer = None
def create_prediction(task_name, s, output_feature_name="targets"): task = dataset_providers.get_mixture_or_task(task_name) return [(0, task.output_features[output_feature_name].vocabulary.encode(s)) ]
def encode_str(task_name, s, output_feature_name="targets"): task = dataset_providers.get_mixture_or_task(task_name) return task.output_features[output_feature_name].vocabulary.encode(s)
def __init__(self, task_name, per_split_data): self._task = dataset_providers.get_mixture_or_task(task_name) self.per_split_data = per_split_data self._saved_source = self._task._source
def __init__(self, mixture_or_task_name: str, feature_converter: FeatureConverter = EncDecFeatureConverter, eval_split: str = "validation", use_cached: bool = False, sequence_lengths: Mapping[str, int] = None, summary_dir: Optional[str] = None): """Evaluator constructor. Args: mixture_or_task_name: a registered task or mixture name. feature_converter: a feature converter object to use to convert the task features to model features. Must be a subclass of seqio.FeatureConverter. eval_split: evaluation split. Typically "validation" or "test". use_cached: whether to use the cached dataset instead of processing it on the fly. sequence_lengths: an optional length specification. If specified, these will be the hard-limit on the evaluation data used for prediction. If unspecified, the maximum length for each feature will be used. These lengths are computed while caching the datasets. summary_dir: an optional directory to save the evaluation results in Event protocol buffer format. """ eval_tasks = dataset_providers.get_subtasks( dataset_providers.get_mixture_or_task(mixture_or_task_name)) self._eval_tasks = get_valid_eval_tasks(eval_tasks, eval_split) if not self._eval_tasks: logging.warning( "No eval task with valid split and metric fn found. Skipping eval." ) return def dataset_fn(task: Task) -> tf.data.Dataset: return task.get_dataset(sequence_length=None, split=eval_split, shuffle=False, use_cached=use_cached) # `task_datasets` have the output features from seqio.Task.get_dataset. # These features will be converted to "model features" by the feature # converter before being cached. cached_targets, cached_task_datasets, max_lengths = ( get_targets_and_examples(tasks=self._eval_tasks, dataset_fn=dataset_fn)) if sequence_lengths is None: logging.info("Setting sequence lengths to %s", max_lengths) lengths = max_lengths elif (sequence_lengths["inputs"] < max_lengths["inputs"] or sequence_lengths["targets"] < max_lengths["targets"]): logging.warning( "Given sequence lengths are insufficient for some evaluation inputs " "or targets. These sequences will be truncated to fit, likely " "leading to sub-optimal results. Consider passing `None` for " "sequence_lengths to have them be automatically computed.\n Got: %s, " "\n Max Lengths:%s", sequence_lengths, max_lengths) lengths = sequence_lengths elif (sequence_lengths["inputs"] > max_lengths["inputs"] or sequence_lengths["targets"] > max_lengths["targets"]): logging.warning( "Given sequence lengths are longer than necessary for some " "evaluation inputs or targets, resulting in wasted computation. " "Consider passing `None` for sequence_lengths to have them be " "automatically computed.\n Got: %s,\n Max Lengths: %s", sequence_lengths, max_lengths) lengths = sequence_lengths self._cached_model_datasets = {} # Convert the task features to model features for task in self._eval_tasks: eval_ds = feature_converter(cached_task_datasets[task.name], lengths) # The eval dataset is enumerated to ensure that the order is preserved # throughout the entire evaluation process. self._cached_model_datasets[task.name] = eval_ds.enumerate() self._cached_targets = cached_targets self._cached_task_datasets = cached_task_datasets if summary_dir: with tf.compat.v1.Graph().as_default(): self._summary_writer = tf.compat.v1.summary.FileWriter( summary_dir)