def prepare_testing_datasets(
config: Config,
speech_featurizer: SpeechFeaturizer,
text_featurizer: TextFeaturizer,
):
test_dataset = asr_dataset.ASRSliceDataset(
speech_featurizer=speech_featurizer,
text_featurizer=text_featurizer,
**vars(config.learning_config.test_dataset_config)
)
return test_dataset
def prepare_training_datasets(
config: Config,
speech_featurizer: SpeechFeaturizer,
text_featurizer: TextFeaturizer,
tfrecords: bool = False,
metadata: str = None,
):
if tfrecords:
train_dataset = asr_dataset.ASRTFRecordDataset(
speech_featurizer=speech_featurizer,
text_featurizer=text_featurizer,
**vars(config.learning_config.train_dataset_config),
indefinite=True
)
eval_dataset = asr_dataset.ASRTFRecordDataset(
speech_featurizer=speech_featurizer,
text_featurizer=text_featurizer,
**vars(config.learning_config.eval_dataset_config),
indefinite=True
)
else:
train_dataset = asr_dataset.ASRSliceDataset(
speech_featurizer=speech_featurizer,
text_featurizer=text_featurizer,
**vars(config.learning_config.train_dataset_config),
indefinite=True
)
eval_dataset = asr_dataset.ASRSliceDataset(
speech_featurizer=speech_featurizer,
text_featurizer=text_featurizer,
**vars(config.learning_config.eval_dataset_config),
indefinite=True
)
train_dataset.load_metadata(metadata)
eval_dataset.load_metadata(metadata)
return train_dataset, eval_dataset
if args.tfrecords:
train_dataset = asr_dataset.ASRTFRecordDataset(
speech_featurizer=speech_featurizer,
text_featurizer=text_featurizer,
**vars(config.learning_config.train_dataset_config),
indefinite=True)
eval_dataset = asr_dataset.ASRTFRecordDataset(
speech_featurizer=speech_featurizer,
text_featurizer=text_featurizer,
**vars(config.learning_config.eval_dataset_config),
indefinite=True)
else:
train_dataset = asr_dataset.ASRSliceDataset(
speech_featurizer=speech_featurizer,
text_featurizer=text_featurizer,
**vars(config.learning_config.train_dataset_config),
indefinite=True)
eval_dataset = asr_dataset.ASRSliceDataset(
speech_featurizer=speech_featurizer,
text_featurizer=text_featurizer,
**vars(config.learning_config.eval_dataset_config),
indefinite=True)
train_dataset.load_metadata(args.metadata)
eval_dataset.load_metadata(args.metadata)
if not args.static_length:
speech_featurizer.reset_length()
text_featurizer.reset_length()
