def prepare_training_datasets(
config: Config,
speech_featurizer: SpeechFeaturizer,
text_featurizer: TextFeaturizer,
tfrecords: bool = False,
metadata: str = None,
):
if tfrecords:
train_dataset = asr_dataset.ASRTFRecordDataset(
speech_featurizer=speech_featurizer,
text_featurizer=text_featurizer,
**vars(config.learning_config.train_dataset_config),
indefinite=True
)
eval_dataset = asr_dataset.ASRTFRecordDataset(
speech_featurizer=speech_featurizer,
text_featurizer=text_featurizer,
**vars(config.learning_config.eval_dataset_config),
indefinite=True
)
else:
train_dataset = asr_dataset.ASRSliceDataset(
speech_featurizer=speech_featurizer,
text_featurizer=text_featurizer,
**vars(config.learning_config.train_dataset_config),
indefinite=True
)
eval_dataset = asr_dataset.ASRSliceDataset(
speech_featurizer=speech_featurizer,
text_featurizer=text_featurizer,
**vars(config.learning_config.eval_dataset_config),
indefinite=True
)
train_dataset.load_metadata(metadata)
eval_dataset.load_metadata(metadata)
return train_dataset, eval_dataset
if args.sentence_piece:
logger.info("Loading SentencePiece model ...")
text_featurizer = text_featurizers.SentencePieceFeaturizer(
config.decoder_config)
elif args.subwords:
logger.info("Loading subwords ...")
text_featurizer = text_featurizers.SubwordFeaturizer(config.decoder_config)
else:
logger.info("Use characters ...")
text_featurizer = text_featurizers.CharFeaturizer(config.decoder_config)
if args.tfrecords:
train_dataset = asr_dataset.ASRTFRecordDataset(
speech_featurizer=speech_featurizer,
text_featurizer=text_featurizer,
**vars(config.learning_config.train_dataset_config),
indefinite=True)
eval_dataset = asr_dataset.ASRTFRecordDataset(
speech_featurizer=speech_featurizer,
text_featurizer=text_featurizer,
**vars(config.learning_config.eval_dataset_config),
indefinite=True)
else:
train_dataset = asr_dataset.ASRSliceDataset(
speech_featurizer=speech_featurizer,
text_featurizer=text_featurizer,
**vars(config.learning_config.train_dataset_config),
indefinite=True)
eval_dataset = asr_dataset.ASRSliceDataset(
speech_featurizer=speech_featurizer,
