print("Loading SentencePiece model ...")
text_featurizer = SentencePieceFeaturizer.load_from_file(
config.decoder_config, args.subwords)
elif args.subwords and os.path.exists(args.subwords):
print("Loading subwords ...")
text_featurizer = SubwordFeaturizer.load_from_file(config.decoder_config,
args.subwords)
else:
print("Generating subwords ...")
text_featurizer = SubwordFeaturizer.build_from_corpus(
config.decoder_config, corpus_files=args.subwords_corpus)
text_featurizer.save_to_file(args.subwords)
train_dataset = ASRTFRecordDatasetKeras(
speech_featurizer=speech_featurizer,
text_featurizer=text_featurizer,
**vars(config.learning_config.train_dataset_config),
indefinite=True)
eval_dataset = ASRTFRecordDatasetKeras(
speech_featurizer=speech_featurizer,
text_featurizer=text_featurizer,
**vars(config.learning_config.eval_dataset_config),
indefinite=True)
if args.compute_lengths:
train_dataset.update_lengths(args.metadata_prefix)
eval_dataset.update_lengths(args.metadata_prefix)
# Update metadata calculated from both train and eval datasets
train_dataset.load_metadata(args.metadata_prefix)
eval_dataset.load_metadata(args.metadata_prefix)
print("Loading SentencePiece model ...")
text_featurizer = SentencePieceFeaturizer.load_from_file(config.decoder_config, args.subwords)
elif args.subwords and os.path.exists(args.subwords):
print("Loading subwords ...")
text_featurizer = SubwordFeaturizer.load_from_file(config.decoder_config, args.subwords)
else:
print("Generating subwords ...")
text_featurizer = SubwordFeaturizer.build_from_corpus(
config.decoder_config,
corpus_files=args.subwords_corpus
)
text_featurizer.save_to_file(args.subwords)
if args.tfrecords:
train_dataset = ASRTFRecordDatasetKeras(
speech_featurizer=speech_featurizer, text_featurizer=text_featurizer,
**vars(config.learning_config.train_dataset_config)
)
eval_dataset = ASRTFRecordDatasetKeras(
speech_featurizer=speech_featurizer, text_featurizer=text_featurizer,
**vars(config.learning_config.eval_dataset_config)
)
else:
train_dataset = ASRSliceDatasetKeras(
speech_featurizer=speech_featurizer, text_featurizer=text_featurizer,
**vars(config.learning_config.train_dataset_config)
)
eval_dataset = ASRSliceDatasetKeras(
speech_featurizer=speech_featurizer, text_featurizer=text_featurizer,
**vars(config.learning_config.train_dataset_config)
)