def main(_):
hparams = {
HP_TOKEN_TYPE: HP_TOKEN_TYPE.domain.values[1],
HP_VOCAB_SIZE: HP_VOCAB_SIZE.domain.values[0],
# Preprocessing
HP_MEL_BINS: HP_MEL_BINS.domain.values[0],
HP_FRAME_LENGTH: HP_FRAME_LENGTH.domain.values[0],
HP_FRAME_STEP: HP_FRAME_STEP.domain.values[0],
HP_HERTZ_LOW: HP_HERTZ_LOW.domain.values[0],
HP_HERTZ_HIGH: HP_HERTZ_HIGH.domain.values[0]
}
_hparams = {k.name: v for k, v in hparams.items()}
texts_gen = common_voice.texts_generator(FLAGS.data_dir)
encoder_fn, decoder_fn, vocab_size = encoding.build_encoder(texts_gen,
output_dir=FLAGS.output_dir, hparams=_hparams)
_hparams[HP_VOCAB_SIZE.name] = vocab_size
train_dataset, train_size = common_voice.load_dataset(
FLAGS.data_dir, 'train')
print('Train size:', train_size)
train_dataset = preprocessing.preprocess_dataset(
train_dataset,
encoder_fn=encoder_fn,
hparams=_hparams)
write_dataset(train_dataset, train_size, 'train')
dev_dataset, dev_size = common_voice.load_dataset(
FLAGS.data_dir, 'dev')
print('Dev size:', dev_size)
dev_dataset = preprocessing.preprocess_dataset(
dev_dataset,
encoder_fn=encoder_fn,
hparams=_hparams)
write_dataset(dev_dataset, dev_size, 'dev')
test_dataset, test_size = common_voice.load_dataset(
FLAGS.data_dir, 'test')
print('Test size:', test_size)
test_dataset = preprocessing.preprocess_dataset(
test_dataset,
encoder_fn=encoder_fn,
hparams=_hparams)
write_dataset(test_dataset, test_size, 'test')
def _dataset_fn(name):
dataset, dataset_size = common_voice.load_dataset(base_path, name)
dataset = preprocessing.preprocess_dataset(dataset,
encoder_fn=encoder_fn,
batch_size=batch_size,
hparams=hparams)
steps_per_epoch = dataset_size // batch_size
return dataset, steps_per_epoch
def _dataset_fn(suffix, base_path, task):
dataset, dataset_size = babi_dialog.load_dataset(suffix=suffix,
base_path=base_path,
hparams=hparams,
task=task)
dataset = preprocessing.preprocess_dataset(
dataset,
tokenizer_fn=tokenizer_fn,
vocab_table=vocab_table,
candidates_table=candidates_table,
hparams=hparams)
return dataset, dataset_size
def main(_):
hparams = {
HP_TOKEN_TYPE: HP_TOKEN_TYPE.domain.values[1],
HP_VOCAB_SIZE: HP_VOCAB_SIZE.domain.values[0],
# Preprocessing
HP_MEL_BINS: HP_MEL_BINS.domain.values[0],
HP_FRAME_LENGTH: HP_FRAME_LENGTH.domain.values[0],
HP_FRAME_STEP: HP_FRAME_STEP.domain.values[0],
HP_HERTZ_LOW: HP_HERTZ_LOW.domain.values[0],
HP_HERTZ_HIGH: HP_HERTZ_HIGH.domain.values[0],
HP_DOWNSAMPLE_FACTOR: HP_DOWNSAMPLE_FACTOR.domain.values[0]
}
train_splits = ['dev-clean']
dev_splits = ['dev-clean']
test_splits = ['dev-clean']
# train_splits = [
# 'train-clean-100',
# 'train-clean-360',
# 'train-other-500'
# ]
# dev_splits = [
# 'dev-clean',
# 'dev-other'
# ]
# test_splits = [
# 'test-clean',
# 'test-other'
# ]
_hparams = {k.name: v for k, v in hparams.items()}
texts_gen = librispeech.texts_generator(FLAGS.data_dir,
split_names=train_splits)
encoder_fn, decoder_fn, vocab_size = encoding.get_encoder(
output_dir=FLAGS.output_dir,
hparams=_hparams,
texts_generator=texts_gen)
_hparams[HP_VOCAB_SIZE.name] = vocab_size
train_dataset = librispeech.load_dataset(FLAGS.data_dir, train_splits)
dev_dataset = librispeech.load_dataset(FLAGS.data_dir, dev_splits)
test_dataset = librispeech.load_dataset(FLAGS.data_dir, test_splits)
train_dataset = preprocessing.preprocess_dataset(
train_dataset,
encoder_fn=encoder_fn,
hparams=_hparams,
max_length=FLAGS.max_length,
save_plots=True)
write_dataset(train_dataset, 'train')
dev_dataset = preprocessing.preprocess_dataset(dev_dataset,
encoder_fn=encoder_fn,
hparams=_hparams,
max_length=FLAGS.max_length)
write_dataset(dev_dataset, 'dev')
test_dataset = preprocessing.preprocess_dataset(
test_dataset,
encoder_fn=encoder_fn,
hparams=_hparams,
max_length=FLAGS.max_length)
write_dataset(test_dataset, 'test')