def _make_processor(src_hparams, tgt_hparams, data_spec, name_prefix):
# Create source data decoder
data_spec_i = data_spec.get_ith_data_spec(0)
src_decoder, src_trans, data_spec_i = MonoTextData._make_processor(
src_hparams, data_spec_i, chained=False)
data_spec.set_ith_data_spec(0, data_spec_i, 2)
# Create target data decoder
tgt_proc_hparams = tgt_hparams
if tgt_hparams["processing_share"]:
tgt_proc_hparams = copy.copy(src_hparams)
try:
tgt_proc_hparams["variable_utterance"] = \
tgt_hparams["variable_utterance"]
except TypeError:
tgt_proc_hparams.variable_utterance = \
tgt_hparams["variable_utterance"]
data_spec_i = data_spec.get_ith_data_spec(1)
tgt_decoder, tgt_trans, data_spec_i = MonoTextData._make_processor(
tgt_proc_hparams, data_spec_i, chained=False)
data_spec.set_ith_data_spec(1, data_spec_i, 2)
tran_fn = dsutils.make_combined_transformation(
[[src_decoder] + src_trans, [tgt_decoder] + tgt_trans],
name_prefix=name_prefix)
data_spec.add_spec(name_prefix=name_prefix)
return tran_fn, data_spec
def _make_processor(dataset_hparams, data_spec, name_prefix):
processors = []
for i, hparams_i in enumerate(dataset_hparams):
data_spec_i = data_spec.get_ith_data_spec(i)
data_type = hparams_i["data_type"]
if _is_text_data(data_type):
tgt_proc_hparams = hparams_i
proc_shr = hparams_i["processing_share_with"]
if proc_shr is not None:
tgt_proc_hparams = copy.copy(dataset_hparams[proc_shr])
try:
tgt_proc_hparams["variable_utterance"] = \
hparams_i["variable_utterance"]
except TypeError:
tgt_proc_hparams.variable_utterance = \
hparams_i["variable_utterance"]
processor, data_spec_i = MonoTextData._make_processor(
tgt_proc_hparams, data_spec_i)
elif _is_scalar_data(data_type):
processor, data_spec_i = ScalarData._make_processor(
hparams_i, data_spec_i, name_prefix='')
else:
raise ValueError("Unsupported data type: %s" % data_type)
processors.append(processor)
data_spec.set_ith_data_spec(i, data_spec_i, len(dataset_hparams))
tran_fn = dsutils.make_combined_transformation(processors,
name_prefix=name_prefix)
data_spec.add_spec(name_prefix=name_prefix)
return tran_fn, data_spec