def test_list_items(self):
r"""Tests the item names of the output data.
"""
text_data = MonoTextData(self._hparams)
self.assertSetEqual(set(text_data.list_items()),
{"text", "text_ids", "length"})
hparams = copy.deepcopy(self._hparams)
hparams["dataset"]["data_name"] = "data"
text_data = MonoTextData(hparams)
self.assertSetEqual(set(text_data.list_items()),
{"data_text", "data_text_ids", "data_length"})
def _run_and_test(self, hparams):
# Construct database
text_data = MonoTextData(hparams)
self.assertEqual(
text_data.vocab.size,
self._vocab_size + len(text_data.vocab.special_tokens))
iterator = DataIterator(text_data)
for data_batch in iterator:
# Run the logics
self.assertEqual(set(data_batch.keys()),
set(text_data.list_items()))
# Test utterance count
utt_ind = np.sum(data_batch["text_ids"], 2) != 0
utt_cnt = np.sum(utt_ind, 1)
self.assertListEqual(
data_batch[text_data.utterance_cnt_name].tolist(),
utt_cnt.tolist())
if text_data.hparams.dataset.pad_to_max_seq_length:
max_l = text_data.hparams.dataset.max_seq_length
max_l += text_data._decoder.added_length
for x in data_batch['text']:
for xx in x:
self.assertEqual(len(xx), max_l)
for x in data_batch['text_ids']:
for xx in x:
self.assertEqual(len(xx), max_l)
def _run_and_test(self,
hparams,
test_batch_size=False,
test_transform=False):
# Construct database
text_data = MonoTextData(hparams)
self.assertEqual(
text_data.vocab.size,
self._vocab_size + len(text_data.vocab.special_tokens))
iterator = DataIterator(text_data)
for data_batch in iterator:
self.assertEqual(set(data_batch.keys()),
set(text_data.list_items()))
if test_batch_size:
self.assertEqual(len(data_batch['text']),
hparams['batch_size'])
if test_transform:
for i in range(len(data_batch['text'])):
text_ = data_batch['text'][i]
self.assertTrue(text_ in self.upper_cased_text)
max_seq_length = text_data.hparams.dataset.max_seq_length
mode = text_data.hparams.dataset.length_filter_mode
max_l = max_seq_length
if max_seq_length is not None:
if text_data.hparams.dataset.eos_token != '':
max_l += 1
if text_data.hparams.dataset.bos_token != '':
max_l += 1
if max_seq_length == 6:
for length in data_batch['length']:
self.assertLessEqual(length, max_l)
if mode == "discard":
for length in data_batch['length']:
self.assertEqual(length, 5)
elif mode == "truncate":
num_length_6 = 0
for length in data_batch['length']:
num_length_6 += int(length == 6)
self.assertGreater(num_length_6, 0)
else:
raise ValueError("Unknown mode: %s" % mode)
if text_data.hparams.dataset.pad_to_max_seq_length:
for x in data_batch['text']:
self.assertEqual(len(x), max_l)
for x in data_batch['text_ids']:
self.assertEqual(len(x), max_l)