def test_bucketing(self):
r"""Tests bucketing.
"""
hparams = copy.deepcopy(self._hparams)
hparams.update({
"bucket_boundaries": [7],
"bucket_batch_sizes": [6, 4]
})
text_data = MonoTextData(hparams)
iterator = DataIterator(text_data)
hparams.update({
"bucket_boundaries": [7],
"bucket_batch_sizes": [7, 7],
"allow_smaller_final_batch": False
})
text_data_1 = MonoTextData(hparams)
iterator_1 = DataIterator(text_data_1)
for data_batch, data_batch_1 in zip(iterator, iterator_1):
length = data_batch['length'][0]
if length < 7:
last_batch_size = hparams['num_epochs'] % 6
self.assertTrue(
len(data_batch['text']) == 6
or len(data_batch['text']) == last_batch_size)
else:
last_batch_size = hparams['num_epochs'] % 4
self.assertTrue(
len(data_batch['text']) == 4
or len(data_batch['text']) == last_batch_size)
self.assertEqual(len(data_batch_1['text']), 7)
def test_list_items(self):
r"""Tests the item names of the output data.
"""
text_data = MonoTextData(self._hparams)
self.assertSetEqual(set(text_data.list_items()),
{"text", "text_ids", "length"})
hparams = copy.deepcopy(self._hparams)
hparams["dataset"]["data_name"] = "data"
text_data = MonoTextData(hparams)
self.assertSetEqual(set(text_data.list_items()),
{"data_text", "data_text_ids", "data_length"})
def _run_and_test(self, hparams):
# Construct database
text_data = MonoTextData(hparams)
self.assertEqual(
text_data.vocab.size,
self._vocab_size + len(text_data.vocab.special_tokens))
iterator = DataIterator(text_data)
for data_batch in iterator:
# Run the logics
self.assertEqual(set(data_batch.keys()),
set(text_data.list_items()))
# Test utterance count
utt_ind = np.sum(data_batch["text_ids"], 2) != 0
utt_cnt = np.sum(utt_ind, 1)
self.assertListEqual(
data_batch[text_data.utterance_cnt_name].tolist(),
utt_cnt.tolist())
if text_data.hparams.dataset.pad_to_max_seq_length:
max_l = text_data.hparams.dataset.max_seq_length
max_l += text_data._decoder.added_length
for x in data_batch['text']:
for xx in x:
self.assertEqual(len(xx), max_l)
for x in data_batch['text_ids']:
for xx in x:
self.assertEqual(len(xx), max_l)
def _run_and_test(self,
hparams,
test_batch_size=False,
test_transform=False):
# Construct database
text_data = MonoTextData(hparams)
self.assertEqual(
text_data.vocab.size,
self._vocab_size + len(text_data.vocab.special_tokens))
iterator = DataIterator(text_data)
for data_batch in iterator:
self.assertEqual(set(data_batch.keys()),
set(text_data.list_items()))
if test_batch_size:
self.assertEqual(len(data_batch['text']),
hparams['batch_size'])
if test_transform:
for i in range(len(data_batch['text'])):
text_ = data_batch['text'][i]
self.assertTrue(text_ in self.upper_cased_text)
max_seq_length = text_data.hparams.dataset.max_seq_length
mode = text_data.hparams.dataset.length_filter_mode
max_l = max_seq_length
if max_seq_length is not None:
if text_data.hparams.dataset.eos_token != '':
max_l += 1
if text_data.hparams.dataset.bos_token != '':
max_l += 1
if max_seq_length == 6:
for length in data_batch['length']:
self.assertLessEqual(length, max_l)
if mode == "discard":
for length in data_batch['length']:
self.assertEqual(length, 5)
elif mode == "truncate":
num_length_6 = 0
for length in data_batch['length']:
num_length_6 += int(length == 6)
self.assertGreater(num_length_6, 0)
else:
raise ValueError("Unknown mode: %s" % mode)
if text_data.hparams.dataset.pad_to_max_seq_length:
for x in data_batch['text']:
self.assertEqual(len(x), max_l)
for x in data_batch['text_ids']:
self.assertEqual(len(x), max_l)
def test_train_test_data_iterator(self):
r"""Tests :class:`texar.torch.data.TrainTestDataIterator`
"""
train = MonoTextData(self._train_hparams)
test = MonoTextData(self._test_hparams)
train_batch_size = self._train_hparams["batch_size"]
test_batch_size = self._test_hparams["batch_size"]
data_iterator = TrainTestDataIterator(train=train, test=test)
data_iterator.switch_to_train_data()
iterator = data_iterator.get_iterator()
for idx, val in enumerate(iterator):
self.assertEqual(len(val), train_batch_size)
number = idx * train_batch_size + 1
self.assertEqual(val.text[0], [str(number)])
# numbers: 1 - 2000, first 4 vocab entries are special tokens
self.assertEqual(val.text_ids[0], torch.tensor(number + 3))
data_iterator.switch_to_test_data()
iterator = data_iterator.get_iterator()
for idx, val in enumerate(iterator):
self.assertEqual(len(val), test_batch_size)
number = idx * test_batch_size + 1001
self.assertEqual(val.text[0], [str(number)])
self.assertEqual(val.text_ids[0], torch.tensor(number + 3))
# test `get_*_iterator` interface
for idx, val in enumerate(data_iterator.get_test_iterator()):
self.assertEqual(len(val), test_batch_size)
number = idx * test_batch_size + 1001
self.assertEqual(val.text[0], [str(number)])
self.assertEqual(val.text_ids[0], torch.tensor(number + 3))
# test exception for invalid dataset name
with self.assertRaises(ValueError) as context:
data_iterator.switch_to_val_data()
def test_iterator_multi_datasets(self):
r"""Tests iterating over multiple datasets.
"""
train = MonoTextData(self._train_hparams)
test = MonoTextData(self._test_hparams)
train_batch_size = self._train_hparams["batch_size"]
test_batch_size = self._test_hparams["batch_size"]
data_iterator = DataIterator({"train": train, "test": test})
data_iterator.switch_to_dataset(dataset_name="train")
iterator = data_iterator.get_iterator()
for idx, val in enumerate(iterator):
self.assertEqual(len(val), train_batch_size)
number = idx * train_batch_size + 1
self.assertEqual(val.text[0], [str(number)])
# numbers: 1 - 2000, first 4 vocab entries are special tokens
self.assertEqual(val.text_ids[0], torch.tensor(number + 3))
data_iterator.switch_to_dataset(dataset_name="test")
iterator = data_iterator.get_iterator()
for idx, val in enumerate(iterator):
self.assertEqual(len(val), test_batch_size)
number = idx * test_batch_size + 1001
self.assertEqual(val.text[0], [str(number)])
self.assertEqual(val.text_ids[0], torch.tensor(number + 3))
# test `get_iterator` interface
for idx, val in enumerate(data_iterator.get_iterator('train')):
self.assertEqual(len(val), train_batch_size)
number = idx * train_batch_size + 1
self.assertEqual(val.text[0], [str(number)])
self.assertEqual(val.text_ids[0], torch.tensor(number + 3))
# test exception for invalid dataset name
with self.assertRaises(ValueError) as context:
data_iterator.switch_to_dataset('val')
self.assertTrue('not found' in str(context.exception))
def test_iterator_single_dataset(self):
r"""Tests iterating over a single dataset.
"""
data = MonoTextData(self._test_hparams)
data_iterator = DataIterator(data)
data_iterator.switch_to_dataset(dataset_name="data")
iterator = data_iterator.get_iterator()
i = 1001
for idx, batch in enumerate(iterator):
self.assertEqual(batch.batch_size, self._test_hparams['batch_size'])
np.testing.assert_array_equal(batch['length'], [1, 1])
for example in batch['text']:
self.assertEqual(example[0], str(i))
i += 1
self.assertEqual(i, 2001)
def test_auto_storage_moving(self):
cuda_tensors = set()
def move_tensor(tensor, device, non_blocking=False):
if isinstance(device, torch.device) and device.type == "cuda":
self.assertTrue(non_blocking)
cuda_tensors.add(id(tensor))
return tensor
device = torch.device("cuda:0")
with patch.object(torch.Tensor, "to", move_tensor):
train = MonoTextData(self._train_hparams, device=device)
iterator = DataIterator(train)
for batch in iterator:
self.assertTrue(id(batch.text_ids) in cuda_tensors)
self.assertTrue(id(batch.length) in cuda_tensors)
def __init__(self, hparams, device: Optional[torch.device] = None):
self._hparams = HParams(hparams, self.default_hparams())
# Defaultizes hyperparameters of each dataset
datasets_hparams = self._hparams.datasets
defaultized_datasets_hparams = []
for hparams_i in datasets_hparams:
data_type = hparams_i.get("data_type", None)
defaultized_ds_hpms = HParams(hparams_i,
_default_dataset_hparams(data_type))
defaultized_datasets_hparams.append(defaultized_ds_hpms)
self._hparams.datasets = defaultized_datasets_hparams
self._vocab = self.make_vocab(self._hparams.datasets)
self._embedding = self.make_embedding(self._hparams.datasets,
self._vocab)
dummy_source = SequenceDataSource[Any]([])
name_prefix: List[str] = []
self._names: List[Dict[str, Any]] = []
sources: List[DataSource] = []
filters: List[Optional[Callable[[str], bool]]] = []
self._databases: List[DataBase] = []
for idx, hparams_i in enumerate(self._hparams.datasets):
data_type = _DataType(hparams_i.data_type)
source_i: DataSource
if _is_text_data(data_type):
source_i = TextLineDataSource(
hparams_i.files,
compression_type=hparams_i.compression_type,
delimiter=hparams_i.delimiter)
sources.append(source_i)
if ((hparams_i.length_filter_mode
== _LengthFilterMode.DISCARD.value)
and hparams_i.max_seq_length is not None):
def _get_filter(max_seq_length):
return lambda x: len(x) <= max_seq_length
filters.append(_get_filter(hparams_i.max_seq_length))
else:
filters.append(None)
self._names.append({
field: connect_name(hparams_i.data_name, field)
for field in ["text", "text_ids", "length"]
})
dataset_hparams = dict_fetch(
hparams_i,
MonoTextData.default_hparams()["dataset"])
dataset_hparams["data_name"] = None
self._databases.append(
MonoTextData(hparams={"dataset": dataset_hparams},
device=device,
vocab=self._vocab[idx],
embedding=self._embedding[idx],
data_source=dummy_source))
elif _is_scalar_data(data_type):
source_i = TextLineDataSource(
hparams_i.files,
compression_type=hparams_i.compression_type)
sources.append(source_i)
filters.append(None)
self._names.append({"data": hparams_i.data_name})
dataset_hparams = dict_fetch(
hparams_i,
ScalarData.default_hparams()["dataset"])
dataset_hparams["data_name"] = "data"
self._databases.append(
ScalarData(hparams={"dataset": dataset_hparams},
device=device,
data_source=dummy_source))
elif _is_record_data(data_type):
source_i = PickleDataSource(file_paths=hparams_i.files)
sources.append(source_i)
self._names.append({
name: connect_name(hparams_i.data_name, name)
for name in hparams_i.feature_original_types.keys()
})
filters.append(None)
dataset_hparams = dict_fetch(
hparams_i,
RecordData.default_hparams()["dataset"])
self._databases.append(
RecordData(hparams={"dataset": dataset_hparams},
device=device,
data_source=dummy_source))
else:
raise ValueError(f"Unknown data type: {hparams_i.data_type}")
# check for duplicate names
for i in range(1, len(name_prefix)):
if name_prefix[i] in name_prefix[:i - 1]:
raise ValueError(f"Duplicate data name: {name_prefix[i]}")
name_prefix.append(hparams_i["data_name"])
self._name_to_id = {v: k for k, v in enumerate(name_prefix)}
data_source: DataSource = ZipDataSource(*sources)
if any(filters):
def filter_fn(data):
return all(
fn(data) for fn, data in zip(filters, data)
if fn is not None)
data_source = FilterDataSource(data_source, filter_fn=filter_fn)
super().__init__(data_source, self._hparams, device)
def __init__(self, hparams, device: Optional[torch.device] = None):
print("Using local texar")
self._hparams = HParams(hparams, self.default_hparams())
# Defaultizes hyperparameters of each dataset
datasets_hparams = self._hparams.datasets
defaultized_datasets_hparams = []
for hparams_i in datasets_hparams:
data_type = hparams_i.get("data_type", None)
#print("data_type:", data_type)
defaultized_ds_hpms = HParams(hparams_i,
_default_dataset_hparams(data_type))
defaultized_datasets_hparams.append(defaultized_ds_hpms)
self._hparams.datasets = defaultized_datasets_hparams
#print("will make_vocab")
self._vocab = self.make_vocab(self._hparams.datasets)
#print("will make_embedding")
self._embedding = self.make_embedding(self._hparams.datasets,
self._vocab)
dummy_source = SequenceDataSource[Any]([])
name_prefix: List[str] = []
self._names: List[Dict[str, Any]] = []
sources: List[DataSource] = []
filters: List[Optional[Callable[[str], bool]]] = []
self._databases: List[DatasetBase] = []
for idx, hparams_i in enumerate(self._hparams.datasets):
data_type = hparams_i.data_type
source_i: DataSource
if _is_text_data(data_type):
#print("will TextLineDataSource")
source_i = TextLineDataSource(
hparams_i.files,
compression_type=hparams_i.compression_type,
delimiter=hparams_i.delimiter)
sources.append(source_i)
if ((hparams_i.length_filter_mode
== _LengthFilterMode.DISCARD.value)
and hparams_i.max_seq_length is not None):
def _get_filter(max_seq_length):
return lambda x: len(x) <= max_seq_length
filters.append(_get_filter(hparams_i.max_seq_length))
else:
filters.append(None)
self._names.append({
field: connect_name(hparams_i.data_name, field)
for field in ["text", "text_ids", "length"]
})
dataset_hparams = dict_fetch(
hparams_i,
MonoTextData.default_hparams()["dataset"])
dataset_hparams["data_name"] = None
self._databases.append(
MonoTextData(hparams={"dataset": dataset_hparams},
device=device,
vocab=self._vocab[idx],
embedding=self._embedding[idx],
data_source=dummy_source))
elif _is_scalar_data(data_type):
source_i = TextLineDataSource(
hparams_i.files,
compression_type=hparams_i.compression_type)
sources.append(source_i)
filters.append(None)
self._names.append({"data": hparams_i.data_name})
dataset_hparams = dict_fetch(
hparams_i,
ScalarData.default_hparams()["dataset"])
dataset_hparams["data_name"] = "data"
self._databases.append(
ScalarData(hparams={"dataset": dataset_hparams},
device=device,
data_source=dummy_source))
elif _is_record_data(data_type):
source_i = PickleDataSource(file_paths=hparams_i.files)
sources.append(source_i)
# TODO: Only check `feature_types` when we finally remove
# `feature_original_types`.
feature_types = (hparams_i.feature_types
or hparams_i.feature_original_types)
self._names.append({
name: connect_name(hparams_i.data_name, name)
for name in feature_types.keys()
})
filters.append(None)
dataset_hparams = dict_fetch(
hparams_i,
RecordData.default_hparams()["dataset"])
self._databases.append(
RecordData(hparams={"dataset": dataset_hparams},
device=device,
data_source=dummy_source))
else:
raise ValueError(f"Unknown data type: {hparams_i.data_type}")
# check for duplicate names
for i in range(1, len(name_prefix)):
if name_prefix[i] in name_prefix[:i - 1]:
raise ValueError(f"Duplicate data name: {name_prefix[i]}")
name_prefix.append(hparams_i["data_name"])
self._name_to_id = {v: k for k, v in enumerate(name_prefix)}
self._processed_cache = []
self._datafile_id = 0 # for training from multiple files
self._index_at_beginning_of_this_dataset = 0
self._datafile_prefix = hparams_i.files
#self._datafile_num = 33 # hparams_i.datafile_num
#self._datafile_num = 64 # hparams_i.datafile_num
#self._datafile_num = 3 # hparams_i.datafile_num
#self._datafile_num = 16 # hparams_i.datafile_num
#self._datafile_num = 26 # hparams_i.datafile_num
self._datafile_num = 1 # hparams_i.datafile_num
#self._datafile_num = 3 # hparams_i.datafile_num
data_source: DataSource = ZipDataSource(*sources)
if any(filters):
def filter_fn(data):
return all(
fn(data) for fn, data in zip(filters, data)
if fn is not None)
data_source = FilterDataSource(data_source, filter_fn=filter_fn)
#print("data init derive done")
super(MultiAlignedData, self).__init__(data_source, self._hparams,
device)
#self._dataset_size = 3000000
#self._dataset_size = 6400000
#self._dataset_size = 16000000
#self._dataset_size = 3802215
#self._dataset_size = 1250000
#self._dataset_size = 3000
self._dataset_size = 834229