def _test_modes_with_workers(self,
lazy_mode: str,
cache_mode: str,
num_workers: int,
parallelize_processing: bool = True,
support_random_access: bool = False,
shuffle: bool = False,
**kwargs):
hparams = {
'batch_size': self.batch_size,
'lazy_strategy': lazy_mode,
'cache_strategy': cache_mode,
'num_parallel_calls': num_workers,
'shuffle': shuffle,
'shuffle_buffer_size': self.size // 5,
'parallelize_processing': parallelize_processing,
'allow_smaller_final_batch': False,
**kwargs,
}
numbers_data = [[x] * self.seq_len for x in range(self.size)]
string_data = [
' '.join(map(str, range(self.seq_len))) for _ in range(self.size)
]
if not support_random_access:
source = ZipDataSource( # type: ignore
IterDataSource(numbers_data), SequenceDataSource(string_data))
else:
source = ZipDataSource(SequenceDataSource(numbers_data),
SequenceDataSource(string_data))
data = MockDataBase(source, hparams) # type: ignore
iterator = DataIterator(data)
if data._hparams.allow_smaller_final_batch:
total_examples = self.size
total_batches = (self.size + self.batch_size -
1) // self.batch_size
else:
total_examples = self.size // self.batch_size * self.batch_size
total_batches = self.size // self.batch_size
def check_batch(idx, batch):
if idx == total_batches - 1:
batch_size = (total_examples - 1) % self.batch_size + 1
else:
batch_size = self.batch_size
self.assertEqual(batch.numbers.shape, (batch_size, self.seq_len))
if not shuffle:
numbers = np.asarray(
[idx * self.batch_size + x + 1 for x in range(batch_size)])
self.assertTrue(np.all(batch.numbers == numbers[:,
np.newaxis]))
# check laziness
if parallelize_processing:
if lazy_mode == 'none':
self.assertEqual(len(data._processed_cache), self.size)
else:
self.assertEqual(len(data._processed_cache), 0)
if not support_random_access:
if lazy_mode == 'process':
self.assertEqual(len(data._cached_source._cache),
self.size)
else:
self.assertEqual(len(data._cached_source._cache), 0)
# first epoch
cnt = 0
for idx, batch in enumerate(iterator):
check_batch(idx, batch)
cnt += 1
self.assertEqual(cnt, total_batches)
# check cache
if parallelize_processing:
if cache_mode == 'none':
self.assertEqual(len(data._processed_cache), 0)
elif cache_mode == 'loaded':
self.assertEqual(len(data._processed_cache), 0)
else:
self.assertEqual(len(data._processed_cache), self.size)
if lazy_mode != 'none' and not support_random_access:
if cache_mode == 'none':
self.assertEqual(len(data._cached_source._cache), 0)
elif cache_mode == 'loaded':
self.assertEqual(len(data._cached_source._cache),
self.size)
else:
self.assertEqual(len(data._cached_source._cache), 0)
# second epoch
cnt = 0
for idx, batch in enumerate(iterator):
check_batch(idx, batch)
cnt += 1
self.assertEqual(cnt, total_batches)
# check again
if parallelize_processing:
if cache_mode == 'none':
self.assertEqual(len(data._processed_cache), 0)
elif cache_mode == 'loaded':
self.assertEqual(len(data._processed_cache), 0)
else:
self.assertEqual(len(data._processed_cache), self.size)
if lazy_mode != 'none' and not support_random_access:
if cache_mode == 'none':
self.assertEqual(len(data._cached_source._cache), 0)
elif cache_mode == 'loaded':
self.assertEqual(len(data._cached_source._cache),
self.size)
else:
self.assertEqual(len(data._cached_source._cache), 0)
def __init__(self, hparams, device: Optional[torch.device] = None):
self._hparams = HParams(hparams, self.default_hparams())
src_hparams = self.hparams.source_dataset
tgt_hparams = self.hparams.target_dataset
# create vocabulary
self._src_bos_token = src_hparams["bos_token"]
self._src_eos_token = src_hparams["eos_token"]
self._src_transforms = src_hparams["other_transformations"]
self._src_vocab = Vocab(src_hparams.vocab_file,
bos_token=src_hparams.bos_token,
eos_token=src_hparams.eos_token)
if tgt_hparams["processing_share"]:
self._tgt_bos_token = src_hparams["bos_token"]
self._tgt_eos_token = src_hparams["eos_token"]
else:
self._tgt_bos_token = tgt_hparams["bos_token"]
self._tgt_eos_token = tgt_hparams["eos_token"]
tgt_bos_token = utils.default_str(self._tgt_bos_token,
SpecialTokens.BOS)
tgt_eos_token = utils.default_str(self._tgt_eos_token,
SpecialTokens.EOS)
if tgt_hparams["vocab_share"]:
if tgt_bos_token == self._src_vocab.bos_token and \
tgt_eos_token == self._src_vocab.eos_token:
self._tgt_vocab = self._src_vocab
else:
self._tgt_vocab = Vocab(src_hparams["vocab_file"],
bos_token=tgt_bos_token,
eos_token=tgt_eos_token)
else:
self._tgt_vocab = Vocab(tgt_hparams["vocab_file"],
bos_token=tgt_bos_token,
eos_token=tgt_eos_token)
# create embeddings
self._src_embedding = MonoTextData.make_embedding(
src_hparams.embedding_init, self._src_vocab.token_to_id_map_py)
if self._hparams.target_dataset.embedding_init_share:
self._tgt_embedding = self._src_embedding
else:
tgt_emb_file = tgt_hparams.embedding_init["file"]
self._tgt_embedding = None
if tgt_emb_file is not None and tgt_emb_file != "":
self._tgt_embedding = MonoTextData.make_embedding(
self._tgt_vocab.token_to_id_map_py,
tgt_hparams.embedding_init)
# create data source
self._src_delimiter = src_hparams.delimiter
self._src_max_seq_length = src_hparams.max_seq_length
self._src_length_filter_mode = _LengthFilterMode(
src_hparams.length_filter_mode)
self._src_pad_length = self._src_max_seq_length
if self._src_pad_length is not None:
self._src_pad_length += sum(
int(x is not None and x != '')
for x in [src_hparams.bos_token, src_hparams.eos_token])
src_data_source = TextLineDataSource(
src_hparams.files, compression_type=src_hparams.compression_type)
self._tgt_transforms = tgt_hparams["other_transformations"]
self._tgt_delimiter = tgt_hparams.delimiter
self._tgt_max_seq_length = tgt_hparams.max_seq_length
self._tgt_length_filter_mode = _LengthFilterMode(
tgt_hparams.length_filter_mode)
self._tgt_pad_length = self._tgt_max_seq_length
if self._tgt_pad_length is not None:
self._tgt_pad_length += sum(
int(x is not None and x != '')
for x in [tgt_hparams.bos_token, tgt_hparams.eos_token])
tgt_data_source = TextLineDataSource(
tgt_hparams.files, compression_type=tgt_hparams.compression_type)
data_source: DataSource[Tuple[List[str], List[str]]]
data_source = ZipDataSource( # type: ignore
src_data_source, tgt_data_source)
if ((self._src_length_filter_mode is _LengthFilterMode.DISCARD
and self._src_max_seq_length is not None)
or (self._tgt_length_filter_mode is _LengthFilterMode.DISCARD
and self._tgt_length_filter_mode is not None)):
max_source_length = self._src_max_seq_length or math.inf
max_tgt_length = self._tgt_max_seq_length or math.inf
def filter_fn(raw_example):
return (len(raw_example[0]) <= max_source_length
and len(raw_example[1]) <= max_tgt_length)
data_source = FilterDataSource(data_source, filter_fn)
super().__init__(data_source, hparams, device=device)
def __init__(self, hparams, device: Optional[torch.device] = None):
self._hparams = HParams(hparams, self.default_hparams())
# Defaultizes hyperparameters of each dataset
datasets_hparams = self._hparams.datasets
defaultized_datasets_hparams = []
for hparams_i in datasets_hparams:
data_type = hparams_i.get("data_type", None)
defaultized_ds_hpms = HParams(hparams_i,
_default_dataset_hparams(data_type))
defaultized_datasets_hparams.append(defaultized_ds_hpms)
self._hparams.datasets = defaultized_datasets_hparams
self._vocab = self.make_vocab(self._hparams.datasets)
self._embedding = self.make_embedding(self._hparams.datasets,
self._vocab)
dummy_source = SequenceDataSource[Any]([])
name_prefix: List[str] = []
self._names: List[Dict[str, Any]] = []
sources: List[DataSource] = []
filters: List[Optional[Callable[[str], bool]]] = []
self._databases: List[DataBase] = []
for idx, hparams_i in enumerate(self._hparams.datasets):
data_type = _DataType(hparams_i.data_type)
source_i: DataSource
if _is_text_data(data_type):
source_i = TextLineDataSource(
hparams_i.files,
compression_type=hparams_i.compression_type,
delimiter=hparams_i.delimiter)
sources.append(source_i)
if ((hparams_i.length_filter_mode
== _LengthFilterMode.DISCARD.value)
and hparams_i.max_seq_length is not None):
def _get_filter(max_seq_length):
return lambda x: len(x) <= max_seq_length
filters.append(_get_filter(hparams_i.max_seq_length))
else:
filters.append(None)
self._names.append({
field: connect_name(hparams_i.data_name, field)
for field in ["text", "text_ids", "length"]
})
dataset_hparams = dict_fetch(
hparams_i,
MonoTextData.default_hparams()["dataset"])
dataset_hparams["data_name"] = None
self._databases.append(
MonoTextData(hparams={"dataset": dataset_hparams},
device=device,
vocab=self._vocab[idx],
embedding=self._embedding[idx],
data_source=dummy_source))
elif _is_scalar_data(data_type):
source_i = TextLineDataSource(
hparams_i.files,
compression_type=hparams_i.compression_type)
sources.append(source_i)
filters.append(None)
self._names.append({"data": hparams_i.data_name})
dataset_hparams = dict_fetch(
hparams_i,
ScalarData.default_hparams()["dataset"])
dataset_hparams["data_name"] = "data"
self._databases.append(
ScalarData(hparams={"dataset": dataset_hparams},
device=device,
data_source=dummy_source))
elif _is_record_data(data_type):
source_i = PickleDataSource(file_paths=hparams_i.files)
sources.append(source_i)
self._names.append({
name: connect_name(hparams_i.data_name, name)
for name in hparams_i.feature_original_types.keys()
})
filters.append(None)
dataset_hparams = dict_fetch(
hparams_i,
RecordData.default_hparams()["dataset"])
self._databases.append(
RecordData(hparams={"dataset": dataset_hparams},
device=device,
data_source=dummy_source))
else:
raise ValueError(f"Unknown data type: {hparams_i.data_type}")
# check for duplicate names
for i in range(1, len(name_prefix)):
if name_prefix[i] in name_prefix[:i - 1]:
raise ValueError(f"Duplicate data name: {name_prefix[i]}")
name_prefix.append(hparams_i["data_name"])
self._name_to_id = {v: k for k, v in enumerate(name_prefix)}
data_source: DataSource = ZipDataSource(*sources)
if any(filters):
def filter_fn(data):
return all(
fn(data) for fn, data in zip(filters, data)
if fn is not None)
data_source = FilterDataSource(data_source, filter_fn=filter_fn)
super().__init__(data_source, self._hparams, device)
def __init__(self, hparams, device: Optional[torch.device] = None):
print("Using local texar")
self._hparams = HParams(hparams, self.default_hparams())
# Defaultizes hyperparameters of each dataset
datasets_hparams = self._hparams.datasets
defaultized_datasets_hparams = []
for hparams_i in datasets_hparams:
data_type = hparams_i.get("data_type", None)
#print("data_type:", data_type)
defaultized_ds_hpms = HParams(hparams_i,
_default_dataset_hparams(data_type))
defaultized_datasets_hparams.append(defaultized_ds_hpms)
self._hparams.datasets = defaultized_datasets_hparams
#print("will make_vocab")
self._vocab = self.make_vocab(self._hparams.datasets)
#print("will make_embedding")
self._embedding = self.make_embedding(self._hparams.datasets,
self._vocab)
dummy_source = SequenceDataSource[Any]([])
name_prefix: List[str] = []
self._names: List[Dict[str, Any]] = []
sources: List[DataSource] = []
filters: List[Optional[Callable[[str], bool]]] = []
self._databases: List[DatasetBase] = []
for idx, hparams_i in enumerate(self._hparams.datasets):
data_type = hparams_i.data_type
source_i: DataSource
if _is_text_data(data_type):
#print("will TextLineDataSource")
source_i = TextLineDataSource(
hparams_i.files,
compression_type=hparams_i.compression_type,
delimiter=hparams_i.delimiter)
sources.append(source_i)
if ((hparams_i.length_filter_mode
== _LengthFilterMode.DISCARD.value)
and hparams_i.max_seq_length is not None):
def _get_filter(max_seq_length):
return lambda x: len(x) <= max_seq_length
filters.append(_get_filter(hparams_i.max_seq_length))
else:
filters.append(None)
self._names.append({
field: connect_name(hparams_i.data_name, field)
for field in ["text", "text_ids", "length"]
})
dataset_hparams = dict_fetch(
hparams_i,
MonoTextData.default_hparams()["dataset"])
dataset_hparams["data_name"] = None
self._databases.append(
MonoTextData(hparams={"dataset": dataset_hparams},
device=device,
vocab=self._vocab[idx],
embedding=self._embedding[idx],
data_source=dummy_source))
elif _is_scalar_data(data_type):
source_i = TextLineDataSource(
hparams_i.files,
compression_type=hparams_i.compression_type)
sources.append(source_i)
filters.append(None)
self._names.append({"data": hparams_i.data_name})
dataset_hparams = dict_fetch(
hparams_i,
ScalarData.default_hparams()["dataset"])
dataset_hparams["data_name"] = "data"
self._databases.append(
ScalarData(hparams={"dataset": dataset_hparams},
device=device,
data_source=dummy_source))
elif _is_record_data(data_type):
source_i = PickleDataSource(file_paths=hparams_i.files)
sources.append(source_i)
# TODO: Only check `feature_types` when we finally remove
# `feature_original_types`.
feature_types = (hparams_i.feature_types
or hparams_i.feature_original_types)
self._names.append({
name: connect_name(hparams_i.data_name, name)
for name in feature_types.keys()
})
filters.append(None)
dataset_hparams = dict_fetch(
hparams_i,
RecordData.default_hparams()["dataset"])
self._databases.append(
RecordData(hparams={"dataset": dataset_hparams},
device=device,
data_source=dummy_source))
else:
raise ValueError(f"Unknown data type: {hparams_i.data_type}")
# check for duplicate names
for i in range(1, len(name_prefix)):
if name_prefix[i] in name_prefix[:i - 1]:
raise ValueError(f"Duplicate data name: {name_prefix[i]}")
name_prefix.append(hparams_i["data_name"])
self._name_to_id = {v: k for k, v in enumerate(name_prefix)}
self._processed_cache = []
self._datafile_id = 0 # for training from multiple files
self._index_at_beginning_of_this_dataset = 0
self._datafile_prefix = hparams_i.files
#self._datafile_num = 33 # hparams_i.datafile_num
#self._datafile_num = 64 # hparams_i.datafile_num
#self._datafile_num = 3 # hparams_i.datafile_num
#self._datafile_num = 16 # hparams_i.datafile_num
#self._datafile_num = 26 # hparams_i.datafile_num
self._datafile_num = 1 # hparams_i.datafile_num
#self._datafile_num = 3 # hparams_i.datafile_num
data_source: DataSource = ZipDataSource(*sources)
if any(filters):
def filter_fn(data):
return all(
fn(data) for fn, data in zip(filters, data)
if fn is not None)
data_source = FilterDataSource(data_source, filter_fn=filter_fn)
#print("data init derive done")
super(MultiAlignedData, self).__init__(data_source, self._hparams,
device)
#self._dataset_size = 3000000
#self._dataset_size = 6400000
#self._dataset_size = 16000000
#self._dataset_size = 3802215
#self._dataset_size = 1250000
#self._dataset_size = 3000
self._dataset_size = 834229