def _test_modes_with_workers(self, lazy_mode: str, cache_mode: str, num_workers: int, parallelize_processing: bool = True, support_random_access: bool = False, shuffle: bool = False, **kwargs): hparams = { 'batch_size': self.batch_size, 'lazy_strategy': lazy_mode, 'cache_strategy': cache_mode, 'num_parallel_calls': num_workers, 'shuffle': shuffle, 'shuffle_buffer_size': self.size // 5, 'parallelize_processing': parallelize_processing, 'allow_smaller_final_batch': False, **kwargs, } numbers_data = [[x] * self.seq_len for x in range(self.size)] string_data = [ ' '.join(map(str, range(self.seq_len))) for _ in range(self.size) ] if not support_random_access: source = ZipDataSource( # type: ignore IterDataSource(numbers_data), SequenceDataSource(string_data)) else: source = ZipDataSource(SequenceDataSource(numbers_data), SequenceDataSource(string_data)) data = MockDataBase(source, hparams) # type: ignore iterator = DataIterator(data) if data._hparams.allow_smaller_final_batch: total_examples = self.size total_batches = (self.size + self.batch_size - 1) // self.batch_size else: total_examples = self.size // self.batch_size * self.batch_size total_batches = self.size // self.batch_size def check_batch(idx, batch): if idx == total_batches - 1: batch_size = (total_examples - 1) % self.batch_size + 1 else: batch_size = self.batch_size self.assertEqual(batch.numbers.shape, (batch_size, self.seq_len)) if not shuffle: numbers = np.asarray( [idx * self.batch_size + x + 1 for x in range(batch_size)]) self.assertTrue(np.all(batch.numbers == numbers[:, np.newaxis])) # check laziness if parallelize_processing: if lazy_mode == 'none': self.assertEqual(len(data._processed_cache), self.size) else: self.assertEqual(len(data._processed_cache), 0) if not support_random_access: if lazy_mode == 'process': self.assertEqual(len(data._cached_source._cache), self.size) else: self.assertEqual(len(data._cached_source._cache), 0) # first epoch cnt = 0 for idx, batch in enumerate(iterator): check_batch(idx, batch) cnt += 1 self.assertEqual(cnt, total_batches) # check cache if parallelize_processing: if cache_mode == 'none': self.assertEqual(len(data._processed_cache), 0) elif cache_mode == 'loaded': self.assertEqual(len(data._processed_cache), 0) else: self.assertEqual(len(data._processed_cache), self.size) if lazy_mode != 'none' and not support_random_access: if cache_mode == 'none': self.assertEqual(len(data._cached_source._cache), 0) elif cache_mode == 'loaded': self.assertEqual(len(data._cached_source._cache), self.size) else: self.assertEqual(len(data._cached_source._cache), 0) # second epoch cnt = 0 for idx, batch in enumerate(iterator): check_batch(idx, batch) cnt += 1 self.assertEqual(cnt, total_batches) # check again if parallelize_processing: if cache_mode == 'none': self.assertEqual(len(data._processed_cache), 0) elif cache_mode == 'loaded': self.assertEqual(len(data._processed_cache), 0) else: self.assertEqual(len(data._processed_cache), self.size) if lazy_mode != 'none' and not support_random_access: if cache_mode == 'none': self.assertEqual(len(data._cached_source._cache), 0) elif cache_mode == 'loaded': self.assertEqual(len(data._cached_source._cache), self.size) else: self.assertEqual(len(data._cached_source._cache), 0)
def __init__(self, hparams, device: Optional[torch.device] = None): self._hparams = HParams(hparams, self.default_hparams()) src_hparams = self.hparams.source_dataset tgt_hparams = self.hparams.target_dataset # create vocabulary self._src_bos_token = src_hparams["bos_token"] self._src_eos_token = src_hparams["eos_token"] self._src_transforms = src_hparams["other_transformations"] self._src_vocab = Vocab(src_hparams.vocab_file, bos_token=src_hparams.bos_token, eos_token=src_hparams.eos_token) if tgt_hparams["processing_share"]: self._tgt_bos_token = src_hparams["bos_token"] self._tgt_eos_token = src_hparams["eos_token"] else: self._tgt_bos_token = tgt_hparams["bos_token"] self._tgt_eos_token = tgt_hparams["eos_token"] tgt_bos_token = utils.default_str(self._tgt_bos_token, SpecialTokens.BOS) tgt_eos_token = utils.default_str(self._tgt_eos_token, SpecialTokens.EOS) if tgt_hparams["vocab_share"]: if tgt_bos_token == self._src_vocab.bos_token and \ tgt_eos_token == self._src_vocab.eos_token: self._tgt_vocab = self._src_vocab else: self._tgt_vocab = Vocab(src_hparams["vocab_file"], bos_token=tgt_bos_token, eos_token=tgt_eos_token) else: self._tgt_vocab = Vocab(tgt_hparams["vocab_file"], bos_token=tgt_bos_token, eos_token=tgt_eos_token) # create embeddings self._src_embedding = MonoTextData.make_embedding( src_hparams.embedding_init, self._src_vocab.token_to_id_map_py) if self._hparams.target_dataset.embedding_init_share: self._tgt_embedding = self._src_embedding else: tgt_emb_file = tgt_hparams.embedding_init["file"] self._tgt_embedding = None if tgt_emb_file is not None and tgt_emb_file != "": self._tgt_embedding = MonoTextData.make_embedding( self._tgt_vocab.token_to_id_map_py, tgt_hparams.embedding_init) # create data source self._src_delimiter = src_hparams.delimiter self._src_max_seq_length = src_hparams.max_seq_length self._src_length_filter_mode = _LengthFilterMode( src_hparams.length_filter_mode) self._src_pad_length = self._src_max_seq_length if self._src_pad_length is not None: self._src_pad_length += sum( int(x is not None and x != '') for x in [src_hparams.bos_token, src_hparams.eos_token]) src_data_source = TextLineDataSource( src_hparams.files, compression_type=src_hparams.compression_type) self._tgt_transforms = tgt_hparams["other_transformations"] self._tgt_delimiter = tgt_hparams.delimiter self._tgt_max_seq_length = tgt_hparams.max_seq_length self._tgt_length_filter_mode = _LengthFilterMode( tgt_hparams.length_filter_mode) self._tgt_pad_length = self._tgt_max_seq_length if self._tgt_pad_length is not None: self._tgt_pad_length += sum( int(x is not None and x != '') for x in [tgt_hparams.bos_token, tgt_hparams.eos_token]) tgt_data_source = TextLineDataSource( tgt_hparams.files, compression_type=tgt_hparams.compression_type) data_source: DataSource[Tuple[List[str], List[str]]] data_source = ZipDataSource( # type: ignore src_data_source, tgt_data_source) if ((self._src_length_filter_mode is _LengthFilterMode.DISCARD and self._src_max_seq_length is not None) or (self._tgt_length_filter_mode is _LengthFilterMode.DISCARD and self._tgt_length_filter_mode is not None)): max_source_length = self._src_max_seq_length or math.inf max_tgt_length = self._tgt_max_seq_length or math.inf def filter_fn(raw_example): return (len(raw_example[0]) <= max_source_length and len(raw_example[1]) <= max_tgt_length) data_source = FilterDataSource(data_source, filter_fn) super().__init__(data_source, hparams, device=device)
def __init__(self, hparams, device: Optional[torch.device] = None): self._hparams = HParams(hparams, self.default_hparams()) # Defaultizes hyperparameters of each dataset datasets_hparams = self._hparams.datasets defaultized_datasets_hparams = [] for hparams_i in datasets_hparams: data_type = hparams_i.get("data_type", None) defaultized_ds_hpms = HParams(hparams_i, _default_dataset_hparams(data_type)) defaultized_datasets_hparams.append(defaultized_ds_hpms) self._hparams.datasets = defaultized_datasets_hparams self._vocab = self.make_vocab(self._hparams.datasets) self._embedding = self.make_embedding(self._hparams.datasets, self._vocab) dummy_source = SequenceDataSource[Any]([]) name_prefix: List[str] = [] self._names: List[Dict[str, Any]] = [] sources: List[DataSource] = [] filters: List[Optional[Callable[[str], bool]]] = [] self._databases: List[DataBase] = [] for idx, hparams_i in enumerate(self._hparams.datasets): data_type = _DataType(hparams_i.data_type) source_i: DataSource if _is_text_data(data_type): source_i = TextLineDataSource( hparams_i.files, compression_type=hparams_i.compression_type, delimiter=hparams_i.delimiter) sources.append(source_i) if ((hparams_i.length_filter_mode == _LengthFilterMode.DISCARD.value) and hparams_i.max_seq_length is not None): def _get_filter(max_seq_length): return lambda x: len(x) <= max_seq_length filters.append(_get_filter(hparams_i.max_seq_length)) else: filters.append(None) self._names.append({ field: connect_name(hparams_i.data_name, field) for field in ["text", "text_ids", "length"] }) dataset_hparams = dict_fetch( hparams_i, MonoTextData.default_hparams()["dataset"]) dataset_hparams["data_name"] = None self._databases.append( MonoTextData(hparams={"dataset": dataset_hparams}, device=device, vocab=self._vocab[idx], embedding=self._embedding[idx], data_source=dummy_source)) elif _is_scalar_data(data_type): source_i = TextLineDataSource( hparams_i.files, compression_type=hparams_i.compression_type) sources.append(source_i) filters.append(None) self._names.append({"data": hparams_i.data_name}) dataset_hparams = dict_fetch( hparams_i, ScalarData.default_hparams()["dataset"]) dataset_hparams["data_name"] = "data" self._databases.append( ScalarData(hparams={"dataset": dataset_hparams}, device=device, data_source=dummy_source)) elif _is_record_data(data_type): source_i = PickleDataSource(file_paths=hparams_i.files) sources.append(source_i) self._names.append({ name: connect_name(hparams_i.data_name, name) for name in hparams_i.feature_original_types.keys() }) filters.append(None) dataset_hparams = dict_fetch( hparams_i, RecordData.default_hparams()["dataset"]) self._databases.append( RecordData(hparams={"dataset": dataset_hparams}, device=device, data_source=dummy_source)) else: raise ValueError(f"Unknown data type: {hparams_i.data_type}") # check for duplicate names for i in range(1, len(name_prefix)): if name_prefix[i] in name_prefix[:i - 1]: raise ValueError(f"Duplicate data name: {name_prefix[i]}") name_prefix.append(hparams_i["data_name"]) self._name_to_id = {v: k for k, v in enumerate(name_prefix)} data_source: DataSource = ZipDataSource(*sources) if any(filters): def filter_fn(data): return all( fn(data) for fn, data in zip(filters, data) if fn is not None) data_source = FilterDataSource(data_source, filter_fn=filter_fn) super().__init__(data_source, self._hparams, device)
def __init__(self, hparams, device: Optional[torch.device] = None): print("Using local texar") self._hparams = HParams(hparams, self.default_hparams()) # Defaultizes hyperparameters of each dataset datasets_hparams = self._hparams.datasets defaultized_datasets_hparams = [] for hparams_i in datasets_hparams: data_type = hparams_i.get("data_type", None) #print("data_type:", data_type) defaultized_ds_hpms = HParams(hparams_i, _default_dataset_hparams(data_type)) defaultized_datasets_hparams.append(defaultized_ds_hpms) self._hparams.datasets = defaultized_datasets_hparams #print("will make_vocab") self._vocab = self.make_vocab(self._hparams.datasets) #print("will make_embedding") self._embedding = self.make_embedding(self._hparams.datasets, self._vocab) dummy_source = SequenceDataSource[Any]([]) name_prefix: List[str] = [] self._names: List[Dict[str, Any]] = [] sources: List[DataSource] = [] filters: List[Optional[Callable[[str], bool]]] = [] self._databases: List[DatasetBase] = [] for idx, hparams_i in enumerate(self._hparams.datasets): data_type = hparams_i.data_type source_i: DataSource if _is_text_data(data_type): #print("will TextLineDataSource") source_i = TextLineDataSource( hparams_i.files, compression_type=hparams_i.compression_type, delimiter=hparams_i.delimiter) sources.append(source_i) if ((hparams_i.length_filter_mode == _LengthFilterMode.DISCARD.value) and hparams_i.max_seq_length is not None): def _get_filter(max_seq_length): return lambda x: len(x) <= max_seq_length filters.append(_get_filter(hparams_i.max_seq_length)) else: filters.append(None) self._names.append({ field: connect_name(hparams_i.data_name, field) for field in ["text", "text_ids", "length"] }) dataset_hparams = dict_fetch( hparams_i, MonoTextData.default_hparams()["dataset"]) dataset_hparams["data_name"] = None self._databases.append( MonoTextData(hparams={"dataset": dataset_hparams}, device=device, vocab=self._vocab[idx], embedding=self._embedding[idx], data_source=dummy_source)) elif _is_scalar_data(data_type): source_i = TextLineDataSource( hparams_i.files, compression_type=hparams_i.compression_type) sources.append(source_i) filters.append(None) self._names.append({"data": hparams_i.data_name}) dataset_hparams = dict_fetch( hparams_i, ScalarData.default_hparams()["dataset"]) dataset_hparams["data_name"] = "data" self._databases.append( ScalarData(hparams={"dataset": dataset_hparams}, device=device, data_source=dummy_source)) elif _is_record_data(data_type): source_i = PickleDataSource(file_paths=hparams_i.files) sources.append(source_i) # TODO: Only check `feature_types` when we finally remove # `feature_original_types`. feature_types = (hparams_i.feature_types or hparams_i.feature_original_types) self._names.append({ name: connect_name(hparams_i.data_name, name) for name in feature_types.keys() }) filters.append(None) dataset_hparams = dict_fetch( hparams_i, RecordData.default_hparams()["dataset"]) self._databases.append( RecordData(hparams={"dataset": dataset_hparams}, device=device, data_source=dummy_source)) else: raise ValueError(f"Unknown data type: {hparams_i.data_type}") # check for duplicate names for i in range(1, len(name_prefix)): if name_prefix[i] in name_prefix[:i - 1]: raise ValueError(f"Duplicate data name: {name_prefix[i]}") name_prefix.append(hparams_i["data_name"]) self._name_to_id = {v: k for k, v in enumerate(name_prefix)} self._processed_cache = [] self._datafile_id = 0 # for training from multiple files self._index_at_beginning_of_this_dataset = 0 self._datafile_prefix = hparams_i.files #self._datafile_num = 33 # hparams_i.datafile_num #self._datafile_num = 64 # hparams_i.datafile_num #self._datafile_num = 3 # hparams_i.datafile_num #self._datafile_num = 16 # hparams_i.datafile_num #self._datafile_num = 26 # hparams_i.datafile_num self._datafile_num = 1 # hparams_i.datafile_num #self._datafile_num = 3 # hparams_i.datafile_num data_source: DataSource = ZipDataSource(*sources) if any(filters): def filter_fn(data): return all( fn(data) for fn, data in zip(filters, data) if fn is not None) data_source = FilterDataSource(data_source, filter_fn=filter_fn) #print("data init derive done") super(MultiAlignedData, self).__init__(data_source, self._hparams, device) #self._dataset_size = 3000000 #self._dataset_size = 6400000 #self._dataset_size = 16000000 #self._dataset_size = 3802215 #self._dataset_size = 1250000 #self._dataset_size = 3000 self._dataset_size = 834229