def _load_cache_item(self, item, transforms): for _transform in transforms: # execute all the deterministic transforms before the first random transform if isinstance(_transform, Randomizable): break item = apply_transform(_transform, item) return item
def __init__(self, data, transform, cache_num=sys.maxsize, cache_rate=1.0): """ Args: data (Iterable): input data to load and transform to generate dataset for model. transform (Callable): transforms to execute operations on input data. cache_num (int): number of items to be cached. Default is `sys.maxsize`. will take the minimum of (cache_num, data_length x cache_rate, data_length). cache_rate (float): percentage of cached data in total, default is 1.0 (cache all). will take the minimum of (cache_num, data_length x cache_rate, data_length). """ if not isinstance(transform, Compose): transform = Compose(transform) super().__init__(data, transform) self.cache_num = min(cache_num, int(len(self) * cache_rate), len(self)) self._cache = list() print('Load and cache transformed data...') for i in range(self.cache_num): process_bar(i + 1, self.cache_num) item = data[i] for _transform in transform.transforms: # execute all the deterministic transforms before the first random transform if isinstance(_transform, Randomizable): break item = apply_transform(_transform, item) self._cache.append(item)
def __getitem__(self, index: int): def to_list(x): return list(x) if isinstance(x, (tuple, list)) else [x] data = list() for dataset in self.data: data.extend(to_list(dataset[index])) if self.transform is not None: data = apply_transform(self.transform, data, map_items=False) # transform the list data return data
def _first_random_and_beyond_transform(self, item_transformed): """ Process the data from before the first random transform to the final state ready for evaluation. Args: item_transformed: The data to be transformed (already process upto the first random transform) Returns: the transformed element through the random transforms """ start_post_randomize_run = False for _transform in self.transform.transforms: if start_post_randomize_run or isinstance(_transform, Randomizable): start_post_randomize_run = True item_transformed = apply_transform(_transform, item_transformed) return item_transformed
def __getitem__(self, index): if index < self.cache_num: # load data from cache and execute from the first random transform start_run = False data = self._cache[index] for _transform in self.transform.transforms: if not start_run and not isinstance(_transform, Randomizable): continue else: start_run = True data = apply_transform(_transform, data) else: # no cache for this data, execute all the transforms directly data = super(CacheDataset, self).__getitem__(index) return data
def _pre_first_random_transform(self, item_transformed): """ Process the data from original state up to the first random element. Args: item_transformed: The data to be transformed Returns: the transformed element up to the first identified random transform object """ for _transform in self.transform.transforms: # execute all the deterministic transforms before the first random transform if isinstance(_transform, Randomizable): break item_transformed = apply_transform(_transform, item_transformed) return item_transformed
def __call__(self, input_): self.randomize() if not self._do_transform: return input_ return apply_transform(self.selected_trans, input_)
def __call__(self, input_): for _transform in self.transforms: input_ = apply_transform(_transform, input_) return input_
def __getitem__(self, index: int): data = self.data[index] if self.transform is not None: data = apply_transform(self.transform, data) return data