def _load_cache_item(self, item, transforms):
for _transform in transforms:
# execute all the deterministic transforms before the first random transform
if isinstance(_transform, Randomizable):
break
item = apply_transform(_transform, item)
return item
def __init__(self, data, transform, cache_num=sys.maxsize, cache_rate=1.0):
"""
Args:
data (Iterable): input data to load and transform to generate dataset for model.
transform (Callable): transforms to execute operations on input data.
cache_num (int): number of items to be cached. Default is `sys.maxsize`.
will take the minimum of (cache_num, data_length x cache_rate, data_length).
cache_rate (float): percentage of cached data in total, default is 1.0 (cache all).
will take the minimum of (cache_num, data_length x cache_rate, data_length).
"""
if not isinstance(transform, Compose):
transform = Compose(transform)
super().__init__(data, transform)
self.cache_num = min(cache_num, int(len(self) * cache_rate), len(self))
self._cache = list()
print('Load and cache transformed data...')
for i in range(self.cache_num):
process_bar(i + 1, self.cache_num)
item = data[i]
for _transform in transform.transforms:
# execute all the deterministic transforms before the first random transform
if isinstance(_transform, Randomizable):
break
item = apply_transform(_transform, item)
self._cache.append(item)
def __getitem__(self, index: int):
def to_list(x):
return list(x) if isinstance(x, (tuple, list)) else [x]
data = list()
for dataset in self.data:
data.extend(to_list(dataset[index]))
if self.transform is not None:
data = apply_transform(self.transform, data, map_items=False) # transform the list data
return data
def _first_random_and_beyond_transform(self, item_transformed):
"""
Process the data from before the first random transform to the final state ready for evaluation.
Args:
item_transformed: The data to be transformed (already process upto the first random transform)
Returns:
the transformed element through the random transforms
"""
start_post_randomize_run = False
for _transform in self.transform.transforms:
if start_post_randomize_run or isinstance(_transform, Randomizable):
start_post_randomize_run = True
item_transformed = apply_transform(_transform, item_transformed)
return item_transformed
def __getitem__(self, index):
if index < self.cache_num:
# load data from cache and execute from the first random transform
start_run = False
data = self._cache[index]
for _transform in self.transform.transforms:
if not start_run and not isinstance(_transform, Randomizable):
continue
else:
start_run = True
data = apply_transform(_transform, data)
else:
# no cache for this data, execute all the transforms directly
data = super(CacheDataset, self).__getitem__(index)
return data
def _pre_first_random_transform(self, item_transformed):
"""
Process the data from original state up to the first random element.
Args:
item_transformed: The data to be transformed
Returns:
the transformed element up to the first identified
random transform object
"""
for _transform in self.transform.transforms:
# execute all the deterministic transforms before the first random transform
if isinstance(_transform, Randomizable):
break
item_transformed = apply_transform(_transform, item_transformed)
return item_transformed
def __call__(self, input_):
self.randomize()
if not self._do_transform:
return input_
return apply_transform(self.selected_trans, input_)
def __call__(self, input_):
for _transform in self.transforms:
input_ = apply_transform(_transform, input_)
return input_
def __getitem__(self, index: int):
data = self.data[index]
if self.transform is not None:
data = apply_transform(self.transform, data)
return data
