def __init__(self, dataset, map_func):
self._dataset = dataset
self._map_func = PicklableWrapper(
map_func) # wrap so that a lambda will work
self._rng = random.Random(42)
self._fallback_candidates = set(range(len(dataset)))
def __init__(self, map_func):
_map_func = PicklableWrapper(
map_func) # wrap so that a lambda will work
mask = make_mask()
data = make_dataset_dicts(mask)
self.data = _map_func(data)
def post_mortem_if_fail_for_main(main_func):
def new_main_func(cfg, output_dir, *args, **kwargs):
pdb_ = (
MultiprocessingPdb(FolderLock(output_dir))
if comm.get_world_size() > 1
else None # fallback to use normal pdb for single process
)
return post_mortem_if_fail(pdb_)(main_func)(cfg, output_dir, *args, **kwargs)
return PicklableWrapper(new_main_func)
def __init__(self,
dataset,
map_func,
in_res=None,
out_res=None,
resize=None):
self._dataset = dataset
self._map_func = PicklableWrapper(
map_func) # wrap so that a lambda will work
self.in_res, self.out_res, self.resize = in_res, out_res, resize
self._rng = random.Random(42)
self._fallback_candidates = set(range(len(dataset)))
def __init__(self, dataset, map_func):
"""
Args:
dataset: a dataset where map function is applied. Can be either
map-style or iterable dataset. When given an iterable dataset,
the returned object will also be an iterable dataset.
map_func: a callable which maps the element in dataset. map_func can
return None to skip the data (e.g. in case of errors).
How None is handled depends on the style of `dataset`.
If `dataset` is map-style, it randomly tries other elements.
If `dataset` is iterable, it skips the data and tries the next.
"""
self._dataset = dataset
self._map_func = PicklableWrapper(
map_func) # wrap so that a lambda will work
self._rng = random.Random(42)
self._fallback_candidates = set(range(len(dataset)))
def __init__(self, dataset, map_func):
self._dataset = dataset
self._map_func = PicklableWrapper(
map_func) # wrap so that a lambda will work
