def __init__(self, dataset, map_func): self._dataset = dataset self._map_func = PicklableWrapper( map_func) # wrap so that a lambda will work self._rng = random.Random(42) self._fallback_candidates = set(range(len(dataset)))
def __init__(self, map_func): _map_func = PicklableWrapper( map_func) # wrap so that a lambda will work mask = make_mask() data = make_dataset_dicts(mask) self.data = _map_func(data)
def post_mortem_if_fail_for_main(main_func): def new_main_func(cfg, output_dir, *args, **kwargs): pdb_ = ( MultiprocessingPdb(FolderLock(output_dir)) if comm.get_world_size() > 1 else None # fallback to use normal pdb for single process ) return post_mortem_if_fail(pdb_)(main_func)(cfg, output_dir, *args, **kwargs) return PicklableWrapper(new_main_func)
def __init__(self, dataset, map_func, in_res=None, out_res=None, resize=None): self._dataset = dataset self._map_func = PicklableWrapper( map_func) # wrap so that a lambda will work self.in_res, self.out_res, self.resize = in_res, out_res, resize self._rng = random.Random(42) self._fallback_candidates = set(range(len(dataset)))
def __init__(self, dataset, map_func): """ Args: dataset: a dataset where map function is applied. Can be either map-style or iterable dataset. When given an iterable dataset, the returned object will also be an iterable dataset. map_func: a callable which maps the element in dataset. map_func can return None to skip the data (e.g. in case of errors). How None is handled depends on the style of `dataset`. If `dataset` is map-style, it randomly tries other elements. If `dataset` is iterable, it skips the data and tries the next. """ self._dataset = dataset self._map_func = PicklableWrapper( map_func) # wrap so that a lambda will work self._rng = random.Random(42) self._fallback_candidates = set(range(len(dataset)))
def __init__(self, dataset, map_func): self._dataset = dataset self._map_func = PicklableWrapper( map_func) # wrap so that a lambda will work