def __init__(self,
*loaders,
path='.',
shuffle=False,
device=None,
bs=None):
"Accepts any number of `DataLoader` and a device"
self.path = path
device = ifnone(device, default_device())
self.device = device
self.c = None
bs = ifnone(bs, min([dl.bs for dl in loaders]))
for i, dl in enumerate(loaders): # ensure all dls have the same bs
dl.bs = bs
dl.shuffle_fn = self.shuffle_fn
if self.c is None and hasattr(dl, "c"): self.c = dl.c
if i == 0: self.dataset = dl.dataset
dl.to(device=device)
self.shuffle = shuffle
if not self.shuffle: self.rng = np.arange(len(self.dataset)).tolist()
self.loaders = loaders
self.count = 0
self.fake_l = _FakeLoader(self, False, 0, 0, 0) if version.parse(
fastai.__version__) >= version.parse("2.1") else _FakeLoader(
self, False, 0, 0)
if sum([len(dl.dataset) for dl in loaders]) > 0:
self._get_idxs() # Do not apply on an empty dataset
def __init__(self, *dls, device=None):
"Accepts any number of `DataLoaders` and a device"
device = ifnone(device, default_device())
self.device = device
self.c = []
bs = 0
for dl in dls: # ensure all dls have the same bs
if bs == 0: bsi = dl.bs
else: dl.bs = bsi
dl.shuffle_fn = self.shuffle_fn
if self.c == [] and hasattr(dl, "c"): self.c = dl.c
dl.to(device=device)
self.dls = dls
self.count = 0
self.fake_l = _FakeLoader(self, False, 0, 0, 0) if version.parse(
fastai.__version__) >= version.parse("2.1") else _FakeLoader(
self, False, 0, 0)
self._get_idxs()
def __init__(self, dl, rank, world_size):
store_attr()
self.bs,self.device,self.num_workers,self.drop_last,self.dataset,self.offs,fake = \
attrgetter('bs','device','num_workers','drop_last','dataset','offs','fake_l')(dl)
self.fake_l = _FakeLoader(self,
fake.pin_memory,
fake.num_workers,
fake.timeout,
persistent_workers=fake.persistent_workers)
self.SERIAL_EXEC = xmp.MpSerialExecutor()
def __init__(self,dl,rank,world_size, seed=42):
store_attr()
self.bs,self.device,self.num_workers, \
self.drop_last,self.dataset,self.offs,fake, self.shuffle = \
attrgetter('bs','device','num_workers',
'drop_last','dataset','offs','fake_l', 'shuffle')(dl)
self.fake_l = _FakeLoader(self, fake.pin_memory, fake.num_workers, fake.timeout,
persistent_workers=fake.persistent_workers)
self.epoch = 0
random.seed(self.seed)
# setting inner dl rng
self.dl.rng = random.Random(random.randint(0,2**32-1))
self.reset_rng()
def __init__(self, dl, rank, world_size):
store_attr('dl,rank,world_size')
self.bs, self.device, self.drop_last, self.dataset = dl.bs, dl.device, dl.drop_last, dl.dataset
self.fake_l = _FakeLoader(self, dl.fake_l.pin_memory, dl.fake_l.num_workers, dl.fake_l.timeout)