def make_loader(self,
batch_size=16,
num_workers=0,
shuffle=False,
pin_memory=False,
resize_rate=10,
drop_last=False):
"""
CommandLine:
python ~/code/netharn/examples/yolo_voc.py YoloVOCDataset.make_loader
Example:
>>> # DISABLE_DOCTSET
>>> torch.random.manual_seed(0)
>>> self = YoloVOCDataset(split='train')
>>> self.augmenter = None
>>> loader = self.make_loader(batch_size=1, shuffle=True)
>>> # training batches should have multiple shapes
>>> shapes = set()
>>> for batch in ub.ProgIter(iter(loader), total=len(loader)):
>>> inputs, labels = batch
>>> # test to see multiscale works
>>> shapes.add(inputs.shape[-1])
>>> if len(shapes) > 1:
>>> break
>>> assert len(shapes) > 1
"""
import torch.utils.data.sampler as torch_sampler
assert len(self) > 0, 'must have some data'
if shuffle:
sampler = torch_sampler.RandomSampler(self)
resample_freq = resize_rate
else:
sampler = torch_sampler.SequentialSampler(self)
resample_freq = None
# use custom sampler that does multiscale training
batch_sampler = multiscale_batch_sampler.MultiScaleBatchSampler(
sampler,
batch_size=batch_size,
resample_freq=resample_freq,
drop_last=drop_last,
)
# torch.utils.data.sampler.WeightedRandomSampler
loader = torch_data.DataLoader(
self,
batch_sampler=batch_sampler,
collate_fn=nh.data.collate.padded_collate,
num_workers=num_workers,
pin_memory=pin_memory)
if loader.batch_size != batch_size:
try:
# Hack: ensure dataloader has batch size attr
loader._DataLoader__initialized = False
loader.batch_size = batch_size
loader._DataLoader__initialized = True
except Exception:
pass
return loader
def make_loader(self, batch_size=16, num_workers=0, shuffle=False,
pin_memory=False):
"""
Example:
>>> torch.random.manual_seed(0)
>>> dset = coco_api.CocoDataset(coco_api.demo_coco_data())
>>> self = YoloCocoDataset(dset, train=1)
>>> loader = self.make_loader(batch_size=1)
>>> train_iter = iter(loader)
>>> # training batches should have multiple shapes
>>> shapes = set()
>>> for batch in train_iter:
>>> shapes.add(batch[0].shape[-1])
>>> if len(shapes) > 1:
>>> break
>>> #assert len(shapes) > 1
>>> vali_loader = iter(loaders['vali'])
>>> vali_iter = iter(loaders['vali'])
>>> # vali batches should have one shape
>>> shapes = set()
>>> for batch, _ in zip(vali_iter, [1, 2, 3, 4]):
>>> shapes.add(batch[0].shape[-1])
>>> assert len(shapes) == 1
"""
assert len(self) > 0, 'must have some data'
if shuffle:
if True:
# If the data is not balanced we need to balance it
index_to_weight = self._training_sample_weights()
num_samples = len(self)
index_to_weight = index_to_weight[:num_samples]
sampler = torch_sampler.WeightedRandomSampler(index_to_weight,
num_samples,
replacement=True)
sampler.data_source = self # hack for use with multiscale
else:
sampler = torch_sampler.RandomSampler(self)
resample_freq = 10
else:
sampler = torch_sampler.SequentialSampler(self)
resample_freq = None
# use custom sampler that does multiscale training
batch_sampler = multiscale_batch_sampler.MultiScaleBatchSampler(
sampler, batch_size=batch_size, resample_freq=resample_freq,
)
# torch.utils.data.sampler.WeightedRandomSampler
loader = torch_data.DataLoader(self, batch_sampler=batch_sampler,
collate_fn=nh.data.collate.padded_collate,
num_workers=num_workers,
pin_memory=pin_memory)
if loader.batch_size != batch_size:
try:
loader.batch_size = batch_size
except Exception:
pass
return loader
def make_loader(self,
batch_size=16,
num_workers=0,
shuffle=False,
pin_memory=False):
"""
CommandLine:
python ~/code/netharn/netharn/examples/yolo_voc.py YoloVOCDataset.make_loader
Example:
>>> torch.random.manual_seed(0)
>>> self = YoloVOCDataset(split='train')
>>> self.augmenter = None
>>> loader = self.make_loader(batch_size=1, shuffle=True)
>>> # training batches should have multiple shapes
>>> shapes = set()
>>> for batch in ub.ProgIter(iter(loader), total=len(loader)):
>>> inputs, labels = batch
>>> # test to see multiscale works
>>> shapes.add(inputs.shape[-1])
>>> if len(shapes) > 1:
>>> break
>>> assert len(shapes) > 1
"""
assert len(self) > 0, 'must have some data'
# use custom sampler that does multiscale training
batch_sampler = multiscale_batch_sampler.MultiScaleBatchSampler(
self, batch_size=batch_size, shuffle=shuffle)
loader = torch_data.DataLoader(self,
batch_sampler=batch_sampler,
collate_fn=collate.padded_collate,
num_workers=num_workers,
pin_memory=pin_memory)
if loader.batch_size != batch_size:
try:
loader.batch_size = batch_size
except Exception:
pass
return loader