def build_dataloader(self, dataset, opts):
training_parameters = self._global_config.training_parameters
num_workers = training_parameters.num_workers
pin_memory = training_parameters.pin_memory
other_args = {}
self._add_extra_args_for_dataloader(dataset, opts, other_args)
loader = DataLoader(dataset=dataset,
pin_memory=pin_memory,
collate_fn=BatchCollator(),
num_workers=num_workers,
**other_args)
loader.dataset_type = self._dataset_type
return loader, other_args.get("sampler", None)
def build_dataloader(self, config, dataset):
training = self._global_config.training
num_workers = training.num_workers
pin_memory = training.pin_memory
other_args = {}
self._add_extra_args_for_dataloader(dataset, config, other_args)
loader = DataLoader(dataset=dataset,
pin_memory=pin_memory,
collate_fn=BatchCollator(dataset.name,
dataset.dataset_type),
num_workers=num_workers,
**other_args)
if num_workers >= 0:
# Suppress leaking semaphore warning
os.environ[
"PYTHONWARNINGS"] = "ignore:semaphore_tracker:UserWarning"
loader.dataset_type = self._dataset_type
return loader, other_args.get("sampler", None)
