def build_dataloader(self, dataset, opts): training_parameters = self._global_config.training_parameters num_workers = training_parameters.num_workers pin_memory = training_parameters.pin_memory other_args = {} self._add_extra_args_for_dataloader(dataset, opts, other_args) loader = DataLoader(dataset=dataset, pin_memory=pin_memory, collate_fn=BatchCollator(), num_workers=num_workers, **other_args) loader.dataset_type = self._dataset_type return loader, other_args.get("sampler", None)
def build_dataloader(self, config, dataset): training = self._global_config.training num_workers = training.num_workers pin_memory = training.pin_memory other_args = {} self._add_extra_args_for_dataloader(dataset, config, other_args) loader = DataLoader(dataset=dataset, pin_memory=pin_memory, collate_fn=BatchCollator(dataset.name, dataset.dataset_type), num_workers=num_workers, **other_args) if num_workers >= 0: # Suppress leaking semaphore warning os.environ[ "PYTHONWARNINGS"] = "ignore:semaphore_tracker:UserWarning" loader.dataset_type = self._dataset_type return loader, other_args.get("sampler", None)