def _setup_dataloader_from_config(self, cfg: DictConfig):
dataset = SquadDataset(
tokenizer=self.tokenizer,
data_file=cfg.file,
doc_stride=self._cfg.dataset.doc_stride,
max_query_length=self._cfg.dataset.max_query_length,
max_seq_length=self._cfg.dataset.max_seq_length,
version_2_with_negative=self._cfg.dataset.version_2_with_negative,
num_samples=cfg.num_samples,
mode=cfg.mode,
use_cache=self._cfg.dataset.use_cache,
)
if cfg.mode == "eval":
self.validation_dataset = dataset
elif cfg.mode == "test":
self.test_dataset = dataset
dl = torch.utils.data.DataLoader(
dataset=dataset,
batch_size=cfg.batch_size,
collate_fn=dataset.collate_fn,
drop_last=cfg.get('drop_last', False),
shuffle=cfg.shuffle,
num_workers=cfg.get('num_workers', 0),
)
return dl
def _setup_dataloader_from_config(self, cfg: DictConfig, mode: str):
dataset = SquadDataset(
tokenizer=self.tokenizer,
data_file=cfg.file,
doc_stride=self._cfg.dataset.doc_stride,
max_query_length=self._cfg.dataset.max_query_length,
max_seq_length=self._cfg.dataset.max_seq_length,
version_2_with_negative=self._cfg.dataset.version_2_with_negative,
num_samples=cfg.num_samples,
mode=mode,
use_cache=self._cfg.dataset.use_cache,
)
dl = torch.utils.data.DataLoader(
dataset=dataset,
batch_size=cfg.batch_size,
collate_fn=dataset.collate_fn,
drop_last=cfg.drop_last,
shuffle=cfg.shuffle,
num_workers=cfg.num_workers,
pin_memory=cfg.pin_memory,
)
return dl
