def _setup_infer_dataloader(
self,
cfg: Dict,
queries: List[str],
max_seq_length: int = -1) -> 'torch.utils.data.DataLoader':
"""
Setup function for a infer data loader.
Args:
cfg: config dictionary containing data loader params like batch_size, num_workers and pin_memory
queries: text
max_seq_length: maximum length of queries, default is -1 for no limit
Returns:
A pytorch DataLoader.
"""
dataset = TextClassificationDataset(tokenizer=self.tokenizer,
queries=queries,
max_seq_length=max_seq_length)
return torch.utils.data.DataLoader(
dataset=dataset,
batch_size=cfg["batch_size"],
shuffle=False,
num_workers=cfg.get("num_workers", 0),
pin_memory=cfg.get("pin_memory", False),
drop_last=False,
collate_fn=dataset.collate_fn,
)
def _setup_dataloader_from_config(
self, cfg: Dict) -> 'torch.utils.data.DataLoader':
input_file = cfg.file_path
if not os.path.exists(input_file):
raise FileNotFoundError(
f'{input_file} not found! The data should be be stored in TAB-separated files \n\
"validation_ds.file_path" and "train_ds.file_path" for train and evaluation respectively. \n\
Each line of the files contains text sequences, where words are separated with spaces. \n\
The label of the example is separated with TAB at the end of each line. \n\
Each line of the files should follow the format: \n\
[WORD][SPACE][WORD][SPACE][WORD][...][TAB][LABEL]')
dataset = TextClassificationDataset(
tokenizer=self.tokenizer,
input_file=input_file,
max_seq_length=self.dataset_cfg.max_seq_length,
num_samples=cfg.get("num_samples", -1),
shuffle=cfg.shuffle,
use_cache=self.dataset_cfg.use_cache,
)
return torch.utils.data.DataLoader(
dataset=dataset,
batch_size=cfg.batch_size,
shuffle=cfg.shuffle,
num_workers=cfg.get("num_workers", 0),
pin_memory=cfg.get("pin_memory", False),
drop_last=cfg.get("drop_last", False),
collate_fn=dataset.collate_fn,
)
