def convert_examples_to_dataset(self, examples, verbose):
return convert_examples_to_dataset(
examples=examples,
feat_spec=self.rparams.feat_spec,
tokenizer=self.model_wrapper.tokenizer,
task=self.task,
verbose=verbose,
)
def get_eval_dataloader(self, eval_examples):
dataset_with_metadata = convert_examples_to_dataset(
examples=eval_examples,
feat_spec=self.rparams.feat_spec,
tokenizer=self.model_wrapper.tokenizer,
task=self.task,
)
eval_sampler = SequentialSampler(dataset_with_metadata.dataset)
eval_dataloader = DataLoader(
dataset=dataset_with_metadata.dataset,
sampler=eval_sampler,
batch_size=self.rparams.eval_batch_size,
)
return HybridLoader(
dataloader=eval_dataloader,
metadata=dataset_with_metadata.metadata,
task=self.task,
)
def get_single_train_dataloader(self, train_examples, batch_size, verbose=True):
dataset_with_metadata = convert_examples_to_dataset(
examples=train_examples,
feat_spec=self.rparams.feat_spec,
tokenizer=self.model_wrapper.tokenizer,
task=self.task,
verbose=verbose,
)
train_sampler = get_sampler(
dataset=dataset_with_metadata.dataset,
local_rank=self.rparams.local_rank,
)
train_dataloader = DataLoader(
dataset=dataset_with_metadata.dataset,
sampler=train_sampler,
batch_size=batch_size,
)
return HybridLoader(
dataloader=train_dataloader,
metadata=dataset_with_metadata.metadata,
task=self.task,
)