def build_training_data_loader(self) -> DataLoader:
train_dataset = data.WikiTextDataset(
self.corpus,
batch_size=self.context.get_per_slot_batch_size(),
)
batch_samp = data.BatchSamp(train_dataset, self.bptt)
return DataLoader(train_dataset, batch_sampler=batch_samp)
def build_validation_data_loader(self) -> DataLoader:
if not self.data_downloaded:
data.download_data(self.download_directory)
self.data_downloaded = True
corpus = data_util.Corpus(self.download_directory)
test_dataset = data.PTBData(
corpus.valid,
self.context.get_hparam("seq_len"),
self.context.get_hparam("eval_batch_size"),
self.context.get_hparam("bptt"),
self.context.get_hparam("max_seq_length_delta"),
)
return DataLoader(
test_dataset,
batch_sampler=data.BatchSamp(
test_dataset,
self.context.get_hparam("bptt"),
self.context.get_hparam("max_seq_length_delta"),
valid=True,
),
collate_fn=data.PadSequence(),
)
def build_validation_data_loader(self) -> DataLoader:
val_dataset = data.WikiTextDataset(
self.corpus,
batch_size=self.eval_batch_size,
valid=True,
)
self.val_data_len = len(val_dataset) - 1
batch_samp = data.BatchSamp(val_dataset, self.bptt)
return DataLoader(val_dataset, batch_sampler=batch_samp)
def build_training_data_loader(self) -> DataLoader:
train_dataset = data.PTBData(
self.corpus.train,
self.context.get_per_slot_batch_size(),
self.hparams.bptt,
self.hparams.max_seq_length_delta,
)
return DataLoader(
train_dataset,
batch_sampler=data.BatchSamp(
train_dataset, self.hparams.bptt, self.hparams.max_seq_length_delta,
),
collate_fn=data.PadSequence(),
)
def build_validation_data_loader(self) -> DataLoader:
test_dataset = data.PTBData(
self.corpus.valid,
self.hparams.eval_batch_size,
self.hparams.bptt,
self.hparams.max_seq_length_delta,
)
return DataLoader(
test_dataset,
batch_sampler=data.BatchSamp(
test_dataset,
self.hparams.bptt,
self.hparams.max_seq_length_delta,
valid=True,
),
collate_fn=data.PadSequence(),
)
def build_training_data_loader(self) -> DataLoader:
if not self.data_downloaded:
data.download_data(self.download_directory)
self.data_downloaded = True
corpus = data_util.Corpus(self.download_directory)
train_dataset = data.PTBData(
corpus.train,
self.context.get_hparam("seq_len"),
self.context.get_per_slot_batch_size(),
self.context.get_hparam("bptt"),
self.context.get_hparam("max_seq_length_delta"),
)
return DataLoader(
train_dataset,
batch_sampler=data.BatchSamp(
train_dataset,
self.context.get_hparam("bptt"),
self.context.get_hparam("max_seq_length_delta"),
),
collate_fn=data.PadSequence(),
)
