Ejemplo n.º 1
0
    def build_validation_data_loader(self) -> DataLoader:
        if not self.data_downloaded:
            data.download_data(self.download_directory)
            self.data_downloaded = True

        corpus = data_util.Corpus(self.download_directory)

        test_dataset = data.PTBData(
            corpus.valid,
            self.context.get_hparam("seq_len"),
            self.context.get_hparam("eval_batch_size"),
            self.context.get_hparam("bptt"),
            self.context.get_hparam("max_seq_length_delta"),
        )

        return DataLoader(
            test_dataset,
            batch_sampler=data.BatchSamp(
                test_dataset,
                self.context.get_hparam("bptt"),
                self.context.get_hparam("max_seq_length_delta"),
                valid=True,
            ),
            collate_fn=data.PadSequence(),
        )
Ejemplo n.º 2
0
 def build_training_data_loader(self) -> DataLoader:
     train_dataset = data.PTBData(
         self.corpus.train,
         self.context.get_per_slot_batch_size(),
         self.hparams.bptt,
         self.hparams.max_seq_length_delta,
     )
     return DataLoader(
         train_dataset,
         batch_sampler=data.BatchSamp(
             train_dataset, self.hparams.bptt, self.hparams.max_seq_length_delta,
         ),
         collate_fn=data.PadSequence(),
     )
Ejemplo n.º 3
0
 def build_validation_data_loader(self) -> DataLoader:
     test_dataset = data.PTBData(
         self.corpus.valid,
         self.hparams.eval_batch_size,
         self.hparams.bptt,
         self.hparams.max_seq_length_delta,
     )
     return DataLoader(
         test_dataset,
         batch_sampler=data.BatchSamp(
             test_dataset,
             self.hparams.bptt,
             self.hparams.max_seq_length_delta,
             valid=True,
         ),
         collate_fn=data.PadSequence(),
     )
Ejemplo n.º 4
0
    def build_training_data_loader(self) -> DataLoader:
        if not self.data_downloaded:
            data.download_data(self.download_directory)
            self.data_downloaded = True

        corpus = data_util.Corpus(self.download_directory)

        train_dataset = data.PTBData(
            corpus.train,
            self.context.get_hparam("seq_len"),
            self.context.get_per_slot_batch_size(),
            self.context.get_hparam("bptt"),
            self.context.get_hparam("max_seq_length_delta"),
        )
        return DataLoader(
            train_dataset,
            batch_sampler=data.BatchSamp(
                train_dataset,
                self.context.get_hparam("bptt"),
                self.context.get_hparam("max_seq_length_delta"),
            ),
            collate_fn=data.PadSequence(),
        )