def get_test_examples(self, data_dir, epoch, batch_size, places, padding_size): test_reader = data_reader((self.data_dir + "/test.tsv"), self.vocab, self.num_examples, "infer", epoch, padding_size) loader = DataLoader.from_generator(capacity=50, return_list=True) loader.set_sample_generator(test_reader, batch_size=batch_size, drop_last=False, places=places) return loader
def get_dev_examples(self, data_dir, epoch, shuffle, batch_size, places, padding_size): dev_reader = data_reader((self.data_dir + "/dev.tsv"), self.vocab, self.num_examples, "dev", epoch, padding_size, shuffle) loader = DataLoader.from_generator(capacity=50, return_list=True) loader.set_sample_generator(dev_reader, batch_size=batch_size, drop_last=False, places=places) return loader