def get_reader(self): logger.info("Get DataLoader") config_abs_dir = self.config.get("config_abs_dir", None) reader_path = self.config.get('runner.train_reader_path', 'reader') reader_path = os.path.join(config_abs_dir, reader_path) logger.info("Reader Path: {}".format(reader_path)) from paddle.io import DataLoader dataset = common.lazy_instance_by_fliename(reader_path, "RecDataset") print("dataset: {}".format(dataset)) use_cuda = int(self.config.get("runner.use_gpu")) batch_size = self.config.get('runner.train_batch_size', None) place = paddle.set_device('gpu' if use_cuda else 'cpu') generator = dataset(self.file_list, self.config) generator.init() loader = DataLoader( generator, batch_size=batch_size, places=place, drop_last=True) return loader
def get_reader_generator(path, reader_name="Reader"): reader_class = common.lazy_instance_by_fliename(path, reader_name)() return reader_class