def get_reader(self):
logger.info("Get DataLoader")
config_abs_dir = self.config.get("config_abs_dir", None)
reader_path = self.config.get('runner.train_reader_path', 'reader')
reader_path = os.path.join(config_abs_dir, reader_path)
logger.info("Reader Path: {}".format(reader_path))
from paddle.io import DataLoader
dataset = common.lazy_instance_by_fliename(reader_path, "RecDataset")
print("dataset: {}".format(dataset))
use_cuda = int(self.config.get("runner.use_gpu"))
batch_size = self.config.get('runner.train_batch_size', None)
place = paddle.set_device('gpu' if use_cuda else 'cpu')
generator = dataset(self.file_list, self.config)
generator.init()
loader = DataLoader(
generator, batch_size=batch_size, places=place, drop_last=True)
return loader
def get_reader_generator(path, reader_name="Reader"):
reader_class = common.lazy_instance_by_fliename(path, reader_name)()
return reader_class