def create_pyreader(args, file_name, feed_list, place, model='lac', reader=None, return_reader=False, mode='train'): # init reader pyreader = fluid.io.PyReader(feed_list=feed_list, capacity=300, use_double_buffer=True, iterable=True) if model == 'lac': if reader == None: reader = Dataset(args) # create lac pyreader if mode == 'train': pyreader.decorate_sample_list_generator(paddle.batch( paddle.reader.shuffle(reader.file_reader(file_name), buf_size=args.traindata_shuffle_buffer), batch_size=args.batch_size), places=place) else: pyreader.decorate_sample_list_generator(paddle.batch( reader.file_reader(file_name, mode=mode), batch_size=args.batch_size), places=place) elif model == 'ernie': # create ernie pyreader if reader == None: reader = task_reader.SequenceLabelReader( vocab_path=args.vocab_path, label_map_config=args.label_map_config, max_seq_len=args.max_seq_len, do_lower_case=args.do_lower_case, in_tokens=False, random_seed=args.random_seed) if mode == 'train': pyreader.decorate_batch_generator(reader.data_generator( file_name, args.batch_size, args.epoch, shuffle=True, phase="train"), places=place) else: pyreader.decorate_batch_generator( reader.data_generator(file_name, args.batch_size, epoch=1, shuffle=False, phase=mode), places=place) if return_reader: return pyreader, reader else: return pyreader
def create_pyreader(args, file_name, feed_list, place, model='lac', reader=None, return_reader=False, mode='train'): # init reader device_count = len(fluid.cuda_places()) if args.use_cuda else len( fluid.cpu_places()) if model == 'lac': pyreader = fluid.io.DataLoader.from_generator(feed_list=feed_list, capacity=50, use_double_buffer=True, iterable=True) if reader == None: reader = Dataset(args) # create lac pyreader if mode == 'train': pyreader.set_sample_list_generator(fluid.io.batch( fluid.io.shuffle(reader.file_reader(file_name), buf_size=args.traindata_shuffle_buffer), batch_size=args.batch_size / device_count), places=place) else: pyreader.set_sample_list_generator(fluid.io.batch( reader.file_reader(file_name, mode=mode), batch_size=args.batch_size / device_count), places=place) elif model == 'ernie': # create ernie pyreader pyreader = fluid.io.DataLoader.from_generator(feed_list=feed_list, capacity=50, use_double_buffer=True, iterable=True) if reader == None: reader = SequenceLabelReader( vocab_path=args.vocab_path, label_map_config=args.label_map_config, max_seq_len=args.max_seq_len, do_lower_case=args.do_lower_case, random_seed=args.random_seed) if mode == 'train': pyreader.set_batch_generator(reader.data_generator(file_name, args.batch_size, args.epoch, shuffle=True, phase="train"), places=place) else: pyreader.set_batch_generator(reader.data_generator(file_name, args.batch_size, epoch=1, shuffle=False, phase=mode), places=place) if return_reader: return pyreader, reader else: return pyreader