def mp_reader(params, batch_size):
"""
multiprocess reader
Args:
params(dict):
"""
check_params(params)
full_lines = get_file_list(params)
if params["mode"] == "train":
full_lines = shuffle_lines(full_lines, seed=None)
# NOTE: multiprocess reader is not supported on windows
if sys.platform == "win32":
return partial_reader(params, full_lines, 0, 1, batch_size)
part_num = 1 if 'num_workers' not in params else params['num_workers']
readers = []
for part_id in range(part_num):
readers.append(
partial_reader(params, full_lines, part_id, part_num, batch_size))
return multiprocess_reader(readers, use_pipe=False)
def mp_reader(params):
"""
multiprocess reader
Args:
params(dict):
"""
check_params(params)
full_lines = get_file_list(params)
if params["mode"] == "train":
full_lines = shuffle_lines(full_lines, seed=None)
part_num = 1 if 'num_workers' not in params else params['num_workers']
readers = []
for part_id in range(part_num):
readers.append(partial_reader(params, full_lines, part_id, part_num))
return multiprocess_reader(readers, use_pipe=False)
def main_impl(self, place, iterable):
def fake_reader():
def __impl__():
for _ in range(40):
if not self.raise_exception:
yield list(np.random.uniform(low=-1, high=1,
size=[10])),
else:
raise ValueError()
return __impl__
with fluid.program_guard(fluid.Program(), fluid.Program()):
image = fluid.layers.data(name='image',
dtype='float32',
shape=[10])
reader = fluid.io.PyReader(feed_list=[image],
capacity=2,
iterable=iterable)
image_p_1 = image + 1
decorated_reader = multiprocess_reader(
[fake_reader(), fake_reader()], use_pipe=self.use_pipe)
if isinstance(place, fluid.CUDAPlace):
reader.decorate_sample_generator(decorated_reader,
batch_size=4,
places=fluid.cuda_places())
else:
reader.decorate_sample_generator(decorated_reader,
batch_size=4,
places=fluid.cpu_places())
exe = fluid.Executor(place)
exe.run(fluid.default_startup_program())
if iterable:
for _ in range(3):
num = 0
for data in reader():
exe.run(feed=data, fetch_list=[image_p_1])
num += 1
if not self.raise_exception:
self.assertEquals(num, 20)
else:
self.assertEquals(num, 0)
raise ValueError('Reader raises exception')
else:
for _ in range(3):
num = 0
reader.start()
try:
while True:
exe.run(fetch_list=[image_p_1])
num += 1
except fluid.core.EOFException:
reader.reset()
if not self.raise_exception:
self.assertEquals(num, 20)
else:
self.assertEquals(num, 0)
raise ValueError('Reader raises exception')
def main_impl(self, place, iterable, use_legacy_py_reader):
sample_num = 40
batch_size = 4
def fake_reader():
def __impl__():
for _ in range(sample_num):
if not self.raise_exception:
yield list(np.random.uniform(low=-1, high=1,
size=[10])),
else:
raise ValueError()
return __impl__
with fluid.program_guard(fluid.Program(), fluid.Program()):
if not use_legacy_py_reader:
image = fluid.data(name='image',
dtype='float32',
shape=[None, 10])
reader = fluid.io.PyReader(feed_list=[image],
capacity=2,
iterable=iterable)
else:
reader = fluid.layers.py_reader(capacity=2,
shapes=[
[-1, 10],
],
dtypes=[
'float32',
])
image = fluid.layers.read_file(reader)
image_p_1 = image + 1
decorated_reader = multiprocess_reader(
[fake_reader(), fake_reader()], use_pipe=self.use_pipe)
if use_legacy_py_reader:
reader.decorate_paddle_reader(
fluid.io.batch(decorated_reader, batch_size=batch_size))
else:
if isinstance(place, fluid.CUDAPlace):
reader.decorate_sample_generator(
decorated_reader,
batch_size=batch_size,
places=fluid.cuda_places())
else:
reader.decorate_sample_generator(decorated_reader,
batch_size=batch_size,
places=fluid.cpu_places())
exe = fluid.Executor(place)
exe.run(fluid.default_startup_program())
batch_num = int(sample_num * 2 / batch_size)
if iterable:
for _ in range(3):
num = 0
try:
for data in reader():
exe.run(feed=data, fetch_list=[image_p_1])
num += 1
self.assertEquals(num, batch_num)
except fluid.core.EnforceNotMet as ex:
self.assertEquals(num, 0)
raise ReaderException()
else:
for _ in range(3):
num = 0
reader.start()
try:
while True:
exe.run(fetch_list=[image_p_1])
num += 1
except fluid.core.EOFException:
reader.reset()
self.assertFalse(self.raise_exception)
self.assertEquals(num, batch_num)
except fluid.core.EnforceNotMet as ex:
self.assertTrue(self.raise_exception)
self.assertEquals(num, 0)
raise ReaderException()
return _impl
if __name__ == '__main__':
# generate sample input files
fake_input_files()
with fluid.program_guard(fluid.Program(), fluid.Program()):
place = fluid.CPUPlace()
# the 1st 2 is batch size
image = fluid.data(name='image', dtype='int64', shape=[2, 1, 2])
fluid.layers.Print(image)
# print detailed tensor info of image variable
reader = fluid.io.PyReader(feed_list=[image], capacity=2)
decorated_reader = multiprocess_reader(
[generate_reader(sample_files[0]), generate_reader(sample_files[1])], False)
reader.decorate_sample_generator(decorated_reader, batch_size=2, places=[place])
exe = fluid.Executor(place)
exe.run(fluid.default_startup_program())
for data in reader():
res = exe.run(feed=data, fetch_list=[image])
print(res[0])
# print below content in this case
# [[[1 2]], [[3 4]]]
# [[[5 6]], [[7 8]]]
# [[[9 10]], [[11 12]]]
# [13,14] will be dropped