def test_numpy_reader_cpu():
with tempfile.TemporaryDirectory() as test_data_root:
rng = np.random.RandomState(12345)
def create_numpy_file(filename, shape, typ, fortran_order):
# generate random array
arr = rng.random_sample(shape) * 10.
arr = arr.astype(typ)
if fortran_order:
arr = np.asfortranarray(arr)
np.save(filename, arr)
num_samples = 20
filenames = []
for index in range(0, num_samples):
filename = os.path.join(test_data_root,
"test_{:02d}.npy".format(index))
filenames.append(filename)
create_numpy_file(filename, (5, 2, 8), np.float32, False)
pipe = Pipeline(batch_size=batch_size, num_threads=4, device_id=None)
processed = fn.numpy_reader(file_root=test_data_root)
pipe.set_outputs(processed)
pipe.build()
for _ in range(3):
pipe.run()
def check_dim_mismatch(device, test_data_root, names):
pipe = Pipeline(2, 2, 0)
pipe.set_outputs(
fn.numpy_reader(device=device, file_root=test_data_root, files=names))
pipe.build()
err = None
try:
pipe.run()
except RuntimeError as thrown:
err = thrown
# asserts should not be in except block to avoid printing nested exception on failure
assert err, "Exception not thrown"
assert "Inconsistent data" in str(
err), "Unexpected error message: {}".format(err)
def test_dim_mismatch():
with tempfile.TemporaryDirectory() as test_data_root:
num_samples = 2
batch_size = 2
names = ["2D.npy", "3D.npy"]
paths = [os.path.join(test_data_root, name) for name in names]
create_numpy_file(paths[0], [3, 4], np.float32, False)
create_numpy_file(paths[1], [2, 3, 4], np.float32, False)
pipe = Pipeline(2, 2, None)
pipe.set_outputs(fn.numpy_reader(file_root=test_data_root,
files=names))
pipe.build()
err = None
try:
pipe.run()
except RuntimeError as thrown:
err = thrown
# asserts should not be in except block to avoid printing nested exception on failure
assert err, "Exception not thrown"
assert "Inconsistent data" in str(
err), "Unexpected error message: {}".format(err)
assert "2 dimensions" in str(err) and "3 dimensions" in str(
err), "Unexpected error message: {}".format(err)
def NumpyReaderPipeline(path,
batch_size,
device="cpu",
file_list=None,
files=None,
path_filter="*.npy",
num_threads=1,
device_id=0,
num_gpus=1,
cache_header_information=False):
pipe = Pipeline(batch_size=batch_size,
num_threads=num_threads,
device_id=0)
data = fn.numpy_reader(device=device,
file_list=file_list,
files=files,
file_root=path,
file_filter=path_filter,
shard_id=0,
num_shards=1,
cache_header_information=cache_header_information)
pipe.set_outputs(data)
return pipe