def test_element_extract_cpu():
pipe = Pipeline(batch_size=batch_size, num_threads=4, device_id=None)
test_data_shape = [5, 10, 20, 3]
def get_data():
out = [
np.random.randint(0, 255, size=test_data_shape, dtype=np.uint8)
for _ in range(batch_size)
]
return out
data = fn.external_source(source=get_data, layout="FHWC")
processed, _ = fn.element_extract(data, element_map=[0, 3])
pipe.set_outputs(processed)
pipe.build()
for _ in range(3):
pipe.run()
def test_sequence_rearrange_cpu():
pipe = Pipeline(batch_size=batch_size, num_threads=4, device_id=None)
test_data_shape = [5, 10, 20, 3]
def get_data():
out = [
np.random.randint(0, 255, size=test_data_shape, dtype=np.uint8)
for _ in range(batch_size)
]
return out
data = fn.external_source(source=get_data, layout="FHWC")
processed = fn.sequence_rearrange(data, new_order=[0, 4, 1, 3, 2])
pipe.set_outputs(processed)
pipe.build()
for _ in range(3):
pipe.run()
def test_lookup_table_cpu():
pipe = Pipeline(batch_size=batch_size, num_threads=4, device_id=None)
test_data_shape = [100]
def get_data():
out = [
np.random.randint(0, 5, size=test_data_shape, dtype=np.uint8)
for _ in range(batch_size)
]
return out
data = fn.external_source(source=get_data)
processed = fn.lookup_table(data, keys=[1, 3], values=[10, 50])
pipe.set_outputs(processed)
pipe.build()
for _ in range(3):
pipe.run()
def test_pad_cpu():
pipe = Pipeline(batch_size=batch_size, num_threads=4, device_id=None)
test_data_shape = [5, 4, 3]
def get_data():
out = [
np.random.randint(0, 255, size=test_data_shape, dtype=np.uint8)
for _ in range(batch_size)
]
return out
data = fn.external_source(source=get_data, layout="HWC")
processed = fn.pad(data, fill_value=-1, axes=(0, ), shape=(10, ))
pipe.set_outputs(processed)
pipe.build()
for _ in range(3):
pipe.run()
def test_bb_flip_cpu():
pipe = Pipeline(batch_size=batch_size, num_threads=4, device_id=None)
test_data_shape = [200, 4]
def get_data():
out = [
(np.random.randint(0, 255, size=test_data_shape, dtype=np.uint8) /
255).astype(dtype=np.float32) for _ in range(batch_size)
]
return out
data = fn.external_source(source=get_data)
processed = fn.bb_flip(data)
pipe.set_outputs(processed)
pipe.build()
for _ in range(3):
pipe.run()
def test_one_hot_cpu():
pipe = Pipeline(batch_size=batch_size, num_threads=4, device_id=None)
test_data_shape = [200]
def get_data():
out = [
np.random.randint(0, 255, size=test_data_shape, dtype=np.uint8)
for _ in range(batch_size)
]
return out
data = fn.external_source(source=get_data)
processed = fn.one_hot(data, num_classes=256)
pipe.set_outputs(processed)
pipe.build()
for _ in range(3):
pipe.run()
def check_type_mismatch(device, test_data_root, names):
pipe = Pipeline(2, 2, 0)
pipe.set_outputs(
fn.numpy_reader(device=device, file_root=test_data_root, files=names))
pipe.build()
err = None
try:
pipe.run()
except RuntimeError as thrown:
err = thrown
# asserts should not be in except block to avoid printing nested exception on failure
assert err, "Exception not thrown"
assert "Inconsistent data" in str(
err), "Unexpected error message: {}".format(err)
assert "int32" in str(err) and "float" in str(
err), "Unexpected error message: {}".format(err)
def test_random_bbox_crop_no_labels():
batch_size = 3
pipe = Pipeline(batch_size=batch_size, num_threads=4, device_id=0)
test_box_shape = [200, 4]
def get_boxes():
out = [(np.random.randint(0, 255, size = test_box_shape, dtype = np.uint8) / 255).astype(dtype = np.float32) for _ in range(batch_size)]
return out
boxes = fn.external_source(source = get_boxes)
processed = fn.random_bbox_crop(boxes,
aspect_ratio=[0.5, 2.0],
thresholds=[0.1, 0.3, 0.5],
scaling=[0.8, 1.0],
bbox_layout="xyXY")
pipe.set_outputs(*processed)
pipe.build()
for _ in range(3):
pipe.run()
def test_external_source_with_iter():
pipe = Pipeline(1, 3, 0)
pipe.set_outputs(fn.external_source(lambda i: [np.array([i + 1.5], dtype=np.float32)]))
pipe.build()
for i in range(10):
check_output(pipe.run(), [np.array([i + 1.5], dtype=np.float32)])
def check_per_sample_gaussian_blur(batch_size,
sigma_dim,
window_size_dim,
shape,
layout,
axes,
op_type="cpu"):
pipe = Pipeline(batch_size=batch_size, num_threads=4, device_id=0)
data = RandomlyShapedDataIterator(batch_size, max_shape=shape)
with pipe:
if sigma_dim is not None:
sigma = fn.random.uniform(range=[0.5, 3], shape=[sigma_dim])
sigma_arg = sigma
else:
# placeholder, so we can return something
sigma = fn.random.coin_flip(probability=0)
sigma_arg = None
if window_size_dim is not None:
window_radius = fn.random.uniform(range=[5, 10],
shape=[window_size_dim])
window_size = fn.cast(window_radius, dtype=types.INT32) * 2 + 1
window_arg = window_size
else:
window_size = fn.random.coin_flip(probability=0)
window_arg = None
input = fn.external_source(data, layout=layout)
if op_type == "gpu":
input = input.gpu()
blurred = fn.gaussian_blur(input,
device=op_type,
sigma=sigma_arg,
window_size=window_arg)
pipe.set_outputs(blurred, input, sigma, window_size)
pipe.build()
for _ in range(test_iters):
result, input, sigma, window_size = pipe.run()
if op_type == "gpu":
result = result.as_cpu()
input = input.as_cpu()
input = to_batch(input, batch_size)
sigma = to_batch(sigma, batch_size)
window_size = to_batch(window_size, batch_size)
baseline = []
for i in range(batch_size):
sigma_arg = sigma[i] if sigma is not None else None
window_arg = window_size[i] if window_size_dim is not None else None
skip_axes = count_skip_axes(layout)
baseline.append(
gaussian_baseline(input[i], sigma_arg, window_arg, axes,
skip_axes))
check_batch(result,
baseline,
batch_size,
max_allowed_error=1,
expected_layout=layout)
def check_container(cont):
pipe = Pipeline(batch_size=1, num_threads=4, device_id=0)
path = os.path.join(video_containers_data_root, cont)
test_videos = [path + '/' + f for f in os.listdir(path)]
with pipe:
# mkv container for some reason fails in DALI VFR heuristics
vid = fn.video_reader(device="gpu",
filenames=test_videos,
sequence_length=10,
skip_vfr_check=True,
stride=1,
name="Reader")
pipe.set_outputs(vid)
pipe.build()
iter_num = pipe.reader_meta("Reader")["epoch_size"]
for _ in range(iter_num):
pipe.run()
def test_external_source_with_iter():
for attempt in range(10):
pipe = Pipeline(1, 3, 0)
pipe.set_outputs(fn.external_source(lambda i: [make_array([attempt * 100 + i * 10 + 1.5], dtype=datapy.float32)]))
pipe.build()
for i in range(10):
check_output(pipe.run(), [np.array([attempt * 100 + i * 10 + 1.5], dtype=np.float32)])
def check_coin_flip(device='cpu',
batch_size=32,
max_shape=[1e5],
p=None,
use_shape_like_input=False):
pipe = Pipeline(batch_size=batch_size,
device_id=0,
num_threads=3,
seed=123456)
with pipe:
def shape_gen_f():
return random_shape(max_shape)
shape_arg = None
inputs = []
shape_out = None
if max_shape is not None:
if use_shape_like_input:
shape_like_in = dali.fn.external_source(
lambda: np.zeros(shape_gen_f()),
device=device,
batch=False)
inputs += [shape_like_in]
shape_out = dali.fn.shapes(shape_like_in)
else:
shape_arg = dali.fn.external_source(shape_gen_f, batch=False)
shape_out = shape_arg
outputs = [
dali.fn.random.coin_flip(*inputs,
device=device,
probability=p,
shape=shape_arg)
]
if shape_out is not None:
outputs += [shape_out]
pipe.set_outputs(*outputs)
pipe.build()
outputs = pipe.run()
data_out = outputs[0].as_cpu() if isinstance(outputs[0],
TensorListGPU) else outputs[0]
shapes_out = None
if max_shape is not None:
shapes_out = outputs[1].as_cpu() if isinstance(
outputs[1], TensorListGPU) else outputs[1]
p = p if p is not None else 0.5
for i in range(batch_size):
data = np.array(data_out[i])
assert np.logical_or(data == 0, data == 1).all()
if max_shape is not None:
sample_shape = np.array(shapes_out[i])
assert (data.shape == sample_shape).all()
total = len(data)
positive = np.count_nonzero(data)
np.testing.assert_allclose(p, positive / total,
atol=0.005) # +/- -.5%
def check_generic_gaussian_blur(batch_size,
sigma,
window_size,
shape,
layout,
axes,
op_type="cpu",
in_dtype=np.uint8,
out_dtype=types.NO_TYPE,
random_shape=True):
pipe = Pipeline(batch_size=batch_size, num_threads=4, device_id=0)
min_shape = None if random_shape else shape
data = RandomlyShapedDataIterator(batch_size,
min_shape=min_shape,
max_shape=shape,
dtype=in_dtype)
# Extract the numpy type from DALI, we can have float32 or the same as input
if out_dtype == types.NO_TYPE:
result_type = in_dtype
elif dali_type(in_dtype) == out_dtype:
result_type = in_dtype
else:
result_type = np.float32
with pipe:
input = fn.external_source(data, layout=layout)
if op_type == "gpu":
input = input.gpu()
blurred = fn.gaussian_blur(input,
device=op_type,
sigma=sigma,
window_size=window_size,
dtype=out_dtype)
pipe.set_outputs(blurred, input)
pipe.build()
for _ in range(test_iters):
result, input = pipe.run()
if op_type == "gpu":
result = result.as_cpu()
input = input.as_cpu()
input = to_batch(input, batch_size)
skip_axes = count_skip_axes(layout)
baseline = [
gaussian_baseline(img,
sigma,
window_size,
axes,
skip_axes,
dtype=result_type) for img in input
]
max_error = 1 if result_type != np.float32 else 1e-04
check_batch(result,
baseline,
batch_size,
max_allowed_error=max_error,
expected_layout=layout)
def test_tfrecord_reader_cpu():
pipe = Pipeline(batch_size=batch_size, num_threads=4, device_id=None)
tfrecord = sorted(glob.glob(os.path.join(tfrecord_dir, '*[!i][!d][!x]')))
tfrecord_idx = sorted(glob.glob(os.path.join(tfrecord_dir, '*idx')))
input = fn.tfrecord_reader(path=tfrecord,
index_path=tfrecord_idx,
shard_id=0,
num_shards=1,
features={
"image/encoded":
tfrec.FixedLenFeature((), tfrec.string, ""),
"image/class/label":
tfrec.FixedLenFeature([1], tfrec.int64, -1)
})
out = input["image/encoded"]
pipe.set_outputs(out)
pipe.build()
for _ in range(3):
pipe.run()
def test_arithm_ops_cpu():
pipe = Pipeline(batch_size=batch_size, num_threads=4, device_id=None)
data = fn.external_source(source=get_data, layout="HWC")
processed = [
data * 2, data + 2, data - 2, data / 2, data // 2, data**2, data == 2,
data != 2, data < 2, data <= 2, data > 2, data >= 2, data & 2,
data | 2, data ^ 2,
dmath.abs(data),
dmath.fabs(data),
dmath.floor(data),
dmath.ceil(data),
dmath.pow(data, 2),
dmath.fpow(data, 1.5),
dmath.min(data, 2),
dmath.max(data, 50),
dmath.clamp(data, 10, 50),
dmath.sqrt(data),
dmath.rsqrt(data),
dmath.cbrt(data),
dmath.exp(data),
dmath.exp(data),
dmath.log(data),
dmath.log2(data),
dmath.log10(data),
dmath.sin(data),
dmath.cos(data),
dmath.tan(data),
dmath.asin(data),
dmath.acos(data),
dmath.atan(data),
dmath.atan2(data, 3),
dmath.sinh(data),
dmath.cosh(data),
dmath.tanh(data),
dmath.asinh(data),
dmath.acosh(data),
dmath.atanh(data)
]
pipe.set_outputs(*processed)
pipe.build()
for _ in range(3):
pipe.run()
def test_nonsilent_region_cpu():
pipe = Pipeline(batch_size=batch_size, num_threads=4, device_id=None)
test_data_shape = [200]
def get_data():
out = [
np.random.randint(0, 255, size=test_data_shape, dtype=np.uint8)
for _ in range(batch_size)
]
out[0][0] = 0
out[1][0] = 0
out[1][1] = 0
return out
data = fn.external_source(source=get_data)
processed, _ = fn.nonsilent_region(data)
pipe.set_outputs(processed)
pipe.build()
for _ in range(3):
pipe.run()
def _test_scalar(device, as_tensors):
"""Test propagation of scalars from external source"""
batch_size = 4
src_pipe = Pipeline(batch_size, 1, 0)
src_ext = fn.external_source(source=lambda i: [np.float32(i * 10 + i + 1) for i in range(batch_size)], device=device)
src_pipe.set_outputs(src_ext)
src_pipe.build()
dst_pipe = Pipeline(batch_size, 1, 0, exec_async=False, exec_pipelined=False)
dst_pipe.set_outputs(fn.external_source(name="ext", device=device))
dst_pipe.build()
for iter in range(3):
src = src_pipe.run()
data = src[0]
if as_tensors:
data = [data[i] for i in range(len(data))]
dst_pipe.feed_input("ext", data)
dst = dst_pipe.run()
check_batch(src[0], dst[0], batch_size, 0, 0, "")
def _test_feed_input(device):
src_pipe, batch_size = build_src_pipe(device)
dst_pipe = Pipeline(batch_size, 1, 0, exec_async=False, exec_pipelined=False)
dst_pipe.set_outputs(fn.external_source(name="ext", device=device))
dst_pipe.build()
for iter in range(3):
out1 = src_pipe.run()
dst_pipe.feed_input("ext", out1[0])
out2 = dst_pipe.run()
check_batch(out2[0], out1[0], batch_size, 0, 0, "XY")
def test_dtype_arg():
batch_size = 2
src_data = [[np.ones((120, 120, 3), dtype=np.uint8)] * batch_size]
src_pipe = Pipeline(batch_size, 1, 0)
src_ext = fn.external_source(source=src_data, dtype=DALIDataType.UINT8)
src_pipe.set_outputs(src_ext)
src_pipe.build()
out, = src_pipe.run()
for i in range(batch_size):
t = out.at(i)
assert t.dtype == np.uint8
np.array_equal(t, np.ones((120, 120, 3), dtype=np.uint8))
def test_external_source_generator():
pipe = Pipeline(1, 3, 0)
def gen():
for i in range(5):
yield [make_array([i + 1.5], dtype=datapy.float32)]
pipe.set_outputs(fn.external_source(gen()))
pipe.build()
for i in range(5):
check_output(pipe.run(), [np.array([i + 1.5], dtype=np.float32)])
def test_operator_coco_reader_same_images():
file_root = os.path.join(test_data_root, 'db', 'coco_pixelwise', 'images')
train_annotations = os.path.join(test_data_root, 'db', 'coco_pixelwise', 'instances.json')
coco_dir = os.path.join(test_data_root, 'db', 'coco')
coco_dir_imgs = os.path.join(coco_dir, 'images')
coco_pixelwise_dir = os.path.join(test_data_root, 'db', 'coco_pixelwise')
coco_pixelwise_dir_imgs = os.path.join(coco_pixelwise_dir, 'images')
for file_root, annotations_file in [ \
(coco_dir_imgs, os.path.join(coco_dir, 'instances.json')),
(coco_pixelwise_dir_imgs, os.path.join(coco_pixelwise_dir, 'instances.json')),
(coco_pixelwise_dir_imgs, os.path.join(coco_pixelwise_dir, 'instances_rle_counts.json'))]:
pipe = Pipeline(batch_size=1, num_threads=4, device_id=0)
with pipe:
inputs1, boxes1, labels1, *other = fn.coco_reader(
file_root=file_root,
annotations_file=train_annotations,
name="reader1",
seed=1234
)
inputs2, boxes2, labels2, *other = fn.coco_reader(
file_root=file_root,
annotations_file=train_annotations,
polygon_masks=True,
name="reader2"
)
inputs3, boxes3, labels3, *other = fn.coco_reader(
file_root=file_root,
annotations_file=train_annotations,
pixelwise_masks=True,
name="reader3"
)
pipe.set_outputs(
inputs1, boxes1, labels1,
inputs2, boxes2, labels2,
inputs3, boxes3, labels3
)
pipe.build()
epoch_sz = pipe.epoch_size("reader1")
assert epoch_sz == pipe.epoch_size("reader2")
assert epoch_sz == pipe.epoch_size("reader3")
for i in range(epoch_sz):
inputs1, boxes1, labels1, inputs2, boxes2, labels2, inputs3, boxes3, labels3 = \
pipe.run()
np.testing.assert_array_equal(inputs1.at(0), inputs2.at(0))
np.testing.assert_array_equal(inputs1.at(0), inputs3.at(0))
np.testing.assert_array_equal(labels1.at(0), labels2.at(0))
np.testing.assert_array_equal(labels1.at(0), labels3.at(0))
np.testing.assert_array_equal(boxes1.at(0), boxes2.at(0))
np.testing.assert_array_equal(boxes1.at(0), boxes3.at(0))
def test_image_decoder_slice_cpu():
anch_shape = [2]
def get_anchors():
out = [(np.random.randint(1, 128, size=anch_shape, dtype=np.uint8) /
255).astype(dtype=np.float32) for _ in range(batch_size)]
return out
def get_shape():
out = [(np.random.randint(1, 128, size=anch_shape, dtype=np.uint8) /
255).astype(dtype=np.float32) for _ in range(batch_size)]
return out
pipe = Pipeline(batch_size=batch_size, num_threads=4, device_id=None)
input, _ = fn.readers.file(file_root=images_dir, shard_id=0, num_shards=1)
anchors = fn.external_source(source=get_anchors)
shape = fn.external_source(source=get_shape)
processed = fn.decoders.image_slice(input, anchors, shape)
pipe.set_outputs(processed)
pipe.build()
for _ in range(3):
pipe.run()
def test_external_source_collection_cycling():
pipe = Pipeline(1, 3, 0)
batches = [[np.array([1.5, 2.5], dtype=np.float32)],
[np.array([-1, 3.5, 4.5], dtype=np.float32)]]
pipe.set_outputs(fn.external_source(batches, cycle=True))
pipe.build()
# epochs are cycles over the source iterable
for epoch in range(3):
for batch in batches:
check_output(pipe.run(), batch)
def test_external_source_gen_function_cycle():
pipe = Pipeline(1, 3, 0)
def gen():
for i in range(5):
yield [make_array([i + 1.5], dtype=datapy.float32)]
pipe.set_outputs(fn.external_source(gen, cycle=True))
pipe.build()
for _ in range(3):
for i in range(5):
check_output(pipe.run(), [np.array([i + 1.5], dtype=np.float32)])
def test_slice_cpu():
anch_shape = [2]
def get_anchors():
out = [(np.random.randint(1, 256, size=anch_shape, dtype=np.uint8) /
255).astype(dtype=np.float32) for _ in range(batch_size)]
return out
def get_shape():
out = [(np.random.randint(1, 256, size=anch_shape, dtype=np.uint8) /
255).astype(dtype=np.float32) for _ in range(batch_size)]
return out
pipe = Pipeline(batch_size=batch_size, num_threads=4, device_id=None)
data = fn.external_source(source=get_data, layout="HWC")
anchors = fn.external_source(source=get_anchors)
shape = fn.external_source(source=get_shape)
processed = fn.slice(data, anchors, shape, out_of_bounds_policy="pad")
pipe.set_outputs(processed)
pipe.build()
for _ in range(3):
pipe.run()
def test_fn_python_function():
pipe = Pipeline(1, 1, 0, exec_pipelined = False, exec_async = False)
batch1 = [np.array([1,2,3])]
batch2 = [np.array([2,3,4])]
# we need a context, because we use an operator with potential side-effects (python_function)
with pipe:
src = fn.external_source([batch1, batch2])
out = fn.python_function(src, function = lambda x: x+1)
pipe.set_outputs(out)
pipe.build()
assert(np.array_equal(pipe.run()[0].at(0), batch1[0] + 1))
assert(np.array_equal(pipe.run()[0].at(0), batch2[0] + 1))
def test_segmentation_select_masks():
def get_data_source(*args, **kwargs):
return lambda: make_batch_select_masks(*args, **kwargs)
pipe = Pipeline(batch_size=batch_size,
num_threads=4,
device_id=None,
seed=1234)
with pipe:
polygons, vertices, selected_masks = fn.external_source(
num_outputs=3,
device='cpu',
source=get_data_source(batch_size,
vertex_ndim=2,
npolygons_range=(1, 5),
nvertices_range=(3, 10)))
out_polygons, out_vertices = fn.segmentation.select_masks(
selected_masks, polygons, vertices, reindex_masks=False)
pipe.set_outputs(polygons, vertices, selected_masks, out_polygons,
out_vertices)
pipe.build()
for _ in range(3):
pipe.run()
def test_bbox_paste_cpu():
pipe = Pipeline(batch_size=batch_size, num_threads=4, device_id=None)
test_data_shape = [200, 4]
def get_data():
out = [
(np.random.randint(0, 255, size=test_data_shape, dtype=np.uint8) /
255).astype(dtype=np.float32) for _ in range(batch_size)
]
return out
data = fn.external_source(source=get_data)
paste_posx = fn.random.uniform(range=(0, 1))
paste_posy = fn.random.uniform(range=(0, 1))
paste_ratio = fn.random.uniform(range=(1, 2))
processed = fn.bbox_paste(data,
paste_x=paste_posx,
paste_y=paste_posy,
ratio=paste_ratio)
pipe.set_outputs(processed)
pipe.build()
for _ in range(3):
pipe.run()
def test_dim_mismatch():
with tempfile.TemporaryDirectory() as test_data_root:
num_samples = 2
batch_size = 2
names = ["2D.npy", "3D.npy"]
paths = [os.path.join(test_data_root, name) for name in names]
create_numpy_file(paths[0], [3, 4], np.float32, False)
create_numpy_file(paths[1], [2, 3, 4], np.float32, False)
pipe = Pipeline(2, 2, None)
pipe.set_outputs(fn.numpy_reader(file_root=test_data_root,
files=names))
pipe.build()
err = None
try:
pipe.run()
except RuntimeError as thrown:
err = thrown
# asserts should not be in except block to avoid printing nested exception on failure
assert err, "Exception not thrown"
assert "Inconsistent data" in str(
err), "Unexpected error message: {}".format(err)
assert "2 dimensions" in str(err) and "3 dimensions" in str(
err), "Unexpected error message: {}".format(err)
