def test_err_k_largest_nonpositive():
with assert_raises(RuntimeError,
glob="``k_largest`` must be at least 1; got -1"):
_test_err_args(k_largest=-1)
with assert_raises(RuntimeError,
glob="``k_largest`` must be at least 1; got 0"):
_test_err_args(k_largest=0)
def test_tl_from_list_of_tensors_empty():
with assert_raises(RuntimeError,
glob='Cannot create TensorList from an empty list.'):
TensorListCPU([])
with assert_raises(RuntimeError,
glob='Cannot create TensorList from an empty list.'):
TensorListGPU([])
def test_arithm_op_context_manager_deep_nested():
tl_1 = tensors.TensorListCPU(np.ones((8, 16, 16)))
tl_2 = tensors.TensorListCPU(np.ones((8, 16, 16)))
expected_sum = np.full(shape=(8, 16, 16), fill_value=2)
eager.arithmetic(True)
assert np.array_equal((tl_1 + tl_2).as_array(), expected_sum)
with eager.arithmetic(False):
with assert_raises(TypeError, glob="unsupported operand type*"):
tl_1 + tl_2
with eager.arithmetic(True):
np.array_equal((tl_1 + tl_2).as_array(), expected_sum)
with eager.arithmetic(False):
with assert_raises(TypeError,
glob="unsupported operand type*"):
tl_1 + tl_2
with assert_raises(TypeError, glob="unsupported operand type*"):
tl_1 + tl_2
assert np.array_equal((tl_1 + tl_2).as_array(), expected_sum)
eager.arithmetic(False)
def test_mmap_dtype_incompatibility():
assert_raises(
RuntimeError,
general_corner_case,
dtypes=[dali.types.INT8, dali.types.FLOAT64],
glob="component size and dtype incompatible",
)
def check_wrong_axes(device, wrong_axes_range=None, named_args=False):
@pipeline_def(batch_size=1, num_threads=1, device_id=0)
def make_pipe():
fake_data = fn.constant(idata=0,
shape=[10, 10, 3],
dtype=types.FLOAT,
device=device)
axes = fn.random.uniform(range=wrong_axes_range,
shape=(2, ),
dtype=types.INT32)
rel_start = fn.random.uniform(range=[0.0, 0.3],
shape=(2, ),
dtype=types.FLOAT)
rel_shape = fn.random.uniform(range=[0.4, 0.6],
shape=(2, ),
dtype=types.FLOAT)
if named_args:
sliced = fn.slice(fake_data,
rel_start=rel_start,
rel_shape=rel_shape,
axes=axes)
else:
sliced = fn.slice(fake_data, rel_start, rel_shape, axes=axes)
return sliced
p = make_pipe()
p.build()
# Note: [[] and []] are '[' and ']' characters.
assert_raises(
RuntimeError,
p.run,
glob="Axis * out of range. Expected range is [[]-3, 2[]] for a 3D input"
)
def _test_too_many_indices(device):
data = [np.uint8([1, 2, 3]), np.uint8([1, 2])]
src = fn.external_source(lambda: data, device=device)
pipe = index_pipe(src, lambda x: x[1, :])
# Verified by tensor_subscript
with assert_raises(RuntimeError, glob="Too many indices"):
pipe.build()
_ = pipe.run()
# Verified by subscript_dim_check
pipe = index_pipe(src, lambda x: x[:, :])
with assert_raises(RuntimeError, glob="Too many indices"):
pipe.build()
_ = pipe.run()
# Verified by expand_dims
pipe = index_pipe(src, lambda x: x[:, :, dali.newaxis])
with assert_raises(RuntimeError, glob="not enough dimensions"):
pipe.build()
_ = pipe.run()
# Verified by subscript_dim_check
pipe = index_pipe(src, lambda x: x[dali.newaxis, :, dali.newaxis, :])
with assert_raises(RuntimeError, glob="Too many indices"):
pipe.build()
_ = pipe.run()
def test_raises():
with assert_raises(
RuntimeError,
glob=
"Input must have at least two dimensions - outermost for sequence and"
" at least one for data elements."):
check_element_extract([4], "F", [1, 3], "cpu")
for shape, layout in [([6, 1], "XF"), ([8, 10, 3], "HWC")]:
with assert_raises(
RuntimeError,
glob=
"Input layout must describe a sequence - it must start with 'F',"
" got '*' instead."):
check_element_extract(shape, layout, [1, 3], "cpu")
with assert_raises(
RuntimeError,
glob="Index `10` from `element_map` is out of bounds for sample with"
" sequence length equal `6`"):
check_element_extract([6, 1], "FX", [10], "cpu")
with assert_raises(
RuntimeError,
glob="Negative indices in `element_map` are not allowed, found: -5"
):
check_element_extract([6, 1], "FX", [-5], "cpu")
def test_wrong_feature_shape():
features = {
'image/encoded': tfrec.FixedLenFeature((), tfrec.string, ""),
'image/object/bbox': tfrec.FixedLenFeature([], tfrec.float32, -1.0),
'image/object/class/label': tfrec.FixedLenFeature([], tfrec.int64, -1),
}
test_dummy_data_path = os.path.join(get_dali_extra_path(), 'db',
'coco_dummy')
pipe = Pipeline(1, 1, 0)
with pipe:
input = fn.readers.tfrecord(path=os.path.join(test_dummy_data_path,
'small_coco.tfrecord'),
index_path=os.path.join(
test_dummy_data_path,
'small_coco_index.idx'),
features=features)
pipe.set_outputs(input['image/encoded'], input['image/object/class/label'],
input['image/object/bbox'])
pipe.build()
# the error is raised because FixedLenFeature is used with insufficient shape to house the input
assert_raises(
RuntimeError,
pipe.run,
glob="Error when executing CPU operator*readers*tfrecord*"
"Output tensor shape is too small*[]*Expected at least 4 elements")
def test_err_classes_weights_length_clash():
error_msg = r"If both ``classes`` and ``class_weights`` are provided, their shapes must match. Got:\s+classes.shape = \{4\}\s+weights.shape = \{3\}"
with assert_raises(RuntimeError, regex=error_msg):
_test_err_args(classes=[0, 1, 2, 3],
class_weights=np.float32([1, 2, 3]))
with assert_raises(RuntimeError, regex=error_msg):
_test_err_args(classes=np.int32([0, 1, 2, 3]), class_weights=[3, 2, 1])
def test_external_source_fail_missing_output():
class ExternalSourcePipeline(Pipeline):
def __init__(self, batch_size, external_s_size, num_threads,
device_id):
super().__init__(batch_size, num_threads, device_id)
self.input = ops.ExternalSource()
self.input_2 = ops.ExternalSource()
self.batch_size_ = batch_size
self.external_s_size_ = external_s_size
def define_graph(self):
self.batch = self.input()
self.batch_2 = self.input_2()
return [self.batch]
def iter_setup(self):
batch = datapy.zeros([self.external_s_size_, 4, 5])
self.feed_input(self.batch, batch)
self.feed_input(self.batch_2, batch)
batch_size = 3
pipe = ExternalSourcePipeline(batch_size, batch_size, 3, 0)
pipe.build()
assert_raises(
RuntimeError,
pipe.run,
regex=
r"Cannot find [\w]+ tensor, it doesn't exists or was pruned as unused one"
)
def _testimpl_image_decoder_slice_error_oob(device):
file_root = os.path.join(test_data_root, good_path, "jpeg")
@pipeline_def(batch_size=batch_size_test, device_id=0, num_threads=4)
def pipe(device):
encoded, _ = fn.readers.file(file_root=file_root)
decoded = fn.decoders.image_slice(encoded, device=device, end=[10000], axes=[1])
return decoded
p = pipe(device)
p.build()
assert_raises(RuntimeError, p.run, glob='cropping window*..*..*is not valid for image dimensions*[*x*]')
def test_tl_from_list_of_tensors_different_backends():
t1 = TensorCPU(np.zeros((1)))
t2 = TensorCPU(np.zeros((1)))._as_gpu()
with assert_raises(
TypeError,
glob='Object at position 1 cannot be converted to TensorCPU'):
TensorListCPU([t1, t2])
with assert_raises(
TypeError,
glob='Object at position 1 cannot be converted to TensorGPU'):
TensorListGPU([t2, t1])
def test_rotate_shape_mismatch():
np_rng = np.random.default_rng(42)
batch_size = 8
num_frames = 50
def mt():
return np.float32(np_rng.uniform(-100, 250, (num_frames, 2, 3)))
mts_inp = [mt() for _ in range(batch_size)]
angles_inp = [
np.array([angle for angle in range(num_frames - i)], dtype=np.float32)
for i in range(batch_size)
]
centers_inp = [
np.array([c for c in range(num_frames + i)], dtype=np.float32)
for i in range(batch_size)
]
with assert_raises(
RuntimeError,
glob="The sample 1 of tensor argument `angle` "
"contains 49 per-frame parameters, but there are 50 frames in "
"the corresponding sample of input 0."):
@pipeline_def
def pipeline():
mts, angles = fn.external_source(lambda _: (mts_inp, angles_inp),
num_outputs=2)
return fn.transforms.rotation(fn.per_frame(mts),
angle=fn.per_frame(angles))
pipe = pipeline(batch_size=batch_size, num_threads=4, device_id=0)
pipe.build()
pipe.run()
with assert_raises(
RuntimeError,
glob="The sample 1 of tensor argument `center` contains 51 "
"per-frame parameters, but there are 49 frames in the corresponding sample "
"of argument `angle`"):
@pipeline_def
def pipeline():
angles, centers = fn.external_source(lambda _:
(angles_inp, centers_inp),
num_outputs=2)
return fn.transforms.rotation(angle=fn.per_frame(angles),
center=fn.per_frame(centers))
pipe = pipeline(batch_size=batch_size, num_threads=4, device_id=0)
pipe.build()
pipe.run()
def test_group_inputs():
e0 = ops._DataNode("op0", "cpu")
e1 = ops._DataNode("op1", "cpu")
inputs = [e0, e1, 10.0, Constant(0).uint8(), 42]
cat_idx, edges, integers, reals = ops._group_inputs(inputs)
assert_equal([("edge", 0), ("edge", 1), ("real", 0),
("integer", 0), ("integer", 1)], cat_idx)
assert_equal([e0, e1], edges)
assert_equal([Constant(0).uint8(), 42], integers)
assert_equal([10.0], reals)
assert_raises(TypeError, ops._group_inputs, [np.complex()],
glob="Expected scalar value of type 'bool', 'int' or 'float', got *.")
_, _, _, none_reals = ops._group_inputs([e0, 10])
assert_equal(None, none_reals)
def test_tensorlist_getitem_cpu():
arr = np.random.rand(3, 5, 6)
tensorlist = TensorListCPU(arr, "NHWC")
list_of_tensors = [x for x in tensorlist]
assert (type(tensorlist.at(0)) == np.ndarray)
assert (type(tensorlist[0]) != np.ndarray)
assert (type(tensorlist[0]) == TensorCPU)
assert (type(tensorlist[-3]) == TensorCPU)
assert (len(list_of_tensors) == len(tensorlist))
with assert_raises(IndexError, glob="out of range"):
tensorlist[len(tensorlist)]
with assert_raises(IndexError, glob="out of range"):
tensorlist[-len(tensorlist) - 1]
def check_bad_device(device_id, error_msg):
test_data_shape = [1, 3, 0, 4]
def get_data():
out = [
np.empty(test_data_shape, dtype=np.uint8)
for _ in range(batch_size)
]
return out
pipe = Pipeline(batch_size=batch_size, num_threads=3, device_id=device_id)
outs = fn.external_source(source=get_data, device="gpu")
pipe.set_outputs(outs)
assert_raises(RuntimeError, pipe.build, glob=error_msg)
def test_tensorlist_getitem_gpu():
arr = np.random.rand(3, 5, 6)
pipe = ExternalSourcePipe(arr.shape[0], arr)
pipe.build()
tensorlist = pipe.run()[0]
list_of_tensors = [x for x in tensorlist]
assert (type(tensorlist[0]) != cp.ndarray)
assert (type(tensorlist[0]) == TensorGPU)
assert (type(tensorlist[-3]) == TensorGPU)
assert (len(list_of_tensors) == len(tensorlist))
with assert_raises(IndexError, glob="TensorListCPU index out of range"):
tensorlist[len(tensorlist)]
with assert_raises(IndexError, glob="TensorListCPU index out of range"):
tensorlist[-len(tensorlist) - 1]
def test_err_classes_ignored():
with assert_raises(
RuntimeError,
glob=
"Class-related arguments * cannot be used when ``ignore_class`` is True"
):
_test_err_args(classes=[0, 1, 2, 3], ignore_class=True)
def test_external_source_collection_cycling_raise():
pipe = Pipeline(1, 3, 0, prefetch_queue_depth=1)
batches = [[make_array([1.5, 2.5], dtype=datapy.float32)],
[make_array([-1, 3.5, 4.5], dtype=datapy.float32)]]
def batch_gen():
for b in batches:
yield b
pipe.set_outputs(fn.external_source(batches, cycle="raise"),
fn.external_source(batch_gen, cycle="raise"))
pipe.build()
# epochs are cycles over the source iterable
for _ in range(3):
for batch in batches:
pipe_out = pipe.run()
batch = asnumpy(batch)
batch = batch, batch
check_output(pipe_out, batch)
with assert_raises(StopIteration):
pipe.run()
pipe.reset()
def check_slice_with_out_of_bounds_error(device, batch_size, input_shape=(100, 200, 3),
normalized_anchor=False, normalized_shape=False):
# This test case is written with HWC layout in mind and "HW" axes in slice arguments
axis_names = "HW"
naxes = len(axis_names)
axes = None
layout = "HWC"
assert(len(input_shape) == 3)
eii = RandomDataIterator(batch_size, shape=input_shape)
eii_arg = SliceArgsIterator(batch_size, len(input_shape), image_shape=input_shape,
image_layout=layout, axes=axes, axis_names=axis_names,
normalized_anchor=normalized_anchor,
normalized_shape=normalized_shape,
min_norm_anchor=-0.5, max_norm_anchor=-0.1,
min_norm_shape=1.1, max_norm_shape=3.6)
pipe = SliceSynthDataPipeline(device, batch_size, layout, iter(eii), iter(eii_arg),
axes=axes, axis_names=axis_names,
normalized_anchor=normalized_anchor,
normalized_shape=normalized_shape,
out_of_bounds_policy="error")
pipe.build()
with assert_raises(RuntimeError, glob="Slice can't be placed out of bounds with current policy. Got:"):
outs = pipe.run()
def test_invalid_number_of_shards():
@pipeline_def(batch_size=1, device_id=0, num_threads=4)
def get_test_pipe():
root = os.path.join(os.environ['DALI_EXTRA_PATH'], 'db/single/mixed')
files, labels = fn.readers.file(file_root=root,
shard_id=0,
num_shards=9999)
return files, labels
pipe = get_test_pipe()
assert_raises(
RuntimeError,
pipe.build,
glob=
"The number of input samples: *, needs to be at least equal to the requested number of shards:*."
) # noqa: E501
def test_set_outputs_err_msg_random_type():
pipe = Pipeline(batch_size=1, num_threads=1, device_id=None)
pipe.set_outputs("test")
with assert_raises(TypeError,
glob='Illegal output type. '
'The output * is a `<class \'str\'>`.'):
pipe.build()
def test_wrong_backend_named_args():
@pipeline_def(batch_size=1, num_threads=1, device_id=0)
def make_pipe():
fake_data = fn.constant(idata=0,
shape=[10, 10, 3],
dtype=types.FLOAT,
device="cpu")
rel_start = fn.random.uniform(range=[0.0, 0.3],
shape=(2, ),
dtype=types.FLOAT,
device="gpu")
rel_shape = fn.random.uniform(range=[0.4, 0.6],
shape=(2, ),
dtype=types.FLOAT,
device="gpu")
sliced = fn.slice(fake_data,
rel_start=rel_start,
rel_shape=rel_shape,
device="cpu")
return sliced
with assert_raises(
RuntimeError,
glob="Named arguments inputs to operators must be CPU data nodes"):
p = make_pipe()
p.build()
p.run()
def _test_out_of_range(device, idx):
data = [np.uint8([1, 2, 3]), np.uint8([1, 2])]
src = fn.external_source(lambda: data, device=device)
pipe = index_pipe(src, lambda x: x[idx])
pipe.build()
with assert_raises(RuntimeError, glob="out of range"):
_ = pipe.run()
def check_slice_named_args_errors(device, batch_size):
test_data_shape = [5, 4, 3]
def get_data():
out = [
np.random.randint(0, 255, size=test_data_shape, dtype=np.uint8)
for _ in range(batch_size)
]
return out
pipe = Pipeline(batch_size=batch_size, num_threads=4, device_id=0)
with pipe:
data = fn.external_source(source=get_data, layout="HWC")
start = np.array([1, 2])
shape = np.array([3, 1])
outs = [
fn.slice(data,
start,
shape,
start=start,
end=start + shape,
shape=shape,
axes=(0, 1)),
]
pipe.set_outputs(*outs)
with assert_raises(
RuntimeError,
glob=
'"end", "rel_end", "shape", and "rel_shape" arguments are mutually exclusive'
):
pipe.build()
for _ in range(1):
outs = pipe.run()
def check_crop_with_out_of_bounds_error(device,
batch_size,
input_shape=(100, 200, 3)):
# This test case is written with HWC layout in mind and "HW" axes in slice arguments
layout = "HWC"
assert len(input_shape) == 3
eii = RandomDataIterator(batch_size, shape=input_shape)
crop_shape = tuple(extent * 2 for extent in input_shape[:2])
crop_y = 0.5
crop_x = 0.5
pipe = CropSynthPipe(device,
batch_size,
iter(eii),
layout=layout,
crop_shape=crop_shape,
crop_x=crop_x,
crop_y=crop_y,
out_of_bounds_policy="error")
pipe.build()
with assert_raises(
RuntimeError,
glob="Slice can't be placed out of bounds with current policy."):
_ = pipe.run()
def check_crop_wrong_layout(device,
batch_size,
input_shape=(100, 200, 3),
layout="ABC"):
assert len(layout) == len(input_shape)
@pipeline_def
def get_pipe():
def get_data():
out = [
np.zeros(input_shape, dtype=np.uint8)
for _ in range(batch_size)
]
return out
data = fn.external_source(source=get_data,
layout=layout,
device=device)
return fn.crop(data, crop_h=10, crop_w=10)
pipe = get_pipe(batch_size=batch_size, device_id=0, num_threads=3)
pipe.build()
with assert_raises(
RuntimeError,
glob=
f"The layout \"{layout}\" does not match any of the allowed layouts"
):
pipe.run()
def test_err_threshold_dim_clash():
with assert_raises(
RuntimeError,
glob=
"Argument \"threshold\" expected shape 2 but got 5 values, which can't be interpreted as the expected shape."
):
_test_err_args(threshold=[1, 2, 3, 4, 5])
def check_tensor_input_fail(batch_size, shape, layout, window_size,
smoothing_size, scale, normalize, dtype,
err_regex):
iterator = RandomlyShapedDataIterator(batch_size,
max_shape=shape,
dtype=np.uint8)
def gen_params():
return np.array(window_size, dtype=np.int32), np.array(smoothing_size, dtype=np.int32), \
np.array(scale, dtype=np.float32)
@pipeline_def
def pipeline():
data = fn.external_source(iterator, layout=layout)
window_size, smoothing_size, scale = fn.external_source(gen_params,
batch=False,
num_outputs=3)
edges = fn.laplacian(data,
window_size=window_size,
smoothing_size=smoothing_size,
scale=scale,
normalized_kernel=normalize,
dtype=dtype)
return edges, data
with assert_raises(RuntimeError, regex=err_regex):
pipe = pipeline(device_id=types.CPU_ONLY_DEVICE_ID,
num_threads=4,
batch_size=batch_size)
pipe.build()
pipe.run()
def check_roi_random_crop_error(shape_like_in=None,
in_shape=None,
crop_shape=None,
roi_start=None,
roi_shape=None,
roi_end=None,
error_msg=""):
batch_size = 3
niter = 3
pipe = dali.pipeline.Pipeline(batch_size=batch_size,
num_threads=4,
device_id=0,
seed=1234)
with pipe:
inputs = [] if shape_like_in is None else [shape_like_in]
out = fn.roi_random_crop(*inputs,
in_shape=in_shape,
crop_shape=crop_shape,
roi_start=roi_start,
roi_shape=roi_shape,
roi_end=roi_end,
device='cpu')
pipe.set_outputs(out)
with assert_raises(RuntimeError, regex=error_msg):
pipe.build()
for _ in range(niter):
pipe.run()
