def test_err_k_largest_nonpositive(): with assert_raises(RuntimeError, glob="``k_largest`` must be at least 1; got -1"): _test_err_args(k_largest=-1) with assert_raises(RuntimeError, glob="``k_largest`` must be at least 1; got 0"): _test_err_args(k_largest=0)
def test_tl_from_list_of_tensors_empty(): with assert_raises(RuntimeError, glob='Cannot create TensorList from an empty list.'): TensorListCPU([]) with assert_raises(RuntimeError, glob='Cannot create TensorList from an empty list.'): TensorListGPU([])
def test_arithm_op_context_manager_deep_nested(): tl_1 = tensors.TensorListCPU(np.ones((8, 16, 16))) tl_2 = tensors.TensorListCPU(np.ones((8, 16, 16))) expected_sum = np.full(shape=(8, 16, 16), fill_value=2) eager.arithmetic(True) assert np.array_equal((tl_1 + tl_2).as_array(), expected_sum) with eager.arithmetic(False): with assert_raises(TypeError, glob="unsupported operand type*"): tl_1 + tl_2 with eager.arithmetic(True): np.array_equal((tl_1 + tl_2).as_array(), expected_sum) with eager.arithmetic(False): with assert_raises(TypeError, glob="unsupported operand type*"): tl_1 + tl_2 with assert_raises(TypeError, glob="unsupported operand type*"): tl_1 + tl_2 assert np.array_equal((tl_1 + tl_2).as_array(), expected_sum) eager.arithmetic(False)
def test_mmap_dtype_incompatibility(): assert_raises( RuntimeError, general_corner_case, dtypes=[dali.types.INT8, dali.types.FLOAT64], glob="component size and dtype incompatible", )
def check_wrong_axes(device, wrong_axes_range=None, named_args=False): @pipeline_def(batch_size=1, num_threads=1, device_id=0) def make_pipe(): fake_data = fn.constant(idata=0, shape=[10, 10, 3], dtype=types.FLOAT, device=device) axes = fn.random.uniform(range=wrong_axes_range, shape=(2, ), dtype=types.INT32) rel_start = fn.random.uniform(range=[0.0, 0.3], shape=(2, ), dtype=types.FLOAT) rel_shape = fn.random.uniform(range=[0.4, 0.6], shape=(2, ), dtype=types.FLOAT) if named_args: sliced = fn.slice(fake_data, rel_start=rel_start, rel_shape=rel_shape, axes=axes) else: sliced = fn.slice(fake_data, rel_start, rel_shape, axes=axes) return sliced p = make_pipe() p.build() # Note: [[] and []] are '[' and ']' characters. assert_raises( RuntimeError, p.run, glob="Axis * out of range. Expected range is [[]-3, 2[]] for a 3D input" )
def _test_too_many_indices(device): data = [np.uint8([1, 2, 3]), np.uint8([1, 2])] src = fn.external_source(lambda: data, device=device) pipe = index_pipe(src, lambda x: x[1, :]) # Verified by tensor_subscript with assert_raises(RuntimeError, glob="Too many indices"): pipe.build() _ = pipe.run() # Verified by subscript_dim_check pipe = index_pipe(src, lambda x: x[:, :]) with assert_raises(RuntimeError, glob="Too many indices"): pipe.build() _ = pipe.run() # Verified by expand_dims pipe = index_pipe(src, lambda x: x[:, :, dali.newaxis]) with assert_raises(RuntimeError, glob="not enough dimensions"): pipe.build() _ = pipe.run() # Verified by subscript_dim_check pipe = index_pipe(src, lambda x: x[dali.newaxis, :, dali.newaxis, :]) with assert_raises(RuntimeError, glob="Too many indices"): pipe.build() _ = pipe.run()
def test_raises(): with assert_raises( RuntimeError, glob= "Input must have at least two dimensions - outermost for sequence and" " at least one for data elements."): check_element_extract([4], "F", [1, 3], "cpu") for shape, layout in [([6, 1], "XF"), ([8, 10, 3], "HWC")]: with assert_raises( RuntimeError, glob= "Input layout must describe a sequence - it must start with 'F'," " got '*' instead."): check_element_extract(shape, layout, [1, 3], "cpu") with assert_raises( RuntimeError, glob="Index `10` from `element_map` is out of bounds for sample with" " sequence length equal `6`"): check_element_extract([6, 1], "FX", [10], "cpu") with assert_raises( RuntimeError, glob="Negative indices in `element_map` are not allowed, found: -5" ): check_element_extract([6, 1], "FX", [-5], "cpu")
def test_wrong_feature_shape(): features = { 'image/encoded': tfrec.FixedLenFeature((), tfrec.string, ""), 'image/object/bbox': tfrec.FixedLenFeature([], tfrec.float32, -1.0), 'image/object/class/label': tfrec.FixedLenFeature([], tfrec.int64, -1), } test_dummy_data_path = os.path.join(get_dali_extra_path(), 'db', 'coco_dummy') pipe = Pipeline(1, 1, 0) with pipe: input = fn.readers.tfrecord(path=os.path.join(test_dummy_data_path, 'small_coco.tfrecord'), index_path=os.path.join( test_dummy_data_path, 'small_coco_index.idx'), features=features) pipe.set_outputs(input['image/encoded'], input['image/object/class/label'], input['image/object/bbox']) pipe.build() # the error is raised because FixedLenFeature is used with insufficient shape to house the input assert_raises( RuntimeError, pipe.run, glob="Error when executing CPU operator*readers*tfrecord*" "Output tensor shape is too small*[]*Expected at least 4 elements")
def test_err_classes_weights_length_clash(): error_msg = r"If both ``classes`` and ``class_weights`` are provided, their shapes must match. Got:\s+classes.shape = \{4\}\s+weights.shape = \{3\}" with assert_raises(RuntimeError, regex=error_msg): _test_err_args(classes=[0, 1, 2, 3], class_weights=np.float32([1, 2, 3])) with assert_raises(RuntimeError, regex=error_msg): _test_err_args(classes=np.int32([0, 1, 2, 3]), class_weights=[3, 2, 1])
def test_external_source_fail_missing_output(): class ExternalSourcePipeline(Pipeline): def __init__(self, batch_size, external_s_size, num_threads, device_id): super().__init__(batch_size, num_threads, device_id) self.input = ops.ExternalSource() self.input_2 = ops.ExternalSource() self.batch_size_ = batch_size self.external_s_size_ = external_s_size def define_graph(self): self.batch = self.input() self.batch_2 = self.input_2() return [self.batch] def iter_setup(self): batch = datapy.zeros([self.external_s_size_, 4, 5]) self.feed_input(self.batch, batch) self.feed_input(self.batch_2, batch) batch_size = 3 pipe = ExternalSourcePipeline(batch_size, batch_size, 3, 0) pipe.build() assert_raises( RuntimeError, pipe.run, regex= r"Cannot find [\w]+ tensor, it doesn't exists or was pruned as unused one" )
def _testimpl_image_decoder_slice_error_oob(device): file_root = os.path.join(test_data_root, good_path, "jpeg") @pipeline_def(batch_size=batch_size_test, device_id=0, num_threads=4) def pipe(device): encoded, _ = fn.readers.file(file_root=file_root) decoded = fn.decoders.image_slice(encoded, device=device, end=[10000], axes=[1]) return decoded p = pipe(device) p.build() assert_raises(RuntimeError, p.run, glob='cropping window*..*..*is not valid for image dimensions*[*x*]')
def test_tl_from_list_of_tensors_different_backends(): t1 = TensorCPU(np.zeros((1))) t2 = TensorCPU(np.zeros((1)))._as_gpu() with assert_raises( TypeError, glob='Object at position 1 cannot be converted to TensorCPU'): TensorListCPU([t1, t2]) with assert_raises( TypeError, glob='Object at position 1 cannot be converted to TensorGPU'): TensorListGPU([t2, t1])
def test_rotate_shape_mismatch(): np_rng = np.random.default_rng(42) batch_size = 8 num_frames = 50 def mt(): return np.float32(np_rng.uniform(-100, 250, (num_frames, 2, 3))) mts_inp = [mt() for _ in range(batch_size)] angles_inp = [ np.array([angle for angle in range(num_frames - i)], dtype=np.float32) for i in range(batch_size) ] centers_inp = [ np.array([c for c in range(num_frames + i)], dtype=np.float32) for i in range(batch_size) ] with assert_raises( RuntimeError, glob="The sample 1 of tensor argument `angle` " "contains 49 per-frame parameters, but there are 50 frames in " "the corresponding sample of input 0."): @pipeline_def def pipeline(): mts, angles = fn.external_source(lambda _: (mts_inp, angles_inp), num_outputs=2) return fn.transforms.rotation(fn.per_frame(mts), angle=fn.per_frame(angles)) pipe = pipeline(batch_size=batch_size, num_threads=4, device_id=0) pipe.build() pipe.run() with assert_raises( RuntimeError, glob="The sample 1 of tensor argument `center` contains 51 " "per-frame parameters, but there are 49 frames in the corresponding sample " "of argument `angle`"): @pipeline_def def pipeline(): angles, centers = fn.external_source(lambda _: (angles_inp, centers_inp), num_outputs=2) return fn.transforms.rotation(angle=fn.per_frame(angles), center=fn.per_frame(centers)) pipe = pipeline(batch_size=batch_size, num_threads=4, device_id=0) pipe.build() pipe.run()
def test_group_inputs(): e0 = ops._DataNode("op0", "cpu") e1 = ops._DataNode("op1", "cpu") inputs = [e0, e1, 10.0, Constant(0).uint8(), 42] cat_idx, edges, integers, reals = ops._group_inputs(inputs) assert_equal([("edge", 0), ("edge", 1), ("real", 0), ("integer", 0), ("integer", 1)], cat_idx) assert_equal([e0, e1], edges) assert_equal([Constant(0).uint8(), 42], integers) assert_equal([10.0], reals) assert_raises(TypeError, ops._group_inputs, [np.complex()], glob="Expected scalar value of type 'bool', 'int' or 'float', got *.") _, _, _, none_reals = ops._group_inputs([e0, 10]) assert_equal(None, none_reals)
def test_tensorlist_getitem_cpu(): arr = np.random.rand(3, 5, 6) tensorlist = TensorListCPU(arr, "NHWC") list_of_tensors = [x for x in tensorlist] assert (type(tensorlist.at(0)) == np.ndarray) assert (type(tensorlist[0]) != np.ndarray) assert (type(tensorlist[0]) == TensorCPU) assert (type(tensorlist[-3]) == TensorCPU) assert (len(list_of_tensors) == len(tensorlist)) with assert_raises(IndexError, glob="out of range"): tensorlist[len(tensorlist)] with assert_raises(IndexError, glob="out of range"): tensorlist[-len(tensorlist) - 1]
def check_bad_device(device_id, error_msg): test_data_shape = [1, 3, 0, 4] def get_data(): out = [ np.empty(test_data_shape, dtype=np.uint8) for _ in range(batch_size) ] return out pipe = Pipeline(batch_size=batch_size, num_threads=3, device_id=device_id) outs = fn.external_source(source=get_data, device="gpu") pipe.set_outputs(outs) assert_raises(RuntimeError, pipe.build, glob=error_msg)
def test_tensorlist_getitem_gpu(): arr = np.random.rand(3, 5, 6) pipe = ExternalSourcePipe(arr.shape[0], arr) pipe.build() tensorlist = pipe.run()[0] list_of_tensors = [x for x in tensorlist] assert (type(tensorlist[0]) != cp.ndarray) assert (type(tensorlist[0]) == TensorGPU) assert (type(tensorlist[-3]) == TensorGPU) assert (len(list_of_tensors) == len(tensorlist)) with assert_raises(IndexError, glob="TensorListCPU index out of range"): tensorlist[len(tensorlist)] with assert_raises(IndexError, glob="TensorListCPU index out of range"): tensorlist[-len(tensorlist) - 1]
def test_err_classes_ignored(): with assert_raises( RuntimeError, glob= "Class-related arguments * cannot be used when ``ignore_class`` is True" ): _test_err_args(classes=[0, 1, 2, 3], ignore_class=True)
def test_external_source_collection_cycling_raise(): pipe = Pipeline(1, 3, 0, prefetch_queue_depth=1) batches = [[make_array([1.5, 2.5], dtype=datapy.float32)], [make_array([-1, 3.5, 4.5], dtype=datapy.float32)]] def batch_gen(): for b in batches: yield b pipe.set_outputs(fn.external_source(batches, cycle="raise"), fn.external_source(batch_gen, cycle="raise")) pipe.build() # epochs are cycles over the source iterable for _ in range(3): for batch in batches: pipe_out = pipe.run() batch = asnumpy(batch) batch = batch, batch check_output(pipe_out, batch) with assert_raises(StopIteration): pipe.run() pipe.reset()
def check_slice_with_out_of_bounds_error(device, batch_size, input_shape=(100, 200, 3), normalized_anchor=False, normalized_shape=False): # This test case is written with HWC layout in mind and "HW" axes in slice arguments axis_names = "HW" naxes = len(axis_names) axes = None layout = "HWC" assert(len(input_shape) == 3) eii = RandomDataIterator(batch_size, shape=input_shape) eii_arg = SliceArgsIterator(batch_size, len(input_shape), image_shape=input_shape, image_layout=layout, axes=axes, axis_names=axis_names, normalized_anchor=normalized_anchor, normalized_shape=normalized_shape, min_norm_anchor=-0.5, max_norm_anchor=-0.1, min_norm_shape=1.1, max_norm_shape=3.6) pipe = SliceSynthDataPipeline(device, batch_size, layout, iter(eii), iter(eii_arg), axes=axes, axis_names=axis_names, normalized_anchor=normalized_anchor, normalized_shape=normalized_shape, out_of_bounds_policy="error") pipe.build() with assert_raises(RuntimeError, glob="Slice can't be placed out of bounds with current policy. Got:"): outs = pipe.run()
def test_invalid_number_of_shards(): @pipeline_def(batch_size=1, device_id=0, num_threads=4) def get_test_pipe(): root = os.path.join(os.environ['DALI_EXTRA_PATH'], 'db/single/mixed') files, labels = fn.readers.file(file_root=root, shard_id=0, num_shards=9999) return files, labels pipe = get_test_pipe() assert_raises( RuntimeError, pipe.build, glob= "The number of input samples: *, needs to be at least equal to the requested number of shards:*." ) # noqa: E501
def test_set_outputs_err_msg_random_type(): pipe = Pipeline(batch_size=1, num_threads=1, device_id=None) pipe.set_outputs("test") with assert_raises(TypeError, glob='Illegal output type. ' 'The output * is a `<class \'str\'>`.'): pipe.build()
def test_wrong_backend_named_args(): @pipeline_def(batch_size=1, num_threads=1, device_id=0) def make_pipe(): fake_data = fn.constant(idata=0, shape=[10, 10, 3], dtype=types.FLOAT, device="cpu") rel_start = fn.random.uniform(range=[0.0, 0.3], shape=(2, ), dtype=types.FLOAT, device="gpu") rel_shape = fn.random.uniform(range=[0.4, 0.6], shape=(2, ), dtype=types.FLOAT, device="gpu") sliced = fn.slice(fake_data, rel_start=rel_start, rel_shape=rel_shape, device="cpu") return sliced with assert_raises( RuntimeError, glob="Named arguments inputs to operators must be CPU data nodes"): p = make_pipe() p.build() p.run()
def _test_out_of_range(device, idx): data = [np.uint8([1, 2, 3]), np.uint8([1, 2])] src = fn.external_source(lambda: data, device=device) pipe = index_pipe(src, lambda x: x[idx]) pipe.build() with assert_raises(RuntimeError, glob="out of range"): _ = pipe.run()
def check_slice_named_args_errors(device, batch_size): test_data_shape = [5, 4, 3] def get_data(): out = [ np.random.randint(0, 255, size=test_data_shape, dtype=np.uint8) for _ in range(batch_size) ] return out pipe = Pipeline(batch_size=batch_size, num_threads=4, device_id=0) with pipe: data = fn.external_source(source=get_data, layout="HWC") start = np.array([1, 2]) shape = np.array([3, 1]) outs = [ fn.slice(data, start, shape, start=start, end=start + shape, shape=shape, axes=(0, 1)), ] pipe.set_outputs(*outs) with assert_raises( RuntimeError, glob= '"end", "rel_end", "shape", and "rel_shape" arguments are mutually exclusive' ): pipe.build() for _ in range(1): outs = pipe.run()
def check_crop_with_out_of_bounds_error(device, batch_size, input_shape=(100, 200, 3)): # This test case is written with HWC layout in mind and "HW" axes in slice arguments layout = "HWC" assert len(input_shape) == 3 eii = RandomDataIterator(batch_size, shape=input_shape) crop_shape = tuple(extent * 2 for extent in input_shape[:2]) crop_y = 0.5 crop_x = 0.5 pipe = CropSynthPipe(device, batch_size, iter(eii), layout=layout, crop_shape=crop_shape, crop_x=crop_x, crop_y=crop_y, out_of_bounds_policy="error") pipe.build() with assert_raises( RuntimeError, glob="Slice can't be placed out of bounds with current policy."): _ = pipe.run()
def check_crop_wrong_layout(device, batch_size, input_shape=(100, 200, 3), layout="ABC"): assert len(layout) == len(input_shape) @pipeline_def def get_pipe(): def get_data(): out = [ np.zeros(input_shape, dtype=np.uint8) for _ in range(batch_size) ] return out data = fn.external_source(source=get_data, layout=layout, device=device) return fn.crop(data, crop_h=10, crop_w=10) pipe = get_pipe(batch_size=batch_size, device_id=0, num_threads=3) pipe.build() with assert_raises( RuntimeError, glob= f"The layout \"{layout}\" does not match any of the allowed layouts" ): pipe.run()
def test_err_threshold_dim_clash(): with assert_raises( RuntimeError, glob= "Argument \"threshold\" expected shape 2 but got 5 values, which can't be interpreted as the expected shape." ): _test_err_args(threshold=[1, 2, 3, 4, 5])
def check_tensor_input_fail(batch_size, shape, layout, window_size, smoothing_size, scale, normalize, dtype, err_regex): iterator = RandomlyShapedDataIterator(batch_size, max_shape=shape, dtype=np.uint8) def gen_params(): return np.array(window_size, dtype=np.int32), np.array(smoothing_size, dtype=np.int32), \ np.array(scale, dtype=np.float32) @pipeline_def def pipeline(): data = fn.external_source(iterator, layout=layout) window_size, smoothing_size, scale = fn.external_source(gen_params, batch=False, num_outputs=3) edges = fn.laplacian(data, window_size=window_size, smoothing_size=smoothing_size, scale=scale, normalized_kernel=normalize, dtype=dtype) return edges, data with assert_raises(RuntimeError, regex=err_regex): pipe = pipeline(device_id=types.CPU_ONLY_DEVICE_ID, num_threads=4, batch_size=batch_size) pipe.build() pipe.run()
def check_roi_random_crop_error(shape_like_in=None, in_shape=None, crop_shape=None, roi_start=None, roi_shape=None, roi_end=None, error_msg=""): batch_size = 3 niter = 3 pipe = dali.pipeline.Pipeline(batch_size=batch_size, num_threads=4, device_id=0, seed=1234) with pipe: inputs = [] if shape_like_in is None else [shape_like_in] out = fn.roi_random_crop(*inputs, in_shape=in_shape, crop_shape=crop_shape, roi_start=roi_start, roi_shape=roi_shape, roi_end=roi_end, device='cpu') pipe.set_outputs(out) with assert_raises(RuntimeError, regex=error_msg): pipe.build() for _ in range(niter): pipe.run()