def check_generic_gaussian_blur(batch_size, sigma, window_size, shape, layout, axes, op_type="cpu", in_dtype=np.uint8, out_dtype=types.NO_TYPE, random_shape=True): pipe = Pipeline(batch_size=batch_size, num_threads=4, device_id=0) min_shape = None if random_shape else shape data = RandomlyShapedDataIterator(batch_size, min_shape=min_shape, max_shape=shape, dtype=in_dtype) # Extract the numpy type from DALI, we can have float32 or the same as input if out_dtype == types.NO_TYPE: result_type = in_dtype elif dali_type(in_dtype) == out_dtype: result_type = in_dtype else: result_type = np.float32 with pipe: input = fn.external_source(data, layout=layout) if op_type == "gpu": input = input.gpu() blurred = fn.gaussian_blur(input, device=op_type, sigma=sigma, window_size=window_size, dtype=out_dtype) pipe.set_outputs(blurred, input) pipe.build() for _ in range(test_iters): result, input = pipe.run() if op_type == "gpu": result = result.as_cpu() input = input.as_cpu() input = to_batch(input, batch_size) skip_axes = count_skip_axes(layout) baseline = [ gaussian_baseline(img, sigma, window_size, axes, skip_axes, dtype=result_type) for img in input ] max_error = 1 if result_type != np.float32 else 1e-04 check_batch(result, baseline, batch_size, max_allowed_error=max_error, expected_layout=layout)
def check_coin_flip(device='cpu', batch_size=32, max_shape=[1e5], p=None, use_shape_like_input=False): pipe = Pipeline(batch_size=batch_size, device_id=0, num_threads=3, seed=123456) with pipe: def shape_gen_f(): return random_shape(max_shape) shape_arg = None inputs = [] shape_out = None if max_shape is not None: if use_shape_like_input: shape_like_in = dali.fn.external_source( lambda: np.zeros(shape_gen_f()), device=device, batch=False) inputs += [shape_like_in] shape_out = dali.fn.shapes(shape_like_in) else: shape_arg = dali.fn.external_source(shape_gen_f, batch=False) shape_out = shape_arg outputs = [ dali.fn.random.coin_flip(*inputs, device=device, probability=p, shape=shape_arg) ] if shape_out is not None: outputs += [shape_out] pipe.set_outputs(*outputs) pipe.build() outputs = pipe.run() data_out = outputs[0].as_cpu() if isinstance(outputs[0], TensorListGPU) else outputs[0] shapes_out = None if max_shape is not None: shapes_out = outputs[1].as_cpu() if isinstance( outputs[1], TensorListGPU) else outputs[1] p = p if p is not None else 0.5 for i in range(batch_size): data = np.array(data_out[i]) assert np.logical_or(data == 0, data == 1).all() if max_shape is not None: sample_shape = np.array(shapes_out[i]) assert (data.shape == sample_shape).all() total = len(data) positive = np.count_nonzero(data) np.testing.assert_allclose(p, positive / total, atol=0.005) # +/- -.5%
def test_mxnet_reader_cpu(): pipe = Pipeline(batch_size=batch_size, num_threads=4, device_id=None) out, _ = fn.readers.mxnet(path=os.path.join(recordio_dir, "train.rec"), index_path=os.path.join(recordio_dir, "train.idx"), shard_id=0, num_shards=1) check_no_input(fn.readers.mxnet, path=os.path.join(recordio_dir, "train.rec"), index_path=os.path.join(recordio_dir, "train.idx"), shard_id=0, num_shards=1)
def test_move_to_device_end(): test_data_shape = [1, 3, 0, 4] def get_data(): out = [ np.empty(test_data_shape, dtype=np.uint8) for _ in range(batch_size) ] return out pipe = Pipeline(batch_size=batch_size, num_threads=3, device_id=None) outs = fn.external_source(source=get_data) pipe.set_outputs(outs.gpu()) assert_raises( RuntimeError, pipe.build, glob= 'Cannot move the data node __ExternalSource_0 to the GPU in a CPU-only pipeline. ' 'The `device_id` parameter is set to `CPU_ONLY_DEVICE_ID`. ' 'Set `device_id` to a valid GPU identifier to enable GPU features in the pipeline.' )
def test_set_outputs_err_msg_random_type(): pipe = Pipeline(batch_size=1, num_threads=1, device_id=None) pipe.set_outputs("test") with assert_raises(TypeError, glob='Illegal output type. ' 'The output * is a `<class \'str\'>`.'): pipe.build()
def get_pipeline(batch_size=4, in_size=None, out_size=None, even_paste_count=False, k=4, dtype=types.UINT8, no_intersections=True, full_input=False, in_anchor_top_left=False, out_anchor_top_left=False, use_gpu=False, num_out_of_bounds=0): pipe = Pipeline(batch_size=batch_size, num_threads=4, device_id=0) with pipe: input, _ = fn.readers.file(file_root=img_dir) decoded = fn.decoders.image(input, device='cpu', output_type=types.RGB) resized = fn.resize(decoded, resize_x=in_size[1], resize_y=in_size[0]) in_idx_l, in_anchors_l, shapes_l, out_anchors_l = prepare_cuts( k, batch_size, in_size, out_size, even_paste_count, no_intersections, full_input, in_anchor_top_left, out_anchor_top_left, num_out_of_bounds) in_idx = fn.external_source(lambda: in_idx_l) in_anchors = fn.external_source(lambda: in_anchors_l) shapes = fn.external_source(lambda: shapes_l) out_anchors = fn.external_source(lambda: out_anchors_l) kwargs = {"in_ids": in_idx, "output_size": out_size, "dtype": dtype} if not full_input: kwargs["shapes"] = shapes if not in_anchor_top_left: kwargs["in_anchors"] = in_anchors if not out_anchor_top_left: kwargs["out_anchors"] = out_anchors pasted = fn.multi_paste(resized.gpu() if use_gpu else resized, **kwargs) pipe.set_outputs(pasted, resized) return pipe, in_idx_l, in_anchors_l, shapes_l, out_anchors_l
def get_pipeline(folder="train", custom_reader=None): pipe = Pipeline(batch_size=64, num_threads=1, device_id=1) if custom_reader: raw_files, labels = custom_reader else: raw_files, labels = fn.file_reader(file_root="%s" % folder, random_shuffle=True) decode = fn.image_decoder(raw_files, device="mixed", output_type=types.GRAY) resize = fn.resize(decode, device="gpu", image_type=types.RGB, interp_type=types.INTERP_LINEAR, resize_x=WIDTH, resize_y=HEIGHT) hsv = fn.hsv(resize, hue=fn.uniform(range=(-10, 10)), saturation=fn.uniform(range=(-.5, .5)), value=fn.uniform(range=(0.9, 1.2)), device="gpu", dtype=types.UINT8) bc = fn.brightness_contrast(hsv, device="gpu", brightness=fn.uniform(range=(.9, 1.1))) cmn = fn.crop_mirror_normalize(bc, device="gpu", output_dtype=types.FLOAT, output_layout=types.NHWC, image_type=types.GRAY, mean=[255 // 2], std=[255 // 2]) rot = fn.rotate(cmn, angle=fn.uniform(range=(-40, 40)), device="gpu", keep_size=True) tpose = fn.transpose(rot, perm=(2, 0, 1), device="gpu") # Reshaping to a format PyTorch likes pipe.set_outputs(tpose, labels) pipe.build() dali_iter = DALIClassificationIterator([pipe], -1) return dali_iter
def get_pipeline(batch_size, num_threads, device, device_id=0, shard_id=0, num_shards=1): test_data_root = os.environ['DALI_EXTRA_PATH'] file_root = os.path.join(test_data_root, 'db', 'coco_dummy', 'images') annotations_file = os.path.join(test_data_root, 'db', 'coco_dummy', 'instances.json') pipe = Pipeline(batch_size, num_threads, device_id) with pipe: jpegs, _, _, image_ids = fn.coco_reader( file_root=file_root, annotations_file=annotations_file, shard_id=shard_id, num_shards=num_shards, ratio=False, image_ids=True) images = fn.image_decoder( jpegs, device=('mixed' if device == 'gpu' else 'cpu'), output_type=types.RGB) images = fn.resize(images, resize_x=224, resize_y=224, interp_type=types.INTERP_LINEAR) images = fn.crop_mirror_normalize(images, dtype=types.FLOAT, mean=[128., 128., 128.], std=[1., 1., 1.]) if device == 'gpu': image_ids = image_ids.gpu() ids_reshaped = fn.reshape(image_ids, shape=[1, 1]) ids_int16 = fn.cast(image_ids, dtype=types.INT16) pipe.set_outputs(images, ids_reshaped, ids_int16) return pipe
def check_dim_mismatch(device, test_data_root, names): pipe = Pipeline(2, 2, 0) pipe.set_outputs( fn.numpy_reader(device=device, file_root=test_data_root, files=names)) pipe.build() err = None try: pipe.run() except RuntimeError as thrown: err = thrown # asserts should not be in except block to avoid printing nested exception on failure assert err, "Exception not thrown" assert "Inconsistent data" in str( err), "Unexpected error message: {}".format(err)
def _test_external_source_callback_split(use_fn_api, batch, as_tensor, device): iter_num = 5 batch_size = 9 pipe = Pipeline(batch_size, 3, 0) # this should produce a two-element list of Tensor(Lists), the first # being 2D, the second being 3D (+ batch dimension) source = TestIterator(iter_num, batch_size, [2, 3], as_tensor) iter_in = iter(source) if batch else iter( SampleIterator(iter(source), True)) if use_fn_api: inputs = fn.external_source(lambda: next(iter_in), 2, device=device, batch=batch) else: ext_source = ops.ExternalSource(lambda: next(iter_in), num_outputs=2, device=device, batch=batch) inputs = ext_source() pipe.set_outputs(*inputs) pipe.build() run_and_check(pipe, source)
def check_transform_scale_op(scale, center=None, has_input = False, reverse_order=False, batch_size=1, num_threads=4, device_id=0): ndim = len(scale) assert center is None or len(center) == ndim pipe = Pipeline(batch_size=batch_size, num_threads=num_threads, device_id=device_id) with pipe: if has_input: T0 = fn.uniform(range=(-1, 1), shape=(ndim, ndim+1), seed = 1234) T1 = fn.transforms.scale(T0, device='cpu', scale=scale, center=center, reverse_order=reverse_order) pipe.set_outputs(T1, T0) else: T1 = fn.transforms.scale(device='cpu', scale=scale, center=center) pipe.set_outputs(T1) pipe.build() outs = pipe.run() ref_mat = scale_affine_mat(scale=scale, center=center) T0 = outs[1] if has_input else None check_results(outs[0], batch_size, ref_mat, T0, reverse_order)
def check_container(cont): pipe = Pipeline(batch_size=1, num_threads=4, device_id=0) path = os.path.join(video_containers_data_root, cont) test_videos = [path + '/' + f for f in os.listdir(path)] with pipe: # mkv container for some reason fails in DALI VFR heuristics vid = fn.video_reader(device="gpu", filenames=test_videos, sequence_length=10, skip_vfr_check=True, stride=1, name="Reader") pipe.set_outputs(vid) pipe.build() iter_num = pipe.reader_meta("Reader")["epoch_size"] for _ in range(iter_num): pipe.run()
def test_python_function_cpu(): def resize(image): return np.array(Image.fromarray(image).resize((50, 10))) pipe = Pipeline(batch_size=batch_size, num_threads=4, device_id=None, exec_async=False, exec_pipelined=False) check_single_input(fn.python_function, function=resize, exec_async=False, exec_pipelined=False)
def test_external_source_iterate_ndarray(): pipe = Pipeline(4, 3, 0) batch = make_array([1.5, 2.5, 2, 3], dtype=datapy.float32) pipe.set_outputs(fn.external_source(batch, batch=False)) pipe.build() run_and_check(pipe, [batch])
def check_corrupted_videos(): corrupted_videos = [corrupted_video_data_root + '/' + f for f in os.listdir(corrupted_video_data_root)] for corrupted in corrupted_videos: pipe = Pipeline(batch_size=BATCH_SIZE, num_threads=4, device_id=0) with pipe: vid = fn.video_reader(device="gpu", filenames=corrupted, sequence_length=1) pipe.set_outputs(vid) pipe.build()
def test_set_outputs_err_msg_unpack(): data = [[[np.random.rand(1, 3, 2)], [np.random.rand(1, 4, 5)]]] pipe = Pipeline(batch_size=1, num_threads=1, device_id=None) pipe.set_outputs(fn.external_source(data, num_outputs=2, cycle='quiet')) with assert_raises(TypeError, glob='Illegal pipeline output type. ' 'The output * contains a nested `DataNode`'): pipe.build()
def check_transform_rotation_op(angle, axis=None, center=None, has_input = False, reverse_order=False, batch_size=1, num_threads=4, device_id=0): assert axis is None or len(axis) == 3 ndim = 3 if axis is not None else 2 assert center is None or len(center) == ndim pipe = Pipeline(batch_size=batch_size, num_threads=num_threads, device_id=device_id) with pipe: if has_input: T0 = fn.uniform(range=(-1, 1), shape=(ndim, ndim+1), seed = 1234) T1 = fn.transforms.rotation(T0, device='cpu', angle=angle, axis=axis, center=center, reverse_order=reverse_order) pipe.set_outputs(T1, T0) else: T1 = fn.transforms.rotation(device='cpu', angle=angle, axis=axis, center=center) pipe.set_outputs(T1) pipe.build() outs = pipe.run() ref_mat = rotate_affine_mat(angle=angle, axis=axis, center=center) T0 = outs[1] if has_input else None check_results(outs[0], batch_size, ref_mat, T0, reverse_order, rtol=1e-5)
def check_per_sample_gaussian_blur(batch_size, sigma_dim, window_size_dim, shape, layout, axes, op_type="cpu"): pipe = Pipeline(batch_size=batch_size, num_threads=4, device_id=0) data = RandomlyShapedDataIterator(batch_size, max_shape=shape) with pipe: if sigma_dim is not None: sigma = fn.uniform(range=[0.5, 3], shape=[sigma_dim]) sigma_arg = sigma else: # placeholder, so we can return something sigma = fn.coin_flip(probability=0) sigma_arg = None if window_size_dim is not None: window_radius = fn.uniform(range=[5, 10], shape=[window_size_dim]) window_size = fn.cast(window_radius, dtype=types.INT32) * 2 + 1 window_arg = window_size else: window_size = fn.coin_flip(probability=0) window_arg = None input = fn.external_source(data, layout=layout) if op_type == "gpu": input = input.gpu() blurred = fn.gaussian_blur(input, device=op_type, sigma=sigma_arg, window_size=window_arg) pipe.set_outputs(blurred, input, sigma, window_size) pipe.build() for _ in range(test_iters): result, input, sigma, window_size = pipe.run() if op_type == "gpu": result = result.as_cpu() input = input.as_cpu() input = to_batch(input, batch_size) sigma = to_batch(sigma, batch_size) window_size = to_batch(window_size, batch_size) baseline = [] for i in range(batch_size): sigma_arg = sigma[i] if sigma is not None else None window_arg = window_size[i] if window_size_dim is not None else None skip_axes = count_skip_axes(layout) baseline.append( gaussian_baseline(input[i], sigma_arg, window_arg, axes, skip_axes)) check_batch(result, baseline, batch_size, max_allowed_error=1)
def test_external_source_generator(): pipe = Pipeline(1, 3, 0) def gen(): for i in range(5): yield [make_array([i + 1.5], dtype=datapy.float32)] pipe.set_outputs(fn.external_source(gen())) pipe.build() for i in range(5): check_output(pipe.run(), [np.array([i + 1.5], dtype=np.float32)])
def test_dtype_arg(): batch_size = 2 src_data = [[np.ones((120, 120, 3), dtype=np.uint8)] * batch_size] src_pipe = Pipeline(batch_size, 1, 0) src_ext = fn.external_source(source=src_data, dtype=DALIDataType.UINT8) src_pipe.set_outputs(src_ext) src_pipe.build() out, = src_pipe.run() for i in range(batch_size): t = out.at(i) assert t.dtype == np.uint8 np.array_equal(t, np.ones((120, 120, 3), dtype=np.uint8))
def NumpyReaderPipeline(path, batch_size, device="cpu", file_list=None, files=None, path_filter="*.npy", num_threads=1, device_id=0, num_gpus=1, cache_header_information=False): pipe = Pipeline(batch_size=batch_size, num_threads=num_threads, device_id=0) data = fn.numpy_reader(device=device, file_list=file_list, files=files, file_root=path, file_filter=path_filter, shard_id=0, num_shards=1, cache_header_information=cache_header_information) pipe.set_outputs(data) return pipe
class DaliChecker: def __init__(self, batch_size, prefetch=2, device="mixed", device_id=0): log.debug("making checker") self.batch_size = batch_size self.prefetch = prefetch self.device = device self.device_id = device_id self.make_pipe() self.pipe.build() def make_pipe(self): log.debug("making pipe") self.pipe = Pipeline(batch_size=self.batch_size, num_threads=2, device_id=self.device_id, prefetch_queue_depth=self.prefetch) with self.pipe: self.files = fn.external_source() images = fn.image_decoder(self.files, device=self.device) self.pipe.set_outputs(images) def feed(self, images): self.pipe.feed_input(self.files, images)
def test_python_function_cpu(): def resize(image): return np.array(Image.fromarray(image).resize((50, 10))) pipe = Pipeline(batch_size=batch_size, num_threads=4, device_id=None, exec_async=False, exec_pipelined=False) with pipe: data = fn.external_source(source=get_data, layout="HWC") processed = fn.python_function(data, function=resize) pipe.set_outputs(processed) pipe.build() for _ in range(3): pipe.run()
def check_transform_shear_op(shear=None, angles=None, center=None, has_input=False, reverse_order=False, batch_size=1, num_threads=4, device_id=0): assert shear is not None or angles is not None if shear is not None: assert len(shear) == 2 or len(shear) == 6 ndim = 3 if len(shear) == 6 else 2 else: assert len(angles) == 2 or len(angles) == 6 ndim = 3 if len(angles) == 6 else 2 assert center is None or len(center) == ndim pipe = Pipeline(batch_size=batch_size, num_threads=num_threads, device_id=device_id, seed=1234) with pipe: if has_input: T0 = fn.random.uniform(range=(-1, 1), shape=(ndim, ndim + 1)) T1 = fn.transforms.shear(T0, device='cpu', shear=shear, angles=angles, center=center, reverse_order=reverse_order) pipe.set_outputs(T1, T0) else: T1 = fn.transforms.shear(device='cpu', shear=shear, angles=angles, center=center) pipe.set_outputs(T1) pipe.build() outs = pipe.run() ref_mat = shear_affine_mat(shear=shear, angles=angles, center=center) T0 = outs[1] if has_input else None check_results(outs[0], batch_size, ref_mat, T0, reverse_order, atol=1e-6)
def test_external_source_collection(): pipe = Pipeline(1, 3, 0) batches = [[make_array([1.5, 2.5], dtype=datapy.float32)], [make_array([-1, 3.5, 4.5], dtype=datapy.float32)]] pipe.set_outputs(fn.external_source(batches)) pipe.build() run_and_check(pipe, batches)
def test_external_source_collection_cycling(): pipe = Pipeline(1, 3, 0) batches = [[np.array([1.5, 2.5], dtype=np.float32)], [np.array([-1, 3.5, 4.5], dtype=np.float32)]] pipe.set_outputs(fn.external_source(batches, cycle=True)) pipe.build() # epochs are cycles over the source iterable for epoch in range(3): for batch in batches: check_output(pipe.run(), batch)
def test_external_source_gen_function_cycle(): pipe = Pipeline(1, 3, 0) def gen(): for i in range(5): yield [make_array([i + 1.5], dtype=datapy.float32)] pipe.set_outputs(fn.external_source(gen, cycle=True)) pipe.build() for _ in range(3): for i in range(5): check_output(pipe.run(), [np.array([i + 1.5], dtype=np.float32)])
def test_element_extract_cpu(): pipe = Pipeline(batch_size=batch_size, num_threads=4, device_id=None) test_data_shape = [5, 10, 20, 3] def get_data(): out = [ np.random.randint(0, 255, size=test_data_shape, dtype=np.uint8) for _ in range(batch_size) ] return out data = fn.external_source(source=get_data, layout="FHWC") processed, _ = fn.element_extract(data, element_map=[0, 3]) pipe.set_outputs(processed) pipe.build() for _ in range(3): pipe.run()
def test_sequence_rearrange_cpu(): pipe = Pipeline(batch_size=batch_size, num_threads=4, device_id=None) test_data_shape = [5, 10, 20, 3] def get_data(): out = [ np.random.randint(0, 255, size=test_data_shape, dtype=np.uint8) for _ in range(batch_size) ] return out data = fn.external_source(source=get_data, layout="FHWC") processed = fn.sequence_rearrange(data, new_order=[0, 4, 1, 3, 2]) pipe.set_outputs(processed) pipe.build() for _ in range(3): pipe.run()
def check_deserialization_with_params(batch_size, num_threads, shape): init_pipe = TestPipeline(batch_size=batch_size, num_threads=num_threads, shape=shape) serialized = init_pipe.serialize() ref_pipe = TestPipeline(batch_size=batch_size**2, num_threads=num_threads + 1, shape=shape) test_pipe = Pipeline.deserialize(serialized, batch_size=batch_size**2, num_threads=num_threads + 1) test_utils.compare_pipelines(ref_pipe, test_pipe, batch_size=batch_size**2, N_iterations=3)