def test_reduce_std_cpu(): pipe = Pipeline(batch_size=batch_size, num_threads=4, device_id=None) data = fn.external_source(source=get_data) mean = fn.reductions.mean(data) reduced = fn.reductions.std_dev(data, mean) pipe.set_outputs(reduced) pipe.build() for _ in range(3): pipe.run()
def test_mel_filter_bank_cpu(): pipe = Pipeline(batch_size=batch_size, num_threads=4, device_id=None) data = fn.external_source(source=get_audio_data) spectrum = fn.spectrogram(data, nfft=60, window_length=50, window_step=25) processed = fn.mel_filter_bank(spectrum) pipe.set_outputs(processed) pipe.build() for _ in range(3): pipe.run()
def _test_invalid_args(device, args, message, run): data = [np.uint8([[1, 2, 3]]), np.uint8([[1, 2]])] pipe = Pipeline(2, 1, 0) src = fn.external_source(lambda: data, device=device) pipe.set_outputs(fn.tensor_subscript(src, **args)) with assert_raises(RuntimeError, glob=message): pipe.build() if run: pipe.run()
def test_batch_permute_cpu(): pipe = Pipeline(batch_size=batch_size, num_threads=3, device_id=None) data = fn.external_source(source=get_data, layout="HWC") perm = fn.batch_permutation(seed=420) processed = fn.permute_batch(data, indices=perm) pipe.set_outputs(processed) pipe.build() for _ in range(3): pipe.run()
def test_external_source_collection(): pipe = Pipeline(1, 3, 0) batches = [[make_array([1.5, 2.5], dtype=datapy.float32)], [make_array([-1, 3.5, 4.5], dtype=datapy.float32)]] pipe.set_outputs(fn.external_source(batches)) pipe.build() run_and_check(pipe, batches)
def check_generic_gaussian_blur(batch_size, sigma, window_size, shape, layout, axes, op_type="cpu", in_dtype=np.uint8, out_dtype=types.NO_TYPE, random_shape=True): pipe = Pipeline(batch_size=batch_size, num_threads=4, device_id=0) min_shape = None if random_shape else shape data = RandomlyShapedDataIterator(batch_size, min_shape=min_shape, max_shape=shape, dtype=in_dtype) # Extract the numpy type from DALI, we can have float32 or the same as input if out_dtype == types.NO_TYPE: result_type = in_dtype elif dali_type(in_dtype) == out_dtype: result_type = in_dtype else: result_type = np.float32 with pipe: input = fn.external_source(data, layout=layout) if op_type == "gpu": input = input.gpu() blurred = fn.gaussian_blur(input, device=op_type, sigma=sigma, window_size=window_size, dtype=out_dtype) pipe.set_outputs(blurred, input) pipe.build() for _ in range(test_iters): result, input = pipe.run() if op_type == "gpu": result = result.as_cpu() input = input.as_cpu() input = to_batch(input, batch_size) skip_axes = count_skip_axes(layout) baseline = [ gaussian_baseline(img, sigma, window_size, axes, skip_axes, dtype=result_type) for img in input ] max_error = 1 if result_type != np.float32 else 1e-04 check_batch(result, baseline, batch_size, max_allowed_error=max_error, expected_layout=layout)
def check_coin_flip(device='cpu', batch_size=32, max_shape=[1e5], p=None, use_shape_like_input=False): pipe = Pipeline(batch_size=batch_size, device_id=0, num_threads=3, seed=123456) with pipe: def shape_gen_f(): return random_shape(max_shape) shape_arg = None inputs = [] shape_out = None if max_shape is not None: if use_shape_like_input: shape_like_in = dali.fn.external_source( lambda: np.zeros(shape_gen_f()), device=device, batch=False) inputs += [shape_like_in] shape_out = dali.fn.shapes(shape_like_in) else: shape_arg = dali.fn.external_source(shape_gen_f, batch=False) shape_out = shape_arg outputs = [ dali.fn.random.coin_flip(*inputs, device=device, probability=p, shape=shape_arg) ] if shape_out is not None: outputs += [shape_out] pipe.set_outputs(*outputs) pipe.build() outputs = pipe.run() data_out = outputs[0].as_cpu() if isinstance(outputs[0], TensorListGPU) else outputs[0] shapes_out = None if max_shape is not None: shapes_out = outputs[1].as_cpu() if isinstance( outputs[1], TensorListGPU) else outputs[1] p = p if p is not None else 0.5 for i in range(batch_size): data = np.array(data_out[i]) assert np.logical_or(data == 0, data == 1).all() if max_shape is not None: sample_shape = np.array(shapes_out[i]) assert (data.shape == sample_shape).all() total = len(data) positive = np.count_nonzero(data) np.testing.assert_allclose(p, positive / total, atol=0.005) # +/- -.5%
def dali_data_iter(batch_size: int, rec_file: str, idx_file: str, num_threads: int, initial_fill=32768, random_shuffle=True, prefetch_queue_depth=1, local_rank=0, name="reader", mean=(127.5, 127.5, 127.5), std=(127.5, 127.5, 127.5)): """ Parameters: ---------- initial_fill: int Size of the buffer that is used for shuffling. If random_shuffle is False, this parameter is ignored. """ rank: int = distributed.get_rank() world_size: int = distributed.get_world_size() import nvidia.dali.fn as fn import nvidia.dali.types as types from nvidia.dali.pipeline import Pipeline from nvidia.dali.plugin.pytorch import DALIClassificationIterator pipe = Pipeline( batch_size=batch_size, num_threads=num_threads, device_id=local_rank, prefetch_queue_depth=prefetch_queue_depth, ) condition_flip = fn.random.coin_flip(probability=0.5) with pipe: jpegs, labels = fn.readers.mxnet(path=rec_file, index_path=idx_file, initial_fill=initial_fill, num_shards=world_size, shard_id=rank, random_shuffle=random_shuffle, pad_last_batch=False, name=name) images = fn.decoders.image(jpegs, device="mixed", output_type=types.RGB) images = fn.crop_mirror_normalize(images, dtype=types.FLOAT, mean=mean, std=std, mirror=condition_flip) pipe.set_outputs(images, labels) pipe.build() return DALIWarper( DALIClassificationIterator( pipelines=[pipe], reader_name=name, ))
def test_set_outputs_err_msg_unpack(): data = [[[np.random.rand(1, 3, 2)], [np.random.rand(1, 4, 5)]]] pipe = Pipeline(batch_size=1, num_threads=1, device_id=None) pipe.set_outputs(fn.external_source(data, num_outputs=2, cycle='quiet')) with assert_raises( TypeError, glob= 'Illegal pipeline output type. The output * contains a nested `DataNode`' ): pipe.build()
def test_incorrect_dtype_arg(): batch_size = 2 src_data = [ [np.ones((120, 120, 3), dtype=np.float32)]*batch_size ] src_pipe = Pipeline(batch_size, 1, 0) src_ext = fn.external_source(source=src_data, device='cpu', dtype=DALIDataType.UINT8) src_pipe.set_outputs(src_ext) src_pipe.build() src_pipe.run()
def test_set_outputs_err_msg_unpack(): data = [[[np.random.rand(1, 3, 2)], [np.random.rand(1, 4, 5)]]] pipe = Pipeline(batch_size = 1, num_threads = 1, device_id = None) pipe.set_outputs(fn.external_source(data, num_outputs = 2, cycle = 'quiet')) pattern = 'Illegal pipeline output type. The output 0 contains a nested `DataNode`. ' + \ 'Missing list/tuple expansion (*) is the likely cause.' try: pipe.build() except TypeError as e: assert(pattern == e.__str__())
def test_ndim_changing(): batch_size = 2 src_data = [[np.ones((120, 120, 3), dtype=np.uint8)] * batch_size, [np.ones((120, 120), dtype=np.uint8)] * batch_size] src_pipe = Pipeline(batch_size, 1, 0) src_ext1 = fn.external_source(source=src_data, dtype=DALIDataType.UINT8) src_pipe.set_outputs(src_ext1) src_pipe.build() src_pipe.run() src_pipe.run()
def test_sequence_reader_cpu(): pipe = Pipeline(batch_size=batch_size, num_threads=4, device_id=None) processed = fn.readers.sequence(file_root=sequence_dir, sequence_length=2, shard_id=0, num_shards=1) pipe.set_outputs(processed) pipe.build() for _ in range(3): pipe.run()
def test_image_decoder_crop_device(): pipe = Pipeline(batch_size=batch_size, num_threads=4, device_id=None) input, _ = fn.readers.file(file_root=images_dir, shard_id=0, num_shards=1) decoded = fn.decoders.image_crop(input, output_type=types.RGB, crop=(10, 10)) pipe.set_outputs(decoded) pipe.build() for _ in range(3): pipe.run()
def test_set_outputs_err_msg_random_type(): pipe = Pipeline(batch_size = 1, num_threads = 1, device_id = None) pipe.set_outputs("test") pattern = "Illegal output type. The output 0 is a `<class 'str'>`. Allowed types are " + \ "``DataNode`` and types convertible to `types.Constant` (numerical constants, " + \ "1D lists/tuple of numbers and ND arrays)." try: pipe.build() except TypeError as e: assert(pattern == e.__str__())
def test_stack_cpu(): pipe = Pipeline(batch_size=batch_size, num_threads=3, device_id=None) data = fn.external_source(source=get_data, layout="HWC") data2 = fn.external_source(source=get_data, layout="HWC") data3 = fn.external_source(source=get_data, layout="HWC") pixel_pos = fn.stack(data, data2, data3) pipe.set_outputs(pixel_pos) pipe.build() for _ in range(3): pipe.run()
def test_changing_dtype(): batch_size = 2 src_data = [[np.ones((120, 120, 3), dtype=np.float32)] * batch_size, [np.ones((120, 120, 3), dtype=np.uint8)] * batch_size] src_pipe = Pipeline(batch_size, 1, 0) src_ext = fn.external_source(source=src_data) src_pipe.set_outputs(src_ext) src_pipe.build() src_pipe.run() src_pipe.run()
def test_invalid_args(): pipeline = Pipeline(batch_size=2, num_threads=4, device_id=0) with pipeline: inputs, _, _, ids = fn.coco_reader(file_root=file_root, annotations_file=train_annotations, image_ids=True, images=images, preprocessed_annotations_dir='/tmp') pipeline.set_outputs(ids) pipeline.build()
def test_coco_reader_cpu(): pipe = Pipeline(batch_size=batch_size, num_threads=4, device_id=None) out, _, _ = fn.readers.coco(file_root=coco_dir, annotations_file=coco_annotation, shard_id=0, num_shards=1) pipe.set_outputs(out) pipe.build() for _ in range(3): pipe.run()
def test_mxnet_iterator_empty_array(): from nvidia.dali.plugin.mxnet import DALIGenericIterator as MXNetIterator import mxnet as mx batch_size = 4 size = 5 all_np_types = [ np.bool_, np.int_, np.intc, np.intp, np.int8, np.int16, np.int32, np.int64, np.uint8, np.uint16, np.uint32, np.uint64, np.float_, np.float32, np.float16, np.short, np.long, np.longlong, np.ushort, np.ulonglong ] np_types = [] # store in np_types only types supported by MXNet for t in all_np_types: try: mx.nd.zeros([2, 2, 2], ctx=None, dtype=t) np_types.append(t) except mx.base.MXNetError: pass test_data_shape = [1, 3, 0, 4] def get_data(): # create batch of [type_a, type_a, type_b, type_b, ...] out = [[np.empty(test_data_shape, dtype=t)] * batch_size for t in np_types] out = [val for pair in zip(out, out) for val in pair] return out pipe = Pipeline(batch_size=batch_size, num_threads=3, device_id=0) outs = fn.external_source(source=get_data, num_outputs=len(np_types) * 2) pipe.set_outputs(*outs) pipe.build() # create map of [(data, type_a), (label, type_a), ...] data_map = [('data_{}'.format(i), MXNetIterator.DATA_TAG) for i, t in enumerate(np_types)] label_map = [('label_{}'.format(i), MXNetIterator.LABEL_TAG) for i, t in enumerate(np_types)] out_map = [val for pair in zip(data_map, label_map) for val in pair] iterator = MXNetIterator(pipe, output_map=out_map, size=size, dynamic_shape=True) for batch in iterator: for d, t in zip(batch[0].data, np_types): shape = d.asnumpy().shape assert shape[0] == batch_size print(shape) assert np.array_equal(shape[1:], test_data_shape) assert d.asnumpy().dtype == t
def test_combine_transforms_cpu(): pipe = Pipeline(batch_size=batch_size, num_threads=4, device_id=None) with pipe: t = fn.transforms.translation(offset=(1, 2)) r = fn.transforms.rotation(angle=30.0) s = fn.transforms.scale(scale=(2, 3)) out = fn.transforms.combine(t, r, s) pipe.set_outputs(out) pipe.build() for _ in range(3): pipe.run()
def test_mxnet_reader_cpu(): pipe = Pipeline(batch_size=batch_size, num_threads=4, device_id=None) out, _ = fn.readers.mxnet(path=os.path.join(recordio_dir, "train.rec"), index_path=os.path.join(recordio_dir, "train.idx"), shard_id=0, num_shards=1) pipe.set_outputs(out) pipe.build() for _ in range(3): pipe.run()
def _test_feed_input(device): src_pipe, batch_size = build_src_pipe(device) dst_pipe = Pipeline(batch_size, 1, 0, exec_async=False, exec_pipelined=False) dst_pipe.set_outputs(fn.external_source(name="ext", device=device)) dst_pipe.build() for iter in range(3): out1 = src_pipe.run() dst_pipe.feed_input("ext", out1[0]) out2 = dst_pipe.run() check_batch(out2[0], out1[0], batch_size, 0, 0, "XY")
def check_no_input(op, get_data=get_data, **kwargs): pipe = Pipeline(batch_size=batch_size, num_threads=4, device_id=None) with pipe: processed = op(**kwargs) if isinstance(processed, Iterable): pipe.set_outputs(*processed) else: pipe.set_outputs(processed) pipe.build() for _ in range(3): pipe.run()
def test_external_source_generator(): pipe = Pipeline(1, 3, 0) def gen(): for i in range(5): yield [make_array([i + 1.5], dtype=datapy.float32)] pipe.set_outputs(fn.external_source(gen())) pipe.build() for i in range(5): check_output(pipe.run(), [np.array([i + 1.5], dtype=np.float32)])
def check_per_sample_gaussian_blur(batch_size, sigma_dim, window_size_dim, shape, layout, axes, op_type="cpu"): pipe = Pipeline(batch_size=batch_size, num_threads=4, device_id=0) data = RandomlyShapedDataIterator(batch_size, max_shape=shape) with pipe: if sigma_dim is not None: sigma = fn.uniform(range=[0.5, 3], shape=[sigma_dim]) sigma_arg = sigma else: # placeholder, so we can return something sigma = fn.coin_flip(probability=0) sigma_arg = None if window_size_dim is not None: window_radius = fn.uniform(range=[5, 10], shape=[window_size_dim]) window_size = fn.cast(window_radius, dtype=types.INT32) * 2 + 1 window_arg = window_size else: window_size = fn.coin_flip(probability=0) window_arg = None input = fn.external_source(data, layout=layout) if op_type == "gpu": input = input.gpu() blurred = fn.gaussian_blur(input, device=op_type, sigma=sigma_arg, window_size=window_arg) pipe.set_outputs(blurred, input, sigma, window_size) pipe.build() for _ in range(test_iters): result, input, sigma, window_size = pipe.run() if op_type == "gpu": result = result.as_cpu() input = input.as_cpu() input = to_batch(input, batch_size) sigma = to_batch(sigma, batch_size) window_size = to_batch(window_size, batch_size) baseline = [] for i in range(batch_size): sigma_arg = sigma[i] if sigma is not None else None window_arg = window_size[i] if window_size_dim is not None else None skip_axes = count_skip_axes(layout) baseline.append( gaussian_baseline(input[i], sigma_arg, window_arg, axes, skip_axes)) check_batch(result, baseline, batch_size, max_allowed_error=1)
def test_operator_coco_reader_same_images(): file_root = os.path.join(test_data_root, 'db', 'coco_pixelwise', 'images') train_annotations = os.path.join(test_data_root, 'db', 'coco_pixelwise', 'instances.json') coco_dir = os.path.join(test_data_root, 'db', 'coco') coco_dir_imgs = os.path.join(coco_dir, 'images') coco_pixelwise_dir = os.path.join(test_data_root, 'db', 'coco_pixelwise') coco_pixelwise_dir_imgs = os.path.join(coco_pixelwise_dir, 'images') for file_root, annotations_file in [ \ (coco_dir_imgs, os.path.join(coco_dir, 'instances.json')), (coco_pixelwise_dir_imgs, os.path.join(coco_pixelwise_dir, 'instances.json')), (coco_pixelwise_dir_imgs, os.path.join(coco_pixelwise_dir, 'instances_rle_counts.json'))]: pipe = Pipeline(batch_size=1, num_threads=4, device_id=0) with pipe: inputs1, boxes1, labels1, *other = fn.coco_reader( file_root=file_root, annotations_file=train_annotations, name="reader1", seed=1234 ) inputs2, boxes2, labels2, *other = fn.coco_reader( file_root=file_root, annotations_file=train_annotations, polygon_masks=True, name="reader2" ) inputs3, boxes3, labels3, *other = fn.coco_reader( file_root=file_root, annotations_file=train_annotations, pixelwise_masks=True, name="reader3" ) pipe.set_outputs( inputs1, boxes1, labels1, inputs2, boxes2, labels2, inputs3, boxes3, labels3 ) pipe.build() epoch_sz = pipe.epoch_size("reader1") assert epoch_sz == pipe.epoch_size("reader2") assert epoch_sz == pipe.epoch_size("reader3") for i in range(epoch_sz): inputs1, boxes1, labels1, inputs2, boxes2, labels2, inputs3, boxes3, labels3 = \ pipe.run() np.testing.assert_array_equal(inputs1.at(0), inputs2.at(0)) np.testing.assert_array_equal(inputs1.at(0), inputs3.at(0)) np.testing.assert_array_equal(labels1.at(0), labels2.at(0)) np.testing.assert_array_equal(labels1.at(0), labels3.at(0)) np.testing.assert_array_equal(boxes1.at(0), boxes2.at(0)) np.testing.assert_array_equal(boxes1.at(0), boxes3.at(0))
def test_dtype_arg(): batch_size = 2 src_data = [[np.ones((120, 120, 3), dtype=np.uint8)] * batch_size] src_pipe = Pipeline(batch_size, 1, 0) src_ext = fn.external_source(source=src_data, dtype=DALIDataType.UINT8) src_pipe.set_outputs(src_ext) src_pipe.build() out, = src_pipe.run() for i in range(batch_size): t = out.at(i) assert t.dtype == np.uint8 np.array_equal(t, np.ones((120, 120, 3), dtype=np.uint8))
def _test_permute_batch_out_of_range(device): batch_size = 10 pipe = Pipeline(batch_size, 4, 0) data = fn.external_source(source=lambda: gen_data(batch_size, np.int32), device=device, layout="abc") perm = fn.batch_permutation() pipe.set_outputs( data, fn.permute_batch(data, indices=[0, 1, 2, 3, 4, 5, 10, 7, 8, 9]), perm) pipe.build() pipe.run()
def test_ndim_data_mismatch(): batch_size = 2 src_data = [[[np.ones((120, 120, 3), dtype=np.uint8)] * batch_size, [np.ones((120, 120), dtype=np.uint8)] * batch_size]] src_pipe = Pipeline(batch_size, 1, 0) src_ext1, src_ext2 = fn.external_source(source=src_data, num_outputs=2, dtype=DALIDataType.UINT8, ndim=3) src_pipe.set_outputs(src_ext1, src_ext2) src_pipe.build() src_pipe.run()