def test_element_extract_cpu(): pipe = Pipeline(batch_size=batch_size, num_threads=4, device_id=None) test_data_shape = [5, 10, 20, 3] def get_data(): out = [ np.random.randint(0, 255, size=test_data_shape, dtype=np.uint8) for _ in range(batch_size) ] return out data = fn.external_source(source=get_data, layout="FHWC") processed, _ = fn.element_extract(data, element_map=[0, 3]) pipe.set_outputs(processed) pipe.build() for _ in range(3): pipe.run()
def test_sequence_rearrange_cpu(): pipe = Pipeline(batch_size=batch_size, num_threads=4, device_id=None) test_data_shape = [5, 10, 20, 3] def get_data(): out = [ np.random.randint(0, 255, size=test_data_shape, dtype=np.uint8) for _ in range(batch_size) ] return out data = fn.external_source(source=get_data, layout="FHWC") processed = fn.sequence_rearrange(data, new_order=[0, 4, 1, 3, 2]) pipe.set_outputs(processed) pipe.build() for _ in range(3): pipe.run()
def test_lookup_table_cpu(): pipe = Pipeline(batch_size=batch_size, num_threads=4, device_id=None) test_data_shape = [100] def get_data(): out = [ np.random.randint(0, 5, size=test_data_shape, dtype=np.uint8) for _ in range(batch_size) ] return out data = fn.external_source(source=get_data) processed = fn.lookup_table(data, keys=[1, 3], values=[10, 50]) pipe.set_outputs(processed) pipe.build() for _ in range(3): pipe.run()
def test_pad_cpu(): pipe = Pipeline(batch_size=batch_size, num_threads=4, device_id=None) test_data_shape = [5, 4, 3] def get_data(): out = [ np.random.randint(0, 255, size=test_data_shape, dtype=np.uint8) for _ in range(batch_size) ] return out data = fn.external_source(source=get_data, layout="HWC") processed = fn.pad(data, fill_value=-1, axes=(0, ), shape=(10, )) pipe.set_outputs(processed) pipe.build() for _ in range(3): pipe.run()
def test_bb_flip_cpu(): pipe = Pipeline(batch_size=batch_size, num_threads=4, device_id=None) test_data_shape = [200, 4] def get_data(): out = [ (np.random.randint(0, 255, size=test_data_shape, dtype=np.uint8) / 255).astype(dtype=np.float32) for _ in range(batch_size) ] return out data = fn.external_source(source=get_data) processed = fn.bb_flip(data) pipe.set_outputs(processed) pipe.build() for _ in range(3): pipe.run()
def test_one_hot_cpu(): pipe = Pipeline(batch_size=batch_size, num_threads=4, device_id=None) test_data_shape = [200] def get_data(): out = [ np.random.randint(0, 255, size=test_data_shape, dtype=np.uint8) for _ in range(batch_size) ] return out data = fn.external_source(source=get_data) processed = fn.one_hot(data, num_classes=256) pipe.set_outputs(processed) pipe.build() for _ in range(3): pipe.run()
def check_type_mismatch(device, test_data_root, names): pipe = Pipeline(2, 2, 0) pipe.set_outputs( fn.numpy_reader(device=device, file_root=test_data_root, files=names)) pipe.build() err = None try: pipe.run() except RuntimeError as thrown: err = thrown # asserts should not be in except block to avoid printing nested exception on failure assert err, "Exception not thrown" assert "Inconsistent data" in str( err), "Unexpected error message: {}".format(err) assert "int32" in str(err) and "float" in str( err), "Unexpected error message: {}".format(err)
def test_random_bbox_crop_no_labels(): batch_size = 3 pipe = Pipeline(batch_size=batch_size, num_threads=4, device_id=0) test_box_shape = [200, 4] def get_boxes(): out = [(np.random.randint(0, 255, size = test_box_shape, dtype = np.uint8) / 255).astype(dtype = np.float32) for _ in range(batch_size)] return out boxes = fn.external_source(source = get_boxes) processed = fn.random_bbox_crop(boxes, aspect_ratio=[0.5, 2.0], thresholds=[0.1, 0.3, 0.5], scaling=[0.8, 1.0], bbox_layout="xyXY") pipe.set_outputs(*processed) pipe.build() for _ in range(3): pipe.run()
def test_external_source_with_iter(): pipe = Pipeline(1, 3, 0) pipe.set_outputs(fn.external_source(lambda i: [np.array([i + 1.5], dtype=np.float32)])) pipe.build() for i in range(10): check_output(pipe.run(), [np.array([i + 1.5], dtype=np.float32)])
def check_per_sample_gaussian_blur(batch_size, sigma_dim, window_size_dim, shape, layout, axes, op_type="cpu"): pipe = Pipeline(batch_size=batch_size, num_threads=4, device_id=0) data = RandomlyShapedDataIterator(batch_size, max_shape=shape) with pipe: if sigma_dim is not None: sigma = fn.random.uniform(range=[0.5, 3], shape=[sigma_dim]) sigma_arg = sigma else: # placeholder, so we can return something sigma = fn.random.coin_flip(probability=0) sigma_arg = None if window_size_dim is not None: window_radius = fn.random.uniform(range=[5, 10], shape=[window_size_dim]) window_size = fn.cast(window_radius, dtype=types.INT32) * 2 + 1 window_arg = window_size else: window_size = fn.random.coin_flip(probability=0) window_arg = None input = fn.external_source(data, layout=layout) if op_type == "gpu": input = input.gpu() blurred = fn.gaussian_blur(input, device=op_type, sigma=sigma_arg, window_size=window_arg) pipe.set_outputs(blurred, input, sigma, window_size) pipe.build() for _ in range(test_iters): result, input, sigma, window_size = pipe.run() if op_type == "gpu": result = result.as_cpu() input = input.as_cpu() input = to_batch(input, batch_size) sigma = to_batch(sigma, batch_size) window_size = to_batch(window_size, batch_size) baseline = [] for i in range(batch_size): sigma_arg = sigma[i] if sigma is not None else None window_arg = window_size[i] if window_size_dim is not None else None skip_axes = count_skip_axes(layout) baseline.append( gaussian_baseline(input[i], sigma_arg, window_arg, axes, skip_axes)) check_batch(result, baseline, batch_size, max_allowed_error=1, expected_layout=layout)
def check_container(cont): pipe = Pipeline(batch_size=1, num_threads=4, device_id=0) path = os.path.join(video_containers_data_root, cont) test_videos = [path + '/' + f for f in os.listdir(path)] with pipe: # mkv container for some reason fails in DALI VFR heuristics vid = fn.video_reader(device="gpu", filenames=test_videos, sequence_length=10, skip_vfr_check=True, stride=1, name="Reader") pipe.set_outputs(vid) pipe.build() iter_num = pipe.reader_meta("Reader")["epoch_size"] for _ in range(iter_num): pipe.run()
def test_external_source_with_iter(): for attempt in range(10): pipe = Pipeline(1, 3, 0) pipe.set_outputs(fn.external_source(lambda i: [make_array([attempt * 100 + i * 10 + 1.5], dtype=datapy.float32)])) pipe.build() for i in range(10): check_output(pipe.run(), [np.array([attempt * 100 + i * 10 + 1.5], dtype=np.float32)])
def check_coin_flip(device='cpu', batch_size=32, max_shape=[1e5], p=None, use_shape_like_input=False): pipe = Pipeline(batch_size=batch_size, device_id=0, num_threads=3, seed=123456) with pipe: def shape_gen_f(): return random_shape(max_shape) shape_arg = None inputs = [] shape_out = None if max_shape is not None: if use_shape_like_input: shape_like_in = dali.fn.external_source( lambda: np.zeros(shape_gen_f()), device=device, batch=False) inputs += [shape_like_in] shape_out = dali.fn.shapes(shape_like_in) else: shape_arg = dali.fn.external_source(shape_gen_f, batch=False) shape_out = shape_arg outputs = [ dali.fn.random.coin_flip(*inputs, device=device, probability=p, shape=shape_arg) ] if shape_out is not None: outputs += [shape_out] pipe.set_outputs(*outputs) pipe.build() outputs = pipe.run() data_out = outputs[0].as_cpu() if isinstance(outputs[0], TensorListGPU) else outputs[0] shapes_out = None if max_shape is not None: shapes_out = outputs[1].as_cpu() if isinstance( outputs[1], TensorListGPU) else outputs[1] p = p if p is not None else 0.5 for i in range(batch_size): data = np.array(data_out[i]) assert np.logical_or(data == 0, data == 1).all() if max_shape is not None: sample_shape = np.array(shapes_out[i]) assert (data.shape == sample_shape).all() total = len(data) positive = np.count_nonzero(data) np.testing.assert_allclose(p, positive / total, atol=0.005) # +/- -.5%
def check_generic_gaussian_blur(batch_size, sigma, window_size, shape, layout, axes, op_type="cpu", in_dtype=np.uint8, out_dtype=types.NO_TYPE, random_shape=True): pipe = Pipeline(batch_size=batch_size, num_threads=4, device_id=0) min_shape = None if random_shape else shape data = RandomlyShapedDataIterator(batch_size, min_shape=min_shape, max_shape=shape, dtype=in_dtype) # Extract the numpy type from DALI, we can have float32 or the same as input if out_dtype == types.NO_TYPE: result_type = in_dtype elif dali_type(in_dtype) == out_dtype: result_type = in_dtype else: result_type = np.float32 with pipe: input = fn.external_source(data, layout=layout) if op_type == "gpu": input = input.gpu() blurred = fn.gaussian_blur(input, device=op_type, sigma=sigma, window_size=window_size, dtype=out_dtype) pipe.set_outputs(blurred, input) pipe.build() for _ in range(test_iters): result, input = pipe.run() if op_type == "gpu": result = result.as_cpu() input = input.as_cpu() input = to_batch(input, batch_size) skip_axes = count_skip_axes(layout) baseline = [ gaussian_baseline(img, sigma, window_size, axes, skip_axes, dtype=result_type) for img in input ] max_error = 1 if result_type != np.float32 else 1e-04 check_batch(result, baseline, batch_size, max_allowed_error=max_error, expected_layout=layout)
def test_tfrecord_reader_cpu(): pipe = Pipeline(batch_size=batch_size, num_threads=4, device_id=None) tfrecord = sorted(glob.glob(os.path.join(tfrecord_dir, '*[!i][!d][!x]'))) tfrecord_idx = sorted(glob.glob(os.path.join(tfrecord_dir, '*idx'))) input = fn.tfrecord_reader(path=tfrecord, index_path=tfrecord_idx, shard_id=0, num_shards=1, features={ "image/encoded": tfrec.FixedLenFeature((), tfrec.string, ""), "image/class/label": tfrec.FixedLenFeature([1], tfrec.int64, -1) }) out = input["image/encoded"] pipe.set_outputs(out) pipe.build() for _ in range(3): pipe.run()
def test_arithm_ops_cpu(): pipe = Pipeline(batch_size=batch_size, num_threads=4, device_id=None) data = fn.external_source(source=get_data, layout="HWC") processed = [ data * 2, data + 2, data - 2, data / 2, data // 2, data**2, data == 2, data != 2, data < 2, data <= 2, data > 2, data >= 2, data & 2, data | 2, data ^ 2, dmath.abs(data), dmath.fabs(data), dmath.floor(data), dmath.ceil(data), dmath.pow(data, 2), dmath.fpow(data, 1.5), dmath.min(data, 2), dmath.max(data, 50), dmath.clamp(data, 10, 50), dmath.sqrt(data), dmath.rsqrt(data), dmath.cbrt(data), dmath.exp(data), dmath.exp(data), dmath.log(data), dmath.log2(data), dmath.log10(data), dmath.sin(data), dmath.cos(data), dmath.tan(data), dmath.asin(data), dmath.acos(data), dmath.atan(data), dmath.atan2(data, 3), dmath.sinh(data), dmath.cosh(data), dmath.tanh(data), dmath.asinh(data), dmath.acosh(data), dmath.atanh(data) ] pipe.set_outputs(*processed) pipe.build() for _ in range(3): pipe.run()
def test_nonsilent_region_cpu(): pipe = Pipeline(batch_size=batch_size, num_threads=4, device_id=None) test_data_shape = [200] def get_data(): out = [ np.random.randint(0, 255, size=test_data_shape, dtype=np.uint8) for _ in range(batch_size) ] out[0][0] = 0 out[1][0] = 0 out[1][1] = 0 return out data = fn.external_source(source=get_data) processed, _ = fn.nonsilent_region(data) pipe.set_outputs(processed) pipe.build() for _ in range(3): pipe.run()
def _test_scalar(device, as_tensors): """Test propagation of scalars from external source""" batch_size = 4 src_pipe = Pipeline(batch_size, 1, 0) src_ext = fn.external_source(source=lambda i: [np.float32(i * 10 + i + 1) for i in range(batch_size)], device=device) src_pipe.set_outputs(src_ext) src_pipe.build() dst_pipe = Pipeline(batch_size, 1, 0, exec_async=False, exec_pipelined=False) dst_pipe.set_outputs(fn.external_source(name="ext", device=device)) dst_pipe.build() for iter in range(3): src = src_pipe.run() data = src[0] if as_tensors: data = [data[i] for i in range(len(data))] dst_pipe.feed_input("ext", data) dst = dst_pipe.run() check_batch(src[0], dst[0], batch_size, 0, 0, "")
def _test_feed_input(device): src_pipe, batch_size = build_src_pipe(device) dst_pipe = Pipeline(batch_size, 1, 0, exec_async=False, exec_pipelined=False) dst_pipe.set_outputs(fn.external_source(name="ext", device=device)) dst_pipe.build() for iter in range(3): out1 = src_pipe.run() dst_pipe.feed_input("ext", out1[0]) out2 = dst_pipe.run() check_batch(out2[0], out1[0], batch_size, 0, 0, "XY")
def test_dtype_arg(): batch_size = 2 src_data = [[np.ones((120, 120, 3), dtype=np.uint8)] * batch_size] src_pipe = Pipeline(batch_size, 1, 0) src_ext = fn.external_source(source=src_data, dtype=DALIDataType.UINT8) src_pipe.set_outputs(src_ext) src_pipe.build() out, = src_pipe.run() for i in range(batch_size): t = out.at(i) assert t.dtype == np.uint8 np.array_equal(t, np.ones((120, 120, 3), dtype=np.uint8))
def test_external_source_generator(): pipe = Pipeline(1, 3, 0) def gen(): for i in range(5): yield [make_array([i + 1.5], dtype=datapy.float32)] pipe.set_outputs(fn.external_source(gen())) pipe.build() for i in range(5): check_output(pipe.run(), [np.array([i + 1.5], dtype=np.float32)])
def test_operator_coco_reader_same_images(): file_root = os.path.join(test_data_root, 'db', 'coco_pixelwise', 'images') train_annotations = os.path.join(test_data_root, 'db', 'coco_pixelwise', 'instances.json') coco_dir = os.path.join(test_data_root, 'db', 'coco') coco_dir_imgs = os.path.join(coco_dir, 'images') coco_pixelwise_dir = os.path.join(test_data_root, 'db', 'coco_pixelwise') coco_pixelwise_dir_imgs = os.path.join(coco_pixelwise_dir, 'images') for file_root, annotations_file in [ \ (coco_dir_imgs, os.path.join(coco_dir, 'instances.json')), (coco_pixelwise_dir_imgs, os.path.join(coco_pixelwise_dir, 'instances.json')), (coco_pixelwise_dir_imgs, os.path.join(coco_pixelwise_dir, 'instances_rle_counts.json'))]: pipe = Pipeline(batch_size=1, num_threads=4, device_id=0) with pipe: inputs1, boxes1, labels1, *other = fn.coco_reader( file_root=file_root, annotations_file=train_annotations, name="reader1", seed=1234 ) inputs2, boxes2, labels2, *other = fn.coco_reader( file_root=file_root, annotations_file=train_annotations, polygon_masks=True, name="reader2" ) inputs3, boxes3, labels3, *other = fn.coco_reader( file_root=file_root, annotations_file=train_annotations, pixelwise_masks=True, name="reader3" ) pipe.set_outputs( inputs1, boxes1, labels1, inputs2, boxes2, labels2, inputs3, boxes3, labels3 ) pipe.build() epoch_sz = pipe.epoch_size("reader1") assert epoch_sz == pipe.epoch_size("reader2") assert epoch_sz == pipe.epoch_size("reader3") for i in range(epoch_sz): inputs1, boxes1, labels1, inputs2, boxes2, labels2, inputs3, boxes3, labels3 = \ pipe.run() np.testing.assert_array_equal(inputs1.at(0), inputs2.at(0)) np.testing.assert_array_equal(inputs1.at(0), inputs3.at(0)) np.testing.assert_array_equal(labels1.at(0), labels2.at(0)) np.testing.assert_array_equal(labels1.at(0), labels3.at(0)) np.testing.assert_array_equal(boxes1.at(0), boxes2.at(0)) np.testing.assert_array_equal(boxes1.at(0), boxes3.at(0))
def test_image_decoder_slice_cpu(): anch_shape = [2] def get_anchors(): out = [(np.random.randint(1, 128, size=anch_shape, dtype=np.uint8) / 255).astype(dtype=np.float32) for _ in range(batch_size)] return out def get_shape(): out = [(np.random.randint(1, 128, size=anch_shape, dtype=np.uint8) / 255).astype(dtype=np.float32) for _ in range(batch_size)] return out pipe = Pipeline(batch_size=batch_size, num_threads=4, device_id=None) input, _ = fn.readers.file(file_root=images_dir, shard_id=0, num_shards=1) anchors = fn.external_source(source=get_anchors) shape = fn.external_source(source=get_shape) processed = fn.decoders.image_slice(input, anchors, shape) pipe.set_outputs(processed) pipe.build() for _ in range(3): pipe.run()
def test_external_source_collection_cycling(): pipe = Pipeline(1, 3, 0) batches = [[np.array([1.5, 2.5], dtype=np.float32)], [np.array([-1, 3.5, 4.5], dtype=np.float32)]] pipe.set_outputs(fn.external_source(batches, cycle=True)) pipe.build() # epochs are cycles over the source iterable for epoch in range(3): for batch in batches: check_output(pipe.run(), batch)
def test_external_source_gen_function_cycle(): pipe = Pipeline(1, 3, 0) def gen(): for i in range(5): yield [make_array([i + 1.5], dtype=datapy.float32)] pipe.set_outputs(fn.external_source(gen, cycle=True)) pipe.build() for _ in range(3): for i in range(5): check_output(pipe.run(), [np.array([i + 1.5], dtype=np.float32)])
def test_slice_cpu(): anch_shape = [2] def get_anchors(): out = [(np.random.randint(1, 256, size=anch_shape, dtype=np.uint8) / 255).astype(dtype=np.float32) for _ in range(batch_size)] return out def get_shape(): out = [(np.random.randint(1, 256, size=anch_shape, dtype=np.uint8) / 255).astype(dtype=np.float32) for _ in range(batch_size)] return out pipe = Pipeline(batch_size=batch_size, num_threads=4, device_id=None) data = fn.external_source(source=get_data, layout="HWC") anchors = fn.external_source(source=get_anchors) shape = fn.external_source(source=get_shape) processed = fn.slice(data, anchors, shape, out_of_bounds_policy="pad") pipe.set_outputs(processed) pipe.build() for _ in range(3): pipe.run()
def test_fn_python_function(): pipe = Pipeline(1, 1, 0, exec_pipelined = False, exec_async = False) batch1 = [np.array([1,2,3])] batch2 = [np.array([2,3,4])] # we need a context, because we use an operator with potential side-effects (python_function) with pipe: src = fn.external_source([batch1, batch2]) out = fn.python_function(src, function = lambda x: x+1) pipe.set_outputs(out) pipe.build() assert(np.array_equal(pipe.run()[0].at(0), batch1[0] + 1)) assert(np.array_equal(pipe.run()[0].at(0), batch2[0] + 1))
def test_segmentation_select_masks(): def get_data_source(*args, **kwargs): return lambda: make_batch_select_masks(*args, **kwargs) pipe = Pipeline(batch_size=batch_size, num_threads=4, device_id=None, seed=1234) with pipe: polygons, vertices, selected_masks = fn.external_source( num_outputs=3, device='cpu', source=get_data_source(batch_size, vertex_ndim=2, npolygons_range=(1, 5), nvertices_range=(3, 10))) out_polygons, out_vertices = fn.segmentation.select_masks( selected_masks, polygons, vertices, reindex_masks=False) pipe.set_outputs(polygons, vertices, selected_masks, out_polygons, out_vertices) pipe.build() for _ in range(3): pipe.run()
def test_bbox_paste_cpu(): pipe = Pipeline(batch_size=batch_size, num_threads=4, device_id=None) test_data_shape = [200, 4] def get_data(): out = [ (np.random.randint(0, 255, size=test_data_shape, dtype=np.uint8) / 255).astype(dtype=np.float32) for _ in range(batch_size) ] return out data = fn.external_source(source=get_data) paste_posx = fn.random.uniform(range=(0, 1)) paste_posy = fn.random.uniform(range=(0, 1)) paste_ratio = fn.random.uniform(range=(1, 2)) processed = fn.bbox_paste(data, paste_x=paste_posx, paste_y=paste_posy, ratio=paste_ratio) pipe.set_outputs(processed) pipe.build() for _ in range(3): pipe.run()
def test_dim_mismatch(): with tempfile.TemporaryDirectory() as test_data_root: num_samples = 2 batch_size = 2 names = ["2D.npy", "3D.npy"] paths = [os.path.join(test_data_root, name) for name in names] create_numpy_file(paths[0], [3, 4], np.float32, False) create_numpy_file(paths[1], [2, 3, 4], np.float32, False) pipe = Pipeline(2, 2, None) pipe.set_outputs(fn.numpy_reader(file_root=test_data_root, files=names)) pipe.build() err = None try: pipe.run() except RuntimeError as thrown: err = thrown # asserts should not be in except block to avoid printing nested exception on failure assert err, "Exception not thrown" assert "Inconsistent data" in str( err), "Unexpected error message: {}".format(err) assert "2 dimensions" in str(err) and "3 dimensions" in str( err), "Unexpected error message: {}".format(err)