def test_output_layout(): pipe = CommonPipeline(1, 1, 0, 999, images_dir) with pipe: images, _ = pipe.load() out1, out2 = fn.python_function(images, function=lambda x: (x, x.mean(2)), num_outputs=2, output_layouts=['ABC', 'DE']) out3, out4 = fn.python_function(images, function=lambda x: (x, x / 2), num_outputs=2, output_layouts='FGH') out5, out6 = fn.python_function(images, function=lambda x: (x, x / 2), num_outputs=2, output_layouts=['IJK']) pipe.set_outputs(out1, out2, out3, out4, out5, out6) pipe.build() out1, out2, out3, out4, out5, out6 = pipe.run() assert (out1.layout() == 'ABC') assert (out2.layout() == 'DE') assert (out3.layout() == 'FGH') assert (out4.layout() == 'FGH') assert (out5.layout() == 'IJK') assert (out6.layout() == '')
def test_invalid_layouts_arg(): pipe = Pipeline(1, 1, 0, 999, exec_async=False, exec_pipelined=False) with pipe: out = fn.python_function(function=lambda: numpy.zeros((1, 1)), output_layouts=['HW', 'HWC']) pipe.set_outputs(out) pipe.build() pipe.run()
def pipe(max_batch_size, input_data, device): pipe = Pipeline(batch_size=max_batch_size, num_threads=4, device_id=0, exec_async=False, exec_pipelined=False) with pipe: data = fn.external_source(source=input_data, cycle=False, device=device) processed = fn.python_function(data, function=resize, num_outputs=1) pipe.set_outputs(processed) return pipe
def bricon_ref_pipe(data_iterator, contrast_center, dtype, has_3_dims=False): brightness, brightness_shift = brightness_params() contrast = contrast_param() inp = fn.external_source(source=data_iterator) layout = "FHWC" if has_3_dims else "HWC" return fn.python_function(inp, brightness, brightness_shift, contrast,\ function=ref_operator(contrast_center, dali_type_to_np(dtype)), output_layouts=layout)
def test_python_function(): pipe = dali.pipeline.Pipeline(3,1,0, exec_async=False, exec_pipelined=False) with pipe: def func(inp): ret = [x*x for x in inp] return [ret] out_cpu = fn.python_function(np.array([[1,2],[3,4]]), function=func, batch_processing=True) pipe.set_outputs(out_cpu) pipe.build() o = pipe.run() assert np.array_equal(o[0].at(0), np.array([[1,4],[9,16]]))
def create_ref_pipe(channel_first, seq_len, interp, dtype, w, h, batch_size=2): pipe = dali.pipeline.Pipeline(batch_size, 1, 0, 0, exec_async=False, exec_pipelined=False) with pipe: layout = "FCHW" if channel_first else "FHWC" ext = fn.external_source(GetSequences(channel_first, seq_len, batch_size), layout=layout) pil_resized = fn.python_function(ext, function=resize_PIL(channel_first, interp, w, h), batch_processing=False) if dtype is not None: # unfortunately, PIL can't quite handle that pil_resized = fn.cast(pil_resized, dtype=dtype) pil_resized = fn.reshape(pil_resized, layout=layout) pipe.set_outputs(pil_resized) return pipe
def test_fn_python_function(): pipe = Pipeline(1, 1, 0, exec_pipelined = False, exec_async = False) batch1 = [np.array([1,2,3])] batch2 = [np.array([2,3,4])] # we need a context, because we use an operator with potential side-effects (python_function) with pipe: src = fn.external_source([batch1, batch2]) out = fn.python_function(src, function = lambda x: x+1) pipe.set_outputs(out) pipe.build() assert(np.array_equal(pipe.run()[0].at(0), batch1[0] + 1)) assert(np.array_equal(pipe.run()[0].at(0), batch2[0] + 1))
def test_python_function_cpu(): def resize(image): return np.array(Image.fromarray(image).resize((50, 10))) pipe = Pipeline(batch_size=batch_size, num_threads=4, device_id=None, exec_async=False, exec_pipelined=False) with pipe: data = fn.external_source(source=get_data, layout="HWC") processed = fn.python_function(data, function=resize) pipe.set_outputs(processed) pipe.build() for _ in range(3): pipe.run()
def test_python_operator_not_allowed_in_tf_dataset_error(): pipeline = Pipeline(1, 1, 0, exec_pipelined=False, exec_async=False) with pipeline: output = fn.python_function(function=lambda: np.zeros((3, 3, 3))) pipeline.set_outputs(output) shapes = ((1, 3, 3, 3)) dtypes = (tf.float32) with tf.device('/cpu:0'): _ = dali_tf.DALIDataset(pipeline=pipeline, batch_size=1, output_shapes=shapes, output_dtypes=dtypes, num_threads=1, device_id=0)
def bricon_ref_pipe(data_iterator, contrast_center, dtype, dev='cpu'): brightness, brightness_shift = brightness_params() contrast = contrast_param() inp = fn.external_source(source=data_iterator) return fn.python_function(inp, brightness, brightness_shift, contrast,\ function=ref_operator(contrast_center, dali_type_to_np(dtype)))