def check_generic_gaussian_blur( batch_size, sigma, window_size, shape, layout, axes, op_type="cpu", in_dtype=np.uint8, out_dtype=types.NO_TYPE, random_shape=True): pipe = Pipeline(batch_size=batch_size, num_threads=4, device_id=0) min_shape = None if random_shape else shape data = RandomlyShapedDataIterator(batch_size, min_shape=min_shape, max_shape=shape, dtype=in_dtype) # Extract the numpy type from DALI, we can have float32 or the same as input if out_dtype == types.NO_TYPE: result_type = in_dtype elif dali_type(in_dtype) == out_dtype: result_type = in_dtype else: result_type = np.float32 with pipe: input = fn.external_source(data, layout=layout) if op_type == "gpu": input = input.gpu() blurred = fn.gaussian_blur(input, device=op_type, sigma=sigma, window_size=window_size, dtype=out_dtype) pipe.set_outputs(blurred, input) pipe.build() for _ in range(test_iters): result, input = pipe.run() if op_type == "gpu": result = result.as_cpu() input = input.as_cpu() input = to_batch(input, batch_size) skip_axes = count_skip_axes(layout) baseline = [ gaussian_baseline(img, sigma, window_size, axes, skip_axes, dtype=result_type) for img in input] max_error = 1 if result_type != np.float32 else 1e-04 check_batch(result, baseline, batch_size, max_allowed_error=max_error, expected_layout=layout)
def __init__(self, device, batch_size, dims, axes, axis_names, batch=False, out_type=None, in_type=None, shift=None, scale=None, num_threads=3, device_id=0, num_gpus=1): super(NormalizePipeline, self).__init__( batch_size, num_threads, device_id, seed=7865, exec_async=False, exec_pipelined=False) common_args = { "device": device, "axes": axes, "axis_names": axis_names, "batch": batch, "dtype": dali_type(out_type), "shift": shift, "scale": scale } self.in_type = in_type self.out_type = out_type self.device = device self.input = ops.ExternalSource() self.add_layout = None if axis_names is not None: layout = '' for i in range(dims): layout += chr(ord('a') + i) self.add_layout = ops.Reshape(layout=layout) self.batch = batch self.dims = dims self.has_axes = axes is not None or axis_names is not None self.scale = scale self.shift = shift self.is_integral = out_type is not None and out_type is not np.float32 if axis_names is not None: axes = [] for a in axis_names: axes.append(ord(a) - ord('a')) self.axes = axes self.axis_names = axis_names self.ddof = 2 if axes is not None and len(axes) > 0 else 0 self.eps = 0.25 self.mean = ops.PythonFunction(function=custom_mean(batch, axes), batch_processing=True) self.stddev = ops.PythonFunction(function=custom_stddev(batch, axes), batch_processing=True) self.normalize = ops.Normalize(**common_args, ddof=self.ddof) self.scalar_mean = ops.Normalize(**common_args, mean=1, ddof=self.ddof, epsilon=self.eps) self.scalar_stddev = ops.Normalize(**common_args, stddev=2, epsilon=self.eps) self.scalar_params = ops.Normalize(**common_args, mean=1, stddev=2)
def test_generic_gaussian_blur(): for dev in ["cpu"]: for t_in in [np.uint8, np.int32, np.float32]: for t_out in [types.NO_TYPE, types.FLOAT, dali_type(t_in)]: for shape, layout, axes in [((20, 20, 30, 3), "DHWC", 3), ((20, 20, 30), "", 3), ((20, 30, 3), "HWC", 2), ((20, 30), "HW", 2), ((3, 30, 20), "CWH", 2), ((5, 20, 30, 3), "FHWC", 2), ((5, 10, 10, 7, 3), "FDHWC", 3), ((5, 3, 20, 30), "FCHW", 2), ((3, 5, 10, 10, 7), "CFDHW", 3)]: for sigma in [1.0, [1.0, 2.0, 3.0]]: for window_size in [3, 5, [7, 5, 9], [3, 5, 9], None]: if isinstance(sigma, list): sigma = sigma[0:axes] if isinstance(window_size, list): window_size = window_size[0:axes] yield check_generic_gaussian_blur, 10, sigma, window_size, shape, layout, axes, dev, t_in, t_out for window_size in [11, 15]: yield check_generic_gaussian_blur, 10, None, window_size, shape, layout, axes, dev, t_in, t_out
def test_generic_gaussian_blur_slow(): for dev in ["cpu", "gpu"]: for t_in in [np.uint8, np.int32, np.float32]: for t_out in [types.NO_TYPE, types.FLOAT, dali_type(t_in)]: yield from generate_generic_cases(dev, t_in, t_out)
def _run_test(device, batch_size, out_dim, in_dim, in_dtype, out_dtype, M_kind, T_kind): pipe = dali.pipeline.Pipeline(batch_size=batch_size, num_threads=4, device_id=0, seed=1234) with pipe: X = fn.external_source(source=get_data_source(batch_size, in_dim, in_dtype), device=device, layout="NX") M = None T = None MT = None if T_kind == "fused": MT = make_param(M_kind, [out_dim, in_dim + 1]) else: M = make_param(M_kind, [out_dim, in_dim]) T = make_param(T_kind, [out_dim]) Y = fn.coord_transform( X, MT=MT.flatten().tolist() if isinstance(MT, np.ndarray) else MT, M=M.flatten().tolist() if isinstance(M, np.ndarray) else M, T=T.flatten().tolist() if isinstance(T, np.ndarray) else T, dtype=dali_type(out_dtype)) if M is None: M = 1 if T is None: T = 0 if MT is None: MT = 0 M, T, MT = (x if isinstance(x, dali.data_node.DataNode) else dali.types.Constant(x, dtype=dali.types.FLOAT) for x in (M, T, MT)) pipe.set_outputs(X, Y, M, T, MT) pipe.build() for iter in range(3): outputs = pipe.run() outputs = [x.as_cpu() if hasattr(x, "as_cpu") else x for x in outputs] ref = [] scale = 1 for idx in range(batch_size): X = outputs[0].at(idx) if T_kind == "fused": MT = outputs[4].at(idx) if MT.size == 1: M = MT T = 0 else: M = MT[:, :-1] T = MT[:, -1] else: M = outputs[2].at(idx) T = outputs[3].at(idx) if M.size == 1: Y = X.astype(np.float32) * M + T else: Y = np.matmul(X.astype(np.float32), M.transpose()) + T if np.issubdtype(out_dtype, np.integer): info = np.iinfo(out_dtype) Y = Y.clip(info.min, info.max) ref.append(Y) scale = max(scale, np.max(np.abs(Y)) - np.min(np.abs(Y))) if Y.size > 0 else 1 avg = 1e-6 * scale eps = 1e-6 * scale if out_dtype != np.float32: # headroom for rounding avg += 0.33 eps += 0.5 check_batch(outputs[1], ref, batch_size, eps, eps, expected_layout="NX", compare_layouts=True)