def check_cmn_vs_numpy(device, batch_size, dtype, output_layout,
mirror_probability, mean, std, scale, shift,
should_pad):
assert mirror_probability in cmn_coin_flip_samples
global cmn_idx
cmn_idx = 0
crop_z, crop_y, crop_x = (0.1, 0.2, 0.3)
crop_d, crop_h, crop_w = (10, 224, 224)
function = partial(crop_mirror_normalize_func, crop_z, crop_y, crop_x,
crop_d, crop_h, crop_w, mirror_probability, should_pad,
mean, std, scale, shift, "HWC", output_layout,
dali_type_to_np(dtype))
iterations = 8 if batch_size == 1 else 1
eps, max_err = (1e-5, 1e-5) if dtype == types.FLOAT else (0.3, 0.6)
compare_pipelines(CropMirrorNormalizePipeline(
device,
batch_size,
dtype=dtype,
output_layout=output_layout,
mirror_probability=mirror_probability,
mean=mean,
std=std,
scale=scale,
shift=shift,
pad_output=should_pad),
PythonOpPipeline(batch_size, function, output_layout),
batch_size=batch_size,
N_iterations=iterations,
eps=eps,
max_allowed_error=max_err)
def _test_rrc(device, max_frames, layout, aspect_ratio_range, area_range,
output_size, input_type, output_type):
batch_size = 4
pipe = dali.pipeline.Pipeline(batch_size, 4, 0)
channel_dim = layout.find('C')
value_range = type_range(test_utils.dali_type_to_np(input_type))
if channel_dim == len(layout) - 1:
channel_dim = -1
input = fn.external_source(source=generator(batch_size, max_frames,
channel_dim, input_type),
layout=layout)
shape = fn.shapes(input)
if device == "gpu":
input = input.gpu()
out = fn.random_resized_crop(input,
random_aspect_ratio=aspect_ratio_range,
random_area=area_range,
size=output_size,
interp_type=dali.types.INTERP_LINEAR,
seed=12321,
dtype=output_type)
pipe.set_outputs(out, shape)
pipe.build()
for iter in range(3):
outputs, input_shapes = pipe.run()
if device == "gpu":
outputs = outputs.as_cpu()
assert outputs.layout() == layout
for i in range(batch_size):
out = outputs.at(i)
input_shape = input_shapes.at(i).tolist()
check_output(out, channel_dim, input_shape, aspect_ratio_range,
area_range, value_range)
def run_decode(data_path, out_type):
batch_size = 4
pipe = Pipeline(batch_size=batch_size, num_threads=4, device_id=0)
input, _ = fn.file_reader(file_root=data_path,
shard_id=0,
num_shards=1,
name="reader")
decoded = fn.image_decoder(input, output_type=types.RGB)
decoded_shape = fn.shapes(decoded)
raw_shape = fn.peek_image_shape(input, type=out_type)
pipe.set_outputs(decoded, decoded_shape, raw_shape)
pipe.build()
samples = 0
length = pipe.reader_meta(name="reader")['epoch_size']
while samples < length:
samples += batch_size
(images, decoded_shape, raw_shape) = pipe.run()
for i in range(batch_size):
# as we are asking for a particular color space it may differ from the source image, so don't compare it
image = images.at(i)
shape_type = dali_type_to_np(out_type)
for d in range(len(image.shape) - 1):
assert image.shape[d] == decoded_shape.at(
i)[d], "{} vs {}".format(image.shape[d],
decoded_shape.at(i)[d])
assert image.shape[d] == raw_shape.at(i)[d], "{} vs {}".format(
image.shape[d],
raw_shape.at(i)[d])
assert raw_shape.at(
i)[d].dtype == shape_type, "{} vs {}".format(
raw_shape.at(i)[d].dtyp, shape_type)
def _test_type_conversion(device, src_type, in_values, dst_type, out_values,
eps):
src_nptype = dali_type_to_np(src_type)
dst_nptype = dali_type_to_np(dst_type)
assert len(out_values) == len(in_values)
in_data = [
np.full((100 + 10 * i, ), x, src_nptype)
for i, x in enumerate(in_values)
]
@pipeline_def(batch_size=len(in_values))
def test_pipe(device):
input = fn.external_source(in_data,
batch=False,
cycle='quiet',
device=device)
return fn.experimental.audio_resample(input,
dtype=dst_type,
scale=1,
quality=0)
pipe = test_pipe(device, device_id=0, num_threads=4)
pipe.build()
is_gpu = device == 'gpu'
for _ in range(2):
out, = pipe.run()
assert len(out) == len(out_values)
assert out.dtype == dst_type
for i in range(len(out_values)):
ref = np.full_like(in_data[i], out_values[i], dst_nptype)
out_arr = as_array(out[i])
if not np.allclose(out_arr, ref, 1e-6, eps):
print("Actual: ", out_arr)
print(out_arr.dtype, out_arr.shape)
print("Reference: ", ref)
print(ref.dtype, ref.shape)
print("Diff: ", out_arr.astype(np.float) - ref)
assert np.allclose(out_arr, ref, 1e-6, eps)
def generator(batch_size, max_frames, channel_dim, type):
type = test_utils.dali_type_to_np(type)
assert max_frames is not None or channel_dim != 1
def generate():
batch = []
for _ in range(batch_size):
frames = None if max_frames is None else np.random.randint(
1, max_frames + 1)
sz = np.random.randint(100, 2000 / (max_frames or 1))
w, h = np.random.randint(sz, 2 * sz, [2])
batch.append(generate_data(frames, w, h, channel_dim, type))
return batch
return generate
def check_cmn_random_data_vs_numpy(device, batch_size, dtype, input_layout,
input_shape, output_layout,
mirror_probability, mean, std, scale, shift,
should_pad):
crop_z, crop_y, crop_x = (0.1, 0.2, 0.3)
crop_d, crop_h, crop_w = (8, 16, 32)
eii1 = RandomDataIterator(batch_size, shape=input_shape)
eii2 = RandomDataIterator(batch_size, shape=input_shape)
assert mirror_probability in cmn_coin_flip_samples
global cmn_idx
cmn_idx = 0
function = partial(crop_mirror_normalize_func, crop_z, crop_y, crop_x,
crop_d, crop_h, crop_w, mirror_probability, should_pad,
mean, std, scale, shift, input_layout, output_layout,
dali_type_to_np(dtype))
cmn_pipe = CMNRandomDataPipeline(device,
batch_size,
input_layout,
iter(eii1),
dtype=dtype,
output_layout=output_layout,
mirror_probability=mirror_probability,
mean=mean,
std=std,
scale=scale,
shift=shift,
pad_output=should_pad)
ref_pipe = CMNRandomDataPythonOpPipeline(function, batch_size,
input_layout, output_layout,
iter(eii2))
eps, max_err = (1e-5, 1e-5) if dtype == types.FLOAT else (0.3, 0.6)
compare_pipelines(cmn_pipe,
ref_pipe,
batch_size,
2,
eps=eps,
max_allowed_error=max_err)
