def test_discard():
bs = 5
pipe = dali.Pipeline(batch_size=bs,
device_id=None,
num_threads=5,
py_num_workers=4,
py_start_method='spawn')
with pipe:
ext1 = dali.fn.external_source([[np.float32(i)
for i in range(bs)]] * 3,
cycle='raise')
ext2 = dali.fn.external_source(ext_cb2, batch=False, parallel=True)
ext3 = dali.fn.external_source(ext_cb2, batch=False, parallel=False)
pipe.set_outputs(ext1, ext2, ext3)
pipe.build()
utils.capture_processes(pipe._py_pool)
sample_in_epoch = 0
iteration = 0
for i in range(10):
try:
e1, e2, e3 = pipe.run()
for i in range(bs):
assert e1.at(i) == i
assert np.array_equal(
e2.at(i), np.array([sample_in_epoch, i, iteration]))
assert np.array_equal(
e3.at(i), np.array([sample_in_epoch, i, iteration]))
sample_in_epoch += 1
iteration += 1
except StopIteration:
sample_in_epoch = 0
iteration = 0
pipe.reset()
def _test_err_args(**kwargs):
pipe = dali.Pipeline(batch_size=1, num_threads=1, device_id=0, seed=1234)
inp = fn.external_source(data, batch=False)
outs = fn.segmentation.random_object_bbox(inp, **kwargs)
pipe.set_outputs(*outs)
pipe.build()
pipe.run()
def _test_random_object_bbox_ignore_class(max_batch_size,
ndim,
dtype,
format=None,
background=None,
threshold=None,
k_largest=None):
pipe = dali.Pipeline(max_batch_size, 4, device_id=None, seed=4321)
background_out = 0 if background is None else background
threshold_out = np.int32([]) if threshold is None else threshold
with pipe:
inp = fn.external_source(batch_generator(max_batch_size, ndim, dtype))
outs = fn.segmentation.random_object_bbox(inp,
format=format,
ignore_class=True,
background=background,
seed=1234,
threshold=threshold,
k_largest=k_largest)
if not isinstance(outs, list):
outs = [outs]
pipe.set_outputs(inp, background_out, threshold_out, *outs)
pipe.build()
format = format or "anchor_shape"
for _ in range(50):
inp, background_out, threshold_out, *outs = pipe.run()
# Iterate over indices instead of elements, because normal iteration
# causes an exception to be thrown in native code, making debugging near impossible.
outs = tuple([np.array(out[i]) for i in range(len(out))]
for out in outs)
boxes = convert_boxes(outs, format)
for i in range(len(inp)):
in_tensor = inp.at(i)
background_label = background_out.at(i)
ref_boxes = all_boxes(in_tensor, None, background_label)
if threshold is not None:
thr = threshold_out.at(i)
ref_boxes = list(
filter(lambda box: np.all(box_extent(box) >= thr),
ref_boxes))
if len(ref_boxes) == 0:
ref_boxes = np.int32([[0] * len(in_tensor.shape) +
list(in_tensor.shape)])
if k_largest is not None:
assert box_in_k_largest(ref_boxes, boxes[i], k_largest)
else:
assert contains_box(ref_boxes, boxes[i])
def _test_random_object_bbox_auto_bg(fg_labels, expected_bg):
"""Checks that a correct backgorund labels is chosen:
0, if 0 is not present in the list of foreground classes
smallest label - 1 if 0 is present
if the smallest label -1 overflows, decrement the label until no collision
"""
pipe = dali.Pipeline(batch_size=1, num_threads=1, device_id=0, seed=1234)
data = np.uint32([0,1,2,3])
box, label = fn.segmentation.random_object_bbox(data, foreground_prob=1e-9, format="box", output_class=1, classes=fg_labels);
pipe.set_outputs(box, label)
pipe.build()
_, labels = pipe.run()
assert int(labels.at(0)) == expected_bg
def ground_truth_pipeline(batch_size, video_reader_params, resize_params):
pipeline = video_reader_pipeline(batch_size, video_reader_params)
def get_next_frame():
pipe_out, = pipeline.run()
sequences_out = pipe_out.as_cpu().as_array()
for sample in range(batch_size):
for frame in range(video_reader_params['sequence_length']):
yield [np.expand_dims(sequences_out[sample][frame], 0)]
gt_pipeline = dali.Pipeline(batch_size=1, **pipeline_params)
with gt_pipeline:
resized_frame = dali.fn.external_source(source=get_next_frame,
num_outputs=1)
resized_frame = resized_frame[0].gpu()
resized_frame = dali.fn.resize(resized_frame, **resize_params)
gt_pipeline.set_outputs(resized_frame)
gt_pipeline.build()
return gt_pipeline
def _test_vs_non_parallel(batch_size, cb_parallel, cb_seq, batch,
py_num_workers):
pipe = dali.Pipeline(batch_size=batch_size,
device_id=None,
num_threads=5,
py_num_workers=py_num_workers,
py_start_method='spawn')
with pipe:
ext_seq = dali.fn.external_source(cb_parallel,
batch=batch,
parallel=False)
ext_par = dali.fn.external_source(cb_seq, batch=batch, parallel=True)
pipe.set_outputs(ext_seq, ext_par)
pipe.build()
utils.capture_processes(pipe._py_pool)
for i in range(10):
seq, par = pipe.run()
for j in range(batch_size):
s = seq.at(j)
p = par.at(j)
assert np.array_equal(s, p)
def _test_random_object_bbox_with_class(max_batch_size,
ndim,
dtype,
format=None,
fg_prob=None,
classes=None,
weights=None,
background=None,
threshold=None,
k_largest=None,
cache=None):
pipe = dali.Pipeline(max_batch_size, 4, device_id=None, seed=4321)
background_out = 0 if background is None else background
classes_out = np.int32([]) if classes is None else classes
weights_out = np.int32([]) if weights is None else weights
threshold_out = np.int32([]) if threshold is None else threshold
if cache:
source = sampled_dataset(2 * max_batch_size, max_batch_size, ndim,
dtype)
else:
source = batch_generator(max_batch_size, ndim, dtype)
with pipe:
inp = fn.external_source(source)
if isinstance(background,
dali.pipeline.DataNode) or (background is not None
and background >= 0):
inp = fn.cast(inp + (background_out + 1),
dtype=np_type_to_dali(dtype))
# preconfigure
op = ops.segmentation.RandomObjectBBox(format=format,
foreground_prob=fg_prob,
classes=classes,
class_weights=weights,
background=background,
threshold=threshold,
k_largest=k_largest,
seed=1234)
outs1 = op(inp, cache_objects=cache)
outs2 = op(inp, output_class=True)
if not isinstance(outs1, list):
outs1 = [outs1]
# the second instance should have always at least 2 outputs
assert isinstance(outs2, (list, tuple))
outputs = [
inp, classes_out, weights_out, background_out, threshold_out,
*outs1, *outs2
]
pipe.set_outputs(*outputs)
pipe.build()
format = format or "anchor_shape"
for _ in range(50):
inp, classes_out, weights_out, background_out, threshold_out, *outs = pipe.run(
)
nout = (len(outs) - 1) // 2
outs1 = outs[:nout]
outs2 = outs[nout:]
for i in range(len(outs1)):
check_batch(outs1[i], outs2[i])
# Iterate over indices instead of elements, because normal iteration
# causes an exception to be thrown in native code, making debugging near impossible.
outs = tuple([np.array(out[i]) for i in range(len(out))]
for out in outs1)
box_class_labels = [
np.int32(outs2[-1][i]) for i in range(len(outs2[-1]))
]
boxes = convert_boxes(outs, format)
for i in range(len(inp)):
in_tensor = inp.at(i)
class_labels = classes_out.at(i)
if background is not None or classes is None:
background_label = background_out.at(i)
else:
background_label = 0 if 0 not in class_labels else np.min(
class_labels) - 1
label = box_class_labels[i]
if classes is not None:
assert label == background_label or label in list(class_labels)
is_foreground = label != background_label
cls_boxes = class_boxes(in_tensor,
label if is_foreground else None)
if is_foreground:
ref_boxes = cls_boxes
if threshold is not None:
extent = box_extent(boxes[i])
thr = threshold_out.at(i)
assert np.all(extent >= thr)
ref_boxes = list(
filter(lambda box: np.all(box_extent(box) >= thr),
cls_boxes))
if k_largest is not None:
assert box_in_k_largest(ref_boxes, boxes[i], k_largest)
assert contains_box(cls_boxes, boxes[i])
