def test_discard(): bs = 5 pipe = dali.Pipeline(batch_size=bs, device_id=None, num_threads=5, py_num_workers=4, py_start_method='spawn') with pipe: ext1 = dali.fn.external_source([[np.float32(i) for i in range(bs)]] * 3, cycle='raise') ext2 = dali.fn.external_source(ext_cb2, batch=False, parallel=True) ext3 = dali.fn.external_source(ext_cb2, batch=False, parallel=False) pipe.set_outputs(ext1, ext2, ext3) pipe.build() utils.capture_processes(pipe._py_pool) sample_in_epoch = 0 iteration = 0 for i in range(10): try: e1, e2, e3 = pipe.run() for i in range(bs): assert e1.at(i) == i assert np.array_equal( e2.at(i), np.array([sample_in_epoch, i, iteration])) assert np.array_equal( e3.at(i), np.array([sample_in_epoch, i, iteration])) sample_in_epoch += 1 iteration += 1 except StopIteration: sample_in_epoch = 0 iteration = 0 pipe.reset()
def _test_err_args(**kwargs): pipe = dali.Pipeline(batch_size=1, num_threads=1, device_id=0, seed=1234) inp = fn.external_source(data, batch=False) outs = fn.segmentation.random_object_bbox(inp, **kwargs) pipe.set_outputs(*outs) pipe.build() pipe.run()
def _test_random_object_bbox_ignore_class(max_batch_size, ndim, dtype, format=None, background=None, threshold=None, k_largest=None): pipe = dali.Pipeline(max_batch_size, 4, device_id=None, seed=4321) background_out = 0 if background is None else background threshold_out = np.int32([]) if threshold is None else threshold with pipe: inp = fn.external_source(batch_generator(max_batch_size, ndim, dtype)) outs = fn.segmentation.random_object_bbox(inp, format=format, ignore_class=True, background=background, seed=1234, threshold=threshold, k_largest=k_largest) if not isinstance(outs, list): outs = [outs] pipe.set_outputs(inp, background_out, threshold_out, *outs) pipe.build() format = format or "anchor_shape" for _ in range(50): inp, background_out, threshold_out, *outs = pipe.run() # Iterate over indices instead of elements, because normal iteration # causes an exception to be thrown in native code, making debugging near impossible. outs = tuple([np.array(out[i]) for i in range(len(out))] for out in outs) boxes = convert_boxes(outs, format) for i in range(len(inp)): in_tensor = inp.at(i) background_label = background_out.at(i) ref_boxes = all_boxes(in_tensor, None, background_label) if threshold is not None: thr = threshold_out.at(i) ref_boxes = list( filter(lambda box: np.all(box_extent(box) >= thr), ref_boxes)) if len(ref_boxes) == 0: ref_boxes = np.int32([[0] * len(in_tensor.shape) + list(in_tensor.shape)]) if k_largest is not None: assert box_in_k_largest(ref_boxes, boxes[i], k_largest) else: assert contains_box(ref_boxes, boxes[i])
def _test_random_object_bbox_auto_bg(fg_labels, expected_bg): """Checks that a correct backgorund labels is chosen: 0, if 0 is not present in the list of foreground classes smallest label - 1 if 0 is present if the smallest label -1 overflows, decrement the label until no collision """ pipe = dali.Pipeline(batch_size=1, num_threads=1, device_id=0, seed=1234) data = np.uint32([0,1,2,3]) box, label = fn.segmentation.random_object_bbox(data, foreground_prob=1e-9, format="box", output_class=1, classes=fg_labels); pipe.set_outputs(box, label) pipe.build() _, labels = pipe.run() assert int(labels.at(0)) == expected_bg
def ground_truth_pipeline(batch_size, video_reader_params, resize_params): pipeline = video_reader_pipeline(batch_size, video_reader_params) def get_next_frame(): pipe_out, = pipeline.run() sequences_out = pipe_out.as_cpu().as_array() for sample in range(batch_size): for frame in range(video_reader_params['sequence_length']): yield [np.expand_dims(sequences_out[sample][frame], 0)] gt_pipeline = dali.Pipeline(batch_size=1, **pipeline_params) with gt_pipeline: resized_frame = dali.fn.external_source(source=get_next_frame, num_outputs=1) resized_frame = resized_frame[0].gpu() resized_frame = dali.fn.resize(resized_frame, **resize_params) gt_pipeline.set_outputs(resized_frame) gt_pipeline.build() return gt_pipeline
def _test_vs_non_parallel(batch_size, cb_parallel, cb_seq, batch, py_num_workers): pipe = dali.Pipeline(batch_size=batch_size, device_id=None, num_threads=5, py_num_workers=py_num_workers, py_start_method='spawn') with pipe: ext_seq = dali.fn.external_source(cb_parallel, batch=batch, parallel=False) ext_par = dali.fn.external_source(cb_seq, batch=batch, parallel=True) pipe.set_outputs(ext_seq, ext_par) pipe.build() utils.capture_processes(pipe._py_pool) for i in range(10): seq, par = pipe.run() for j in range(batch_size): s = seq.at(j) p = par.at(j) assert np.array_equal(s, p)
def _test_random_object_bbox_with_class(max_batch_size, ndim, dtype, format=None, fg_prob=None, classes=None, weights=None, background=None, threshold=None, k_largest=None, cache=None): pipe = dali.Pipeline(max_batch_size, 4, device_id=None, seed=4321) background_out = 0 if background is None else background classes_out = np.int32([]) if classes is None else classes weights_out = np.int32([]) if weights is None else weights threshold_out = np.int32([]) if threshold is None else threshold if cache: source = sampled_dataset(2 * max_batch_size, max_batch_size, ndim, dtype) else: source = batch_generator(max_batch_size, ndim, dtype) with pipe: inp = fn.external_source(source) if isinstance(background, dali.pipeline.DataNode) or (background is not None and background >= 0): inp = fn.cast(inp + (background_out + 1), dtype=np_type_to_dali(dtype)) # preconfigure op = ops.segmentation.RandomObjectBBox(format=format, foreground_prob=fg_prob, classes=classes, class_weights=weights, background=background, threshold=threshold, k_largest=k_largest, seed=1234) outs1 = op(inp, cache_objects=cache) outs2 = op(inp, output_class=True) if not isinstance(outs1, list): outs1 = [outs1] # the second instance should have always at least 2 outputs assert isinstance(outs2, (list, tuple)) outputs = [ inp, classes_out, weights_out, background_out, threshold_out, *outs1, *outs2 ] pipe.set_outputs(*outputs) pipe.build() format = format or "anchor_shape" for _ in range(50): inp, classes_out, weights_out, background_out, threshold_out, *outs = pipe.run( ) nout = (len(outs) - 1) // 2 outs1 = outs[:nout] outs2 = outs[nout:] for i in range(len(outs1)): check_batch(outs1[i], outs2[i]) # Iterate over indices instead of elements, because normal iteration # causes an exception to be thrown in native code, making debugging near impossible. outs = tuple([np.array(out[i]) for i in range(len(out))] for out in outs1) box_class_labels = [ np.int32(outs2[-1][i]) for i in range(len(outs2[-1])) ] boxes = convert_boxes(outs, format) for i in range(len(inp)): in_tensor = inp.at(i) class_labels = classes_out.at(i) if background is not None or classes is None: background_label = background_out.at(i) else: background_label = 0 if 0 not in class_labels else np.min( class_labels) - 1 label = box_class_labels[i] if classes is not None: assert label == background_label or label in list(class_labels) is_foreground = label != background_label cls_boxes = class_boxes(in_tensor, label if is_foreground else None) if is_foreground: ref_boxes = cls_boxes if threshold is not None: extent = box_extent(boxes[i]) thr = threshold_out.at(i) assert np.all(extent >= thr) ref_boxes = list( filter(lambda box: np.all(box_extent(box) >= thr), cls_boxes)) if k_largest is not None: assert box_in_k_largest(ref_boxes, boxes[i], k_largest) assert contains_box(cls_boxes, boxes[i])