def get_operand(self, operand, kind, operand_type): if kind == "const": return types.Constant(magic_number, np_type_to_dali(operand_type)) elif "cpu" in kind: return operand elif "gpu" in kind: return operand.gpu()
def run_dali(reduce_fn, batch_fn, keep_dims, axes, output_type, add_mean_input=False, ddof=0): batch_size = batch_fn.batch_size() # Needed due to how ExternalSource API works. It fails on methods, partials. def get_batch(): return batch_fn() result_cpu = [] result_gpu = [] pipe = Pipeline(batch_size=batch_size, num_threads=4, device_id=0) args = {'keep_dims': keep_dims, 'axes': axes} if output_type is not None: args['dtype'] = np_type_to_dali(output_type) with pipe: input = fn.external_source(source=get_batch) if not add_mean_input: reduced_cpu = reduce_fn(input, **args) reduced_gpu = reduce_fn(input.gpu(), **args) else: mean = fn.reductions.mean(input, **args) args['ddof'] = ddof reduced_cpu = reduce_fn(input, mean, **args) reduced_gpu = reduce_fn(input.gpu(), mean.gpu(), **args) pipe.set_outputs(reduced_cpu, reduced_gpu) pipe.build() for _ in range(batch_fn.num_iter()): output = pipe.run() reduced_cpu = output[0].as_array() reduced_gpu = output[1].as_cpu().as_array() result_cpu.append(reduced_cpu) result_gpu.append(reduced_gpu) return result_cpu, result_gpu
def _test_random_object_bbox_with_class(max_batch_size, ndim, dtype, format=None, fg_prob=None, classes=None, weights=None, background=None, threshold=None, k_largest=None, cache=None): pipe = dali.Pipeline(max_batch_size, 4, device_id=None, seed=4321) background_out = 0 if background is None else background classes_out = np.int32([]) if classes is None else classes weights_out = np.int32([]) if weights is None else weights threshold_out = np.int32([]) if threshold is None else threshold if cache: source = sampled_dataset(2 * max_batch_size, max_batch_size, ndim, dtype) else: source = batch_generator(max_batch_size, ndim, dtype) with pipe: inp = fn.external_source(source) if isinstance(background, dali.pipeline.DataNode) or (background is not None and background >= 0): inp = fn.cast(inp + (background_out + 1), dtype=np_type_to_dali(dtype)) # preconfigure op = ops.segmentation.RandomObjectBBox(format=format, foreground_prob=fg_prob, classes=classes, class_weights=weights, background=background, threshold=threshold, k_largest=k_largest, seed=1234) outs1 = op(inp, cache_objects=cache) outs2 = op(inp, output_class=True) if not isinstance(outs1, list): outs1 = [outs1] # the second instance should have always at least 2 outputs assert isinstance(outs2, (list, tuple)) outputs = [ inp, classes_out, weights_out, background_out, threshold_out, *outs1, *outs2 ] pipe.set_outputs(*outputs) pipe.build() format = format or "anchor_shape" for _ in range(50): inp, classes_out, weights_out, background_out, threshold_out, *outs = pipe.run( ) nout = (len(outs) - 1) // 2 outs1 = outs[:nout] outs2 = outs[nout:] for i in range(len(outs1)): check_batch(outs1[i], outs2[i]) # Iterate over indices instead of elements, because normal iteration # causes an exception to be thrown in native code, making debugging near impossible. outs = tuple([np.array(out[i]) for i in range(len(out))] for out in outs1) box_class_labels = [ np.int32(outs2[-1][i]) for i in range(len(outs2[-1])) ] boxes = convert_boxes(outs, format) for i in range(len(inp)): in_tensor = inp.at(i) class_labels = classes_out.at(i) if background is not None or classes is None: background_label = background_out.at(i) else: background_label = 0 if 0 not in class_labels else np.min( class_labels) - 1 label = box_class_labels[i] if classes is not None: assert label == background_label or label in list(class_labels) is_foreground = label != background_label cls_boxes = class_boxes(in_tensor, label if is_foreground else None) if is_foreground: ref_boxes = cls_boxes if threshold is not None: extent = box_extent(boxes[i]) thr = threshold_out.at(i) assert np.all(extent >= thr) ref_boxes = list( filter(lambda box: np.all(box_extent(box) >= thr), cls_boxes)) if k_largest is not None: assert box_in_k_largest(ref_boxes, boxes[i], k_largest) assert contains_box(cls_boxes, boxes[i])
def cast_pipe(): inp = fn.external_source(src) inp_dev = inp.gpu() if device == 'gpu' else inp return inp, fn.cast(inp_dev, dtype=np_type_to_dali(out_dtype))