def test_deprecated_double_def(): error_msg = ("Usage of `{}` is deprecated in favor of `output_{}`*only `output_{}` " "should be provided.") shapes_error_msg = error_msg.format(*(("shapes",) * 3)) yield raises(ValueError, shapes_error_msg)(dali_pipe_deprecated), \ {"shapes": 2, "output_shapes": 2, "dtypes": tf.uint8}, 2, tf.uint8, dali_types.UINT8, 1, 2 dtypes_error_msg = error_msg.format(*(("dtypes",) * 3)) yield raises(ValueError, dtypes_error_msg)(dali_pipe_deprecated), \ {"shapes": 2, "dtypes": tf.uint8, "output_dtypes": tf.uint8}, \ 2, tf.uint8, dali_types.UINT8, 1, 2
def test_queue_large_failure(): max_int32 = 2**31 - 1 max_uint32 = 2**32 - 1 error_message = "Failed to serialize object as C-like structure. " \ "Tried to populate following fields:" for start_method in ("spawn", "fork"): yield raises(RuntimeError, error_message)(_test_queue_large), \ start_method, [(max_int32 + 1, 0, max_uint32, max_uint32, max_uint32)] yield raises(RuntimeError, error_message)(_test_queue_large), \ start_method, [(max_int32, max_int32, -1, 0, 0)]
def run_checks(samples_allowed, batches_allowed, samples_disallowed, batches_disallowed): for sample, baseline in samples_allowed: yield passes_assert, external_source_impl.assert_cpu_sample_data_type, sample yield converts, external_source_impl.sample_to_numpy, sample, baseline for sample, baseline in samples_allowed + batches_allowed: yield passes_assert, external_source_impl.assert_cpu_batch_data_type, sample yield converts, external_source_impl.batch_to_numpy, sample, baseline for sample in samples_disallowed: yield raises(TypeError, "Unsupported callback return type.")( external_source_impl.assert_cpu_sample_data_type), sample for sample in samples_disallowed + batches_disallowed: yield raises(TypeError, "Unsupported callback return type")( external_source_impl.assert_cpu_batch_data_type), sample
def test_batch_1_wrong_shape(): for shape in [(2, None, None, None), (None, None, 4), (2, None, None, 4), (None, 0, None, 3)]: yield raises( tf.errors.InvalidArgumentError, "The shape provided for output `0` is not compatible with the " "shape returned by DALI Pipeline" )(dali_pipe_batch_1), shape, tf.uint8
def test_parallel_fork(): epoch_size = 250 callback = ExtCallback((4, 5), epoch_size, np.int32) pipes = [( create_pipe( callback, 'cpu', batch_size, py_num_workers=num_workers, py_start_method='fork', parallel=True), create_pipe(callback, 'cpu', batch_size, parallel=False), dtype, batch_size) for dtype in [np.float32, np.int16] for num_workers in [1, 3, 4] for batch_size in [1, 16, 150, 250]] pipes.append(( create_pipe( Iterable(32, (4, 5), dtype=np.int16), 'cpu', 32, py_num_workers=1, py_start_method='fork', parallel=True, batch=True), create_pipe(Iterable(32, (4, 5), dtype=np.int16), 'cpu', 32, parallel=False, batch=True), np.int16, 32)) for parallel_pipe, _, _, _ in pipes: parallel_pipe.start_py_workers() for parallel_pipe, pipe, dtype, batch_size in pipes: yield check_callback, parallel_pipe, pipe, epoch_size, batch_size, dtype # explicitely call py_pool close as nose might still reference parallel_pipe from the yield above parallel_pipe._py_pool.close() # test that another pipline with forking initialization fails as there is CUDA contexts already initialized parallel_pipe = create_pipe(callback, 'cpu', 16, py_num_workers=4, py_start_method='fork', parallel=True) yield raises(RuntimeError, "Cannot fork a process when the CUDA has been initialized in the process.")( build_and_run_pipeline), parallel_pipe, 1
def test_batch_info_flag_default(): batch_size = 5 cb_int = BatchCb(False, batch_size, 1) yield _test_batch_info_flag_default, cb_int, batch_size cb_batch_info = BatchCb(True, batch_size, 1) yield raises(AssertionError, "Expected BatchInfo instance as cb argument")( _test_batch_info_flag_default), cb_batch_info, batch_size
def test_artificial_no_match(): batch = 10 for shape in [(batch + 1, None, None, None), (None, None, 3), (batch, 2, 1, 1)]: yield raises( tf.errors.InvalidArgumentError, "The shape provided for output `0` is not compatible with the shape returned by DALI Pipeline" )(dali_pipe_artificial_shape), shape, tf.uint8, dali_types.UINT8, batch
def test_nan_check(): err_msg = "Argument 'fdata' for operator 'constant' unexpectedly changed value from*" for values in [[np.nan, 1], [1, np.nan]]: yield raises(RuntimeError, glob=err_msg)(_test_nan_check), values for values in [[1, 1], [np.nan, np.nan]]: yield _test_nan_check, values
def test_fail_sequence_rearrange(): shape = [5, 1] orders = [([6, 7], False), ([-1], False), ([], False), ([np.int32([0]), np.int32([])], True), ([np.int32([6, 7]), np.int32([0])], True), ([np.int32([-1]), np.int32([0])], True), ([np.int32([[1], [2]]), np.int32([[1], [2]])], True)] error_msgs = [ 'new_order[[]*[]] must be between * and input_sequence_length = * for sample *, but it is: *', 'new_order[[]*[]] must be between * and input_sequence_length = * for sample *, but it is: *', 'Empty result sequences are not allowed', 'Empty `new_order` for sample * is not allowed', 'new_order[[]*[]] must be between * and input_sequence_length = * for sample *, but it is: *', 'new_order[[]*[]] must be between * and input_sequence_length = * for sample *, but it is: *', 'Input with dimension * cannot be converted to dimension *' ] assert len(orders) == len(error_msgs) for dev in ["cpu", "gpu"]: for [new_order, per_sample], error_msg in zip(orders, error_msgs): yield raises( RuntimeError, glob=error_msg)(check_fail_sequence_rearrange ), 2, shape, new_order, per_sample, dev
def test_non_uniform_batch(): batches_disallowed = [ [test_array, np.array([[42, 42]], dtype=np.uint8)], non_uniform_tl() ] for b in batches_disallowed: yield raises(ValueError, "Uniform input is required (batch of tensors of equal shapes)")( external_source_impl.batch_to_numpy), b
def test_type_returns(): for tf_t, dali_t in matching_types: yield dali_pipe_types, tf_t, dali_t for tf_t, dali_t in not_matching_types: yield raises( tf.errors.InvalidArgumentError, "The type provided for output `0` is not compatible with the type returned by DALI Pipeline" )(dali_pipe_types), tf_t, dali_t
def test_slice_with_out_of_bounds_error(): in_shape = (40, 80, 3) for device in ['gpu', 'cpu']: for batch_size in [1, 3]: yield raises( RuntimeError, "Slice can't be placed out of bounds with current policy." )(check_cmn_with_out_of_bounds_error), device, batch_size, in_shape
def test_cycle_no_resetting(): batch_size = 20 for epoch_size, cb in [ (1, Iterable(batch_size, (4, 5), epoch_size=1)), (4, Iterable(batch_size, (4, 5), epoch_size=4)), (1, generator_epoch_size_1), (4, generator_epoch_size_4),]: for reader_queue_size in (1, 2, 6): yield raises(StopIteration)(_test_cycle_no_resetting), cb, batch_size, epoch_size, reader_queue_size
def test_queue_full_assertion(): for start_method in ("spawn", "fork"): for capacity in [1, 4]: for one_by_one in (True, False): mp = multiprocessing.get_context(start_method) queue = ShmQueue(mp, capacity) msgs = [ ShmMessageDesc(i, i, i, i, i) for i in range(capacity + 1) ] yield raises( RuntimeError, "The queue is full")(_put_msgs), queue, msgs, one_by_one
def test_wrong_layouts_sequence_rearrange(): shape = [5, 1] new_order = [0, 2, 1, 3, 4] per_sample = False for dev in ["cpu", "gpu"]: for layout in ["HF", "HW"]: yield raises( RuntimeError, glob= 'Expected sequence as the input, where outermost dimension represents frames dimension `F`, ' 'got data with layout = "H[WF]"')( check_fail_sequence_rearrange ), 5, shape, new_order, per_sample, dev, layout
def test_wrong_source(): callable_msg = ("Callable passed to External Source in parallel mode (when `parallel=True`) " "must accept exactly one argument*. Got {} instead.") batch_required_msg = "Parallel external source with {} must be run in a batch mode" disallowed_sources = [ (no_arg_fun, (TypeError, callable_msg.format("a callable that does not accept arguments"))), (multi_arg_fun, (TypeError, "External source callback must be a callable with 0 or 1 argument")), (Iterable(), (TypeError, batch_required_msg.format("an iterable"))), (generator_fun, (TypeError, batch_required_msg.format("a generator function"))), (generator_fun(), (TypeError, batch_required_msg.format("an iterable"))), ] for source, (error_type, error_msg) in disallowed_sources: yield raises(error_type, error_msg)(check_source_build), source
def test_squeeze_throw_error(): args_list = [ ([1], None, None, [(300, 1, 200), (10, 10, 10)]), (None, "C", "XYZ", [(2, 3, 4), (4, 2, 3)]), (None, "Z", "XYZ", [(1, 1, 10)]), ([2], "Z", "XYZ", [[1, 1, 10]]), ([2, 1], None, "XYZ", [(100, 0, 0)]), ([1, 1], None, "XYZ", [(300, 1, 200), (10, 1, 10)]), ] expected_errors = [ "Requested a shape with 100 elements but the original shape has 1000 elements.", "Axis 'C' is not present in the input layout", "Requested a shape with 1 elements but the original shape has 10 elements.", "Provided both ``axes`` and ``axis_names`` arguments", "Requested a shape with 100 elements but the original shape has 0 elements.", "Specified at least twice same dimension to remove." ] assert len(expected_errors) == len(args_list) for (axes, axis_names, layout, shapes), error_msg in zip(args_list, expected_errors): yield raises(RuntimeError, error_msg)( _test_squeeze_throw_error), axes, axis_names, layout, shapes
def test_no_output_dtypes(): expected_msg = ("`output_dtypes` should be provided as single tf.DType value or a tuple of " "tf.DType values") yield raises(TypeError, expected_msg)(dali_pipe_deprecated), \ {"shapes": 2}, 2, tf.uint8, dali_types.UINT8, 1, 2
def _test_exception_propagation(callback, batch_size, num_workers, expected): pipe = create_pipe( callback, 'cpu', batch_size, py_num_workers=num_workers, py_start_method='spawn', parallel=True) raises(expected)(build_and_run_pipeline)(pipe, None)
def test_no_output_dtypes(): yield raises(TypeError, "`output_dtypes` should be provided as single tf.DType value or a tuple of tf.DType values")( dali_pipe_deprecated), { "shapes": 2, }, 2, tf.uint8, dali_types.UINT8, 1, 2
def test_multiple_input_invalid(): for batch in [1, 10]: for shapes in [(None,), (batch, 200, 200, 3, None), (None, None, None)]: yield raises(ValueError, "The two structures don't have the same sequence length.")( dali_pipe_multiple_out), shapes, (tf.uint8, tf.uint8), batch
def test_zero_dim_not_allowed(): expected_msg = "Cannot mark zero-dimensional input as a sequence" for device in ["cpu", "gpu"]: yield raises(RuntimeError, expected_msg)(run_pipeline), device, 0