def test_random_choice(): """ Test RandomChoice op """ ds.config.set_seed(0) def test_config(arr, op_list): try: data = ds.NumpySlicesDataset(arr, column_names="col", shuffle=False) data = data.map(operations=ops.RandomChoice(op_list), input_columns=["col"]) res = [] for i in data.create_dict_iterator(num_epochs=1, output_numpy=True): res.append(i["col"].tolist()) return res except (TypeError, ValueError) as e: return str(e) # Test whether an operation would be randomly chosen. # In order to prevent random failure, both results need to be checked. res1 = test_config([[0, 1, 2]], [ops.PadEnd([4], 0), ops.Slice([0, 2])]) assert res1 in [[[0, 1, 2, 0]], [[0, 2]]] # Test nested structure res2 = test_config([[0, 1, 2]], [ ops.Compose([ops.Duplicate(), ops.Concatenate()]), ops.Compose([ops.Slice([0, 1]), ops.OneHot(2)]) ]) assert res2 in [[[[1, 0], [0, 1]]], [[0, 1, 2, 0, 1, 2]]] # Test RandomChoice where there is only 1 operation assert test_config([[4, 3], [2, 1]], [ops.Slice([0])]) == [[4], [2]]
def test_compose(): """ Test C++ and Python Compose Op """ ds.config.set_seed(0) def test_config(arr, op_list): try: data = ds.NumpySlicesDataset(arr, column_names="col", shuffle=False) data = data.map(input_columns=["col"], operations=op_list) res = [] for i in data.create_dict_iterator(output_numpy=True): res.append(i["col"].tolist()) return res except (TypeError, ValueError) as e: return str(e) # Test simple compose with only 1 op, this would generate a warning assert test_config([[1, 0], [3, 4]], ops.Compose([ops.Fill(2)])) == [[2, 2], [2, 2]] # Test 1 column -> 2 columns -> 1 -> 2 -> 1 assert test_config([[1, 0]], ops.Compose([ops.Duplicate(), ops.Concatenate(), ops.Duplicate(), ops.Concatenate()])) \ == [[1, 0] * 4] # Test one Python transform followed by a C transform. Type after OneHot is a float (mixed use-case) assert test_config( [1, 0], ops.Compose([py_ops.OneHotOp(2), ops.TypeCast(mstype.int32)])) == [[[0, 1]], [[1, 0]]] # Test exceptions. with pytest.raises(TypeError) as error_info: ops.Compose([1, ops.TypeCast(mstype.int32)]) assert "op_list[0] is not a c_transform op (TensorOp) nor a callable pyfunc." in str( error_info.value) # Test empty op list with pytest.raises(ValueError) as error_info: test_config([1, 0], ops.Compose([])) assert "op_list can not be empty." in str(error_info.value) # Test Python compose op assert test_config([1, 0], py_ops.Compose([py_ops.OneHotOp(2)])) == [[[0, 1]], [[1, 0]]] assert test_config([1, 0], py_ops.Compose([py_ops.OneHotOp(2), (lambda x: x + x)])) == [[[0, 2]], [[2, 0]]] # Test nested Python compose op assert test_config([1, 0], py_ops.Compose([py_ops.Compose([py_ops.OneHotOp(2)]), (lambda x: x + x)])) \ == [[[0, 2]], [[2, 0]]] with pytest.raises(TypeError) as error_info: py_ops.Compose([(lambda x: x + x)])() assert "Compose was called without an image. Fix invocation (avoid it being invoked as Compose([...])())." in str( error_info.value)
def test_random_apply(): ds.config.set_seed(0) def test_config(arr, op_list, prob=0.5): try: data = ds.NumpySlicesDataset(arr, column_names="col", shuffle=False) data = data.map(input_columns=["col"], operations=ops.RandomApply(op_list, prob)) res = [] for i in data.create_dict_iterator(): res.append(i["col"].tolist()) return res except (TypeError, ValueError) as e: return str(e) res1 = test_config([[0, 1]], [ops.Duplicate(), ops.Concatenate()]) assert res1 in [[[0, 1]], [[0, 1, 0, 1]]] # test single nested compose assert test_config([[0, 1, 2]], [ ops.Compose([ops.Duplicate(), ops.Concatenate(), ops.Slice([0, 1, 2])]) ]) == [[0, 1, 2]] # test exception assert "is not of type (<class 'list'>" in test_config([1, 0], ops.TypeCast( mstype.int32)) assert "Input prob is not within the required interval" in test_config( [0, 1], [ops.Slice([0, 1])], 1.1) assert "is not of type (<class 'float'>" in test_config( [1, 0], [ops.TypeCast(mstype.int32)], None) assert "op_list with value None is not of type (<class 'list'>" in test_config( [1, 0], None)
def test_random_select_subpolicy(): ds.config.set_seed(0) def test_config(arr, policy): try: data = ds.NumpySlicesDataset(arr, column_names="col", shuffle=False) data = data.map(operations=visions.RandomSelectSubpolicy(policy), input_columns=["col"]) res = [] for i in data.create_dict_iterator(num_epochs=1, output_numpy=True): res.append(i["col"].tolist()) return res except (TypeError, ValueError) as e: return str(e) # 3 possible outcomes policy1 = [[(ops.PadEnd([4], 0), 0.5), (ops.Compose([ops.Duplicate(), ops.Concatenate()]), 1)], [(ops.Slice([0, 1]), 0.5), (ops.Duplicate(), 1), (ops.Concatenate(), 1)]] res1 = test_config([[1, 2, 3]], policy1) assert res1 in [[[1, 2, 1, 2]], [[1, 2, 3, 1, 2, 3]], [[1, 2, 3, 0, 1, 2, 3, 0]]] # test exceptions assert "policy can not be empty." in test_config([[1, 2, 3]], []) assert "policy[0] can not be empty." in test_config([[1, 2, 3]], [[]]) assert "op of (op, prob) in policy[1][0] is neither a c_transform op (TensorOperation) nor a callable pyfunc" \ in test_config([[1, 2, 3]], [[(ops.PadEnd([4], 0), 0.5)], [(1, 0.4)]]) assert "prob of (op, prob) policy[1][0] is not within the required interval of [0, 1]" in test_config( [[1]], [[(ops.Duplicate(), 0)], [(ops.Duplicate(), -0.1)]])
def test_config(arr, op_list): try: data = ds.NumpySlicesDataset(arr, column_names="col", shuffle=False) data = data.map(input_columns=["col"], operations=ops.Compose(op_list)) res = [] for i in data.create_dict_iterator(): res.append(i["col"].tolist()) return res except (TypeError, ValueError) as e: return str(e)