def test_c_py_compose_transforms_module(): """ Test combining Python and C++ transforms """ ds.config.set_seed(0) def test_config(arr, input_columns, output_cols, op_list): data = ds.NumpySlicesDataset(arr, column_names=input_columns, shuffle=False) data = data.map(operations=op_list, input_columns=input_columns, output_columns=output_cols, column_order=output_cols) res = [] for i in data.create_dict_iterator(output_numpy=True): for col_name in output_cols: res.append(i[col_name].tolist()) return res arr = [1, 0] assert test_config(arr, ["cols"], ["cols"], [py_transforms.OneHotOp(2), c_transforms.Mask(c_transforms.Relational.EQ, 1)]) == \ [[[False, True]], [[True, False]]] assert test_config(arr, ["cols"], ["cols"], [py_transforms.OneHotOp(2), (lambda x: x + x), c_transforms.Fill(1)]) \ == [[[1, 1]], [[1, 1]]] assert test_config(arr, ["cols"], ["cols"], [py_transforms.OneHotOp(2), (lambda x: x + x), c_transforms.Fill(1), (lambda x: x + x)]) \ == [[[2, 2]], [[2, 2]]] assert test_config([[1, 3]], ["cols"], ["cols"], [c_transforms.PadEnd([3], -1), (lambda x: x + x)]) \ == [[2, 6, -2]] arr = ([[1]], [[3]]) assert test_config(arr, ["col0", "col1"], ["a"], [(lambda x, y: x + y), c_transforms.PadEnd([2], -1)]) == [[4, -1]]
def test_compose(): """ Test C++ and Python Compose Op """ ds.config.set_seed(0) def test_config(arr, op_list): try: data = ds.NumpySlicesDataset(arr, column_names="col", shuffle=False) data = data.map(input_columns=["col"], operations=op_list) res = [] for i in data.create_dict_iterator(output_numpy=True): res.append(i["col"].tolist()) return res except (TypeError, ValueError) as e: return str(e) # Test simple compose with only 1 op, this would generate a warning assert test_config([[1, 0], [3, 4]], ops.Compose([ops.Fill(2)])) == [[2, 2], [2, 2]] # Test 1 column -> 2 columns -> 1 -> 2 -> 1 assert test_config([[1, 0]], ops.Compose([ops.Duplicate(), ops.Concatenate(), ops.Duplicate(), ops.Concatenate()])) \ == [[1, 0] * 4] # Test one Python transform followed by a C transform. Type after OneHot is a float (mixed use-case) assert test_config( [1, 0], ops.Compose([py_ops.OneHotOp(2), ops.TypeCast(mstype.int32)])) == [[[0, 1]], [[1, 0]]] # Test exceptions. with pytest.raises(TypeError) as error_info: ops.Compose([1, ops.TypeCast(mstype.int32)]) assert "op_list[0] is not a c_transform op (TensorOp) nor a callable pyfunc." in str( error_info.value) # Test empty op list with pytest.raises(ValueError) as error_info: test_config([1, 0], ops.Compose([])) assert "op_list can not be empty." in str(error_info.value) # Test Python compose op assert test_config([1, 0], py_ops.Compose([py_ops.OneHotOp(2)])) == [[[0, 1]], [[1, 0]]] assert test_config([1, 0], py_ops.Compose([py_ops.OneHotOp(2), (lambda x: x + x)])) == [[[0, 2]], [[2, 0]]] # Test nested Python compose op assert test_config([1, 0], py_ops.Compose([py_ops.Compose([py_ops.OneHotOp(2)]), (lambda x: x + x)])) \ == [[[0, 2]], [[2, 0]]] with pytest.raises(TypeError) as error_info: py_ops.Compose([(lambda x: x + x)])() assert "Compose was called without an image. Fix invocation (avoid it being invoked as Compose([...])())." in str( error_info.value)
def test_py_transforms_with_c_vision(): """ These examples will fail, as py_transforms.Random(Apply/Choice/Order) expect callable functions """ ds.config.set_seed(0) def test_config(op_list): data_dir = "../data/dataset/testImageNetData/train/" data = ds.ImageFolderDataset(dataset_dir=data_dir, shuffle=False) data = data.map(operations=op_list) res = [] for i in data.create_dict_iterator(output_numpy=True): for col_name in output_cols: res.append(i[col_name].tolist()) return res with pytest.raises(ValueError) as error_info: test_config(py_transforms.RandomApply([c_vision.Resize(200)])) assert "transforms[0] is not callable." in str(error_info.value) with pytest.raises(ValueError) as error_info: test_config(py_transforms.RandomChoice([c_vision.Resize(200)])) assert "transforms[0] is not callable." in str(error_info.value) with pytest.raises(ValueError) as error_info: test_config(py_transforms.RandomOrder([np.array, c_vision.Resize(200)])) assert "transforms[1] is not callable." in str(error_info.value) with pytest.raises(RuntimeError) as error_info: test_config([py_transforms.OneHotOp(20, 0.1)]) assert "The truth value of an array with more than one element is ambiguous. Use a.any() or a.all()" in str( error_info.value)
def test_one_hot_op(): """ Test one hot encoding op """ logger.info("Test one hot encoding op") # define map operations # ds = de.ImageFolderDataset(DATA_DIR, schema=SCHEMA_DIR) dataset = ds.ImageFolderDatasetV2(DATA_DIR) num_classes = 2 epsilon_para = 0.1 transforms = [ f.OneHotOp(num_classes=num_classes, smoothing_rate=epsilon_para), ] transform_label = py_vision.ComposeOp(transforms) dataset = dataset.map(input_columns=["label"], operations=transform_label()) golden_label = np.ones(num_classes) * epsilon_para / num_classes golden_label[1] = 1 - epsilon_para / num_classes for data in dataset.create_dict_iterator(): label = data["label"] logger.info("label is {}".format(label)) logger.info("golden_label is {}".format(golden_label)) assert (label.all() == golden_label.all()) logger.info("====test one hot op ok====")
def test_compose(): ds.config.set_seed(0) def test_config(arr, op_list): try: data = ds.NumpySlicesDataset(arr, column_names="col", shuffle=False) data = data.map(operations=ops.Compose(op_list), input_columns=["col"]) res = [] for i in data.create_dict_iterator(num_epochs=1, output_numpy=True): res.append(i["col"].tolist()) return res except (TypeError, ValueError) as e: return str(e) # test simple compose with only 1 op, this would generate a warning assert test_config([[1, 0], [3, 4]], [ops.Fill(2)]) == [[2, 2], [2, 2]] # test 1 column -> 2columns -> 1 -> 2 -> 1 assert test_config([[1, 0]], [ ops.Duplicate(), ops.Concatenate(), ops.Duplicate(), ops.Concatenate() ]) == [[1, 0] * 4] # test one python transform followed by a C transform. type after oneHot is float (mixed use-case) assert test_config( [1, 0], [py_ops.OneHotOp(2), ops.TypeCast(mstype.int32)]) == [[[0, 1]], [[1, 0]]] # test exceptions. compose, randomApply randomChoice use the same validator assert "op_list[0] is not a c_transform op" in test_config( [1, 0], [1, ops.TypeCast(mstype.int32)]) # test empty op list assert "op_list can not be empty." in test_config([1, 0], [])