def test_random_sharpness_one_channel_c(degrees=(1.4, 1.4), plot=False):
"""
Test Random Sharpness cpp op with one channel
"""
logger.info(
"Test RandomSharpness C Op With MNIST Dataset (Grayscale images)")
c_op = C.RandomSharpness()
if degrees is not None:
c_op = C.RandomSharpness(degrees)
# RandomSharpness Images
data = de.MnistDataset(dataset_dir=MNIST_DATA_DIR,
num_samples=2,
shuffle=False)
ds_random_sharpness_c = data.map(operations=c_op, input_columns="image")
# Original images
data = de.MnistDataset(dataset_dir=MNIST_DATA_DIR,
num_samples=2,
shuffle=False)
images = []
images_trans = []
labels = []
for _, (data_orig,
data_trans) in enumerate(zip(data, ds_random_sharpness_c)):
image_orig, label_orig = data_orig
image_trans, _ = data_trans
images.append(image_orig.asnumpy())
labels.append(label_orig.asnumpy())
images_trans.append(image_trans.asnumpy())
if plot:
visualize_one_channel_dataset(images, images_trans, labels)
def test_equalize_mnist_c(plot=False):
"""
Test Equalize C op with MNIST dataset (Grayscale images)
"""
logger.info("Test Equalize C Op With MNIST Images")
ds = de.MnistDataset(dataset_dir=MNIST_DATA_DIR,
num_samples=2,
shuffle=False)
ds_equalize_c = ds.map(input_columns="image", operations=C.Equalize())
ds_orig = de.MnistDataset(dataset_dir=MNIST_DATA_DIR,
num_samples=2,
shuffle=False)
images = []
images_trans = []
labels = []
for _, (data_orig, data_trans) in enumerate(zip(ds_orig, ds_equalize_c)):
image_orig, label_orig = data_orig
image_trans, _ = data_trans
images.append(image_orig)
labels.append(label_orig)
images_trans.append(image_trans)
# Compare with expected md5 from images
filename = "equalize_mnist_result_c.npz"
save_and_check_md5(ds_equalize_c,
filename,
generate_golden=GENERATE_GOLDEN)
if plot:
visualize_one_channel_dataset(images, images_trans, labels)
def test_random_solarize_mnist(plot=False, run_golden=True):
"""
Test RandomSolarize op with MNIST dataset (Grayscale images)
"""
mnist_1 = de.MnistDataset(dataset_dir=MNIST_DATA_DIR,
num_samples=2,
shuffle=False)
mnist_2 = de.MnistDataset(dataset_dir=MNIST_DATA_DIR,
num_samples=2,
shuffle=False)
mnist_2 = mnist_2.map(input_columns="image",
operations=vision.RandomSolarize((0, 255)))
images = []
images_trans = []
labels = []
for _, (data_orig, data_trans) in enumerate(zip(mnist_1, mnist_2)):
image_orig, label_orig = data_orig
image_trans, _ = data_trans
images.append(image_orig)
labels.append(label_orig)
images_trans.append(image_trans)
if plot:
visualize_one_channel_dataset(images, images_trans, labels)
if run_golden:
filename = "random_solarize_02_result.npz"
save_and_check_md5(mnist_2, filename, generate_golden=GENERATE_GOLDEN)
def test_random_color_c_errors():
"""
Test that Cpp RandomColor errors with bad input
"""
with pytest.raises(TypeError) as error_info:
vision.RandomColor((12))
assert "degrees must be either a tuple or a list." in str(error_info.value)
with pytest.raises(TypeError) as error_info:
vision.RandomColor(("col", 3))
assert "Argument degrees[0] with value col is not of type (<class 'int'>, <class 'float'>)." in str(
error_info.value)
with pytest.raises(ValueError) as error_info:
vision.RandomColor((0.9, 0.1))
assert "degrees should be in (min,max) format. Got (max,min)." in str(
error_info.value)
with pytest.raises(ValueError) as error_info:
vision.RandomColor((0.9, ))
assert "degrees must be a sequence with length 2." in str(error_info.value)
# RandomColor Cpp Op will fail with one channel input
mnist_ds = de.MnistDataset(dataset_dir=MNIST_DATA_DIR,
num_samples=2,
shuffle=False)
mnist_ds = mnist_ds.map(input_columns="image",
operations=vision.RandomColor())
with pytest.raises(RuntimeError) as error_info:
for _ in enumerate(mnist_ds):
pass
assert "Invalid number of channels in input image" in str(error_info.value)
def create_dataset(data_path, batch_size, repeat_num):
if repeat_num != 1:
raise RuntimeError('repeat_num must be 1')
num_parallel_workers = 1
device_num = 1
shard_id = 0
# TODO: replace with kungfu dataset
ds = de.MnistDataset(
data_path,
# num_parallel_workers=num_parallel_workers,
shuffle=False,
num_shards=device_num,
shard_id=shard_id,
)
# define map operations
label_trans = [
C2.TypeCast(mstype.int32),
]
imgage_trans = [
# ValueError: For 'MatMul' evaluator shapes of inputs can not do this operator, got 256 and 400,
# C.Resize((28, 28)),
#
CV.Resize((32, 32), interpolation=Inter.LINEAR), #
CV.Rescale(1.0 / 255.0, shift=0.0),
CV.Rescale(1 / 0.3081,
shift=-1 * 0.1307 / 0.3081), # NOT converge if removed
CV.HWC2CHW(),
]
ds = ds.map(operations=label_trans,
input_columns="label",
num_parallel_workers=num_parallel_workers)
ds = ds.map(operations=imgage_trans,
input_columns="image",
num_parallel_workers=num_parallel_workers)
# apply batch operations
ds = ds.batch(batch_size, drop_remainder=True)
# apply dataset repeat operation
ds = ds.repeat(repeat_num) # repeat_num is deprecated
return ds
def create_dataset(data_path, is_train=True, batch_size=32):
# import
import mindspore.common.dtype as mstype
import mindspore.dataset.engine as de
import mindspore.dataset.transforms.c_transforms as C2
import mindspore.dataset.vision.c_transforms as C
from mindspore.common import set_seed
from mindspore.dataset.vision import Inter
set_seed(1)
# shard
num_shards = shard_id = None
rand_size = os.getenv("RANK_SIZE")
rand_id = os.getenv("RANK_ID")
if rand_size is not None and rand_id is not None:
num_shards = int(rand_size)
shard_id = int(rand_id)
# define dataset
data_path = os.path.join(data_path, "train" if is_train else "test")
ds = de.MnistDataset(data_path, num_shards=num_shards, shard_id=shard_id)
# define ops
comps_ops = list()
comps_ops.append(C.Resize((32, 32), interpolation=Inter.LINEAR))
comps_ops.append(C.Rescale(1 / 0.3081, -1 * 0.1307 / 0.3081))
comps_ops.append(C.Rescale(1 / 255., 0.))
comps_ops.append(C.HWC2CHW())
# map ops
ds = ds.map(input_columns=["image"],
operations=comps_ops,
num_parallel_workers=8)
ds = ds.map(input_columns=["label"],
operations=C2.TypeCast(mstype.int32),
num_parallel_workers=8)
# batch & repeat
ds = ds.shuffle(buffer_size=1000)
ds = ds.batch(batch_size=batch_size, drop_remainder=is_train)
ds = ds.repeat(count=1)
return ds
