def util_test_normalize(mean, std, op_type):
"""
Utility function for testing Normalize. Input arguments are given by other tests
"""
if op_type == "cpp":
# define map operations
decode_op = c_vision.Decode()
normalize_op = c_vision.Normalize(mean, std)
# Generate dataset
data = ds.TFRecordDataset(DATA_DIR, SCHEMA_DIR, columns_list=["image"], shuffle=False)
data = data.map(input_columns=["image"], operations=decode_op)
data = data.map(input_columns=["image"], operations=normalize_op)
elif op_type == "python":
# define map operations
transforms = [
py_vision.Decode(),
py_vision.ToTensor(),
py_vision.Normalize(mean, std)
]
transform = py_vision.ComposeOp(transforms)
# Generate dataset
data = ds.TFRecordDataset(DATA_DIR, SCHEMA_DIR, columns_list=["image"], shuffle=False)
data = data.map(input_columns=["image"], operations=transform())
else:
raise ValueError("Wrong parameter value")
return data
def test_normalize_op_py(plot=False):
"""
Test Normalize in python transformations
"""
logger.info("Test Normalize in python")
mean = [0.475, 0.45, 0.392]
std = [0.275, 0.267, 0.278]
# define map operations
transforms = [
py_vision.Decode(),
py_vision.ToTensor()
]
transform = py_vision.ComposeOp(transforms)
normalize_op = py_vision.Normalize(mean, std)
# First dataset
data1 = ds.TFRecordDataset(DATA_DIR, SCHEMA_DIR, columns_list=["image"], shuffle=False)
data1 = data1.map(input_columns=["image"], operations=transform())
data1 = data1.map(input_columns=["image"], operations=normalize_op)
# Second dataset
data2 = ds.TFRecordDataset(DATA_DIR, SCHEMA_DIR, columns_list=["image"], shuffle=False)
data2 = data2.map(input_columns=["image"], operations=transform())
num_iter = 0
for item1, item2 in zip(data1.create_dict_iterator(), data2.create_dict_iterator()):
image_de_normalized = (item1["image"].transpose(1, 2, 0) * 255).astype(np.uint8)
image_np_normalized = (normalize_np(item2["image"].transpose(1, 2, 0), mean, std) * 255).astype(np.uint8)
image_original = (item2["image"].transpose(1, 2, 0) * 255).astype(np.uint8)
mse = diff_mse(image_de_normalized, image_np_normalized)
logger.info("image_{}, mse: {}".format(num_iter + 1, mse))
assert mse < 0.01
if plot:
visualize_mse(image_de_normalized, image_np_normalized, mse, image_original)
num_iter += 1
def create_dataset_py(dataset_path, do_train, config, device_target, repeat_num=1, batch_size=32):
"""
create a train or eval dataset
Args:
dataset_path(string): the path of dataset.
do_train(bool): whether dataset is used for train or eval.
repeat_num(int): the repeat times of dataset. Default: 1.
batch_size(int): the batch size of dataset. Default: 32.
Returns:
dataset
"""
if device_target == "Ascend":
rank_size = int(os.getenv("RANK_SIZE"))
rank_id = int(os.getenv("RANK_ID"))
if do_train:
if rank_size == 1:
ds = de.ImageFolderDatasetV2(dataset_path, num_parallel_workers=8, shuffle=True)
else:
ds = de.ImageFolderDatasetV2(dataset_path, num_parallel_workers=8, shuffle=True,
num_shards=rank_size, shard_id=rank_id)
else:
ds = de.ImageFolderDatasetV2(dataset_path, num_parallel_workers=8, shuffle=False)
else:
raise ValueError("Unsupported device target.")
resize_height = config.image_height
if do_train:
buffer_size = 20480
# apply shuffle operations
ds = ds.shuffle(buffer_size=buffer_size)
# define map operations
decode_op = P.Decode()
resize_crop_op = P.RandomResizedCrop(resize_height, scale=(0.08, 1.0), ratio=(0.75, 1.333))
horizontal_flip_op = P.RandomHorizontalFlip(prob=0.5)
resize_op = P.Resize(256)
center_crop = P.CenterCrop(resize_height)
to_tensor = P.ToTensor()
normalize_op = P.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225])
if do_train:
trans = [decode_op, resize_crop_op, horizontal_flip_op, to_tensor, normalize_op]
else:
trans = [decode_op, resize_op, center_crop, to_tensor, normalize_op]
compose = P.ComposeOp(trans)
ds = ds.map(input_columns="image", operations=compose(), num_parallel_workers=8, python_multiprocessing=True)
# apply batch operations
ds = ds.batch(batch_size, drop_remainder=True)
# apply dataset repeat operation
ds = ds.repeat(repeat_num)
return ds
def test_normalize_exception_invalid_range_py():
"""
Test Normalize in python transformation: value is not in range [0,1]
expected to raise ValueError
"""
logger.info("test_normalize_exception_invalid_range_py")
try:
_ = py_vision.Normalize([0.75, 1.25, 0.5], [0.1, 0.18, 1.32])
except ValueError as e:
logger.info("Got an exception in DE: {}".format(str(e)))
assert "Input is not within the required range" in str(e)
def test_normalize_exception_unequal_size_py():
"""
Test Normalize in python transformation: len(mean) != len(std)
expected to raise ValueError
"""
logger.info("test_normalize_exception_unequal_size_py")
try:
_ = py_vision.Normalize([0.50, 0.30, 0.75], [0.18, 0.32, 0.71, 0.72])
except ValueError as e:
logger.info("Got an exception in DE: {}".format(str(e)))
assert str(e) == "Length of mean and std must be equal"
def create_imagenet_dataset(imagenet_dir):
ds = de.ImageFolderDatasetV2(imagenet_dir)
transform = F.ComposeOp([
F.Decode(),
F.RandomHorizontalFlip(0.5),
F.ToTensor(),
F.Normalize((0.491, 0.482, 0.447), (0.247, 0.243, 0.262)),
F.RandomErasing()
])
ds = ds.map(input_columns="image", operations=transform())
ds = ds.shuffle(buffer_size=5)
ds = ds.repeat(3)
return ds
def util_test_normalize_grayscale(num_output_channels, mean, std):
"""
Utility function for testing Normalize. Input arguments are given by other tests
"""
transforms = [
py_vision.Decode(),
py_vision.Grayscale(num_output_channels),
py_vision.ToTensor(),
py_vision.Normalize(mean, std)
]
transform = py_vision.ComposeOp(transforms)
# Generate dataset
data = ds.TFRecordDataset(DATA_DIR, SCHEMA_DIR, columns_list=["image"], shuffle=False)
data = data.map(input_columns=["image"], operations=transform())
return data
def create_icdar_train_dataset(img_path, gt_path, batch_size=32, repeat_num=10,
is_training=True, num_parallel_workers=1, length=512, scale=0.25):
dataloader = ds.GeneratorDataset(source=custom_dataset(img_path, gt_path, scale=scale, length=length), column_names=["img", "score_map", "geo_map", "ignored_map"], shuffle=is_training, num_parallel_workers=num_parallel_workers)
dataloader.set_dataset_size(1000)
transform = py_transforms.ComposeOp([py_transforms.RandomColorAdjust(brightness=0.5, contrast=0.5, saturation=0.5, hue=0.25), \
py_transforms.ToTensor(), \
py_transforms.Normalize(mean=(0.5,0.5,0.5),std=(0.5,0.5,0.5))])
dataloader = dataloader.map(input_columns="img", operations=transform, num_parallel_workers=num_parallel_workers, python_multiprocessing=is_training)
dataloader = dataloader.batch(batch_size, drop_remainder=True)
return dataloader
def create_dataset(dataset_path,
do_train,
config,
platform,
repeat_num=1,
batch_size=100):
"""
create a train or eval dataset
Args:
dataset_path(string): the path of dataset.
do_train(bool): whether dataset is used for train or eval.
repeat_num(int): the repeat times of dataset. Default: 1
batch_size(int): the batch size of dataset. Default: 32
Returns:
dataset
"""
if platform == "Ascend":
rank_size = int(os.getenv("RANK_SIZE"))
rank_id = int(os.getenv("RANK_ID"))
if rank_size == 1:
ds = de.MindDataset(dataset_path,
num_parallel_workers=8,
shuffle=True)
else:
ds = de.MindDataset(dataset_path,
num_parallel_workers=8,
shuffle=True,
num_shards=rank_size,
shard_id=rank_id)
elif platform == "GPU":
if do_train:
from mindspore.communication.management import get_rank, get_group_size
ds = de.MindDataset(dataset_path,
num_parallel_workers=8,
shuffle=True,
num_shards=get_group_size(),
shard_id=get_rank())
else:
ds = de.MindDataset(dataset_path,
num_parallel_workers=8,
shuffle=False)
else:
raise ValueError("Unsupport platform.")
resize_height = config.image_height
buffer_size = 1000
# define map operations
resize_crop_op = C.RandomCropDecodeResize(resize_height,
scale=(0.08, 1.0),
ratio=(0.75, 1.333))
horizontal_flip_op = C.RandomHorizontalFlip(prob=0.5)
color_op = C.RandomColorAdjust(brightness=0.4,
contrast=0.4,
saturation=0.4)
rescale_op = C.Rescale(1 / 255.0, 0)
normalize_op = C.Normalize(mean=[0.485, 0.456, 0.406],
std=[0.229, 0.224, 0.225])
change_swap_op = C.HWC2CHW()
# define python operations
decode_p = P.Decode()
resize_p = P.Resize(256, interpolation=Inter.BILINEAR)
center_crop_p = P.CenterCrop(224)
totensor = P.ToTensor()
normalize_p = P.Normalize((0.485, 0.456, 0.406), (0.229, 0.224, 0.225))
composeop = P.ComposeOp(
[decode_p, resize_p, center_crop_p, totensor, normalize_p])
if do_train:
trans = [
resize_crop_op, horizontal_flip_op, color_op, rescale_op,
normalize_op, change_swap_op
]
else:
trans = composeop()
type_cast_op = C2.TypeCast(mstype.int32)
ds = ds.map(input_columns="image",
operations=trans,
num_parallel_workers=8)
ds = ds.map(input_columns="label_list",
operations=type_cast_op,
num_parallel_workers=8)
# apply shuffle operations
ds = ds.shuffle(buffer_size=buffer_size)
# apply batch operations
ds = ds.batch(batch_size, drop_remainder=True)
# apply dataset repeat operation
ds = ds.repeat(repeat_num)
return ds
def create_dataset_py(dataset_path,
do_train,
repeat_num=1,
batch_size=32,
target="Ascend"):
"""
create a train or eval dataset
Args:
dataset_path(string): the path of dataset.
do_train(bool): whether dataset is used for train or eval.
repeat_num(int): the repeat times of dataset. Default: 1
batch_size(int): the batch size of dataset. Default: 32
target(str): the device target. Default: Ascend
Returns:
dataset
"""
if target == "Ascend":
device_num = int(os.getenv("RANK_SIZE"))
rank_id = int(os.getenv("RANK_ID"))
else:
init("nccl")
rank_id = get_rank()
device_num = get_group_size()
if do_train:
if device_num == 1:
ds = de.ImageFolderDatasetV2(dataset_path,
num_parallel_workers=8,
shuffle=True)
else:
ds = de.ImageFolderDatasetV2(dataset_path,
num_parallel_workers=8,
shuffle=True,
num_shards=device_num,
shard_id=rank_id)
else:
ds = de.ImageFolderDatasetV2(dataset_path,
num_parallel_workers=8,
shuffle=False)
image_size = 224
# define map operations
decode_op = P.Decode()
resize_crop_op = P.RandomResizedCrop(image_size,
scale=(0.08, 1.0),
ratio=(0.75, 1.333))
horizontal_flip_op = P.RandomHorizontalFlip(prob=0.5)
resize_op = P.Resize(256)
center_crop = P.CenterCrop(image_size)
to_tensor = P.ToTensor()
normalize_op = P.Normalize(mean=[0.485, 0.456, 0.406],
std=[0.229, 0.224, 0.225])
# define map operations
if do_train:
trans = [
decode_op, resize_crop_op, horizontal_flip_op, to_tensor,
normalize_op
]
else:
trans = [decode_op, resize_op, center_crop, to_tensor, normalize_op]
compose = P.ComposeOp(trans)
ds = ds.map(input_columns="image",
operations=compose(),
num_parallel_workers=8,
python_multiprocessing=True)
# apply batch operations
ds = ds.batch(batch_size, drop_remainder=True)
# apply dataset repeat operation
ds = ds.repeat(repeat_num)
return ds