def create_dataset(dataset_path, do_train, repeat_num=1, batch_size=32, target="Ascend"):
"""
create a train or eval imagenet2012 dataset for resnet50
Args:
dataset_path(string): the path of dataset.
do_train(bool): whether dataset is used for train or eval.
repeat_num(int): the repeat times of dataset. Default: 1
batch_size(int): the batch size of dataset. Default: 32
target(str): the device target. Default: Ascend
Returns:
dataset
"""
if target == "Ascend":
device_num, rank_id = _get_rank_info()
else:
init("nccl")
rank_id = get_rank()
device_num = get_group_size()
if device_num == 1:
ds = de.ImageFolderDatasetV2(dataset_path, num_parallel_workers=8, shuffle=True)
else:
ds = de.ImageFolderDatasetV2(dataset_path, num_parallel_workers=8, shuffle=True,
num_shards=device_num, shard_id=rank_id)
image_size = 224
mean = [0.485 * 255, 0.456 * 255, 0.406 * 255]
std = [0.229 * 255, 0.224 * 255, 0.225 * 255]
# define map operations
if do_train:
trans = [
C.RandomCropDecodeResize(image_size, scale=(0.08, 1.0), ratio=(0.75, 1.333)),
C.RandomHorizontalFlip(prob=0.5),
C.Normalize(mean=mean, std=std),
C.HWC2CHW()
]
else:
trans = [
C.Decode(),
C.Resize(256),
C.CenterCrop(image_size),
C.Normalize(mean=mean, std=std),
C.HWC2CHW()
]
type_cast_op = C2.TypeCast(mstype.int32)
ds = ds.map(input_columns="image", num_parallel_workers=8, operations=trans)
ds = ds.map(input_columns="label", num_parallel_workers=8, operations=type_cast_op)
# apply batch operations
ds = ds.batch(batch_size, drop_remainder=True)
# apply dataset repeat operation
ds = ds.repeat(repeat_num)
return ds
def create_dataset(dataset_path, do_train, repeat_num=1, batch_size=32):
"""
create a train or eval dataset
Args:
dataset_path(string): the path of dataset.
do_train(bool): whether dataset is used for train or eval.
repeat_num(int): the repeat times of dataset. Default: 1
batch_size(int): the batch size of dataset. Default: 32
Returns:
dataset
"""
device_num = int(os.getenv("RANK_SIZE"))
rank_id = int(os.getenv("RANK_ID"))
if device_num == 1:
ds = de.ImageFolderDatasetV2(dataset_path, num_parallel_workers=8, shuffle=False)
else:
ds = de.ImageFolderDatasetV2(dataset_path, num_parallel_workers=8, shuffle=True,
num_shards=device_num, shard_id=rank_id)
image_size = 224
mean = [0.485 * 255, 0.456 * 255, 0.406 * 255]
std = [0.229 * 255, 0.224 * 255, 0.225 * 255]
if do_train:
transform_img = [
V_C.RandomCropDecodeResize(image_size, scale=(0.08, 1.0), ratio=(0.75, 1.333)),
V_C.RandomHorizontalFlip(prob=0.5),
V_C.Normalize(mean=mean, std=std),
V_C.HWC2CHW()
]
else:
transform_img = [
V_C.Decode(),
V_C.Resize((256, 256)),
V_C.CenterCrop(image_size),
V_C.Normalize(mean=mean, std=std),
V_C.HWC2CHW()
]
# type_cast_op = C2.TypeCast(mstype.float16)
type_cast_op = C2.TypeCast(mstype.int32)
ds = ds.map(input_columns="image", operations=transform_img, num_parallel_workers=8)
ds = ds.map(input_columns="label", operations=type_cast_op, num_parallel_workers=8)
# apply shuffle operations
# ds = ds.shuffle(buffer_size=config.buffer_size)
# apply batch operations
ds = ds.batch(batch_size, drop_remainder=True)
# apply dataset repeat operation
ds = ds.repeat(repeat_num)
return ds
def create_icdar_train_dataset(mindrecord_file='icdar_train.mindrecord', batch_size=32, repeat_num=10,
is_training=True, num_parallel_workers=1, length=512, scale=0.25):
dataset = ds.MindDataset(mindrecord_file, columns_list=['image', 'annotation', 'label'],
num_parallel_workers=num_parallel_workers, shuffle=False)
dataset.set_dataset_size(1000)
decode = C.Decode()
dataset = dataset.map(input_columns=["image"], operations=decode, python_multiprocessing=is_training, num_parallel_workers=num_parallel_workers)
change_swap_op = C.HWC2CHW()
normalize_op = C.Normalize(mean=[0.485*255, 0.456*255, 0.406*255], std=[0.229*255, 0.224*255, 0.225*255])
color_adjust_op = C.RandomColorAdjust(brightness=0.5, contrast=0.5, saturation=0.5, hue=0.25)
compose_map_func = (lambda image, annotation, label: preprocess(image, annotation, label, length, scale))
output_columns = ["image", "score_map", "geo_map", "ignored_map"]
dataset = dataset.map(input_columns=["image", "annotation", "label"],
output_columns=output_columns, columns_order=output_columns,
operations=compose_map_func, python_multiprocessing=is_training,
num_parallel_workers=num_parallel_workers)
trans = [normalize_op, change_swap_op]
dataset = dataset.map(input_columns=["image"], operations=trans, python_multiprocessing=is_training,
num_parallel_workers=num_parallel_workers)
dataset = dataset.batch(batch_size, drop_remainder=True)
dataset = dataset.repeat(repeat_num)
return dataset
def test_normalize_op_c(plot=False):
"""
Test Normalize in cpp transformations
"""
logger.info("Test Normalize in cpp")
mean = [121.0, 115.0, 100.0]
std = [70.0, 68.0, 71.0]
# define map operations
decode_op = c_vision.Decode()
normalize_op = c_vision.Normalize(mean, std)
# First dataset
data1 = ds.TFRecordDataset(DATA_DIR, SCHEMA_DIR, columns_list=["image"], shuffle=False)
data1 = data1.map(input_columns=["image"], operations=decode_op)
data1 = data1.map(input_columns=["image"], operations=normalize_op)
# Second dataset
data2 = ds.TFRecordDataset(DATA_DIR, SCHEMA_DIR, columns_list=["image"], shuffle=False)
data2 = data2.map(input_columns=["image"], operations=decode_op)
num_iter = 0
for item1, item2 in zip(data1.create_dict_iterator(), data2.create_dict_iterator()):
image_de_normalized = item1["image"]
image_original = item2["image"]
image_np_normalized = normalize_np(image_original, mean, std)
mse = diff_mse(image_de_normalized, image_np_normalized)
logger.info("image_{}, mse: {}".format(num_iter + 1, mse))
assert mse < 0.01
if plot:
visualize_mse(image_de_normalized, image_np_normalized, mse, image_original)
num_iter += 1
def test_decode_normalize_op():
logger.info("Test [Decode, Normalize] in one Map")
data1 = ds.TFRecordDataset(DATA_DIR,
SCHEMA_DIR,
columns_list=["image", "label"],
num_parallel_workers=1,
shuffle=False)
# define map operations
decode_op = vision.Decode()
normalize_op = vision.Normalize([121.0, 115.0, 100.0], [70.0, 68.0, 71.0])
# apply map operations on images
data1 = data1.map(input_columns=["image"],
operations=[decode_op, normalize_op])
num_iter = 0
image = None
for item in data1.create_dict_iterator():
logger.info("Looping inside iterator {}".format(num_iter))
image = item["image"]
# plt.subplot(131)
# plt.imshow(image)
# plt.title("DE image")
# plt.show()
num_iter += 1
def create_dataset(data_path, batch_size=32, repeat_size=1, mode="train"):
"""
create dataset for train or test
"""
cifar_ds = ds.Cifar10Dataset(data_path)
rescale = 1.0 / 255.0
shift = 0.0
resize_op = CV.Resize((cfg.image_height, cfg.image_width))
rescale_op = CV.Rescale(rescale, shift)
normalize_op = CV.Normalize((0.4914, 0.4822, 0.4465), (0.2023, 0.1994, 0.2010))
if mode == "train":
random_crop_op = CV.RandomCrop([32, 32], [4, 4, 4, 4])
random_horizontal_op = CV.RandomHorizontalFlip()
channel_swap_op = CV.HWC2CHW()
typecast_op = C.TypeCast(mstype.int32)
cifar_ds = cifar_ds.map(input_columns="label", operations=typecast_op)
if mode == "train":
cifar_ds = cifar_ds.map(input_columns="image", operations=random_crop_op)
cifar_ds = cifar_ds.map(input_columns="image", operations=random_horizontal_op)
cifar_ds = cifar_ds.map(input_columns="image", operations=resize_op)
cifar_ds = cifar_ds.map(input_columns="image", operations=rescale_op)
cifar_ds = cifar_ds.map(input_columns="image", operations=normalize_op)
cifar_ds = cifar_ds.map(input_columns="image", operations=channel_swap_op)
cifar_ds = cifar_ds.shuffle(buffer_size=cfg.buffer_size)
cifar_ds = cifar_ds.batch(batch_size, drop_remainder=True)
cifar_ds = cifar_ds.repeat(repeat_size)
return cifar_ds
def create_ssd_dataset(mindrecord_file, batch_size=32, repeat_num=10, device_num=1, rank=0,
is_training=True, num_parallel_workers=4):
"""Creatr SSD dataset with MindDataset."""
ds = de.MindDataset(mindrecord_file, columns_list=["img_id", "image", "annotation"], num_shards=device_num,
shard_id=rank, num_parallel_workers=num_parallel_workers, shuffle=is_training)
decode = C.Decode()
ds = ds.map(input_columns=["image"], operations=decode)
change_swap_op = C.HWC2CHW()
normalize_op = C.Normalize(mean=[0.485*255, 0.456*255, 0.406*255], std=[0.229*255, 0.224*255, 0.225*255])
color_adjust_op = C.RandomColorAdjust(brightness=0.4, contrast=0.4, saturation=0.4)
compose_map_func = (lambda img_id, image, annotation: preprocess_fn(img_id, image, annotation, is_training))
if is_training:
output_columns = ["image", "box", "label", "num_match"]
trans = [color_adjust_op, normalize_op, change_swap_op]
else:
output_columns = ["img_id", "image", "image_shape"]
trans = [normalize_op, change_swap_op]
ds = ds.map(input_columns=["img_id", "image", "annotation"],
output_columns=output_columns, columns_order=output_columns,
operations=compose_map_func, python_multiprocessing=is_training,
num_parallel_workers=num_parallel_workers)
ds = ds.map(input_columns=["image"], operations=trans, python_multiprocessing=is_training,
num_parallel_workers=num_parallel_workers)
ds = ds.batch(batch_size, drop_remainder=True)
ds = ds.repeat(repeat_num)
return ds
def test_decode_normalize_op():
"""
Test Decode op followed by Normalize op
"""
logger.info("Test [Decode, Normalize] in one Map")
data1 = ds.TFRecordDataset(DATA_DIR,
SCHEMA_DIR,
columns_list=["image", "label"],
num_parallel_workers=1,
shuffle=False)
# define map operations
decode_op = c_vision.Decode()
normalize_op = c_vision.Normalize([121.0, 115.0, 100.0],
[70.0, 68.0, 71.0])
# apply map operations on images
data1 = data1.map(input_columns=["image"],
operations=[decode_op, normalize_op])
num_iter = 0
for item in data1.create_dict_iterator():
logger.info("Looping inside iterator {}".format(num_iter))
_ = item["image"]
num_iter += 1
def create_dataset(dataset_path, do_train, repeat_num=1, batch_size=32):
"""
Create a train or eval dataset.
Args:
dataset_path (str): The path of dataset.
do_train (bool): Whether dataset is used for train or eval.
repeat_num (int): The repeat times of dataset. Default: 1.
batch_size (int): The batch size of dataset. Default: 32.
Returns:
Dataset.
"""
if do_train:
dataset_path = os.path.join(dataset_path, 'train')
do_shuffle = True
else:
dataset_path = os.path.join(dataset_path, 'eval')
do_shuffle = False
if device_num == 1 or not do_train:
ds = de.Cifar10Dataset(dataset_path, num_parallel_workers=8, shuffle=do_shuffle)
else:
ds = de.Cifar10Dataset(dataset_path, num_parallel_workers=8, shuffle=do_shuffle,
num_shards=device_num, shard_id=device_id)
resize_height = 224
resize_width = 224
buffer_size = 100
rescale = 1.0 / 255.0
shift = 0.0
# define map operations
random_crop_op = C.RandomCrop((32, 32), (4, 4, 4, 4))
random_horizontal_flip_op = C.RandomHorizontalFlip(device_id / (device_id + 1))
resize_op = C.Resize((resize_height, resize_width))
rescale_op = C.Rescale(rescale, shift)
normalize_op = C.Normalize([0.4914, 0.4822, 0.4465], [0.2023, 0.1994, 0.2010])
change_swap_op = C.HWC2CHW()
trans = []
if do_train:
trans += [random_crop_op, random_horizontal_flip_op]
trans += [resize_op, rescale_op, normalize_op, change_swap_op]
type_cast_op = C2.TypeCast(mstype.int32)
ds = ds.map(input_columns="label", num_parallel_workers=8, operations=type_cast_op)
ds = ds.map(input_columns="image", num_parallel_workers=8, operations=trans)
# apply batch operations
ds = ds.batch(batch_size, drop_remainder=True)
# apply dataset repeat operation
ds = ds.repeat(repeat_num)
return ds
def util_test_normalize(mean, std, op_type):
"""
Utility function for testing Normalize. Input arguments are given by other tests
"""
if op_type == "cpp":
# define map operations
decode_op = c_vision.Decode()
normalize_op = c_vision.Normalize(mean, std)
# Generate dataset
data = ds.TFRecordDataset(DATA_DIR, SCHEMA_DIR, columns_list=["image"], shuffle=False)
data = data.map(input_columns=["image"], operations=decode_op)
data = data.map(input_columns=["image"], operations=normalize_op)
elif op_type == "python":
# define map operations
transforms = [
py_vision.Decode(),
py_vision.ToTensor(),
py_vision.Normalize(mean, std)
]
transform = py_vision.ComposeOp(transforms)
# Generate dataset
data = ds.TFRecordDataset(DATA_DIR, SCHEMA_DIR, columns_list=["image"], shuffle=False)
data = data.map(input_columns=["image"], operations=transform())
else:
raise ValueError("Wrong parameter value")
return data
def create_dataset(dataset_path, do_train, repeat_num=1, batch_size=32):
"""
create a train or eval dataset
Args:
dataset_path(string): the path of dataset.
do_train(bool): whether dataset is used for train or eval.
repeat_num(int): the repeat times of dataset. Default: 1
batch_size(int): the batch size of dataset. Default: 32
Returns:
dataset
"""
device_num = int(os.getenv("DEVICE_NUM"))
rank_id = int(os.getenv("RANK_ID"))
if device_num == 1:
ds = de.Cifar10Dataset(dataset_path,
num_parallel_workers=8,
shuffle=True)
else:
ds = de.Cifar10Dataset(dataset_path,
num_parallel_workers=8,
shuffle=True,
num_shards=device_num,
shard_id=rank_id)
# define map operations
trans = []
if do_train:
trans += [
C.RandomCrop((32, 32), (4, 4, 4, 4)),
C.RandomHorizontalFlip(prob=0.5)
]
trans += [
C.Resize((config.image_height, config.image_width)),
C.Rescale(1.0 / 255.0, 0.0),
C.Normalize([0.4914, 0.4822, 0.4465], [0.2023, 0.1994, 0.2010]),
C.HWC2CHW()
]
type_cast_op = C2.TypeCast(mstype.int32)
ds = ds.map(input_columns="label",
num_parallel_workers=8,
operations=type_cast_op)
ds = ds.map(input_columns="image",
num_parallel_workers=8,
operations=trans)
# apply batch operations
ds = ds.batch(batch_size, drop_remainder=True)
# apply dataset repeat operation
ds = ds.repeat(repeat_num)
return ds
def create_dataset(dataset_path, do_train, config, device_target, repeat_num=1, batch_size=32):
"""
create a train or eval dataset
Args:
dataset_path(string): the path of dataset.
do_train(bool): whether dataset is used for train or eval.
repeat_num(int): the repeat times of dataset. Default: 1
batch_size(int): the batch size of dataset. Default: 32
Returns:
dataset
"""
if device_target == "GPU":
if do_train:
from mindspore.communication.management import get_rank, get_group_size
ds = de.ImageFolderDatasetV2(dataset_path, num_parallel_workers=8, shuffle=True,
num_shards=get_group_size(), shard_id=get_rank())
else:
ds = de.ImageFolderDatasetV2(dataset_path, num_parallel_workers=8, shuffle=True)
else:
raise ValueError("Unsupported device_target.")
resize_height = config.image_height
resize_width = config.image_width
buffer_size = 1000
# define map operations
decode_op = C.Decode()
resize_crop_op = C.RandomCropDecodeResize(resize_height, scale=(0.08, 1.0), ratio=(0.75, 1.333))
horizontal_flip_op = C.RandomHorizontalFlip(prob=0.5)
resize_op = C.Resize(256)
center_crop = C.CenterCrop(resize_width)
rescale_op = C.RandomColorAdjust(brightness=0.4, contrast=0.4, saturation=0.4)
normalize_op = C.Normalize(mean=[0.485*255, 0.456*255, 0.406*255], std=[0.229*255, 0.224*255, 0.225*255])
change_swap_op = C.HWC2CHW()
if do_train:
trans = [resize_crop_op, horizontal_flip_op, rescale_op, normalize_op, change_swap_op]
else:
trans = [decode_op, resize_op, center_crop, normalize_op, change_swap_op]
type_cast_op = C2.TypeCast(mstype.int32)
ds = ds.map(input_columns="image", operations=trans, num_parallel_workers=8)
ds = ds.map(input_columns="label", operations=type_cast_op, num_parallel_workers=8)
# apply shuffle operations
ds = ds.shuffle(buffer_size=buffer_size)
# apply batch operations
ds = ds.batch(batch_size, drop_remainder=True)
# apply dataset repeat operation
ds = ds.repeat(repeat_num)
return ds
def create_dataset(dataset_path, do_train, repeat_num=1, batch_size=32):
"""
create a train or eval dataset
Args:
dataset_path(string): the path of dataset.
do_train(bool): whether dataset is used for train or eval.
repeat_num(int): the repeat times of dataset. Default: 1
batch_size(int): the batch size of dataset. Default: 32
Returns:
dataset
"""
rank_size = int(os.getenv("RANK_SIZE"))
rank_id = int(os.getenv("RANK_ID"))
if rank_size == 1:
ds = de.ImageFolderDatasetV2(dataset_path, num_parallel_workers=8, shuffle=True)
else:
ds = de.ImageFolderDatasetV2(dataset_path, num_parallel_workers=8, shuffle=True,
num_shards=rank_size, shard_id=rank_id)
resize_height = config.image_height
resize_width = config.image_width
buffer_size = 1000
# define map operations
decode_op = C.Decode()
resize_crop_op = C.RandomCropDecodeResize(resize_height, scale=(0.08, 1.0), ratio=(0.75, 1.333))
horizontal_flip_op = C.RandomHorizontalFlip(prob=0.5)
resize_op = C.Resize((256, 256))
center_crop = C.CenterCrop(resize_width)
rescale_op = C.RandomColorAdjust(brightness=0.4, contrast=0.4, saturation=0.4)
normalize_op = C.Normalize(mean=[0.485*255, 0.456*255, 0.406*255], std=[0.229*255, 0.224*255, 0.225*255])
change_swap_op = C.HWC2CHW()
if do_train:
trans = [resize_crop_op, horizontal_flip_op, rescale_op, normalize_op, change_swap_op]
else:
trans = [decode_op, resize_op, center_crop, rescale_op, normalize_op, change_swap_op]
type_cast_op = C2.TypeCast(mstype.int32)
ds = ds.map(input_columns="image", operations=trans, num_parallel_workers=8)
ds = ds.map(input_columns="label", operations=type_cast_op, num_parallel_workers=8)
# apply shuffle operations
ds = ds.shuffle(buffer_size=buffer_size)
# apply batch operations
ds = ds.batch(batch_size, drop_remainder=True)
# apply dataset repeat operation
ds = ds.repeat(repeat_num)
return ds
def create_dataset(data_home, repeat_num=1, training=True):
"""Data operations."""
ds.config.set_seed(1)
rank_size = int(
os.environ.get("RANK_SIZE")) if os.environ.get("RANK_SIZE") else None
device_num = int(os.getenv("RANK_SIZE"))
rank_id = int(os.getenv("DEVICE_ID"))
if device_num == 1:
data_set = ds.Cifar10Dataset(data_home,
num_parallel_workers=8,
shuffle=True)
else:
data_set = ds.Cifar10Dataset(data_home,
num_parallel_workers=8,
shuffle=True,
num_shards=device_num,
shard_id=rank_id)
resize_height = cfg.image_height
resize_width = cfg.image_width
rescale = 1.0 / 255.0
shift = 0.0
# define map operations
random_crop_op = vision.RandomCrop(
(32, 32), (4, 4, 4, 4)) # padding_mode default CONSTANT
random_horizontal_op = vision.RandomHorizontalFlip()
resize_op = vision.Resize(
(resize_height, resize_width)) # interpolation default BILINEAR
rescale_op = vision.Rescale(rescale, shift)
normalize_op = vision.Normalize((0.4465, 0.4822, 0.4914),
(0.2010, 0.1994, 0.2023))
changeswap_op = vision.HWC2CHW()
type_cast_op = C.TypeCast(mstype.int32)
c_trans = []
if training:
c_trans = [random_crop_op, random_horizontal_op]
c_trans += [resize_op, rescale_op, normalize_op, changeswap_op]
# apply map operations on images
data_set = data_set.map(input_columns="label", operations=type_cast_op)
data_set = data_set.map(input_columns="image", operations=c_trans)
# apply repeat operations
data_set = data_set.repeat(repeat_num)
# apply shuffle operations
data_set = data_set.shuffle(buffer_size=10)
# apply batch operations
data_set = data_set.batch(batch_size=cfg.batch_size, drop_remainder=True)
return data_set
def create_dataset(dataset_path, config, do_train, repeat_num=1):
"""
create a train or eval dataset
Args:
dataset_path(string): the path of dataset.
config(dict): config of dataset.
do_train(bool): whether dataset is used for train or eval.
repeat_num(int): the repeat times of dataset. Default: 1.
Returns:
dataset
"""
rank = config.rank
group_size = config.group_size
if group_size == 1:
ds = de.ImageFolderDatasetV2(dataset_path,
num_parallel_workers=config.work_nums,
shuffle=True)
else:
ds = de.ImageFolderDatasetV2(dataset_path,
num_parallel_workers=config.work_nums,
shuffle=True,
num_shards=group_size,
shard_id=rank)
# define map operations
if do_train:
trans = [
C.RandomCropDecodeResize(config.image_size),
C.RandomHorizontalFlip(prob=0.5),
C.RandomColorAdjust(brightness=0.4, saturation=0.5) # fast mode
#C.RandomColorAdjust(brightness=0.4, contrast=0.5, saturation=0.5, hue=0.2)
]
else:
trans = [
C.Decode(),
C.Resize(int(config.image_size / 0.875)),
C.CenterCrop(config.image_size)
]
trans += [
C.Rescale(1.0 / 255.0, 0.0),
C.Normalize(mean=[0.5, 0.5, 0.5], std=[0.5, 0.5, 0.5]),
C.HWC2CHW()
]
type_cast_op = C2.TypeCast(mstype.int32)
ds = ds.map(input_columns="image",
operations=trans,
num_parallel_workers=config.work_nums)
ds = ds.map(input_columns="label",
operations=type_cast_op,
num_parallel_workers=config.work_nums)
# apply batch operations
ds = ds.batch(config.batch_size, drop_remainder=True)
# apply dataset repeat operation
ds = ds.repeat(repeat_num)
return ds
def test_normalize_exception_unequal_size_c():
"""
Test Normalize in c transformation: len(mean) != len(std)
expected to raise ValueError
"""
logger.info("test_normalize_exception_unequal_size_c")
try:
_ = c_vision.Normalize([100, 250, 125], [50, 50, 75, 75])
except ValueError as e:
logger.info("Got an exception in DE: {}".format(str(e)))
assert str(e) == "Length of mean and std must be equal"
def create_dataset(repeat_num=1,
training=True,
batch_size=32,
rank_id=0,
rank_size=1,
enable_hccl=False):
data_dir = data_home + "/cifar-10-batches-bin"
if not training:
data_dir = data_home + "/cifar-10-verify-bin"
data_set = ds.Cifar10Dataset(data_dir)
if enable_hccl:
rank_id = rank_id
rank_size = rank_size
data_set = ds.Cifar10Dataset(data_dir,
num_shards=rank_size,
shard_id=rank_id)
resize_height = 224
resize_width = 224
rescale = 1.0 / 255.0
shift = 0.0
# define map operations
random_crop_op = vision.RandomCrop(
(32, 32), (4, 4, 4, 4)) # padding_mode default CONSTANT
random_horizontal_op = vision.RandomHorizontalFlip()
# interpolation default BILINEAR
resize_op = vision.Resize((resize_height, resize_width))
rescale_op = vision.Rescale(rescale, shift)
normalize_op = vision.Normalize((0.4465, 0.4822, 0.4914),
(0.2010, 0.1994, 0.2023))
changeswap_op = vision.HWC2CHW()
type_cast_op = C.TypeCast(mstype.int32)
c_trans = []
if training:
c_trans = [random_crop_op, random_horizontal_op]
c_trans += [resize_op, rescale_op, normalize_op, changeswap_op]
# apply map operations on images
data_set = data_set.map(input_columns="label", operations=type_cast_op)
data_set = data_set.map(input_columns="image", operations=c_trans)
# apply shuffle operations
data_set = data_set.shuffle(buffer_size=1000)
# apply batch operations
data_set = data_set.batch(batch_size=batch_size, drop_remainder=True)
# apply repeat operations
data_set = data_set.repeat(repeat_num)
return data_set
def create_dataset(dataset_path, do_train, rank, group_size, repeat_num=1):
"""
create a train or eval dataset
Args:
dataset_path(string): the path of dataset.
do_train(bool): whether dataset is used for train or eval.
rank (int): The shard ID within num_shards (default=None).
group_size (int): Number of shards that the dataset should be divided into (default=None).
repeat_num(int): the repeat times of dataset. Default: 1.
Returns:
dataset
"""
if group_size == 1:
ds = de.ImageFolderDatasetV2(dataset_path,
num_parallel_workers=cfg.work_nums,
shuffle=True)
else:
ds = de.ImageFolderDatasetV2(dataset_path,
num_parallel_workers=cfg.work_nums,
shuffle=True,
num_shards=group_size,
shard_id=rank)
# define map operations
if do_train:
trans = [
C.RandomCropDecodeResize(299,
scale=(0.08, 1.0),
ratio=(0.75, 1.333)),
C.RandomHorizontalFlip(prob=0.5),
C.RandomColorAdjust(brightness=0.4, contrast=0.4, saturation=0.4)
]
else:
trans = [C.Decode(), C.Resize(299), C.CenterCrop(299)]
trans += [
C.Rescale(1.0 / 255.0, 0.0),
C.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]),
C.HWC2CHW()
]
type_cast_op = C2.TypeCast(mstype.int32)
ds = ds.map(input_columns="image",
operations=trans,
num_parallel_workers=cfg.work_nums)
ds = ds.map(input_columns="label",
operations=type_cast_op,
num_parallel_workers=cfg.work_nums)
# apply batch operations
ds = ds.batch(cfg.batch_size, drop_remainder=True)
# apply dataset repeat operation
ds = ds.repeat(repeat_num)
return ds
def create_dataset(repeat_num=1, training=True):
"""
create data for next use such as training or infering
"""
cifar_ds = ds.Cifar10Dataset(data_home)
if args_opt.run_distribute:
rank_id = int(os.getenv('RANK_ID'))
rank_size = int(os.getenv('RANK_SIZE'))
cifar_ds = ds.Cifar10Dataset(data_home,
num_shards=rank_size,
shard_id=rank_id)
resize_height = 224
resize_width = 224
rescale = 1.0 / 255.0
shift = 0.0
# define map operations
random_crop_op = C.RandomCrop(
(32, 32), (4, 4, 4, 4)) # padding_mode default CONSTANT
random_horizontal_op = C.RandomHorizontalFlip()
resize_op = C.Resize(
(resize_height, resize_width)) # interpolation default BILINEAR
rescale_op = C.Rescale(rescale, shift)
normalize_op = C.Normalize((0.4914, 0.4822, 0.4465),
(0.2023, 0.1994, 0.2010))
changeswap_op = C.HWC2CHW()
type_cast_op = C2.TypeCast(mstype.int32)
c_trans = []
if training:
c_trans = [random_crop_op, random_horizontal_op]
c_trans += [resize_op, rescale_op, normalize_op, changeswap_op]
# apply map operations on images
cifar_ds = cifar_ds.map(input_columns="label", operations=type_cast_op)
cifar_ds = cifar_ds.map(input_columns="image", operations=c_trans)
# apply repeat operations
cifar_ds = cifar_ds.repeat(repeat_num)
# apply shuffle operations
cifar_ds = cifar_ds.shuffle(buffer_size=10)
# apply batch operations
cifar_ds = cifar_ds.batch(batch_size=args_opt.batch_size,
drop_remainder=True)
return cifar_ds
def create_dataset(data_home, repeat_num=1, training=True):
"""Data operations."""
ds.config.set_seed(1)
data_dir = os.path.join(data_home, "cifar-10-batches-bin")
if not training:
data_dir = os.path.join(data_home, "cifar-10-verify-bin")
rank_size, rank_id = _get_rank_info()
if training:
data_set = ds.Cifar10Dataset(data_dir,
num_shards=rank_size,
shard_id=rank_id,
shuffle=True)
else:
data_set = ds.Cifar10Dataset(data_dir,
num_shards=rank_size,
shard_id=rank_id,
shuffle=False)
resize_height = cfg.image_height
resize_width = cfg.image_width
# define map operations
random_crop_op = vision.RandomCrop(
(32, 32), (4, 4, 4, 4)) # padding_mode default CONSTANT
random_horizontal_op = vision.RandomHorizontalFlip()
resize_op = vision.Resize(
(resize_height, resize_width)) # interpolation default BILINEAR
rescale_op = vision.Rescale(1.0 / 255.0, 0.0)
normalize_op = vision.Normalize((0.4914, 0.4822, 0.4465),
(0.2023, 0.1994, 0.2010))
changeswap_op = vision.HWC2CHW()
type_cast_op = C.TypeCast(mstype.int32)
c_trans = []
if training:
c_trans = [random_crop_op, random_horizontal_op]
c_trans += [resize_op, rescale_op, normalize_op, changeswap_op]
# apply map operations on images
data_set = data_set.map(input_columns="label", operations=type_cast_op)
data_set = data_set.map(input_columns="image", operations=c_trans)
# apply batch operations
data_set = data_set.batch(batch_size=cfg.batch_size, drop_remainder=True)
# apply repeat operations
data_set = data_set.repeat(repeat_num)
return data_set
def vgg_create_dataset(data_home,
image_size,
batch_size,
rank_id=0,
rank_size=1,
repeat_num=1,
training=True):
"""Data operations."""
de.config.set_seed(1)
data_dir = os.path.join(data_home, "cifar-10-batches-bin")
if not training:
data_dir = os.path.join(data_home, "cifar-10-verify-bin")
data_set = de.Cifar10Dataset(data_dir,
num_shards=rank_size,
shard_id=rank_id)
rescale = 1.0 / 255.0
shift = 0.0
# define map operations
random_crop_op = vision.RandomCrop(
(32, 32), (4, 4, 4, 4)) # padding_mode default CONSTANT
random_horizontal_op = vision.RandomHorizontalFlip()
resize_op = vision.Resize(image_size) # interpolation default BILINEAR
rescale_op = vision.Rescale(rescale, shift)
normalize_op = vision.Normalize((0.4465, 0.4822, 0.4914),
(0.2010, 0.1994, 0.2023))
changeswap_op = vision.HWC2CHW()
type_cast_op = C.TypeCast(mstype.int32)
c_trans = []
if training:
c_trans = [random_crop_op, random_horizontal_op]
c_trans += [resize_op, rescale_op, normalize_op, changeswap_op]
# apply map operations on images
data_set = data_set.map(input_columns="label", operations=type_cast_op)
data_set = data_set.map(input_columns="image", operations=c_trans)
# apply repeat operations
data_set = data_set.repeat(repeat_num)
# apply shuffle operations
data_set = data_set.shuffle(buffer_size=10)
# apply batch operations
data_set = data_set.batch(batch_size=batch_size, drop_remainder=True)
return data_set
def test_normalize_op():
"""
Test Normalize
"""
logger.info("Test Normalize")
# define map operations
decode_op = vision.Decode()
normalize_op = vision.Normalize([121.0, 115.0, 100.0], [70.0, 68.0, 71.0])
# First dataset
data1 = ds.TFRecordDataset(DATA_DIR,
SCHEMA_DIR,
columns_list=["image"],
shuffle=False)
data1 = data1.map(input_columns=["image"], operations=decode_op)
data1 = data1.map(input_columns=["image"], operations=normalize_op)
# Second dataset
data2 = ds.TFRecordDataset(DATA_DIR,
SCHEMA_DIR,
columns_list=["image"],
shuffle=False)
data2 = data2.map(input_columns=["image"], operations=decode_op)
num_iter = 0
for item1, item2 in zip(data1.create_dict_iterator(),
data2.create_dict_iterator()):
image_de_normalized = item1["image"]
image_np_normalized = normalize_np(item2["image"])
diff = image_de_normalized - image_np_normalized
mse = np.sum(np.power(diff, 2))
logger.info("image_{}, mse: {}".format(num_iter + 1, mse))
assert mse < 0.01
# Uncomment these blocks to see visual results
# plt.subplot(131)
# plt.imshow(image_de_normalized)
# plt.title("DE normalize image")
#
# plt.subplot(132)
# plt.imshow(image_np_normalized)
# plt.title("Numpy normalized image")
#
# plt.subplot(133)
# plt.imshow(diff)
# plt.title("Difference image, mse : {}".format(mse))
#
# plt.show()
num_iter += 1
def create_cifar10_dataset(cifar_dir, num_parallel_workers=1):
"""
Creat the cifar10 dataset.
"""
ds = de.Cifar10Dataset(cifar_dir)
training = True
resize_height = 224
resize_width = 224
rescale = 1.0 / 255.0
shift = 0.0
repeat_num = 10
batch_size = 32
# define map operations
random_crop_op = vision.RandomCrop(
(32, 32), (4, 4, 4, 4)) # padding_mode default CONSTANT
random_horizontal_op = vision.RandomHorizontalFlip()
resize_op = vision.Resize(
(resize_height, resize_width)) # interpolation default BILINEAR
rescale_op = vision.Rescale(rescale, shift)
normalize_op = vision.Normalize((0.4465, 0.4822, 0.4914),
(0.2010, 0.1994, 0.2023))
changeswap_op = vision.HWC2CHW()
type_cast_op = C.TypeCast(mstype.int32)
if training:
c_trans = [random_crop_op, random_horizontal_op]
c_trans += [resize_op, rescale_op, normalize_op, changeswap_op]
# apply map operations on images
ds = ds.map(input_columns="label", operations=type_cast_op)
ds = ds.map(input_columns="image", operations=c_trans)
# apply repeat operations
ds = ds.repeat(repeat_num)
# apply shuffle operations
ds = ds.shuffle(buffer_size=10)
# apply batch operations
ds = ds.batch(batch_size=batch_size, drop_remainder=True)
return ds
def create_dataset(data_path,
repeat_num=1,
batch_size=32,
rank_id=0,
rank_size=1):
resize_height = 224
resize_width = 224
rescale = 1.0 / 255.0
shift = 0.0
# get rank_id and rank_size
rank_id = get_rank()
rank_size = get_group_size()
data_set = ds.Cifar10Dataset(data_path,
num_shards=rank_size,
shard_id=rank_id)
# define map operations
random_crop_op = vision.RandomCrop((32, 32), (4, 4, 4, 4))
random_horizontal_op = vision.RandomHorizontalFlip()
resize_op = vision.Resize((resize_height, resize_width))
rescale_op = vision.Rescale(rescale, shift)
normalize_op = vision.Normalize((0.4465, 0.4822, 0.4914),
(0.2010, 0.1994, 0.2023))
changeswap_op = vision.HWC2CHW()
type_cast_op = C.TypeCast(mstype.int32)
c_trans = [random_crop_op, random_horizontal_op]
c_trans += [resize_op, rescale_op, normalize_op, changeswap_op]
# apply map operations on images
data_set = data_set.map(input_columns="label", operations=type_cast_op)
data_set = data_set.map(input_columns="image", operations=c_trans)
# apply shuffle operations
data_set = data_set.shuffle(buffer_size=10)
# apply batch operations
data_set = data_set.batch(batch_size=batch_size, drop_remainder=True)
# apply repeat operations
data_set = data_set.repeat(repeat_num)
return data_set
def test_apply_imagefolder_case():
# apply dataset map operations
data1 = ds.ImageFolderDatasetV2(DATA_DIR, num_shards=4, shard_id=3)
data2 = ds.ImageFolderDatasetV2(DATA_DIR, num_shards=4, shard_id=3)
decode_op = vision.Decode()
normalize_op = vision.Normalize([121.0, 115.0, 100.0], [70.0, 68.0, 71.0])
def dataset_fn(ds):
ds = ds.map(operations = decode_op)
ds = ds.map(operations = normalize_op)
ds = ds.repeat(2)
return ds
data1 = data1.apply(dataset_fn)
data2 = data2.map(operations = decode_op)
data2 = data2.map(operations = normalize_op)
data2 = data2.repeat(2)
for item1, item2 in zip(data1.create_dict_iterator(), data2.create_dict_iterator()):
assert np.array_equal(item1["image"], item2["image"])
def create_dataset(dataset_path,
batch_size=1,
num_shards=1,
shard_id=0,
device_target='Ascend'):
"""
create train or evaluation dataset for warpctc
Args:
dataset_path(int): dataset path
batch_size(int): batch size of generated dataset, default is 1
num_shards(int): number of devices
shard_id(int): rank id
device_target(str): platform of training, support Ascend and GPU
"""
dataset = _CaptchaDataset(dataset_path, cf.max_captcha_digits,
device_target)
ds = de.GeneratorDataset(dataset, ["image", "label"],
shuffle=True,
num_shards=num_shards,
shard_id=shard_id)
ds.set_dataset_size(m.ceil(len(dataset) / num_shards))
image_trans = [
vc.Rescale(1.0 / 255.0, 0.0),
vc.Normalize([0.9010, 0.9049, 0.9025], std=[0.1521, 0.1347, 0.1458]),
vc.Resize((m.ceil(cf.captcha_height / 16) * 16, cf.captcha_width)),
vc.HWC2CHW()
]
label_trans = [c.TypeCast(mstype.int32)]
ds = ds.map(input_columns=["image"],
num_parallel_workers=8,
operations=image_trans)
ds = ds.map(input_columns=["label"],
num_parallel_workers=8,
operations=label_trans)
ds = ds.batch(batch_size, drop_remainder=True)
return ds
def create_dataset(dataset_path, repeat_num=1, batch_size=1):
"""
create train or evaluation dataset for warpctc
Args:
dataset_path(int): dataset path
repeat_num(int): dataset repetition num, default is 1
batch_size(int): batch size of generated dataset, default is 1
"""
rank_size = int(
os.environ.get("RANK_SIZE")) if os.environ.get("RANK_SIZE") else 1
rank_id = int(
os.environ.get("RANK_ID")) if os.environ.get("RANK_ID") else 0
dataset = _CaptchaDataset(dataset_path, cf.max_captcha_digits)
ds = de.GeneratorDataset(dataset, ["image", "label"],
shuffle=True,
num_shards=rank_size,
shard_id=rank_id)
ds.set_dataset_size(m.ceil(len(dataset) / rank_size))
image_trans = [
vc.Rescale(1.0 / 255.0, 0.0),
vc.Normalize([0.9010, 0.9049, 0.9025], std=[0.1521, 0.1347, 0.1458]),
vc.Resize((m.ceil(cf.captcha_height / 16) * 16, cf.captcha_width)),
vc.HWC2CHW()
]
label_trans = [c.TypeCast(mstype.int32)]
ds = ds.map(input_columns=["image"],
num_parallel_workers=8,
operations=image_trans)
ds = ds.map(input_columns=["label"],
num_parallel_workers=8,
operations=label_trans)
ds = ds.batch(batch_size)
ds = ds.repeat(repeat_num)
return ds
def create_dataset(data_path, batch_size=32, repeat_size=1, status="train"):
"""
create dataset for train or test
"""
rank_id = int(os.getenv('DEVICE_ID'))
device_num = int(os.getenv('RANK_SIZE'))
if device_num == 1:
cifar_ds = ds.Cifar10Dataset(data_path, num_parallel_workers=8, shuffle=True)
else:
cifar_ds = ds.Cifar10Dataset(data_path, num_parallel_workers=8, shuffle=True,
num_shards=device_num, shard_id=rank_id)
rescale = 1.0 / 255.0
shift = 0.0
resize_op = CV.Resize((cfg.image_height, cfg.image_width))
rescale_op = CV.Rescale(rescale, shift)
normalize_op = CV.Normalize((0.4914, 0.4822, 0.4465), (0.2023, 0.1994, 0.2010))
if status == "train":
random_crop_op = CV.RandomCrop([32, 32], [4, 4, 4, 4])
random_horizontal_op = CV.RandomHorizontalFlip()
channel_swap_op = CV.HWC2CHW()
typecast_op = C.TypeCast(mstype.int32)
cifar_ds = cifar_ds.map(input_columns="label", operations=typecast_op)
if status == "train":
cifar_ds = cifar_ds.map(input_columns="image", operations=random_crop_op)
cifar_ds = cifar_ds.map(input_columns="image", operations=random_horizontal_op)
cifar_ds = cifar_ds.map(input_columns="image", operations=resize_op)
cifar_ds = cifar_ds.map(input_columns="image", operations=rescale_op)
cifar_ds = cifar_ds.map(input_columns="image", operations=normalize_op)
cifar_ds = cifar_ds.map(input_columns="image", operations=channel_swap_op)
cifar_ds = cifar_ds.shuffle(buffer_size=cfg.buffer_size)
cifar_ds = cifar_ds.batch(batch_size, drop_remainder=True)
cifar_ds = cifar_ds.repeat(repeat_size)
return cifar_ds
def create_fasterrcnn_dataset(mindrecord_file,
batch_size=2,
repeat_num=12,
device_num=1,
rank_id=0,
is_training=True,
num_parallel_workers=8):
"""Creatr FasterRcnn dataset with MindDataset."""
ds = de.MindDataset(mindrecord_file,
columns_list=["image", "annotation"],
num_shards=device_num,
shard_id=rank_id,
num_parallel_workers=num_parallel_workers,
shuffle=is_training)
decode = C.Decode()
ds = ds.map(input_columns=["image"], operations=decode)
compose_map_func = (lambda image, annotation: preprocess_fn(
image, annotation, is_training))
hwc_to_chw = C.HWC2CHW()
normalize_op = C.Normalize((123.675, 116.28, 103.53),
(58.395, 57.12, 57.375))
horizontally_op = C.RandomHorizontalFlip(1)
type_cast0 = CC.TypeCast(mstype.float32)
type_cast1 = CC.TypeCast(mstype.float16)
type_cast2 = CC.TypeCast(mstype.int32)
type_cast3 = CC.TypeCast(mstype.bool_)
if is_training:
ds = ds.map(input_columns=["image", "annotation"],
output_columns=[
"image", "image_shape", "box", "label", "valid_num"
],
columns_order=[
"image", "image_shape", "box", "label", "valid_num"
],
operations=compose_map_func,
num_parallel_workers=4)
ds = ds.map(input_columns=["image"],
operations=[normalize_op, type_cast0],
num_parallel_workers=num_parallel_workers)
flip = (np.random.rand() < config.flip_ratio)
if flip:
ds = ds.map(input_columns=["image"],
operations=[horizontally_op],
num_parallel_workers=num_parallel_workers)
ds = ds.map(input_columns=[
"image", "image_shape", "box", "label", "valid_num"
],
operations=flipped_generation,
num_parallel_workers=4)
# transpose_column from python to c
ds = ds.map(input_columns=["image"],
operations=[hwc_to_chw, type_cast1])
ds = ds.map(input_columns=["image_shape"], operations=[type_cast1])
ds = ds.map(input_columns=["box"], operations=[type_cast1])
ds = ds.map(input_columns=["label"], operations=[type_cast2])
ds = ds.map(input_columns=["valid_num"], operations=[type_cast3])
ds = ds.batch(batch_size, drop_remainder=True)
ds = ds.repeat(repeat_num)
else:
ds = ds.map(input_columns=["image", "annotation"],
output_columns=[
"image", "image_shape", "box", "label", "valid_num"
],
columns_order=[
"image", "image_shape", "box", "label", "valid_num"
],
operations=compose_map_func,
num_parallel_workers=num_parallel_workers)
# transpose_column from python to c
ds = ds.map(input_columns=["image"],
operations=[hwc_to_chw, type_cast1])
ds = ds.map(input_columns=["image_shape"], operations=[type_cast1])
ds = ds.map(input_columns=["box"], operations=[type_cast1])
ds = ds.map(input_columns=["label"], operations=[type_cast2])
ds = ds.map(input_columns=["valid_num"], operations=[type_cast3])
ds = ds.batch(batch_size, drop_remainder=True)
ds = ds.repeat(repeat_num)
return ds
def create_dataset(dataset_path, do_train, repeat_num=1, batch_size=32):
"""
create a train or eval dataset
Args:
dataset_path(string): the path of dataset.
do_train(bool): whether dataset is used for train or eval.
repeat_num(int): the repeat times of dataset. Default: 1
batch_size(int): the batch size of dataset. Default: 32
Returns:
dataset
"""
device_num = int(os.getenv("RANK_SIZE"))
try:
# global_rank_id = int(os.getenv('RANK_ID').split("-")[1].split("custom")[1])
global_rank_id = int(os.getenv('RANK_ID').split("-")[-1])
except:
global_rank_id = 0
rank_id = int(os.getenv('DEVICE_ID')) + global_rank_id * 8
columns_list = ["data", "label"]
if do_train:
ds = de.MindDataset(dataset_path + '/imagenet_train.mindrecord00',
columns_list,
num_parallel_workers=8,
shuffle=True,
num_shards=device_num,
shard_id=rank_id)
print("train dataset size", ds.get_dataset_size())
else:
padded_sample = {}
white_io = BytesIO()
Image.new('RGB', (224, 224), (255, 255, 255)).save(white_io, 'JPEG')
padded_sample['data'] = white_io.getvalue()
padded_sample['label'] = -1
batch_per_step = batch_size * device_num
print("eval batch per step:", batch_per_step)
if batch_per_step < 50000:
if 50000 % batch_per_step == 0:
num_padded = 0
else:
num_padded = batch_per_step - (50000 % batch_per_step)
else:
num_padded = batch_per_step - 50000
print("Padded samples:", num_padded)
ds = de.MindDataset(dataset_path + '/imagenet_eval.mindrecord0',
columns_list,
num_parallel_workers=8,
shuffle=False,
num_shards=device_num,
shard_id=rank_id,
padded_sample=padded_sample,
num_padded=num_padded)
print("eval dataset size", ds.get_dataset_size())
image_size = 224
mean = [0.485 * 255, 0.456 * 255, 0.406 * 255]
std = [0.229 * 255, 0.224 * 255, 0.225 * 255]
# define map operations
if do_train:
trans = [
C.RandomCropDecodeResize(image_size,
scale=(0.08, 1.0),
ratio=(0.75, 1.333)),
C.RandomHorizontalFlip(prob=0.5),
C.Normalize(mean=mean, std=std),
C.HWC2CHW()
]
else:
trans = [
C.Decode(),
C.Resize((365, 365)),
C.CenterCrop(320),
C.Normalize(mean=mean, std=std),
C.HWC2CHW()
]
type_cast_op = C2.TypeCast(mstype.int32)
# ds = ds.shuffle(buffer_size=100000)
ds = ds.map(input_columns="data", num_parallel_workers=8, operations=trans)
ds = ds.map(input_columns="label",
num_parallel_workers=8,
operations=type_cast_op)
# apply batch operations
ds = ds.batch(batch_size, drop_remainder=True)
# apply dataset repeat operation
ds = ds.repeat(repeat_num)
return ds