def util_test_random_color_adjust_error(brightness=(1, 1), contrast=(1, 1), saturation=(1, 1), hue=(0, 0)):
"""
Util function that tests the error message in case of grayscale images
"""
transforms = [
py_vision.Decode(),
py_vision.Grayscale(1),
py_vision.ToTensor(),
(lambda image: (image.transpose(1, 2, 0) * 255).astype(np.uint8))
]
transform = mindspore.dataset.transforms.py_transforms.Compose(transforms)
data1 = ds.TFRecordDataset(DATA_DIR, SCHEMA_DIR, columns_list=["image"], shuffle=False)
data1 = data1.map(operations=transform, input_columns=["image"])
# if input is grayscale, the output dimensions should be single channel, the following should fail
random_adjust_op = c_vision.RandomColorAdjust(brightness=brightness, contrast=contrast, saturation=saturation,
hue=hue)
with pytest.raises(RuntimeError) as info:
data1 = data1.map(operations=random_adjust_op, input_columns=["image"])
dataset_shape_1 = []
for item1 in data1.create_dict_iterator(num_epochs=1):
c_image = item1["image"]
dataset_shape_1.append(c_image.shape)
error_msg = "image shape is not <H,W,C>"
assert error_msg in str(info.value)
def create_ssd_dataset(mindrecord_file, batch_size=32, repeat_num=10, device_num=1, rank=0,
is_training=True, num_parallel_workers=4, use_multiprocessing=True):
"""Create SSD dataset with MindDataset."""
ds = de.MindDataset(mindrecord_file, columns_list=["img_id", "image", "annotation"], num_shards=device_num,
shard_id=rank, num_parallel_workers=num_parallel_workers, shuffle=is_training)
decode = C.Decode()
ds = ds.map(operations=decode, input_columns=["image"])
change_swap_op = C.HWC2CHW()
normalize_op = C.Normalize(mean=[0.485 * 255, 0.456 * 255, 0.406 * 255],
std=[0.229 * 255, 0.224 * 255, 0.225 * 255])
color_adjust_op = C.RandomColorAdjust(brightness=0.4, contrast=0.4, saturation=0.4)
compose_map_func = (lambda img_id, image, annotation: preprocess_fn(img_id, image, annotation, is_training))
if is_training:
output_columns = ["image", "box", "label", "num_match"]
trans = [color_adjust_op, normalize_op, change_swap_op]
else:
output_columns = ["img_id", "image", "image_shape"]
trans = [normalize_op, change_swap_op]
ds = ds.map(operations=compose_map_func, input_columns=["img_id", "image", "annotation"],
output_columns=output_columns, column_order=output_columns,
python_multiprocessing=use_multiprocessing,
num_parallel_workers=num_parallel_workers)
ds = ds.map(operations=trans, input_columns=["image"], python_multiprocessing=use_multiprocessing,
num_parallel_workers=num_parallel_workers)
ds = ds.batch(batch_size, drop_remainder=True)
ds = ds.repeat(repeat_num)
return ds
def test_random_color_adjust_md5():
"""
Test RandomColorAdjust with md5 check
"""
logger.info("Test RandomColorAdjust with md5 check")
original_seed = config_get_set_seed(10)
original_num_parallel_workers = config_get_set_num_parallel_workers(1)
# First dataset
data1 = ds.TFRecordDataset(DATA_DIR, SCHEMA_DIR, columns_list=["image"], shuffle=False)
decode_op = c_vision.Decode()
random_adjust_op = c_vision.RandomColorAdjust(0.4, 0.4, 0.4, 0.1)
data1 = data1.map(operations=decode_op, input_columns=["image"])
data1 = data1.map(operations=random_adjust_op, input_columns=["image"])
# Second dataset
transforms = [
py_vision.Decode(),
py_vision.RandomColorAdjust(0.4, 0.4, 0.4, 0.1),
py_vision.ToTensor()
]
transform = mindspore.dataset.transforms.py_transforms.Compose(transforms)
data2 = ds.TFRecordDataset(DATA_DIR, SCHEMA_DIR, columns_list=["image"], shuffle=False)
data2 = data2.map(operations=transform, input_columns=["image"])
# Compare with expected md5 from images
filename = "random_color_adjust_01_c_result.npz"
save_and_check_md5(data1, filename, generate_golden=GENERATE_GOLDEN)
filename = "random_color_adjust_01_py_result.npz"
save_and_check_md5(data2, filename, generate_golden=GENERATE_GOLDEN)
# Restore configuration
ds.config.set_seed(original_seed)
ds.config.set_num_parallel_workers(original_num_parallel_workers)
def get_de_dataset(args):
'''Get de_dataset.'''
transform_label = [C.TypeCast(mstype.int32)]
transform_img = [
VC.Decode(),
VC.Resize((96, 64)),
VC.RandomColorAdjust(brightness=0.3,
contrast=0.3,
saturation=0.3,
hue=0),
VC.RandomHorizontalFlip(),
VC.Normalize((127.5, 127.5, 127.5), (127.5, 127.5, 127.5)),
VC.HWC2CHW()
]
de_dataset = de.ImageFolderDataset(dataset_dir=args.data_dir,
num_shards=args.world_size,
shard_id=args.local_rank,
shuffle=True)
de_dataset = de_dataset.map(input_columns="image",
operations=transform_img)
de_dataset = de_dataset.map(input_columns="label",
operations=transform_label)
de_dataset = de_dataset.project(columns=["image", "label"])
de_dataset = de_dataset.batch(args.per_batch_size, drop_remainder=True)
num_iter_per_gpu = de_dataset.get_dataset_size()
de_dataset = de_dataset.repeat(args.max_epoch)
num_classes = de_dataset.num_classes()
return de_dataset, num_iter_per_gpu, num_classes
def test_cpp_uniform_augment(plot=False, num_ops=2):
"""
Test UniformAugment
"""
logger.info("Test CPP UniformAugment")
# Original Images
data_set = ds.ImageFolderDataset(dataset_dir=DATA_DIR, shuffle=False)
transforms_original = [C.Decode(), C.Resize(size=[224, 224]),
F.ToTensor()]
ds_original = data_set.map(operations=transforms_original, input_columns="image")
ds_original = ds_original.batch(512)
for idx, (image, _) in enumerate(ds_original):
if idx == 0:
images_original = np.transpose(image.asnumpy(), (0, 2, 3, 1))
else:
images_original = np.append(images_original,
np.transpose(image.asnumpy(), (0, 2, 3, 1)),
axis=0)
# UniformAugment Images
data_set = ds.ImageFolderDataset(dataset_dir=DATA_DIR, shuffle=False)
transforms_ua = [C.RandomCrop(size=[224, 224], padding=[32, 32, 32, 32]),
C.RandomHorizontalFlip(),
C.RandomVerticalFlip(),
C.RandomColorAdjust(),
C.RandomRotation(degrees=45)]
uni_aug = C.UniformAugment(transforms=transforms_ua, num_ops=num_ops)
transforms_all = [C.Decode(), C.Resize(size=[224, 224]),
uni_aug,
F.ToTensor()]
ds_ua = data_set.map(operations=transforms_all, input_columns="image", num_parallel_workers=1)
ds_ua = ds_ua.batch(512)
for idx, (image, _) in enumerate(ds_ua):
if idx == 0:
images_ua = np.transpose(image.asnumpy(), (0, 2, 3, 1))
else:
images_ua = np.append(images_ua,
np.transpose(image.asnumpy(), (0, 2, 3, 1)),
axis=0)
if plot:
visualize_list(images_original, images_ua)
num_samples = images_original.shape[0]
mse = np.zeros(num_samples)
for i in range(num_samples):
mse[i] = diff_mse(images_ua[i], images_original[i])
logger.info("MSE= {}".format(str(np.mean(mse))))
def create_dataset(dataset_path, config, do_train, repeat_num=1):
"""
create a train or eval dataset
Args:
dataset_path(string): the path of dataset.
config(dict): config of dataset.
do_train(bool): whether dataset is used for train or eval.
repeat_num(int): the repeat times of dataset. Default: 1.
Returns:
dataset
"""
rank = config.rank
group_size = config.group_size
if group_size == 1:
ds = de.ImageFolderDataset(dataset_path,
num_parallel_workers=config.work_nums,
shuffle=True)
else:
ds = de.ImageFolderDataset(dataset_path,
num_parallel_workers=config.work_nums,
shuffle=True,
num_shards=group_size,
shard_id=rank)
# define map operations
if do_train:
trans = [
C.RandomCropDecodeResize(config.image_size),
C.RandomHorizontalFlip(prob=0.5),
C.RandomColorAdjust(brightness=0.4, saturation=0.5) # fast mode
]
else:
trans = [
C.Decode(),
C.Resize(int(config.image_size / 0.875)),
C.CenterCrop(config.image_size)
]
trans += [
C.Rescale(1.0 / 255.0, 0.0),
C.Normalize(mean=[0.5, 0.5, 0.5], std=[0.5, 0.5, 0.5]),
C.HWC2CHW()
]
type_cast_op = C2.TypeCast(mstype.int32)
ds = ds.map(operations=trans,
input_columns="image",
num_parallel_workers=config.work_nums)
ds = ds.map(operations=type_cast_op,
input_columns="label",
num_parallel_workers=config.work_nums)
# apply batch operations
ds = ds.batch(config.batch_size, drop_remainder=True)
# apply dataset repeat operation
ds = ds.repeat(repeat_num)
return ds
def create_dataset(dataset_path,
do_train,
batch_size=16,
device_num=1,
rank=0):
"""
create a train or eval dataset
Args:
dataset_path(string): the path of dataset.
do_train(bool): whether dataset is used for train or eval.
batch_size(int): the batch size of dataset. Default: 16.
device_num (int): Number of shards that the dataset should be divided into (default=1).
rank (int): The shard ID within num_shards (default=0).
Returns:
dataset
"""
if device_num == 1:
ds = de.ImageFolderDataset(dataset_path,
num_parallel_workers=8,
shuffle=True)
else:
ds = de.ImageFolderDataset(dataset_path,
num_parallel_workers=8,
shuffle=True,
num_shards=device_num,
shard_id=rank)
# define map operations
if do_train:
trans = [
C.RandomCropDecodeResize(299),
C.RandomHorizontalFlip(prob=0.5),
C.RandomColorAdjust(brightness=0.4, contrast=0.4, saturation=0.4)
]
else:
trans = [C.Decode(), C.Resize(320), C.CenterCrop(299)]
trans += [
C.Normalize(mean=[127.5, 127.5, 127.5], std=[127.5, 127.5, 127.5]),
C.HWC2CHW(),
C2.TypeCast(mstype.float32)
]
type_cast_op = C2.TypeCast(mstype.int32)
ds = ds.map(input_columns="image",
operations=trans,
num_parallel_workers=8)
ds = ds.map(input_columns="label",
operations=type_cast_op,
num_parallel_workers=8)
# apply batch operations
ds = ds.batch(batch_size, drop_remainder=True)
return ds
def create_dataset(dataset_path, do_train, repeat_num=1, batch_size=32):
"""
Create a train or eval dataset.
Args:
dataset_path (str): The path of dataset.
do_train (bool): Whether dataset is used for train or eval.
repeat_num (int): The repeat times of dataset. Default: 1.
batch_size (int): The batch size of dataset. Default: 32.
Returns:
Dataset.
"""
do_shuffle = bool(do_train)
if device_num == 1 or not do_train:
ds = de.ImageFolderDataset(dataset_path, num_parallel_workers=config.work_nums, shuffle=do_shuffle)
else:
ds = de.ImageFolderDataset(dataset_path, num_parallel_workers=config.work_nums,
shuffle=do_shuffle, num_shards=device_num, shard_id=device_id)
image_length = 299
if do_train:
trans = [
C.RandomCropDecodeResize(image_length, scale=(0.08, 1.0), ratio=(0.75, 1.333)),
C.RandomHorizontalFlip(prob=0.5),
C.RandomColorAdjust(brightness=0.4, contrast=0.4, saturation=0.4)
]
else:
trans = [
C.Decode(),
C.Resize(image_length),
C.CenterCrop(image_length)
]
trans += [
C.Rescale(1.0 / 255.0, 0.0),
C.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]),
C.HWC2CHW()
]
type_cast_op = C2.TypeCast(mstype.int32)
ds = ds.map(input_columns="label", operations=type_cast_op, num_parallel_workers=config.work_nums)
ds = ds.map(input_columns="image", operations=trans, num_parallel_workers=config.work_nums)
# apply batch operations
ds = ds.batch(batch_size, drop_remainder=True)
# apply dataset repeat operation
ds = ds.repeat(repeat_num)
return ds
def create_dataset(dataset_path, do_train, rank, group_size, repeat_num=1):
"""
create a train or eval dataset
Args:
dataset_path(string): the path of dataset.
do_train(bool): whether dataset is used for train or eval.
rank (int): The shard ID within num_shards (default=None).
group_size (int): Number of shards that the dataset should be divided into (default=None).
repeat_num(int): the repeat times of dataset. Default: 1.
Returns:
dataset
"""
if group_size == 1:
ds = de.ImageFolderDataset(dataset_path,
num_parallel_workers=cfg.work_nums,
shuffle=True)
else:
ds = de.ImageFolderDataset(dataset_path,
num_parallel_workers=cfg.work_nums,
shuffle=True,
num_shards=group_size,
shard_id=rank)
# define map operations
if do_train:
trans = [
C.RandomCropDecodeResize(299,
scale=(0.08, 1.0),
ratio=(0.75, 1.333)),
C.RandomHorizontalFlip(prob=0.5),
C.RandomColorAdjust(brightness=0.4, contrast=0.4, saturation=0.4)
]
else:
trans = [C.Decode(), C.Resize(299), C.CenterCrop(299)]
trans += [
C.Rescale(1.0 / 255.0, 0.0),
C.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]),
C.HWC2CHW()
]
type_cast_op = C2.TypeCast(mstype.int32)
ds = ds.map(operations=trans,
input_columns="image",
num_parallel_workers=cfg.work_nums)
ds = ds.map(operations=type_cast_op,
input_columns="label",
num_parallel_workers=cfg.work_nums)
# apply batch operations
ds = ds.batch(cfg.batch_size, drop_remainder=True)
# apply dataset repeat operation
ds = ds.repeat(repeat_num)
return ds
def util_test_random_color_adjust_op(brightness=(1, 1), contrast=(1, 1), saturation=(1, 1), hue=(0, 0), plot=False):
"""
Util function that tests RandomColorAdjust for a specific argument
"""
# First dataset
data1 = ds.TFRecordDataset(DATA_DIR, SCHEMA_DIR, columns_list=["image"], shuffle=False)
decode_op = c_vision.Decode()
random_adjust_op = c_vision.RandomColorAdjust(brightness=brightness, contrast=contrast, saturation=saturation,
hue=hue)
ctrans = [decode_op,
random_adjust_op,
]
data1 = data1.map(operations=ctrans, input_columns=["image"])
# Second dataset
transforms = [
py_vision.Decode(),
py_vision.RandomColorAdjust(brightness=brightness, contrast=contrast, saturation=saturation,
hue=hue),
py_vision.ToTensor()
]
transform = mindspore.dataset.transforms.py_transforms.Compose(transforms)
data2 = ds.TFRecordDataset(DATA_DIR, SCHEMA_DIR, columns_list=["image"], shuffle=False)
data2 = data2.map(operations=transform, input_columns=["image"])
num_iter = 0
for item1, item2 in zip(data1.create_dict_iterator(num_epochs=1, output_numpy=True),
data2.create_dict_iterator(num_epochs=1, output_numpy=True)):
num_iter += 1
c_image = item1["image"]
py_image = (item2["image"].transpose(1, 2, 0) * 255).astype(np.uint8)
logger.info("shape of c_image: {}".format(c_image.shape))
logger.info("shape of py_image: {}".format(py_image.shape))
logger.info("dtype of c_image: {}".format(c_image.dtype))
logger.info("dtype of py_image: {}".format(py_image.dtype))
mse = diff_mse(c_image, py_image)
logger.info("mse is {}".format(mse))
logger.info("random_rotation_op_{}, mse: {}".format(num_iter + 1, mse))
assert mse < 0.01
if plot:
visualize_image(c_image, py_image, mse)
def create_dataset(dataset_path, do_train, device_num=1, rank=0):
"""
create a train or eval dataset
Args:
dataset_path(string): the path of dataset.
do_train(bool): whether dataset is used for train or eval.
rank (int): The shard ID within num_shards (default=None).
group_size (int): Number of shards that the dataset should be divided into (default=None).
repeat_num(int): the repeat times of dataset. Default: 1.
Returns:
dataset
"""
if device_num == 1:
data_set = ds.ImageFolderDataset(dataset_path,
num_parallel_workers=8,
shuffle=True)
else:
data_set = ds.ImageFolderDataset(dataset_path,
num_parallel_workers=8,
shuffle=True,
num_shards=device_num,
shard_id=rank)
# define map operations
if do_train:
trans = [
C.RandomCropDecodeResize(224),
C.RandomHorizontalFlip(prob=0.5),
C.RandomColorAdjust(brightness=0.4, contrast=0.4, saturation=0.4)
]
else:
trans = [C.Decode(), C.Resize(239), C.CenterCrop(224)]
trans += [
C.Normalize(mean=[0.485 * 255, 0.456 * 255, 0.406 * 255],
std=[0.229 * 255, 0.224 * 255, 0.225 * 255]),
C.HWC2CHW(),
]
type_cast_op = C2.TypeCast(mstype.int32)
data_set = data_set.map(input_columns="image",
operations=trans,
num_parallel_workers=8)
data_set = data_set.map(input_columns="label",
operations=type_cast_op,
num_parallel_workers=8)
# apply batch operations
data_set = data_set.batch(config.batch_size, drop_remainder=True)
return data_set
def create_dataset(data_path, is_train=True, batch_size=32):
# import
import mindspore.common.dtype as mstype
import mindspore.dataset.engine as de
import mindspore.dataset.transforms.c_transforms as C2
import mindspore.dataset.vision.c_transforms as C
from mindspore.common import set_seed
set_seed(1)
# shard
num_shards = shard_id = None
rand_size = os.getenv("RANK_SIZE")
rand_id = os.getenv("RANK_ID")
if rand_size is not None and rand_id is not None:
num_shards = int(rand_size)
shard_id = int(rand_id)
# define dataset
data_path = os.path.join(data_path, "train" if is_train else "val")
ds = de.ImageFolderDataset(data_path, shuffle=True, num_parallel_workers=8,
num_shards=num_shards, shard_id=shard_id, num_samples=None)
# define ops
comps_ops = list()
# train or val
if is_train:
comps_ops.append(C.RandomCropDecodeResize(224, scale=(0.08, 1.0), ratio=(0.75, 1.333)))
comps_ops.append(C.RandomHorizontalFlip(prob=0.5))
comps_ops.append(C.RandomColorAdjust(brightness=0.4, contrast=0.4, saturation=0.4))
else:
comps_ops.append(C.Decode())
comps_ops.append(C.Resize(224))
comps_ops.append(C.CenterCrop(224))
comps_ops.append(C.Rescale(1 / 255.0, 0.))
comps_ops.append(C.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]))
comps_ops.append(C.HWC2CHW())
# map ops
ds = ds.map(input_columns=["image"], operations=comps_ops, num_parallel_workers=8)
ds = ds.map(input_columns=["label"], operations=C2.TypeCast(mstype.int32), num_parallel_workers=8)
# batch & repeat
ds = ds.batch(batch_size=batch_size, drop_remainder=is_train)
ds = ds.repeat(count=1)
return ds
def test_cpp_uniform_augment_exception_float_numops(num_ops=2.5):
"""
Test UniformAugment invalid float number of ops
"""
logger.info("Test CPP UniformAugment invalid float num_ops exception")
transforms_ua = [C.RandomCrop(size=[224, 224], padding=[32, 32, 32, 32]),
C.RandomHorizontalFlip(),
C.RandomVerticalFlip(),
C.RandomColorAdjust(),
C.RandomRotation(degrees=45)]
try:
_ = C.UniformAugment(transforms=transforms_ua, num_ops=num_ops)
except Exception as e:
logger.info("Got an exception in DE: {}".format(str(e)))
assert "Argument num_ops with value 2.5 is not of type (<class 'int'>,)" in str(e)
def test_cpp_uniform_augment_exception_nonpositive_numops(num_ops=0):
"""
Test UniformAugment invalid non-positive number of ops
"""
logger.info("Test CPP UniformAugment invalid non-positive num_ops exception")
transforms_ua = [C.RandomCrop(size=[224, 224], padding=[32, 32, 32, 32]),
C.RandomHorizontalFlip(),
C.RandomVerticalFlip(),
C.RandomColorAdjust(),
C.RandomRotation(degrees=45)]
try:
_ = C.UniformAugment(transforms=transforms_ua, num_ops=num_ops)
except Exception as e:
logger.info("Got an exception in DE: {}".format(str(e)))
assert "Input num_ops must be greater than 0" in str(e)
def test_cpp_uniform_augment_exception_pyops(num_ops=2):
"""
Test UniformAugment invalid op in operations
"""
logger.info("Test CPP UniformAugment invalid OP exception")
transforms_ua = [C.RandomCrop(size=[224, 224], padding=[32, 32, 32, 32]),
C.RandomHorizontalFlip(),
C.RandomVerticalFlip(),
C.RandomColorAdjust(),
C.RandomRotation(degrees=45),
F.Invert()]
with pytest.raises(TypeError) as e:
C.UniformAugment(transforms=transforms_ua, num_ops=num_ops)
logger.info("Got an exception in DE: {}".format(str(e)))
assert "Type of Transforms[5] must be c_transform" in str(e.value)
def test_serdes_voc_dataset(remove_json_files=True):
"""
Test serdes on VOC dataset pipeline.
"""
data_dir = "../data/dataset/testVOC2012"
original_seed = config_get_set_seed(1)
original_num_parallel_workers = config_get_set_num_parallel_workers(1)
# define map operations
random_color_adjust_op = vision.RandomColorAdjust(brightness=(0.5, 0.5))
random_rotation_op = vision.RandomRotation((0, 90),
expand=True,
resample=Inter.BILINEAR,
center=(50, 50),
fill_value=150)
data1 = ds.VOCDataset(data_dir,
task="Detection",
usage="train",
decode=True)
data1 = data1.map(operations=random_color_adjust_op,
input_columns=["image"])
data1 = data1.map(operations=random_rotation_op, input_columns=["image"])
data1 = data1.skip(2)
data2 = util_check_serialize_deserialize_file(data1,
"voc_dataset_pipeline",
remove_json_files)
num_samples = 0
# Iterate and compare the data in the original pipeline (data1) against the deserialized pipeline (data2)
for item1, item2 in zip(
data1.create_dict_iterator(num_epochs=1, output_numpy=True),
data2.create_dict_iterator(num_epochs=1, output_numpy=True)):
np.testing.assert_array_equal(item1['image'], item2['image'])
num_samples += 1
assert num_samples == 7
# Restore configuration num_parallel_workers
ds.config.set_seed(original_seed)
ds.config.set_num_parallel_workers(original_num_parallel_workers)
def test_cpp_uniform_augment_exception_pyops(num_ops=2):
"""
Test UniformAugment invalid op in operations
"""
logger.info("Test CPP UniformAugment invalid OP exception")
transforms_ua = [C.RandomCrop(size=[224, 224], padding=[32, 32, 32, 32]),
C.RandomHorizontalFlip(),
C.RandomVerticalFlip(),
C.RandomColorAdjust(),
C.RandomRotation(degrees=45),
F.Invert()]
with pytest.raises(TypeError) as e:
C.UniformAugment(transforms=transforms_ua, num_ops=num_ops)
logger.info("Got an exception in DE: {}".format(str(e)))
assert "Argument transforms[5] with value" \
" <mindspore.dataset.vision.py_transforms.Invert" in str(e.value)
assert "is not of type (<class 'mindspore._c_dataengine.TensorOp'>,"\
" <class 'mindspore._c_dataengine.TensorOperation'>)" in str(e.value)
def create_dataset_cifar(dataset_path,
do_train,
repeat_num=1,
batch_size=32,
target="Ascend"):
"""
create a train or evaluate cifar10 dataset
Args:
dataset_path(string): the path of dataset.
do_train(bool): whether dataset is used for train or eval.
repeat_num(int): the repeat times of dataset. Default: 1
batch_size(int): the batch size of dataset. Default: 32
target(str): the device target. Default: Ascend
Returns:
dataset
"""
if target == "Ascend":
device_num, rank_id = _get_rank_info()
elif target == "CPU":
device_num = 1
else:
init()
rank_id = get_rank()
device_num = get_group_size()
if device_num == 1:
data_set = ds.Cifar10Dataset(dataset_path,
num_parallel_workers=8,
shuffle=True)
else:
data_set = ds.Cifar10Dataset(dataset_path,
num_parallel_workers=8,
shuffle=True,
num_shards=device_num,
shard_id=rank_id)
# define map operations
if do_train:
trans = [
C.RandomCrop((32, 32), (4, 4, 4, 4)),
C.RandomHorizontalFlip(prob=0.5),
C.RandomColorAdjust(brightness=0.4, contrast=0.4, saturation=0.4),
C.Resize((227, 227)),
C.Rescale(1.0 / 255.0, 0.0),
C.Normalize([0.4914, 0.4822, 0.4465], [0.2023, 0.1994, 0.2010]),
C.CutOut(112),
C.HWC2CHW()
]
else:
trans = [
C.Resize((227, 227)),
C.Rescale(1.0 / 255.0, 0.0),
C.Normalize([0.4914, 0.4822, 0.4465], [0.2023, 0.1994, 0.2010]),
C.HWC2CHW()
]
type_cast_op = C2.TypeCast(mstype.int32)
data_set = data_set.map(operations=type_cast_op,
input_columns="label",
num_parallel_workers=8)
data_set = data_set.map(operations=trans,
input_columns="image",
num_parallel_workers=8)
# apply batch operations
data_set = data_set.batch(batch_size, drop_remainder=True)
# apply dataset repeat operation
data_set = data_set.repeat(repeat_num)
return data_set
def create_dataset(dataset_path, config, buffer_size=1000):
"""
create a train or eval dataset
Args:
dataset_path(string): the path of dataset.
config(struct): the config of train and eval in diffirent platform.
repeat_num(int): the repeat times of dataset. Default: 1.
Returns:
train_dataset, val_dataset
"""
ds = de.ImageFolderDataset(dataset_path, num_parallel_workers=4)
train_ds, eval_ds = ds.split(config.data_split, randomize=True)
resize_height = config.image_height
resize_width = config.image_width
# define operations mapping to each sample
normalize_op = C.Normalize(mean=[0.485 * 255, 0.456 * 255, 0.406 * 255],
std=[0.229 * 255, 0.224 * 255, 0.225 * 255])
change_swap_op = C.HWC2CHW()
type_cast_op = C2.TypeCast(mstype.int32)
# operations for training
crop_decode_resize = C.RandomCropDecodeResize(resize_height,
scale=(0.08, 1.0),
ratio=(0.75, 1.333))
horizontal_flip_op = C.RandomHorizontalFlip(prob=0.5)
color_adjust = C.RandomColorAdjust(brightness=0.4,
contrast=0.4,
saturation=0.4)
# operations for inference
decode_op = C.Decode()
resize_op = C.Resize((256, 256))
center_crop = C.CenterCrop(resize_width)
train_trans = [
crop_decode_resize, horizontal_flip_op, color_adjust, normalize_op,
change_swap_op
]
train_ds = train_ds.map(input_columns="image",
operations=train_trans,
num_parallel_workers=4)
train_ds = train_ds.map(input_columns="label",
operations=type_cast_op,
num_parallel_workers=4)
eval_trans = [
decode_op, resize_op, center_crop, normalize_op, change_swap_op
]
eval_ds = eval_ds.map(input_columns="image",
operations=eval_trans,
num_parallel_workers=4)
eval_ds = eval_ds.map(input_columns="label",
operations=type_cast_op,
num_parallel_workers=4)
# apply shuffle operations
train_ds = train_ds.shuffle(buffer_size=buffer_size)
# apply batch operations
train_ds = train_ds.batch(config.batch_size, drop_remainder=True)
eval_ds = eval_ds.batch(config.eval_batch_size, drop_remainder=True)
return train_ds, eval_ds
def create_dataset_imagenet(dataset_path,
do_train,
repeat_num=1,
batch_size=32,
target="Ascend"):
"""
create a train or eval imagenet dataset
Args:
dataset_path(string): the path of dataset.
do_train(bool): whether dataset is used for train or eval.
repeat_num(int): the repeat times of dataset. Default: 1
batch_size(int): the batch size of dataset. Default: 32
target(str): the device target. Default: Ascend
Returns:
dataset
"""
if target == "Ascend":
device_num, rank_id = _get_rank_info()
else:
init()
rank_id = get_rank()
device_num = get_group_size()
if device_num == 1:
data_set = ds.ImageFolderDataset(dataset_path, shuffle=True)
else:
data_set = ds.ImageFolderDataset(dataset_path,
shuffle=True,
num_shards=device_num,
shard_id=rank_id)
image_size = 227
mean = [0.485 * 255, 0.456 * 255, 0.406 * 255]
std = [0.229 * 255, 0.224 * 255, 0.225 * 255]
# define map operations
if do_train:
trans = [
C.RandomCropDecodeResize(image_size,
scale=(0.08, 1.0),
ratio=(0.75, 1.333)),
C.RandomHorizontalFlip(prob=0.5),
C.RandomColorAdjust(brightness=0.4, contrast=0.4, saturation=0.4),
C.Normalize(mean=mean, std=std),
C.CutOut(112),
C.HWC2CHW()
]
else:
trans = [
C.Decode(),
C.Resize((256, 256)),
C.CenterCrop(image_size),
C.Normalize(mean=mean, std=std),
C.HWC2CHW()
]
type_cast_op = C2.TypeCast(mstype.int32)
data_set = data_set.map(operations=type_cast_op, input_columns="label")
data_set = data_set.map(operations=trans,
input_columns="image",
num_parallel_workers=10)
# apply batch operations
data_set = data_set.batch(batch_size, drop_remainder=True)
# apply dataset repeat operation
data_set = data_set.repeat(repeat_num)
return data_set
def create_dataset(dataset_path,
do_train,
config,
device_target,
repeat_num=1,
batch_size=32,
run_distribute=False):
"""
create a train or eval dataset
Args:
dataset_path(string): the path of dataset.
do_train(bool): whether dataset is used for train or eval.
repeat_num(int): the repeat times of dataset. Default: 1
batch_size(int): the batch size of dataset. Default: 32
Returns:
dataset
"""
if device_target == "GPU":
if do_train:
if run_distribute:
from mindspore.communication.management import get_rank, get_group_size
data_set = ds.ImageFolderDataset(dataset_path,
num_parallel_workers=8,
shuffle=True,
num_shards=get_group_size(),
shard_id=get_rank())
else:
data_set = ds.ImageFolderDataset(dataset_path,
num_parallel_workers=8,
shuffle=True)
else:
data_set = ds.ImageFolderDataset(dataset_path,
num_parallel_workers=8,
shuffle=True)
else:
raise ValueError("Unsupported device_target.")
resize_height = config.image_height
resize_width = config.image_width
buffer_size = 1000
# define map operations
decode_op = C.Decode()
resize_crop_op = C.RandomCropDecodeResize(resize_height,
scale=(0.08, 1.0),
ratio=(0.75, 1.333))
horizontal_flip_op = C.RandomHorizontalFlip(prob=0.5)
resize_op = C.Resize(256)
center_crop = C.CenterCrop(resize_width)
rescale_op = C.RandomColorAdjust(brightness=0.4,
contrast=0.4,
saturation=0.4)
normalize_op = C.Normalize(mean=[0.485 * 255, 0.456 * 255, 0.406 * 255],
std=[0.229 * 255, 0.224 * 255, 0.225 * 255])
change_swap_op = C.HWC2CHW()
if do_train:
trans = [
resize_crop_op, horizontal_flip_op, rescale_op, normalize_op,
change_swap_op
]
else:
trans = [
decode_op, resize_op, center_crop, normalize_op, change_swap_op
]
type_cast_op = C2.TypeCast(mstype.int32)
data_set = data_set.map(operations=trans,
input_columns="image",
num_parallel_workers=8)
data_set = data_set.map(operations=type_cast_op,
input_columns="label",
num_parallel_workers=8)
# apply shuffle operations
data_set = data_set.shuffle(buffer_size=buffer_size)
# apply batch operations
data_set = data_set.batch(batch_size, drop_remainder=True)
# apply dataset repeat operation
data_set = data_set.repeat(repeat_num)
return data_set
def create_dataset(dataset_path, do_train, config, repeat_num=1):
"""
create a train or eval dataset
Args:
dataset_path(string): the path of dataset.
do_train(bool): whether dataset is used for train or eval.
config(struct): the config of train and eval in diffirent platform.
repeat_num(int): the repeat times of dataset. Default: 1.
Returns:
dataset
"""
if config.platform == "Ascend":
rank_size = int(os.getenv("RANK_SIZE", '1'))
rank_id = int(os.getenv("RANK_ID", '0'))
if rank_size == 1:
ds = de.ImageFolderDataset(dataset_path,
num_parallel_workers=8,
shuffle=True)
else:
ds = de.ImageFolderDataset(dataset_path,
num_parallel_workers=8,
shuffle=True,
num_shards=rank_size,
shard_id=rank_id)
elif config.platform == "GPU":
if do_train:
if config.run_distribute:
from mindspore.communication.management import get_rank, get_group_size
ds = de.ImageFolderDataset(dataset_path,
num_parallel_workers=8,
shuffle=True,
num_shards=get_group_size(),
shard_id=get_rank())
else:
ds = de.ImageFolderDataset(dataset_path,
num_parallel_workers=8,
shuffle=True)
else:
ds = de.ImageFolderDataset(dataset_path,
num_parallel_workers=8,
shuffle=True)
elif config.platform == "CPU":
ds = de.ImageFolderDataset(dataset_path,
num_parallel_workers=8,
shuffle=True)
resize_height = config.image_height
resize_width = config.image_width
buffer_size = 1000
# define map operations
decode_op = C.Decode()
resize_crop_op = C.RandomCropDecodeResize(resize_height,
scale=(0.08, 1.0),
ratio=(0.75, 1.333))
horizontal_flip_op = C.RandomHorizontalFlip(prob=0.5)
resize_op = C.Resize((256, 256))
center_crop = C.CenterCrop(resize_width)
rescale_op = C.RandomColorAdjust(brightness=0.4,
contrast=0.4,
saturation=0.4)
normalize_op = C.Normalize(mean=[0.485 * 255, 0.456 * 255, 0.406 * 255],
std=[0.229 * 255, 0.224 * 255, 0.225 * 255])
change_swap_op = C.HWC2CHW()
if do_train:
trans = [
resize_crop_op, horizontal_flip_op, rescale_op, normalize_op,
change_swap_op
]
else:
trans = [
decode_op, resize_op, center_crop, normalize_op, change_swap_op
]
type_cast_op = C2.TypeCast(mstype.int32)
ds = ds.map(operations=trans,
input_columns="image",
num_parallel_workers=8)
ds = ds.map(operations=type_cast_op,
input_columns="label",
num_parallel_workers=8)
# apply shuffle operations
ds = ds.shuffle(buffer_size=buffer_size)
# apply batch operations
ds = ds.batch(config.batch_size, drop_remainder=True)
# apply dataset repeat operation
ds = ds.repeat(repeat_num)
return ds
def create_dataset(args, dataset_mode, repeat_num=1):
"""
create a train or evaluate cifar10 dataset for SimCLR
"""
if args.dataset_name != "cifar10":
raise ValueError("Unsupported dataset.")
if dataset_mode in ("train_endcoder", "train_classifier"):
dataset_path = args.train_dataset_path
else:
dataset_path = args.eval_dataset_path
if args.run_distribute and args.device_target == "Ascend":
data_set = ds.Cifar10Dataset(dataset_path,
num_parallel_workers=8,
shuffle=True,
num_shards=args.device_num,
shard_id=args.device_id)
else:
data_set = ds.Cifar10Dataset(dataset_path,
num_parallel_workers=8,
shuffle=True)
# define map operations
trans = []
if dataset_mode == "train_endcoder":
if args.use_crop:
trans += [C.Resize(256, interpolation=Inter.BICUBIC)]
trans += [
C.RandomResizedCrop(size=(32, 32),
scale=(0.31, 1),
interpolation=Inter.BICUBIC,
max_attempts=100)
]
if args.use_flip:
trans += [C.RandomHorizontalFlip(prob=0.5)]
if args.use_color_jitter:
scale = 0.6
color_jitter = C.RandomColorAdjust(0.8 * scale, 0.8 * scale,
0.8 * scale, 0.2 * scale)
trans += [C2.RandomApply([color_jitter], prob=0.8)]
if args.use_color_gray:
trans += [
py_vision.ToPIL(),
py_vision.RandomGrayscale(prob=0.2), np.array
] # need to convert PIL image to a NumPy array to pass it to C++ operation
if args.use_blur:
trans += [C2.RandomApply([gaussian_blur], prob=0.8)]
if args.use_norm:
trans += [
C.Normalize([0.4914, 0.4822, 0.4465], [0.2023, 0.1994, 0.2010])
]
trans += [C2.TypeCast(mstype.float32), C.HWC2CHW()]
else:
trans += [C.Resize(32)]
trans += [C2.TypeCast(mstype.float32)]
if args.use_norm:
trans += [
C.Normalize([0.4914, 0.4822, 0.4465], [0.2023, 0.1994, 0.2010])
]
trans += [C.HWC2CHW()]
type_cast_op = C2.TypeCast(mstype.int32)
data_set = data_set.map(operations=type_cast_op,
input_columns="label",
num_parallel_workers=8)
data_set = data_set.map(operations=copy_column,
input_columns=["image", "label"],
output_columns=["image1", "image2", "label"],
column_order=["image1", "image2", "label"],
num_parallel_workers=8)
data_set = data_set.map(operations=trans,
input_columns=["image1"],
num_parallel_workers=8)
data_set = data_set.map(operations=trans,
input_columns=["image2"],
num_parallel_workers=8)
# apply batch operations
data_set = data_set.batch(args.batch_size, drop_remainder=True)
# apply dataset repeat operation
data_set = data_set.repeat(repeat_num)
return data_set
def classification_dataset(data_dir,
image_size,
per_batch_size,
max_epoch,
rank,
group_size,
mode='train',
input_mode='folder',
root='',
num_parallel_workers=None,
shuffle=None,
sampler=None,
class_indexing=None,
drop_remainder=True,
transform=None,
target_transform=None):
"""
A function that returns a dataset for classification. The mode of input dataset could be "folder" or "txt".
If it is "folder", all images within one folder have the same label. If it is "txt", all paths of images
are written into a textfile.
Args:
data_dir (str): Path to the root directory that contains the dataset for "input_mode="folder"".
Or path of the textfile that contains every image's path of the dataset.
image_size (Union(int, sequence)): Size of the input images.
per_batch_size (int): the batch size of evey step during training.
max_epoch (int): the number of epochs.
rank (int): The shard ID within num_shards (default=None).
group_size (int): Number of shards that the dataset should be divided
into (default=None).
mode (str): "train" or others. Default: " train".
input_mode (str): The form of the input dataset. "folder" or "txt". Default: "folder".
root (str): the images path for "input_mode="txt"". Default: " ".
num_parallel_workers (int): Number of workers to read the data. Default: None.
shuffle (bool): Whether or not to perform shuffle on the dataset
(default=None, performs shuffle).
sampler (Sampler): Object used to choose samples from the dataset. Default: None.
class_indexing (dict): A str-to-int mapping from folder name to index
(default=None, the folder names will be sorted
alphabetically and each class will be given a
unique index starting from 0).
Examples:
>>> from src.dataset import classification_dataset
>>> # path to imagefolder directory. This directory needs to contain sub-directories which contain the images
>>> data_dir = "/path/to/imagefolder_directory"
>>> de_dataset = classification_dataset(data_dir, image_size=[224, 244],
>>> per_batch_size=64, max_epoch=100,
>>> rank=0, group_size=4)
>>> # Path of the textfile that contains every image's path of the dataset.
>>> data_dir = "/path/to/dataset/images/train.txt"
>>> images_dir = "/path/to/dataset/images"
>>> de_dataset = classification_dataset(data_dir, image_size=[224, 244],
>>> per_batch_size=64, max_epoch=100,
>>> rank=0, group_size=4,
>>> input_mode="txt", root=images_dir)
"""
mean = [0.485 * 255, 0.456 * 255, 0.406 * 255]
std = [0.229 * 255, 0.224 * 255, 0.225 * 255]
if transform is None:
if mode == 'train':
transform_img = [
V_C.RandomCropDecodeResize(image_size,
scale=(0.08, 1.0),
ratio=(0.75, 1.333)),
V_C.RandomHorizontalFlip(prob=0.5),
V_C.RandomColorAdjust(brightness=0.4,
contrast=0.4,
saturation=0.4),
V_C.Normalize(mean=mean, std=std),
V_C.HWC2CHW()
]
else:
transform_img = [
V_C.Decode(),
V_C.Resize((256, 256)),
V_C.CenterCrop(image_size),
V_C.Normalize(mean=mean, std=std),
V_C.HWC2CHW()
]
else:
transform_img = transform
if target_transform is None:
transform_label = [C.TypeCast(mstype.int32)]
else:
transform_label = target_transform
if input_mode == 'folder':
de_dataset = de.ImageFolderDataset(
data_dir,
num_parallel_workers=num_parallel_workers,
shuffle=shuffle,
sampler=sampler,
class_indexing=class_indexing,
num_shards=group_size,
shard_id=rank)
else:
dataset = TxtDataset(root, data_dir)
sampler = DistributedSampler(dataset,
rank,
group_size,
shuffle=shuffle)
de_dataset = de.GeneratorDataset(dataset, ["image", "label"],
sampler=sampler)
de_dataset = de_dataset.map(operations=transform_img,
input_columns="image",
num_parallel_workers=num_parallel_workers)
de_dataset = de_dataset.map(operations=transform_label,
input_columns="label",
num_parallel_workers=num_parallel_workers)
columns_to_project = ["image", "label"]
de_dataset = de_dataset.project(columns=columns_to_project)
de_dataset = de_dataset.batch(per_batch_size,
drop_remainder=drop_remainder)
de_dataset = de_dataset.repeat(max_epoch)
return de_dataset
def classification_dataset_cifar10(data_dir,
image_size,
per_batch_size,
max_epoch,
rank,
group_size,
mode='train',
num_parallel_workers=None,
shuffle=None,
sampler=None,
drop_remainder=True,
transform=None,
target_transform=None):
"""
A function that returns cifar10 dataset for classification.
Args:
data_dir (str): Path to the root directory that contains the dataset's bin files.
image_size (Union(int, sequence)): Size of the input images.
per_batch_size (int): the batch size of evey step during training.
max_epoch (int): the number of epochs.
rank (int): The shard ID within num_shards (default=None).
group_size (int): Number of shards that the dataset should be divided
into (default=None).
mode (str): "train" or others. Default: " train".
input_mode (str): The form of the input dataset. "folder" or "txt". Default: "folder".
root (str): the images path for "input_mode="txt"". Default: " ".
num_parallel_workers (int): Number of workers to read the data. Default: None.
shuffle (bool): Whether or not to perform shuffle on the dataset
(default=None, performs shuffle).
sampler (Sampler): Object used to choose samples from the dataset. Default: None.
Examples:
>>> from src.datasets.classification import classification_dataset_cifar10
>>> # path to imagefolder directory. This directory needs to contain bin files of data.
>>> data_dir = "/path/to/datafolder_directory"
>>> de_dataset = classification_dataset_cifar10(data_dir, image_size=[32, 32],
>>> per_batch_size=64, max_epoch=100,
>>> rank=0, group_size=1)
"""
mean = [0.5 * 255, 0.5 * 255, 0.5 * 255]
std = [0.5 * 255, 0.5 * 255, 0.5 * 255]
if transform is None:
if mode == 'train':
transform_img = [
vision_C.RandomCrop(image_size, padding=4),
vision_C.RandomHorizontalFlip(prob=0.5),
vision_C.RandomColorAdjust(brightness=0.4,
contrast=0.4,
saturation=0.4),
vision_C.Normalize(mean=mean, std=std),
vision_C.HWC2CHW()
]
else:
transform_img = [
vision_C.Normalize(mean=mean, std=std),
vision_C.HWC2CHW()
]
else:
transform_img = transform
if target_transform is None:
transform_label = [normal_C.TypeCast(mstype.int32)]
else:
transform_label = target_transform
de_dataset = de.Cifar10Dataset(data_dir,
num_parallel_workers=num_parallel_workers,
shuffle=shuffle,
sampler=sampler,
num_shards=group_size,
shard_id=rank)
de_dataset = de_dataset.map(input_columns="image",
num_parallel_workers=8,
operations=transform_img)
de_dataset = de_dataset.map(input_columns="label",
num_parallel_workers=8,
operations=transform_label)
columns_to_project = ["image", "label"]
de_dataset = de_dataset.project(columns=columns_to_project)
de_dataset = de_dataset.batch(per_batch_size,
drop_remainder=drop_remainder)
de_dataset = de_dataset.repeat(1)
return de_dataset
def create_dataset_imagenet(dataset_path,
repeat_num=1,
training=True,
num_parallel_workers=None,
shuffle=None):
"""
create a train or eval imagenet2012 dataset for resnet50
Args:
dataset_path(string): the path of dataset.
do_train(bool): whether dataset is used for train or eval.
repeat_num(int): the repeat times of dataset. Default: 1
batch_size(int): the batch size of dataset. Default: 32
target(str): the device target. Default: Ascend
Returns:
dataset
"""
device_num, rank_id = _get_rank_info()
if device_num == 1:
data_set = ds.ImageFolderDataset(
dataset_path,
num_parallel_workers=num_parallel_workers,
shuffle=shuffle)
else:
data_set = ds.ImageFolderDataset(
dataset_path,
num_parallel_workers=num_parallel_workers,
shuffle=shuffle,
num_shards=device_num,
shard_id=rank_id)
assert imagenet_cfg.image_height == imagenet_cfg.image_width, "image_height not equal image_width"
image_size = imagenet_cfg.image_height
mean = [0.485 * 255, 0.456 * 255, 0.406 * 255]
std = [0.229 * 255, 0.224 * 255, 0.225 * 255]
# define map operations
if training:
transform_img = [
vision.RandomCropDecodeResize(image_size,
scale=(0.08, 1.0),
ratio=(0.75, 1.333)),
vision.RandomHorizontalFlip(prob=0.5),
vision.RandomColorAdjust(0.4, 0.4, 0.4, 0.1),
vision.Normalize(mean=mean, std=std),
vision.HWC2CHW()
]
else:
transform_img = [
vision.Decode(),
vision.Resize(256),
vision.CenterCrop(image_size),
vision.Normalize(mean=mean, std=std),
vision.HWC2CHW()
]
transform_label = [C.TypeCast(mstype.int32)]
data_set = data_set.map(input_columns="image",
num_parallel_workers=12,
operations=transform_img)
data_set = data_set.map(input_columns="label",
num_parallel_workers=4,
operations=transform_label)
# apply batch operations
data_set = data_set.batch(imagenet_cfg.batch_size, drop_remainder=True)
# apply dataset repeat operation
data_set = data_set.repeat(repeat_num)
return data_set
def create_dataset(dataset_path,
do_train,
config,
platform,
repeat_num=1,
batch_size=100):
"""
create a train or eval dataset
Args:
dataset_path(string): the path of dataset.
do_train(bool): whether dataset is used for train or eval.
repeat_num(int): the repeat times of dataset. Default: 1
batch_size(int): the batch size of dataset. Default: 32
Returns:
dataset
"""
if platform == "Ascend":
rank_size = int(os.getenv("RANK_SIZE"))
rank_id = int(os.getenv("RANK_ID"))
if rank_size == 1:
data_set = ds.MindDataset(dataset_path,
num_parallel_workers=8,
shuffle=True)
else:
data_set = ds.MindDataset(dataset_path,
num_parallel_workers=8,
shuffle=True,
num_shards=rank_size,
shard_id=rank_id)
elif platform == "GPU":
if do_train:
from mindspore.communication.management import get_rank, get_group_size
data_set = ds.MindDataset(dataset_path,
num_parallel_workers=8,
shuffle=True,
num_shards=get_group_size(),
shard_id=get_rank())
else:
data_set = ds.MindDataset(dataset_path,
num_parallel_workers=8,
shuffle=False)
else:
raise ValueError("Unsupported platform.")
resize_height = config.image_height
buffer_size = 1000
# define map operations
resize_crop_op = C.RandomCropDecodeResize(resize_height,
scale=(0.08, 1.0),
ratio=(0.75, 1.333))
horizontal_flip_op = C.RandomHorizontalFlip(prob=0.5)
color_op = C.RandomColorAdjust(brightness=0.4,
contrast=0.4,
saturation=0.4)
rescale_op = C.Rescale(1 / 255.0, 0)
normalize_op = C.Normalize(mean=[0.485, 0.456, 0.406],
std=[0.229, 0.224, 0.225])
change_swap_op = C.HWC2CHW()
# define python operations
decode_p = P.Decode()
resize_p = P.Resize(256, interpolation=Inter.BILINEAR)
center_crop_p = P.CenterCrop(224)
totensor = P.ToTensor()
normalize_p = P.Normalize((0.485, 0.456, 0.406), (0.229, 0.224, 0.225))
composeop = P2.Compose(
[decode_p, resize_p, center_crop_p, totensor, normalize_p])
if do_train:
trans = [
resize_crop_op, horizontal_flip_op, color_op, rescale_op,
normalize_op, change_swap_op
]
else:
trans = composeop
type_cast_op = C2.TypeCast(mstype.int32)
data_set = data_set.map(input_columns="image",
operations=trans,
num_parallel_workers=8)
data_set = data_set.map(input_columns="label_list",
operations=type_cast_op,
num_parallel_workers=8)
# apply shuffle operations
data_set = data_set.shuffle(buffer_size=buffer_size)
# apply batch operations
data_set = data_set.batch(batch_size, drop_remainder=True)
# apply dataset repeat operation
data_set = data_set.repeat(repeat_num)
return data_set
