def create_dataset(repeat_num=1, training=True):
data_dir = data_home + "/cifar-10-batches-bin"
if not training:
data_dir = data_home + "/cifar-10-verify-bin"
ds = de.Cifar10Dataset(data_dir)
if args_opt.run_distribute:
rank_id = int(os.getenv('RANK_ID'))
rank_size = int(os.getenv('RANK_SIZE'))
ds = de.Cifar10Dataset(data_dir,
num_shards=rank_size,
shard_id=rank_id)
resize_height = 224
resize_width = 224
rescale = 1.0 / 255.0
shift = 0.0
# define map operations
random_crop_op = vision.RandomCrop(
(32, 32), (4, 4, 4, 4)) # padding_mode default CONSTANT
random_horizontal_op = vision.RandomHorizontalFlip()
resize_op = vision.Resize(
(resize_height, resize_width)) # interpolation default BILINEAR
rescale_op = vision.Rescale(rescale, shift)
normalize_op = vision.Normalize((0.4465, 0.4822, 0.4914),
(0.2010, 0.1994, 0.2023))
changeswap_op = vision.HWC2CHW()
type_cast_op = C.TypeCast(mstype.int32)
c_trans = []
if training:
c_trans = [random_crop_op, random_horizontal_op]
c_trans += [resize_op, rescale_op, normalize_op, changeswap_op]
# apply map operations on images
ds = ds.map(input_columns="label", operations=type_cast_op)
ds = ds.map(input_columns="image", operations=c_trans)
# apply repeat operations
ds = ds.repeat(repeat_num)
# apply shuffle operations
ds = ds.shuffle(buffer_size=10)
# apply batch operations
ds = ds.batch(batch_size=args_opt.batch_size, drop_remainder=True)
return ds
def test_cache_nomap_disallowed_share1():
"""
It is not allowed to share the cache between the following two trees:
Cache Cache
| |
Map(decode) Map(rescale)
| |
TFReader TFReader
"""
logger.info("Test cache nomap disallowed share1")
# This dataset has 3 records in it only
some_cache = ds.DatasetCache(session_id=1, size=0, spilling=True)
decode_op = c_vision.Decode()
rescale_op = c_vision.Rescale(1.0 / 255.0, -1.0)
ds1 = ds.TFRecordDataset(DATA_DIR,
SCHEMA_DIR,
columns_list=["image"],
shuffle=False)
ds1 = ds1.map(input_columns=["image"],
operations=decode_op,
cache=some_cache)
ds2 = ds.TFRecordDataset(DATA_DIR,
SCHEMA_DIR,
columns_list=["image"],
shuffle=False)
ds2 = ds2.map(input_columns=["image"],
operations=rescale_op,
cache=some_cache)
num_iter = 0
for _ in ds1.create_dict_iterator():
num_iter += 1
logger.info("Number of data in ds1: {} ".format(num_iter))
assert num_iter == 3
try:
sum([1 for _ in ds2])
except RuntimeError as e:
logger.info("Got an exception in DE: {}".format(str(e)))
assert "Attempt to re-use a cache for a different tree!" in str(e)
logger.info("test_cache_nomap_disallowed_share1 Ended.\n")
def test_case_3():
"""
Test Map
"""
logger.info("Test Map Rescale and Resize, then Shuffle")
data1 = ds.TFRecordDataset(DATA_DIR, SCHEMA_DIR, shuffle=False)
# define data augmentation parameters
rescale = 1.0 / 255.0
shift = 0.0
resize_height, resize_width = 224, 224
# define map operations
decode_op = vision.Decode()
rescale_op = vision.Rescale(rescale, shift)
# resize_op = vision.Resize(resize_height, resize_width,
# InterpolationMode.DE_INTER_LINEAR) # Bilinear mode
resize_op = vision.Resize((resize_height, resize_width))
# apply map operations on images
data1 = data1.map(input_columns=["image"], operations=decode_op)
data1 = data1.map(input_columns=["image"], operations=rescale_op)
data1 = data1.map(input_columns=["image"], operations=resize_op)
# # apply ont-hot encoding on labels
num_classes = 4
one_hot_encode = data_trans.OneHot(
num_classes) # num_classes is input argument
data1 = data1.map(input_columns=["label"], operations=one_hot_encode)
#
# # apply Datasets
buffer_size = 100
seed = 10
batch_size = 2
ds.config.set_seed(seed)
data1 = data1.shuffle(
buffer_size=buffer_size) # 10000 as in imageNet train script
data1 = data1.batch(batch_size, drop_remainder=True)
num_iter = 0
for item in data1.create_dict_iterator(): # each data is a dictionary
# in this example, each dictionary has keys "image" and "label"
logger.info("image is: {}".format(item["image"]))
logger.info("label is: {}".format(item["label"]))
num_iter += 1
logger.info("Number of data in data1: {}".format(num_iter))
def vgg_create_dataset(data_home, repeat_num=1, training=True):
"""Data operations."""
ds.config.set_seed(1)
data_dir = os.path.join(data_home, "cifar-10-batches-bin")
if not training:
data_dir = os.path.join(data_home, "cifar-10-verify-bin")
rank_size = int(os.environ.get("RANK_SIZE")) if os.environ.get("RANK_SIZE") else None
rank_id = int(os.environ.get("RANK_ID")) if os.environ.get("RANK_ID") else None
data_set = ds.Cifar10Dataset(data_dir, num_shards=rank_size, shard_id=rank_id)
resize_height = cfg.image_height
resize_width = cfg.image_width
rescale = 1.0 / 255.0
shift = 0.0
# define map operations
random_crop_op = vision.RandomCrop((32, 32), (4, 4, 4, 4)) # padding_mode default CONSTANT
random_horizontal_op = vision.RandomHorizontalFlip()
resize_op = vision.Resize((resize_height, resize_width)) # interpolation default BILINEAR
rescale_op = vision.Rescale(rescale, shift)
normalize_op = vision.Normalize((0.4465, 0.4822, 0.4914), (0.2010, 0.1994, 0.2023))
changeswap_op = vision.HWC2CHW()
type_cast_op = C.TypeCast(mstype.int32)
c_trans = []
if training:
c_trans = [random_crop_op, random_horizontal_op]
c_trans += [resize_op, rescale_op, normalize_op,
changeswap_op]
# apply map operations on images
data_set = data_set.map(input_columns="label", operations=type_cast_op)
data_set = data_set.map(input_columns="image", operations=c_trans)
# apply repeat operations
data_set = data_set.repeat(repeat_num)
# apply shuffle operations
data_set = data_set.shuffle(buffer_size=10)
# apply batch operations
data_set = data_set.batch(batch_size=cfg.batch_size, drop_remainder=True)
return data_set
def create_dataset(data_path,
repeat_num=1,
batch_size=32,
rank_id=0,
rank_size=1):
resize_height = 224
resize_width = 224
rescale = 1.0 / 255.0
shift = 0.0
# get rank_id and rank_size
rank_id = get_rank()
rank_size = get_group_size()
data_set = ds.Cifar10Dataset(data_path,
num_shards=rank_size,
shard_id=rank_id)
# define map operations
random_crop_op = vision.RandomCrop((32, 32), (4, 4, 4, 4))
random_horizontal_op = vision.RandomHorizontalFlip()
resize_op = vision.Resize((resize_height, resize_width))
rescale_op = vision.Rescale(rescale, shift)
normalize_op = vision.Normalize((0.4465, 0.4822, 0.4914),
(0.2010, 0.1994, 0.2023))
changeswap_op = vision.HWC2CHW()
type_cast_op = C.TypeCast(mstype.int32)
c_trans = [random_crop_op, random_horizontal_op]
c_trans += [resize_op, rescale_op, normalize_op, changeswap_op]
# apply map operations on images
data_set = data_set.map(input_columns="label", operations=type_cast_op)
data_set = data_set.map(input_columns="image", operations=c_trans)
# apply shuffle operations
data_set = data_set.shuffle(buffer_size=10)
# apply batch operations
data_set = data_set.batch(batch_size=batch_size, drop_remainder=True)
# apply repeat operations
data_set = data_set.repeat(repeat_num)
return data_set
def create_cifar10_dataset(cifar_dir, num_parallel_workers=1):
"""
Creat the cifar10 dataset.
"""
ds = de.Cifar10Dataset(cifar_dir)
training = True
resize_height = 224
resize_width = 224
rescale = 1.0 / 255.0
shift = 0.0
repeat_num = 10
batch_size = 32
# define map operations
random_crop_op = vision.RandomCrop(
(32, 32), (4, 4, 4, 4)) # padding_mode default CONSTANT
random_horizontal_op = vision.RandomHorizontalFlip()
resize_op = vision.Resize(
(resize_height, resize_width)) # interpolation default BILINEAR
rescale_op = vision.Rescale(rescale, shift)
normalize_op = vision.Normalize((0.4465, 0.4822, 0.4914),
(0.2010, 0.1994, 0.2023))
changeswap_op = vision.HWC2CHW()
type_cast_op = C.TypeCast(mstype.int32)
if training:
c_trans = [random_crop_op, random_horizontal_op]
c_trans += [resize_op, rescale_op, normalize_op, changeswap_op]
# apply map operations on images
ds = ds.map(input_columns="label", operations=type_cast_op)
ds = ds.map(input_columns="image", operations=c_trans)
# apply repeat operations
ds = ds.repeat(repeat_num)
# apply shuffle operations
ds = ds.shuffle(buffer_size=10)
# apply batch operations
ds = ds.batch(batch_size=batch_size, drop_remainder=True)
return ds
def create_dataset(repeat_num=1, training=True, batch_size=32):
data_home = "/home/workspace/mindspore_dataset"
data_dir = data_home + "/cifar-10-batches-bin"
if not training:
data_dir = data_home + "/cifar-10-verify-bin"
data_set = ds.Cifar10Dataset(data_dir)
resize_height = 224
resize_width = 224
rescale = 1.0 / 255.0
shift = 0.0
# define map operations
random_crop_op = vision.RandomCrop(
(32, 32), (4, 4, 4, 4)) # padding_mode default CONSTANT
random_horizontal_op = vision.RandomHorizontalFlip()
# interpolation default BILINEAR
resize_op = vision.Resize((resize_height, resize_width))
rescale_op = vision.Rescale(rescale, shift)
normalize_op = vision.Normalize((0.4465, 0.4822, 0.4914),
(0.2010, 0.1994, 0.2023))
changeswap_op = vision.HWC2CHW()
type_cast_op = C.TypeCast(mstype.int32)
c_trans = []
if training:
c_trans = [random_crop_op, random_horizontal_op]
c_trans += [resize_op, rescale_op, normalize_op, changeswap_op]
# apply map operations on images
data_set = data_set.map(input_columns="label", operations=type_cast_op)
data_set = data_set.map(input_columns="image", operations=c_trans)
# apply shuffle operations
data_set = data_set.shuffle(buffer_size=1000)
# apply batch operations
data_set = data_set.batch(batch_size=batch_size, drop_remainder=True)
# apply repeat operations
data_set = data_set.repeat(repeat_num)
return data_set
def test_cutmix_batch_success2(plot=False):
"""
Test CutMixBatch op with default values for alpha and prob on a batch of rescaled HWC images
"""
logger.info("test_cutmix_batch_success2")
# Original Images
ds_original = ds.Cifar10Dataset(DATA_DIR, num_samples=10, shuffle=False)
ds_original = ds_original.batch(5, drop_remainder=True)
images_original = None
for idx, (image, _) in enumerate(ds_original):
if idx == 0:
images_original = image
else:
images_original = np.append(images_original, image, axis=0)
# CutMix Images
data1 = ds.Cifar10Dataset(DATA_DIR, num_samples=10, shuffle=False)
one_hot_op = data_trans.OneHot(num_classes=10)
data1 = data1.map(input_columns=["label"], operations=one_hot_op)
rescale_op = vision.Rescale((1.0 / 255.0), 0.0)
data1 = data1.map(input_columns=["image"], operations=rescale_op)
cutmix_batch_op = vision.CutMixBatch(mode.ImageBatchFormat.NHWC)
data1 = data1.batch(5, drop_remainder=True)
data1 = data1.map(input_columns=["image", "label"],
operations=cutmix_batch_op)
images_cutmix = None
for idx, (image, _) in enumerate(data1):
if idx == 0:
images_cutmix = image
else:
images_cutmix = np.append(images_cutmix, image, axis=0)
if plot:
visualize_list(images_original, images_cutmix)
num_samples = images_original.shape[0]
mse = np.zeros(num_samples)
for i in range(num_samples):
mse[i] = diff_mse(images_cutmix[i], images_original[i])
logger.info("MSE= {}".format(str(np.mean(mse))))
def test_one_hot_post_aug():
"""
Test One Hot Encoding after Multiple Data Augmentation Operators
"""
logger.info("test_one_hot_post_aug")
data1 = ds.TFRecordDataset(DATA_DIR, SCHEMA_DIR, shuffle=False)
# Define data augmentation parameters
rescale = 1.0 / 255.0
shift = 0.0
resize_height, resize_width = 224, 224
# Define map operations
decode_op = c_vision.Decode()
rescale_op = c_vision.Rescale(rescale, shift)
resize_op = c_vision.Resize((resize_height, resize_width))
# Apply map operations on images
data1 = data1.map(input_columns=["image"], operations=decode_op)
data1 = data1.map(input_columns=["image"], operations=rescale_op)
data1 = data1.map(input_columns=["image"], operations=resize_op)
# Apply one-hot encoding on labels
depth = 4
one_hot_encode = data_trans.OneHot(depth)
data1 = data1.map(input_columns=["label"], operations=one_hot_encode)
# Apply datasets ops
buffer_size = 100
seed = 10
batch_size = 2
ds.config.set_seed(seed)
data1 = data1.shuffle(buffer_size=buffer_size)
data1 = data1.batch(batch_size, drop_remainder=True)
num_iter = 0
for item in data1.create_dict_iterator():
logger.info("image is: {}".format(item["image"]))
logger.info("label is: {}".format(item["label"]))
num_iter += 1
assert num_iter == 1
def generate_mnist_dataset(data_path, batch_size=32, repeat_size=1,
num_parallel_workers=1, sparse=True):
"""
create dataset for training or testing
"""
# define dataset
ds1 = ds.MnistDataset(data_path)
# define operation parameters
resize_height, resize_width = 32, 32
rescale = 1.0 / 255.0
shift = 0.0
# define map operations
resize_op = CV.Resize((resize_height, resize_width),
interpolation=Inter.LINEAR)
rescale_op = CV.Rescale(rescale, shift)
hwc2chw_op = CV.HWC2CHW()
type_cast_op = C.TypeCast(mstype.int32)
# apply map operations on images
if not sparse:
one_hot_enco = C.OneHot(10)
ds1 = ds1.map(input_columns="label", operations=one_hot_enco,
num_parallel_workers=num_parallel_workers)
type_cast_op = C.TypeCast(mstype.float32)
ds1 = ds1.map(input_columns="label", operations=type_cast_op,
num_parallel_workers=num_parallel_workers)
ds1 = ds1.map(input_columns="image", operations=resize_op,
num_parallel_workers=num_parallel_workers)
ds1 = ds1.map(input_columns="image", operations=rescale_op,
num_parallel_workers=num_parallel_workers)
ds1 = ds1.map(input_columns="image", operations=hwc2chw_op,
num_parallel_workers=num_parallel_workers)
# apply DatasetOps
buffer_size = 10000
ds1 = ds1.shuffle(buffer_size=buffer_size)
ds1 = ds1.batch(batch_size, drop_remainder=True)
ds1 = ds1.repeat(repeat_size)
return ds1
def test_rescale_with_random_posterize():
"""
Test RandomPosterize: only support CV_8S/CV_8U
"""
logger.info("test_rescale_with_random_posterize")
DATA_DIR_10 = "../data/dataset/testCifar10Data"
dataset = ds.Cifar10Dataset(DATA_DIR_10)
rescale_op = c_vision.Rescale((1.0 / 255.0), 0.0)
dataset = dataset.map(input_columns=["image"], operations=rescale_op)
random_posterize_op = c_vision.RandomPosterize((4, 8))
dataset = dataset.map(input_columns=["image"], operations=random_posterize_op, num_parallel_workers=1)
try:
_ = dataset.output_shapes()
except RuntimeError as e:
logger.info("Got an exception in DE: {}".format(str(e)))
assert "Input image data type can not be float" in str(e)
def test_rescale_md5():
"""
Test Rescale with md5 comparison
"""
logger.info("Test Rescale with md5 comparison")
# generate dataset
data = ds.TFRecordDataset(DATA_DIR,
SCHEMA_DIR,
columns_list=["image"],
shuffle=False)
decode_op = vision.Decode()
rescale_op = vision.Rescale(1.0 / 255.0, -1.0)
# apply map operations on images
data = data.map(input_columns=["image"], operations=decode_op)
data = data.map(input_columns=["image"], operations=rescale_op)
# check results with md5 comparison
filename = "rescale_01_result.npz"
save_and_check_md5(data, filename, generate_golden=GENERATE_GOLDEN)
def create_dataset(data_path,
batch_size=32,
repeat_size=1,
num_parallel_workers=1):
"""Create dataset for train or test."""
# define dataset
mnist_ds = dataset.MnistDataset(data_path)
resize_height, resize_width = 32, 32
rescale = 1.0 / 255.0
shift = 0.0
# define map operations
resize_op = transforms.Resize((resize_height, resize_width),
interpolation=Inter.LINEAR) # Bilinear mode
rescale_op = transforms.Rescale(rescale, shift)
hwc2chw_op = transforms.HWC2CHW()
type_cast_op = C.TypeCast(mstype.int32)
# apply map operations on images
mnist_ds = mnist_ds.map(input_columns="label",
operations=type_cast_op,
num_parallel_workers=num_parallel_workers)
mnist_ds = mnist_ds.map(input_columns="image",
operations=resize_op,
num_parallel_workers=num_parallel_workers)
mnist_ds = mnist_ds.map(input_columns="image",
operations=rescale_op,
num_parallel_workers=num_parallel_workers)
mnist_ds = mnist_ds.map(input_columns="image",
operations=hwc2chw_op,
num_parallel_workers=num_parallel_workers)
# apply DatasetOps
buffer_size = 10000
mnist_ds = mnist_ds.shuffle(buffer_size=buffer_size)
mnist_ds = mnist_ds.batch(batch_size, drop_remainder=True)
mnist_ds = mnist_ds.repeat(repeat_size)
return mnist_ds
def create_dataset(dataset_path,
batch_size=1,
num_shards=1,
shard_id=0,
device_target='Ascend'):
"""
create train or evaluation dataset for warpctc
Args:
dataset_path(int): dataset path
batch_size(int): batch size of generated dataset, default is 1
num_shards(int): number of devices
shard_id(int): rank id
device_target(str): platform of training, support Ascend and GPU
"""
dataset = _CaptchaDataset(dataset_path, cf.max_captcha_digits,
device_target)
ds = de.GeneratorDataset(dataset, ["image", "label"],
shuffle=True,
num_shards=num_shards,
shard_id=shard_id)
ds.set_dataset_size(m.ceil(len(dataset) / num_shards))
image_trans = [
vc.Rescale(1.0 / 255.0, 0.0),
vc.Normalize([0.9010, 0.9049, 0.9025], std=[0.1521, 0.1347, 0.1458]),
vc.Resize((m.ceil(cf.captcha_height / 16) * 16, cf.captcha_width)),
vc.HWC2CHW()
]
label_trans = [c.TypeCast(mstype.int32)]
ds = ds.map(input_columns=["image"],
num_parallel_workers=8,
operations=image_trans)
ds = ds.map(input_columns=["label"],
num_parallel_workers=8,
operations=label_trans)
ds = ds.batch(batch_size, drop_remainder=True)
return ds
def create_dataset(data_dir,
training=True,
batch_size=32,
resize=(32, 32),
repeat=1,
rescale=1 / (255 * 0.3081),
shift=-0.1307 / 0.3081,
buffer_size=64):
data_train = os.path.join(data_dir, 'train') # 训练集信息
data_test = os.path.join(data_dir, 'test') # 测试集信息
ds = ms.dataset.MnistDataset(data_train if training else data_test)
ds = ds.map(input_columns=["image"],
operations=[
CV.Resize(resize),
CV.Rescale(rescale, shift),
CV.HWC2CHW()
])
ds = ds.map(input_columns=["label"], operations=C.TypeCast(ms.int32))
ds = ds.shuffle(buffer_size=buffer_size).batch(
batch_size, drop_remainder=True).repeat(repeat)
return ds
def create_dataset(data_path, batch_size=32, repeat_size=1,
num_parallel_workers=1):
""" create dataset for train or test
Args:
data_path: Data path
batch_size: The number of data records in each group
repeat_size: The number of replicated data records
num_parallel_workers: The number of parallel workers
"""
# define dataset
mnist_ds = ds.MnistDataset(data_path)
#mnist_ds = ds.MnistDataset(data_path,num_samples=32)
# define operation parameters
resize_height, resize_width = 32, 32
rescale = 1.0 / 255.0
shift = 0.0
# define map operations
resize_op = CV.Resize((resize_height, resize_width), interpolation=Inter.LINEAR) # resize images to (32, 32)
rescale_op = CV.Rescale(rescale, shift) # rescale images
hwc2chw_op = CV.HWC2CHW() # change shape from (height, width, channel) to (channel, height, width) to fit network.
type_cast_op = C.TypeCast(mstype.int32) # change data type of label to int32 to fit network
# apply map operations on images
mnist_ds = mnist_ds.map(input_columns="label", operations=type_cast_op, num_parallel_workers=num_parallel_workers)
mnist_ds = mnist_ds.map(input_columns="image", operations=resize_op, num_parallel_workers=num_parallel_workers)
mnist_ds = mnist_ds.map(input_columns="image", operations=rescale_op, num_parallel_workers=num_parallel_workers)
mnist_ds = mnist_ds.map(input_columns="image", operations=hwc2chw_op, num_parallel_workers=num_parallel_workers)
# apply DatasetOps
buffer_size = 10000
mnist_ds = mnist_ds.shuffle(buffer_size=buffer_size) # 10000 as in LeNet train script
mnist_ds = mnist_ds.batch(batch_size, drop_remainder=True)
mnist_ds = mnist_ds.repeat(repeat_size)
return mnist_ds
def create_dataset(data_path, batch_size=32, repeat_size=1, status="train"):
"""
create dataset for train or test
"""
rank_id = int(os.getenv('DEVICE_ID'))
device_num = int(os.getenv('RANK_SIZE'))
if device_num == 1:
cifar_ds = ds.Cifar10Dataset(data_path, num_parallel_workers=8, shuffle=True)
else:
cifar_ds = ds.Cifar10Dataset(data_path, num_parallel_workers=8, shuffle=True,
num_shards=device_num, shard_id=rank_id)
rescale = 1.0 / 255.0
shift = 0.0
resize_op = CV.Resize((cfg.image_height, cfg.image_width))
rescale_op = CV.Rescale(rescale, shift)
normalize_op = CV.Normalize((0.4914, 0.4822, 0.4465), (0.2023, 0.1994, 0.2010))
if status == "train":
random_crop_op = CV.RandomCrop([32, 32], [4, 4, 4, 4])
random_horizontal_op = CV.RandomHorizontalFlip()
channel_swap_op = CV.HWC2CHW()
typecast_op = C.TypeCast(mstype.int32)
cifar_ds = cifar_ds.map(input_columns="label", operations=typecast_op)
if status == "train":
cifar_ds = cifar_ds.map(input_columns="image", operations=random_crop_op)
cifar_ds = cifar_ds.map(input_columns="image", operations=random_horizontal_op)
cifar_ds = cifar_ds.map(input_columns="image", operations=resize_op)
cifar_ds = cifar_ds.map(input_columns="image", operations=rescale_op)
cifar_ds = cifar_ds.map(input_columns="image", operations=normalize_op)
cifar_ds = cifar_ds.map(input_columns="image", operations=channel_swap_op)
cifar_ds = cifar_ds.shuffle(buffer_size=cfg.buffer_size)
cifar_ds = cifar_ds.batch(batch_size, drop_remainder=True)
cifar_ds = cifar_ds.repeat(repeat_size)
return cifar_ds
def create_dataset(dataset_path, repeat_num=1, batch_size=1):
"""
create train or evaluation dataset for warpctc
Args:
dataset_path(int): dataset path
repeat_num(int): dataset repetition num, default is 1
batch_size(int): batch size of generated dataset, default is 1
"""
rank_size = int(
os.environ.get("RANK_SIZE")) if os.environ.get("RANK_SIZE") else 1
rank_id = int(
os.environ.get("RANK_ID")) if os.environ.get("RANK_ID") else 0
dataset = _CaptchaDataset(dataset_path, cf.max_captcha_digits)
ds = de.GeneratorDataset(dataset, ["image", "label"],
shuffle=True,
num_shards=rank_size,
shard_id=rank_id)
ds.set_dataset_size(m.ceil(len(dataset) / rank_size))
image_trans = [
vc.Rescale(1.0 / 255.0, 0.0),
vc.Normalize([0.9010, 0.9049, 0.9025], std=[0.1521, 0.1347, 0.1458]),
vc.Resize((m.ceil(cf.captcha_height / 16) * 16, cf.captcha_width)),
vc.HWC2CHW()
]
label_trans = [c.TypeCast(mstype.int32)]
ds = ds.map(input_columns=["image"],
num_parallel_workers=8,
operations=image_trans)
ds = ds.map(input_columns=["label"],
num_parallel_workers=8,
operations=label_trans)
ds = ds.batch(batch_size)
ds = ds.repeat(repeat_num)
return ds
def create_dataset(dataset_path, do_train, repeat_num=1, batch_size=32):
"""
create a train or eval dataset
Args:
dataset_path(string): the path of dataset.
do_train(bool): whether dataset is used for train or eval.
repeat_num(int): the repeat times of dataset. Default: 1
batch_size(int): the batch size of dataset. Default: 32
Returns:
dataset
"""
device_num = int(os.getenv("RANK_SIZE"))
rank_id = int(os.getenv("DEVICE_ID"))
data_path_list = []
for root, dirs, files in os.walk(dataset_path):
for file in files:
data_path_list.append(os.path.join(dataset_path, file))
schema = dt.Schema()
schema.add_column('image', de_type=mstype.uint8,
shape=[112, 112,
1]) # Binary data usually use uint8 here.
schema.add_column('label', de_type=mstype.int64, shape=[])
if device_num == 1:
ds = dt.TFRecordDataset(data_path_list,
num_parallel_workers=4,
shuffle=True,
schema=schema,
shard_equal_rows=True)
else:
ds = dt.TFRecordDataset(data_path_list,
num_parallel_workers=4,
shuffle=True,
num_shards=device_num,
shard_id=rank_id,
schema=schema,
shard_equal_rows=True)
resize_height = config.image_height
resize_width = config.image_width
rescale = 1.0 / 255.0
shift = 0.0
resize_op = C.Resize((resize_height, resize_width))
rescale_op = C.Rescale(rescale, shift)
change_swap_op = C.HWC2CHW()
trans = []
type_cast_op_train = C2.TypeCast(mstype.float32)
trans += [resize_op, rescale_op, type_cast_op_train, change_swap_op]
#trans += [resize_op, rescale_op, type_cast_op_train]
#trans += [resize_op, type_cast_op_train, change_swap_op]
type_cast_op = C2.TypeCast(mstype.int32)
ds = ds.map(input_columns="label", operations=type_cast_op)
ds = ds.map(input_columns="image", operations=trans)
ds = ds.shuffle(buffer_size=config.buffer_size)
ds = ds.batch(batch_size, drop_remainder=True)
ds = ds.repeat(repeat_num)
return ds
def create_dataset(dataset_path,
do_train,
config,
platform,
repeat_num=1,
batch_size=100):
"""
create a train or eval dataset
Args:
dataset_path(string): the path of dataset.
do_train(bool): whether dataset is used for train or eval.
repeat_num(int): the repeat times of dataset. Default: 1
batch_size(int): the batch size of dataset. Default: 32
Returns:
dataset
"""
if platform == "Ascend":
rank_size = int(os.getenv("RANK_SIZE"))
rank_id = int(os.getenv("RANK_ID"))
if rank_size == 1:
ds = de.MindDataset(dataset_path,
num_parallel_workers=8,
shuffle=True)
else:
ds = de.MindDataset(dataset_path,
num_parallel_workers=8,
shuffle=True,
num_shards=rank_size,
shard_id=rank_id)
elif platform == "GPU":
if do_train:
from mindspore.communication.management import get_rank, get_group_size
ds = de.MindDataset(dataset_path,
num_parallel_workers=8,
shuffle=True,
num_shards=get_group_size(),
shard_id=get_rank())
else:
ds = de.MindDataset(dataset_path,
num_parallel_workers=8,
shuffle=False)
else:
raise ValueError("Unsupport platform.")
resize_height = config.image_height
buffer_size = 1000
# define map operations
resize_crop_op = C.RandomCropDecodeResize(resize_height,
scale=(0.08, 1.0),
ratio=(0.75, 1.333))
horizontal_flip_op = C.RandomHorizontalFlip(prob=0.5)
color_op = C.RandomColorAdjust(brightness=0.4,
contrast=0.4,
saturation=0.4)
rescale_op = C.Rescale(1 / 255.0, 0)
normalize_op = C.Normalize(mean=[0.485, 0.456, 0.406],
std=[0.229, 0.224, 0.225])
change_swap_op = C.HWC2CHW()
# define python operations
decode_p = P.Decode()
resize_p = P.Resize(256, interpolation=Inter.BILINEAR)
center_crop_p = P.CenterCrop(224)
totensor = P.ToTensor()
normalize_p = P.Normalize((0.485, 0.456, 0.406), (0.229, 0.224, 0.225))
composeop = P.ComposeOp(
[decode_p, resize_p, center_crop_p, totensor, normalize_p])
if do_train:
trans = [
resize_crop_op, horizontal_flip_op, color_op, rescale_op,
normalize_op, change_swap_op
]
else:
trans = composeop()
type_cast_op = C2.TypeCast(mstype.int32)
ds = ds.map(input_columns="image",
operations=trans,
num_parallel_workers=8)
ds = ds.map(input_columns="label_list",
operations=type_cast_op,
num_parallel_workers=8)
# apply shuffle operations
ds = ds.shuffle(buffer_size=buffer_size)
# apply batch operations
ds = ds.batch(batch_size, drop_remainder=True)
# apply dataset repeat operation
ds = ds.repeat(repeat_num)
return ds
def create_dataset(dataset_path, do_train, repeat_num=1, batch_size=32):
"""
create a train or eval dataset
Args:
dataset_path(string): the path of dataset.
do_train(bool): whether dataset is used for train or eval.
repeat_num(int): the repeat times of dataset. Default: 1
batch_size(int): the batch size of dataset. Default: 32
Returns:
dataset
"""
rank_size = int(os.getenv("RANK_SIZE"))
rank_id = int(os.getenv("RANK_ID"))
if rank_size == 1:
ds = de.ImageFolderDatasetV2(dataset_path,
num_parallel_workers=16,
shuffle=True)
else:
ds = de.ImageFolderDatasetV2(dataset_path,
num_parallel_workers=16,
shuffle=True,
num_shards=rank_size,
shard_id=rank_id)
resize_height = config.image_height
resize_width = config.image_width
rescale = 1.0 / 255.0
shift = 0.0
buffer_size = 1000
# define map operations
decode_op = C.Decode()
resize_crop_op = C.RandomResizedCrop(resize_height,
scale=(0.08, 1.0),
ratio=(0.75, 1.333))
horizontal_flip_op = C.RandomHorizontalFlip(prob=0.5)
resize_op = C.Resize((256, 256))
center_crop = C.CenterCrop(resize_width)
rescale_op = C.Rescale(rescale, shift)
normalize_op = C.Normalize(mean=[0.485, 0.456, 0.406],
std=[0.229, 0.224, 0.225])
change_swap_op = C.HWC2CHW()
if do_train:
trans = [
decode_op, resize_crop_op, horizontal_flip_op, rescale_op,
normalize_op, change_swap_op
]
else:
trans = [
decode_op, resize_op, center_crop, rescale_op, normalize_op,
change_swap_op
]
type_cast_op = C2.TypeCast(mstype.int32)
ds = ds.map(input_columns="image", operations=trans)
ds = ds.map(input_columns="label", operations=type_cast_op)
# apply shuffle operations
ds = ds.shuffle(buffer_size=buffer_size)
# apply batch operations
ds = ds.batch(batch_size, drop_remainder=True)
# apply dataset repeat operation
ds = ds.repeat(repeat_num)
return ds
def create_dataset(dataset_path, do_train, repeat_num=1, batch_size=32):
"""
create a train or eval dataset
Args:
dataset_path(string): the path of dataset.
do_train(bool): whether dataset is used for train or eval.
repeat_num(int): the repeat times of dataset. Default: 1
batch_size(int): the batch size of dataset. Default: 32
Returns:
dataset
"""
device_num = int(os.getenv("DEVICE_NUM"))
rank_id = int(os.getenv("RANK_ID"))
if device_num == 1:
ds = de.Cifar10Dataset(dataset_path,
num_parallel_workers=4,
shuffle=True)
else:
ds = de.Cifar10Dataset(dataset_path,
num_parallel_workers=4,
shuffle=True,
num_shards=device_num,
shard_id=rank_id)
resize_height = config.image_height
resize_width = config.image_width
rescale = 1.0 / 255.0
shift = 0.0
# define map operations
random_crop_op = C.RandomCrop((32, 32), (4, 4, 4, 4))
random_horizontal_flip_op = C.RandomHorizontalFlip(rank_id / (rank_id + 1))
resize_op = C.Resize((resize_height, resize_width))
rescale_op = C.Rescale(rescale, shift)
normalize_op = C.Normalize([0.4914, 0.4822, 0.4465],
[0.2023, 0.1994, 0.2010])
change_swap_op = C.HWC2CHW()
trans = []
if do_train:
trans += [random_crop_op, random_horizontal_flip_op]
trans += [resize_op, rescale_op, normalize_op, change_swap_op]
type_cast_op = C2.TypeCast(mstype.int32)
ds = ds.map(input_columns="label", operations=type_cast_op)
ds = ds.map(input_columns="image", operations=trans)
# apply shuffle operations
ds = ds.shuffle(buffer_size=config.buffer_size)
# apply batch operations
ds = ds.batch(batch_size, drop_remainder=True)
# apply dataset repeat operation
ds = ds.repeat(repeat_num)
return ds
def create_dataset(dataset_path, do_train, config, repeat_num=1):
"""
Create a train or eval dataset.
Args:
dataset_path (string): The path of dataset.
do_train (bool): Whether dataset is used for train or eval.
config: configuration
repeat_num (int): The repeat times of dataset. Default: 1.
Returns:
Dataset.
"""
if do_train:
dataset_path = os.path.join(dataset_path, 'train')
do_shuffle = True
else:
dataset_path = os.path.join(dataset_path, 'eval')
do_shuffle = False
device_id = 0
device_num = 1
if config.platform == "GPU":
if config.run_distribute:
from mindspore.communication.management import get_rank, get_group_size
device_id = get_rank()
device_num = get_group_size()
elif config.platform == "Ascend":
device_id = int(os.getenv('DEVICE_ID'))
device_num = int(os.getenv('RANK_SIZE'))
if device_num == 1 or not do_train:
ds = de.Cifar10Dataset(dataset_path,
num_parallel_workers=4,
shuffle=do_shuffle)
else:
ds = de.Cifar10Dataset(dataset_path,
num_parallel_workers=4,
shuffle=do_shuffle,
num_shards=device_num,
shard_id=device_id)
resize_height = config.image_height
resize_width = config.image_width
buffer_size = 100
rescale = 1.0 / 255.0
shift = 0.0
# define map operations
random_crop_op = C.RandomCrop((32, 32), (4, 4, 4, 4))
random_horizontal_flip_op = C.RandomHorizontalFlip(device_id /
(device_id + 1))
resize_op = C.Resize((resize_height, resize_width))
rescale_op = C.Rescale(rescale, shift)
normalize_op = C.Normalize([0.4914, 0.4822, 0.4465],
[0.2023, 0.1994, 0.2010])
change_swap_op = C.HWC2CHW()
trans = []
if do_train:
trans += [random_crop_op, random_horizontal_flip_op]
trans += [resize_op, rescale_op, normalize_op, change_swap_op]
type_cast_op = C2.TypeCast(mstype.int32)
ds = ds.map(input_columns="label", operations=type_cast_op)
ds = ds.map(input_columns="image", operations=trans)
# apply shuffle operations
ds = ds.shuffle(buffer_size=buffer_size)
# apply batch operations
ds = ds.batch(config.batch_size, drop_remainder=True)
# apply dataset repeat operation
ds = ds.repeat(repeat_num)
return ds
def create_dataset3(dataset_path, do_train, repeat_num=1, batch_size=32):
"""
create a train or eval imagenet2012 dataset for resnet101
Args:
dataset_path(string): the path of dataset.
do_train(bool): whether dataset is used for train or eval.
repeat_num(int): the repeat times of dataset. Default: 1
batch_size(int): the batch size of dataset. Default: 32
Returns:
dataset
"""
device_num = int(os.getenv("RANK_SIZE"))
rank_id = int(os.getenv("RANK_ID"))
if device_num == 1:
ds = de.ImageFolderDatasetV2(dataset_path, num_parallel_workers=8, shuffle=True)
else:
ds = de.ImageFolderDatasetV2(dataset_path, num_parallel_workers=8, shuffle=True,
num_shards=device_num, shard_id=rank_id)
resize_height = 224
rescale = 1.0 / 255.0
shift = 0.0
# define map operations
decode_op = C.Decode()
random_resize_crop_op = C.RandomResizedCrop(resize_height, (0.08, 1.0), (0.75, 1.33), max_attempts=100)
horizontal_flip_op = C.RandomHorizontalFlip(rank_id / (rank_id + 1))
resize_op_256 = C.Resize((256, 256))
center_crop = C.CenterCrop(224)
rescale_op = C.Rescale(rescale, shift)
normalize_op = C.Normalize((0.475, 0.451, 0.392), (0.275, 0.267, 0.278))
changeswap_op = C.HWC2CHW()
if do_train:
trans = [decode_op,
random_resize_crop_op,
horizontal_flip_op,
rescale_op,
normalize_op,
changeswap_op]
else:
trans = [decode_op,
resize_op_256,
center_crop,
rescale_op,
normalize_op,
changeswap_op]
type_cast_op = C2.TypeCast(mstype.int32)
ds = ds.map(input_columns="image", operations=trans, num_parallel_workers=8)
ds = ds.map(input_columns="label", operations=type_cast_op, num_parallel_workers=8)
# apply batch operations
ds = ds.batch(batch_size, drop_remainder=True)
# apply dataset repeat operation
ds = ds.repeat(repeat_num)
return ds
def create_dataset(dataset_path,
do_train,
repeat_num=1,
batch_size=32,
target="Ascend"):
"""
create a train or eval dataset
Args:
dataset_path(string): the path of dataset.
do_train(bool): whether dataset is used for train or eval.
repeat_num(int): the repeat times of dataset. Default: 1
batch_size(int): the batch size of dataset. Default: 32
target(str): the device target. Default: Ascend
Returns:
dataset
"""
if target == "Ascend":
device_num = int(os.getenv("DEVICE_NUM"))
rank_id = int(os.getenv("RANK_ID"))
else:
init("nccl")
rank_id = get_rank()
device_num = get_group_size()
if device_num == 1:
ds = de.Cifar10Dataset(dataset_path,
num_parallel_workers=8,
shuffle=True)
else:
ds = de.Cifar10Dataset(dataset_path,
num_parallel_workers=8,
shuffle=True,
num_shards=device_num,
shard_id=rank_id)
# define map operations
trans = []
if do_train:
trans += [
C.RandomCrop((32, 32), (4, 4, 4, 4)),
C.RandomHorizontalFlip(prob=0.5)
]
trans += [
C.Resize((config.image_height, config.image_width)),
C.Rescale(1.0 / 255.0, 0.0),
C.Normalize([0.4914, 0.4822, 0.4465], [0.2023, 0.1994, 0.2010]),
C.HWC2CHW()
]
type_cast_op = C2.TypeCast(mstype.int32)
ds = ds.map(input_columns="label",
num_parallel_workers=8,
operations=type_cast_op)
ds = ds.map(input_columns="image",
num_parallel_workers=8,
operations=trans)
# apply batch operations
ds = ds.batch(batch_size, drop_remainder=True)
# apply dataset repeat operation
ds = ds.repeat(repeat_num)
return ds
def create_dataset(dataset_path, do_train, repeat_num=1, batch_size=32):
"""
Create a train or eval dataset.
Args:
dataset_path (str): The path of dataset.
do_train (bool): Whether dataset is used for train or eval.
repeat_num (int): The repeat times of dataset. Default: 1.
batch_size (int): The batch size of dataset. Default: 32.
Returns:
Dataset.
"""
if do_train:
dataset_path = os.path.join(dataset_path, 'train')
do_shuffle = True
else:
dataset_path = os.path.join(dataset_path, 'eval')
do_shuffle = False
if device_num == 1 or not do_train:
ds = de.Cifar10Dataset(dataset_path,
num_parallel_workers=4,
shuffle=do_shuffle)
else:
ds = de.Cifar10Dataset(dataset_path,
num_parallel_workers=4,
shuffle=do_shuffle,
num_shards=device_num,
shard_id=device_id)
resize_height = 224
resize_width = 224
buffer_size = 100
rescale = 1.0 / 255.0
shift = 0.0
# define map operations
random_crop_op = C.RandomCrop((32, 32), (4, 4, 4, 4))
random_horizontal_flip_op = C.RandomHorizontalFlip(device_id /
(device_id + 1))
resize_op = C.Resize((resize_height, resize_width))
rescale_op = C.Rescale(rescale, shift)
normalize_op = C.Normalize([0.4914, 0.4822, 0.4465],
[0.2023, 0.1994, 0.2010])
change_swap_op = C.HWC2CHW()
trans = []
if do_train:
trans += [random_crop_op, random_horizontal_flip_op]
trans += [resize_op, rescale_op, normalize_op, change_swap_op]
type_cast_op = C2.TypeCast(mstype.int32)
ds = ds.map(input_columns="label", operations=type_cast_op)
ds = ds.map(input_columns="image", operations=trans)
# apply shuffle operations
ds = ds.shuffle(buffer_size=buffer_size)
# apply batch operations
ds = ds.batch(batch_size, drop_remainder=True)
# apply dataset repeat operation
ds = ds.repeat(repeat_num)
return ds
def test_imagefolder(remove_json_files=True):
"""
Test simulating resnet50 dataset pipeline.
"""
data_dir = "../data/dataset/testPK/data"
ds.config.set_seed(1)
# define data augmentation parameters
rescale = 1.0 / 255.0
shift = 0.0
resize_height, resize_width = 224, 224
weights = [1.0, 0.1, 0.02, 0.3, 0.4, 0.05, 1.2, 0.13, 0.14, 0.015, 0.16, 1.1]
# Constructing DE pipeline
sampler = ds.WeightedRandomSampler(weights, 11)
data1 = ds.ImageFolderDatasetV2(data_dir, sampler=sampler)
data1 = data1.repeat(1)
data1 = data1.map(input_columns=["image"], operations=[vision.Decode(True)])
rescale_op = vision.Rescale(rescale, shift)
resize_op = vision.Resize((resize_height, resize_width), Inter.LINEAR)
data1 = data1.map(input_columns=["image"], operations=[rescale_op, resize_op])
data1 = data1.batch(2)
# Serialize the dataset pre-processing pipeline.
# data1 should still work after saving.
ds.serialize(data1, "imagenet_dataset_pipeline.json")
ds1_dict = ds.serialize(data1)
assert validate_jsonfile("imagenet_dataset_pipeline.json") is True
# Print the serialized pipeline to stdout
ds.show(data1)
# Deserialize the serialized json file
data2 = ds.deserialize(json_filepath="imagenet_dataset_pipeline.json")
# Serialize the pipeline we just deserialized.
# The content of the json file should be the same to the previous serialize.
ds.serialize(data2, "imagenet_dataset_pipeline_1.json")
assert validate_jsonfile("imagenet_dataset_pipeline_1.json") is True
assert filecmp.cmp('imagenet_dataset_pipeline.json', 'imagenet_dataset_pipeline_1.json')
# Deserialize the latest json file again
data3 = ds.deserialize(json_filepath="imagenet_dataset_pipeline_1.json")
data4 = ds.deserialize(input_dict=ds1_dict)
num_samples = 0
# Iterate and compare the data in the original pipeline (data1) against the deserialized pipeline (data2)
for item1, item2, item3, item4 in zip(data1.create_dict_iterator(), data2.create_dict_iterator(),
data3.create_dict_iterator(), data4.create_dict_iterator()):
assert np.array_equal(item1['image'], item2['image'])
assert np.array_equal(item1['image'], item3['image'])
assert np.array_equal(item1['label'], item2['label'])
assert np.array_equal(item1['label'], item3['label'])
assert np.array_equal(item3['image'], item4['image'])
assert np.array_equal(item3['label'], item4['label'])
num_samples += 1
logger.info("Number of data in data1: {}".format(num_samples))
assert num_samples == 6
# Remove the generated json file
if remove_json_files:
delete_json_files()
