def create_dataset2(dataset_path, do_train=True, repeat_num=1, batch_size=32, target="gpu", rank=0, size=1):
"""
create a train or eval imagenet2012 dataset for resnet50
Args:
dataset_path(string): the path of dataset.
do_train(bool): whether dataset is used for train or eval.
repeat_num(int): the repeat times of dataset. Default: 1
batch_size(int): the batch size of dataset. Default: 32
target(str): the device target. Default: Ascend
Returns:
dataset
"""
if target == "Ascend":
device_num, rank_id = _get_rank_info()
else:
init()
rank_id = rank
device_num = size
file_list = [os.path.join(dataset_path, f'train-{num:05d}-of-01024') for num in range(1024)]
if device_num == 1:
ds = msds.MindDataset(dataset_file=file_list, num_parallel_workers=8, shuffle=True)
else:
ds = msds.MindDataset(dataset_file=file_list, num_parallel_workers=8, shuffle=True,
num_shards=device_num, shard_id=rank_id)
image_size = 224
mean = [0.485 * 255, 0.456 * 255, 0.406 * 255]
std = [0.229 * 255, 0.224 * 255, 0.225 * 255]
# define map operations
if do_train:
trans = [
C.RandomCropDecodeResize(image_size, scale=(0.08, 1.0), ratio=(0.75, 1.333)),
C.RandomHorizontalFlip(prob=0.5),
C.Normalize(mean=mean, std=std),
C.HWC2CHW(),
C2.TypeCast(mstype.float16)
]
else:
trans = [
C.Decode(),
C.Resize(256),
C.CenterCrop(image_size),
C.Normalize(mean=mean, std=std),
C.HWC2CHW()
]
type_cast_op = C2.TypeCast(mstype.int32)
ds = ds.map(operations=trans, input_columns="image", num_parallel_workers=8)
ds = ds.map(operations=type_cast_op, input_columns="label", num_parallel_workers=8)
# apply batch operations
ds = ds.batch(batch_size, drop_remainder=True)
# apply dataset repeat operation
ds = ds.repeat(repeat_num)
return ds
def create_deeptext_dataset(mindrecord_file,
batch_size=2,
repeat_num=12,
device_num=1,
rank_id=0,
is_training=True,
num_parallel_workers=4):
"""Creatr deeptext dataset with MindDataset."""
ds = de.MindDataset(mindrecord_file,
columns_list=["image", "annotation"],
num_shards=device_num,
shard_id=rank_id,
num_parallel_workers=1,
shuffle=is_training)
decode = C.Decode()
ds = ds.map(operations=decode,
input_columns=["image"],
num_parallel_workers=1)
compose_map_func = (lambda image, annotation: preprocess_fn(
image, annotation, is_training))
hwc_to_chw = C.HWC2CHW()
normalize_op = C.Normalize((123.675, 116.28, 103.53),
(58.395, 57.12, 57.375))
horizontally_op = C.RandomHorizontalFlip(1)
type_cast0 = CC.TypeCast(mstype.float32)
type_cast1 = CC.TypeCast(mstype.float32)
type_cast2 = CC.TypeCast(mstype.int32)
type_cast3 = CC.TypeCast(mstype.bool_)
if is_training:
ds = ds.map(
operations=compose_map_func,
input_columns=["image", "annotation"],
output_columns=[
"image", "image_shape", "box", "label", "valid_num"
],
column_order=["image", "image_shape", "box", "label", "valid_num"],
num_parallel_workers=num_parallel_workers)
flip = (np.random.rand() < config.flip_ratio)
if flip:
ds = ds.map(operations=[normalize_op, type_cast0, horizontally_op],
input_columns=["image"],
num_parallel_workers=12)
ds = ds.map(operations=flipped_generation,
input_columns=[
"image", "image_shape", "box", "label", "valid_num"
],
num_parallel_workers=num_parallel_workers)
else:
ds = ds.map(operations=[normalize_op, type_cast0],
input_columns=["image"],
num_parallel_workers=12)
ds = ds.map(operations=[hwc_to_chw, type_cast1],
input_columns=["image"],
num_parallel_workers=12)
else:
ds = ds.map(
operations=compose_map_func,
input_columns=["image", "annotation"],
output_columns=[
"image", "image_shape", "box", "label", "valid_num"
],
column_order=["image", "image_shape", "box", "label", "valid_num"],
num_parallel_workers=num_parallel_workers)
ds = ds.map(operations=[normalize_op, hwc_to_chw, type_cast1],
input_columns=["image"],
num_parallel_workers=24)
# transpose_column from python to c
ds = ds.map(operations=[type_cast1], input_columns=["image_shape"])
ds = ds.map(operations=[type_cast1], input_columns=["box"])
ds = ds.map(operations=[type_cast2], input_columns=["label"])
ds = ds.map(operations=[type_cast3], input_columns=["valid_num"])
ds = ds.batch(batch_size, drop_remainder=True)
ds = ds.repeat(repeat_num)
return ds
def create_dataset4(dataset_path,
do_train,
repeat_num=1,
batch_size=32,
target="Ascend"):
"""
create a train or eval imagenet2012 dataset for se-resnet50
Args:
dataset_path(string): the path of dataset.
do_train(bool): whether dataset is used for train or eval.
repeat_num(int): the repeat times of dataset. Default: 1
batch_size(int): the batch size of dataset. Default: 32
target(str): the device target. Default: Ascend
Returns:
dataset
"""
if target == "Ascend":
device_num, rank_id = _get_rank_info()
if device_num == 1:
ds = de.ImageFolderDataset(dataset_path,
num_parallel_workers=12,
shuffle=True)
else:
ds = de.ImageFolderDataset(dataset_path,
num_parallel_workers=12,
shuffle=True,
num_shards=device_num,
shard_id=rank_id)
image_size = 224
mean = [123.68, 116.78, 103.94]
std = [1.0, 1.0, 1.0]
# define map operations
if do_train:
trans = [
C.RandomCropDecodeResize(image_size,
scale=(0.08, 1.0),
ratio=(0.75, 1.333)),
C.RandomHorizontalFlip(prob=0.5),
C.Normalize(mean=mean, std=std),
C.HWC2CHW()
]
else:
trans = [
C.Decode(),
C.Resize(292),
C.CenterCrop(256),
C.Normalize(mean=mean, std=std),
C.HWC2CHW()
]
type_cast_op = C2.TypeCast(mstype.int32)
ds = ds.map(operations=trans,
input_columns="image",
num_parallel_workers=12)
ds = ds.map(operations=type_cast_op,
input_columns="label",
num_parallel_workers=12)
# apply batch operations
ds = ds.batch(batch_size, drop_remainder=True)
# apply dataset repeat operation
ds = ds.repeat(repeat_num)
return ds
def create_dataset3(dataset_path,
do_train,
repeat_num=1,
batch_size=32,
target="Ascend",
distribute=False):
"""
create a train or eval imagenet2012 dataset for resnet101
Args:
dataset_path(string): the path of dataset.
do_train(bool): whether dataset is used for train or eval.
repeat_num(int): the repeat times of dataset. Default: 1
batch_size(int): the batch size of dataset. Default: 32
target(str): the device target. Default: Ascend
distribute(bool): data for distribute or not. Default: False
Returns:
dataset
"""
if not do_train:
dataset_path = os.path.join(dataset_path, 'val')
else:
dataset_path = os.path.join(dataset_path, 'train')
if target == "Ascend":
device_num, rank_id = _get_rank_info()
else:
if distribute:
init()
rank_id = get_rank()
device_num = get_group_size()
else:
device_num = 1
rank_id = 1
if device_num == 1:
ds = de.ImageFolderDataset(dataset_path,
num_parallel_workers=8,
shuffle=True)
else:
ds = de.ImageFolderDataset(dataset_path,
num_parallel_workers=8,
shuffle=True,
num_shards=device_num,
shard_id=rank_id)
image_size = 224
mean = [0.475 * 255, 0.451 * 255, 0.392 * 255]
std = [0.275 * 255, 0.267 * 255, 0.278 * 255]
# define map operations
if do_train:
trans = [
C.RandomCropDecodeResize(image_size,
scale=(0.08, 1.0),
ratio=(0.75, 1.333)),
C.RandomHorizontalFlip(rank_id / (rank_id + 1)),
C.Normalize(mean=mean, std=std),
C.HWC2CHW()
]
else:
trans = [
C.Decode(),
C.Resize(256),
C.CenterCrop(image_size),
C.Normalize(mean=mean, std=std),
C.HWC2CHW()
]
type_cast_op = C2.TypeCast(mstype.int32)
ds = ds.map(operations=trans,
input_columns="image",
num_parallel_workers=8)
ds = ds.map(operations=type_cast_op,
input_columns="label",
num_parallel_workers=8)
# apply batch operations
ds = ds.batch(batch_size, drop_remainder=True)
# apply dataset repeat operation
ds = ds.repeat(repeat_num)
return ds
def create_dataset(dataset_path, do_train, repeat_num=1, batch_size=32):
"""
Create a train or eval dataset.
Args:
dataset_path (str): The path of dataset.
do_train (bool): Whether dataset is used for train or eval.
repeat_num (int): The repeat times of dataset. Default: 1.
batch_size (int): The batch size of dataset. Default: 32.
Returns:
Dataset.
"""
if do_train:
dataset_path = os.path.join(dataset_path, 'train')
do_shuffle = True
else:
dataset_path = os.path.join(dataset_path, 'eval')
do_shuffle = False
if device_num == 1 or not do_train:
ds = de.Cifar10Dataset(dataset_path,
num_parallel_workers=8,
shuffle=do_shuffle)
else:
ds = de.Cifar10Dataset(dataset_path,
num_parallel_workers=8,
shuffle=do_shuffle,
num_shards=device_num,
shard_id=device_id)
resize_height = 224
resize_width = 224
buffer_size = 100
rescale = 1.0 / 255.0
shift = 0.0
# define map operations
random_crop_op = C.RandomCrop((32, 32), (4, 4, 4, 4))
random_horizontal_flip_op = C.RandomHorizontalFlip(device_id /
(device_id + 1))
resize_op = C.Resize((resize_height, resize_width))
rescale_op = C.Rescale(rescale, shift)
normalize_op = C.Normalize([0.4914, 0.4822, 0.4465],
[0.2023, 0.1994, 0.2010])
change_swap_op = C.HWC2CHW()
trans = []
if do_train:
trans += [random_crop_op, random_horizontal_flip_op]
trans += [resize_op, rescale_op, normalize_op, change_swap_op]
type_cast_op = C2.TypeCast(mstype.int32)
ds = ds.map(operations=type_cast_op,
input_columns="label",
num_parallel_workers=8)
ds = ds.map(operations=trans,
input_columns="image",
num_parallel_workers=8)
# apply batch operations
ds = ds.batch(batch_size, drop_remainder=True)
# apply dataset repeat operation
ds = ds.repeat(repeat_num)
return ds
def create_dataset1(dataset_path,
do_train,
repeat_num=1,
batch_size=32,
target="Ascend",
distribute=False,
enable_cache=False,
cache_session_id=None):
"""
create a train or evaluate cifar10 dataset for resnet50
Args:
dataset_path(string): the path of dataset.
do_train(bool): whether dataset is used for train or eval.
repeat_num(int): the repeat times of dataset. Default: 1
batch_size(int): the batch size of dataset. Default: 32
target(str): the device target. Default: Ascend
distribute(bool): data for distribute or not. Default: False
enable_cache(bool): whether tensor caching service is used for eval. Default: False
cache_session_id(int): If enable_cache, cache session_id need to be provided. Default: None
Returns:
dataset
"""
if target == "Ascend":
device_num, rank_id = _get_rank_info()
else:
if distribute:
init()
rank_id = get_rank()
device_num = get_group_size()
else:
device_num = 1
if device_num == 1:
data_set = ds.Cifar10Dataset(dataset_path,
num_parallel_workers=8,
shuffle=True)
else:
data_set = ds.Cifar10Dataset(dataset_path,
num_parallel_workers=8,
shuffle=True,
num_shards=device_num,
shard_id=rank_id)
# define map operations
trans = []
if do_train:
trans += [
C.RandomCrop((32, 32), (4, 4, 4, 4)),
C.RandomHorizontalFlip(prob=0.5)
]
trans += [
C.Resize((224, 224)),
C.Rescale(1.0 / 255.0, 0.0),
C.Normalize([0.4914, 0.4822, 0.4465], [0.2023, 0.1994, 0.2010]),
C.HWC2CHW()
]
type_cast_op = C2.TypeCast(mstype.int32)
data_set = data_set.map(operations=type_cast_op,
input_columns="label",
num_parallel_workers=8)
# only enable cache for eval
if do_train:
enable_cache = False
if enable_cache:
if not cache_session_id:
raise ValueError(
"A cache session_id must be provided to use cache.")
eval_cache = ds.DatasetCache(session_id=int(cache_session_id), size=0)
data_set = data_set.map(operations=trans,
input_columns="image",
num_parallel_workers=8,
cache=eval_cache)
else:
data_set = data_set.map(operations=trans,
input_columns="image",
num_parallel_workers=8)
# apply batch operations
data_set = data_set.batch(batch_size, drop_remainder=True)
# apply dataset repeat operation
data_set = data_set.repeat(repeat_num)
return data_set
def create_dataset(dataset_path, do_train, config, repeat_num=1):
"""
create a train or eval dataset
Args:
dataset_path(string): the path of dataset.
do_train(bool): whether dataset is used for train or eval.
config(struct): the config of train and eval in diffirent platform.
repeat_num(int): the repeat times of dataset. Default: 1.
Returns:
dataset
"""
if config.platform == "Ascend":
rank_size = int(os.getenv("RANK_SIZE", '1'))
rank_id = int(os.getenv("RANK_ID", '0'))
if rank_size == 1:
ds = de.ImageFolderDataset(dataset_path,
num_parallel_workers=8,
shuffle=True)
else:
ds = de.ImageFolderDataset(dataset_path,
num_parallel_workers=8,
shuffle=True,
num_shards=rank_size,
shard_id=rank_id)
elif config.platform == "GPU":
if do_train:
if config.run_distribute:
from mindspore.communication.management import get_rank, get_group_size
ds = de.ImageFolderDataset(dataset_path,
num_parallel_workers=8,
shuffle=True,
num_shards=get_group_size(),
shard_id=get_rank())
else:
ds = de.ImageFolderDataset(dataset_path,
num_parallel_workers=8,
shuffle=True)
else:
ds = de.ImageFolderDataset(dataset_path,
num_parallel_workers=8,
shuffle=True)
elif config.platform == "CPU":
ds = de.ImageFolderDataset(dataset_path,
num_parallel_workers=8,
shuffle=True)
resize_height = config.image_height
resize_width = config.image_width
buffer_size = 1000
# define map operations
decode_op = C.Decode()
resize_crop_op = C.RandomCropDecodeResize(resize_height,
scale=(0.08, 1.0),
ratio=(0.75, 1.333))
horizontal_flip_op = C.RandomHorizontalFlip(prob=0.5)
resize_op = C.Resize((256, 256))
center_crop = C.CenterCrop(resize_width)
rescale_op = C.RandomColorAdjust(brightness=0.4,
contrast=0.4,
saturation=0.4)
normalize_op = C.Normalize(mean=[0.485 * 255, 0.456 * 255, 0.406 * 255],
std=[0.229 * 255, 0.224 * 255, 0.225 * 255])
change_swap_op = C.HWC2CHW()
if do_train:
trans = [
resize_crop_op, horizontal_flip_op, rescale_op, normalize_op,
change_swap_op
]
else:
trans = [
decode_op, resize_op, center_crop, normalize_op, change_swap_op
]
type_cast_op = C2.TypeCast(mstype.int32)
ds = ds.map(operations=trans,
input_columns="image",
num_parallel_workers=8)
ds = ds.map(operations=type_cast_op,
input_columns="label",
num_parallel_workers=8)
# apply shuffle operations
ds = ds.shuffle(buffer_size=buffer_size)
# apply batch operations
ds = ds.batch(config.batch_size, drop_remainder=True)
# apply dataset repeat operation
ds = ds.repeat(repeat_num)
return ds
def create_dataset_imagenet(dataset_path,
repeat_num=1,
training=True,
num_parallel_workers=None,
shuffle=None):
"""
create a train or eval imagenet2012 dataset for resnet50
Args:
dataset_path(string): the path of dataset.
do_train(bool): whether dataset is used for train or eval.
repeat_num(int): the repeat times of dataset. Default: 1
batch_size(int): the batch size of dataset. Default: 32
target(str): the device target. Default: Ascend
Returns:
dataset
"""
device_num, rank_id = _get_rank_info()
if device_num == 1:
data_set = ds.ImageFolderDatasetV2(
dataset_path,
num_parallel_workers=num_parallel_workers,
shuffle=shuffle)
else:
data_set = ds.ImageFolderDatasetV2(
dataset_path,
num_parallel_workers=num_parallel_workers,
shuffle=shuffle,
num_shards=device_num,
shard_id=rank_id)
assert imagenet_cfg.image_height == imagenet_cfg.image_width, "image_height not equal image_width"
image_size = imagenet_cfg.image_height
mean = [0.485 * 255, 0.456 * 255, 0.406 * 255]
std = [0.229 * 255, 0.224 * 255, 0.225 * 255]
# define map operations
if training:
transform_img = [
vision.RandomCropDecodeResize(image_size,
scale=(0.08, 1.0),
ratio=(0.75, 1.333)),
vision.RandomHorizontalFlip(prob=0.5),
vision.RandomColorAdjust(0.4, 0.4, 0.4, 0.1),
vision.Normalize(mean=mean, std=std),
vision.HWC2CHW()
]
else:
transform_img = [
vision.Decode(),
vision.Resize(256),
vision.CenterCrop(image_size),
vision.Normalize(mean=mean, std=std),
vision.HWC2CHW()
]
transform_label = [C.TypeCast(mstype.int32)]
data_set = data_set.map(input_columns="image",
num_parallel_workers=8,
operations=transform_img)
data_set = data_set.map(input_columns="label",
num_parallel_workers=8,
operations=transform_label)
# apply batch operations
data_set = data_set.batch(imagenet_cfg.batch_size, drop_remainder=True)
# apply dataset repeat operation
data_set = data_set.repeat(repeat_num)
return data_set
def create_dataset_imagenet(dataset_path,
do_train,
repeat_num=1,
batch_size=32,
target="Ascend"):
"""
create a train or eval imagenet dataset
Args:
dataset_path(string): the path of dataset.
do_train(bool): whether dataset is used for train or eval.
repeat_num(int): the repeat times of dataset. Default: 1
batch_size(int): the batch size of dataset. Default: 32
target(str): the device target. Default: Ascend
Returns:
dataset
"""
if target == "Ascend":
device_num, rank_id = _get_rank_info()
else:
init()
rank_id = get_rank()
device_num = get_group_size()
if device_num == 1:
data_set = ds.ImageFolderDataset(dataset_path,
num_parallel_workers=8,
shuffle=True)
else:
data_set = ds.ImageFolderDataset(dataset_path,
num_parallel_workers=8,
shuffle=True,
num_shards=device_num,
shard_id=rank_id)
image_size = 227
mean = [0.485 * 255, 0.456 * 255, 0.406 * 255]
std = [0.229 * 255, 0.224 * 255, 0.225 * 255]
# define map operations
if do_train:
trans = [
C.RandomCropDecodeResize(image_size,
scale=(0.08, 1.0),
ratio=(0.75, 1.333)),
C.RandomHorizontalFlip(prob=0.5),
C.RandomColorAdjust(brightness=0.4, contrast=0.4, saturation=0.4),
C.Normalize(mean=mean, std=std),
C.CutOut(112),
C.HWC2CHW()
]
else:
trans = [
C.Decode(),
C.Resize((256, 256)),
C.CenterCrop(image_size),
C.Normalize(mean=mean, std=std),
C.HWC2CHW()
]
type_cast_op = C2.TypeCast(mstype.int32)
data_set = data_set.map(operations=type_cast_op,
input_columns="label",
num_parallel_workers=8)
data_set = data_set.map(operations=trans,
input_columns="image",
num_parallel_workers=8)
# apply batch operations
data_set = data_set.batch(batch_size, drop_remainder=True)
# apply dataset repeat operation
data_set = data_set.repeat(repeat_num)
return data_set
def create_dataset(dataset_path, config, buffer_size=1000):
"""
create a train or eval dataset
Args:
dataset_path(string): the path of dataset.
config(struct): the config of train and eval in diffirent platform.
repeat_num(int): the repeat times of dataset. Default: 1.
Returns:
train_dataset, val_dataset
"""
ds = de.ImageFolderDataset(dataset_path, num_parallel_workers=4)
train_ds, eval_ds = ds.split(config.data_split, randomize=True)
resize_height = config.image_height
resize_width = config.image_width
# define operations mapping to each sample
normalize_op = C.Normalize(mean=[0.485 * 255, 0.456 * 255, 0.406 * 255],
std=[0.229 * 255, 0.224 * 255, 0.225 * 255])
change_swap_op = C.HWC2CHW()
type_cast_op = C2.TypeCast(mstype.int32)
# operations for training
crop_decode_resize = C.RandomCropDecodeResize(resize_height,
scale=(0.08, 1.0),
ratio=(0.75, 1.333))
horizontal_flip_op = C.RandomHorizontalFlip(prob=0.5)
color_adjust = C.RandomColorAdjust(brightness=0.4,
contrast=0.4,
saturation=0.4)
# operations for inference
decode_op = C.Decode()
resize_op = C.Resize((256, 256))
center_crop = C.CenterCrop(resize_width)
train_trans = [
crop_decode_resize, horizontal_flip_op, color_adjust, normalize_op,
change_swap_op
]
train_ds = train_ds.map(input_columns="image",
operations=train_trans,
num_parallel_workers=4)
train_ds = train_ds.map(input_columns="label",
operations=type_cast_op,
num_parallel_workers=4)
eval_trans = [
decode_op, resize_op, center_crop, normalize_op, change_swap_op
]
eval_ds = eval_ds.map(input_columns="image",
operations=eval_trans,
num_parallel_workers=4)
eval_ds = eval_ds.map(input_columns="label",
operations=type_cast_op,
num_parallel_workers=4)
# apply shuffle operations
train_ds = train_ds.shuffle(buffer_size=buffer_size)
# apply batch operations
train_ds = train_ds.batch(config.batch_size, drop_remainder=True)
eval_ds = eval_ds.batch(config.eval_batch_size, drop_remainder=True)
return train_ds, eval_ds
def create_dataset(dataset_path,
do_train,
repeat_num=1,
batch_size=32,
target="Ascend"):
"""
create a train or eval dataset
Args:
dataset_path(string): the path of dataset.
do_train(bool): whether dataset is used for train or eval.
repeat_num(int): the repeat times of dataset. Default: 1
batch_size(int): the batch size of dataset. Default: 32
target(str): the device target. Default: Ascend
Returns:
dataset
"""
if target == "Ascend":
device_num = int(os.getenv("RANK_SIZE"))
rank_id = int(os.getenv("RANK_ID"))
else:
init()
rank_id = get_rank()
device_num = get_group_size()
columns_list = ['image', 'label']
if config.data_load_mode == "mindrecord":
load_func = partial(de.MindDataset, dataset_path, columns_list)
else:
load_func = partial(de.ImageFolderDataset, dataset_path)
if device_num == 1:
ds = load_func(num_parallel_workers=8, shuffle=True)
else:
ds = load_func(num_parallel_workers=8,
shuffle=True,
num_shards=device_num,
shard_id=rank_id)
image_size = config.image_height
mean = [0.485 * 255, 0.456 * 255, 0.406 * 255]
std = [0.229 * 255, 0.224 * 255, 0.225 * 255]
# define map operations
if do_train:
trans = [
C.RandomCropDecodeResize(image_size,
scale=(0.08, 1.0),
ratio=(0.75, 1.333)),
C.RandomHorizontalFlip(prob=0.5),
C.Normalize(mean=mean, std=std),
C.HWC2CHW()
]
else:
trans = [
C.Decode(),
C.Resize(256),
C.CenterCrop(image_size),
C.Normalize(mean=mean, std=std),
C.HWC2CHW()
]
type_cast_op = C2.TypeCast(mstype.int32)
ds = ds.map(operations=trans,
input_columns="image",
num_parallel_workers=8)
ds = ds.map(operations=type_cast_op,
input_columns="label",
num_parallel_workers=8)
# apply batch operations
ds = ds.batch(batch_size, drop_remainder=True)
# apply dataset repeat operation
ds = ds.repeat(repeat_num)
return ds
def create_dataset(dataset_path,
do_train,
config,
device_target,
repeat_num=1,
batch_size=32):
"""
create a train or eval dataset
Args:
dataset_path(string): the path of dataset.
do_train(bool): whether dataset is used for train or eval.
repeat_num(int): the repeat times of dataset. Default: 1.
batch_size(int): the batch size of dataset. Default: 32.
Returns:
dataset
"""
if device_target == "Ascend":
rank_size = int(os.getenv("RANK_SIZE"))
rank_id = int(os.getenv("RANK_ID"))
columns_list = ['image', 'label']
if config.data_load_mode == "mindrecord":
load_func = partial(ds.MindDataset, dataset_path, columns_list)
else:
load_func = partial(ds.ImageFolderDataset, dataset_path)
if do_train:
if rank_size == 1:
data_set = load_func(num_parallel_workers=8, shuffle=True)
else:
data_set = load_func(num_parallel_workers=8,
shuffle=True,
num_shards=rank_size,
shard_id=rank_id)
else:
data_set = load_func(num_parallel_workers=8, shuffle=False)
elif device_target == "GPU":
if do_train:
from mindspore.communication.management import get_rank, get_group_size
data_set = ds.ImageFolderDataset(dataset_path,
num_parallel_workers=8,
shuffle=True,
num_shards=get_group_size(),
shard_id=get_rank())
else:
data_set = ds.ImageFolderDataset(dataset_path,
num_parallel_workers=8,
shuffle=True)
else:
raise ValueError("Unsupported device_target.")
resize_height = 224
if do_train:
buffer_size = 20480
# apply shuffle operations
data_set = data_set.shuffle(buffer_size=buffer_size)
# define map operations
decode_op = C.Decode()
resize_crop_decode_op = C.RandomCropDecodeResize(resize_height,
scale=(0.08, 1.0),
ratio=(0.75, 1.333))
horizontal_flip_op = C.RandomHorizontalFlip(prob=0.5)
resize_op = C.Resize(256)
center_crop = C.CenterCrop(resize_height)
normalize_op = C.Normalize(mean=[0.485 * 255, 0.456 * 255, 0.406 * 255],
std=[0.229 * 255, 0.224 * 255, 0.225 * 255])
change_swap_op = C.HWC2CHW()
if do_train:
trans = [
resize_crop_decode_op, horizontal_flip_op, normalize_op,
change_swap_op
]
else:
trans = [
decode_op, resize_op, center_crop, normalize_op, change_swap_op
]
type_cast_op = C2.TypeCast(mstype.int32)
data_set = data_set.map(operations=trans,
input_columns="image",
num_parallel_workers=16)
data_set = data_set.map(operations=type_cast_op,
input_columns="label",
num_parallel_workers=8)
# apply batch operations
data_set = data_set.batch(batch_size, drop_remainder=True)
# apply dataset repeat operation
data_set = data_set.repeat(repeat_num)
return data_set
def create_dataset(dataset_path,
do_train,
repeat_num=1,
batch_size=32,
shard_id=0):
"""
Create a train or eval dataset.
Args:
dataset_path (str): The path of dataset.
do_train (bool): Whether dataset is used for train or eval.
repeat_num (int): The repeat times of dataset. Default: 1.
batch_size (int): The batch size of dataset. Default: 32.
Returns:
Dataset.
"""
do_shuffle = bool(do_train)
if device_num == 1 or not do_train:
ds = de.ImageFolderDataset(dataset_path,
num_parallel_workers=config.work_nums,
shuffle=do_shuffle)
else:
ds = de.ImageFolderDataset(dataset_path,
num_parallel_workers=config.work_nums,
shuffle=do_shuffle,
num_shards=device_num,
shard_id=shard_id)
image_length = 299
if do_train:
trans = [
C.RandomCropDecodeResize(image_length,
scale=(0.08, 1.0),
ratio=(0.75, 1.333)),
C.RandomHorizontalFlip(prob=0.5),
C.RandomColorAdjust(brightness=0.4, contrast=0.4, saturation=0.4)
]
else:
trans = [
C.Decode(),
C.Resize(image_length),
C.CenterCrop(image_length)
]
trans += [
C.Rescale(1.0 / 255.0, 0.0),
C.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]),
C.HWC2CHW()
]
type_cast_op = C2.TypeCast(mstype.int32)
ds = ds.map(input_columns="label",
operations=type_cast_op,
num_parallel_workers=config.work_nums)
ds = ds.map(input_columns="image",
operations=trans,
num_parallel_workers=config.work_nums)
# apply batch operations
ds = ds.batch(batch_size, drop_remainder=True)
# apply dataset repeat operation
ds = ds.repeat(repeat_num)
return ds
def keypoint_dataset(config,
ann_file=None,
image_path=None,
bbox_file=None,
rank=0,
group_size=1,
train_mode=True,
num_parallel_workers=8,
transform=None,
shuffle=None):
"""
A function that returns an imagenet dataset for classification. The mode of input dataset should be "folder" .
Args:
rank (int): The shard ID within num_shards (default=None).
group_size (int): Number of shards that the dataset should be divided
into (default=None).
mode (str): "train" or others. Default: " train".
num_parallel_workers (int): Number of workers to read the data. Default: None.
"""
# config
per_batch_size = config.TRAIN.BATCH_SIZE if train_mode else config.TEST.BATCH_SIZE
image_path = image_path if image_path else os.path.join(
config.DATASET.ROOT,
config.DATASET.TRAIN_SET if train_mode else config.DATASET.TEST_SET)
print('loading dataset from {}'.format(image_path))
ann_file = ann_file if ann_file else os.path.join(
config.DATASET.ROOT, 'annotations/person_keypoints_{}2017.json'.format(
'train' if train_mode else 'val'))
shuffle = shuffle if shuffle is not None else train_mode
# gen dataset db
dataset_generator = KeypointDatasetGenerator(config, is_train=train_mode)
if not train_mode and not config.TEST.USE_GT_BBOX:
print('loading bbox file from {}'.format(bbox_file))
dataset_generator.load_detect_dataset(image_path, ann_file, bbox_file)
else:
dataset_generator.load_gt_dataset(image_path, ann_file)
# construct dataset
de_dataset = de.GeneratorDataset(dataset_generator,
column_names=[
"image", "target", "weight", "scale",
"center", "score", "id"
],
num_parallel_workers=num_parallel_workers,
num_shards=group_size,
shard_id=rank,
shuffle=shuffle)
# inputs map functions
if transform is None:
transform_img = [
V_C.Rescale(1.0 / 255.0, 0.0),
V_C.Normalize(mean=[0.485, 0.456, 0.406],
std=[0.229, 0.224, 0.225]),
V_C.HWC2CHW()
]
else:
transform_img = transform
de_dataset = de_dataset.map(input_columns="image",
num_parallel_workers=num_parallel_workers,
operations=transform_img)
# batch
de_dataset = de_dataset.batch(per_batch_size, drop_remainder=train_mode)
return de_dataset, dataset_generator
def create_dataset(dataset_path,
do_train,
config,
device_target,
repeat_num=1,
batch_size=32,
run_distribute=False):
"""
create a train or eval dataset
Args:
dataset_path(string): the path of dataset.
do_train(bool): whether dataset is used for train or eval.
repeat_num(int): the repeat times of dataset. Default: 1
batch_size(int): the batch size of dataset. Default: 32
Returns:
dataset
"""
if device_target == "GPU":
if do_train:
if run_distribute:
from mindspore.communication.management import get_rank, get_group_size
data_set = ds.ImageFolderDataset(dataset_path,
num_parallel_workers=8,
shuffle=True,
num_shards=get_group_size(),
shard_id=get_rank())
else:
data_set = ds.ImageFolderDataset(dataset_path,
num_parallel_workers=8,
shuffle=True)
else:
data_set = ds.ImageFolderDataset(dataset_path,
num_parallel_workers=8,
shuffle=True)
else:
raise ValueError("Unsupported device_target.")
resize_height = config.image_height
resize_width = config.image_width
buffer_size = 1000
# define map operations
decode_op = C.Decode()
resize_crop_op = C.RandomCropDecodeResize(resize_height,
scale=(0.08, 1.0),
ratio=(0.75, 1.333))
horizontal_flip_op = C.RandomHorizontalFlip(prob=0.5)
resize_op = C.Resize(256)
center_crop = C.CenterCrop(resize_width)
rescale_op = C.RandomColorAdjust(brightness=0.4,
contrast=0.4,
saturation=0.4)
normalize_op = C.Normalize(mean=[0.485 * 255, 0.456 * 255, 0.406 * 255],
std=[0.229 * 255, 0.224 * 255, 0.225 * 255])
change_swap_op = C.HWC2CHW()
if do_train:
trans = [
resize_crop_op, horizontal_flip_op, rescale_op, normalize_op,
change_swap_op
]
else:
trans = [
decode_op, resize_op, center_crop, normalize_op, change_swap_op
]
type_cast_op = C2.TypeCast(mstype.int32)
data_set = data_set.map(operations=trans,
input_columns="image",
num_parallel_workers=8)
data_set = data_set.map(operations=type_cast_op,
input_columns="label",
num_parallel_workers=8)
# apply shuffle operations
data_set = data_set.shuffle(buffer_size=buffer_size)
# apply batch operations
data_set = data_set.batch(batch_size, drop_remainder=True)
# apply dataset repeat operation
data_set = data_set.repeat(repeat_num)
return data_set
def create_dataset_imagenet(dataset_path,
batch_size=32,
repeat_num=1,
training=True,
num_parallel_workers=None,
shuffle=None,
sampler=None,
class_indexing=None):
"""
create a train or eval imagenet2012 dataset for resnet50
Args:
dataset_path(string): the path of dataset.
do_train(bool): whether dataset is used for train or eval.
repeat_num(int): the repeat times of dataset. Default: 1
batch_size(int): the batch size of dataset. Default: 32
target(str): the device target. Default: Ascend
Returns:
dataset
"""
device_num, rank_id = _get_rank_info()
cfg = alexnet_imagenet_cfg
num_parallel_workers = 16
if device_num == 1:
num_parallel_workers = 48
ds.config.set_prefetch_size(8)
else:
ds.config.set_numa_enable(True)
data_set = ds.ImageFolderDataset(dataset_path,
num_parallel_workers=4,
shuffle=shuffle,
sampler=sampler,
class_indexing=class_indexing,
num_shards=device_num,
shard_id=rank_id)
assert cfg.image_height == cfg.image_width, "imagenet_cfg.image_height not equal imagenet_cfg.image_width"
image_size = cfg.image_height
mean = [0.485 * 255, 0.456 * 255, 0.406 * 255]
std = [0.229 * 255, 0.224 * 255, 0.225 * 255]
# define map operations
if training:
transform_img = [
CV.RandomCropDecodeResize(image_size,
scale=(0.08, 1.0),
ratio=(0.75, 1.333)),
CV.RandomHorizontalFlip(prob=0.5),
CV.Normalize(mean=mean, std=std),
CV.HWC2CHW()
]
else:
transform_img = [
CV.Decode(),
CV.Resize((256, 256)),
CV.CenterCrop(image_size),
CV.Normalize(mean=mean, std=std),
CV.HWC2CHW()
]
data_set = data_set.map(input_columns="image",
num_parallel_workers=num_parallel_workers,
operations=transform_img)
data_set = data_set.batch(batch_size, drop_remainder=True)
# apply dataset repeat operation
if repeat_num > 1:
data_set = data_set.repeat(repeat_num)
return data_set
def create_ctpn_dataset(mindrecord_file,
batch_size=1,
repeat_num=1,
device_num=1,
rank_id=0,
is_training=True,
num_parallel_workers=12):
"""Creatr ctpn dataset with MindDataset."""
ds = de.MindDataset(mindrecord_file, columns_list=["image", "annotation"], num_shards=device_num, shard_id=rank_id,\
num_parallel_workers=num_parallel_workers, shuffle=is_training)
decode = C.Decode()
ds = ds.map(operations=decode,
input_columns=["image"],
num_parallel_workers=num_parallel_workers)
compose_map_func = (lambda image, annotation: preprocess_fn(
image, annotation, is_training))
hwc_to_chw = C.HWC2CHW()
normalize_op = C.Normalize((123.675, 116.28, 103.53),
(58.395, 57.12, 57.375))
type_cast0 = CC.TypeCast(mstype.float32)
type_cast1 = CC.TypeCast(mstype.float16)
type_cast2 = CC.TypeCast(mstype.int32)
type_cast3 = CC.TypeCast(mstype.bool_)
if is_training:
ds = ds.map(
operations=compose_map_func,
input_columns=["image", "annotation"],
output_columns=[
"image", "box", "label", "valid_num", "image_shape"
],
column_order=["image", "box", "label", "valid_num", "image_shape"],
num_parallel_workers=num_parallel_workers,
python_multiprocessing=True)
ds = ds.map(operations=[normalize_op, type_cast0],
input_columns=["image"],
num_parallel_workers=num_parallel_workers,
python_multiprocessing=True)
ds = ds.map(operations=[hwc_to_chw, type_cast1],
input_columns=["image"],
num_parallel_workers=num_parallel_workers,
python_multiprocessing=True)
else:
ds = ds.map(
operations=compose_map_func,
input_columns=["image", "annotation"],
output_columns=[
"image", "box", "label", "valid_num", "image_shape"
],
column_order=["image", "box", "label", "valid_num", "image_shape"],
num_parallel_workers=8,
python_multiprocessing=True)
ds = ds.map(operations=[normalize_op, hwc_to_chw, type_cast1],
input_columns=["image"],
num_parallel_workers=8)
# transpose_column from python to c
ds = ds.map(operations=[type_cast1], input_columns=["image_shape"])
ds = ds.map(operations=[type_cast1], input_columns=["box"])
ds = ds.map(operations=[type_cast2], input_columns=["label"])
ds = ds.map(operations=[type_cast3], input_columns=["valid_num"])
ds = ds.batch(batch_size, drop_remainder=True)
ds = ds.repeat(repeat_num)
return ds
def create_dataset(dataset_path, do_train, repeat_num=1, batch_size=32):
"""
create a train or eval dataset.
Args:
dataset_path(string): the path of dataset.
do_train(bool): whether dataset is used for train or eval.
repeat_num(int): the repeat times of dataset. Default: 1
batch_size(int): the batch size of dataset. Default: 32
Returns:
dataset
"""
device_num = int(os.getenv("RANK_SIZE"))
rank_id = int(os.getenv("RANK_ID"))
if device_num == 1:
ds = de.ImageFolderDataset(dataset_path,
num_parallel_workers=8,
shuffle=True)
else:
ds = de.ImageFolderDataset(dataset_path,
num_parallel_workers=8,
shuffle=True,
num_shards=device_num,
shard_id=rank_id)
image_size = 224
mean = [0.485 * 255, 0.456 * 255, 0.406 * 255]
std = [0.229 * 255, 0.224 * 255, 0.225 * 255]
# define map operations
if do_train:
trans = [
C.RandomCropDecodeResize(image_size,
scale=(0.08, 1.0),
ratio=(0.75, 1.333)),
C.RandomHorizontalFlip(prob=0.5),
C.Normalize(mean=mean, std=std),
C.HWC2CHW()
]
else:
trans = [
C.Decode(),
C.Resize((256, 256)),
C.CenterCrop(image_size),
C.Normalize(mean=mean, std=std),
C.HWC2CHW()
]
type_cast_op = C2.TypeCast(mstype.int32)
ds = ds.map(operations=trans,
input_columns="image",
num_parallel_workers=8)
ds = ds.map(operations=type_cast_op,
input_columns="label",
num_parallel_workers=8)
# apply batch operations
ds = ds.batch(batch_size, drop_remainder=True)
# apply dataset repeat operation
ds = ds.repeat(repeat_num)
return ds
def vgg_create_dataset100(data_home,
image_size,
batch_size,
rank_id=0,
rank_size=1,
repeat_num=1,
training=True,
num_samples=None,
shuffle=True):
"""Data operations."""
ds.config.set_seed(1)
data_dir = os.path.join(data_home, "train")
if not training:
data_dir = os.path.join(data_home, "test")
if num_samples is not None:
data_set = ds.Cifar100Dataset(data_dir,
num_shards=rank_size,
shard_id=rank_id,
num_samples=num_samples,
shuffle=shuffle)
else:
data_set = ds.Cifar100Dataset(data_dir,
num_shards=rank_size,
shard_id=rank_id)
input_columns = ["fine_label"]
output_columns = ["label"]
data_set = data_set.rename(input_columns=input_columns,
output_columns=output_columns)
data_set = data_set.project(["image", "label"])
rescale = 1.0 / 255.0
shift = 0.0
# define map operations
random_crop_op = CV.RandomCrop(
(32, 32), (4, 4, 4, 4)) # padding_mode default CONSTANT
random_horizontal_op = CV.RandomHorizontalFlip()
resize_op = CV.Resize(image_size) # interpolation default BILINEAR
rescale_op = CV.Rescale(rescale, shift)
normalize_op = CV.Normalize((0.4465, 0.4822, 0.4914),
(0.2010, 0.1994, 0.2023))
changeswap_op = CV.HWC2CHW()
type_cast_op = C.TypeCast(mstype.int32)
c_trans = []
if training:
c_trans = [random_crop_op, random_horizontal_op]
c_trans += [resize_op, rescale_op, normalize_op, changeswap_op]
# apply map operations on images
data_set = data_set.map(input_columns="label", operations=type_cast_op)
data_set = data_set.map(input_columns="image", operations=c_trans)
# apply repeat operations
data_set = data_set.repeat(repeat_num)
# apply shuffle operations
# data_set = data_set.shuffle(buffer_size=1000)
# apply batch operations
data_set = data_set.batch(batch_size=batch_size, drop_remainder=True)
return data_set
def create_dataset(dataset_path,
do_train,
config,
platform,
repeat_num=1,
batch_size=100):
"""
create a train or eval dataset
Args:
dataset_path(string): the path of dataset.
do_train(bool): whether dataset is used for train or eval.
repeat_num(int): the repeat times of dataset. Default: 1
batch_size(int): the batch size of dataset. Default: 32
Returns:
dataset
"""
if platform == "Ascend":
rank_size = int(os.getenv("RANK_SIZE"))
rank_id = int(os.getenv("RANK_ID"))
if rank_size == 1:
data_set = ds.MindDataset(dataset_path,
num_parallel_workers=8,
shuffle=True)
else:
data_set = ds.MindDataset(dataset_path,
num_parallel_workers=8,
shuffle=True,
num_shards=rank_size,
shard_id=rank_id)
elif platform == "GPU":
if do_train:
from mindspore.communication.management import get_rank, get_group_size
data_set = ds.MindDataset(dataset_path,
num_parallel_workers=8,
shuffle=True,
num_shards=get_group_size(),
shard_id=get_rank())
else:
data_set = ds.MindDataset(dataset_path,
num_parallel_workers=8,
shuffle=False)
else:
raise ValueError("Unsupported platform.")
resize_height = config.image_height
buffer_size = 1000
# define map operations
resize_crop_op = C.RandomCropDecodeResize(resize_height,
scale=(0.08, 1.0),
ratio=(0.75, 1.333))
horizontal_flip_op = C.RandomHorizontalFlip(prob=0.5)
color_op = C.RandomColorAdjust(brightness=0.4,
contrast=0.4,
saturation=0.4)
rescale_op = C.Rescale(1 / 255.0, 0)
normalize_op = C.Normalize(mean=[0.485, 0.456, 0.406],
std=[0.229, 0.224, 0.225])
change_swap_op = C.HWC2CHW()
# define python operations
decode_p = P.Decode()
resize_p = P.Resize(256, interpolation=Inter.BILINEAR)
center_crop_p = P.CenterCrop(224)
totensor = P.ToTensor()
normalize_p = P.Normalize((0.485, 0.456, 0.406), (0.229, 0.224, 0.225))
composeop = P2.Compose(
[decode_p, resize_p, center_crop_p, totensor, normalize_p])
if do_train:
trans = [
resize_crop_op, horizontal_flip_op, color_op, rescale_op,
normalize_op, change_swap_op
]
else:
trans = composeop
type_cast_op = C2.TypeCast(mstype.int32)
data_set = data_set.map(input_columns="image",
operations=trans,
num_parallel_workers=8)
data_set = data_set.map(input_columns="label_list",
operations=type_cast_op,
num_parallel_workers=8)
# apply shuffle operations
data_set = data_set.shuffle(buffer_size=buffer_size)
# apply batch operations
data_set = data_set.batch(batch_size, drop_remainder=True)
# apply dataset repeat operation
data_set = data_set.repeat(repeat_num)
return data_set
def classification_dataset(data_dir,
image_size,
per_batch_size,
rank=0,
group_size=1,
mode='train',
input_mode='folder',
root='',
num_parallel_workers=None,
shuffle=None,
sampler=None,
repeat_num=1,
class_indexing=None,
drop_remainder=True,
transform=None,
target_transform=None):
"""
A function that returns a dataset for classification. The mode of input dataset could be "folder" or "txt".
If it is "folder", all images within one folder have the same label. If it is "txt", all paths of images
are written into a textfile.
Args:
data_dir (str): Path to the root directory that contains the dataset for "input_mode="folder"".
Or path of the textfile that contains every image's path of the dataset.
image_size (str): Size of the input images.
per_batch_size (int): the batch size of evey step during training.
rank (int): The shard ID within num_shards (default=None).
group_size (int): Number of shards that the dataset should be divided
into (default=None).
mode (str): "train" or others. Default: " train".
input_mode (str): The form of the input dataset. "folder" or "txt". Default: "folder".
root (str): the images path for "input_mode="txt"". Default: " ".
num_parallel_workers (int): Number of workers to read the data. Default: None.
shuffle (bool): Whether or not to perform shuffle on the dataset
(default=None, performs shuffle).
sampler (Sampler): Object used to choose samples from the dataset. Default: None.
repeat_num (int): the num of repeat dataset.
class_indexing (dict): A str-to-int mapping from folder name to index
(default=None, the folder names will be sorted
alphabetically and each class will be given a
unique index starting from 0).
Examples:
>>> from mindvision.common.datasets.classification import classification_dataset
>>> # path to imagefolder directory. This directory needs to contain sub-directories which contain the images
>>> dataset_dir = "/path/to/imagefolder_directory"
>>> de_dataset = classification_dataset(train_data_dir, image_size=[224, 244],
>>> per_batch_size=64, rank=0, group_size=4)
>>> # Path of the textfile that contains every image's path of the dataset.
>>> dataset_dir = "/path/to/dataset/images/train.txt"
>>> images_dir = "/path/to/dataset/images"
>>> de_dataset = classification_dataset(train_data_dir, image_size=[224, 244],
>>> per_batch_size=64, rank=0, group_size=4,
>>> input_mode="txt", root=images_dir)
"""
mean = [0.485 * 255, 0.456 * 255, 0.406 * 255]
std = [0.229 * 255, 0.224 * 255, 0.225 * 255]
if transform is None:
if mode == 'train':
transform_img = [
vision.RandomCropDecodeResize(image_size, scale=(0.08, 1.0)),
vision.RandomHorizontalFlip(prob=0.5),
vision.Normalize(mean=mean, std=std),
vision.HWC2CHW()
]
else:
transform_img = [
vision.Decode(),
vision.Resize((256, 256)),
vision.CenterCrop(image_size),
vision.Normalize(mean=mean, std=std),
vision.HWC2CHW()
]
else:
transform_img = transform
if target_transform is None:
transform_label = [C.TypeCast(mstype.int32)]
else:
transform_label = target_transform
if input_mode == 'folder':
de_dataset = de.ImageFolderDataset(
data_dir,
num_parallel_workers=num_parallel_workers,
shuffle=shuffle,
sampler=sampler,
class_indexing=class_indexing,
num_shards=group_size,
shard_id=rank)
else:
dataset = TxtDataset(root, data_dir)
sampler = DistributedSampler(dataset,
rank,
group_size,
shuffle=shuffle)
de_dataset = de.GeneratorDataset(dataset, ["image", "label"],
sampler=sampler)
de_dataset = de_dataset.map(operations=transform_img,
input_columns="image",
num_parallel_workers=8)
de_dataset = de_dataset.map(operations=transform_label,
input_columns="label",
num_parallel_workers=8)
columns_to_project = ["image", "label"]
de_dataset = de_dataset.project(columns=columns_to_project)
de_dataset = de_dataset.batch(per_batch_size,
drop_remainder=drop_remainder)
de_dataset = de_dataset.repeat(repeat_num)
return de_dataset
def create_dataset4(dataset_path,
do_train,
repeat_num=1,
batch_size=32,
target="Ascend",
distribute=False,
enable_cache=False,
cache_session_id=None):
"""
create a train or eval imagenet2012 dataset for se-resnet50
Args:
dataset_path(string): the path of dataset.
do_train(bool): whether dataset is used for train or eval.
repeat_num(int): the repeat times of dataset. Default: 1
batch_size(int): the batch size of dataset. Default: 32
target(str): the device target. Default: Ascend
distribute(bool): data for distribute or not. Default: False
enable_cache(bool): whether tensor caching service is used for eval. Default: False
cache_session_id(int): If enable_cache, cache session_id need to be provided. Default: None
Returns:
dataset
"""
if target == "Ascend":
device_num, rank_id = _get_rank_info()
else:
if distribute:
init()
rank_id = get_rank()
device_num = get_group_size()
else:
device_num = 1
if device_num == 1:
data_set = ds.ImageFolderDataset(dataset_path,
num_parallel_workers=12,
shuffle=True)
else:
data_set = ds.ImageFolderDataset(dataset_path,
num_parallel_workers=12,
shuffle=True,
num_shards=device_num,
shard_id=rank_id)
image_size = 224
mean = [123.68, 116.78, 103.94]
std = [1.0, 1.0, 1.0]
# define map operations
if do_train:
trans = [
C.RandomCropDecodeResize(image_size,
scale=(0.08, 1.0),
ratio=(0.75, 1.333)),
C.RandomHorizontalFlip(prob=0.5),
C.Normalize(mean=mean, std=std),
C.HWC2CHW()
]
else:
trans = [
C.Decode(),
C.Resize(292),
C.CenterCrop(256),
C.Normalize(mean=mean, std=std),
C.HWC2CHW()
]
type_cast_op = C2.TypeCast(mstype.int32)
data_set = data_set.map(operations=trans,
input_columns="image",
num_parallel_workers=12)
# only enable cache for eval
if do_train:
enable_cache = False
if enable_cache:
if not cache_session_id:
raise ValueError(
"A cache session_id must be provided to use cache.")
eval_cache = ds.DatasetCache(session_id=int(cache_session_id), size=0)
data_set = data_set.map(operations=type_cast_op,
input_columns="label",
num_parallel_workers=12,
cache=eval_cache)
else:
data_set = data_set.map(operations=type_cast_op,
input_columns="label",
num_parallel_workers=12)
# apply batch operations
data_set = data_set.batch(batch_size, drop_remainder=True)
# apply dataset repeat operation
data_set = data_set.repeat(repeat_num)
return data_set
def create_dataset(args):
"""Create dataset"""
dataroot = args.dataroot
phase = args.phase
batch_size = args.batch_size
device_num = args.device_num
rank = args.rank
shuffle = args.use_random
max_dataset_size = args.max_dataset_size
cores = multiprocessing.cpu_count()
num_parallel_workers = min(8, int(cores / device_num))
image_size = args.image_size
mean = [0.5 * 255] * 3
std = [0.5 * 255] * 3
if phase == "train":
dataset = UnalignedDataset(dataroot,
phase,
max_dataset_size=max_dataset_size,
use_random=args.use_random)
distributed_sampler = DistributedSampler(len(dataset),
device_num,
rank,
shuffle=shuffle)
ds = de.GeneratorDataset(dataset,
column_names=["image_A", "image_B"],
sampler=distributed_sampler,
num_parallel_workers=num_parallel_workers)
if args.use_random:
trans = [
C.RandomResizedCrop(image_size,
scale=(0.5, 1.0),
ratio=(0.75, 1.333)),
C.RandomHorizontalFlip(prob=0.5),
C.Normalize(mean=mean, std=std),
C.HWC2CHW()
]
else:
trans = [
C.Resize((image_size, image_size)),
C.Normalize(mean=mean, std=std),
C.HWC2CHW()
]
ds = ds.map(operations=trans,
input_columns=["image_A"],
num_parallel_workers=num_parallel_workers)
ds = ds.map(operations=trans,
input_columns=["image_B"],
num_parallel_workers=num_parallel_workers)
ds = ds.batch(batch_size, drop_remainder=True)
ds = ds.repeat(1)
else:
datadir = os.path.join(dataroot, args.data_dir)
dataset = ImageFolderDataset(datadir,
max_dataset_size=max_dataset_size)
ds = de.GeneratorDataset(dataset,
column_names=["image", "image_name"],
num_parallel_workers=num_parallel_workers)
trans = [
C.Resize((image_size, image_size)),
C.Normalize(mean=mean, std=std),
C.HWC2CHW()
]
ds = ds.map(operations=trans,
input_columns=["image"],
num_parallel_workers=num_parallel_workers)
ds = ds.batch(1, drop_remainder=True)
ds = ds.repeat(1)
args.dataset_size = len(dataset)
return ds
def create_dataset1(dataset_path,
do_train,
repeat_num=1,
batch_size=32,
target="Ascend"):
"""
create a train or evaluate cifar10 dataset for resnet50
Args:
dataset_path(string): the path of dataset.
do_train(bool): whether dataset is used for train or eval.
repeat_num(int): the repeat times of dataset. Default: 1
batch_size(int): the batch size of dataset. Default: 32
target(str): the device target. Default: Ascend
Returns:
dataset
"""
if target == "Ascend":
device_num, rank_id = _get_rank_info()
else:
init()
rank_id = get_rank()
device_num = get_group_size()
if device_num == 1:
ds = de.Cifar10Dataset(dataset_path,
num_parallel_workers=8,
shuffle=True)
else:
ds = de.Cifar10Dataset(dataset_path,
num_parallel_workers=8,
shuffle=True,
num_shards=device_num,
shard_id=rank_id)
# define map operations
trans = []
if do_train:
trans += [
C.RandomCrop((32, 32), (4, 4, 4, 4)),
C.RandomHorizontalFlip(prob=0.5)
]
trans += [
C.Resize((224, 224)),
C.Rescale(1.0 / 255.0, 0.0),
C.Normalize([0.4914, 0.4822, 0.4465], [0.2023, 0.1994, 0.2010]),
C.HWC2CHW()
]
type_cast_op = C2.TypeCast(mstype.int32)
ds = ds.map(operations=type_cast_op,
input_columns="label",
num_parallel_workers=8)
ds = ds.map(operations=trans,
input_columns="image",
num_parallel_workers=8)
# apply batch operations
ds = ds.batch(batch_size, drop_remainder=True)
# apply dataset repeat operation
ds = ds.repeat(repeat_num)
return ds
def create_dataset(dataset_path, do_train, config, repeat_num=1):
"""
Create a train or eval dataset.
Args:
dataset_path (string): The path of dataset.
do_train (bool): Whether dataset is used for train or eval.
config: configuration
repeat_num (int): The repeat times of dataset. Default: 1.
Returns:
Dataset.
"""
if do_train:
dataset_path = os.path.join(dataset_path, 'train')
do_shuffle = True
else:
dataset_path = os.path.join(dataset_path, 'eval')
do_shuffle = False
device_id = 0
device_num = 1
if config.platform == "GPU":
if config.run_distribute:
from mindspore.communication.management import get_rank, get_group_size
device_id = get_rank()
device_num = get_group_size()
elif config.platform == "Ascend":
device_id = int(os.getenv('DEVICE_ID'))
device_num = int(os.getenv('RANK_SIZE'))
if device_num == 1 or not do_train:
ds = de.Cifar10Dataset(dataset_path, num_parallel_workers=4, shuffle=do_shuffle)
else:
ds = de.Cifar10Dataset(dataset_path, num_parallel_workers=4, shuffle=do_shuffle,
num_shards=device_num, shard_id=device_id)
resize_height = config.image_height
resize_width = config.image_width
buffer_size = 100
rescale = 1.0 / 255.0
shift = 0.0
# define map operations
random_crop_op = C.RandomCrop((32, 32), (4, 4, 4, 4))
random_horizontal_flip_op = C.RandomHorizontalFlip(device_id / (device_id + 1))
resize_op = C.Resize((resize_height, resize_width))
rescale_op = C.Rescale(rescale, shift)
normalize_op = C.Normalize([0.4914, 0.4822, 0.4465], [0.2023, 0.1994, 0.2010])
change_swap_op = C.HWC2CHW()
trans = []
if do_train:
trans += [random_crop_op, random_horizontal_flip_op]
trans += [resize_op, rescale_op, normalize_op, change_swap_op]
type_cast_op = C2.TypeCast(mstype.int32)
ds = ds.map(input_columns="label", operations=type_cast_op)
ds = ds.map(input_columns="image", operations=trans)
# apply shuffle operations
ds = ds.shuffle(buffer_size=buffer_size)
# apply batch operations
ds = ds.batch(config.batch_size, drop_remainder=True)
# apply dataset repeat operation
ds = ds.repeat(repeat_num)
return ds
