def test_lambdas():
"""
Test Multi Column Python Compose Op
"""
ds.config.set_seed(0)
def test_config(arr, input_columns, output_cols, op_list):
data = ds.NumpySlicesDataset(arr,
column_names=input_columns,
shuffle=False)
data = data.map(operations=op_list,
input_columns=input_columns,
output_columns=output_cols,
column_order=output_cols)
res = []
for i in data.create_dict_iterator(output_numpy=True):
for col_name in output_cols:
res.append(i[col_name].tolist())
return res
arr = ([[1]], [[3]])
assert test_config(arr, ["col0", "col1"], ["a"],
py_transforms.Compose([(lambda x, y: x)])) == [[1]]
assert test_config(arr, ["col0", "col1"], ["a"],
py_transforms.Compose([lambda x, y: x,
lambda x: x])) == [[1]]
assert test_config(arr, ["col0", "col1"], ["a", "b"],
py_transforms.Compose([lambda x, y: x, lambda x: (x, x * 2)])) == \
[[1], [2]]
assert test_config(arr, ["col0", "col1"], ["a", "b"],
[lambda x, y: (x, x + y), lambda x, y:
(x, y * 2)]) == [[1], [8]]
def test_compose():
"""
Test C++ and Python Compose Op
"""
ds.config.set_seed(0)
def test_config(arr, op_list):
try:
data = ds.NumpySlicesDataset(arr,
column_names="col",
shuffle=False)
data = data.map(input_columns=["col"], operations=op_list)
res = []
for i in data.create_dict_iterator(output_numpy=True):
res.append(i["col"].tolist())
return res
except (TypeError, ValueError) as e:
return str(e)
# Test simple compose with only 1 op, this would generate a warning
assert test_config([[1, 0], [3, 4]],
ops.Compose([ops.Fill(2)])) == [[2, 2], [2, 2]]
# Test 1 column -> 2 columns -> 1 -> 2 -> 1
assert test_config([[1, 0]],
ops.Compose([ops.Duplicate(), ops.Concatenate(), ops.Duplicate(), ops.Concatenate()])) \
== [[1, 0] * 4]
# Test one Python transform followed by a C transform. Type after OneHot is a float (mixed use-case)
assert test_config(
[1, 0], ops.Compose([py_ops.OneHotOp(2),
ops.TypeCast(mstype.int32)])) == [[[0, 1]],
[[1, 0]]]
# Test exceptions.
with pytest.raises(TypeError) as error_info:
ops.Compose([1, ops.TypeCast(mstype.int32)])
assert "op_list[0] is not a c_transform op (TensorOp) nor a callable pyfunc." in str(
error_info.value)
# Test empty op list
with pytest.raises(ValueError) as error_info:
test_config([1, 0], ops.Compose([]))
assert "op_list can not be empty." in str(error_info.value)
# Test Python compose op
assert test_config([1, 0],
py_ops.Compose([py_ops.OneHotOp(2)])) == [[[0, 1]],
[[1, 0]]]
assert test_config([1, 0],
py_ops.Compose([py_ops.OneHotOp(2),
(lambda x: x + x)])) == [[[0, 2]],
[[2, 0]]]
# Test nested Python compose op
assert test_config([1, 0],
py_ops.Compose([py_ops.Compose([py_ops.OneHotOp(2)]), (lambda x: x + x)])) \
== [[[0, 2]], [[2, 0]]]
with pytest.raises(TypeError) as error_info:
py_ops.Compose([(lambda x: x + x)])()
assert "Compose was called without an image. Fix invocation (avoid it being invoked as Compose([...])())." in str(
error_info.value)
def data_generator(args):
'''Build train dataloader.'''
mindrecord_path = args.mindrecord_path
dst_w = args.dst_w
dst_h = args.dst_h
batch_size = args.per_batch_size
attri_num = args.attri_num
max_epoch = args.max_epoch
transform_img = F2.Compose([
F.Decode(),
F.Resize((dst_w, dst_h)),
F.RandomHorizontalFlip(prob=0.5),
F.ToTensor(),
F.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5))
])
de_dataset = de.MindDataset(mindrecord_path + "0",
columns_list=["image", "label"],
num_shards=args.world_size,
shard_id=args.local_rank)
de_dataset = de_dataset.map(input_columns="image",
operations=transform_img,
num_parallel_workers=args.workers,
python_multiprocessing=True)
de_dataset = de_dataset.batch(batch_size, drop_remainder=True)
steps_per_epoch = de_dataset.get_dataset_size()
de_dataset = de_dataset.repeat(max_epoch)
de_dataloader = de_dataset.create_tuple_iterator(output_numpy=True)
num_classes = attri_num
return de_dataloader, steps_per_epoch, num_classes
def test_random_apply_exception_random_crop_badinput():
"""
Test RandomApply: test invalid input for one of the transform functions,
expected to raise error
"""
logger.info("test_random_apply_exception_random_crop_badinput")
original_seed = config_get_set_seed(200)
original_num_parallel_workers = config_get_set_num_parallel_workers(1)
# define map operations
transforms_list = [
py_vision.Resize([32, 32]),
py_vision.RandomCrop(100), # crop size > image size
py_vision.RandomRotation(30)
]
transforms = [
py_vision.Decode(),
py_transforms.RandomApply(transforms_list, prob=0.6),
py_vision.ToTensor()
]
transform = py_transforms.Compose(transforms)
# Generate dataset
data = ds.TFRecordDataset(DATA_DIR,
SCHEMA_DIR,
columns_list=["image"],
shuffle=False)
data = data.map(operations=transform, input_columns=["image"])
try:
_ = data.create_dict_iterator(num_epochs=1).get_next()
except RuntimeError as e:
logger.info("Got an exception in DE: {}".format(str(e)))
assert "Crop size" in str(e)
# Restore configuration
ds.config.set_seed(original_seed)
ds.config.set_num_parallel_workers(original_num_parallel_workers)
def test_serdes_pyvision(remove_json_files=True):
"""
Test serdes on py_transform pipeline.
"""
data_dir = [
"../data/dataset/test_tf_file_3_images/train-0000-of-0001.data"
]
schema_file = "../data/dataset/test_tf_file_3_images/datasetSchema.json"
data1 = ds.TFRecordDataset(data_dir,
schema_file,
columns_list=["image", "label"],
shuffle=False)
transforms = [
py_vision.Decode(),
py_vision.CenterCrop([32, 32]),
py_vision.ToTensor()
]
data1 = data1.map(operations=py.Compose(transforms),
input_columns=["image"])
# Current python function derialization will be failed for pickle, so we disable this testcase
# as an exception testcase.
try:
util_check_serialize_deserialize_file(data1,
"pyvision_dataset_pipeline",
remove_json_files)
assert False
except NotImplementedError as e:
assert "python function is not yet supported" in str(e)
def test_one_hot_op():
"""
Test one hot encoding op
"""
logger.info("Test one hot encoding op")
# define map operations
# ds = de.ImageFolderDataset(DATA_DIR, schema=SCHEMA_DIR)
dataset = ds.ImageFolderDataset(DATA_DIR)
num_classes = 2
epsilon_para = 0.1
transforms = [f.OneHotOp(num_classes=num_classes, smoothing_rate=epsilon_para)]
transform_label = f.Compose(transforms)
dataset = dataset.map(operations=transform_label, input_columns=["label"])
golden_label = np.ones(num_classes) * epsilon_para / num_classes
golden_label[1] = 1 - epsilon_para / num_classes
for data in dataset.create_dict_iterator(num_epochs=1, output_numpy=True):
label = data["label"]
logger.info("label is {}".format(label))
logger.info("golden_label is {}".format(golden_label))
assert label.all() == golden_label.all()
logger.info("====test one hot op ok====")
def test_random_apply_md5():
"""
Test RandomApply op with md5 check
"""
logger.info("test_random_apply_md5")
original_seed = config_get_set_seed(10)
original_num_parallel_workers = config_get_set_num_parallel_workers(1)
# define map operations
transforms_list = [py_vision.CenterCrop(64), py_vision.RandomRotation(30)]
transforms = [
py_vision.Decode(),
# Note: using default value "prob=0.5"
py_transforms.RandomApply(transforms_list),
py_vision.ToTensor()
]
transform = py_transforms.Compose(transforms)
# Generate dataset
data = ds.TFRecordDataset(DATA_DIR,
SCHEMA_DIR,
columns_list=["image"],
shuffle=False)
data = data.map(operations=transform, input_columns=["image"])
# check results with md5 comparison
filename = "random_apply_01_result.npz"
save_and_check_md5(data, filename, generate_golden=GENERATE_GOLDEN)
# Restore configuration
ds.config.set_seed(original_seed)
ds.config.set_num_parallel_workers((original_num_parallel_workers))
def create_dataset(batch_size,
train_data_url='',
workers=8,
distributed=False,
input_size=224,
color_jitter=0.4):
"""Create ImageNet training dataset"""
if not os.path.exists(train_data_url):
raise ValueError('Path not exists')
decode_op = py_vision.Decode()
type_cast_op = c_transforms.TypeCast(mstype.int32)
random_resize_crop_bicubic = py_vision.RandomResizedCrop(
size=(input_size, input_size),
scale=SCALE,
ratio=RATIO,
interpolation=Inter.BICUBIC)
random_horizontal_flip_op = py_vision.RandomHorizontalFlip(0.5)
adjust_range = (max(0, 1 - color_jitter), 1 + color_jitter)
random_color_jitter_op = py_vision.RandomColorAdjust(
brightness=adjust_range,
contrast=adjust_range,
saturation=adjust_range)
to_tensor = py_vision.ToTensor()
normalize_op = py_vision.Normalize(IMAGENET_DEFAULT_MEAN,
IMAGENET_DEFAULT_STD)
# assemble all the transforms
image_ops = py_transforms.Compose([
decode_op, random_resize_crop_bicubic, random_horizontal_flip_op,
random_color_jitter_op, to_tensor, normalize_op
])
rank_id = get_rank() if distributed else 0
rank_size = get_group_size() if distributed else 1
dataset_train = ds.ImageFolderDataset(train_data_url,
num_parallel_workers=workers,
shuffle=True,
num_shards=rank_size,
shard_id=rank_id)
dataset_train = dataset_train.map(input_columns=["image"],
operations=image_ops,
num_parallel_workers=workers)
dataset_train = dataset_train.map(input_columns=["label"],
operations=type_cast_op,
num_parallel_workers=workers)
# batch dealing
ds_train = dataset_train.batch(batch_size,
per_batch_map=split_imgs_and_labels,
input_columns=["image", "label"],
num_parallel_workers=2,
drop_remainder=True)
ds_train = ds_train.repeat(1)
return ds_train
def create_dataset_py(dataset_path, do_train, repeat_num=1, batch_size=32, target="Ascend"):
"""
create a train or eval dataset
Args:
dataset_path(string): the path of dataset.
do_train(bool): whether dataset is used for train or eval.
repeat_num(int): the repeat times of dataset. Default: 1
batch_size(int): the batch size of dataset. Default: 32
target(str): the device target. Default: Ascend
Returns:
dataset
"""
if target == "Ascend":
device_num = int(os.getenv("RANK_SIZE"))
rank_id = int(os.getenv("RANK_ID"))
else:
init()
rank_id = get_rank()
device_num = get_group_size()
if do_train:
if device_num == 1:
ds = de.ImageFolderDataset(dataset_path, num_parallel_workers=8, shuffle=True)
else:
ds = de.ImageFolderDataset(dataset_path, num_parallel_workers=8, shuffle=True,
num_shards=device_num, shard_id=rank_id)
else:
ds = de.ImageFolderDataset(dataset_path, num_parallel_workers=8, shuffle=False)
image_size = 224
# define map operations
decode_op = P.Decode()
resize_crop_op = P.RandomResizedCrop(image_size, scale=(0.08, 1.0), ratio=(0.75, 1.333))
horizontal_flip_op = P.RandomHorizontalFlip(prob=0.5)
resize_op = P.Resize(256)
center_crop = P.CenterCrop(image_size)
to_tensor = P.ToTensor()
normalize_op = P.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225])
# define map operations
if do_train:
trans = [decode_op, resize_crop_op, horizontal_flip_op, to_tensor, normalize_op]
else:
trans = [decode_op, resize_op, center_crop, to_tensor, normalize_op]
compose = P2.Compose(trans)
ds = ds.map(operations=compose, input_columns="image", num_parallel_workers=8, python_multiprocessing=True)
# apply batch operations
ds = ds.batch(batch_size, drop_remainder=True)
# apply dataset repeat operation
ds = ds.repeat(repeat_num)
return ds
def test_random_apply_op(plot=False):
"""
Test RandomApply in python transformations
"""
logger.info("test_random_apply_op")
# define map operations
transforms_list = [py_vision.CenterCrop(64), py_vision.RandomRotation(30)]
transforms1 = [
py_vision.Decode(),
py_transforms.RandomApply(transforms_list, prob=0.6),
py_vision.ToTensor()
]
transform1 = py_transforms.Compose(transforms1)
transforms2 = [py_vision.Decode(), py_vision.ToTensor()]
transform2 = py_transforms.Compose(transforms2)
# First dataset
data1 = ds.TFRecordDataset(DATA_DIR,
SCHEMA_DIR,
columns_list=["image"],
shuffle=False)
data1 = data1.map(operations=transform1, input_columns=["image"])
# Second dataset
data2 = ds.TFRecordDataset(DATA_DIR,
SCHEMA_DIR,
columns_list=["image"],
shuffle=False)
data2 = data2.map(operations=transform2, input_columns=["image"])
image_apply = []
image_original = []
for item1, item2 in zip(
data1.create_dict_iterator(num_epochs=1, output_numpy=True),
data2.create_dict_iterator(num_epochs=1, output_numpy=True)):
image1 = (item1["image"].transpose(1, 2, 0) * 255).astype(np.uint8)
image2 = (item2["image"].transpose(1, 2, 0) * 255).astype(np.uint8)
image_apply.append(image1)
image_original.append(image2)
if plot:
visualize_list(image_original, image_apply)
def test_random_choice_comp(plot=False):
"""
Test RandomChoice and compare with single CenterCrop results
"""
logger.info("test_random_choice_comp")
# define map operations
transforms_list = [py_vision.CenterCrop(64)]
transforms1 = [
py_vision.Decode(),
py_transforms.RandomChoice(transforms_list),
py_vision.ToTensor()
]
transform1 = py_transforms.Compose(transforms1)
transforms2 = [
py_vision.Decode(),
py_vision.CenterCrop(64),
py_vision.ToTensor()
]
transform2 = py_transforms.Compose(transforms2)
# First dataset
data1 = ds.TFRecordDataset(DATA_DIR, SCHEMA_DIR, columns_list=["image"], shuffle=False)
data1 = data1.map(operations=transform1, input_columns=["image"])
# Second dataset
data2 = ds.TFRecordDataset(DATA_DIR, SCHEMA_DIR, columns_list=["image"], shuffle=False)
data2 = data2.map(operations=transform2, input_columns=["image"])
image_choice = []
image_original = []
for item1, item2 in zip(data1.create_dict_iterator(num_epochs=1, output_numpy=True),
data2.create_dict_iterator(num_epochs=1, output_numpy=True)):
image1 = (item1["image"].transpose(1, 2, 0) * 255).astype(np.uint8)
image2 = (item2["image"].transpose(1, 2, 0) * 255).astype(np.uint8)
image_choice.append(image1)
image_original.append(image2)
mse = diff_mse(image1, image2)
assert mse == 0
if plot:
visualize_list(image_original, image_choice)
def get_de_dataset(args):
'''get_de_dataset'''
lbl_transforms = [F.ToType(np.int32)]
transform_label = F2.Compose(lbl_transforms)
drop_remainder = False
transforms = [
F.ToPIL(),
F.RandomHorizontalFlip(),
F.ToTensor(),
F.Normalize(mean=[0.5], std=[0.5])
]
transform = F2.Compose(transforms)
cache_path = os.path.join('cache', os.path.basename(args.data_dir),
'data_cache.pkl')
print(cache_path)
if not os.path.exists(os.path.dirname(cache_path)):
os.makedirs(os.path.dirname(cache_path))
dataset = CustomDataset(args.data_dir, cache_path, args.is_distributed)
args.logger.info("dataset len:{}".format(dataset.__len__()))
sampler = DistributedCustomSampler(dataset,
num_replicas=args.world_size,
rank=args.local_rank,
is_distributed=args.is_distributed)
de_dataset = de.GeneratorDataset(dataset, ["image", "label"],
sampler=sampler)
args.logger.info("after sampler de_dataset datasize :{}".format(
de_dataset.get_dataset_size()))
de_dataset = de_dataset.map(input_columns="image", operations=transform)
de_dataset = de_dataset.map(input_columns="label",
operations=transform_label)
de_dataset = de_dataset.project(columns=["image", "label"])
de_dataset = de_dataset.batch(args.per_batch_size,
drop_remainder=drop_remainder)
num_iter_per_npu = math.ceil(
len(dataset) * 1.0 / args.world_size / args.per_batch_size)
num_classes = len(dataset.classes)
return de_dataset, num_iter_per_npu, num_classes
def create_dataset_val(batch_size=128,
val_data_url='',
workers=8,
distributed=False,
input_size=224):
"""Create ImageNet validation dataset"""
if not os.path.exists(val_data_url):
raise ValueError('Path not exists')
rank_id = get_rank() if distributed else 0
rank_size = get_group_size() if distributed else 1
dataset = ds.ImageFolderDataset(val_data_url,
num_parallel_workers=workers,
num_shards=rank_size,
shard_id=rank_id)
scale_size = None
if isinstance(input_size, tuple):
assert len(input_size) == 2
if input_size[-1] == input_size[-2]:
scale_size = int(math.floor(input_size[0] / DEFAULT_CROP_PCT))
else:
scale_size = tuple([int(x / DEFAULT_CROP_PCT) for x in input_size])
else:
scale_size = int(math.floor(input_size / DEFAULT_CROP_PCT))
type_cast_op = c_transforms.TypeCast(mstype.int32)
decode_op = py_vision.Decode()
resize_op = py_vision.Resize(size=scale_size, interpolation=Inter.BICUBIC)
center_crop = py_vision.CenterCrop(size=input_size)
to_tensor = py_vision.ToTensor()
normalize_op = py_vision.Normalize(IMAGENET_DEFAULT_MEAN,
IMAGENET_DEFAULT_STD)
image_ops = py_transforms.Compose(
[decode_op, resize_op, center_crop, to_tensor, normalize_op])
dataset = dataset.map(input_columns=["label"],
operations=type_cast_op,
num_parallel_workers=workers)
dataset = dataset.map(input_columns=["image"],
operations=image_ops,
num_parallel_workers=workers)
dataset = dataset.batch(batch_size,
per_batch_map=split_imgs_and_labels,
input_columns=["image", "label"],
num_parallel_workers=2,
drop_remainder=True)
dataset = dataset.repeat(1)
return dataset
def faceqa_dataset(imlist, per_batch_size, local_rank, world_size):
'''faceqa dataset'''
transform_img = F2.Compose([F.ToTensor()])
dataset = MdFaceDataset(imlist)
sampler = DistributedSampler(dataset, local_rank, world_size)
de_dataset = ds.GeneratorDataset(dataset, ["image", "label"],
sampler=sampler,
num_parallel_workers=8,
python_multiprocessing=True)
de_dataset = de_dataset.map(input_columns="image",
operations=transform_img,
num_parallel_workers=8,
python_multiprocessing=True)
de_dataset = de_dataset.batch(per_batch_size, drop_remainder=True)
return de_dataset
def load_images(paths, batch_size=128):
'''Load images.'''
ll = []
resize = V.Resize((96, 64))
transform = T.Compose(
[V.ToTensor(),
V.Normalize([0.5, 0.5, 0.5], [0.5, 0.5, 0.5])])
for i, _ in enumerate(paths):
im = Image.open(paths[i])
im = resize(im)
img = np.array(im)
ts = transform(img)
ll.append(ts[0])
if len(ll) == batch_size:
yield np.stack(ll, axis=0)
ll.clear()
if ll:
yield np.stack(ll, axis=0)
def test_serdes_pyvision(remove_json_files=True):
"""
Test serdes on py_transform pipeline.
"""
data_dir = [
"../data/dataset/test_tf_file_3_images/train-0000-of-0001.data"
]
schema_file = "../data/dataset/test_tf_file_3_images/datasetSchema.json"
data1 = ds.TFRecordDataset(data_dir,
schema_file,
columns_list=["image", "label"],
shuffle=False)
transforms = [
py_vision.Decode(),
py_vision.CenterCrop([32, 32]),
py_vision.ToTensor()
]
data1 = data1.map(operations=py.Compose(transforms),
input_columns=["image"])
util_check_serialize_deserialize_file(data1, "pyvision_dataset_pipeline",
remove_json_files)
def data_generator_eval(args):
'''Build eval dataloader.'''
mindrecord_path = args.mindrecord_path
dst_w = args.dst_w
dst_h = args.dst_h
batch_size = 1
attri_num = args.attri_num
transform_img = F2.Compose([F.Decode(),
F.Resize((dst_w, dst_h)),
F.ToTensor(),
F.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5))])
de_dataset = de.MindDataset(mindrecord_path + "0", columns_list=["image", "label"])
de_dataset = de_dataset.map(input_columns="image", operations=transform_img, num_parallel_workers=args.workers,
python_multiprocessing=True)
de_dataset = de_dataset.batch(batch_size)
de_dataloader = de_dataset.create_tuple_iterator(output_numpy=True)
steps_per_epoch = de_dataset.get_dataset_size()
print("image number:{0}".format(steps_per_epoch))
num_classes = attri_num
return de_dataloader, steps_per_epoch, num_classes
def test_random_choice_exception_random_crop_badinput():
"""
Test RandomChoice: hit error in RandomCrop with greater crop size,
expected to raise error
"""
logger.info("test_random_choice_exception_random_crop_badinput")
# define map operations
# note: crop size[5000, 5000] > image size[4032, 2268]
transforms_list = [py_vision.RandomCrop(5000)]
transforms = [
py_vision.Decode(),
py_transforms.RandomChoice(transforms_list),
py_vision.ToTensor()
]
transform = py_transforms.Compose(transforms)
# Generate dataset
data = ds.TFRecordDataset(DATA_DIR, SCHEMA_DIR, columns_list=["image"], shuffle=False)
data = data.map(operations=transform, input_columns=["image"])
try:
_ = data.create_dict_iterator(num_epochs=1).get_next()
except RuntimeError as e:
logger.info("Got an exception in DE: {}".format(str(e)))
assert "Crop size" in str(e)
def create_dataset(dataset_path,
do_train,
config,
platform,
repeat_num=1,
batch_size=100):
"""
create a train or eval dataset
Args:
dataset_path(string): the path of dataset.
do_train(bool): whether dataset is used for train or eval.
repeat_num(int): the repeat times of dataset. Default: 1
batch_size(int): the batch size of dataset. Default: 32
Returns:
dataset
"""
if platform == "Ascend":
rank_size = int(os.getenv("RANK_SIZE"))
rank_id = int(os.getenv("RANK_ID"))
if rank_size == 1:
data_set = ds.MindDataset(dataset_path,
num_parallel_workers=8,
shuffle=True)
else:
data_set = ds.MindDataset(dataset_path,
num_parallel_workers=8,
shuffle=True,
num_shards=rank_size,
shard_id=rank_id)
elif platform == "GPU":
if do_train:
from mindspore.communication.management import get_rank, get_group_size
data_set = ds.MindDataset(dataset_path,
num_parallel_workers=8,
shuffle=True,
num_shards=get_group_size(),
shard_id=get_rank())
else:
data_set = ds.MindDataset(dataset_path,
num_parallel_workers=8,
shuffle=False)
else:
raise ValueError("Unsupported platform.")
resize_height = config.image_height
buffer_size = 1000
# define map operations
resize_crop_op = C.RandomCropDecodeResize(resize_height,
scale=(0.08, 1.0),
ratio=(0.75, 1.333))
horizontal_flip_op = C.RandomHorizontalFlip(prob=0.5)
color_op = C.RandomColorAdjust(brightness=0.4,
contrast=0.4,
saturation=0.4)
rescale_op = C.Rescale(1 / 255.0, 0)
normalize_op = C.Normalize(mean=[0.485, 0.456, 0.406],
std=[0.229, 0.224, 0.225])
change_swap_op = C.HWC2CHW()
# define python operations
decode_p = P.Decode()
resize_p = P.Resize(256, interpolation=Inter.BILINEAR)
center_crop_p = P.CenterCrop(224)
totensor = P.ToTensor()
normalize_p = P.Normalize((0.485, 0.456, 0.406), (0.229, 0.224, 0.225))
composeop = P2.Compose(
[decode_p, resize_p, center_crop_p, totensor, normalize_p])
if do_train:
trans = [
resize_crop_op, horizontal_flip_op, color_op, rescale_op,
normalize_op, change_swap_op
]
else:
trans = composeop
type_cast_op = C2.TypeCast(mstype.int32)
data_set = data_set.map(input_columns="image",
operations=trans,
num_parallel_workers=8)
data_set = data_set.map(input_columns="label_list",
operations=type_cast_op,
num_parallel_workers=8)
# apply shuffle operations
data_set = data_set.shuffle(buffer_size=buffer_size)
# apply batch operations
data_set = data_set.batch(batch_size, drop_remainder=True)
# apply dataset repeat operation
data_set = data_set.repeat(repeat_num)
return data_set
def main(args):
if not os.path.exists(args.test_dir):
args.logger.info('ERROR, test_dir is not exists, please set test_dir in config.py.')
return 0
all_start_time = time.time()
net = get_model(args)
compile_time_used = time.time() - all_start_time
args.logger.info('INFO, graph compile finished, time used:{:.2f}s, start calculate img embedding'.
format(compile_time_used))
img_transforms = transforms.Compose([vision.ToTensor(), vision.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5))])
#for test images
args.logger.info('INFO, start step1, calculate test img embedding, weight file = {}'.format(args.weight))
step1_start_time = time.time()
ds, img_tot, all_labels = get_dataloader(args.test_img_predix, args.test_img_list,
args.test_batch_size, img_transforms)
args.logger.info('INFO, dataset total test img:{}, total test batch:{}'.format(img_tot, ds.get_dataset_size()))
test_embedding_tot_np = np.zeros((img_tot, args.emb_size))
test_img_labels = all_labels
data_loader = ds.create_dict_iterator(output_numpy=True, num_epochs=1)
for i, data in enumerate(data_loader):
img, idxs = data["image"], data["index"]
out = net(Tensor(img)).asnumpy().astype(np.float32)
embeddings = l2normalize(out)
for batch in range(embeddings.shape[0]):
test_embedding_tot_np[idxs[batch]] = embeddings[batch]
try:
check_minmax(args, np.linalg.norm(test_embedding_tot_np, ord=2, axis=1))
except ValueError:
return 0
test_embedding_tot = {}
for idx, label in enumerate(test_img_labels):
test_embedding_tot[label] = test_embedding_tot_np[idx]
step2_start_time = time.time()
step1_time_used = step2_start_time - step1_start_time
args.logger.info('INFO, step1 finished, time used:{:.2f}s, start step2, calculate dis img embedding'.
format(step1_time_used))
# for dis images
ds_dis, img_tot, _ = get_dataloader(args.dis_img_predix, args.dis_img_list, args.dis_batch_size, img_transforms)
dis_embedding_tot_np = np.zeros((img_tot, args.emb_size))
total_batch = ds_dis.get_dataset_size()
args.logger.info('INFO, dataloader total dis img:{}, total dis batch:{}'.format(img_tot, total_batch))
start_time = time.time()
img_per_gpu = int(math.ceil(1.0 * img_tot / args.world_size))
delta_num = img_per_gpu * args.world_size - img_tot
start_idx = img_per_gpu * args.local_rank - max(0, args.local_rank - (args.world_size - delta_num))
data_loader = ds_dis.create_dict_iterator(output_numpy=True, num_epochs=1)
for idx, data in enumerate(data_loader):
img = data["image"]
out = net(Tensor(img)).asnumpy().astype(np.float32)
embeddings = l2normalize(out)
dis_embedding_tot_np[start_idx:(start_idx + embeddings.shape[0])] = embeddings
start_idx += embeddings.shape[0]
if args.local_rank % 8 == 0 and idx % args.log_interval == 0 and idx > 0:
speed = 1.0 * (args.dis_batch_size * args.log_interval * args.world_size) / (time.time() - start_time)
time_left = (total_batch - idx - 1) * args.dis_batch_size *args.world_size / speed
args.logger.info('INFO, processed [{}/{}], speed: {:.2f} img/s, left:{:.2f}s'.
format(idx, total_batch, speed, time_left))
start_time = time.time()
try:
check_minmax(args, np.linalg.norm(dis_embedding_tot_np, ord=2, axis=1))
except ValueError:
return 0
step3_start_time = time.time()
step2_time_used = step3_start_time - step2_start_time
args.logger.info('INFO, step2 finished, time used:{:.2f}s, start step3, calculate top1 acc'.format(step2_time_used))
# clear npu memory
img = None
net = None
dis_embedding_tot_np = np.transpose(dis_embedding_tot_np, (1, 0))
args.logger.info('INFO, calculate top1 acc dis_embedding_tot_np shape:{}'.format(dis_embedding_tot_np.shape))
# find best match
assert len(args.test_img_list) % 2 == 0
task_num = int(len(args.test_img_list) / 2)
correct = np.array([0] * (2 * task_num))
tot = np.array([0] * task_num)
for i in range(int(len(args.test_img_list) / 2)):
jk_list = args.test_img_list[2 * i]
zj_list = args.test_img_list[2 * i + 1]
zj2jk_pairs = sorted(generate_test_pair(jk_list, zj_list))
sampler = DistributedSampler(zj2jk_pairs)
args.logger.info('INFO, calculate top1 acc sampler len:{}'.format(len(sampler)))
for idx in sampler:
out1, out2 = cal_topk(args, idx, zj2jk_pairs, test_embedding_tot, dis_embedding_tot_np)
correct[2 * i] += out1[0]
correct[2 * i + 1] += out1[1]
tot[i] += out2[0]
args.logger.info('local_rank={},tot={},correct={}'.format(args.local_rank, tot, correct))
step3_time_used = time.time() - step3_start_time
args.logger.info('INFO, step3 finished, time used:{:.2f}s'.format(step3_time_used))
args.logger.info('weight:{}'.format(args.weight))
for i in range(int(len(args.test_img_list) / 2)):
test_set_name = 'test_dataset'
zj2jk_acc = correct[2 * i] / tot[i]
jk2zj_acc = correct[2 * i + 1] / tot[i]
avg_acc = (zj2jk_acc + jk2zj_acc) / 2
results = '[{}]: zj2jk={:.4f}, jk2zj={:.4f}, avg={:.4f}'.format(test_set_name, zj2jk_acc, jk2zj_acc, avg_acc)
args.logger.info(results)
args.logger.info('INFO, tot time used: {:.2f}s'.format(time.time() - all_start_time))
return 0
def create_dataset_py(dataset_path,
do_train,
config,
device_target,
repeat_num=1,
batch_size=32):
"""
create a train or eval dataset
Args:
dataset_path(string): the path of dataset.
do_train(bool): whether dataset is used for train or eval.
repeat_num(int): the repeat times of dataset. Default: 1.
batch_size(int): the batch size of dataset. Default: 32.
Returns:
dataset
"""
if device_target == "Ascend":
rank_size = int(os.getenv("RANK_SIZE"))
rank_id = int(os.getenv("RANK_ID"))
if do_train:
if rank_size == 1:
data_set = ds.ImageFolderDataset(dataset_path,
num_parallel_workers=8,
shuffle=True)
else:
data_set = ds.ImageFolderDataset(dataset_path,
num_parallel_workers=8,
shuffle=True,
num_shards=rank_size,
shard_id=rank_id)
else:
data_set = ds.ImageFolderDataset(dataset_path,
num_parallel_workers=8,
shuffle=False)
else:
raise ValueError("Unsupported device target.")
resize_height = 224
if do_train:
buffer_size = 20480
# apply shuffle operations
data_set = data_set.shuffle(buffer_size=buffer_size)
# define map operations
decode_op = P.Decode()
resize_crop_op = P.RandomResizedCrop(resize_height,
scale=(0.08, 1.0),
ratio=(0.75, 1.333))
horizontal_flip_op = P.RandomHorizontalFlip(prob=0.5)
resize_op = P.Resize(256)
center_crop = P.CenterCrop(resize_height)
to_tensor = P.ToTensor()
normalize_op = P.Normalize(mean=[0.485, 0.456, 0.406],
std=[0.229, 0.224, 0.225])
if do_train:
trans = [
decode_op, resize_crop_op, horizontal_flip_op, to_tensor,
normalize_op
]
else:
trans = [decode_op, resize_op, center_crop, to_tensor, normalize_op]
compose = P2.Compose(trans)
data_set = data_set.map(operations=compose,
input_columns="image",
num_parallel_workers=8,
python_multiprocessing=True)
# apply batch operations
data_set = data_set.batch(batch_size, drop_remainder=True)
# apply dataset repeat operation
data_set = data_set.repeat(repeat_num)
return data_set