def test_random_crop_op_py(plot=False):
"""
Test RandomCrop op in py transforms
"""
logger.info("test_random_crop_op_py")
# First dataset
data1 = ds.TFRecordDataset(DATA_DIR, SCHEMA_DIR, columns_list=["image"], shuffle=False)
transforms1 = [
py_vision.Decode(),
py_vision.RandomCrop([512, 512], [200, 200, 200, 200]),
py_vision.ToTensor()
]
transform1 = mindspore.dataset.transforms.py_transforms.Compose(transforms1)
data1 = data1.map(operations=transform1, input_columns=["image"])
# Second dataset
# Second dataset for comparison
data2 = ds.TFRecordDataset(DATA_DIR, SCHEMA_DIR, columns_list=["image"], shuffle=False)
transforms2 = [
py_vision.Decode(),
py_vision.ToTensor()
]
transform2 = mindspore.dataset.transforms.py_transforms.Compose(transforms2)
data2 = data2.map(operations=transform2, input_columns=["image"])
crop_images = []
original_images = []
for item1, item2 in zip(data1.create_dict_iterator(num_epochs=1, output_numpy=True),
data2.create_dict_iterator(num_epochs=1, output_numpy=True)):
crop = (item1["image"].transpose(1, 2, 0) * 255).astype(np.uint8)
original = (item2["image"].transpose(1, 2, 0) * 255).astype(np.uint8)
crop_images.append(crop)
original_images.append(original)
if plot:
visualize_list(original_images, crop_images)
def test_random_crop_05_py():
"""
Test RandomCrop op with py_transforms:
input image size < crop size but pad_if_needed is enabled,
expected to pass
"""
logger.info("test_random_crop_05_py")
original_seed = config_get_set_seed(0)
original_num_parallel_workers = config_get_set_num_parallel_workers(1)
# Generate dataset
data = ds.TFRecordDataset(DATA_DIR, SCHEMA_DIR, columns_list=["image"], shuffle=False)
# Note: The size of the image is 4032*2268
transforms = [
py_vision.Decode(),
py_vision.RandomCrop([2268, 4033], [200, 200, 200, 200], pad_if_needed=True),
py_vision.ToTensor()
]
transform = mindspore.dataset.transforms.py_transforms.Compose(transforms)
data = data.map(operations=transform, input_columns=["image"])
filename = "random_crop_05_py_result.npz"
save_and_check_md5(data, filename, generate_golden=GENERATE_GOLDEN)
# Restore config setting
ds.config.set_seed(original_seed)
ds.config.set_num_parallel_workers(original_num_parallel_workers)
def test_random_crop_comp(plot=False):
"""
Test RandomCrop and compare between python and c image augmentation
"""
logger.info("Test RandomCrop with c_transform and py_transform comparison")
cropped_size = 512
# First dataset
data1 = ds.TFRecordDataset(DATA_DIR, SCHEMA_DIR, columns_list=["image"], shuffle=False)
random_crop_op = c_vision.RandomCrop(cropped_size)
decode_op = c_vision.Decode()
data1 = data1.map(operations=decode_op, input_columns=["image"])
data1 = data1.map(operations=random_crop_op, input_columns=["image"])
# Second dataset
data2 = ds.TFRecordDataset(DATA_DIR, SCHEMA_DIR, columns_list=["image"], shuffle=False)
transforms = [
py_vision.Decode(),
py_vision.RandomCrop(cropped_size),
py_vision.ToTensor()
]
transform = mindspore.dataset.transforms.py_transforms.Compose(transforms)
data2 = data2.map(operations=transform, input_columns=["image"])
image_c_cropped = []
image_py_cropped = []
for item1, item2 in zip(data1.create_dict_iterator(num_epochs=1, output_numpy=True),
data2.create_dict_iterator(num_epochs=1, output_numpy=True)):
c_image = item1["image"]
py_image = (item2["image"].transpose(1, 2, 0) * 255).astype(np.uint8)
image_c_cropped.append(c_image)
image_py_cropped.append(py_image)
if plot:
visualize_list(image_c_cropped, image_py_cropped, visualize_mode=2)
def test_random_crop_08_py():
"""
Test RandomCrop op with py_transforms: padding_mode is Border.EDGE,
expected to pass
"""
logger.info("test_random_crop_08_py")
original_seed = config_get_set_seed(0)
original_num_parallel_workers = config_get_set_num_parallel_workers(1)
# Generate dataset
data = ds.TFRecordDataset(DATA_DIR, SCHEMA_DIR, columns_list=["image"], shuffle=False)
# Note: The padding_mode is Border.EDGE.
transforms = [
py_vision.Decode(),
py_vision.RandomCrop(512, [200, 200, 200, 200], padding_mode=mode.Border.EDGE),
py_vision.ToTensor()
]
transform = mindspore.dataset.transforms.py_transforms.Compose(transforms)
data = data.map(operations=transform, input_columns=["image"])
filename = "random_crop_08_py_result.npz"
save_and_check_md5(data, filename, generate_golden=GENERATE_GOLDEN)
# Restore config setting
ds.config.set_seed(original_seed)
ds.config.set_num_parallel_workers(original_num_parallel_workers)
def test_random_crop_01_py():
"""
Test RandomCrop op with py_transforms: size is a single integer, expected to pass
"""
logger.info("test_random_crop_01_py")
original_seed = config_get_set_seed(0)
original_num_parallel_workers = config_get_set_num_parallel_workers(1)
# Generate dataset
data = ds.TFRecordDataset(DATA_DIR, SCHEMA_DIR, columns_list=["image"], shuffle=False)
# Note: If size is an int, a square crop of size (size, size) is returned.
transforms = [
py_vision.Decode(),
py_vision.RandomCrop(512),
py_vision.ToTensor()
]
transform = mindspore.dataset.transforms.py_transforms.Compose(transforms)
data = data.map(operations=transform, input_columns=["image"])
filename = "random_crop_01_py_result.npz"
save_and_check_md5(data, filename, generate_golden=GENERATE_GOLDEN)
# Restore config setting
ds.config.set_seed(original_seed)
ds.config.set_num_parallel_workers(original_num_parallel_workers)
def test_random_crop_02_py():
"""
Test RandomCrop op with py_transforms: size is a list/tuple with length 2, expected to pass
"""
logger.info("test_random_crop_02_py")
original_seed = config_get_set_seed(0)
original_num_parallel_workers = config_get_set_num_parallel_workers(1)
# Generate dataset
data = ds.TFRecordDataset(DATA_DIR, SCHEMA_DIR, columns_list=["image"], shuffle=False)
# Note: If size is a sequence of length 2, it should be (height, width).
transforms = [
py_vision.Decode(),
py_vision.RandomCrop([512, 375]),
py_vision.ToTensor()
]
transform = mindspore.dataset.transforms.py_transforms.Compose(transforms)
data = data.map(operations=transform, input_columns=["image"])
filename = "random_crop_02_py_result.npz"
save_and_check_md5(data, filename, generate_golden=GENERATE_GOLDEN)
# Restore config setting
ds.config.set_seed(original_seed)
ds.config.set_num_parallel_workers(original_num_parallel_workers)
def test_random_crop_04_py():
"""
Test RandomCrop op with py_transforms:
input image size < crop size, expected to fail
"""
logger.info("test_random_crop_04_py")
# Generate dataset
data = ds.TFRecordDataset(DATA_DIR,
SCHEMA_DIR,
columns_list=["image"],
shuffle=False)
# Note: The size of the image is 4032*2268
transforms = [
py_vision.Decode(),
py_vision.RandomCrop([2268, 4033]),
py_vision.ToTensor()
]
transform = mindspore.dataset.transforms.py_transforms.Compose(transforms)
data = data.map(operations=transform, input_columns=["image"])
try:
data.create_dict_iterator(num_epochs=1).__next__()
except RuntimeError as e:
logger.info("Got an exception in DE: {}".format(str(e)))
assert "Crop size" in str(e)
def test_random_crop_06_py():
"""
Test RandomCrop op with py_transforms:
invalid size, expected to raise TypeError
"""
logger.info("test_random_crop_06_py")
# Generate dataset
data = ds.TFRecordDataset(DATA_DIR,
SCHEMA_DIR,
columns_list=["image"],
shuffle=False)
try:
# Note: if size is neither an int nor a list of length 2, an exception will raise
transforms = [
py_vision.Decode(),
py_vision.RandomCrop([512, 512, 375]),
py_vision.ToTensor()
]
transform = mindspore.dataset.transforms.py_transforms.Compose(
transforms)
data = data.map(operations=transform, input_columns=["image"])
except TypeError as e:
logger.info("Got an exception in DE: {}".format(str(e)))
assert "Size should be a single integer" in str(e)
def test_random_apply_exception_random_crop_badinput():
"""
Test RandomApply: test invalid input for one of the transform functions,
expected to raise error
"""
logger.info("test_random_apply_exception_random_crop_badinput")
original_seed = config_get_set_seed(200)
original_num_parallel_workers = config_get_set_num_parallel_workers(1)
# define map operations
transforms_list = [
py_vision.Resize([32, 32]),
py_vision.RandomCrop(100), # crop size > image size
py_vision.RandomRotation(30)
]
transforms = [
py_vision.Decode(),
py_transforms.RandomApply(transforms_list, prob=0.6),
py_vision.ToTensor()
]
transform = py_transforms.Compose(transforms)
# Generate dataset
data = ds.TFRecordDataset(DATA_DIR,
SCHEMA_DIR,
columns_list=["image"],
shuffle=False)
data = data.map(operations=transform, input_columns=["image"])
try:
_ = data.create_dict_iterator(num_epochs=1).get_next()
except RuntimeError as e:
logger.info("Got an exception in DE: {}".format(str(e)))
assert "Crop size" in str(e)
# Restore configuration
ds.config.set_seed(original_seed)
ds.config.set_num_parallel_workers(original_num_parallel_workers)
def test_random_order_md5():
"""
Test RandomOrder op with md5 check
"""
logger.info("test_random_order_md5")
original_seed = config_get_set_seed(8)
original_num_parallel_workers = config_get_set_num_parallel_workers(1)
# define map operations
transforms_list = [py_vision.RandomCrop(64), py_vision.RandomRotation(30)]
transforms = [
py_vision.Decode(),
py_transforms.RandomOrder(transforms_list),
py_vision.ToTensor()
]
transform = py_transforms.Compose(transforms)
# Generate dataset
data = ds.TFRecordDataset(DATA_DIR,
SCHEMA_DIR,
columns_list=["image"],
shuffle=False)
data = data.map(operations=transform, input_columns=["image"])
# check results with md5 comparison
filename = "random_order_01_result.npz"
save_and_check_md5(data, filename, generate_golden=GENERATE_GOLDEN)
# Restore configuration
ds.config.set_seed(original_seed)
ds.config.set_num_parallel_workers((original_num_parallel_workers))
def test_deterministic_python_seed_multi_thread():
"""
Test deterministic execution with seed in python, this fails with multi-thread pyfunc run
"""
logger.info("test_deterministic_python_seed_multi_thread")
# Save original configuration values
num_parallel_workers_original = ds.config.get_num_parallel_workers()
seed_original = ds.config.get_seed()
ds.config.set_num_parallel_workers(3)
ds.config.set_seed(0)
# when we set the seed all operations within our dataset should be deterministic
# First dataset
data1 = ds.TFRecordDataset(DATA_DIR,
SCHEMA_DIR,
columns_list=["image"],
shuffle=False)
transforms = [
py_vision.Decode(),
py_vision.RandomCrop([512, 512], [200, 200, 200, 200]),
py_vision.ToTensor(),
]
transform = mindspore.dataset.transforms.py_transforms.Compose(transforms)
data1 = data1.map(operations=transform,
input_columns=["image"],
python_multiprocessing=True)
data1_output = []
# config.set_seed() calls random.seed()
for data_one in data1.create_dict_iterator(num_epochs=1,
output_numpy=True):
data1_output.append(data_one["image"])
# Second dataset
data2 = ds.TFRecordDataset(DATA_DIR,
SCHEMA_DIR,
columns_list=["image"],
shuffle=False)
# If seed is set up on constructor
data2 = data2.map(operations=transform,
input_columns=["image"],
python_multiprocessing=True)
# config.set_seed() calls random.seed()
ds.config.set_seed(0)
data2_output = []
for data_two in data2.create_dict_iterator(num_epochs=1,
output_numpy=True):
data2_output.append(data_two["image"])
try:
np.testing.assert_equal(data1_output, data2_output)
except Exception as e:
# expect output to not match during multi-threaded execution
logger.info("Got an exception in DE: {}".format(str(e)))
assert "Array" in str(e)
# Restore original configuration values
ds.config.set_num_parallel_workers(num_parallel_workers_original)
ds.config.set_seed(seed_original)
def test_deterministic_python_seed():
"""
Test deterministic execution with seed in python
"""
logger.info("test_deterministic_python_seed")
# Save original configuration values
num_parallel_workers_original = ds.config.get_num_parallel_workers()
seed_original = ds.config.get_seed()
ds.config.set_seed(0)
ds.config.set_num_parallel_workers(1)
# First dataset
data1 = ds.TFRecordDataset(DATA_DIR,
SCHEMA_DIR,
columns_list=["image"],
shuffle=False)
transforms = [
py_vision.Decode(),
py_vision.RandomCrop([512, 512], [200, 200, 200, 200]),
py_vision.ToTensor(),
]
transform = mindspore.dataset.transforms.py_transforms.Compose(transforms)
data1 = data1.map(operations=transform, input_columns=["image"])
data1_output = []
# config.set_seed() calls random.seed()
for data_one in data1.create_dict_iterator(num_epochs=1,
output_numpy=True):
data1_output.append(data_one["image"])
# Second dataset
data2 = ds.TFRecordDataset(DATA_DIR,
SCHEMA_DIR,
columns_list=["image"],
shuffle=False)
data2 = data2.map(operations=transform, input_columns=["image"])
# config.set_seed() calls random.seed(), resets seed for next dataset iterator
ds.config.set_seed(0)
data2_output = []
for data_two in data2.create_dict_iterator(num_epochs=1,
output_numpy=True):
data2_output.append(data_two["image"])
np.testing.assert_equal(data1_output, data2_output)
# Restore original configuration values
ds.config.set_num_parallel_workers(num_parallel_workers_original)
ds.config.set_seed(seed_original)
def test_random_crop_09():
"""
Test RandomCrop op: invalid type of input image (not PIL), expected to raise TypeError
"""
logger.info("test_random_crop_09")
# Generate dataset
data = ds.TFRecordDataset(DATA_DIR, SCHEMA_DIR, columns_list=["image"], shuffle=False)
transforms = [
py_vision.Decode(),
py_vision.ToTensor(),
# Note: if input is not PIL image, TypeError will raise
py_vision.RandomCrop(512)
]
transform = mindspore.dataset.transforms.py_transforms.Compose(transforms)
data = data.map(operations=transform, input_columns=["image"])
try:
data.create_dict_iterator(num_epochs=1).__next__()
except RuntimeError as e:
logger.info("Got an exception in DE: {}".format(str(e)))
assert "should be PIL image" in str(e)
def test_random_choice_exception_random_crop_badinput():
"""
Test RandomChoice: hit error in RandomCrop with greater crop size,
expected to raise error
"""
logger.info("test_random_choice_exception_random_crop_badinput")
# define map operations
# note: crop size[5000, 5000] > image size[4032, 2268]
transforms_list = [py_vision.RandomCrop(5000)]
transforms = [
py_vision.Decode(),
py_transforms.RandomChoice(transforms_list),
py_vision.ToTensor()
]
transform = py_transforms.Compose(transforms)
# Generate dataset
data = ds.TFRecordDataset(DATA_DIR, SCHEMA_DIR, columns_list=["image"], shuffle=False)
data = data.map(operations=transform, input_columns=["image"])
try:
_ = data.create_dict_iterator(num_epochs=1).get_next()
except RuntimeError as e:
logger.info("Got an exception in DE: {}".format(str(e)))
assert "Crop size" in str(e)
def set_random_seed(img_name, seed):
random.seed(seed)
return img_name
ds.config.set_seed(8)
DATA_DIR = opt.data_url
train_dataset_generator = DatasetGenerator(DATA_DIR, train=True)
train_dataset = ds.GeneratorDataset(train_dataset_generator, ["hazy", "gt", "img_name"], shuffle=True)
test_dataset_generator = DatasetGenerator(DATA_DIR, train=False)
test_dataset = ds.GeneratorDataset(test_dataset_generator, ["hazy", "gt", "img_name"], shuffle=False)
transforms_list = [
decode,
(lambda img_name: set_random_seed(img_name, dataset_generator.get_seed())),
py_trans.RandomCrop(opt.crop_size),
py_trans.ToTensor(),
]
compose_trans = Compose(transforms_list)
train_dataset = train_dataset.map(operations=compose_trans, input_columns=["hazy"])
train_dataset = train_dataset.map(operations=compose_trans, input_columns=["gt"])
if __name__ == '__main__':
dataset_generator = DatasetGenerator(DATA_DIR, train=True, size=192)
dataset = ds.GeneratorDataset(dataset_generator, ["hazy", "gt", "img_name"], shuffle=False)
transforms_list = [
decode,
(lambda img_name: set_random_seed(img_name, dataset_generator.get_seed())),
py_trans.RandomCrop(192),
data_path = args.data_path
best_acc = 0
best_acc = 0 # best test accuracy
start_epoch = 0
feature_dim = args.low_dim
wG = 0
start_time = time.time()
print("==> Loading data")
# Data Loading code
transform_train = Compose([
decode,
py_trans.Pad(10),
py_trans.RandomCrop((args.img_h, args.img_w)),
py_trans.RandomHorizontalFlip(),
py_trans.ToTensor(),
py_trans.Normalize(mean=[0.485, 0.456, 0.406],
std=[0.229, 0.224, 0.225])
])
transform_test = Compose([
decode,
py_trans.Resize((args.img_h, args.img_w)),
py_trans.ToTensor(),
py_trans.Normalize(mean=[0.485, 0.456, 0.406],
std=[0.229, 0.224, 0.225])
])
ifDebug_dic = {"yes": True, "no": False}
