def test_affine(self):
# Tests on square and rectangular images
scripted_affine = torch.jit.script(F.affine)
data = [self._create_data(26, 26, device=self.device), self._create_data(32, 26, device=self.device)]
for tensor, pil_img in data:
for dt in [None, torch.float32, torch.float64, torch.float16]:
if dt == torch.float16 and torch.device(self.device).type == "cpu":
# skip float16 on CPU case
continue
if dt is not None:
tensor = tensor.to(dtype=dt)
self._test_affine_identity_map(tensor, scripted_affine)
if pil_img.size[0] == pil_img.size[1]:
self._test_affine_square_rotations(tensor, pil_img, scripted_affine)
else:
self._test_affine_rect_rotations(tensor, pil_img, scripted_affine)
self._test_affine_translations(tensor, pil_img, scripted_affine)
self._test_affine_all_ops(tensor, pil_img, scripted_affine)
batch_tensors = self._create_data_batch(26, 36, num_samples=4, device=self.device)
if dt is not None:
batch_tensors = batch_tensors.to(dtype=dt)
self._test_fn_on_batch(
batch_tensors, F.affine, angle=-43, translate=[-3, 4], scale=1.2, shear=[4.0, 5.0]
)
tensor, pil_img = data[0]
# assert deprecation warning and non-BC
with self.assertWarnsRegex(UserWarning, r"Argument resample is deprecated and will be removed"):
res1 = F.affine(tensor, 45, translate=[0, 0], scale=1.0, shear=[0.0, 0.0], resample=2)
res2 = F.affine(tensor, 45, translate=[0, 0], scale=1.0, shear=[0.0, 0.0], interpolation=BILINEAR)
self.assertTrue(res1.equal(res2))
# assert changed type warning
with self.assertWarnsRegex(UserWarning, r"Argument interpolation should be of type InterpolationMode"):
res1 = F.affine(tensor, 45, translate=[0, 0], scale=1.0, shear=[0.0, 0.0], interpolation=2)
res2 = F.affine(tensor, 45, translate=[0, 0], scale=1.0, shear=[0.0, 0.0], interpolation=BILINEAR)
self.assertTrue(res1.equal(res2))
with self.assertWarnsRegex(UserWarning, r"Argument fillcolor is deprecated and will be removed"):
res1 = F.affine(pil_img, 45, translate=[0, 0], scale=1.0, shear=[0.0, 0.0], fillcolor=10)
res2 = F.affine(pil_img, 45, translate=[0, 0], scale=1.0, shear=[0.0, 0.0], fill=10)
self.assertEqual(res1, res2)
def spatial_transform(self, input1, label1):
vflip = torch.rand(1).item() > 0.5
hflip = torch.rand(1).item() > 0.5
deg = torch.LongTensor(1).random_(-20, 20).item()
scale = torch.FloatTensor(1).uniform_(0.8, 1.2)
image_list = [input1, label1]
for i, p in enumerate(image_list):
dtype = p.dtype
p_min = p.min()
p_max = p.max()
p = (p - p_min) / ((p_max - p_min) + 0.001)
p = TF.to_pil_image(p.float())
if vflip:
p = TF.vflip(p)
if hflip:
p = TF.hflip(p)
p = TF.affine(p,
deg,
scale=scale,
translate=(0, 0),
shear=0,
resample=BILINEAR)
p = TF.to_tensor(p).squeeze()
p = (p * ((p_max - p_min) + 0.001)) + p_min
if dtype == torch.int64:
p = p.round()
p = p.to(dtype=dtype)
image_list[i] = p.clone()
input1, mask = image_list
return input1, mask
def _test_affine_identity_map(self, tensor, scripted_affine):
# 1) identity map
out_tensor = F.affine(tensor,
angle=0,
translate=[0, 0],
scale=1.0,
shear=[0.0, 0.0],
resample=0)
self.assertTrue(tensor.equal(out_tensor),
msg="{} vs {}".format(out_tensor[0, :5, :5],
tensor[0, :5, :5]))
out_tensor = scripted_affine(tensor,
angle=0,
translate=[0, 0],
scale=1.0,
shear=[0.0, 0.0],
resample=0)
self.assertTrue(tensor.equal(out_tensor),
msg="{} vs {}".format(out_tensor[0, :5, :5],
tensor[0, :5, :5]))
def translate_y(
x: torch.Tensor,
interval: float, # [0, interval]
magnitude: float,
max_magnitude: int,
fill: float,
) -> torch.Tensor:
# Differ from: https://github.com/tensorflow/tpu/blob/3679ca6b979349dde6da7156be2528428b000c7c/models/official/efficientnet/autoaugment.py#L503-L507
height = x.shape[-2]
pixels = height * (magnitude / max_magnitude) * interval
pixels = int(_to_negative(pixels))
return TF.affine(x,
angle=0.,
translate=[0, pixels],
shear=[0., 0.],
scale=1.,
fill=[
fill,
])
def __call__(self, img, mask):
assert img.size == mask.size
x_offset = int(2 * (random.random() - 0.5) * self.offset[0])
y_offset = int(2 * (random.random() - 0.5) * self.offset[1])
x_crop_offset = x_offset
y_crop_offset = y_offset
if x_offset < 0:
x_crop_offset = 0
if y_offset < 0:
y_crop_offset = 0
cropped_img = tf.crop(img,
y_crop_offset,
x_crop_offset,
img.size[1]-abs(y_offset),
img.size[0]-abs(x_offset))
if x_offset >= 0 and y_offset >= 0:
padding_tuple = (0, 0, x_offset, y_offset)
elif x_offset >= 0 and y_offset < 0:
padding_tuple = (0, abs(y_offset), x_offset, 0)
elif x_offset < 0 and y_offset >= 0:
padding_tuple = (abs(x_offset), 0, 0, y_offset)
elif x_offset < 0 and y_offset < 0:
padding_tuple = (abs(x_offset), abs(y_offset), 0, 0)
return (
tf.pad(cropped_img,
padding_tuple,
padding_mode='reflect'),
tf.affine(mask,
translate=(-x_offset, -y_offset),
scale=1.0,
angle=0.0,
shear=0.0,
fillcolor=250))
def _test_affine_square_rotations(self, tensor, pil_img, scripted_affine):
# 2) Test rotation
test_configs = [
(90, torch.rot90(tensor, k=1, dims=(-1, -2))),
(45, None),
(30, None),
(-30, None),
(-45, None),
(-90, torch.rot90(tensor, k=-1, dims=(-1, -2))),
(180, torch.rot90(tensor, k=2, dims=(-1, -2))),
]
for a, true_tensor in test_configs:
out_pil_img = F.affine(
pil_img, angle=a, translate=[0, 0], scale=1.0, shear=[0.0, 0.0], interpolation=NEAREST
)
out_pil_tensor = torch.from_numpy(np.array(out_pil_img).transpose((2, 0, 1))).to(self.device)
for fn in [F.affine, scripted_affine]:
out_tensor = fn(
tensor, angle=a, translate=[0, 0], scale=1.0, shear=[0.0, 0.0], interpolation=NEAREST
)
if true_tensor is not None:
self.assertTrue(
true_tensor.equal(out_tensor),
msg="{}\n{} vs \n{}".format(a, out_tensor[0, :5, :5], true_tensor[0, :5, :5])
)
if out_tensor.dtype != torch.uint8:
out_tensor = out_tensor.to(torch.uint8)
num_diff_pixels = (out_tensor != out_pil_tensor).sum().item() / 3.0
ratio_diff_pixels = num_diff_pixels / out_tensor.shape[-1] / out_tensor.shape[-2]
# Tolerance : less than 6% of different pixels
self.assertLess(
ratio_diff_pixels,
0.06,
msg="{}\n{} vs \n{}".format(
ratio_diff_pixels, out_tensor[0, :7, :7], out_pil_tensor[0, :7, :7]
)
)
def zoom_blur_p(dataroot, saveroot, frameNum):
for index, folder in enumerate(sorted(os.listdir(dataroot))):
for img_loc in os.listdir(dataroot + folder):
img = trn.Resize(256)(PILImage.open(dataroot + folder + '/' +
img_loc)).convert('RGB')
z = trn.CenterCrop(224)(img)
avg = trn.ToTensor()(z)
for i in range(1, frameNum):
z = trn.CenterCrop(224)(trn_F.affine(
img,
angle=0,
translate=(0, 0),
scale=1 + 0.004 * i,
shear=0,
resample=PILImage.BILINEAR))
avg += trn.ToTensor()(z)
save_dir = saveroot + '/zoom_blur/' + folder + '/' + img_loc + '/'
if not os.path.exists(save_dir):
os.makedirs(save_dir)
trn.ToPILImage()(avg / (i + 1)).save(
os.path.join(save_dir, 'img' + str(i) + '.png'))
def __call__(self, img: Image.Image, label=None):
"""
img (PIL Image): Image to be rotated.
Returns:
PIL Image: Rotated image.
"""
if label is None:
return super(RandomAffineWithLabel, self).__call__(img)
rot_angle, translate, scale, shear = \
self.get_params(self.degrees, self.translate, self.scale, self.shear, img.size)
img = TF.affine(img,
rot_angle,
translate,
scale,
shear,
resample=self.resample,
fillcolor=self.fillcolor)
center = (img.size[0] * 0.5 + 0.5, img.size[1] * 0.5 + 0.5)
affine_transform_matrix = _get_affine_matrix(center, rot_angle,
translate, scale, shear)
label = _affine_transform_label(label, affine_transform_matrix)
return img, label
def affine_img_mask(self):
"""
This step is aimed to avoid cropping image with
black regions
"""
""" Generate a mask with the same size of original image """
mask = np.ones((self.row, self.col))
pil_mask = Image.fromarray(mask)
""" Do the same transformations"""
if self.all_param[0] == 'True':
mask_hflip = TF.hflip(pil_mask)
else:
mask_hflip = pil_mask
mask_affine = TF.affine(mask_hflip, self.all_param[1],
self.all_param[3], self.all_param[2],
self.all_param[4])
np_mask = np.uint8(mask_affine)
""" Select random rectangular regions without black parts """
while (1):
occ_i = int(random.uniform(0, 0.8) * self.row)
occ_j = int(random.uniform(0, 0.8) * self.col)
occ_h = int(random.uniform(0.2, 0.5) * self.row)
occ_w = int(random.uniform(0.2, 0.5) * self.col)
MaxRow = min(self.row - 1, occ_i + occ_h)
MaxCol = min(self.col - 1, occ_j + occ_w)
mask_crop = np_mask[occ_i:MaxRow, occ_j:MaxCol]
mask_crop_pixel = (MaxRow - occ_i) * (MaxCol - occ_j)
mask_crop_sum = sum(sum(mask_crop))
if (mask_crop_sum == mask_crop_pixel):
break
return occ_i, occ_j, MaxRow, MaxCol
def __call__(self, img: Image.Image, label=None):
"""
img (PIL Image): Image to be rotated.
Returns:
PIL Image: Rotated image.
"""
if label is None:
return super(RandomAffineWithLabel, self).__call__(img)
rot_angle, translate, scale, shear = \
self.get_params(self.degrees, self.translate, self.scale, self.shear, img.size)
# TODO: git issue shearing, otherwise throws .any()/.all() error
old_shear = shear.copy()
assert shear[1] == 0
shear = shear[0]
# from utils import pil_imshow
# pil_imshow(img, label.numpy()[:, 0:4], multiple_rectangles=True, window_name="original", wait_key=False)
img = TF.affine(img,
rot_angle,
translate,
scale,
shear,
resample=self.resample,
fillcolor=self.fillcolor)
center = (img.size[0] * 0.5 + 0.5, img.size[1] * 0.5 + 0.5)
affine_transform_matrix = _get_affine_matrix(center, rot_angle,
translate, scale, shear)
label = _affine_transform_label(label, affine_transform_matrix)
# pil_imshow(img, label.numpy()[:, 0:4], multiple_rectangles=True, window_name="skewed", wait_key=True)
return img, label
def _test_affine_all_ops(self, tensor, pil_img, scripted_affine):
# 4) Test rotation + translation + scale + share
test_configs = [
(45.5, [5, 6], 1.0, [0.0, 0.0], None),
(33, (5, -4), 1.0, [0.0, 0.0], [0, 0, 0]),
(45, [-5, 4], 1.2, [0.0, 0.0], (1, 2, 3)),
(33, (-4, -8), 2.0, [0.0, 0.0], [255, 255, 255]),
(85, (10, -10), 0.7, [0.0, 0.0], [1, ]),
(0, [0, 0], 1.0, [35.0, ], (2.0, )),
(-25, [0, 0], 1.2, [0.0, 15.0], None),
(-45, [-10, 0], 0.7, [2.0, 5.0], None),
(-45, [-10, -10], 1.2, [4.0, 5.0], None),
(-90, [0, 0], 1.0, [0.0, 0.0], None),
]
for r in [NEAREST, ]:
for a, t, s, sh, f in test_configs:
f_pil = int(f[0]) if f is not None and len(f) == 1 else f
out_pil_img = F.affine(pil_img, angle=a, translate=t, scale=s, shear=sh, interpolation=r, fill=f_pil)
out_pil_tensor = torch.from_numpy(np.array(out_pil_img).transpose((2, 0, 1)))
for fn in [F.affine, scripted_affine]:
out_tensor = fn(tensor, angle=a, translate=t, scale=s, shear=sh, interpolation=r, fill=f).cpu()
if out_tensor.dtype != torch.uint8:
out_tensor = out_tensor.to(torch.uint8)
num_diff_pixels = (out_tensor != out_pil_tensor).sum().item() / 3.0
ratio_diff_pixels = num_diff_pixels / out_tensor.shape[-1] / out_tensor.shape[-2]
# Tolerance : less than 5% (cpu), 6% (cuda) of different pixels
tol = 0.06 if self.device == "cuda" else 0.05
self.assertLess(
ratio_diff_pixels,
tol,
msg="{}: {}\n{} vs \n{}".format(
(r, a, t, s, sh, f), ratio_diff_pixels, out_tensor[0, :7, :7], out_pil_tensor[0, :7, :7]
)
)
def translate_image(input, translate, show=False):
img = input[0]
plt.imshow(img)
data = input[1]
rect_coords = data[1:]
width, height = img.size
new_rect = np.zeros(4)
print rect_coords[0], rect_coords[0] + float(translate[0]) / width
new_rect[0] = max(0., rect_coords[0] + float(translate[0]) / width)
new_rect[1] = min(1., rect_coords[1] + float(translate[0]) / width)
new_rect[2] = max(0., rect_coords[2] + float(translate[1]) / height)
new_rect[3] = min(1., rect_coords[3] + float(translate[1]) / height)
print new_rect
img = TF.affine(img, 0, translate, 1, 0)
if show:
draw = ImageDraw.Draw(img)
draw.rectangle(((new_rect[0] * width, new_rect[2] * height),
(new_rect[1] * width, new_rect[3] * height)))
plt.imshow(img)
plt.show()
return (img, np.insert(new_rect, 0, data[0]))
def _transform(
self, x: "torch.Tensor", y: Optional["torch.Tensor"], **kwargs
) -> Tuple["torch.Tensor", Optional["torch.Tensor"]]:
"""
Apply random affine transforms to an image.
:param x: Input samples.
:param y: Label of the samples `x`.
:return: Transformed samples and labels.
"""
import torch
import torchvision.transforms.functional as F
img_size = x.shape[:2]
angle = float(
torch.empty(1)
.uniform_(float(self.degree_range[0]), float(self.degree_range[1]))
.item()
)
max_dx = float(self.translate[0] * img_size[1])
max_dy = float(self.translate[1] * img_size[0])
tx = int(round(torch.empty(1).uniform_(-max_dx, max_dx).item()))
ty = int(round(torch.empty(1).uniform_(-max_dy, max_dy).item()))
translations = (tx, ty)
scale = float(torch.empty(1).uniform_(self.scale[0], self.scale[1]).item())
# x needs to have channel first
x = x.permute(2, 0, 1)
x = F.affine(
img=x, angle=angle, translate=translations, scale=scale, shear=(0.0, 0.0)
)
x = x.permute(1, 2, 0)
return torch.clamp(x, min=self.clip_values[0], max=self.clip_values[1]), y
def __call__(self, *in_images, fillcolor=None):
# Check that all input images share the same size which also
# match the declared one
assert np.all([i.shape[:2] == self.size for i in in_images])
fill = fillcolor
if fillcolor is None:
fill = self.fillcolor
output = []
for image in in_images:
image = Image.fromarray(image)
x = F.affine(image,
*self.ret,
resample=self.resample,
fillcolor=fill)
output.append(np.asarray(x))
if len(output) == 1:
output = output[0]
else:
output = np.stack(output, axis=0)
return output
def _test_transformation(a, t, s, sh):
a_rad = math.radians(a)
s_rad = math.radians(sh)
# 1) Check transformation matrix:
c_matrix = np.array([[1.0, 0.0, cnt[0]], [0.0, 1.0, cnt[1]], [0.0, 0.0, 1.0]])
c_inv_matrix = np.linalg.inv(c_matrix)
t_matrix = np.array([[1.0, 0.0, t[0]],
[0.0, 1.0, t[1]],
[0.0, 0.0, 1.0]])
r_matrix = np.array([[s * math.cos(a_rad), -s * math.sin(a_rad + s_rad), 0.0],
[s * math.sin(a_rad), s * math.cos(a_rad + s_rad), 0.0],
[0.0, 0.0, 1.0]])
true_matrix = np.dot(t_matrix, np.dot(c_matrix, np.dot(r_matrix, c_inv_matrix)))
result_matrix = _to_3x3_inv(F._get_inverse_affine_matrix(center=cnt, angle=a,
translate=t, scale=s, shear=sh))
assert np.sum(np.abs(true_matrix - result_matrix)) < 1e-10
# 2) Perform inverse mapping:
true_result = np.zeros((200, 200, 3), dtype=np.uint8)
inv_true_matrix = np.linalg.inv(true_matrix)
for y in range(true_result.shape[0]):
for x in range(true_result.shape[1]):
res = np.dot(inv_true_matrix, [x, y, 1])
_x = int(res[0] + 0.5)
_y = int(res[1] + 0.5)
if 0 <= _x < input_img.shape[1] and 0 <= _y < input_img.shape[0]:
true_result[y, x, :] = input_img[_y, _x, :]
result = F.affine(pil_img, angle=a, translate=t, scale=s, shear=sh)
assert result.size == pil_img.size
# Compute number of different pixels:
np_result = np.array(result)
n_diff_pixels = np.sum(np_result != true_result) / 3
# Accept 3 wrong pixels
assert n_diff_pixels < 3, \
"a={}, t={}, s={}, sh={}\n".format(a, t, s, sh) +\
"n diff pixels={}\n".format(np.sum(np.array(result)[:, :, 0] != true_result[:, :, 0]))
def _test_affine_translations(self, tensor, pil_img, scripted_affine):
# 3) Test translation
test_configs = [[10, 12], (-12, -13)]
for t in test_configs:
out_pil_img = F.affine(pil_img,
angle=0,
translate=t,
scale=1.0,
shear=[0.0, 0.0],
resample=0)
for fn in [F.affine, scripted_affine]:
out_tensor = fn(tensor,
angle=0,
translate=t,
scale=1.0,
shear=[0.0, 0.0],
resample=0)
if out_tensor.dtype != torch.uint8:
out_tensor = out_tensor.to(torch.uint8)
self.compareTensorToPIL(out_tensor, out_pil_img)
def transform_image(self,
image,
resize_param,
crop_param=False,
method=Image.BICUBIC,
angle=None,
normalize=True,
toTensor=True):
if crop_param:
image = F.crop(image, crop_param['top'], crop_param['left'],
crop_param['height'], crop_param['width'])
if angle is not None:
image = F.affine(image,
angle=angle,
translate=(0, 0),
scale=1,
shear=0)
image = F.resize(image, resize_param, interpolation=method)
if toTensor:
image = F.to_tensor(image)
if normalize:
image = F.normalize(image, (0.5, 0.5, 0.5), (0.5, 0.5, 0.5))
return image
def __call__(self, samples):
"""samples: pil images"""
transformations = transforms.Compose([
MaxSizeResizer(224),
SquarePad(),
transforms.ToPILImage(),
])
samples = [transformations(s) for s in samples]
if self.augment:
rotation, translation, scale, shear = transforms.RandomAffine.get_params(
(-30, 30), (0.1, 0.1), (0.8, 1.2), None, samples[0].size)
brightness = np.random.uniform(0.5, 1.5)
contrast = np.random.uniform(0.7, 1.3)
hue = np.random.uniform(-0.2, 0.2)
flip = np.random.uniform(0, 1) > 0.5
for i, s in enumerate(samples):
s = F.affine(s, rotation, translation, scale, shear)
s = F.adjust_brightness(s, brightness)
s = F.adjust_contrast(s, contrast)
s = F.adjust_hue(s, hue)
if flip > 0.5:
s = F.hflip(s)
samples[i] = s
# plt.imshow(np.array(samples[0]))
# plt.show()
transformations = transforms.Compose([
transforms.Grayscale(),
transforms.ToTensor(),
transforms.Normalize(mean=0.456, std=0.224),
])
samples = torch.cat([transformations(s) for s in samples], 0)
return samples
def transform_image(self,
image,
resize_param,
method=Image.BICUBIC,
affine=None,
normalize=True,
toTensor=True,
fillWhiteColor=False):
image = F.resize(image, resize_param, interpolation=method)
if affine is not None:
angle, translate, scale = affine['angle'], affine['shift'], affine[
'scale']
fillcolor = None if not fillWhiteColor else (255, 255, 255)
image = F.affine(image,
angle=angle,
translate=translate,
scale=scale,
shear=0,
fillcolor=fillcolor)
if toTensor:
image = F.to_tensor(image)
if normalize:
image = F.normalize(image, (0.5, 0.5, 0.5), (0.5, 0.5, 0.5))
return image
def test_warnings(self, device):
tensor, pil_img = _create_data(26, 26, device=device)
# assert deprecation warning and non-BC
with pytest.warns(UserWarning, match=r"Argument resample is deprecated and will be removed"):
res1 = F.affine(tensor, 45, translate=[0, 0], scale=1.0, shear=[0.0, 0.0], resample=2)
res2 = F.affine(tensor, 45, translate=[0, 0], scale=1.0, shear=[0.0, 0.0], interpolation=BILINEAR)
assert_equal(res1, res2)
# assert changed type warning
with pytest.warns(UserWarning, match=r"Argument interpolation should be of type InterpolationMode"):
res1 = F.affine(tensor, 45, translate=[0, 0], scale=1.0, shear=[0.0, 0.0], interpolation=2)
res2 = F.affine(tensor, 45, translate=[0, 0], scale=1.0, shear=[0.0, 0.0], interpolation=BILINEAR)
assert_equal(res1, res2)
with pytest.warns(UserWarning, match=r"Argument fillcolor is deprecated and will be removed"):
res1 = F.affine(pil_img, 45, translate=[0, 0], scale=1.0, shear=[0.0, 0.0], fillcolor=10)
res2 = F.affine(pil_img, 45, translate=[0, 0], scale=1.0, shear=[0.0, 0.0], fill=10)
# we convert the PIL images to numpy as assert_equal doesn't work on PIL images.
assert_equal(np.asarray(res1), np.asarray(res2))
def __call__(self, img, target=None, mask=None):
"""
img (PIL Image): Image to be rotated.
target (PIL Image): (optional) Target to be rotated
mask (PIL Image): (optional) Mask to be rotated
Returns:
PIL Image: Rotated image(s).
"""
ret = self.get_params(self.degrees, self.translate, self.scale, self.shear, img.size)
if target is not None and mask is None:
return F.affine(img, *ret, resample=self.resample, fillcolor=self.fillcolor), \
F.affine(target, *ret, resample=self.resample_tg, fillcolor=self.fillcolor)
# resample = False is by default nearest, appropriate for targets
if target is not None and mask is not None:
return F.affine(img, *ret, resample=self.resample, fillcolor=self.fillcolor), \
F.affine(target, *ret, resample=self.resample_tg, fillcolor=self.fillcolor), \
F.affine(mask, *ret, resample=self.resample_mask, fillcolor=self.fillcolor)
# resample = False is by default nearest, appropriate for targets
return F.affine(img, *ret, resample=self.resample, fillcolor=self.fillcolor)
def test_affine(self):
input_img = np.zeros((200, 200, 3), dtype=np.uint8)
pts = []
cnt = [100, 100]
for pt in [(80, 80), (100, 80), (100, 100)]:
for i in range(-5, 5):
for j in range(-5, 5):
input_img[pt[0] + i, pt[1] + j, :] = [255, 155, 55]
pts.append((pt[0] + i, pt[1] + j))
pts = list(set(pts))
with self.assertRaises(TypeError):
F.affine(input_img, 10)
pil_img = F.to_pil_image(input_img)
def _to_3x3_inv(inv_result_matrix):
result_matrix = np.zeros((3, 3))
result_matrix[:2, :] = np.array(inv_result_matrix).reshape((2, 3))
result_matrix[2, 2] = 1
return np.linalg.inv(result_matrix)
def _test_transformation(a, t, s, sh):
a_rad = math.radians(a)
s_rad = math.radians(sh)
# 1) Check transformation matrix:
c_matrix = np.array([[1.0, 0.0, cnt[0]], [0.0, 1.0, cnt[1]], [0.0, 0.0, 1.0]])
c_inv_matrix = np.linalg.inv(c_matrix)
t_matrix = np.array([[1.0, 0.0, t[0]],
[0.0, 1.0, t[1]],
[0.0, 0.0, 1.0]])
r_matrix = np.array([[s * math.cos(a_rad), -s * math.sin(a_rad + s_rad), 0.0],
[s * math.sin(a_rad), s * math.cos(a_rad + s_rad), 0.0],
[0.0, 0.0, 1.0]])
true_matrix = np.dot(t_matrix, np.dot(c_matrix, np.dot(r_matrix, c_inv_matrix)))
result_matrix = _to_3x3_inv(F._get_inverse_affine_matrix(center=cnt, angle=a,
translate=t, scale=s, shear=sh))
assert np.sum(np.abs(true_matrix - result_matrix)) < 1e-10
# 2) Perform inverse mapping:
true_result = np.zeros((200, 200, 3), dtype=np.uint8)
inv_true_matrix = np.linalg.inv(true_matrix)
for y in range(true_result.shape[0]):
for x in range(true_result.shape[1]):
res = np.dot(inv_true_matrix, [x, y, 1])
_x = int(res[0] + 0.5)
_y = int(res[1] + 0.5)
if 0 <= _x < input_img.shape[1] and 0 <= _y < input_img.shape[0]:
true_result[y, x, :] = input_img[_y, _x, :]
result = F.affine(pil_img, angle=a, translate=t, scale=s, shear=sh)
assert result.size == pil_img.size
# Compute number of different pixels:
np_result = np.array(result)
n_diff_pixels = np.sum(np_result != true_result) / 3
# Accept 3 wrong pixels
assert n_diff_pixels < 3, \
"a={}, t={}, s={}, sh={}\n".format(a, t, s, sh) +\
"n diff pixels={}\n".format(np.sum(np.array(result)[:, :, 0] != true_result[:, :, 0]))
# Test rotation
a = 45
_test_transformation(a=a, t=(0, 0), s=1.0, sh=0.0)
# Test translation
t = [10, 15]
_test_transformation(a=0.0, t=t, s=1.0, sh=0.0)
# Test scale
s = 1.2
_test_transformation(a=0.0, t=(0.0, 0.0), s=s, sh=0.0)
# Test shear
sh = 45.0
_test_transformation(a=0.0, t=(0.0, 0.0), s=1.0, sh=sh)
# Test rotation, scale, translation, shear
for a in range(-90, 90, 25):
for t1 in range(-10, 10, 5):
for s in [0.75, 0.98, 1.0, 1.1, 1.2]:
for sh in range(-15, 15, 5):
_test_transformation(a=a, t=(t1, t1), s=s, sh=sh)
def test_affine(self):
input_img = np.zeros((200, 200, 3), dtype=np.uint8)
pts = []
cnt = [100, 100]
for pt in [(80, 80), (100, 80), (100, 100)]:
for i in range(-5, 5):
for j in range(-5, 5):
input_img[pt[0] + i, pt[1] + j, :] = [255, 155, 55]
pts.append((pt[0] + i, pt[1] + j))
pts = list(set(pts))
with self.assertRaises(TypeError):
F.affine(input_img, 10)
pil_img = F.to_pil_image(input_img)
def _to_3x3_inv(inv_result_matrix):
result_matrix = np.zeros((3, 3))
result_matrix[:2, :] = np.array(inv_result_matrix).reshape((2, 3))
result_matrix[2, 2] = 1
return np.linalg.inv(result_matrix)
def _test_transformation(a, t, s, sh):
a_rad = math.radians(a)
s_rad = math.radians(sh)
# 1) Check transformation matrix:
c_matrix = np.array([[1.0, 0.0, cnt[0]], [0.0, 1.0, cnt[1]],
[0.0, 0.0, 1.0]])
c_inv_matrix = np.linalg.inv(c_matrix)
t_matrix = np.array([[1.0, 0.0, t[0]], [0.0, 1.0, t[1]],
[0.0, 0.0, 1.0]])
r_matrix = np.array(
[[s * math.cos(a_rad), -s * math.sin(a_rad + s_rad), 0.0],
[s * math.sin(a_rad), s * math.cos(a_rad + s_rad), 0.0],
[0.0, 0.0, 1.0]])
true_matrix = np.dot(
t_matrix, np.dot(c_matrix, np.dot(r_matrix, c_inv_matrix)))
result_matrix = _to_3x3_inv(
F._get_inverse_affine_matrix(center=cnt,
angle=a,
translate=t,
scale=s,
shear=sh))
assert np.sum(np.abs(true_matrix - result_matrix)) < 1e-10
# 2) Perform inverse mapping:
true_result = np.zeros((200, 200, 3), dtype=np.uint8)
inv_true_matrix = np.linalg.inv(true_matrix)
for y in range(true_result.shape[0]):
for x in range(true_result.shape[1]):
res = np.dot(inv_true_matrix, [x, y, 1])
_x = int(res[0] + 0.5)
_y = int(res[1] + 0.5)
if 0 <= _x < input_img.shape[
1] and 0 <= _y < input_img.shape[0]:
true_result[y, x, :] = input_img[_y, _x, :]
result = F.affine(pil_img, angle=a, translate=t, scale=s, shear=sh)
assert result.size == pil_img.size
# Compute number of different pixels:
np_result = np.array(result)
n_diff_pixels = np.sum(np_result != true_result) / 3
# Accept 3 wrong pixels
assert n_diff_pixels < 3, \
"a={}, t={}, s={}, sh={}\n".format(a, t, s, sh) +\
"n diff pixels={}\n".format(np.sum(np.array(result)[:, :, 0] != true_result[:, :, 0]))
# Test rotation
a = 45
_test_transformation(a=a, t=(0, 0), s=1.0, sh=0.0)
# Test translation
t = [10, 15]
_test_transformation(a=0.0, t=t, s=1.0, sh=0.0)
# Test scale
s = 1.2
_test_transformation(a=0.0, t=(0.0, 0.0), s=s, sh=0.0)
# Test shear
sh = 45.0
_test_transformation(a=0.0, t=(0.0, 0.0), s=1.0, sh=sh)
# Test rotation, scale, translation, shear
for a in range(-90, 90, 25):
for t1 in range(-10, 10, 5):
for s in [0.75, 0.98, 1.0, 1.1, 1.2]:
for sh in range(-15, 15, 5):
_test_transformation(a=a, t=(t1, t1), s=s, sh=sh)
def torchvision(self, img):
return torchvision.affine(img, angle=45, translate=(50, 50), scale=2, shear=0, resample=Image.BILINEAR)
def torchvision_transform(self, img):
return torchvision.affine(img, angle=45, translate=(50, 50), scale=2, shear=0, resample=Image.BILINEAR)
def sample_augment(self, input_data, params):
input_data = tvF.affine(input_data,
*params,
resample=self.resample,
fillcolor=self.fillcolor)
return input_data
def test_affine(self):
# Tests on square and rectangular images
scripted_affine = torch.jit.script(F.affine)
for tensor, pil_img in [
self._create_data(26, 26),
self._create_data(32, 26)
]:
# 1) identity map
out_tensor = F.affine(tensor,
angle=0,
translate=[0, 0],
scale=1.0,
shear=[0.0, 0.0],
resample=0)
self.assertTrue(tensor.equal(out_tensor),
msg="{} vs {}".format(out_tensor[0, :5, :5],
tensor[0, :5, :5]))
out_tensor = scripted_affine(tensor,
angle=0,
translate=[0, 0],
scale=1.0,
shear=[0.0, 0.0],
resample=0)
self.assertTrue(tensor.equal(out_tensor),
msg="{} vs {}".format(out_tensor[0, :5, :5],
tensor[0, :5, :5]))
if pil_img.size[0] == pil_img.size[1]:
# 2) Test rotation
test_configs = [
(90, torch.rot90(tensor, k=1, dims=(-1, -2))),
(45, None),
(30, None),
(-30, None),
(-45, None),
(-90, torch.rot90(tensor, k=-1, dims=(-1, -2))),
(180, torch.rot90(tensor, k=2, dims=(-1, -2))),
]
for a, true_tensor in test_configs:
for fn in [F.affine, scripted_affine]:
out_tensor = fn(tensor,
angle=a,
translate=[0, 0],
scale=1.0,
shear=[0.0, 0.0],
resample=0)
if true_tensor is not None:
self.assertTrue(true_tensor.equal(out_tensor),
msg="{}\n{} vs \n{}".format(
a, out_tensor[0, :5, :5],
true_tensor[0, :5, :5]))
else:
true_tensor = out_tensor
out_pil_img = F.affine(pil_img,
angle=a,
translate=[0, 0],
scale=1.0,
shear=[0.0, 0.0],
resample=0)
out_pil_tensor = torch.from_numpy(
np.array(out_pil_img).transpose((2, 0, 1)))
num_diff_pixels = (true_tensor !=
out_pil_tensor).sum().item() / 3.0
ratio_diff_pixels = num_diff_pixels / true_tensor.shape[
-1] / true_tensor.shape[-2]
# Tolerance : less than 6% of different pixels
self.assertLess(ratio_diff_pixels,
0.06,
msg="{}\n{} vs \n{}".format(
ratio_diff_pixels,
true_tensor[0, :7, :7],
out_pil_tensor[0, :7, :7]))
else:
test_configs = [90, 45, 15, -30, -60, -120]
for a in test_configs:
for fn in [F.affine, scripted_affine]:
out_tensor = fn(tensor,
angle=a,
translate=[0, 0],
scale=1.0,
shear=[0.0, 0.0],
resample=0)
out_pil_img = F.affine(pil_img,
angle=a,
translate=[0, 0],
scale=1.0,
shear=[0.0, 0.0],
resample=0)
out_pil_tensor = torch.from_numpy(
np.array(out_pil_img).transpose((2, 0, 1)))
num_diff_pixels = (out_tensor !=
out_pil_tensor).sum().item() / 3.0
ratio_diff_pixels = num_diff_pixels / out_tensor.shape[
-1] / out_tensor.shape[-2]
# Tolerance : less than 3% of different pixels
self.assertLess(ratio_diff_pixels,
0.03,
msg="{}: {}\n{} vs \n{}".format(
a, ratio_diff_pixels,
out_tensor[0, :7, :7],
out_pil_tensor[0, :7, :7]))
# 3) Test translation
test_configs = [[10, 12], (-12, -13)]
for t in test_configs:
for fn in [F.affine, scripted_affine]:
out_tensor = fn(tensor,
angle=0,
translate=t,
scale=1.0,
shear=[0.0, 0.0],
resample=0)
out_pil_img = F.affine(pil_img,
angle=0,
translate=t,
scale=1.0,
shear=[0.0, 0.0],
resample=0)
self.compareTensorToPIL(out_tensor, out_pil_img)
# 3) Test rotation + translation + scale + share
test_configs = [
(45, [5, 6], 1.0, [0.0, 0.0]),
(33, (5, -4), 1.0, [0.0, 0.0]),
(45, [-5, 4], 1.2, [0.0, 0.0]),
(33, (-4, -8), 2.0, [0.0, 0.0]),
(85, (10, -10), 0.7, [0.0, 0.0]),
(0, [0, 0], 1.0, [
35.0,
]),
(-25, [0, 0], 1.2, [0.0, 15.0]),
(-45, [-10, 0], 0.7, [2.0, 5.0]),
(-45, [-10, -10], 1.2, [4.0, 5.0]),
(-90, [0, 0], 1.0, [0.0, 0.0]),
]
for r in [
0,
]:
for a, t, s, sh in test_configs:
out_pil_img = F.affine(pil_img,
angle=a,
translate=t,
scale=s,
shear=sh,
resample=r)
out_pil_tensor = torch.from_numpy(
np.array(out_pil_img).transpose((2, 0, 1)))
for fn in [F.affine, scripted_affine]:
out_tensor = fn(tensor,
angle=a,
translate=t,
scale=s,
shear=sh,
resample=r)
num_diff_pixels = (out_tensor !=
out_pil_tensor).sum().item() / 3.0
ratio_diff_pixels = num_diff_pixels / out_tensor.shape[
-1] / out_tensor.shape[-2]
# Tolerance : less than 5% of different pixels
self.assertLess(ratio_diff_pixels,
0.05,
msg="{}: {}\n{} vs \n{}".format(
(r, a, t, s, sh),
ratio_diff_pixels,
out_tensor[0, :7, :7],
out_pil_tensor[0, :7, :7]))
def affine(im, angle, translate, scale, shear):
assert isPIL(im), f"Got type {type(im)}."
assert -180 <= angle <= 180, f"Invalid angle: {angle}"
return TFF.affine(im, angle, translate, scale, shear)
def affine(self, x, params, resample=Image.BILINEAR):
return TF.affine(x, *params, resample=resample, fillcolor=0)
def imageTransform(img, transformVals):
if transformVals[0] == 1:
img = TF.hflip(img)
img = TF.affine(img, transformVals[1], transformVals[2], transformVals[3],0)
return img
def forward(self, img: Tensor):
"""
img (PIL Image or Tensor): Image to be transformed.
Returns:
PIL Image or Tensor: AutoAugmented image.
"""
fill = self.fill
if isinstance(img, Tensor):
if isinstance(fill, (int, float)):
fill = [float(fill)] * F._get_image_num_channels(img)
elif fill is not None:
fill = [float(f) for f in fill]
transform_id, probs, signs = self.get_params(len(self.transforms))
for i, (op_name, p, magnitude_id) in enumerate(self.transforms[transform_id]):
if probs[i] <= p:
magnitudes, signed = self._get_op_meta(op_name)
magnitude = float(magnitudes[magnitude_id].item()) \
if magnitudes is not None and magnitude_id is not None else 0.0
if signed is not None and signed and signs[i] == 0:
magnitude *= -1.0
if op_name == "Pairing":
img =
if op_name == "ShearX":
img = F.affine(img, angle=0.0, translate=[0, 0], scale=1.0, shear=[math.degrees(magnitude), 0.0],
interpolation=self.interpolation, fill=fill)
elif op_name == "ShearY":
img = F.affine(img, angle=0.0, translate=[0, 0], scale=1.0, shear=[0.0, math.degrees(magnitude)],
interpolation=self.interpolation, fill=fill)
elif op_name == "TranslateX":
img = F.affine(img, angle=0.0, translate=[int(F._get_image_size(img)[0] * magnitude), 0], scale=1.0,
interpolation=self.interpolation, shear=[0.0, 0.0], fill=fill)
elif op_name == "TranslateY":
img = F.affine(img, angle=0.0, translate=[0, int(F._get_image_size(img)[1] * magnitude)], scale=1.0,
interpolation=self.interpolation, shear=[0.0, 0.0], fill=fill)
elif op_name == "Rotate":
img = F.rotate(
img, magnitude, interpolation=self.interpolation, fill=fill)
elif op_name == "Flip":
img = F.hflip(img)
elif op_name == "Brightness":
img = F.adjust_brightness(img, 1.0 + magnitude)
elif op_name == "Color":
img = F.adjust_saturation(img, 1.0 + magnitude)
elif op_name == "Contrast":
img = F.adjust_contrast(img, 1.0 + magnitude)
elif op_name == "Sharpness":
img = F.adjust_sharpness(img, 1.0 + magnitude)
elif op_name == "Posterize":
img = F.posterize(img, int(magnitude))
elif op_name == "Solarize":
img = F.solarize(img, magnitude)
elif op_name == "AutoContrast":
img = F.autocontrast(img)
elif op_name == "Equalize":
img = F.equalize(img)
elif op_name == "Invert":
img = F.invert(img)
else:
raise ValueError(
"The provided operator {} is not recognized.".format(op_name))
return img
def affine(pil, angle, translate, scale, shear):
assert isPIL(pil), f"Got type {type(pil)}."
return TFF.affine(pil, angle, translate, scale, shear)