def data_ThinPlateSpline(image, mask, height, width, ratio=0.05):
if random.random() < set_ratio:
return image, mask
bias = np.random.randint(-int(height * ratio), int(width * ratio), 16)
tps = cv2.createThinPlateSplineShapeTransformer()
sshape = np.array(
[[0 + bias[0], 0 + bias[1]], [height + bias[2], 0 + bias[3]],
[0 + bias[4], width + bias[5]], [height + bias[6], width + bias[7]]],
np.float32)
tshape = np.array(
[[0 + bias[8], 0 + bias[9]], [height + bias[10], 0 + bias[11]],
[0 + bias[12], width + bias[13]],
[height + bias[14], width + bias[15]]], np.float32)
sshape = sshape.reshape(1, -1, 2)
tshape = tshape.reshape(1, -1, 2)
matches = list()
matches.append(cv2.DMatch(0, 0, 0))
matches.append(cv2.DMatch(1, 1, 0))
matches.append(cv2.DMatch(2, 2, 0))
matches.append(cv2.DMatch(3, 3, 0))
tps.estimateTransformation(tshape, sshape, matches)
res = tps.warpImage(image)
res_mask = tps.warpImage(mask)
return res, res_mask
def thin_plate_spline(img, init_points):
tps = cv2.createThinPlateSplineShapeTransformer()
sshape = init_points.astype(np.float32)
l = sshape.shape[0]
print(l)
delta = np.empty((0, 2))
for n in range(l):
randx = random.randint(-50, 50) * 1
randy = random.randint(-100, 100) * 1
delta = np.concatenate([delta, np.array([[randx, randy]])])
# delta = np.array([[-10,100],[10,100],[-10,-100],[10,-100]],np.float32)
tshape = sshape + delta
sshape = sshape.reshape(1, -1, 2)
tshape = tshape.reshape(1, -1, 2)
matches = list()
for n in range(l):
matches.append(cv2.DMatch(n, n, 0))
tps.estimateTransformation(tshape, sshape, matches)
out_img = tps.warpImage(img)
return out_img
def data_ThinPlateSpline_prior(prior, height, width, ratio=ratio_do_transform):
if random.random() < ratio_return_unchanged:
return prior
bias = np.random.randint(-int(height * ratio), int(width * ratio), 16)
tps = cv2.createThinPlateSplineShapeTransformer()
sshape = np.array(
[[0 + bias[0], 0 + bias[1]], [height + bias[2], 0 + bias[3]],
[0 + bias[4], width + bias[5]], [height + bias[6], width + bias[7]]],
np.float32)
tshape = np.array(
[[0 + bias[8], 0 + bias[9]], [height + bias[10], 0 + bias[11]],
[0 + bias[12], width + bias[13]],
[height + bias[14], width + bias[15]]], np.float32)
sshape = sshape.reshape(1, -1, 2)
tshape = tshape.reshape(1, -1, 2)
matches = list()
matches.append(cv2.DMatch(0, 0, 0))
matches.append(cv2.DMatch(1, 1, 0))
matches.append(cv2.DMatch(2, 2, 0))
matches.append(cv2.DMatch(3, 3, 0))
tps.estimateTransformation(tshape, sshape, matches)
prior = tps.warpImage(prior)
return prior
def __call__(self, img, tps=None):
"""
accepts a PIL image
must convert to numpy array to apply TPS
converts back to PIL image before returning
"""
# construct the transformed grid from the regular grid
img_as_arr = np.transpose(img.numpy(), (1, 2, 0))
if tps is None:
warp_matrix, t_x, t_y = self.sample_warp()
perturb_mat = self.random_perturb()
center = np.array([[[self.grid[:, :, 0].max()/2.0 + t_x, self.grid[:, :, 1].max()/2.0 + t_y]]])
target_grid = np.matmul((self.grid - center), warp_matrix) + perturb_mat + center
tps = cv2.createThinPlateSplineShapeTransformer()
tps.estimateTransformation(self.grid, target_grid, self.matches)
img_as_arr = tps.warpImage(img_as_arr, borderMode=cv2.BORDER_REPLICATE)
dims = img_as_arr.shape
if self.append_offset_channels: # extract ground truth warping offsets
full_grid_x, full_grid_y = np.meshgrid(np.arange(dims[1]), np.arange(dims[0]))
dims_half_x = dims[1]/2.0
dims_half_y = dims[0]/2.0
full_grid_x = (full_grid_x - dims_half_x)/dims_half_x
full_grid_y = (full_grid_y - dims_half_y)/dims_half_y
full_grid = np.concatenate((np.expand_dims(full_grid_x, 2), np.expand_dims(full_grid_y, 2)), axis=2)
img_coord_arr = tps.warpImage(full_grid.astype(np.float32), borderValue=-1024)
displacement = img_coord_arr
img_as_arr = np.concatenate((img_as_arr, displacement), 2)
# convert back to PIL and return
out_img = torch.from_numpy(img_as_arr).permute(2, 0, 1)
return out_img
def keypoint_guided_tps():
num_sample = 64
pair_list = io.load_json(
'datasets/DF_Pose/Label/pair_split.json')['test'][0:num_sample]
pose_label = io.load_data('datasets/DF_Pose/Label/pose_label.pkl')
image_dir = 'datasets/DF_Pose/Img/img_df/'
seg_dir = 'datasets/DF_Pose/Img/seg-lip_df_revised/'
output_dir = 'temp/patch_matching/output/tps_keypoint/'
io.mkdir_if_missing(output_dir)
tps = cv2.createThinPlateSplineShapeTransformer()
for i, (id_1, id_2) in enumerate(tqdm.tqdm(pair_list)):
kp_1 = np.array(pose_label[id_1][1:14],
dtype=np.float64).reshape(1, -1, 2)
kp_2 = np.array(pose_label[id_2][1:14],
dtype=np.float64).reshape(1, -1, 2)
kp_matches = []
for j in range(kp_1.shape[1]):
if (kp_1[0, j] >= 0).all() and (kp_2[0, j] >= 0).all():
kp_matches.append(cv2.DMatch(j, j, 0))
if len(kp_matches) == 0:
continue
tps.estimateTransformation(kp_2, kp_1, kp_matches)
img_1 = cv2.imread(image_dir + id_1 + '.jpg')
img_2 = cv2.imread(image_dir + id_2 + '.jpg')
img_w = tps.warpImage(img_1)
seg = cv2.imread(seg_dir + id_2 + '.bmp', cv2.IMREAD_GRAYSCALE)
mask = ((seg == 3) | (seg == 7)).astype(img_w.dtype)[:, :, np.newaxis]
img_out = img_w * mask + img_2 * (1 - mask)
cv2.imwrite(output_dir + '%d_%s_%s.jpg' % (i, id_1, id_2), img_out)
cv2.imwrite(output_dir + 'w%d_%s_%s.jpg' % (i, id_1, id_2), img_w)
def tps(img, label, attention):
if np.sum(label) == 0:
return img, label, attention
tps = cv2.createThinPlateSplineShapeTransformer()
multi = False
points, _ = nms(label[:, :, :5])
if len(points[0]) > 1:
multi = True
p_ = []
for p in points:
for e in list(map(list, p)):
p_.append(e)
points = np.array(p_).reshape(1, -1, 2)
t_points = points.copy()
t_points[:, 0, 0] -= np.random.randint(-2, 2, size=1)
t_points[:, 0, 1] -= np.random.randint(-2, 2, size=1)
if multi:
t_points[:, 1, 0] -= np.random.randint(-2, 2, size=1)
t_points[:, 1, 1] -= np.random.randint(-2, 2, size=1)
matches = [cv2.DMatch(i, i, 0) for i in range(5)]
tps.estimateTransformation(t_points, points, matches)
return tps.warpImage(img), tps.warpImage(label), attention
def tps(sourcePoints, targetPoints, img):
# thinplate spline deform
tps = cv2.createThinPlateSplineShapeTransformer()
sshape = sourcePoints.reshape(1, -1, 2)
tshape = targetPoints.reshape(1, -1, 2)
matches = []
for i in range(sshape.shape[1]):
matches.append(cv2.DMatch(i, i, 0))
tps.estimateTransformation(tshape.copy(), sshape.copy(), matches)
return tps.warpImage(img)
def tps_warp(source, target, img):
tps = cv2.createThinPlateSplineShapeTransformer()
source = source[np.newaxis, :, :]
target = target[np.newaxis, :, :]
matches = list()
for i in range(0, len(source[0])):
matches.append(cv2.DMatch(i, i, 0))
tps.estimateTransformation(target, source, matches)
new_img = tps.warpImage(img)
return new_img
def _tps_transform(self, img, box, pad_width):
l, u, r, b = np.array([
np.min(box[:, 0]),
np.min(box[:, 1]),
np.max(box[:, 0]),
np.max(box[:, 1])
],
dtype=np.int32)
N = 5
circle = False
matches = []
for i in range(1, N + 1):
matches.append(cv2.DMatch(i, i, 0))
jitter_size = pad_width // 4
def get_src_tar_shape(N):
src_points = []
dx = (r - l) // (N - 1)
for i in range(N):
src_points.append((l + dx * i, u))
src_points.append((l + dx * i, b))
source_shape = np.array(src_points, np.int32)
source_shape = np.reshape(source_shape, (1, 2 * N, 2))
jitters = (np.random.rand(1, 2 * N, 2) - 0.5) * jitter_size
target_shape = np.copy(source_shape) - jitters
target_shape = target_shape.astype(np.int32)
return source_shape, target_shape
source_shape, target_shape = get_src_tar_shape(N)
if circle:
for i in range(2 * N):
cv2.circle(img, tuple(source_shape[0, i, :]), 1, (255, 0, 0),
5)
tps = cv2.createThinPlateSplineShapeTransformer()
tps.estimateTransformation(target_shape, source_shape, matches)
new_img = tps.warpImage(img)
if circle:
for i in range(2 * N):
cv2.circle(new_img, tuple(target_shape[0, i, :]), 1,
(0, 255, 0), 2)
bbox = self.min_rectangle(np.squeeze(target_shape, axis=0),
relaxation=jitter_size // 2)
return new_img, bbox
def __call__(self, sample_img):
image = sample_img
h, w = image.shape[:2]
assert self.distortion < min(int(h / 5), int(w / 5))
tps = createThinPlateSplineShapeTransformer()
corner1 = [0, 0]
corner2 = [h, 0]
corner3 = [0, w]
corner4 = [h, w]
#choose the rest of the grid points around the center to avoid distorting the background
grid_upper = int(h / 4)
grid_lower = 3 * grid_upper
grid_left = int(w / 4)
grid_right = 3 * grid_left
if self.rand_point:
n_points = np.random.randint(3, self.max_point)
else:
n_points = self.max_point
tpoints = np.empty((n_points, 2), dtype=np.int32)
spoints = np.empty((n_points, 2), dtype=np.int32)
for i in range(0, n_points):
x = np.random.randint(grid_upper, grid_lower)
y = np.random.randint(grid_left, grid_right)
d_y = np.random.randint(1, self.distortion)
d_x = np.random.randint(1, self.distortion)
tpoints[i] = np.asarray([x, y])
spoints[i] = np.asarray([x + d_x, y + d_y])
tpoints = np.concatenate(
[tpoints, [corner1, corner2, corner3, corner4]], axis=0)
spoints = np.concatenate(
[spoints, [corner1, corner2, corner3, corner4]], axis=0)
spoints = spoints.reshape(1, -1, 2)
tpoints = tpoints.reshape(1, -1, 2)
#landmarks = landmarks.reshape(1,-1,2)
matches = list()
for i in range(n_points + 4):
matches.append(DMatch(i, i, 0))
tps.estimateTransformation(tpoints, spoints, matches)
out_img = tps.warpImage(image).astype(np.float32)
#CAUTION!!! landmarks not transformed!!!
return out_img
def estimate_tps_transform(sources, targets):
assert sources.shape[0] == targets.shape[0]
N = sources.shape[0]
matches = list()
for i in range(N):
matches.append(cv.DMatch(i, i, 0))
sources = np.array(sources).reshape((1, -1, 2)).astype(np.float32)
targets = np.array(targets).reshape((1, -1, 2)).astype(np.float32)
tps = cv.createThinPlateSplineShapeTransformer(regularizationParameter=10)
tps.estimateTransformation(sources, targets, matches)
retval, test_pnts = tps.applyTransformation(sources)
print("mean error = ", np.mean(test_pnts - targets))
return tps
def warpImageTPS(source,target,img):
tps = cv2.createThinPlateSplineShapeTransformer()
source=source.reshape(-1,len(source),2)
target=target.reshape(-1,len(target),2)
matches=list()
for i in range(0,len(source[0])):
matches.append(cv2.DMatch(i,i,0))
tps.estimateTransformation(target,source,matches)
new_img = tps.warpImage(img)
return new_img
def tps_cv2(source, target, img):
"""
使用cv2自带的tps处理
"""
tps = cv2.createThinPlateSplineShapeTransformer()
source_cv2 = source.reshape(1, -1, 2)
target_cv2 = target.reshape(1, -1, 2)
matches = list()
for i in range(0, len(source_cv2[0])):
matches.append(cv2.DMatch(i, i, 0))
tps.estimateTransformation(target_cv2, source_cv2, matches)
new_img_cv2 = tps.warpImage(img)
return new_img_cv2
def apply_tps(body_shape, cloth_shape, matches, img):
"""
:param body_shape: target points for TPS
:param cloth_shape: source points for TPS
:param matches: point matches
:param img: clothing image
:return: matched/warped image
"""
# Forward TPS
tps = cv2.createThinPlateSplineShapeTransformer(regularizationParameter=0)
tps.estimateTransformation(body_shape, cloth_shape, matches)
tps.applyTransformation(body_shape)
# forward warping
warped_img = tps.warpImage(img)
return warped_img
def calc_df(
width: int,
height: int,
ptrs_from: List[np.ndarray],
ptrs_to: List[np.ndarray]):
num_disps = min(len(ptrs_from), len(ptrs_to))
grid = get_grid(width, height)
tps = cv2.createThinPlateSplineShapeTransformer()
arr_src = np.expand_dims(np.array(ptrs_from), 0)
arr_dst = np.expand_dims(np.array(ptrs_to), 0)
matches = [cv2.DMatch(i, i, 0) for i in range(num_disps)]
tps.estimateTransformation(arr_src, arr_dst, matches)
grid_warped = tps.applyTransformation(
grid.reshape(1, -1, 2))[1].reshape(height, width, 2)
return grid_warped
def pipeline_step(original_cloth_path, actual_cloth_path):
"""[summary]
Args:
original_cloth_path ([type]): [description]
actual_cloth_path ([type]): [description]
Returns:
cv image: shape(height, width, 3). BGR. Range 0-255. uint8
"""
# Read image
cloth = cv2.imread(original_cloth_path, 1)
cnts_cloth, hierarchy = process_contour_cloth(cloth)
img = cv2.imread(actual_cloth_path, 0)
cnts_pred, hierarchy = process_contour_actual(img)
min_cost, points_cloth, points_pred, init_source_id, init_target_id = \
compute_indices(cnts_cloth, cnts_pred)
source, target = \
get_match_points(points_cloth, points_pred, init_source_id, init_target_id, dropout_rate=0.3)
# Cartesian metrics
tps = cv2.createThinPlateSplineShapeTransformer()
source = np.asarray(source, dtype=np.int32)
target = np.asarray(target, dtype=np.int32)
source = source.reshape(-1, len(source), 2)
target = target.reshape(-1, len(target), 2)
matches = list()
for i in range(0, len(source[0])):
matches.append(cv2.DMatch(i, i, 0))
tps.setRegularizationParameter(beta=0.5)
tps.estimateTransformation(target, source, matches)
prep_cloth = mask_background(cloth)
new_img = tps.warpImage(prep_cloth)
return new_img # some shape. range [0 - 255]
def do_tps_trans_and_warp(self):
if len(self.pixel_points) == len(
self.fix_points) and len(self.pixel_points) >= 3:
matches = list()
for i in range(0, len(self.pixel_points)):
matches.append(cv2.DMatch(i, i, 0))
pixel_points = np.float32(self.pixel_points).reshape(1, -1, 2)
fix_points = np.float32(self.fix_points).reshape(1, -1, 2)
if self.tps_transformer is None:
self.tps_transformer = cv2.createThinPlateSplineShapeTransformer(
0)
self.tps_transformer.estimateTransformation(
pixel_points, fix_points, matches)
ret, output = self.tps_transformer.applyTransformation(
pixel_points)
image = self.tps_transformer.warpImage(
self.image,
flags=self.warp_method,
borderMode=cv2.BORDER_CONSTANT)
cv2.imshow('image', image)
trans = np.zeros((self.image.shape[0], self.image.shape[1], 1),
np.uint8)
for i in range(0, int(self.image.shape[1] / 10) + 1):
cv2.line(trans, (10 * i, 0), (10 * i, self.image.shape[0]),
(255, 255, 255), 1)
for i in range(0, int(self.image.shape[0] / 10) + 1):
cv2.line(trans, (0, 10 * i), (self.image.shape[1], 10 * i),
(255, 255, 255), 1)
trans = self.tps_transformer.warpImage(
trans, flags=self.warp_method, borderMode=cv2.BORDER_CONSTANT)
cv2.imshow('transformation', trans)
else:
messagebox.showinfo("Not enough points!",
"Please set at least 3 point pairs!")
def _warp_image(array: np.ndarray, orig: ty.Sequence[ty.Tuple[int, int]],
dest: ty.Sequence[ty.Tuple[int, int]]) -> np.ndarray:
newarray = array.copy()
newarray = newarray.T # opencv convention on columns and rows is inverted
newarray_min = newarray.min()
newarray_max = newarray.max()
newarray = (newarray - newarray_min) / (newarray_max - newarray_min)
orig_p = np.array(orig, np.int32)
orig_p = orig_p.reshape(
1, -1, 2) # Don't ask why. https://stackoverflow.com/a/47114049
dest_p = np.array(dest, np.int32)
dest_p = dest_p.reshape(
1, -1, 2) # Don't ask why. https://stackoverflow.com/a/47114049
good_matches = [cv2.DMatch(p, p, 0) for p in range(len(orig))]
tps = cv2.createThinPlateSplineShapeTransformer()
tps.estimateTransformation(dest_p, orig_p, good_matches)
warped = None
warped = tps.warpImage(newarray, warped, cv2.INTER_LINEAR,
cv2.BORDER_CONSTANT, np.inf)
warped = warped * (newarray_max - newarray_min) + newarray_min
warped = warped.T
return warped
def thin_plate_transform(x,
y,
offw,
offh,
imshape,
shift_l=-0.05,
shift_r=0.05,
num_points=5,
offsetMatrix=False):
rand_p = np.random.choice(x.size, num_points, replace=False)
movingPoints = np.zeros((1, num_points, 2), dtype='float32')
fixedPoints = np.zeros((1, num_points, 2), dtype='float32')
movingPoints[:, :, 0] = x[rand_p]
movingPoints[:, :, 1] = y[rand_p]
fixedPoints[:, :,
0] = movingPoints[:, :,
0] + offw * (np.random.rand(num_points) *
(shift_r - shift_l) + shift_l)
fixedPoints[:, :,
1] = movingPoints[:, :,
1] + offh * (np.random.rand(num_points) *
(shift_r - shift_l) + shift_l)
tps = cv2.createThinPlateSplineShapeTransformer()
good_matches = [cv2.DMatch(i, i, 0) for i in xrange(num_points)]
tps.estimateTransformation(movingPoints, fixedPoints, good_matches)
imh, imw = imshape
x, y = np.meshgrid(np.arange(imw), np.arange(imh))
x, y = x.astype('float32'), y.astype('float32')
newxy = tps.applyTransformation(np.dstack((x.ravel(), y.ravel())))[1]
newxy = newxy.reshape([imh, imw, 2])
if offsetMatrix:
return newxy, newxy - np.dstack((x, y))
else:
return newxy
def __call__(self, data):
assert "image" in data and "label" in data, "`image` and `label` in data is required by this process"
if random.random() < 1 - self.p:
return data
image = data["image"]
label = data["label"]
height, width = image.shape[:2]
bias = np.random.randint(-int(height * self.ratio),
int(width * self.ratio), 16)
tps = cv2.createThinPlateSplineShapeTransformer()
sshape = np.array(
[[0 + bias[0], 0 + bias[1]], [height + bias[2], 0 + bias[3]],
[0 + bias[4], width + bias[5]],
[height + bias[6], width + bias[7]]], np.float32)
tshape = np.array(
[[0 + bias[8], 0 + bias[9]], [height + bias[10], 0 + bias[11]],
[0 + bias[12], width + bias[13]],
[height + bias[14], width + bias[15]]], np.float32)
sshape = sshape.reshape((1, -1, 2))
tshape = tshape.reshape((1, -1, 2))
matches = list()
matches.append(cv2.DMatch(0, 0, 0))
matches.append(cv2.DMatch(1, 1, 0))
matches.append(cv2.DMatch(2, 2, 0))
matches.append(cv2.DMatch(3, 3, 0))
tps.estimateTransformation(tshape, sshape, matches)
image_thin_plate_spline = tps.warpImage(image)
label_thin_plate_spline = tps.warpImage(label)
data["image"] = image_thin_plate_spline
data["label"] = label_thin_plate_spline
return data
def distoration_image(mask, part, pattern, iuv_path, test=False):
iuv = cv2.imread(iuv_path)
iuv = merge_body(iuv)
if test:
draw_new_coutour(mask, iuv)
x, y = mask.shape
ori_pattern = np.zeros((x * 2, y * 2, 3), np.uint8)
for i in range(2):
for j in range(2):
ori_pattern[i * x:(i + 1) * x,
j * y:(j + 1) * y, :] = pattern[:, :, :]
if part in top:
seq = top_body
else:
seq = under_body
body_box = [0, 0, 0, 0]
for p in seq:
n_mask = np.where(iuv[:, :, 0] == p, 1, 0) * mask
coord = np.where((n_mask == 1))
if len(coord[0]) < 200:
continue
source, box = getsource_grid(n_mask) #1
box = list(box)
source, dst = getdes_grid(n_mask, iuv, source, test=False)
l, _ = source.shape
if l >= 4:
#warped = warp_image_cv(ori_pattern, (source/2.0)+0.25, (dst/2.0)+0.25, dshape=(x*2, y*2))
n, _ = source.shape
source = source.reshape(1, -1, 2)
dst = dst.reshape(1, -1, 2)
matches = []
for i in range(1, n + 1):
matches.append(cv2.DMatch(i, i, 0))
tps = cv2.createThinPlateSplineShapeTransformer()
tps.estimateTransformation(dst, source, matches)
warped = tps.warpImage(ori_pattern)[x // 2:-x // 2, y // 2:-y // 2]
#plt.imshow(warped)
#plt.show()
#plt.close()
else:
warped = copy.deepcopy(pattern)
if p == 2 or p == 7:
body_box = box
n_mask = mask
r_mask = np.where(mask == 1, 0, 1)
for i in range(3):
pattern[:, :,
i] = warped[:, :, i] * n_mask + pattern[:, :,
i] * r_mask
#possion blending
mask_box = get_box(n_mask)
cen_x, cen_y = (mask_box[0] + mask_box[1]) // 2, (mask_box[2] +
mask_box[3]) // 2
x, y = n_mask.shape
#print()
#cv2.seamlessClone(warped,pattern,255*n_mask,(cen_y,cen_x),cv2.NORMAL_CLONE,pattern)
#pattern=blend_image(pattern, warped, 255 * n_mask)
else:
out_body = out_body_part(body_box, box, x, y)
out_body = np.where((out_body + n_mask) > 0, 1, 0) * mask
r_body = np.where(out_body == 1, 0, 1)
body_box[0] = max(box[0], body_box[0])
body_box[1] = min(box[1], box[1])
body_box[2] = max(body_box[2], box[2])
body_box[3] = min(box[3], body_box[3])
for i in range(3):
pattern[:, :,
i] = warped[:, :, i] * out_body + pattern[:, :,
i] * r_body
mask_box = get_box(out_body)
cen_x, cen_y = int((mask_box[0] + mask_box[1]) / 2), int(
(mask_box[2] + mask_box[3]) / 2)
#print(cen_x,cen_y)
#cv2.seamlessClone(warped, pattern, n_mask, (cen_y, cen_x), cv2.NORMAL_CLONE)
#pattern = blend_image(pattern, warped, 255 * out_body)
#plt.imshow(np.where(out_body==1,255,0))
#plt.imshow(warped)
#plt.show()
#plt.close()
g_pattern = cv2.cvtColor(np.uint8(pattern), cv2.COLOR_RGB2GRAY)
ori_pattern = ori_pattern[x // 2:-x // 2, y // 2:-y // 2, :]
n_mask = np.where(g_pattern == 0, 1, 0)
#n_mask = cv2.medianBlur(n_mask.reshape(), 7)
r_mask = np.where(n_mask == 1, 0, 1)
for i in range(3):
pattern[:, :,
i] = pattern[:, :, i] * r_mask + ori_pattern[:, :, i] * n_mask
#pattern = blend_image(pattern, ori_pattern, 255 * n_mask)
if test:
plt.imshow(pattern)
plt.show()
plt.close()
return pattern
#iuv=cv2.imread("IUV/output_177_IUV.png")
#iuv=cv2.imread("IUV/output_94_IUV.png")
#iuv=cv2.imread("IUV/output_129_IUV.png")
#111,273,20,171
#mask=cv2.imread("sample_mask_image/output_177_221.png",cv2.IMREAD_GRAYSCALE)
#pattern=preparepattern(mask)
#mask=np.where(mask==255,1,0)
#distoration_image(mask,169,pattern,"IUV/output_177_IUV.png",test=True)
def try_tps_transformer():
tps = cv2.createThinPlateSplineShapeTransformer()
def process(fname):
img = cv.imread('00.bmp', 0)
(w, h) = img.shape
img = cv.resize(img, (4*w, 4*h), interpolation=cv.INTER_AREA)
(w, h) = img.shape
img = cv.GaussianBlur(img,(5,5),0)
imgnorm = normalize_image(img)
cv.imwrite("n00.bmp", imgnorm)
thimg1, pos1, neg1, bool1 = findboundaries(img, inv=True, rotate=False)
thimg2, pos2, neg2, bool2 = findboundaries(img, inv=False, rotate=False)
# print("----")
thimg3, pos3, neg3, bool3 = findboundaries(img, inv=True, rotate=True)
thimg4, pos4, neg4, bool4 = findboundaries(img, inv=False, rotate=True)
maxidx = np.max(img.shape)
# print('max:', maxidx)
template0 = np.zeros((maxidx, maxidx))
xx1, yy1 = firstfit(thimg1, True, template0, False)
xx2, yy2 = firstfit(thimg2, False, template0, False)
xx3, yy3 = firstfit(thimg3, True, template0, True)
xx4, yy4 = firstfit(thimg4, False, template0, True)
points = np.where(template0 > 129)
# print(points)
# ################ 2, find the 4 corners to rectify
template = np.zeros((maxidx, maxidx))
shift = 15
ptr_src = []
ptr_dst = []
for idx in range(len(points[0])):
ptr_src.append((points[1][idx], points[0][idx]))
ptr_dst.append((15, 15))
ptr_dst.append((15+(template.shape[0]-2*shift), 15))
ptr_dst.append((15+(template.shape[0]-2*shift), 15+(template.shape[0]-2*shift)))
ptr_dst.append((15, 15+(template.shape[0]-2*shift)))
template[0:img.shape[0], 0:img.shape[1]] = img
template_dst = np.zeros(template.shape)
#template_dst = cv.remap(template, template_x, template_y, cv.INTER_LINEAR)
sourceshape = np.array(ptr_src,np.int32)
sourceshape=sourceshape.reshape(1,-1,2)
targetshape = np.array(ptr_dst,np.int32)
targetshape=targetshape.reshape(1,-1,2)
# print(sourceshape, targetshape)
matches =[]
for idx in range(len(ptr_src)):
# print(idx, ptr_src[idx], ptr_dst[idx], template.shape)
matches.append(cv.DMatch(idx, idx, 0)) #
# ################ 3, rectify (part1)
tps = cv.createThinPlateSplineShapeTransformer()
tps.estimateTransformation(targetshape ,sourceshape, matches)
tps.warpImage(template, template_dst, cv.INTER_CUBIC, cv.BORDER_REPLICATE)
barcode1 = pyzbar.decode(imgnorm)
barcode2 = pyzbar.decode(template_dst)
barcodeData1 = ""
barcodeData2 = ""
for barcode in barcode1:
# # extract the bounding box location of the barcode and draw the
# # bounding box surrounding the barcode on the image
# (x, y, w, h) = barcode.rect
# cv2.rectangle(image, (x, y), (x + w, y + h), (0, 0, 255), 2)
# the barcode data is a bytes object so if we want to draw it on
# our output image we need to convert it to a string first
barcodeData1 = barcode.data.decode("utf-8")
barcodeType = barcode.type
# # draw the barcode data and barcode type on the image
# text = "{} ({})".format(barcodeData, barcodeType)
# cv2.putText(image, text, (x, y - 10), cv2.FONT_HERSHEY_SIMPLEX,
# 0.5, (0, 0, 255), 2)
# print the barcode type and data to the terminal
# text1 = "Before: {}".format(barcodeData)
# print("[INFO] File {} Found {} barcode: {}".format(fname, barcodeType, barcodeData))
for barcode in barcode2:
barcodeData2 = barcode.data.decode("utf-8")
barcodeType = barcode.type
#text2 = barcodeData
# print("[INFO] File {} Found {} barcode: {}".format(fname, barcodeType, barcodeData))
fnameo = fname.split('.')[0]+"_d.bmp"
print(fname, fnameo, "------\"", barcodeData1, "\" -> \"", barcodeData2, "\"")
cv.imwrite(fnameo, template_dst)
diff = template.shape[1]-img.shape[1]
def wbia_plugin_kaggle7_chip_depc(depc, aid_list, config):
r"""
Refine localizations for CurvRank with Dependency Cache (depc)
CommandLine:
python -m wbia_kaggle7._plugin --test-wbia_plugin_kaggle7_chip_depc
python -m wbia_kaggle7._plugin --test-wbia_plugin_kaggle7_chip_depc:0
Example:
>>> # DISABLE_DOCTEST
>>> from wbia_kaggle7._plugin import * # NOQA
>>> import wbia
>>> from wbia.init import sysres
>>> dbdir = sysres.ensure_testdb_kaggle7()
>>> ibs = wbia.opendb(dbdir=dbdir, allow_newdir=True)
>>> aid_list = ibs.get_image_aids(1)
>>> images = ibs.depc_annot.get('KaggleSevenChip', aid_list, 'image')
>>> image = images[0]
>>> assert ut.hash_data(image) in ['imlkoiskkykpbwozghmpidlqwbmglzhw']
"""
padding = config['chip_padding']
tips_list = depc.get('Notch_Tips', aid_list)
size_list = depc.get('chips', aid_list, ('width', 'height'))
config_ = {
'dim_size': 1550,
'resize_dim': 'width',
'ext': '.jpg',
}
chip_list = depc.get('chips', aid_list, 'img', config=config_, ensure=True)
tps = cv2.createThinPlateSplineShapeTransformer()
zipped = list(zip(aid_list, tips_list, size_list, chip_list))
for aid, tip_list, size, chip in zipped:
h0, w0, c0 = chip.shape
notch = tip_list[0].copy()
left = tip_list[1].copy()
right = tip_list[2].copy()
size = np.array(size, dtype=np.float32)
notch /= size
left /= size
right /= size
size = np.array([w0, h0], dtype=np.float32)
notch *= size
left *= size
right *= size
chip_ = chip.copy()
h0, w0, c0 = chip_.shape
left += padding
notch += padding
right += padding
pad = np.zeros((h0, padding, 3), dtype=chip_.dtype)
chip_ = np.hstack((pad, chip_, pad))
h, w, c = chip_.shape
pad = np.zeros((padding, w, 3), dtype=chip_.dtype)
chip_ = np.vstack((pad, chip_, pad))
h, w, c = chip_.shape
delta = right - left
radian = np.arctan2(delta[1], delta[0])
degree = np.degrees(radian)
M = cv2.getRotationMatrix2D((left[1], left[0]), degree, 1)
chip_ = cv2.warpAffine(chip_, M, (w, h), flags=cv2.INTER_LANCZOS4)
H = np.vstack((M, [0, 0, 1]))
vert_list = np.array([notch, left, right])
vert_list_ = vt.transform_points_with_homography(H, vert_list.T).T
notch, left, right = vert_list_
left[0] -= padding // 2
left[1] -= padding // 2
notch[1] += padding // 2
right[0] += padding // 2
right[1] -= padding // 2
sshape = np.array([left, notch, right], np.float32)
tshape = np.array([[0, 0], [w0 // 2, h0], [w0, 0]], np.float32)
sshape = sshape.reshape(1, -1, 2)
tshape = tshape.reshape(1, -1, 2)
matches = [
cv2.DMatch(0, 0, 0),
cv2.DMatch(1, 1, 0),
cv2.DMatch(2, 2, 0),
]
tps.clear()
tps.estimateTransformation(tshape, sshape, matches)
chip_ = tps.warpImage(chip_)
chip_ = chip_[:h0, :w0, :]
chip_h, chip_w = chip_.shape[:2]
yield (
chip_,
chip_w,
chip_h,
)
font = ImageFont.truetype(opt.font_path, opt.font_size)
tps_width = opt.font_size
scale_height = opt.spare_height
with open(opt.word_document) as to_read, open(opt.file_list_document,
'w') as to_write:
for line in to_read:
line = line.strip()
font_width, font_height = font.getsize(line)
image = Image.new('RGBA',
(font_width, font_height + opt.spare_height * 2),
bgcolor)
draw = ImageDraw.Draw(image)
draw.text((0, opt.spare_height), line, font=font, fill=fontcolor)
img = np.array(image)
tps = cv2.createThinPlateSplineShapeTransformer()
# tps.setRegularizationParameter(0.3)
height, width, channel = img.shape
pt_1 = [[x,opt.spare_height] for x in range(0,width,tps_width)] +\
[[x,height-opt.spare_height] for x in range(0,width,tps_width)]
pt_2 = [[
pt[0],
random.randint(-scale_height, scale_height) + pt[1]
] for pt in pt_1]
pt_size = len(pt_1)
matches = [cv2.DMatch(i, i, 0) for i in range(pt_size)]
M = tps.estimateTransformation(
np.array(pt_1).reshape((1, pt_size, 2)),
np.array(pt_2).reshape((1, pt_size, 2)), matches)
res = tps.warpImage(img, M)
res = 255 - res
def augment_image_and_mask(self,
gt_arr,
gt_path=None,
affine_transformation_path=None,
non_rigid_deform_path=None):
#gt_arr shape (H,W) and binary(0,1)
# let us do non-rigid deformation
N = 5
Delta = 0.05
H, W = gt_arr.shape
#get the target boundary
kernel = cv2.getStructuringElement(cv2.MORPH_ELLIPSE, (3, 3))
boundary = cv2.dilate(gt_arr, kernel) - gt_arr
boundindex = np.where(boundary == 1)
num_index = boundindex[0].shape[0]
if num_index > N:
maxH, minH = max(boundindex[0]), min(boundindex[0])
tarH = maxH - minH
maxW, minW = max(boundindex[1]), min(boundindex[1])
tarW = maxW - minW
# thin plate spline coord num
randindex = [random.randint(0, num_index - 1) for _ in range(N)]
sourcepoints = []
targetpoints = []
for i in range(N):
sourcepoints.append(
(boundindex[1][randindex[i]], boundindex[0][randindex[i]]))
x = boundindex[1][randindex[i]] + int(
random.uniform(-Delta, Delta) * tarW)
y = boundindex[0][randindex[i]] + int(
random.uniform(-Delta, Delta) * tarH)
targetpoints.append((x, y))
sourceshape = np.array(sourcepoints, np.int32)
sourceshape = sourceshape.reshape(1, -1, 2)
targetshape = np.array(targetpoints, np.int32)
targetshape = targetshape.reshape(1, -1, 2)
matches = []
for i in range(0, N):
matches.append(cv2.DMatch(i, i, 0))
tps = cv2.createThinPlateSplineShapeTransformer()
tps.estimateTransformation(targetshape, sourceshape, matches)
no_grid_img = tps.warpImage(gt_arr)
kernel = cv2.getStructuringElement(cv2.MORPH_ELLIPSE, (5, 5))
no_grid_img = cv2.dilate(no_grid_img, kernel)
#for p in sourcepoints:
# cv2.circle(gt_arr,p,2,(2),2)
#for p in targetpoints:
# cv2.circle(no_grid_img,p,2,(2),2)
gt_out = gt_arr * 255
no_grid_img = no_grid_img * 255
gt_out = Image.fromarray(gt_out)
no_grid_out = Image.fromarray(no_grid_img)
#let us do affine transformation
scale = 0.98
randScale = random.uniform(scale, 1 / scale)
M = cv2.getRotationMatrix2D(
((maxH + minH) * 0.5, (maxW - minW) * 0.5), 0, randScale)
dx = round(random.uniform(-0.05, 0.05) * tarW)
dy = round(random.uniform(-0.05, 0.05) * tarH)
M[0, 2] += dx
M[1, 2] += dy
affine_out = cv2.warpAffine(gt_arr, M, (W, H)) * 255
affine_out = Image.fromarray(affine_out)
else:
gt_out = Image.fromarray(gt_arr * 255)
no_grid_out = Image.fromarray(gt_arr * 255)
affine_out = Image.fromarray(gt_arr * 255)
#save gt and none_rigid_deform and affine_tran
gt_out.save(gt_path)
no_grid_out.save(non_rigid_deform_path)
affine_out.save(affine_transformation_path)
def create_descriptor_inhomogeneous_seg():
'''
only used with segment guide
'''
tps = cv2.createThinPlateSplineShapeTransformer()
image_info = io.load_json('temp/patch_matching/label/image_info.json')
pose_label = io.load_data('datasets/DF_Pose/Label/pose_label.pkl')
pose_np = np.array(pose_label.values())
valid_rshoulder = pose_np[:, 2, 0] >= 0
mean_p_rshoulder = pose_np[valid_rshoulder, 2, :].mean(axis=0)
valid_lshoulder = pose_np[:, 5, 0] >= 0
mean_p_lshoulder = pose_np[valid_lshoulder, 5, :].mean(axis=0)
valid_rhip = pose_np[:, 8, 0] >= 0
mean_p_rhip = pose_np[valid_rhip, 8, :].mean(axis=0)
valid_lhip = pose_np[:, 11, 0] >= 0
mean_p_lhip = pose_np[valid_lhip, 11, :].mean(axis=0)
img_size = 256
gx, gy = np.meshgrid(np.linspace(-1, 1, img_size),
np.linspace(-1, 1, img_size),
indexing='xy')
grid_std = np.stack((gx, gy), axis=2)
kp_std = np.stack(
[mean_p_rshoulder, mean_p_rhip, mean_p_lhip,
mean_p_lshoulder]).reshape(1, -1, 2).astype(np.float64)
desc = {}
for subset in ['1', '2', 'gen']:
print(subset)
desc[subset] = []
for i in range(num_sample):
if subset == '1':
pose = np.array(pose_label[image_info['id_1'][i]])
else:
pose = np.array(pose_label[image_info['id_2'][i]])
kp_tar = pose[[2, 8, 11, 5], :].reshape(1, -1,
2).astype(np.float64)
kp_matches = []
for j in range(kp_tar.shape[1]):
if (kp_tar[0, j] >= 0).all():
kp_matches.append(cv2.DMatch(j, j, 0))
if len(kp_matches) <= 1:
grid_tar = grid_std.copy()
else:
tps.estimateTransformation(kp_tar, kp_std, kp_matches)
grid_tar = tps.warpImage(grid_std.copy())
desc[subset].append(grid_tar)
desc[subset] = np.stack(desc[subset])
data_dict_gt = {
'desc_1': desc['1'],
'desc_2': desc['2'],
'name': 'gt_inhomoseg'
}
data_dict_gen = {
'desc_1': desc['1'],
'desc_2': desc['gen'],
'name': 'gen_inhomoseg'
}
scipy.io.matlab.savemat(
'temp/patch_matching/descriptor/desc_gt_inhomoseg.mat', data_dict_gt)
scipy.io.matlab.savemat(
'temp/patch_matching/descriptor/desc_gen_inhomoseg.mat', data_dict_gen)
def __init__(self):
self.tps = cv2.createThinPlateSplineShapeTransformer()
self.translateXAbs = TranslateXAbs()
self.translateYAbs = TranslateYAbs()
def deformation(self, label_path=None, save_path=None):
label_img = Image.open(label_path)
palette = label_img.getpalette()
gt_arr = np.array(label_img)
# let us do non-rigid deformation
N = 5
Delta = 0.05
#get the target boundary
kernel = cv2.getStructuringElement(cv2.MORPH_ELLIPSE, (3, 3))
boundary = cv2.dilate(gt_arr, kernel) - gt_arr
boundindex = np.where(boundary == 1)
num_index = boundindex[0].shape[0]
if num_index > N:
maxH, minH = max(boundindex[0]), min(boundindex[0])
tarH = maxH - minH
maxW, minW = max(boundindex[1]), min(boundindex[1])
tarW = maxW - minW
# thin plate spline coord num
randindex = [random.randint(0, num_index - 1) for _ in range(N)]
sourcepoints = []
targetpoints = []
for i in range(N):
sourcepoints.append(
(boundindex[1][randindex[i]], boundindex[0][randindex[i]]))
x = boundindex[1][randindex[i]] + int(
random.uniform(-Delta, Delta) * tarW)
y = boundindex[0][randindex[i]] + int(
random.uniform(-Delta, Delta) * tarH)
targetpoints.append((x, y))
sourceshape = np.array(sourcepoints, np.int32)
sourceshape = sourceshape.reshape(1, -1, 2)
targetshape = np.array(targetpoints, np.int32)
targetshape = targetshape.reshape(1, -1, 2)
matches = []
for i in range(0, N):
matches.append(cv2.DMatch(i, i, 0))
tps = cv2.createThinPlateSplineShapeTransformer()
tps.estimateTransformation(targetshape, sourceshape, matches)
no_grid_img = tps.warpImage(gt_arr)
kernel = cv2.getStructuringElement(cv2.MORPH_ELLIPSE, (5, 5))
no_grid_img = cv2.dilate(no_grid_img, kernel)
if self.show_point:
for p in sourcepoints:
cv2.circle(gt_arr, p, 4, (2), 2)
for p in targetpoints:
cv2.circle(no_grid_img, p, 4, (2), 2)
no_grid_out = Image.fromarray(no_grid_img)
gt_out = Image.fromarray(gt_arr)
else:
no_grid_out = Image.fromarray(gt_arr)
gt_out = Image.fromarray(gt_arr)
#save gt and none_rigid_deform and affine_tran
no_grid_out.putpalette(palette)
no_grid_out.save(save_path)
if self.show_point:
gt_out.putpalette(palette)
gt_out.save(label_path[:-4] + '_gt' + '.png')
no_grid_out.show('Deformation result')
gt_out.show('source label')
