def alignment(src_img, src_pts, output_size=(96, 112)): """Warp a face image so that its features align with a canoncial reference set of landmarks. The alignment is performed with an affine warp Args: src_img (ndarray): an HxWx3 RGB containing a face src_pts (ndarray): a 5x2 array of landmark locations output_size (tuple): the dimensions (oH, oW) of the output image Returns: (ndarray): an (oH x oW x 3) warped RGB image. """ ref_pts = [ [30.2946, 51.6963], [65.5318, 51.5014], [48.0252, 71.7366], [33.5493, 92.3655], [62.7299, 92.2041], ] src_pts = np.array(src_pts).reshape(5, 2) s = np.array(src_pts).astype(np.float32) r = np.array(ref_pts).astype(np.float32) tfm = get_similarity_transform_for_cv2(s, r) face_img = cv2.warpAffine(src_img, tfm, output_size) return face_img
def align(src_img, src_pts, ref_pts, image_size, scale=1.0, transpose_input=False, landmarks_label=None): w, h = image_size = tuple(image_size) # Actual offset = new center - old center (scaled) scale_ = max(w, h) * scale cx_ref = cy_ref = 0. offset_x = 0.5 * w - cx_ref * scale_ offset_y = 0.5 * h - cy_ref * scale_ s = np.array(src_pts).astype(np.float32).reshape([-1, 2]) r = np.array(ref_pts).astype(np.float32) * scale_ + np.array( [[offset_x, offset_y]]) if transpose_input: s = s.reshape([2, -1]).T tfm = get_similarity_transform_for_cv2(s, r) dst_img = cv2.warpAffine(src_img, tfm, image_size) s_new = np.concatenate([s.reshape([2, -1]), np.ones((1, s.shape[0]))]) lm_new = np.concatenate( [landmarks_label.T, np.ones((1, landmarks_label.shape[0]))]) lm_new = np.matmul(tfm, lm_new) lm_new = lm_new.T s_new = np.matmul(tfm, s_new) s_new = s_new.reshape([-1]) if transpose_input else s_new.T.reshape([-1]) tfm = tfm.reshape([-1]) return dst_img, lm_new, tfm
def alignment(src_img, src_pts): #校正人臉 of = 0 #ref_pts = [ [30.2946+of, 51.6963+of],[65.5318+of, 51.5014+of],[48.0252+of, 71.7366+of] ] ref_pts = [[86 + of, 140 + of], [172 + of, 140 + of], [129 + of, 170 + of]] #[29,80][159,80][96,157] crop_size = (304, 320) #(96+of*2, 112+of*2)#(144+of*2,160+of*2) # ref_pts = [ [86+of,80+of],[300+of,80+of],[129+of,120+of]] # crop_size =(304,320)#src_img.shape[:2]#(160,144) xx1, xx2, yy1, yy2 = 76, 182, 105, 200 #20,75,46,76 s = np.array(src_pts).astype(np.float32) r = np.array(ref_pts).astype(np.float32) tfm = get_similarity_transform_for_cv2(s, r) M = cv2.getAffineTransform(s, r) face_img = cv2.warpAffine(src_img, tfm, crop_size) #face_img = cv2.warpAffine(src_img, M, crop_size) plt_show(face_img) rx, ry = 0, 0 crop_face_img = face_img[yy1 - ry:yy2 + ry, xx1 - rx:xx2 + rx] plt_show(crop_face_img) #crop_face_img=face_img[yy1-ry:yy2+ry,xx1-rx:xx2+rx] resize = (160, 144) #(96, 112) #plt_show(face_img) #plt_show(crop_face_img) resize_face_img = cv2.resize(crop_face_img, resize) #crop_face_img #cv2.imwrite('output.jpg', resize_face_img) return resize_face_img
def alignment(self,src_img,src_pts): ref_pts = [ [30.2946, 51.6963],[65.5318, 51.5014],[48.0252, 71.7366],[33.5493, 92.3655],[62.7299, 92.2041]] crop_size = (96, 112) src_pts = np.array(src_pts).reshape(5,2) s = np.array(src_pts).astype(np.float32) r = np.array(ref_pts).astype(np.float32) tfm = get_similarity_transform_for_cv2(s, r) face_img = cv2.warpAffine(src_img, tfm, crop_size) return face_img
def alignment(src_img,src_pts): of = 2 ref_pts = [ [30.2946+of, 51.6963+of],[65.5318+of, 51.5014+of], [48.0252+of, 71.7366+of],[33.5493+of, 92.3655+of],[62.7299+of, 92.2041+of] ] crop_size = (96+of*2, 112+of*2) s = np.array(src_pts).astype(np.float32) r = np.array(ref_pts).astype(np.float32) print s tfm = get_similarity_transform_for_cv2(s, r) face_img = cv2.warpAffine(src_img, tfm, crop_size) return face_img
def alignment_test(src_img,src_pts): of = 0 ref_pts = [ [30.2946+of, 51.6963+of],[65.5318+of, 51.5014+of],[48.0252+of, 71.7366+of]] crop_size =(96+of*2, 112+of*2) xx1,xx2,yy1,yy2=30,65,51,92 #print(src_img.shape[1],src_img.shape[0]) s = np.array(src_pts).astype(np.float32) r = np.array(ref_pts).astype(np.float32) tfm = get_similarity_transform_for_cv2(s, r) face_img = cv2.warpAffine(src_img, tfm, crop_size) plt_show(face_img) rx,ry=12,14 crop_face_img=face_img[yy1-ry:yy2+ry,xx1-rx:xx2+rx] resize_face_img=cv2.resize(crop_face_img, (96, 112)) return resize_face_img
def alignment(src_img, src_pts): ref_pts = [[30.2946, 51.6963], [65.5318, 51.5014], [48.0252, 71.7366], [33.5493, 92.3655], [62.7299, 92.2041]] crop_size = (96, 112) src_pts = [ src_pts["keypoints"]["left_eye"], src_pts["keypoints"]["right_eye"], src_pts["keypoints"]["nose"], src_pts["keypoints"]["mouth_left"], src_pts["keypoints"]["mouth_right"] ] src_pts = np.array(src_pts) s = np.array(src_pts).astype(np.float32) r = np.array(ref_pts).astype(np.float32) tfm = get_similarity_transform_for_cv2(s, r) face_img = cv2.warpAffine(src_img, tfm, crop_size) return face_img
def alignment(src_img, src_pts, size=None): ref_pts = [[30.2946, 51.6963], [65.5318, 51.5014], [48.0252, 71.7366], [33.5493, 92.3655], [62.7299, 92.2041]] if size is not None: ref_pts = np.array(ref_pts) ref_pts[:,0] = ref_pts[:,0] * size/96 ref_pts[:,1] = ref_pts[:,1] * size/96 crop_size = (int(size), int(112/(96/size))) else: crop_size = (96, 112) src_pts = np.array(src_pts).reshape(5, 2) s = np.array(src_pts).astype(np.float32) r = np.array(ref_pts).astype(np.float32) tfm = get_similarity_transform_for_cv2(s, r) face_img = cv2.warpAffine(src_img, tfm, crop_size, flags=cv2.INTER_CUBIC) if size is not None: face_img = cv2.resize(face_img, dsize=(96, 112), interpolation=cv2.INTER_CUBIC) return face_img
def alignment(src_img, src_pts): min_y = min(src_pts["keypoints"]["left_eye"][1], src_pts["keypoints"]["right_eye"][1]) max_y = max(src_pts["keypoints"]["mouth_left"][1], src_pts["keypoints"]["mouth_right"][1]) min_x = min(src_pts["keypoints"]["left_eye"][0], src_pts["keypoints"]["mouth_left"][0]) max_x = max(src_pts["keypoints"]["right_eye"][0], src_pts["keypoints"]["mouth_right"][0]) # print(src_pts["keypoints"]["right_eye"]) height = max_y - min_y width = max_x - min_x db_img = src_img[ max(0, int(min_y - height * 1.8)):min(src_img.shape[0], int(max_y + height * 1.5)), max(0, int(min_x - width * 1.2)):min(src_img.shape[1], int(max_x + width * 1.2))] #cv2.imshow("windows", db_img) #cv2.waitKey(0) ref_pts = [[30.2946, 51.6963], [65.5318, 51.5014], [48.0252, 71.7366], [33.5493, 92.3655], [62.7299, 92.2041]] crop_size = (96, 112) src_pts = [ src_pts["keypoints"]["left_eye"], src_pts["keypoints"]["right_eye"], src_pts["keypoints"]["nose"], src_pts["keypoints"]["mouth_left"], src_pts["keypoints"]["mouth_right"] ] src_pts = np.array(src_pts) s = np.array(src_pts).astype(np.float32) r = np.array(ref_pts).astype(np.float32) tfm = get_similarity_transform_for_cv2(s, r) face_img = cv2.warpAffine(src_img, tfm, crop_size) return face_img, db_img
def return_warp_vid(vidname, t=120): path = return_path(vidname) openposename = os.listdir(openpose_coordinates_dir+path)[0] openposename =openposename.split('_keypoints.json')[0][:-13] start_frame, end_frame = return_start_end(vidname) for frame_num in range(start_frame, end_frame+1): ##################### read openpose openpose_path = openpose_coordinates_dir+path+'/'+openposename+'_'+'{:012d}'.format(frame_num)+'_keypoints.json' data = json.load(open(openpose_path)) arr=data['people'][0]['face_keypoints_2d'] coords_list=[(round(arr[i*3]),round(arr[i*3+1])) for i in range(int(len(arr)/3))] ##################### find face coordinates bb_x_min = min([item[0] for item in coords_list]) bb_x_max = max([item[0] for item in coords_list]) bb_y_min = min([item[1] for item in coords_list]) bb_y_max = max([item[1] for item in coords_list]) c1,c2 = int(np.mean([bb_x_min,bb_x_max])), int(np.mean([bb_y_min,bb_y_max])) face_coords = [c1-t,c1+t,c2-t,c2+t] #x_min, x_max, y_min, y_max #####################crop the frame from high resolution to 240x240 frame_path = return_highres_path(vidname)+'/frame_{}.png'.format(frame_num) im = cv2.imread(frame_path) cropped_img = im[face_coords[2]:face_coords[3], face_coords[0]:face_coords[1]] ##################### shift the coordinates x_org,y_org = [c1-t,c2-t] shifted_coords_list = [(x-x_org,y-y_org) for (x,y) in coords_list] coord_arr = np.array(shifted_coords_list).astype(np.float32) mean_coords_list = [(71, 92), (71, 105), (72, 118), (74, 131), (79, 143), (86, 153), (96, 161), (108, 167), (121, 169), (133, 167), (144, 162), (154, 154), (161, 144), (165, 132), (167, 120), (169, 108), (170, 95), (82, 77), (89, 73), (97, 71), (106, 72), (114, 75), (131, 76), (139, 73), (147, 73), (155, 76), (160, 81), (122, 87), (122, 94), (122, 100), (122, 107), (112, 116), (117, 118), (122, 119), (126, 118), (131, 117), (92, 89), (97, 86), (103, 86), (108, 89), (103, 91), (97, 91), (135, 91), (140, 88), (146, 88), (151, 91), (146, 93), (140, 92), (104, 135), (110, 130), (116, 127), (121, 128), (126, 127), (133, 130), (138, 136), (133, 140), (126, 142), (121, 142), (115, 142), (109, 139), (107, 134), (116, 132), (121, 133), (126, 133), (135, 135), (126, 135), (121, 135), (116, 135), (100, 88), (143, 90)] mean_arr = np.array(mean_coords_list).astype(np.float32) ##################### warp the cropped image tfm = get_similarity_transform_for_cv2(coord_arr,mean_arr) warped_img = cv2.warpAffine(cropped_img, tfm, (240, 240)) ##################### crop the warped image crop_warped_img = warped_img[41: 169, 71:170, :] ##################### write the outout image if not os.path.exists('output/'+vidname): os.mkdir('output/'+vidname) des_path = 'output/'+vidname+'/frame_{}.png'.format(frame_num) cv2.imwrite(des_path,crop_warped_img)
def alignment_orig(src_img, src_pts, ncols=96, nrows=112, custom_align=None): """ Original alignment function for MTCNN :param src_img: input image :param src_pts: landmark points :return: """ from matlab_cp2tform import get_similarity_transform_for_cv2 ref_pts = [[30.2946, 51.6963], [65.5318, 51.5014], [48.0252, 71.7366], [33.5493, 92.3655], [62.7299, 92.2041]] if custom_align is not None: row[0] += custom_align[0] row[1] += custom_align[1] elif ncols == 112: for row in ref_pts: row[0] += 8.0 elif ncols == 128: for row in ref_pts: row[0] += 16.0 if nrows == 128 and custom_align is None: for row in ref_pts: row[1] += 16.0 # print(ref_pts) crop_size = (ncols, nrows) src_pts = np.array(src_pts).reshape(5, 2) s = np.array(src_pts).astype(np.float32) r = np.array(ref_pts).astype(np.float32) tfm = get_similarity_transform_for_cv2(s, r) face_img = cv2.warpAffine(src_img, tfm, crop_size) return face_img
def warp_and_crop_face(src_img, facial_pts, reference_pts = None, crop_size=(96, 112), align_type = 'smilarity'): """ Function: ---------- apply affine transform 'trans' to uv Parameters: ---------- @src_img: 3x3 np.array input image @facial_pts: could be 1)a list of K coordinates (x,y) or 2) Kx2 or 2xK np.array each row or col is a pair of coordinates (x, y) @reference_pts: could be 1) a list of K coordinates (x,y) or 2) Kx2 or 2xK np.array each row or col is a pair of coordinates (x, y) or 3) None if None, use default reference facial points @crop_size: (w, h) output face image size @align_type: transform type, could be one of 1) 'similarity': use similarity transform 2) 'cv2_affine': use the first 3 points to do affine transform, by calling cv2.getAffineTransform() 3) 'affine': use all points to do affine transform Returns: ---------- @face_img: output face image with size (w, h) = @crop_size """ if reference_pts is None: if crop_size[0] == 96 and crop_size[1] == 112: reference_pts = REFERENCE_FACIAL_POINTS else: default_square = False inner_padding_factor = 0 outer_padding = (0, 0) output_size = crop_size reference_pts = get_reference_facial_points(output_size, inner_padding_factor, outer_padding, default_square) ref_pts = np.float32(reference_pts) ref_pts_shp = ref_pts.shape if max(ref_pts_shp) < 3 or min(ref_pts_shp) != 2: raise FaceWarpException( 'reference_pts.shape must be (K,2) or (2,K) and K>2') if ref_pts_shp[0] == 2: ref_pts = ref_pts.T src_pts = np.float32(facial_pts) src_pts_shp = src_pts.shape if max(src_pts_shp) < 3 or min(src_pts_shp) != 2: raise FaceWarpException( 'facial_pts.shape must be (K,2) or (2,K) and K>2') if src_pts_shp[0] == 2: src_pts = src_pts.T # #print('--->src_pts:\n', src_pts # #print('--->ref_pts\n', ref_pts if src_pts.shape != ref_pts.shape: raise FaceWarpException( 'facial_pts and reference_pts must have the same shape') if align_type is 'cv2_affine': tfm = cv2.getAffineTransform(src_pts[0:3], ref_pts[0:3]) # #print(('cv2.getAffineTransform() returns tfm=\n' + str(tfm)) elif align_type is 'affine': tfm = get_affine_transform_matrix(src_pts, ref_pts) # #print(('get_affine_transform_matrix() returns tfm=\n' + str(tfm)) else: tfm = get_similarity_transform_for_cv2(src_pts, ref_pts) # #print(('get_similarity_transform_for_cv2() returns tfm=\n' + str(tfm)) # #print('--->Transform matrix: ' # #print(('type(tfm):' + str(type(tfm))) # #print(('tfm.dtype:' + str(tfm.dtype)) # #print( tfm face_img = cv2.warpAffine(src_img, tfm, (crop_size[0], crop_size[1])) return face_img
def warp_and_crop_face(src_img, facial_pts, reference_pts=None, crop_size=(96, 112), align_type='smilarity'): """ Function: ---------- apply affine transform 'trans' to uv Parameters: ---------- @src_img: 3x3 np.array input image @facial_pts: could be 1)a list of K coordinates (x,y) or 2) Kx2 or 2xK np.array each row or col is a pair of coordinates (x, y) @reference_pts: could be 1) a list of K coordinates (x,y) or 2) Kx2 or 2xK np.array each row or col is a pair of coordinates (x, y) or 3) None if None, use default reference facial points @crop_size: (w, h) output face image size @align_type: transform type, could be one of 1) 'similarity': use similarity transform 2) 'cv2_affine': use the first 3 points to do affine transform, by calling cv2.getAffineTransform() 3) 'affine': use all points to do affine transform Returns: ---------- @face_img: output face image with size (w, h) = @crop_size """ if reference_pts is None: if crop_size[0] == 96 and crop_size[1] == 112: reference_pts = REFERENCE_FACIAL_POINTS else: default_square = False inner_padding_factor = 0 outer_padding = (0, 0) output_size = crop_size reference_pts = get_reference_facial_points( output_size, inner_padding_factor, outer_padding, default_square) ref_pts = np.float32(reference_pts) ref_pts_shp = ref_pts.shape if max(ref_pts_shp) < 3 or min(ref_pts_shp) != 2: raise FaceWarpException( 'reference_pts.shape must be (K,2) or (2,K) and K>2') if ref_pts_shp[0] == 2: ref_pts = ref_pts.T src_pts = np.float32(facial_pts) src_pts_shp = src_pts.shape if max(src_pts_shp) < 3 or min(src_pts_shp) != 2: raise FaceWarpException( 'facial_pts.shape must be (K,2) or (2,K) and K>2') if src_pts_shp[0] == 2: src_pts = src_pts.T if src_pts.shape != ref_pts.shape: raise FaceWarpException( 'facial_pts and reference_pts must have the same shape') if align_type is 'cv2_affine': tfm = cv2.getAffineTransform(src_pts[0:3], ref_pts[0:3]) elif align_type is 'affine': tfm = get_affine_transform_matrix(src_pts, ref_pts) else: tfm = get_similarity_transform_for_cv2(src_pts, ref_pts) face_img = cv2.warpAffine(src_img, tfm, (crop_size[0], crop_size[1])) return face_img
def warp_and_crop_face(src_img, facial_pts, reference_pts=None, crop_size=(96, 112), align_type='smilarity'): """ Function: ---------- apply affine transform 'trans' to uv Parameters: ---------- @src_img: 3x3 np.array input image @facial_pts: could be 1)a list of K coordinates (x,y) or 2) Kx2 or 2xK np.array each row or col is a pair of coordinates (x, y) @reference_pts: could be 1) a list of K coordinates (x,y) or 2) Kx2 or 2xK np.array each row or col is a pair of coordinates (x, y) or 3) None if None, use default reference facial points @crop_size: (w, h) output face image size @align_type: transform type, could be one of 1) 'similarity': use similarity transform 2) 'cv2_affine': use the first 3 points to do affine transform, by calling cv2.getAffineTransform() 3) 'affine': use all points to do affine transform Returns: ---------- @face_img: output face image with size (w, h) = @crop_size """ if reference_pts is None: reference_pts = REFERENCE_FACIAL_POINTS ref_pts = np.float32(reference_pts) ref_pts_shp = ref_pts.shape if max(ref_pts_shp) < 3 or min(ref_pts_shp) != 2: raise FaceWarpException( 'reference_pts.shape must be (K,2) or (2,K) and K>2') if ref_pts_shp[0] == 2: ref_pts = ref_pts.T src_pts = np.float32(facial_pts) src_pts_shp = src_pts.shape if max(src_pts_shp) < 3 or min(src_pts_shp) != 2: raise FaceWarpException( 'facial_pts.shape must be (K,2) or (2,K) and K>2') if src_pts_shp[0] == 2: src_pts = src_pts.T # print '--->src_pts:\n', src_pts # print '--->ref_pts\n', ref_pts if src_pts.shape != ref_pts.shape: raise FaceWarpException( 'facial_pts and reference_pts must have the same shape') if align_type is 'cv2_affine': tfm = cv2.getAffineTransform(src_pts[0:3], ref_pts[0:3]) # print('cv2.getAffineTransform() returns tfm=\n' + str(tfm)) elif align_type is 'affine': tfm = get_affine_transform_matrix(src_pts, ref_pts) # print('get_affine_transform_matrix() returns tfm=\n' + str(tfm)) else: tfm = get_similarity_transform_for_cv2(src_pts, ref_pts) # print('get_similarity_transform_for_cv2() returns tfm=\n' + str(tfm)) # print '--->Transform matrix: ' # print('type(tfm):' + str(type(tfm))) # print('tfm.dtype:' + str(tfm.dtype)) # print tfm face_img = cv2.warpAffine(src_img, tfm, (crop_size[0], crop_size[1])) return face_img