def getFaceTextureCoords(img, mean3DShape, blendshapes, idxs2D, idxs3D, detector, predictor, maxImgSizeForDetection=320): projectionModel = models.OrthographicProjectionBlendshapes( blendshapes.shape[0]) keypoints = getFaceKeypoints(img, detector, predictor, maxImgSizeForDetection)[0] modelParams = projectionModel.getInitialParameters(mean3DShape[:, idxs3D], keypoints[:, idxs2D]) modelParams = NonLinearLeastSquares.GaussNewton( modelParams, projectionModel.residual, projectionModel.jacobian, ([mean3DShape[:, idxs3D], blendshapes[:, :, idxs3D] ], keypoints[:, idxs2D]), verbose=0) textureCoords = projectionModel.fun([mean3DShape, blendshapes], modelParams) return textureCoords
def faceswap(): cameraImg = cap.read()[1] rows, cols = cameraImg.shape[:2] translation_matrix = np.float32([[1, 0, int(0.5 * cols)], [0, 1, int(0.5 * rows)]]) img_t = cv2.warpAffine(cameraImg, translation_matrix, (2 * cols, 2 * rows)) rotation_matrix = cv2.getRotationMatrix2D((cols, rows), -90, 1) img_r = cv2.warpAffine(img_t, rotation_matrix, (2 * cols, 2 * rows)) translation_matrix = np.float32([[1, 0, -int(cols - (0.5 * rows))], [0, 1, -int(rows - (0.5 * cols))]]) cameraImg = cv2.warpAffine(img_r, translation_matrix, (rows, cols)) shapes2D = utils.getFaceKeypoints(cameraImg, detector, predictor, maxImageSizeForDetection) if shapes2D is not None: for shape2D in shapes2D: #3D model parameter initialization modelParams = projectionModel.getInitialParameters( mean3DShape[:, idxs3D], shape2D[:, idxs2D]) #3D model parameter optimization modelParams = NonLinearLeastSquares.GaussNewton( modelParams, projectionModel.residual, projectionModel.jacobian, ([mean3DShape[:, idxs3D], blendshapes[:, :, idxs3D] ], shape2D[:, idxs2D]), verbose=0) #rendering the model to an image shape3D = utils.getShape3D(mean3DShape, blendshapes, modelParams) renderedImg = renderer.render(shape3D) #blending of the rendered face with the image mask = np.copy(renderedImg[:, :, 0]) renderedImg = ImageProcessing.colorTransfer( cameraImg, renderedImg, mask) cameraImg = ImageProcessing.blendImages(renderedImg, cameraImg, mask) #drawing of the mesh and keypoints if drawOverlay: drawPoints(cameraImg, shape2D.T) drawProjectedShape(cameraImg, [mean3DShape, blendshapes], projectionModel, mesh, modelParams, lockedTranslation) if writer is not None: writer.write(cameraImg) cv2.imshow('image', cameraImg) return cameraImg key = cv2.waitKey(1) if key == 27: return []
def draw_face_mesh(self, frame, roi, landmarks, head_pose): # print('draw_face_mesh-------------------') if self.shapes2D is not None: for shape2D in self.shapes2D: print(shape2D[0], len(shape2D[0]) ) # list of [[x1,x2,x3.....xn],[y1,y2,y3.....yn]] #3D model parameter initialization modelParams = self.projectionModel.getInitialParameters( self.mean3DShape[:, self.idxs3D], shape2D[:, self.idxs2D]) #3D model parameter optimization modelParams = NonLinearLeastSquares.GaussNewton( modelParams, self.projectionModel.residual, self.projectionModel.jacobian, ([ self.mean3DShape[:, self.idxs3D], self.blendshapes[:, :, self.idxs3D] ], shape2D[:, self.idxs2D]), verbose=0) #rendering the model to an image shape3D = utils.getShape3D(self.mean3DShape, self.blendshapes, modelParams) renderedImg = self.renderer.render(shape3D) #blending of the rendered face with the image mask = np.copy(renderedImg[:, :, 0]) renderedImg = ImageProcessing.colorTransfer( frame, renderedImg, mask) # apply rendered image on frame frame = ImageProcessing.blendImages(renderedImg, frame, mask) # cv2.imshow('frame', frame) #drawing of the mesh and keypoints if self.drawOverlay: drawPoints(frame, shape2D.T) drawProjectedShape(frame, [self.mean3DShape, self.blendshapes], self.projectionModel, self.mesh, modelParams, self.lockedTranslation) cv2.imshow('Face recognition demo', frame) if self.writer is not None: # output_stream.write(frame) self.writer.write(frame)
def calcTextureCoords(self, landmarks): #projectionModel = models.OrthographicProjectionBlendshapes(self.blendshapes.shape[0]) modelParams = self.projectionModel.getInitialParameters( self.mean3DShape[:, self.idxs3D], landmarks[:, self.idxs2D]) modelParams = NonLinearLeastSquares.GaussNewton( modelParams, self.projectionModel.residual, self.projectionModel.jacobian, ([ self.mean3DShape[:, self.idxs3D], self.blendshapes[:, :, self.idxs3D] ], landmarks[:, self.idxs2D]), verbose=0) textureCoords = self.projectionModel.fun( [self.mean3DShape, self.blendshapes], modelParams) return textureCoords
def getFaceTextureCoords(img, mean3DShape, blendshapes, idxs2D, idxs3D, openpose): projectionModel = models.OrthographicProjectionBlendshapes( blendshapes.shape[0]) arr, arr2, output_image = openpose.forward(img, False) for shape2D in arr2: keypoints = shape2D[:, :2].T modelParams = projectionModel.getInitialParameters(mean3DShape[:, idxs3D], keypoints[:, idxs2D]) modelParams = NonLinearLeastSquares.GaussNewton( modelParams, projectionModel.residual, projectionModel.jacobian, ([mean3DShape[:, idxs3D], blendshapes[:, :, idxs3D] ], keypoints[:, idxs2D]), verbose=0) textureCoords = projectionModel.fun([mean3DShape, blendshapes], modelParams) return textureCoords
def swap(self, target_img, target_seg=None): landmarks_set = self.calcLandmarks(target_img) if landmarks_set is None: return None for landmarks in landmarks_set: # 3D model parameter optimization modelParams = self.projectionModel.getInitialParameters( self.mean3DShape[:, self.idxs3D], landmarks[:, self.idxs2D]) modelParams = NonLinearLeastSquares.GaussNewton( modelParams, self.projectionModel.residual, self.projectionModel.jacobian, ([ self.mean3DShape[:, self.idxs3D], self.blendshapes[:, :, self.idxs3D] ], landmarks[:, self.idxs2D]), verbose=0) # Render the model to the target image shape3D = utils.getShape3D(self.mean3DShape, self.blendshapes, modelParams) rendered_img = self.renderer.render(shape3D) # Blend images mask = np.zeros(rendered_img.shape, rendered_img.dtype) red, green, blue = rendered_img[:, :, 0], rendered_img[:, :, 1], rendered_img[:, :, 2] binmask = (red != 0) | (green != 0) | (blue != 0) if target_seg is not None: binmask = binmask & (target_seg != 0) mask[:, :, :3][binmask] = [255, 255, 255] cmin, rmin, cmax, rmax = self.calcBBox(binmask) center = ((cmin + cmax) / 2, (rmin + rmax) / 2) output = cv2.seamlessClone(rendered_img, target_img, mask, center, cv2.NORMAL_CLONE) return output
def process_video(self, in_filename, out_filename, face_filename, keep_audio=True): # extract audio clip from src if keep_audio == True: clip = VideoFileClip(in_filename) clip.audio.write_audiofile("./temp/src_audio.mp3", verbose=False) predictor_path = "./models/shape_predictor_68_face_landmarks.dat" # predictor_path = "./models/shape_predictor_81_face_landmarks.dat" # the smaller this value gets the faster the detection will work # if it is too small, the user's face might not be detected maxImageSizeForDetection = 320 detector = dlib.get_frontal_face_detector() predictor = dlib.shape_predictor(predictor_path) mean3DShape, blendshapes, mesh, idxs3D, idxs2D = utils.load3DFaceModel( "./models/candide.npz") projectionModel = models.OrthographicProjectionBlendshapes( blendshapes.shape[0]) # open source video vidcap = cv2.VideoCapture(in_filename) # get some parameters from input video width = int(vidcap.get(cv2.CAP_PROP_FRAME_WIDTH)) height = int(vidcap.get(cv2.CAP_PROP_FRAME_HEIGHT)) fps = int(vidcap.get(cv2.CAP_PROP_FPS)) frames_count = int(vidcap.get(cv2.CAP_PROP_FRAME_COUNT)) # create a video writer for output res_filename = "./output/" + out_filename vidwriter = cv2.VideoWriter(res_filename, cv2.VideoWriter_fourcc('M', 'J', 'P', 'G'), fps, (width, height)) cameraImg = vidcap.read()[1] textureImg = cv2.imread(face_filename) textureCoords = utils.getFaceTextureCoords(textureImg, mean3DShape, blendshapes, idxs2D, idxs3D, detector, predictor) renderer = FaceRendering.FaceRenderer(cameraImg, textureImg, textureCoords, mesh) destShapes2D = utils.getFaceKeypoints(cameraImg, detector, predictor, maxImageSizeForDetection) destShape = destShapes2D[0] modelParams = projectionModel.getInitialParameters( mean3DShape[:, idxs3D], destShape[:, idxs2D]) # 3D model parameter optimization modelParams = NonLinearLeastSquares.GaussNewton( modelParams, projectionModel.residual, projectionModel.jacobian, ([mean3DShape[:, idxs3D], blendshapes[:, :, idxs3D] ], destShape[:, idxs2D]), verbose=0) # rendering the model to an image destShape3D = utils.getShape3D(mean3DShape, blendshapes, modelParams) destRenderedImg = renderer.render(destShape3D) self.progressBar.setRange(0, frames_count - 1) # iterate over the frames and apply the face swap for i in tqdm(range(frames_count - 1)): success, cameraImg = vidcap.read() self.progressBar.setValue(i + 1) if success != True: # no frames left => break break shapes2D = utils.getFaceKeypoints(cameraImg, detector, predictor, maxImageSizeForDetection) newImg = cameraImg try: if shapes2D is not None: for shape2D in shapes2D: # 3D model parameter initialization modelParams = projectionModel.getInitialParameters( mean3DShape[:, idxs3D], shape2D[:, idxs2D]) # 3D model parameter optimization modelParams = NonLinearLeastSquares.GaussNewton( modelParams, projectionModel.residual, projectionModel.jacobian, ([ mean3DShape[:, idxs3D], blendshapes[:, :, idxs3D] ], shape2D[:, idxs2D]), verbose=0) # rendering the model to an image shape3D = utils.getShape3D(mean3DShape, blendshapes, modelParams) renderedImg = renderer.render(shape3D) # blending of the rendered face with the image mask = np.copy(renderedImg[:, :, 0]) mask1 = np.copy(destRenderedImg[:, :, 0]) renderedImg = ImageProcessing.colorTransfer( cameraImg, renderedImg, mask) # newImg = ImageProcessing.blendImages(renderedImg, cameraImg, mask) newImg = ImageProcessing.blendImages0( renderedImg, cameraImg, mask, mask1) except: pass vidwriter.write(newImg) # releas video capture and writer vidcap.release() vidwriter.release() renderer.release() # apply audio clip to generated video if keep_audio == True: video = VideoFileClip("./output/proc_video.avi") video.write_videofile(out_filename, audio="./output/src_audio.mp3", progress_bar=False, verbose=False)
def guide_facechange(): def tmp(): print("") t = threading.Timer(GUIDE_SHOW_TIME, tmp) t.start() t_wait = threading.Timer(GUIDE_WAIT_TIME, tmp) t_wait.start() print("Press T to draw the keypoints and the 3D model") print("Press W to start recording to a video file") print("Press R to restart") print("Press Q or ESC to Quit") modelParams = None lockedTranslation = False drawOverlay = False global BACKGROUND_VID_PATH_NUM #cap = cv2.VideoCapture(VIDEO_CAPTURE_CAM_NUM) cap = cv2.VideoCapture(BACKGROUND_VID_PATH[BACKGROUND_VID_PATH_NUM]) writer = None cameraImg = cap.read()[1] # face swap하여 붙일 영상의 img textureImg = cv2.VideoCapture(VIDEO_CAPTURE_CAM_NUM).read()[1] #print("광고영상 shape : \t\t",cameraImg.shape[1],cameraImg.shape[0]) #print("카메라 캡쳐영상 shape : ",textureImg.shape[1],textureImg.shape[0]) ###### face detection with guide cap_guide_cam = cv2.VideoCapture(VIDEO_CAPTURE_CAM_NUM) if (cap_guide_cam.isOpened() == False): print("Unable to read camera feed") frame_width = int(cap_guide_cam.get(3)) frame_height = int(cap_guide_cam.get(4)) str="match your face" str2="O" str3="ATTENTION" while(True): ret, frame = cap_guide_cam.read() frame_org = frame cv2.putText(frame,str,(int(frame_width/3),int(frame_height/6)),cv2.FONT_HERSHEY_SIMPLEX,int(frame_width/600),(0,0,0),int(frame_width/300)) cv2.putText(frame,str2,(int(frame_width/3),int(frame_width/2)),cv2.FONT_HERSHEY_SIMPLEX,int(frame_width/60),(0,0,255),int(frame_width/300)) cv2.putText(frame,str3,(int((frame_width*2)/3),int((frame_height*2)/3)),cv2.FONT_HERSHEY_SIMPLEX,int(frame_width/650),(0,0,0),int(frame_width/300)) cv2.namedWindow("frame", cv2.WND_PROP_FULLSCREEN) cv2.setWindowProperty("frame", cv2.WND_PROP_FULLSCREEN, 1) cv2.imshow('frame',frame) if cv2.waitKey(1) & 0xFF == ord('q'): break if not t_wait.isAlive(): dets = detector(frame_org, 1) #처음 camera로 촬영한 캡쳐를 넣어서 얼굴을 찾음. if len(dets) > 0: print("detected") break else: print("now detecting") if not t.isAlive(): video() # 찍은 영상의 캡쳐를 3D로 재구성하여 합침 textureCoords = utils.getFaceTextureCoords(textureImg, mean3DShape, blendshapes, idxs2D, idxs3D, detector, predictor) # 찍은 얼굴의 데이터를 영상의 얼굴에 rendering renderer = FaceRendering.FaceRenderer(cameraImg, textureImg, textureCoords, mesh) doProcess=False meanTime=[[0]*4 for i in range(4)] while True: #영상 캡쳐 cameraImg = cap.read()[1] shapes2D = utils.getFaceKeypoints(cameraImg, detector, predictor, maxImageSizeForDetection) doProcess = not doProcess if doProcess is not True: continue else: if shapes2D is not None: for shape2D in shapes2D: start = timeit.default_timer() #3D model parameter initialization modelParams = projectionModel.getInitialParameters(mean3DShape[:, idxs3D], shape2D[:, idxs2D]) stop = timeit.default_timer() meanTime[0][0]+=stop-start meanTime[0][1]+=1 #print(1, float(meanTime[0][0]/meanTime[0][1])) start = timeit.default_timer() #3D model parameter optimization modelParams = NonLinearLeastSquares.GaussNewton(modelParams, projectionModel.residual, projectionModel.jacobian, ([mean3DShape[:, idxs3D], blendshapes[:, :, idxs3D]], shape2D[:, idxs2D]), verbose=0) stop = timeit.default_timer() meanTime[1][0]+=stop-start meanTime[1][1]+=1 #print(2, float(meanTime[1][0]/meanTime[1][1])) start = timeit.default_timer() #rendering the model to an image #다듬기 shape3D = utils.getShape3D(mean3DShape, blendshapes, modelParams) renderedImg = renderer.render(shape3D) stop = timeit.default_timer() meanTime[2][0]+=stop-start meanTime[2][1]+=1 #print(3, float(meanTime[2][0]/meanTime[2][1]))\ start = timeit.default_timer() #blending of the rendered face with the image mask = np.copy(renderedImg[:, :, 0]) renderedImg = ImageProcessing.colorTransfer(cameraImg, renderedImg, mask) cameraImg = ImageProcessing.blendImages(renderedImg, cameraImg, mask) stop = timeit.default_timer() meanTime[3][0] += stop - start meanTime[3][1] += 1 #print(4, float(meanTime[3][0] / meanTime[3][1])) #drawing of the mesh and keypoints # 't'를 누를 때, 적용. facepoint가 표시됨. if drawOverlay: drawPoints(cameraImg, shape2D.T) drawProjectedShape(cameraImg, [mean3DShape, blendshapes], projectionModel, mesh, modelParams, lockedTranslation) if writer is not None: writer.write(cameraImg) cv2.namedWindow("image", cv2.WND_PROP_FULLSCREEN) cv2.setWindowProperty("image", cv2.WND_PROP_FULLSCREEN, 1) cv2.imshow('image',cameraImg) key = cv2.waitKey(1) if key == 27 or key == ord('q'): break if key == ord('t'): drawOverlay = not drawOverlay if key == ord('r'): cv2.destroyAllWindows() video() if key == ord('1'): cv2.destroyAllWindows() BACKGROUND_VID_PATH_NUM = 0 guide_facechange() break if key == ord('2'): cv2.destroyAllWindows() BACKGROUND_VID_PATH_NUM = 1 guide_facechange() break if key == ord('3'): cv2.destroyAllWindows() BACKGROUND_VID_PATH_NUM = 2 guide_facechange() break if key == ord('4'): cv2.destroyAllWindows() BACKGROUND_VID_PATH_NUM = 3 guide_facechange() break if key == ord('5'): cv2.destroyAllWindows() BACKGROUND_VID_PATH_NUM = 4 guide_facechange() break if key == ord('w'): if writer is None: print("Starting video writer") writer = cv2.VideoWriter(SAVE_VID_PATH, cv2.VideoWriter_fourcc('X', 'V', 'I', 'D'), 13, (cameraImg.shape[1], cameraImg.shape[0])) if writer.isOpened(): print("Writer succesfully opened") else: writer = None print("Writer opening failed") else: print("Stopping video writer") writer.release() writer = None cap.release() cap_intro_vid.release() cap_guide_cam.release() cv2.destroyAllWindows()
remapped_mouth = ImageProcessing.colorTransfer(cameraImg, remapped_mouth, mask_mouth) cameraImg = ImageProcessing.blendImages(remapped_mouth, cameraImg, mask_mouth) except: pass else: try: cameraImg = ImageProcessing.blendImages(remapped_mouth, cameraImg, mask_mouth) except: pass # Face rendering if count % 10 == 0: # Init params of 3D model, then perform optimisation modelParams = projectionModel.getInitialParameters(mean3DShape[:, idxs3D], shape2D[:, idxs2D]) modelParams = NonLinearLeastSquares.GaussNewton(modelParams, projectionModel.residual, projectionModel.jacobian, ([mean3DShape[:, idxs3D], blendshapes[:, :, idxs3D]], shape2D[:, idxs2D]), verbose=0) # Build 3D model for rendering shape3D = utils.getShape3D(mean3DShape, blendshapes, modelParams) # Get rendered image renderedImg = renderer.render(shape3D) # Blend of rendered face with original frame mask_renderedImg = np.copy(renderedImg[:, :, 0]) renderedImg = ImageProcessing.colorTransfer(cameraImg, renderedImg, mask_renderedImg) cameraImg = ImageProcessing.blendImages(renderedImg, cameraImg, mask_renderedImg) else: try: cameraImg = ImageProcessing.blendImages(renderedImg, cameraImg, mask_renderedImg) except:
def mix_video(): data = {} if os.path.getsize('global.pickle') > 0: with open('global.pickle', 'rb') as pf: data = pickle.load(pf) else: win32api.MessageBox(0, '값이 제대로 전달되지 않았습니다. 처음부터 다시 시작해주세요', 'warning', 0x00001000) sys.exit(0) backgroundvideo = "testvideo1" genderNum = data['gender'] backgroundvideo = data['backgroundvideo'] userfolder = data['userfolder'] personnumber = data['personnumber'] backgroundvideo_name = data['backgroundvideo_name'] print('gender:', genderNum, 'vid_dir:', backgroundvideo, 'user_dir:', userfolder, 'peonson#:', personnumber, 'vid_name:', backgroundvideo_name) user_shape_data = {} with open("input/" + userfolder + "/" + 'userdata.pickle', 'rb') as pf: user_shape_data = pickle.load(pf) print( "Press T to draw the keypoints and the 3D model") # 결과 영상에 얼굴합성 과정 표시 print("Press R to start recording to a video file") # 녹화 # shape_predictor_64_face_landmarks.dat 여기서 다운받아서 압축해제 # http://sourceforge.net/projects/dclib/files/dlib/v18.10/shape_predictor_68_face_landmarks.dat.bz2rr # 키포인트 인식 모델 predictor_path = "data/shape_predictor_68_face_landmarks.dat" # 이미지 사이즈가 작을수록 처리속도가 빨라짐 # 너무 작으면 얼굴 인식이 안됨 maxImageSizeForDetection = 320 # 얼굴 인식기 로드 detector = dlib.get_frontal_face_detector() # detector = dlib.cnn_face_detection_model_v1("data/mmod_human_face_detector.dat") predictor = dlib.shape_predictor(predictor_path) mean3DShape, blendshapes, mesh, idxs3D, idxs2D = utils.load3DFaceModel( "data/candide.npz") projectionModel = models.OrthographicProjectionBlendshapes( blendshapes.shape[0]) modelParams = None lockedTranslation = False drawOverlay = False writer = None # 콤비네이션: VC:Video,Cam | CI: Cam,Image | CC: Cam, Cam | VI: Video,Image cap_background = cv2.VideoCapture(backgroundvideo) # Video for background # cap_background = cv2.VideoCapture("input/" + backgroundvideo + ".mp4") # Video for background # cap_background = cv2.VideoCapture(0) # WebCAM for background cameraImg = cap_background.read()[1] # 본인의 여러 얼굴을 메모리에 저장함 (0~9) textureImgs = [] for i in range(0, 9): textureImgs.append( cv2.imread("input/" + userfolder + "/" + str(i) + ".jpg")) # Image for face # textureImgs.append(cv2.imread("input/user_test1_images/" + str(i) + ".jpg")) # Image for face if not os.path.isdir('output'): os.mkdir('output') output_video_name = 'output/' + backgroundvideo + '_' + datetime.datetime.now( ).strftime('%Y%m%d_%H%M%S') + '.avi' writer = cv2.VideoWriter(output_video_name, cv2.VideoWriter_fourcc(*'XVID'), 25, (cameraImg.shape[1], cameraImg.shape[0]), True) modelParams = np.zeros(20) startTime = time.time() full_shapes2D_csv = selectGender.reading_csv(backgroundvideo_name + "_annotation") # full_shapes2D_csv = selectGender.reading_csv("testvideo1_annotation") for framecnt, shapes2D_csv in enumerate(full_shapes2D_csv): print("frame number:", framecnt) # 배경으로 사용할 영상의 프레임 이미지 읽기 cap_background.set(cv2.CAP_PROP_POS_FRAMES, framecnt) ret, cameraImg = cap_background.read() try: background_user_shapes2D = utils.getFaceKeypoints( cameraImg, detector, predictor, maxImageSizeForDetection) if shapes2D_csv is not None: # 영상이 끝나면 반복문 탈출 if ret == False: break # textureImg, user_face_angle, user_shapes2D, textureCoords = user_shape_data[utils.getFaceAngle(background_user_shapes2D)] # 저장된 유저 얼굴 정보로 부터 해당 각도의 데이터를 가저옴 background_face_angle = utils.getFaceAngle( background_user_shapes2D) print("user_face_angle: {}".format(background_face_angle)) textureImg = textureImgs[background_face_angle] user_shapes2D = utils.getFaceKeypoints( textureImg, detector, predictor, maxImageSizeForDetection) textureCoords = utils.getFaceTextureCoords_v2( textureImg, mean3DShape, blendshapes, idxs2D, idxs3D, user_shapes2D, predictor) renderer = FaceRendering.FaceRenderer(cameraImg, textureImg, textureCoords, mesh) for shape2D in background_user_shapes2D: #[shapes2D_csv]: # 3D 모델 파라미터 초기화 (영상에서 인식된 얼굴로부터 3D 모델 생성을 위해) modelParams = projectionModel.getInitialParameters( mean3DShape[:, idxs3D], shape2D[:, idxs2D]) # 3D 모델 파라미터 최적화 기능 (배경의 얼굴과 입력된 얼굴의 키포인트 간의 거리를 최소화 하도록 머신러닝으로 최적화) modelParams = NonLinearLeastSquares.GaussNewton( modelParams, projectionModel.residual, projectionModel.jacobian, ([mean3DShape[:, idxs3D], blendshapes[:, :, idxs3D] ], shape2D[:, idxs2D]), verbose=0) # 위의 모델을 이용해 입력된 이미지의 얼굴을 3D객체로 바꿔 배경의 얼굴 위치로 보정 shape3D = utils.getShape3D(mean3DShape, blendshapes, modelParams) renderedImg = renderer.render(shape3D) # 배경 영상과 입력된 얼굴 이미지를 합성 (합성과정에서 색변환, 이미지 블랜딩 기법 사용) mask = np.copy(renderedImg[:, :, 0]) renderedImg = ImageProcessing.colorTransfer( cameraImg, renderedImg, mask) cameraImg = ImageProcessing.blendImages( renderedImg, cameraImg, mask) # 3D매쉬와 키포인트를 화면 위에 그림 if drawOverlay: drawPoints(cameraImg, shape2D.T) # 초록색 drawProjectedShape(cameraImg, [mean3DShape, blendshapes], projectionModel, mesh, modelParams, lockedTranslation) writer.write(cameraImg) # 얼굴 합성된 영상 출력 cv2.imshow('image', cameraImg) except: pass # 걸린 시간 (초) 출력 writer.release() endTime = time.time() - startTime print("endTime: ", endTime)
def guide_facechange(): def tmp(): print("") t = threading.Timer(GUIDE_SHOW_TIME, tmp) t.start() print("Press T to draw the keypoints and the 3D model") print("Press R to start recording to a video file") #loading the keypoint detection model, the image and the 3D model predictor_path = "../shape_predictor_68_face_landmarks.dat" image_name = "../data/jolie.jpg" #the smaller this value gets the faster the detection will work #if it is too small, the user's face might not be detected maxImageSizeForDetection = 320 #카메라로 찍히는 영상에서 얼굴을 찾는 detector detector = dlib.get_frontal_face_detector() #사람 얼굴을 찾는 입과 눈의 구석, 코의 끝과 같은 중요한 얼굴 표식의 위치를 식별하는 점들의 집합 predictor = dlib.shape_predictor(predictor_path) # candide = 3D face model source # mean3Dshape : 얼굴의 중립상태에 해당하는 정점 리스트 # blendshapes : 중립상태인 얼굴에서 추가하여 수정할 수 있는 얼굴 # ex) 미소, 눈썹 올라가는 부분 # candide에 정의된 애니메이션 Units에서 파생된다. # mesh : Candide가 얼굴 목록으로 제공한 원래의 mesh # idxs3D, idxs2D: Candide 모델(idxs3D)과 얼굴 정렬점 세트(idxs2D)사이에 해당하는 지점들의 인덱스들이다. mean3DShape, blendshapes, mesh, idxs3D, idxs2D = utils.load3DFaceModel( "../candide.npz") # projectionModel = models.OrthographicProjectionBlendshapes( blendshapes.shape[0]) modelParams = None lockedTranslation = False drawOverlay = False #cap = cv2.VideoCapture(0) cap = cv2.VideoCapture("../ROEM 2014 Spring SUZY 320p.mp4") writer = None cameraImg = cap.read()[1] # face swap하여 붙일 영상의 img textureImg = cv2.VideoCapture(0).read()[1] #cv2.imread(image_name) print("광고영상 shape : \t\t", cameraImg.shape[1], cameraImg.shape[0]) print("카메라 캡쳐영상 shape : ", textureImg.shape[1], textureImg.shape[0]) ###### face detection with guide cap_guide_cam = cv2.VideoCapture(0) if (cap_guide_cam.isOpened() == False): print("Unable to read camera feed") frame_width = int(cap_guide_cam.get(3)) frame_height = int(cap_guide_cam.get(4)) str = "match your face" str2 = "O" str3 = "ATTENTION" while (True): ret, frame = cap_guide_cam.read() frame_org = frame cv2.putText(frame, str, (int(frame_width / 3), int(frame_height / 6)), cv2.FONT_HERSHEY_SIMPLEX, int(frame_width / 600), (0, 0, 0), int(frame_width / 300)) cv2.putText(frame, str2, (int(frame_width / 3), int(frame_width / 2)), cv2.FONT_HERSHEY_SIMPLEX, int(frame_width / 60), (0, 0, 255), int(frame_width / 300)) cv2.putText(frame, str3, (int( (frame_width * 2) / 3), int( (frame_height * 2) / 3)), cv2.FONT_HERSHEY_SIMPLEX, int(frame_width / 650), (0, 0, 0), int(frame_width / 300)) cv2.imshow('frame', frame) if cv2.waitKey(1) & 0xFF == ord('q'): break dets = detector(frame_org, 1) #처음 camera로 촬영한 캡쳐를 넣어서 얼굴을 찾음. if len(dets) > 0: print("detected") break else: print("now detecting") if not t.isAlive(): video() break # 찍은 영상의 캡쳐를 3D로 재구성하여 합침 textureCoords = utils.getFaceTextureCoords(textureImg, mean3DShape, blendshapes, idxs2D, idxs3D, detector, predictor) # 찍은 얼굴의 데이터를 영상의 얼굴에 rendering renderer = FaceRendering.FaceRenderer(cameraImg, textureImg, textureCoords, mesh) doProcess = False meanTime = [[0] * 4 for i in range(4)] while True: #영상 캡쳐 cameraImg = cap.read()[1] shapes2D = utils.getFaceKeypoints(cameraImg, detector, predictor, maxImageSizeForDetection) doProcess = not doProcess if doProcess is not True: continue else: if shapes2D is not None: for shape2D in shapes2D: start = timeit.default_timer() #3D model parameter initialization modelParams = projectionModel.getInitialParameters( mean3DShape[:, idxs3D], shape2D[:, idxs2D]) stop = timeit.default_timer() meanTime[0][0] += stop - start meanTime[0][1] += 1 #print(1, float(meanTime[0][0]/meanTime[0][1])) start = timeit.default_timer() #3D model parameter optimization modelParams = NonLinearLeastSquares.GaussNewton( modelParams, projectionModel.residual, projectionModel.jacobian, ([mean3DShape[:, idxs3D], blendshapes[:, :, idxs3D] ], shape2D[:, idxs2D]), verbose=0) stop = timeit.default_timer() meanTime[1][0] += stop - start meanTime[1][1] += 1 #print(2, float(meanTime[1][0]/meanTime[1][1])) start = timeit.default_timer() #rendering the model to an image #다듬기 shape3D = utils.getShape3D(mean3DShape, blendshapes, modelParams) renderedImg = renderer.render(shape3D) stop = timeit.default_timer() meanTime[2][0] += stop - start meanTime[2][1] += 1 #print(3, float(meanTime[2][0]/meanTime[2][1])) start = timeit.default_timer() #blending of the rendered face with the image mask = np.copy(renderedImg[:, :, 0]) renderedImg = ImageProcessing.colorTransfer( cameraImg, renderedImg, mask) cameraImg = ImageProcessing.blendImages( renderedImg, cameraImg, mask) stop = timeit.default_timer() meanTime[3][0] += stop - start meanTime[3][1] += 1 #print(4, float(meanTime[3][0] / meanTime[3][1])) #drawing of the mesh and keypoints # 't'를 누를 때, 적용. facepoint가 표시됨. if drawOverlay: drawPoints(cameraImg, shape2D.T) drawProjectedShape(cameraImg, [mean3DShape, blendshapes], projectionModel, mesh, modelParams, lockedTranslation) if writer is not None: writer.write(cameraImg) cv2.namedWindow("image", cv2.WND_PROP_FULLSCREEN) cv2.setWindowProperty("image", cv2.WND_PROP_FULLSCREEN, 1) cv2.imshow('image', cameraImg) key = cv2.waitKey(1) if key == 27 or key == ord('q'): break if key == ord('t'): drawOverlay = not drawOverlay if key == ord('r'): cv2.destroyAllWindows() video() if key == ord('w'): if writer is None: print("Starting video writer") writer = cv2.VideoWriter( "../out.avi", cv2.VideoWriter_fourcc('X', 'V', 'I', 'D'), 13, (cameraImg.shape[1], cameraImg.shape[0])) if writer.isOpened(): print("Writer succesfully opened") else: writer = None print("Writer opening failed") else: print("Stopping video writer") writer.release() writer = None cap.release() cap_intro_vid.release() cap_guide_cam.release() cv2.destroyAllWindows()
def main(): os.makedirs('original', exist_ok=True) os.makedirs('landmarks', exist_ok=True) mean3DShape, blendshapes, mesh, idxs3D, idxs2D = utils.load3DFaceModel( "candide.npz") projectionModel = models.OrthographicProjectionBlendshapes( blendshapes.shape[0]) files = [] for fl_type in ['png', 'jpg']: [ files.append(fl) for fl in glob.glob(args.Directory + '/*.' + fl_type) ] print(len(files)) count = 0 imgParams = {} for fl in files: frame = cv2.imread(fl) print(fl) frame_resize = cv2.resize(frame, None, fx=1 / DOWNSAMPLE_RATIO, fy=1 / DOWNSAMPLE_RATIO) gray = cv2.cvtColor(frame_resize, cv2.COLOR_BGR2GRAY) faces = detector(gray, 1) black_image = np.zeros(frame.shape, np.uint8) t = time.time() # Perform if there is a face detecte shapes2D = getFaceKeypoints(frame, detector, predictor) if shapes2D is None: continue if len(shapes2D) == 1: # 3D model parameter initialization modelParams = projectionModel.getInitialParameters( mean3DShape[:, idxs3D], shapes2D[0][:, idxs2D]) # 3D model parameter optimization modelParams = NonLinearLeastSquares.GaussNewton( modelParams, projectionModel.residual, projectionModel.jacobian, ([mean3DShape[:, idxs3D], blendshapes[:, :, idxs3D] ], shapes2D[0][:, idxs2D]), verbose=0) drawProjectedShape(black_image, [mean3DShape, blendshapes], projectionModel, mesh, modelParams, lockedTranslation) # Display the resulting frame imgParams[fl] = modelParams count += 1 print(count) cv2.imwrite("original/{}.png".format(count), frame) cv2.imwrite("landmarks/{}.png".format(count), black_image) print('[INFO] elapsed time: {:.2f}'.format(time.time() - t)) else: print("No face detected")
def main(): os.makedirs('original', exist_ok=True) os.makedirs('landmarks', exist_ok=True) mean3DShape, blendshapes, mesh, idxs3D, idxs2D = utils.load3DFaceModel("candide.npz") projectionModel = models.OrthographicProjectionBlendshapes(blendshapes.shape[0]) cap = cv2.VideoCapture(args.filename) fps = video.FPS().start() count = 0 while cap.isOpened(): ret, frame = cap.read() if frame is None: break frame_resize = cv2.resize(frame, None, fx=1 / DOWNSAMPLE_RATIO, fy=1 / DOWNSAMPLE_RATIO) gray = cv2.cvtColor(frame_resize, cv2.COLOR_BGR2GRAY) faces = detector(gray, 1) black_image = np.zeros(frame.shape, np.uint8) t = time.time() # Perform if there is a face detecte shapes2D = getFaceKeypoints(frame, detector, predictor) if shapes2D is None: continue if len(shapes2D) == 1: # 3D model parameter initialization modelParams = projectionModel.getInitialParameters(mean3DShape[:, idxs3D], shapes2D[0][:, idxs2D]) # 3D model parameter optimization modelParams = NonLinearLeastSquares.GaussNewton(modelParams, projectionModel.residual, projectionModel.jacobian, ( [mean3DShape[:, idxs3D], blendshapes[:, :, idxs3D]], shapes2D[0][:, idxs2D]), verbose=0) drawProjectedShape(black_image, [mean3DShape, blendshapes], projectionModel, mesh, modelParams, lockedTranslation) # Display the resulting frame count += 1 print(count) cv2.imwrite("original/{}.png".format(count), frame) cv2.imwrite("landmarks/{}.png".format(count), black_image) fps.update() print('[INFO] elapsed time: {:.2f}'.format(time.time() - t)) if count == args.number: # only take 400 photos break elif cv2.waitKey(1) & 0xFF == ord('q'): break else: print("No face detected") fps.stop() print('[INFO] elapsed time (total): {:.2f}'.format(fps.elapsed())) print('[INFO] approx. FPS: {:.2f}'.format(fps.fps())) cap.release() cv2.destroyAllWindows()
def face_Fusion(self): print("----- Face fusion function is start -----") print(self.status) # cameraImg = self.cap.read()[1] # import image path self.textureImg = cv2.imread(self.image_name) maxImageSizeForDetection = 320 detector = dlib.get_frontal_face_detector() mean3DShape, blendshapes, mesh, idxs3D, idxs2D = utils_face_fusion.load3DFaceModel( "./fw/candide.npz") projectionModel = models_face_fusion.OrthographicProjectionBlendshapes( blendshapes.shape[0]) modelParams = None lockedTranslation = False drawOverlay = False # cap = cv2.VideoCapture(cv2.CAP_DSHOW) writer = None cameraImg = self.cap.read()[1] self.textureImg = cv2.imread(self.image_name) # textureCoords = utils_face_fusion.getFaceTextureCoords(self.textureImg, mean3DShape, blendshapes, idxs2D, idxs3D, detector, predictor) # renderer = FaceRendering.FaceRenderer(cameraImg, self.textureImg, textureCoords, mesh) while True: print("----- Face fusion function is running -----") cameraImg = self.cap.read()[1] if (self.status != "2"): print("end2") break textureCoords = utils_face_fusion.getFaceTextureCoords( self.textureImg, mean3DShape, blendshapes, idxs2D, idxs3D, detector, predictor) renderer = FaceRendering.FaceRenderer(cameraImg, self.textureImg, textureCoords, mesh) shapes2D = utils_face_fusion.getFaceKeypoints( cameraImg, detector, predictor, maxImageSizeForDetection) if shapes2D is not None: for shape2D in shapes2D: # 3D model parameter initialization modelParams = projectionModel.getInitialParameters( mean3DShape[:, idxs3D], shape2D[:, idxs2D]) # cameraImg = self.cap.read()[1] # 3D model parameter optimization modelParams = NonLinearLeastSquares.GaussNewton( modelParams, projectionModel.residual, projectionModel.jacobian, ([mean3DShape[:, idxs3D], blendshapes[:, :, idxs3D] ], shape2D[:, idxs2D]), verbose=0) # rendering the model to an image shape3D = utils_face_fusion.getShape3D( mean3DShape, blendshapes, modelParams) renderedImg = renderer.render(shape3D) # blending of the rendered face with the image mask = np.copy(renderedImg[:, :, 0]) renderedImg = ImageProcessing.colorTransfer( cameraImg, renderedImg, mask) cameraImg = ImageProcessing.blendImages( renderedImg, cameraImg, mask) # drawing of the mesh and keypoints if drawOverlay: drawPoints(cameraImg, shape2D.T) drawProjectedShape(cameraImg, [mean3DShape, blendshapes], projectionModel, mesh, modelParams, lockedTranslation) imgg = self.textureImg imgg = cv2.resize(imgg, (100, 100)) cameraImg = Add_image(cameraImg, imgg) if writer is not None: writer.write(cameraImg) self.image = cameraImg show = cv2.resize(self.image, (1080, 960)) # 把读到的帧的大小重新设置为 640x480 # 视频色彩转换回RGB,这样才是现实的颜色 show = cv2.cvtColor(show, cv2.COLOR_BGR2RGB) showImage = QtGui.QImage( show.data, show.shape[1], show.shape[0], QtGui.QImage.Format_RGB888) # 把读取到的视频数据变成QImage形式 self.label_show_camera.setPixmap( QtGui.QPixmap.fromImage(showImage)) if (self.status != "2"): self.label_show_camera.clear() print("end1") return None
def main(): # fourcc = cv2.VideoWriter_fourcc(*'XVID') fourcc = cv2.VideoWriter_fourcc(*'MP4V') writer = None if writer is None: print("Starting video writer") writer = cv2.VideoWriter("./out.mp4", fourcc, 30.0, (CROP_SIZE*2, CROP_SIZE)) if writer.isOpened(): print("Writer succesfully opened") else: writer = None print("Writer opening failed") else: print("Stopping video writer") writer.release() writer = None # TensorFlow graph = load_graph(args.frozen_model_file) image_tensor = graph.get_tensor_by_name('image_tensor:0') output_tensor = graph.get_tensor_by_name('generate_output/output:0') sess = tf.Session(graph=graph) # OpenCV # cap = cv2.VideoCapture(args.video_source) cap = cv2.VideoCapture(args.video_dir) fps = video.FPS().start() mean3DShape, blendshapes, mesh, idxs3D, idxs2D = utils.load3DFaceModel("candide.npz") projectionModel = models.OrthographicProjectionBlendshapes(blendshapes.shape[0]) while True: ret, frame = cap.read() if frame is None: break # resize image and detect face frame_resize = cv2.resize(frame, None, fx=1 / DOWNSAMPLE_RATIO, fy=1 / DOWNSAMPLE_RATIO) gray = cv2.cvtColor(frame_resize, cv2.COLOR_BGR2GRAY) #get frame face label # faces = detector(gray, 1) black_image = np.zeros(frame.shape, np.uint8) # for face in faces: # detected_landmarks = predictor(gray, face).parts() # landmarks = [[p.x * DOWNSAMPLE_RATIO, p.y * DOWNSAMPLE_RATIO] for p in detected_landmarks] # # jaw = reshape_for_polyline(landmarks[0:17]) # left_eyebrow = reshape_for_polyline(landmarks[22:27]) # right_eyebrow = reshape_for_polyline(landmarks[17:22]) # nose_bridge = reshape_for_polyline(landmarks[27:31]) # lower_nose = reshape_for_polyline(landmarks[30:35]) # left_eye = reshape_for_polyline(landmarks[42:48]) # right_eye = reshape_for_polyline(landmarks[36:42]) # outer_lip = reshape_for_polyline(landmarks[48:60]) # inner_lip = reshape_for_polyline(landmarks[60:68]) # # color = (255, 255, 255) # thickness = 3 # # cv2.polylines(black_image, [jaw], False, color, thickness) # cv2.polylines(black_image, [left_eyebrow], False, color, thickness) # cv2.polylines(black_image, [right_eyebrow], False, color, thickness) # cv2.polylines(black_image, [nose_bridge], False, color, thickness) # cv2.polylines(black_image, [lower_nose], True, color, thickness) # cv2.polylines(black_image, [left_eye], True, color, thickness) # cv2.polylines(black_image, [right_eye], True, color, thickness) # cv2.polylines(black_image, [outer_lip], True, color, thickness) # cv2.polylines(black_image, [inner_lip], True, color, thickness) shapes2D = getFaceKeypoints(frame, detector, predictor) if shapes2D is None: continue # 3D model parameter initialization modelParams = projectionModel.getInitialParameters(mean3DShape[:, idxs3D], shapes2D[0][:, idxs2D]) # 3D model parameter optimization modelParams = NonLinearLeastSquares.GaussNewton(modelParams, projectionModel.residual, projectionModel.jacobian, ( [mean3DShape[:, idxs3D], blendshapes[:, :, idxs3D]], shapes2D[0][:, idxs2D]), verbose=0) drawProjectedShape(black_image, [mean3DShape, blendshapes], projectionModel, mesh, modelParams, lockedTranslation) # generate prediction combined_image = np.concatenate([resize(black_image), resize(frame_resize)], axis=1) image_rgb = cv2.cvtColor(combined_image, cv2.COLOR_BGR2RGB) # OpenCV uses BGR instead of RGB generated_image = sess.run(output_tensor, feed_dict={image_tensor: image_rgb}) image_bgr = cv2.cvtColor(np.squeeze(generated_image), cv2.COLOR_RGB2BGR) image_normal = np.concatenate([resize(frame_resize), image_bgr], axis=1) image_landmark = np.concatenate([resize(black_image), image_bgr], axis=1) if args.display_landmark == 0: cv2.imshow('frame', image_normal) else: cv2.imshow('frame', image_landmark) if writer is not None: writer.write(image_normal) fps.update() if cv2.waitKey(1) & 0xFF == ord('q'): break writer.release() fps.stop() print('[INFO] elapsed time (total): {:.2f}'.format(fps.elapsed())) print('[INFO] approx. FPS: {:.2f}'.format(fps.fps())) sess.close() cap.release() cv2.destroyAllWindows()