def faceswap(): cameraImg = cap.read()[1] rows, cols = cameraImg.shape[:2] translation_matrix = np.float32([[1, 0, int(0.5 * cols)], [0, 1, int(0.5 * rows)]]) img_t = cv2.warpAffine(cameraImg, translation_matrix, (2 * cols, 2 * rows)) rotation_matrix = cv2.getRotationMatrix2D((cols, rows), -90, 1) img_r = cv2.warpAffine(img_t, rotation_matrix, (2 * cols, 2 * rows)) translation_matrix = np.float32([[1, 0, -int(cols - (0.5 * rows))], [0, 1, -int(rows - (0.5 * cols))]]) cameraImg = cv2.warpAffine(img_r, translation_matrix, (rows, cols)) shapes2D = utils.getFaceKeypoints(cameraImg, detector, predictor, maxImageSizeForDetection) if shapes2D is not None: for shape2D in shapes2D: #3D model parameter initialization modelParams = projectionModel.getInitialParameters( mean3DShape[:, idxs3D], shape2D[:, idxs2D]) #3D model parameter optimization modelParams = NonLinearLeastSquares.GaussNewton( modelParams, projectionModel.residual, projectionModel.jacobian, ([mean3DShape[:, idxs3D], blendshapes[:, :, idxs3D] ], shape2D[:, idxs2D]), verbose=0) #rendering the model to an image shape3D = utils.getShape3D(mean3DShape, blendshapes, modelParams) renderedImg = renderer.render(shape3D) #blending of the rendered face with the image mask = np.copy(renderedImg[:, :, 0]) renderedImg = ImageProcessing.colorTransfer( cameraImg, renderedImg, mask) cameraImg = ImageProcessing.blendImages(renderedImg, cameraImg, mask) #drawing of the mesh and keypoints if drawOverlay: drawPoints(cameraImg, shape2D.T) drawProjectedShape(cameraImg, [mean3DShape, blendshapes], projectionModel, mesh, modelParams, lockedTranslation) if writer is not None: writer.write(cameraImg) cv2.imshow('image', cameraImg) return cameraImg key = cv2.waitKey(1) if key == 27: return []
def process(self, input_stream, output_stream): frame_rate = 10 prev = 0 self.input_stream = input_stream self.output_stream = output_stream # cameraImg = input_stream.read()[1] while input_stream.isOpened(): time_elapsed = time.time() - prev has_frame, frame = input_stream.read() # print ("##"*100) # to be removed self.shapes2D = utils.getFaceKeypoints( frame, self.detector, self.predictor, self.maxImageSizeForDetection) # if time_elapsed > 1./frame_rate: # prev = time.time() # else: # continue if not has_frame: break # frame = self.fliped(frame) # manually added by SG to make mirror like effect # if self.input_crop is not None: # frame = Visualizer.center_crop(frame, self.input_crop) detections = self.frame_processor.process(frame) # print("detections------------", detections) self.draw_detections(frame, detections) # self.draw_status(frame, detections) if output_stream: output_stream.write(frame) if self.display: self.display_interactive_window(frame) if self.should_stop_display(): break self.update_fps() self.frame_num += 1
def process_video(self, in_filename, out_filename, face_filename, keep_audio=True): # extract audio clip from src if keep_audio == True: clip = VideoFileClip(in_filename) clip.audio.write_audiofile("./temp/src_audio.mp3", verbose=False) predictor_path = "./models/shape_predictor_68_face_landmarks.dat" # predictor_path = "./models/shape_predictor_81_face_landmarks.dat" # the smaller this value gets the faster the detection will work # if it is too small, the user's face might not be detected maxImageSizeForDetection = 320 detector = dlib.get_frontal_face_detector() predictor = dlib.shape_predictor(predictor_path) mean3DShape, blendshapes, mesh, idxs3D, idxs2D = utils.load3DFaceModel( "./models/candide.npz") projectionModel = models.OrthographicProjectionBlendshapes( blendshapes.shape[0]) # open source video vidcap = cv2.VideoCapture(in_filename) # get some parameters from input video width = int(vidcap.get(cv2.CAP_PROP_FRAME_WIDTH)) height = int(vidcap.get(cv2.CAP_PROP_FRAME_HEIGHT)) fps = int(vidcap.get(cv2.CAP_PROP_FPS)) frames_count = int(vidcap.get(cv2.CAP_PROP_FRAME_COUNT)) # create a video writer for output res_filename = "./output/" + out_filename vidwriter = cv2.VideoWriter(res_filename, cv2.VideoWriter_fourcc('M', 'J', 'P', 'G'), fps, (width, height)) cameraImg = vidcap.read()[1] textureImg = cv2.imread(face_filename) textureCoords = utils.getFaceTextureCoords(textureImg, mean3DShape, blendshapes, idxs2D, idxs3D, detector, predictor) renderer = FaceRendering.FaceRenderer(cameraImg, textureImg, textureCoords, mesh) destShapes2D = utils.getFaceKeypoints(cameraImg, detector, predictor, maxImageSizeForDetection) destShape = destShapes2D[0] modelParams = projectionModel.getInitialParameters( mean3DShape[:, idxs3D], destShape[:, idxs2D]) # 3D model parameter optimization modelParams = NonLinearLeastSquares.GaussNewton( modelParams, projectionModel.residual, projectionModel.jacobian, ([mean3DShape[:, idxs3D], blendshapes[:, :, idxs3D] ], destShape[:, idxs2D]), verbose=0) # rendering the model to an image destShape3D = utils.getShape3D(mean3DShape, blendshapes, modelParams) destRenderedImg = renderer.render(destShape3D) self.progressBar.setRange(0, frames_count - 1) # iterate over the frames and apply the face swap for i in tqdm(range(frames_count - 1)): success, cameraImg = vidcap.read() self.progressBar.setValue(i + 1) if success != True: # no frames left => break break shapes2D = utils.getFaceKeypoints(cameraImg, detector, predictor, maxImageSizeForDetection) newImg = cameraImg try: if shapes2D is not None: for shape2D in shapes2D: # 3D model parameter initialization modelParams = projectionModel.getInitialParameters( mean3DShape[:, idxs3D], shape2D[:, idxs2D]) # 3D model parameter optimization modelParams = NonLinearLeastSquares.GaussNewton( modelParams, projectionModel.residual, projectionModel.jacobian, ([ mean3DShape[:, idxs3D], blendshapes[:, :, idxs3D] ], shape2D[:, idxs2D]), verbose=0) # rendering the model to an image shape3D = utils.getShape3D(mean3DShape, blendshapes, modelParams) renderedImg = renderer.render(shape3D) # blending of the rendered face with the image mask = np.copy(renderedImg[:, :, 0]) mask1 = np.copy(destRenderedImg[:, :, 0]) renderedImg = ImageProcessing.colorTransfer( cameraImg, renderedImg, mask) # newImg = ImageProcessing.blendImages(renderedImg, cameraImg, mask) newImg = ImageProcessing.blendImages0( renderedImg, cameraImg, mask, mask1) except: pass vidwriter.write(newImg) # releas video capture and writer vidcap.release() vidwriter.release() renderer.release() # apply audio clip to generated video if keep_audio == True: video = VideoFileClip("./output/proc_video.avi") video.write_videofile(out_filename, audio="./output/src_audio.mp3", progress_bar=False, verbose=False)
def guide_facechange(): def tmp(): print("") t = threading.Timer(GUIDE_SHOW_TIME, tmp) t.start() t_wait = threading.Timer(GUIDE_WAIT_TIME, tmp) t_wait.start() print("Press T to draw the keypoints and the 3D model") print("Press W to start recording to a video file") print("Press R to restart") print("Press Q or ESC to Quit") modelParams = None lockedTranslation = False drawOverlay = False global BACKGROUND_VID_PATH_NUM #cap = cv2.VideoCapture(VIDEO_CAPTURE_CAM_NUM) cap = cv2.VideoCapture(BACKGROUND_VID_PATH[BACKGROUND_VID_PATH_NUM]) writer = None cameraImg = cap.read()[1] # face swap하여 붙일 영상의 img textureImg = cv2.VideoCapture(VIDEO_CAPTURE_CAM_NUM).read()[1] #print("광고영상 shape : \t\t",cameraImg.shape[1],cameraImg.shape[0]) #print("카메라 캡쳐영상 shape : ",textureImg.shape[1],textureImg.shape[0]) ###### face detection with guide cap_guide_cam = cv2.VideoCapture(VIDEO_CAPTURE_CAM_NUM) if (cap_guide_cam.isOpened() == False): print("Unable to read camera feed") frame_width = int(cap_guide_cam.get(3)) frame_height = int(cap_guide_cam.get(4)) str="match your face" str2="O" str3="ATTENTION" while(True): ret, frame = cap_guide_cam.read() frame_org = frame cv2.putText(frame,str,(int(frame_width/3),int(frame_height/6)),cv2.FONT_HERSHEY_SIMPLEX,int(frame_width/600),(0,0,0),int(frame_width/300)) cv2.putText(frame,str2,(int(frame_width/3),int(frame_width/2)),cv2.FONT_HERSHEY_SIMPLEX,int(frame_width/60),(0,0,255),int(frame_width/300)) cv2.putText(frame,str3,(int((frame_width*2)/3),int((frame_height*2)/3)),cv2.FONT_HERSHEY_SIMPLEX,int(frame_width/650),(0,0,0),int(frame_width/300)) cv2.namedWindow("frame", cv2.WND_PROP_FULLSCREEN) cv2.setWindowProperty("frame", cv2.WND_PROP_FULLSCREEN, 1) cv2.imshow('frame',frame) if cv2.waitKey(1) & 0xFF == ord('q'): break if not t_wait.isAlive(): dets = detector(frame_org, 1) #처음 camera로 촬영한 캡쳐를 넣어서 얼굴을 찾음. if len(dets) > 0: print("detected") break else: print("now detecting") if not t.isAlive(): video() # 찍은 영상의 캡쳐를 3D로 재구성하여 합침 textureCoords = utils.getFaceTextureCoords(textureImg, mean3DShape, blendshapes, idxs2D, idxs3D, detector, predictor) # 찍은 얼굴의 데이터를 영상의 얼굴에 rendering renderer = FaceRendering.FaceRenderer(cameraImg, textureImg, textureCoords, mesh) doProcess=False meanTime=[[0]*4 for i in range(4)] while True: #영상 캡쳐 cameraImg = cap.read()[1] shapes2D = utils.getFaceKeypoints(cameraImg, detector, predictor, maxImageSizeForDetection) doProcess = not doProcess if doProcess is not True: continue else: if shapes2D is not None: for shape2D in shapes2D: start = timeit.default_timer() #3D model parameter initialization modelParams = projectionModel.getInitialParameters(mean3DShape[:, idxs3D], shape2D[:, idxs2D]) stop = timeit.default_timer() meanTime[0][0]+=stop-start meanTime[0][1]+=1 #print(1, float(meanTime[0][0]/meanTime[0][1])) start = timeit.default_timer() #3D model parameter optimization modelParams = NonLinearLeastSquares.GaussNewton(modelParams, projectionModel.residual, projectionModel.jacobian, ([mean3DShape[:, idxs3D], blendshapes[:, :, idxs3D]], shape2D[:, idxs2D]), verbose=0) stop = timeit.default_timer() meanTime[1][0]+=stop-start meanTime[1][1]+=1 #print(2, float(meanTime[1][0]/meanTime[1][1])) start = timeit.default_timer() #rendering the model to an image #다듬기 shape3D = utils.getShape3D(mean3DShape, blendshapes, modelParams) renderedImg = renderer.render(shape3D) stop = timeit.default_timer() meanTime[2][0]+=stop-start meanTime[2][1]+=1 #print(3, float(meanTime[2][0]/meanTime[2][1]))\ start = timeit.default_timer() #blending of the rendered face with the image mask = np.copy(renderedImg[:, :, 0]) renderedImg = ImageProcessing.colorTransfer(cameraImg, renderedImg, mask) cameraImg = ImageProcessing.blendImages(renderedImg, cameraImg, mask) stop = timeit.default_timer() meanTime[3][0] += stop - start meanTime[3][1] += 1 #print(4, float(meanTime[3][0] / meanTime[3][1])) #drawing of the mesh and keypoints # 't'를 누를 때, 적용. facepoint가 표시됨. if drawOverlay: drawPoints(cameraImg, shape2D.T) drawProjectedShape(cameraImg, [mean3DShape, blendshapes], projectionModel, mesh, modelParams, lockedTranslation) if writer is not None: writer.write(cameraImg) cv2.namedWindow("image", cv2.WND_PROP_FULLSCREEN) cv2.setWindowProperty("image", cv2.WND_PROP_FULLSCREEN, 1) cv2.imshow('image',cameraImg) key = cv2.waitKey(1) if key == 27 or key == ord('q'): break if key == ord('t'): drawOverlay = not drawOverlay if key == ord('r'): cv2.destroyAllWindows() video() if key == ord('1'): cv2.destroyAllWindows() BACKGROUND_VID_PATH_NUM = 0 guide_facechange() break if key == ord('2'): cv2.destroyAllWindows() BACKGROUND_VID_PATH_NUM = 1 guide_facechange() break if key == ord('3'): cv2.destroyAllWindows() BACKGROUND_VID_PATH_NUM = 2 guide_facechange() break if key == ord('4'): cv2.destroyAllWindows() BACKGROUND_VID_PATH_NUM = 3 guide_facechange() break if key == ord('5'): cv2.destroyAllWindows() BACKGROUND_VID_PATH_NUM = 4 guide_facechange() break if key == ord('w'): if writer is None: print("Starting video writer") writer = cv2.VideoWriter(SAVE_VID_PATH, cv2.VideoWriter_fourcc('X', 'V', 'I', 'D'), 13, (cameraImg.shape[1], cameraImg.shape[0])) if writer.isOpened(): print("Writer succesfully opened") else: writer = None print("Writer opening failed") else: print("Stopping video writer") writer.release() writer = None cap.release() cap_intro_vid.release() cap_guide_cam.release() cv2.destroyAllWindows()
sr_model = torch.load("model_scale_3_batch_4_epoch_500.pth") # Start running count = 0 while True: # Skip frame if count % 5 != 0: count += 1 continue t0 = time() cameraImg = cap.read()[1] cameraImg = cv2.resize(cameraImg, (cameraImg.shape[1] / 4, cameraImg.shape[0] / 4)) # Get 68 facial keypoints shapes2D = utils.getFaceKeypoints(cameraImg, detector, predictor, maxImageSizeForDetection) if shapes2D is not None: for shape2D in shapes2D: # Mouth generating if count % 10 == 0: # Build input for GAN mouth_landmarks, mouth_leftup_coord, m_size= gan_func.getMouthKeypoints(shape2D, 64) combined_image = np.concatenate([mouth_landmarks, mouth_landmarks], axis=1) image_rgb = cv2.cvtColor(combined_image, cv2.COLOR_GRAY2RGB) try: # Generate mouth generated_image = sess.run(output_tensor, feed_dict={image_tensor: image_rgb}) generated_image = np.squeeze(generated_image) # 64 * 64 in RGB
def mix_video(): data = {} if os.path.getsize('global.pickle') > 0: with open('global.pickle', 'rb') as pf: data = pickle.load(pf) else: win32api.MessageBox(0, '값이 제대로 전달되지 않았습니다. 처음부터 다시 시작해주세요', 'warning', 0x00001000) sys.exit(0) backgroundvideo = "testvideo1" genderNum = data['gender'] backgroundvideo = data['backgroundvideo'] userfolder = data['userfolder'] personnumber = data['personnumber'] backgroundvideo_name = data['backgroundvideo_name'] print('gender:', genderNum, 'vid_dir:', backgroundvideo, 'user_dir:', userfolder, 'peonson#:', personnumber, 'vid_name:', backgroundvideo_name) user_shape_data = {} with open("input/" + userfolder + "/" + 'userdata.pickle', 'rb') as pf: user_shape_data = pickle.load(pf) print( "Press T to draw the keypoints and the 3D model") # 결과 영상에 얼굴합성 과정 표시 print("Press R to start recording to a video file") # 녹화 # shape_predictor_64_face_landmarks.dat 여기서 다운받아서 압축해제 # http://sourceforge.net/projects/dclib/files/dlib/v18.10/shape_predictor_68_face_landmarks.dat.bz2rr # 키포인트 인식 모델 predictor_path = "data/shape_predictor_68_face_landmarks.dat" # 이미지 사이즈가 작을수록 처리속도가 빨라짐 # 너무 작으면 얼굴 인식이 안됨 maxImageSizeForDetection = 320 # 얼굴 인식기 로드 detector = dlib.get_frontal_face_detector() # detector = dlib.cnn_face_detection_model_v1("data/mmod_human_face_detector.dat") predictor = dlib.shape_predictor(predictor_path) mean3DShape, blendshapes, mesh, idxs3D, idxs2D = utils.load3DFaceModel( "data/candide.npz") projectionModel = models.OrthographicProjectionBlendshapes( blendshapes.shape[0]) modelParams = None lockedTranslation = False drawOverlay = False writer = None # 콤비네이션: VC:Video,Cam | CI: Cam,Image | CC: Cam, Cam | VI: Video,Image cap_background = cv2.VideoCapture(backgroundvideo) # Video for background # cap_background = cv2.VideoCapture("input/" + backgroundvideo + ".mp4") # Video for background # cap_background = cv2.VideoCapture(0) # WebCAM for background cameraImg = cap_background.read()[1] # 본인의 여러 얼굴을 메모리에 저장함 (0~9) textureImgs = [] for i in range(0, 9): textureImgs.append( cv2.imread("input/" + userfolder + "/" + str(i) + ".jpg")) # Image for face # textureImgs.append(cv2.imread("input/user_test1_images/" + str(i) + ".jpg")) # Image for face if not os.path.isdir('output'): os.mkdir('output') output_video_name = 'output/' + backgroundvideo + '_' + datetime.datetime.now( ).strftime('%Y%m%d_%H%M%S') + '.avi' writer = cv2.VideoWriter(output_video_name, cv2.VideoWriter_fourcc(*'XVID'), 25, (cameraImg.shape[1], cameraImg.shape[0]), True) modelParams = np.zeros(20) startTime = time.time() full_shapes2D_csv = selectGender.reading_csv(backgroundvideo_name + "_annotation") # full_shapes2D_csv = selectGender.reading_csv("testvideo1_annotation") for framecnt, shapes2D_csv in enumerate(full_shapes2D_csv): print("frame number:", framecnt) # 배경으로 사용할 영상의 프레임 이미지 읽기 cap_background.set(cv2.CAP_PROP_POS_FRAMES, framecnt) ret, cameraImg = cap_background.read() try: background_user_shapes2D = utils.getFaceKeypoints( cameraImg, detector, predictor, maxImageSizeForDetection) if shapes2D_csv is not None: # 영상이 끝나면 반복문 탈출 if ret == False: break # textureImg, user_face_angle, user_shapes2D, textureCoords = user_shape_data[utils.getFaceAngle(background_user_shapes2D)] # 저장된 유저 얼굴 정보로 부터 해당 각도의 데이터를 가저옴 background_face_angle = utils.getFaceAngle( background_user_shapes2D) print("user_face_angle: {}".format(background_face_angle)) textureImg = textureImgs[background_face_angle] user_shapes2D = utils.getFaceKeypoints( textureImg, detector, predictor, maxImageSizeForDetection) textureCoords = utils.getFaceTextureCoords_v2( textureImg, mean3DShape, blendshapes, idxs2D, idxs3D, user_shapes2D, predictor) renderer = FaceRendering.FaceRenderer(cameraImg, textureImg, textureCoords, mesh) for shape2D in background_user_shapes2D: #[shapes2D_csv]: # 3D 모델 파라미터 초기화 (영상에서 인식된 얼굴로부터 3D 모델 생성을 위해) modelParams = projectionModel.getInitialParameters( mean3DShape[:, idxs3D], shape2D[:, idxs2D]) # 3D 모델 파라미터 최적화 기능 (배경의 얼굴과 입력된 얼굴의 키포인트 간의 거리를 최소화 하도록 머신러닝으로 최적화) modelParams = NonLinearLeastSquares.GaussNewton( modelParams, projectionModel.residual, projectionModel.jacobian, ([mean3DShape[:, idxs3D], blendshapes[:, :, idxs3D] ], shape2D[:, idxs2D]), verbose=0) # 위의 모델을 이용해 입력된 이미지의 얼굴을 3D객체로 바꿔 배경의 얼굴 위치로 보정 shape3D = utils.getShape3D(mean3DShape, blendshapes, modelParams) renderedImg = renderer.render(shape3D) # 배경 영상과 입력된 얼굴 이미지를 합성 (합성과정에서 색변환, 이미지 블랜딩 기법 사용) mask = np.copy(renderedImg[:, :, 0]) renderedImg = ImageProcessing.colorTransfer( cameraImg, renderedImg, mask) cameraImg = ImageProcessing.blendImages( renderedImg, cameraImg, mask) # 3D매쉬와 키포인트를 화면 위에 그림 if drawOverlay: drawPoints(cameraImg, shape2D.T) # 초록색 drawProjectedShape(cameraImg, [mean3DShape, blendshapes], projectionModel, mesh, modelParams, lockedTranslation) writer.write(cameraImg) # 얼굴 합성된 영상 출력 cv2.imshow('image', cameraImg) except: pass # 걸린 시간 (초) 출력 writer.release() endTime = time.time() - startTime print("endTime: ", endTime)
def guide_facechange(): def tmp(): print("") t = threading.Timer(GUIDE_SHOW_TIME, tmp) t.start() print("Press T to draw the keypoints and the 3D model") print("Press R to start recording to a video file") #loading the keypoint detection model, the image and the 3D model predictor_path = "../shape_predictor_68_face_landmarks.dat" image_name = "../data/jolie.jpg" #the smaller this value gets the faster the detection will work #if it is too small, the user's face might not be detected maxImageSizeForDetection = 320 #카메라로 찍히는 영상에서 얼굴을 찾는 detector detector = dlib.get_frontal_face_detector() #사람 얼굴을 찾는 입과 눈의 구석, 코의 끝과 같은 중요한 얼굴 표식의 위치를 식별하는 점들의 집합 predictor = dlib.shape_predictor(predictor_path) # candide = 3D face model source # mean3Dshape : 얼굴의 중립상태에 해당하는 정점 리스트 # blendshapes : 중립상태인 얼굴에서 추가하여 수정할 수 있는 얼굴 # ex) 미소, 눈썹 올라가는 부분 # candide에 정의된 애니메이션 Units에서 파생된다. # mesh : Candide가 얼굴 목록으로 제공한 원래의 mesh # idxs3D, idxs2D: Candide 모델(idxs3D)과 얼굴 정렬점 세트(idxs2D)사이에 해당하는 지점들의 인덱스들이다. mean3DShape, blendshapes, mesh, idxs3D, idxs2D = utils.load3DFaceModel( "../candide.npz") # projectionModel = models.OrthographicProjectionBlendshapes( blendshapes.shape[0]) modelParams = None lockedTranslation = False drawOverlay = False #cap = cv2.VideoCapture(0) cap = cv2.VideoCapture("../ROEM 2014 Spring SUZY 320p.mp4") writer = None cameraImg = cap.read()[1] # face swap하여 붙일 영상의 img textureImg = cv2.VideoCapture(0).read()[1] #cv2.imread(image_name) print("광고영상 shape : \t\t", cameraImg.shape[1], cameraImg.shape[0]) print("카메라 캡쳐영상 shape : ", textureImg.shape[1], textureImg.shape[0]) ###### face detection with guide cap_guide_cam = cv2.VideoCapture(0) if (cap_guide_cam.isOpened() == False): print("Unable to read camera feed") frame_width = int(cap_guide_cam.get(3)) frame_height = int(cap_guide_cam.get(4)) str = "match your face" str2 = "O" str3 = "ATTENTION" while (True): ret, frame = cap_guide_cam.read() frame_org = frame cv2.putText(frame, str, (int(frame_width / 3), int(frame_height / 6)), cv2.FONT_HERSHEY_SIMPLEX, int(frame_width / 600), (0, 0, 0), int(frame_width / 300)) cv2.putText(frame, str2, (int(frame_width / 3), int(frame_width / 2)), cv2.FONT_HERSHEY_SIMPLEX, int(frame_width / 60), (0, 0, 255), int(frame_width / 300)) cv2.putText(frame, str3, (int( (frame_width * 2) / 3), int( (frame_height * 2) / 3)), cv2.FONT_HERSHEY_SIMPLEX, int(frame_width / 650), (0, 0, 0), int(frame_width / 300)) cv2.imshow('frame', frame) if cv2.waitKey(1) & 0xFF == ord('q'): break dets = detector(frame_org, 1) #처음 camera로 촬영한 캡쳐를 넣어서 얼굴을 찾음. if len(dets) > 0: print("detected") break else: print("now detecting") if not t.isAlive(): video() break # 찍은 영상의 캡쳐를 3D로 재구성하여 합침 textureCoords = utils.getFaceTextureCoords(textureImg, mean3DShape, blendshapes, idxs2D, idxs3D, detector, predictor) # 찍은 얼굴의 데이터를 영상의 얼굴에 rendering renderer = FaceRendering.FaceRenderer(cameraImg, textureImg, textureCoords, mesh) doProcess = False meanTime = [[0] * 4 for i in range(4)] while True: #영상 캡쳐 cameraImg = cap.read()[1] shapes2D = utils.getFaceKeypoints(cameraImg, detector, predictor, maxImageSizeForDetection) doProcess = not doProcess if doProcess is not True: continue else: if shapes2D is not None: for shape2D in shapes2D: start = timeit.default_timer() #3D model parameter initialization modelParams = projectionModel.getInitialParameters( mean3DShape[:, idxs3D], shape2D[:, idxs2D]) stop = timeit.default_timer() meanTime[0][0] += stop - start meanTime[0][1] += 1 #print(1, float(meanTime[0][0]/meanTime[0][1])) start = timeit.default_timer() #3D model parameter optimization modelParams = NonLinearLeastSquares.GaussNewton( modelParams, projectionModel.residual, projectionModel.jacobian, ([mean3DShape[:, idxs3D], blendshapes[:, :, idxs3D] ], shape2D[:, idxs2D]), verbose=0) stop = timeit.default_timer() meanTime[1][0] += stop - start meanTime[1][1] += 1 #print(2, float(meanTime[1][0]/meanTime[1][1])) start = timeit.default_timer() #rendering the model to an image #다듬기 shape3D = utils.getShape3D(mean3DShape, blendshapes, modelParams) renderedImg = renderer.render(shape3D) stop = timeit.default_timer() meanTime[2][0] += stop - start meanTime[2][1] += 1 #print(3, float(meanTime[2][0]/meanTime[2][1])) start = timeit.default_timer() #blending of the rendered face with the image mask = np.copy(renderedImg[:, :, 0]) renderedImg = ImageProcessing.colorTransfer( cameraImg, renderedImg, mask) cameraImg = ImageProcessing.blendImages( renderedImg, cameraImg, mask) stop = timeit.default_timer() meanTime[3][0] += stop - start meanTime[3][1] += 1 #print(4, float(meanTime[3][0] / meanTime[3][1])) #drawing of the mesh and keypoints # 't'를 누를 때, 적용. facepoint가 표시됨. if drawOverlay: drawPoints(cameraImg, shape2D.T) drawProjectedShape(cameraImg, [mean3DShape, blendshapes], projectionModel, mesh, modelParams, lockedTranslation) if writer is not None: writer.write(cameraImg) cv2.namedWindow("image", cv2.WND_PROP_FULLSCREEN) cv2.setWindowProperty("image", cv2.WND_PROP_FULLSCREEN, 1) cv2.imshow('image', cameraImg) key = cv2.waitKey(1) if key == 27 or key == ord('q'): break if key == ord('t'): drawOverlay = not drawOverlay if key == ord('r'): cv2.destroyAllWindows() video() if key == ord('w'): if writer is None: print("Starting video writer") writer = cv2.VideoWriter( "../out.avi", cv2.VideoWriter_fourcc('X', 'V', 'I', 'D'), 13, (cameraImg.shape[1], cameraImg.shape[0])) if writer.isOpened(): print("Writer succesfully opened") else: writer = None print("Writer opening failed") else: print("Stopping video writer") writer.release() writer = None cap.release() cap_intro_vid.release() cap_guide_cam.release() cv2.destroyAllWindows()
textureCoords2 = utils.getFaceTextureCoords( cameraImg, mean3DShape, blendshapes, idxs2D, idxs3D, detector, predictor) textureImg2 = cameraImg print "Second face scanned." except: print "Please try again." pass renderer = FaceRendering.FaceRenderer(cameraImg, mesh) renderer.setFirstFaceImageAndCoordinates(textureImg1, textureCoords1) renderer.setSecondFaceImageAndCoordinates(textureImg2, textureCoords2) while True: cameraImg = cap.read()[1] shapes2D = utils.getFaceKeypoints(cameraImg, detector, predictor, 320) if shapes2D is not None and len(shapes2D) == 2: avg = sum(shapes2D[0][0]) / len(shapes2D[0][0]) if avg > 250: leftFace = shapes2D[0] rightFace = shapes2D[1] else: leftFace = shapes2D[1] rightFace = shapes2D[0] for index, shape2D in enumerate([leftFace, rightFace]): modelParams = projectionModel.getInitialParameters( mean3DShape[:, idxs3D], shape2D[:, idxs2D])
projectionModel = models.OrthographicProjectionBlendshapes(blendshapes.shape[0]) modelParams = None lockedTranslation = False drawOverlay = False cap = cv2.VideoCapture(0) writer = None cameraImg = cap.read()[1] textureImg = cv2.imread(image_name) textureCoords = utils.getFaceTextureCoords(textureImg, mean3DShape, blendshapes, idxs2D, idxs3D, detector, predictor) renderer = FaceRendering.FaceRenderer(cameraImg, textureImg, textureCoords, mesh) while True: cameraImg = cap.read()[1] shapes2D = utils.getFaceKeypoints(cameraImg, detector, predictor, maxImageSizeForDetection) if shapes2D is not None: for shape2D in shapes2D: #3D model parameter initialization modelParams = projectionModel.getInitialParameters(mean3DShape[:, idxs3D], shape2D[:, idxs2D]) #3D model parameter optimization modelParams = NonLinearLeastSquares.GaussNewton(modelParams, projectionModel.residual, projectionModel.jacobian, ([mean3DShape[:, idxs3D], blendshapes[:, :, idxs3D]], shape2D[:, idxs2D]), verbose=0) #rendering the model to an image shape3D = utils.getShape3D(mean3DShape, blendshapes, modelParams) renderedImg = renderer.render(shape3D) #blending of the rendered face with the image mask = np.copy(renderedImg[:, :, 0])
projectionModel = models.OrthographicProjectionBlendshapes(blendshapes.shape[0]) modelParams = None lockedTranslation = False drawOverlay = False cap = cv2.VideoCapture(0) writer = None cameraImg = cap.read()[1] textureImg = cv2.imread(image_name) textureCoords = utils.getFaceTextureCoords(textureImg, mean3DShape, blendshapes, idxs2D, idxs3D, detector, predictor) renderer = FaceRendering.FaceRenderer(cameraImg, textureImg, textureCoords, mesh) while True: cameraImg = cap.read()[1] shapes2D = utils.getFaceKeypoints(cameraImg, detector, predictor) if shapes2D is not None: for shape2D in shapes2D: #3D model parameter initialization modelParams = projectionModel.getInitialParameters(mean3DShape[:, idxs3D], shape2D[:, idxs2D]) #3D model parameter optimization modelParams = NonLinearLeastSquares.GaussNewton(modelParams, projectionModel.residual, projectionModel.jacobian, ([mean3DShape[:, idxs3D], blendshapes[:, :, idxs3D]], shape2D[:, idxs2D]), verbose=0) #rendering the model to an image shape3D = utils.getShape3D(mean3DShape, blendshapes, modelParams) renderedImg = renderer.render(shape3D) #blending of the rendered face with the image mask = np.copy(renderedImg[:, :, 0])
def video_to_images(user_video): vidcap = cv2.VideoCapture("./input/" + str(user_video)) if not vidcap.isOpened(): raise Exception("Could not open video device") angle = 0 #각도 0~8 까지 모두 추출해 낼 수 있는지 확인 video_check = [] folder_name = "./input/"+str(user_video)+"_images" #폴더가 없으면 만든다 if not os.path.isdir(folder_name): os.mkdir(folder_name) while(vidcap.isOpened()): ret, cameraImg = vidcap.read() #유저 쉐입 user_shapes2D = utils.getFaceKeypoints(cameraImg, detector, predictor, maxImageSizeForDetection) #영상이 끝나면 반복문 탈출 if ret == False: break #얼굴 정면이 모두 인식된다면 if user_shapes2D: face_angle = utils.getFaceAngle(user_shapes2D) #유저 텍스처 if face_angle == angle: #캡쳐된 이미지를 저장하는 함수 cv2.imwrite(folder_name+"/%d.jpg" % angle,cameraImg) print('angle %d.jpg saved' % angle) video_check.append(angle) textureCoords = utils.getFaceTextureCoords_v2(cameraImg, mean3DShape, blendshapes, idxs2D, idxs3D, user_shapes2D, predictor)#, maxImgSizeForDetection=320) data_save.append((cameraImg, face_angle, user_shapes2D, textureCoords)) if angle == 8: #8 break else : angle += 1 else : pass else : print("no face") vidcap.release() with open(folder_name +'/userdata.pickle', 'wb') as f: pickle.dump(data_save,f) #모든 각도의 값이 저장되었는지 확인 check = True print (video_check) if len(video_check) == 9: for i in range (len(video_check)): if not video_check[i] == i: check = False else : check = False #모든각도 추출 못했으면 폴더 지우기 if not check: try: shutil.rmtree(folder_name) except OSError as e: if e.errno == 2: # 파일이나 디렉토리가 없음! print ("No such file or directory to remove") else: raise return check