def faceswap():
cameraImg = cap.read()[1]
rows, cols = cameraImg.shape[:2]
translation_matrix = np.float32([[1, 0, int(0.5 * cols)],
[0, 1, int(0.5 * rows)]])
img_t = cv2.warpAffine(cameraImg, translation_matrix, (2 * cols, 2 * rows))
rotation_matrix = cv2.getRotationMatrix2D((cols, rows), -90, 1)
img_r = cv2.warpAffine(img_t, rotation_matrix, (2 * cols, 2 * rows))
translation_matrix = np.float32([[1, 0, -int(cols - (0.5 * rows))],
[0, 1, -int(rows - (0.5 * cols))]])
cameraImg = cv2.warpAffine(img_r, translation_matrix, (rows, cols))
shapes2D = utils.getFaceKeypoints(cameraImg, detector, predictor,
maxImageSizeForDetection)
if shapes2D is not None:
for shape2D in shapes2D:
#3D model parameter initialization
modelParams = projectionModel.getInitialParameters(
mean3DShape[:, idxs3D], shape2D[:, idxs2D])
#3D model parameter optimization
modelParams = NonLinearLeastSquares.GaussNewton(
modelParams,
projectionModel.residual,
projectionModel.jacobian,
([mean3DShape[:, idxs3D], blendshapes[:, :, idxs3D]
], shape2D[:, idxs2D]),
verbose=0)
#rendering the model to an image
shape3D = utils.getShape3D(mean3DShape, blendshapes, modelParams)
renderedImg = renderer.render(shape3D)
#blending of the rendered face with the image
mask = np.copy(renderedImg[:, :, 0])
renderedImg = ImageProcessing.colorTransfer(
cameraImg, renderedImg, mask)
cameraImg = ImageProcessing.blendImages(renderedImg, cameraImg,
mask)
#drawing of the mesh and keypoints
if drawOverlay:
drawPoints(cameraImg, shape2D.T)
drawProjectedShape(cameraImg, [mean3DShape, blendshapes],
projectionModel, mesh, modelParams,
lockedTranslation)
if writer is not None:
writer.write(cameraImg)
cv2.imshow('image', cameraImg)
return cameraImg
key = cv2.waitKey(1)
if key == 27:
return []
def draw_face_mesh(self, frame, roi, landmarks, head_pose):
# print('draw_face_mesh-------------------')
if self.shapes2D is not None:
for shape2D in self.shapes2D:
print(shape2D[0], len(shape2D[0])
) # list of [[x1,x2,x3.....xn],[y1,y2,y3.....yn]]
#3D model parameter initialization
modelParams = self.projectionModel.getInitialParameters(
self.mean3DShape[:, self.idxs3D], shape2D[:, self.idxs2D])
#3D model parameter optimization
modelParams = NonLinearLeastSquares.GaussNewton(
modelParams,
self.projectionModel.residual,
self.projectionModel.jacobian, ([
self.mean3DShape[:, self.idxs3D],
self.blendshapes[:, :, self.idxs3D]
], shape2D[:, self.idxs2D]),
verbose=0)
#rendering the model to an image
shape3D = utils.getShape3D(self.mean3DShape, self.blendshapes,
modelParams)
renderedImg = self.renderer.render(shape3D)
#blending of the rendered face with the image
mask = np.copy(renderedImg[:, :, 0])
renderedImg = ImageProcessing.colorTransfer(
frame, renderedImg, mask)
# apply rendered image on frame
frame = ImageProcessing.blendImages(renderedImg, frame, mask)
# cv2.imshow('frame', frame)
#drawing of the mesh and keypoints
if self.drawOverlay:
drawPoints(frame, shape2D.T)
drawProjectedShape(frame,
[self.mean3DShape, self.blendshapes],
self.projectionModel, self.mesh,
modelParams, self.lockedTranslation)
cv2.imshow('Face recognition demo', frame)
if self.writer is not None:
# output_stream.write(frame)
self.writer.write(frame)
def process_video(self,
in_filename,
out_filename,
face_filename,
keep_audio=True):
# extract audio clip from src
if keep_audio == True:
clip = VideoFileClip(in_filename)
clip.audio.write_audiofile("./temp/src_audio.mp3", verbose=False)
predictor_path = "./models/shape_predictor_68_face_landmarks.dat"
# predictor_path = "./models/shape_predictor_81_face_landmarks.dat"
# the smaller this value gets the faster the detection will work
# if it is too small, the user's face might not be detected
maxImageSizeForDetection = 320
detector = dlib.get_frontal_face_detector()
predictor = dlib.shape_predictor(predictor_path)
mean3DShape, blendshapes, mesh, idxs3D, idxs2D = utils.load3DFaceModel(
"./models/candide.npz")
projectionModel = models.OrthographicProjectionBlendshapes(
blendshapes.shape[0])
# open source video
vidcap = cv2.VideoCapture(in_filename)
# get some parameters from input video
width = int(vidcap.get(cv2.CAP_PROP_FRAME_WIDTH))
height = int(vidcap.get(cv2.CAP_PROP_FRAME_HEIGHT))
fps = int(vidcap.get(cv2.CAP_PROP_FPS))
frames_count = int(vidcap.get(cv2.CAP_PROP_FRAME_COUNT))
# create a video writer for output
res_filename = "./output/" + out_filename
vidwriter = cv2.VideoWriter(res_filename,
cv2.VideoWriter_fourcc('M', 'J', 'P', 'G'),
fps, (width, height))
cameraImg = vidcap.read()[1]
textureImg = cv2.imread(face_filename)
textureCoords = utils.getFaceTextureCoords(textureImg, mean3DShape,
blendshapes, idxs2D, idxs3D,
detector, predictor)
renderer = FaceRendering.FaceRenderer(cameraImg, textureImg,
textureCoords, mesh)
destShapes2D = utils.getFaceKeypoints(cameraImg, detector, predictor,
maxImageSizeForDetection)
destShape = destShapes2D[0]
modelParams = projectionModel.getInitialParameters(
mean3DShape[:, idxs3D], destShape[:, idxs2D])
# 3D model parameter optimization
modelParams = NonLinearLeastSquares.GaussNewton(
modelParams,
projectionModel.residual,
projectionModel.jacobian,
([mean3DShape[:, idxs3D], blendshapes[:, :, idxs3D]
], destShape[:, idxs2D]),
verbose=0)
# rendering the model to an image
destShape3D = utils.getShape3D(mean3DShape, blendshapes, modelParams)
destRenderedImg = renderer.render(destShape3D)
self.progressBar.setRange(0, frames_count - 1)
# iterate over the frames and apply the face swap
for i in tqdm(range(frames_count - 1)):
success, cameraImg = vidcap.read()
self.progressBar.setValue(i + 1)
if success != True:
# no frames left => break
break
shapes2D = utils.getFaceKeypoints(cameraImg, detector, predictor,
maxImageSizeForDetection)
newImg = cameraImg
try:
if shapes2D is not None:
for shape2D in shapes2D:
# 3D model parameter initialization
modelParams = projectionModel.getInitialParameters(
mean3DShape[:, idxs3D], shape2D[:, idxs2D])
# 3D model parameter optimization
modelParams = NonLinearLeastSquares.GaussNewton(
modelParams,
projectionModel.residual,
projectionModel.jacobian, ([
mean3DShape[:, idxs3D], blendshapes[:, :,
idxs3D]
], shape2D[:, idxs2D]),
verbose=0)
# rendering the model to an image
shape3D = utils.getShape3D(mean3DShape, blendshapes,
modelParams)
renderedImg = renderer.render(shape3D)
# blending of the rendered face with the image
mask = np.copy(renderedImg[:, :, 0])
mask1 = np.copy(destRenderedImg[:, :, 0])
renderedImg = ImageProcessing.colorTransfer(
cameraImg, renderedImg, mask)
# newImg = ImageProcessing.blendImages(renderedImg, cameraImg, mask)
newImg = ImageProcessing.blendImages0(
renderedImg, cameraImg, mask, mask1)
except:
pass
vidwriter.write(newImg)
# releas video capture and writer
vidcap.release()
vidwriter.release()
renderer.release()
# apply audio clip to generated video
if keep_audio == True:
video = VideoFileClip("./output/proc_video.avi")
video.write_videofile(out_filename,
audio="./output/src_audio.mp3",
progress_bar=False,
verbose=False)
def guide_facechange():
def tmp():
print("")
t = threading.Timer(GUIDE_SHOW_TIME, tmp)
t.start()
t_wait = threading.Timer(GUIDE_WAIT_TIME, tmp)
t_wait.start()
print("Press T to draw the keypoints and the 3D model")
print("Press W to start recording to a video file")
print("Press R to restart")
print("Press Q or ESC to Quit")
modelParams = None
lockedTranslation = False
drawOverlay = False
global BACKGROUND_VID_PATH_NUM
#cap = cv2.VideoCapture(VIDEO_CAPTURE_CAM_NUM)
cap = cv2.VideoCapture(BACKGROUND_VID_PATH[BACKGROUND_VID_PATH_NUM])
writer = None
cameraImg = cap.read()[1] # face swap하여 붙일 영상의 img
textureImg = cv2.VideoCapture(VIDEO_CAPTURE_CAM_NUM).read()[1]
#print("광고영상 shape : \t\t",cameraImg.shape[1],cameraImg.shape[0])
#print("카메라 캡쳐영상 shape : ",textureImg.shape[1],textureImg.shape[0])
###### face detection with guide
cap_guide_cam = cv2.VideoCapture(VIDEO_CAPTURE_CAM_NUM)
if (cap_guide_cam.isOpened() == False):
print("Unable to read camera feed")
frame_width = int(cap_guide_cam.get(3))
frame_height = int(cap_guide_cam.get(4))
str="match your face"
str2="O"
str3="ATTENTION"
while(True):
ret, frame = cap_guide_cam.read()
frame_org = frame
cv2.putText(frame,str,(int(frame_width/3),int(frame_height/6)),cv2.FONT_HERSHEY_SIMPLEX,int(frame_width/600),(0,0,0),int(frame_width/300))
cv2.putText(frame,str2,(int(frame_width/3),int(frame_width/2)),cv2.FONT_HERSHEY_SIMPLEX,int(frame_width/60),(0,0,255),int(frame_width/300))
cv2.putText(frame,str3,(int((frame_width*2)/3),int((frame_height*2)/3)),cv2.FONT_HERSHEY_SIMPLEX,int(frame_width/650),(0,0,0),int(frame_width/300))
cv2.namedWindow("frame", cv2.WND_PROP_FULLSCREEN)
cv2.setWindowProperty("frame", cv2.WND_PROP_FULLSCREEN, 1)
cv2.imshow('frame',frame)
if cv2.waitKey(1) & 0xFF == ord('q'):
break
if not t_wait.isAlive():
dets = detector(frame_org, 1) #처음 camera로 촬영한 캡쳐를 넣어서 얼굴을 찾음.
if len(dets) > 0:
print("detected")
break
else:
print("now detecting")
if not t.isAlive():
video()
# 찍은 영상의 캡쳐를 3D로 재구성하여 합침
textureCoords = utils.getFaceTextureCoords(textureImg, mean3DShape, blendshapes, idxs2D, idxs3D, detector, predictor)
# 찍은 얼굴의 데이터를 영상의 얼굴에 rendering
renderer = FaceRendering.FaceRenderer(cameraImg, textureImg, textureCoords, mesh)
doProcess=False
meanTime=[[0]*4 for i in range(4)]
while True:
#영상 캡쳐
cameraImg = cap.read()[1]
shapes2D = utils.getFaceKeypoints(cameraImg, detector, predictor, maxImageSizeForDetection)
doProcess = not doProcess
if doProcess is not True:
continue
else:
if shapes2D is not None:
for shape2D in shapes2D:
start = timeit.default_timer()
#3D model parameter initialization
modelParams = projectionModel.getInitialParameters(mean3DShape[:, idxs3D], shape2D[:, idxs2D])
stop = timeit.default_timer()
meanTime[0][0]+=stop-start
meanTime[0][1]+=1
#print(1, float(meanTime[0][0]/meanTime[0][1]))
start = timeit.default_timer()
#3D model parameter optimization
modelParams = NonLinearLeastSquares.GaussNewton(modelParams, projectionModel.residual, projectionModel.jacobian, ([mean3DShape[:, idxs3D], blendshapes[:, :, idxs3D]], shape2D[:, idxs2D]), verbose=0)
stop = timeit.default_timer()
meanTime[1][0]+=stop-start
meanTime[1][1]+=1
#print(2, float(meanTime[1][0]/meanTime[1][1]))
start = timeit.default_timer()
#rendering the model to an image
#다듬기
shape3D = utils.getShape3D(mean3DShape, blendshapes, modelParams)
renderedImg = renderer.render(shape3D)
stop = timeit.default_timer()
meanTime[2][0]+=stop-start
meanTime[2][1]+=1
#print(3, float(meanTime[2][0]/meanTime[2][1]))\
start = timeit.default_timer()
#blending of the rendered face with the image
mask = np.copy(renderedImg[:, :, 0])
renderedImg = ImageProcessing.colorTransfer(cameraImg, renderedImg, mask)
cameraImg = ImageProcessing.blendImages(renderedImg, cameraImg, mask)
stop = timeit.default_timer()
meanTime[3][0] += stop - start
meanTime[3][1] += 1
#print(4, float(meanTime[3][0] / meanTime[3][1]))
#drawing of the mesh and keypoints
# 't'를 누를 때, 적용. facepoint가 표시됨.
if drawOverlay:
drawPoints(cameraImg, shape2D.T)
drawProjectedShape(cameraImg, [mean3DShape, blendshapes], projectionModel, mesh, modelParams, lockedTranslation)
if writer is not None:
writer.write(cameraImg)
cv2.namedWindow("image", cv2.WND_PROP_FULLSCREEN)
cv2.setWindowProperty("image", cv2.WND_PROP_FULLSCREEN, 1)
cv2.imshow('image',cameraImg)
key = cv2.waitKey(1)
if key == 27 or key == ord('q'):
break
if key == ord('t'):
drawOverlay = not drawOverlay
if key == ord('r'):
cv2.destroyAllWindows()
video()
if key == ord('1'):
cv2.destroyAllWindows()
BACKGROUND_VID_PATH_NUM = 0
guide_facechange()
break
if key == ord('2'):
cv2.destroyAllWindows()
BACKGROUND_VID_PATH_NUM = 1
guide_facechange()
break
if key == ord('3'):
cv2.destroyAllWindows()
BACKGROUND_VID_PATH_NUM = 2
guide_facechange()
break
if key == ord('4'):
cv2.destroyAllWindows()
BACKGROUND_VID_PATH_NUM = 3
guide_facechange()
break
if key == ord('5'):
cv2.destroyAllWindows()
BACKGROUND_VID_PATH_NUM = 4
guide_facechange()
break
if key == ord('w'):
if writer is None:
print("Starting video writer")
writer = cv2.VideoWriter(SAVE_VID_PATH, cv2.VideoWriter_fourcc('X', 'V', 'I', 'D'), 13, (cameraImg.shape[1], cameraImg.shape[0]))
if writer.isOpened():
print("Writer succesfully opened")
else:
writer = None
print("Writer opening failed")
else:
print("Stopping video writer")
writer.release()
writer = None
cap.release()
cap_intro_vid.release()
cap_guide_cam.release()
cv2.destroyAllWindows()
out_img_y = out_img_y.clip(0, 255)
out_img_y = Image.fromarray(np.uint8(out_img_y[0]), mode='L')
out_img_cb = cb.resize(out_img_y.size, Image.BICUBIC)
out_img_cr = cr.resize(out_img_y.size, Image.BICUBIC)
out_img = Image.merge('YCbCr', [out_img_y, out_img_cb, out_img_cr]).convert('RGB')
out_img = np.array(out_img)
image_bgr = cv2.cvtColor(np.squeeze(out_img), cv2.COLOR_RGB2BGR)
image_bgr = cv2.resize(image_bgr, (image_bgr.shape[1] / 3, image_bgr.shape[0] / 3))
image_bgr[image_bgr < 15] = 0
# Remap generated mouth to original size and position
remapped_mouth = gan_func.setMouth(cameraImg, image_bgr, mouth_leftup_coord, m_size) # Put generated mouth back to renderedImg
# Blend remapped mouth with original frame
mask_mouth = np.copy(remapped_mouth[:,:,0])
remapped_mouth = ImageProcessing.colorTransfer(cameraImg, remapped_mouth, mask_mouth)
cameraImg = ImageProcessing.blendImages(remapped_mouth, cameraImg, mask_mouth)
except:
pass
else:
try:
cameraImg = ImageProcessing.blendImages(remapped_mouth, cameraImg, mask_mouth)
except:
pass
# Face rendering
if count % 10 == 0:
# Init params of 3D model, then perform optimisation
modelParams = projectionModel.getInitialParameters(mean3DShape[:, idxs3D], shape2D[:, idxs2D])
modelParams = NonLinearLeastSquares.GaussNewton(modelParams, projectionModel.residual, projectionModel.jacobian, ([mean3DShape[:, idxs3D], blendshapes[:, :, idxs3D]], shape2D[:, idxs2D]), verbose=0)
modelParams = projectionModel.getInitialParameters(
mean3DShape[:, idxs3D], shape2D[:, idxs2D])
modelParams = NonLinearLeastSquares.GaussNewton(
modelParams,
projectionModel.residual,
projectionModel.jacobian,
([mean3DShape[:, idxs3D], blendshapes[:, :, idxs3D]
], shape2D[:, idxs2D]),
verbose=0)
shape3D = utils.getShape3D(mean3DShape, blendshapes, modelParams)
renderedImg = renderer.render(shape3D)
mask = np.copy(renderedImg[:, :, 0])
renderedImg = ImageProcessing.colorTransfer(
cameraImg, renderedImg, mask)
cameraImg = ImageProcessing.blendImages(renderedImg, cameraImg,
mask)
if drawOverlay:
drawPoints(cameraImg, shape2D.T)
drawProjectedShape(cameraImg, [mean3DShape, blendshapes],
projectionModel, mesh, modelParams,
lockedTranslation)
if writer is not None:
writer.write(cameraImg)
writer = cv2.VideoWriter(
os.path.join(os.path.dirname(__file__), "..", "out.avi"),
cv2.VideoWriter_fourcc('X', 'V', 'I', 'D'), 25,
(cameraImg.shape[1], cameraImg.shape[0]))
def mix_video():
data = {}
if os.path.getsize('global.pickle') > 0:
with open('global.pickle', 'rb') as pf:
data = pickle.load(pf)
else:
win32api.MessageBox(0, '값이 제대로 전달되지 않았습니다. 처음부터 다시 시작해주세요', 'warning',
0x00001000)
sys.exit(0)
backgroundvideo = "testvideo1"
genderNum = data['gender']
backgroundvideo = data['backgroundvideo']
userfolder = data['userfolder']
personnumber = data['personnumber']
backgroundvideo_name = data['backgroundvideo_name']
print('gender:', genderNum, 'vid_dir:', backgroundvideo, 'user_dir:',
userfolder, 'peonson#:', personnumber, 'vid_name:',
backgroundvideo_name)
user_shape_data = {}
with open("input/" + userfolder + "/" + 'userdata.pickle', 'rb') as pf:
user_shape_data = pickle.load(pf)
print(
"Press T to draw the keypoints and the 3D model") # 결과 영상에 얼굴합성 과정 표시
print("Press R to start recording to a video file") # 녹화
# shape_predictor_64_face_landmarks.dat 여기서 다운받아서 압축해제
# http://sourceforge.net/projects/dclib/files/dlib/v18.10/shape_predictor_68_face_landmarks.dat.bz2rr
# 키포인트 인식 모델
predictor_path = "data/shape_predictor_68_face_landmarks.dat"
# 이미지 사이즈가 작을수록 처리속도가 빨라짐
# 너무 작으면 얼굴 인식이 안됨
maxImageSizeForDetection = 320
# 얼굴 인식기 로드
detector = dlib.get_frontal_face_detector()
# detector = dlib.cnn_face_detection_model_v1("data/mmod_human_face_detector.dat")
predictor = dlib.shape_predictor(predictor_path)
mean3DShape, blendshapes, mesh, idxs3D, idxs2D = utils.load3DFaceModel(
"data/candide.npz")
projectionModel = models.OrthographicProjectionBlendshapes(
blendshapes.shape[0])
modelParams = None
lockedTranslation = False
drawOverlay = False
writer = None
# 콤비네이션: VC:Video,Cam | CI: Cam,Image | CC: Cam, Cam | VI: Video,Image
cap_background = cv2.VideoCapture(backgroundvideo) # Video for background
# cap_background = cv2.VideoCapture("input/" + backgroundvideo + ".mp4") # Video for background
# cap_background = cv2.VideoCapture(0) # WebCAM for background
cameraImg = cap_background.read()[1]
# 본인의 여러 얼굴을 메모리에 저장함 (0~9)
textureImgs = []
for i in range(0, 9):
textureImgs.append(
cv2.imread("input/" + userfolder + "/" + str(i) +
".jpg")) # Image for face
# textureImgs.append(cv2.imread("input/user_test1_images/" + str(i) + ".jpg")) # Image for face
if not os.path.isdir('output'):
os.mkdir('output')
output_video_name = 'output/' + backgroundvideo + '_' + datetime.datetime.now(
).strftime('%Y%m%d_%H%M%S') + '.avi'
writer = cv2.VideoWriter(output_video_name,
cv2.VideoWriter_fourcc(*'XVID'), 25,
(cameraImg.shape[1], cameraImg.shape[0]), True)
modelParams = np.zeros(20)
startTime = time.time()
full_shapes2D_csv = selectGender.reading_csv(backgroundvideo_name +
"_annotation")
# full_shapes2D_csv = selectGender.reading_csv("testvideo1_annotation")
for framecnt, shapes2D_csv in enumerate(full_shapes2D_csv):
print("frame number:", framecnt)
# 배경으로 사용할 영상의 프레임 이미지 읽기
cap_background.set(cv2.CAP_PROP_POS_FRAMES, framecnt)
ret, cameraImg = cap_background.read()
try:
background_user_shapes2D = utils.getFaceKeypoints(
cameraImg, detector, predictor, maxImageSizeForDetection)
if shapes2D_csv is not None:
# 영상이 끝나면 반복문 탈출
if ret == False:
break
# textureImg, user_face_angle, user_shapes2D, textureCoords = user_shape_data[utils.getFaceAngle(background_user_shapes2D)]
# 저장된 유저 얼굴 정보로 부터 해당 각도의 데이터를 가저옴
background_face_angle = utils.getFaceAngle(
background_user_shapes2D)
print("user_face_angle: {}".format(background_face_angle))
textureImg = textureImgs[background_face_angle]
user_shapes2D = utils.getFaceKeypoints(
textureImg, detector, predictor, maxImageSizeForDetection)
textureCoords = utils.getFaceTextureCoords_v2(
textureImg, mean3DShape, blendshapes, idxs2D, idxs3D,
user_shapes2D, predictor)
renderer = FaceRendering.FaceRenderer(cameraImg, textureImg,
textureCoords, mesh)
for shape2D in background_user_shapes2D: #[shapes2D_csv]:
# 3D 모델 파라미터 초기화 (영상에서 인식된 얼굴로부터 3D 모델 생성을 위해)
modelParams = projectionModel.getInitialParameters(
mean3DShape[:, idxs3D], shape2D[:, idxs2D])
# 3D 모델 파라미터 최적화 기능 (배경의 얼굴과 입력된 얼굴의 키포인트 간의 거리를 최소화 하도록 머신러닝으로 최적화)
modelParams = NonLinearLeastSquares.GaussNewton(
modelParams,
projectionModel.residual,
projectionModel.jacobian,
([mean3DShape[:, idxs3D], blendshapes[:, :, idxs3D]
], shape2D[:, idxs2D]),
verbose=0)
# 위의 모델을 이용해 입력된 이미지의 얼굴을 3D객체로 바꿔 배경의 얼굴 위치로 보정
shape3D = utils.getShape3D(mean3DShape, blendshapes,
modelParams)
renderedImg = renderer.render(shape3D)
# 배경 영상과 입력된 얼굴 이미지를 합성 (합성과정에서 색변환, 이미지 블랜딩 기법 사용)
mask = np.copy(renderedImg[:, :, 0])
renderedImg = ImageProcessing.colorTransfer(
cameraImg, renderedImg, mask)
cameraImg = ImageProcessing.blendImages(
renderedImg, cameraImg, mask)
# 3D매쉬와 키포인트를 화면 위에 그림
if drawOverlay:
drawPoints(cameraImg, shape2D.T) # 초록색
drawProjectedShape(cameraImg,
[mean3DShape, blendshapes],
projectionModel, mesh, modelParams,
lockedTranslation)
writer.write(cameraImg)
# 얼굴 합성된 영상 출력
cv2.imshow('image', cameraImg)
except:
pass
# 걸린 시간 (초) 출력
writer.release()
endTime = time.time() - startTime
print("endTime: ", endTime)
def guide_facechange():
def tmp():
print("")
t = threading.Timer(GUIDE_SHOW_TIME, tmp)
t.start()
print("Press T to draw the keypoints and the 3D model")
print("Press R to start recording to a video file")
#loading the keypoint detection model, the image and the 3D model
predictor_path = "../shape_predictor_68_face_landmarks.dat"
image_name = "../data/jolie.jpg"
#the smaller this value gets the faster the detection will work
#if it is too small, the user's face might not be detected
maxImageSizeForDetection = 320
#카메라로 찍히는 영상에서 얼굴을 찾는 detector
detector = dlib.get_frontal_face_detector()
#사람 얼굴을 찾는 입과 눈의 구석, 코의 끝과 같은 중요한 얼굴 표식의 위치를 식별하는 점들의 집합
predictor = dlib.shape_predictor(predictor_path)
# candide = 3D face model source
# mean3Dshape : 얼굴의 중립상태에 해당하는 정점 리스트
# blendshapes : 중립상태인 얼굴에서 추가하여 수정할 수 있는 얼굴
# ex) 미소, 눈썹 올라가는 부분
# candide에 정의된 애니메이션 Units에서 파생된다.
# mesh : Candide가 얼굴 목록으로 제공한 원래의 mesh
# idxs3D, idxs2D: Candide 모델(idxs3D)과 얼굴 정렬점 세트(idxs2D)사이에 해당하는 지점들의 인덱스들이다.
mean3DShape, blendshapes, mesh, idxs3D, idxs2D = utils.load3DFaceModel(
"../candide.npz")
#
projectionModel = models.OrthographicProjectionBlendshapes(
blendshapes.shape[0])
modelParams = None
lockedTranslation = False
drawOverlay = False
#cap = cv2.VideoCapture(0)
cap = cv2.VideoCapture("../ROEM 2014 Spring SUZY 320p.mp4")
writer = None
cameraImg = cap.read()[1] # face swap하여 붙일 영상의 img
textureImg = cv2.VideoCapture(0).read()[1] #cv2.imread(image_name)
print("광고영상 shape : \t\t", cameraImg.shape[1], cameraImg.shape[0])
print("카메라 캡쳐영상 shape : ", textureImg.shape[1], textureImg.shape[0])
###### face detection with guide
cap_guide_cam = cv2.VideoCapture(0)
if (cap_guide_cam.isOpened() == False):
print("Unable to read camera feed")
frame_width = int(cap_guide_cam.get(3))
frame_height = int(cap_guide_cam.get(4))
str = "match your face"
str2 = "O"
str3 = "ATTENTION"
while (True):
ret, frame = cap_guide_cam.read()
frame_org = frame
cv2.putText(frame, str, (int(frame_width / 3), int(frame_height / 6)),
cv2.FONT_HERSHEY_SIMPLEX, int(frame_width / 600),
(0, 0, 0), int(frame_width / 300))
cv2.putText(frame, str2, (int(frame_width / 3), int(frame_width / 2)),
cv2.FONT_HERSHEY_SIMPLEX, int(frame_width / 60),
(0, 0, 255), int(frame_width / 300))
cv2.putText(frame, str3, (int(
(frame_width * 2) / 3), int(
(frame_height * 2) / 3)), cv2.FONT_HERSHEY_SIMPLEX,
int(frame_width / 650), (0, 0, 0), int(frame_width / 300))
cv2.imshow('frame', frame)
if cv2.waitKey(1) & 0xFF == ord('q'):
break
dets = detector(frame_org, 1) #처음 camera로 촬영한 캡쳐를 넣어서 얼굴을 찾음.
if len(dets) > 0:
print("detected")
break
else:
print("now detecting")
if not t.isAlive():
video()
break
# 찍은 영상의 캡쳐를 3D로 재구성하여 합침
textureCoords = utils.getFaceTextureCoords(textureImg, mean3DShape,
blendshapes, idxs2D, idxs3D,
detector, predictor)
# 찍은 얼굴의 데이터를 영상의 얼굴에 rendering
renderer = FaceRendering.FaceRenderer(cameraImg, textureImg, textureCoords,
mesh)
doProcess = False
meanTime = [[0] * 4 for i in range(4)]
while True:
#영상 캡쳐
cameraImg = cap.read()[1]
shapes2D = utils.getFaceKeypoints(cameraImg, detector, predictor,
maxImageSizeForDetection)
doProcess = not doProcess
if doProcess is not True:
continue
else:
if shapes2D is not None:
for shape2D in shapes2D:
start = timeit.default_timer()
#3D model parameter initialization
modelParams = projectionModel.getInitialParameters(
mean3DShape[:, idxs3D], shape2D[:, idxs2D])
stop = timeit.default_timer()
meanTime[0][0] += stop - start
meanTime[0][1] += 1
#print(1, float(meanTime[0][0]/meanTime[0][1]))
start = timeit.default_timer()
#3D model parameter optimization
modelParams = NonLinearLeastSquares.GaussNewton(
modelParams,
projectionModel.residual,
projectionModel.jacobian,
([mean3DShape[:, idxs3D], blendshapes[:, :, idxs3D]
], shape2D[:, idxs2D]),
verbose=0)
stop = timeit.default_timer()
meanTime[1][0] += stop - start
meanTime[1][1] += 1
#print(2, float(meanTime[1][0]/meanTime[1][1]))
start = timeit.default_timer()
#rendering the model to an image
#다듬기
shape3D = utils.getShape3D(mean3DShape, blendshapes,
modelParams)
renderedImg = renderer.render(shape3D)
stop = timeit.default_timer()
meanTime[2][0] += stop - start
meanTime[2][1] += 1
#print(3, float(meanTime[2][0]/meanTime[2][1]))
start = timeit.default_timer()
#blending of the rendered face with the image
mask = np.copy(renderedImg[:, :, 0])
renderedImg = ImageProcessing.colorTransfer(
cameraImg, renderedImg, mask)
cameraImg = ImageProcessing.blendImages(
renderedImg, cameraImg, mask)
stop = timeit.default_timer()
meanTime[3][0] += stop - start
meanTime[3][1] += 1
#print(4, float(meanTime[3][0] / meanTime[3][1]))
#drawing of the mesh and keypoints
# 't'를 누를 때, 적용. facepoint가 표시됨.
if drawOverlay:
drawPoints(cameraImg, shape2D.T)
drawProjectedShape(cameraImg,
[mean3DShape, blendshapes],
projectionModel, mesh, modelParams,
lockedTranslation)
if writer is not None:
writer.write(cameraImg)
cv2.namedWindow("image", cv2.WND_PROP_FULLSCREEN)
cv2.setWindowProperty("image", cv2.WND_PROP_FULLSCREEN, 1)
cv2.imshow('image', cameraImg)
key = cv2.waitKey(1)
if key == 27 or key == ord('q'):
break
if key == ord('t'):
drawOverlay = not drawOverlay
if key == ord('r'):
cv2.destroyAllWindows()
video()
if key == ord('w'):
if writer is None:
print("Starting video writer")
writer = cv2.VideoWriter(
"../out.avi",
cv2.VideoWriter_fourcc('X', 'V', 'I', 'D'), 13,
(cameraImg.shape[1], cameraImg.shape[0]))
if writer.isOpened():
print("Writer succesfully opened")
else:
writer = None
print("Writer opening failed")
else:
print("Stopping video writer")
writer.release()
writer = None
cap.release()
cap_intro_vid.release()
cap_guide_cam.release()
cv2.destroyAllWindows()
renderer = FaceRendering.FaceRenderer(targetImg, textureImg, textureCoords, mesh)
shapes2D = utils.getFaceKeypoints(targetImg, detector, predictor, maxImageSizeForDetection)
if shapes2D is not None:
for shape2D in shapes2D:
#3D model parameter initialization
modelParams = projectionModel.getInitialParameters(mean3DShape[:, idxs3D], shape2D[:, idxs2D])
#3D model parameter optimization
modelParams = NonLinearLeastSquares.GaussNewton(modelParams, projectionModel.residual, projectionModel.jacobian, ([mean3DShape[:, idxs3D], blendshapes[:, :, idxs3D]], shape2D[:, idxs2D]), verbose=0)
#rendering the model to an image
shape3D = utils.getShape3D(mean3DShape, blendshapes, modelParams)
renderedImg = renderer.render(shape3D)
#blending of the rendered face with the image
mask = np.copy(renderedImg[:, :, 0])
renderedImg = ImageProcessing.colorTransfer(targetImg, renderedImg, mask)
targetImg = ImageProcessing.blendImages(renderedImg, targetImg, mask)
#drawing of the mesh and keypoints
if drawOverlay:
drawPoints(targetImg, shape2D.T)
drawProjectedShape(targetImg, [mean3DShape, blendshapes], projectionModel, mesh, modelParams, lockedTranslation)
cv2.imshow("img", targetImg)
cv2.imwrite("output.jpg", targetImg)
import numpy as np
import cv2
import ImageProcessing
handImg = cv2.imread("../data/hand.png")
eyeImg = cv2.imread("../data/eye.png")
maskImg = cv2.imread("../data/mask.png")
#zmiana obrazka kolorowego na obrazek w skali szarosci
mask = np.mean(maskImg, axis=2)
eyeImg = ImageProcessing.colorTransfer(handImg, eyeImg, mask)
blendedImg = ImageProcessing.blendImages(eyeImg, handImg, mask)
cv2.imwrite("../eyeHandBlend.jpg", blendedImg)
if shapes2D is not None:
for shape2D in shapes2D:
#3D model parameter initialization
modelParams = projectionModel.getInitialParameters(mean3DShape[:, idxs3D], shape2D[:, idxs2D])
#3D model parameter optimization
modelParams = NonLinearLeastSquares.GaussNewton(modelParams, projectionModel.residual, projectionModel.jacobian, ([mean3DShape[:, idxs3D], blendshapes[:, :, idxs3D]], shape2D[:, idxs2D]), verbose=0)
#rendering the model to an image
shape3D = utils.getShape3D(mean3DShape, blendshapes, modelParams)
renderedImg = renderer.render(shape3D)
#blending of the rendered face with the image
mask = np.copy(renderedImg[:, :, 0])
renderedImg = ImageProcessing.colorTransfer(cameraImg, renderedImg, mask)
cameraImg = ImageProcessing.blendImages(renderedImg, cameraImg, mask)
#drawing of the mesh and keypoints
if drawOverlay:
drawPoints(cameraImg, shape2D.T)
drawProjectedShape(cameraImg, [mean3DShape, blendshapes], projectionModel, mesh, modelParams, lockedTranslation)
if writer is not None:
writer.write(cameraImg)
cv2.imshow('image', cameraImg)
key = cv2.waitKey(1)
if key == 27:
def face_Fusion(self):
print("----- Face fusion function is start -----")
print(self.status)
# cameraImg = self.cap.read()[1]
# import image path
self.textureImg = cv2.imread(self.image_name)
maxImageSizeForDetection = 320
detector = dlib.get_frontal_face_detector()
mean3DShape, blendshapes, mesh, idxs3D, idxs2D = utils_face_fusion.load3DFaceModel(
"./fw/candide.npz")
projectionModel = models_face_fusion.OrthographicProjectionBlendshapes(
blendshapes.shape[0])
modelParams = None
lockedTranslation = False
drawOverlay = False
# cap = cv2.VideoCapture(cv2.CAP_DSHOW)
writer = None
cameraImg = self.cap.read()[1]
self.textureImg = cv2.imread(self.image_name)
# textureCoords = utils_face_fusion.getFaceTextureCoords(self.textureImg, mean3DShape, blendshapes, idxs2D, idxs3D, detector, predictor)
# renderer = FaceRendering.FaceRenderer(cameraImg, self.textureImg, textureCoords, mesh)
while True:
print("----- Face fusion function is running -----")
cameraImg = self.cap.read()[1]
if (self.status != "2"):
print("end2")
break
textureCoords = utils_face_fusion.getFaceTextureCoords(
self.textureImg, mean3DShape, blendshapes, idxs2D, idxs3D,
detector, predictor)
renderer = FaceRendering.FaceRenderer(cameraImg, self.textureImg,
textureCoords, mesh)
shapes2D = utils_face_fusion.getFaceKeypoints(
cameraImg, detector, predictor, maxImageSizeForDetection)
if shapes2D is not None:
for shape2D in shapes2D:
# 3D model parameter initialization
modelParams = projectionModel.getInitialParameters(
mean3DShape[:, idxs3D], shape2D[:, idxs2D])
# cameraImg = self.cap.read()[1]
# 3D model parameter optimization
modelParams = NonLinearLeastSquares.GaussNewton(
modelParams,
projectionModel.residual,
projectionModel.jacobian,
([mean3DShape[:, idxs3D], blendshapes[:, :, idxs3D]
], shape2D[:, idxs2D]),
verbose=0)
# rendering the model to an image
shape3D = utils_face_fusion.getShape3D(
mean3DShape, blendshapes, modelParams)
renderedImg = renderer.render(shape3D)
# blending of the rendered face with the image
mask = np.copy(renderedImg[:, :, 0])
renderedImg = ImageProcessing.colorTransfer(
cameraImg, renderedImg, mask)
cameraImg = ImageProcessing.blendImages(
renderedImg, cameraImg, mask)
# drawing of the mesh and keypoints
if drawOverlay:
drawPoints(cameraImg, shape2D.T)
drawProjectedShape(cameraImg,
[mean3DShape, blendshapes],
projectionModel, mesh, modelParams,
lockedTranslation)
imgg = self.textureImg
imgg = cv2.resize(imgg, (100, 100))
cameraImg = Add_image(cameraImg, imgg)
if writer is not None:
writer.write(cameraImg)
self.image = cameraImg
show = cv2.resize(self.image, (1080, 960)) # 把读到的帧的大小重新设置为 640x480
# 视频色彩转换回RGB，这样才是现实的颜色
show = cv2.cvtColor(show, cv2.COLOR_BGR2RGB)
showImage = QtGui.QImage(
show.data, show.shape[1], show.shape[0],
QtGui.QImage.Format_RGB888) # 把读取到的视频数据变成QImage形式
self.label_show_camera.setPixmap(
QtGui.QPixmap.fromImage(showImage))
if (self.status != "2"):
self.label_show_camera.clear()
print("end1")
return None
