def lr2hr3DModel(imgLR, imgHR, zRatio, objPath): '''1. read LR and HR img''' # imgLRPath = '../../github/vrn-07231340/examples/scaled/trump-12.jpg' # imgLR = cv2.imread(imgLRPath) # imgHRPath = '../../github/vrn-07231340/examples/trump-12.jpg' # imgHR = cv2.imread(imgHRPath) '''2. face landmark detection on LR and HR img''' landmarkLR = getLandmark2D(imgLR) landmarkHR = getLandmark2D(imgHR) landmarkLR1 = np.float32([[landmarkLR[0][0], landmarkLR[0][1]], [landmarkLR[8][0], landmarkLR[8][1]], [landmarkLR[16][0], landmarkLR[16][1]]]) landmarkHR1 = np.float32([[landmarkHR[0][0], landmarkHR[0][1]], [landmarkHR[8][0], landmarkHR[8][1]], [landmarkHR[16][0], landmarkHR[16][1]]]) landmarkLR2 = np.float32([[landmarkLR[27][0], landmarkLR[27][1]], [landmarkLR[48][0], landmarkLR[48][1]], [landmarkLR[54][0], landmarkLR[54][1]]]) landmarkHR2 = np.float32([[landmarkHR[27][0], landmarkHR[27][1]], [landmarkHR[48][0], landmarkHR[48][1]], [landmarkHR[54][0], landmarkHR[54][1]]]) landmarkLR3 = np.float32([[landmarkLR[36][0], landmarkLR[36][1]], [landmarkLR[45][0], landmarkLR[45][1]], [landmarkLR[33][0], landmarkLR[33][1]]]) landmarkHR3 = np.float32([[landmarkHR[36][0], landmarkHR[36][1]], [landmarkHR[45][0], landmarkHR[45][1]], [landmarkHR[33][0], landmarkHR[33][1]]]) '''3. affine transform''' M1 = cv2.getAffineTransform(landmarkLR1, landmarkHR1) M2 = cv2.getAffineTransform(landmarkLR2, landmarkHR2) M3 = cv2.getAffineTransform(landmarkLR3, landmarkHR3) M = (M1 + M2 + M3) / 3 objLines, _, _ = readObj(objPath) newObjLines = [] newVLines = [] for objLine in objLines: if objLine.split()[0] == 'v': v, x, y, z, r, g, b = objLine.split() l = np.array([float(x), float(y), 1]) dst = np.dot(M, l.T) newX = float(dst[0]) newY = float(dst[1]) newZ = float(z) * zRatio objLine = '{} {} {} {} {} {} {}'.format(v, newX, newY, newZ, r, g, b) newVLines.append(objLine) newObjLines.append(objLine) saveObjFile('HR', newObjLines, objPath) landmarkHR = [] for landmark in landmarkLR: l = np.array([landmark[0], landmark[1], 1]) dst = np.dot(M, l.T) landmarkHR.append((dst[0], dst[1])) return newObjLines, newVLines, landmarkLR, landmarkHR
def mainLR2HR(): imgLRPath = '../../github/vrn-07231340/examples/scaled/trump-12.jpg' imgLR = cv2.imread(imgLRPath) imgHRPath = '../../github/vrn-07231340/examples/trump-12.jpg' imgHR = cv2.imread(imgHRPath) srcLandmarkLR = getLandmark2D(imgLR) lr2hr(imgLR, imgHR, srcLandmarkLR)
def lr2hr(imgLR, imgHR, srcLandmarkLR): '''1. read LR and HR img''' # imgLRPath = '../../github/vrn-07231340/examples/scaled/trump-12.jpg' # imgLR = cv2.imread(imgLRPath) # imgHRPath = '../../github/vrn-07231340/examples/trump-12.jpg' # imgHR = cv2.imread(imgHRPath) '''2. face landmark detection on LR and HR img''' landmarkLR = getLandmark2D(imgLR) landmarkHR = getLandmark2D(imgHR) landmarkLR1 = np.float32([[landmarkLR[0][0], landmarkLR[0][1]], [landmarkLR[8][0], landmarkLR[8][1]], [landmarkLR[16][0], landmarkLR[16][1]]]) landmarkHR1 = np.float32([[landmarkHR[0][0], landmarkHR[0][1]], [landmarkHR[8][0], landmarkHR[8][1]], [landmarkHR[16][0], landmarkHR[16][1]]]) landmarkLR2 = np.float32([[landmarkLR[27][0], landmarkLR[27][1]], [landmarkLR[48][0], landmarkLR[48][1]], [landmarkLR[54][0], landmarkLR[54][1]]]) landmarkHR2 = np.float32([[landmarkHR[27][0], landmarkHR[27][1]], [landmarkHR[48][0], landmarkHR[48][1]], [landmarkHR[54][0], landmarkHR[54][1]]]) landmarkLR3 = np.float32([[landmarkLR[36][0], landmarkLR[36][1]], [landmarkLR[45][0], landmarkLR[45][1]], [landmarkLR[33][0], landmarkLR[33][1]]]) landmarkHR3 = np.float32([[landmarkHR[36][0], landmarkHR[36][1]], [landmarkHR[45][0], landmarkHR[45][1]], [landmarkHR[33][0], landmarkHR[33][1]]]) '''3. affine transform''' M1 = cv2.getAffineTransform(landmarkLR1, landmarkHR1) M2 = cv2.getAffineTransform(landmarkLR2, landmarkHR2) M3 = cv2.getAffineTransform(landmarkLR3, landmarkHR3) M = (M1 + M2 + M3) / 3 landmarkHR = [] for landmark in srcLandmarkLR: l = np.array([landmark[0], landmark[1], 1]) dst = np.dot(M, l.T) landmarkHR.append((dst[0], dst[1])) # print landmarkHR # drawPointsOnImg(landmarkHR, imgHR, 'r') return landmarkHR
def main2D_3D_2D_LowResolution(): '''1. read origin obj and image''' objPath = '../../github/vrn-07231340/obj/trump-12.obj' objLines, vLines, fLines = readObj(objPath) imgPath = '../../github/vrn-07231340/examples/scaled/trump-12.jpg' img = cv2.imread(imgPath) '''2. origin 2D landmark --> origin 3D landmark''' ''' BUT ONE 2D LANDMARK --> MORE THAN ONE 3D LANDMARK''' ''' HOW TO CHOOSE THE BEST ONE ?''' ''' OPTION 1: 选Z最大的3D landmark''' '''2.1 origin 2D landmark [x, y]''' originLandmark2DList = getLandmark2D(img) '''2.2 origin 3D landmark list [[[x, y], [(line1, vLine1), (line2, vLine2)]], ... ]''' originLandmark3DList = getLandmark3D(originLandmark2DList, vLines) '''2.3 OPTION 1: 选Z最大的3D landmark''' originLandmark3DList = optionChooseZMax(originLandmark3DList) '''3. nod''' nodAngle = 15 nodCenterMode = 'maxY' nodedObjLines = nod(objPath, nodAngle, nodCenterMode) '''4. noded 3D landmark list''' '''nodedLandmark3DList: [[x0, y0], [x0, y0, z0], [x1, y1], [x1, y1, z1], (line, vLine)]''' '''len(nodedLandmark3DList): number of landmarks both on 2D and 3D''' '''[x0, y0]: original landmark on 2D img''' '''[x0, y0, z0]: original landmark on 3D model''' '''[x1, y1]: noded landmark on 2D img''' '''[x1, y1, z1]: noded landmark on 3D model''' '''(line, vLine): the line(th) noded vLine''' nodedVLines = objLines2vLines(nodedObjLines) nodedLandmark3DList = getNodedLandmark3D(nodedVLines, originLandmark3DList) '''5. noded 3D landmark --> noded 2D landmark''' nodedLandmark2D = [] for nodedLandmark3D in nodedLandmark3DList: nodedLandmark2D.append(nodedLandmark3D[2]) nodedLandmark2D = addEdgeLandmark(nodedLandmark2D, img) '''6. delaunay triangle for (origin landmarks which are both on 2D and 3D) + (eight edge points)''' originLandmark2D = [] for nodedLandmark3D in nodedLandmark3DList: originLandmark2D.append((nodedLandmark3D[0][0], nodedLandmark3D[0][1])) originLandmark2D = addEdgeLandmark(originLandmark2D, img) assert len(originLandmark2D) == len(nodedLandmark2D) triList = generateTriList(originLandmark2D, img) '''7. warp''' imgMorph = morph_modify_for_2D3D2D_low_resolution(originLandmark2D, nodedLandmark2D, img, triList) imshow('imgMorph', imgMorph) cv2.imwrite('./noded.jpg', imgMorph)
def main2D_3D_2D_TurnFace(): '''1. read img and obj''' imgPath = '../../github/vrn-07231340/examples/scaled/trump-12.jpg' img = cv2.imread(imgPath) objPath = '../../github/vrn-07231340/obj/trump-12.obj' objLines, vLines, fLines = readObj(objPath) '''2. origin 2D landmark --> origin 3D landmark''' ''' BUT ONE 2D LANDMARK --> MORE THAN ONE 3D LANDMARK''' ''' HOW TO CHOOSE THE BEST ONE ?''' ''' OPTION 1: 选Z最大的3D landmark''' '''2.1 origin 2D landmark [x, y]''' originLandmark2DList = getLandmark2D(img) '''2.2 origin 3D landmark list [[[x, y], [(line1, vLine1), (line2, vLine2)]], ... ]''' originLandmark3DList = getLandmark3D(originLandmark2DList, vLines) '''2.3 OPTION 1: 选Z最大的3D landmark''' originLandmark3DList = optionChooseZMax(originLandmark3DList) '''3. turn''' turnAngle = 15 turnCenterMode = 'maxY' turnDirection = 'right' turnObjLines = turn(objPath, turnAngle, turnDirection, turnCenterMode) # projection(turnObjLines, imgPath) '''4. turn 3D landmark list''' '''turnLandmark3DList: [[x0, y0], [x0, y0, z0], [x1, y1], [x1, y1, z1], (line, vLine)]''' '''len(turnLandmark3DList): number of landmarks both on 2D and 3D''' '''[x0, y0]: original landmark on 2D img''' '''[x0, y0, z0]: original landmark on 3D model''' '''[x1, y1]: turn landmark on 2D img''' '''[x1, y1, z1]: turn landmark on 3D model''' '''(line, vLine): the line(th) turn vLine''' turnVLines = objLines2vLines(turnObjLines) turnLandmark3DList = getTurnLandmark3D(turnVLines, originLandmark3DList) '''5. turn 3D landmark --> turn 2D landmark''' turnLandmark2D = [] for turnLandmark3D in turnLandmark3DList: turnLandmark2D.append(turnLandmark3D[2]) '''6. origin 2D landmarks''' originLandmark2D = [] for turnLandmark3D in turnLandmark3DList: originLandmark2D.append((turnLandmark3D[0][0], turnLandmark3D[0][1])) assert len(originLandmark2D) == len(turnLandmark2D) print len(turnLandmark2D)
def main2D_3D_2D_LR_MeshHair(): '''1. read obj and img''' objPath = '../../github/vrn-07231340/obj/trump-12.obj' objLines, vLines, fLines = readObj(objPath) imgPath = '../../github/vrn-07231340/examples/scaled/trump-12.jpg' img = cv2.imread(imgPath) '''2. 2D face landmark''' oriFaceLandmark2DList = getLandmark2D(img) '''3. create source hair region mesh''' gridSize = 15 rows, cols = img.shape[0], img.shape[1] src_rows = np.linspace(0, rows, gridSize) # 10 rows src_cols = np.linspace(0, cols, gridSize) # 20 columns src_rows, src_cols = np.meshgrid(src_rows, src_cols) src = np.dstack([src_cols.flat, src_rows.flat])[0] '''3.1 create head region ellipse''' '''!!!手动修改了椭圆中心点,因为川普本来就是侧脸!!!''' ellipseParam = { 'ellipseCenterX': oriFaceLandmark2DList[ELLIPSE_CENTER][1], 'ellipseCenterY': oriFaceLandmark2DList[ELLIPSE_CENTER][0] - 10, 'ellipseSemiX': 70, 'ellipseSemiY': 90, 'orientation': np.pi * 1.5 } rr, cc = ellipse_perimeter(ellipseParam['ellipseCenterY'], ellipseParam['ellipseCenterX'], ellipseParam['ellipseSemiY'], ellipseParam['ellipseSemiX'], orientation=ellipseParam['orientation']) '''3.2 mesh points in ellipse''' ellipseVerts = np.dstack([rr, cc])[0] '''3.3 add outer ellipse edge points''' edgePoints, outerEllipseVerts = addOuterEllipseEdgeLandmark( ellipseParam, img) # drawPointsOnImg(ellipseVerts, img, 'g', cover=True) # drawPointsOnImg(edgePoints, img, 'r') mask = points_in_poly(src, ellipseVerts) pointsInEllipseList = [] indexInEllipseList = [] for i, (s, m) in enumerate(zip(src, mask)): if m == True: pointsInEllipseList.append(s) indexInEllipseList.append(i) assert len(pointsInEllipseList) == len(indexInEllipseList) '''3.3 mesh points in face region''' faceVerts = [] for FACE_CONTOUR_INDICE in FACE_CONTOUR_INDICES: faceVerts.append([ oriFaceLandmark2DList[FACE_CONTOUR_INDICE][0], oriFaceLandmark2DList[FACE_CONTOUR_INDICE][1] ]) mask = points_in_poly(src, faceVerts) pointsInFaceList = [] indexInFaceList = [] for i, (s, m) in enumerate(zip(src, mask)): if m == True: pointsInFaceList.append(s) indexInFaceList.append(i) assert len(pointsInFaceList) == len(indexInFaceList) '''3.4 mesh points between (face region) and (ellipse)''' for i, ((pointInEllipseX, pointInEllipseY), indexInEllise) in enumerate( zip(pointsInEllipseList, indexInEllipseList)): for (pointInFaceX, pointInFaceY) in pointsInFaceList: if pointInEllipseX == pointInFaceX and pointInEllipseY == pointInFaceY: pointsInEllipseList[i] = np.array([-1, -1]) indexInEllipseList[i] = -1 pointsInHairList = [] for (pointInEllipseX, pointInEllipseY) in pointsInEllipseList: if pointInEllipseX != -1 and pointInEllipseY != -1: pointsInHairList.append([pointInEllipseX, pointInEllipseY]) indexInHairList = [x for x in indexInEllipseList if x != -1] assert len(pointsInHairList) == len(indexInHairList) '''3.5 mesh points above minY''' minY = minYInLandmark(oriFaceLandmark2DList) hairMeshPointsInEllipseList = [] hairMeshIndexInEllipseList = [] for (s, i) in zip(pointsInHairList, indexInHairList): srcY = s[1] srcX = s[0] if srcY < minY and srcY >= 0: hairMeshPointsInEllipseList.append((srcX, srcY)) hairMeshIndexInEllipseList.append(i) assert len(hairMeshPointsInEllipseList) == len(hairMeshIndexInEllipseList) # drawPointsOnImg(hairMeshPointsInEllipseList, img, 'r') '''4. 2D hair mesh points --> 3D hair mesh points''' '''4.1 3D hair landmark list: (x, y, z)''' # maxminDict = maxminXYZ(objLines) # minZ = float(maxminDict['maxZCoord'][2]) minZ = 60.0 hairLandmark3DList = [] for hairLandmark2D in hairMeshPointsInEllipseList: hairLandmark3D = (float(hairLandmark2D[0]), float(hairLandmark2D[1]), minZ) hairLandmark3DList.append(hairLandmark3D) '''4.2 hair landmark color list: (r, g, b)''' colorList = [] for hairLandmark2D in hairMeshPointsInEllipseList: r = img[int(hairLandmark2D[0]), int(hairLandmark2D[1]), 2] / 255.0 g = img[int(hairLandmark2D[0]), int(hairLandmark2D[1]), 1] / 255.0 b = img[int(hairLandmark2D[0]), int(hairLandmark2D[1]), 0] / 255.0 color = (r, g, b) colorList.append(color) '''4.3 3D vLines''' hairVLines = [] for (x, y, z), (r, g, b) in zip(hairLandmark3DList, colorList): hairVLine = 'v {} {} {} {} {} {}'.format(x, y, z, r, g, b) hairVLines.append(hairVLine) '''4.4 write vLines txt''' hairVLinesTxtPath = './hairVLines.txt' hairVLinesTxt = open(hairVLinesTxtPath, 'w') for hairVLine in hairVLines: hairVLinesTxt.write(hairVLine + '\n') hairVLinesTxt.close() '''5. nod hair''' nodAngle = 15 nodCenterMode = 'maxY' nodHairVLines = nodHair(objPath, nodAngle, nodCenterMode, hairVLines, hairVLinesTxtPath) '''6. 2D nod hair landmark''' nodHairLandmark2DList = [] for nodHairVLine in nodHairVLines: _, x, y, _, _, _, _ = nodHairVLine.split() nodHairLandmark2DList.append((float(x), float(y))) assert len(hairMeshPointsInEllipseList) == len(nodHairLandmark2DList) # print hairMeshPointsInEllipseList # print nodHairLandmark2DList return hairMeshPointsInEllipseList, nodHairLandmark2DList, edgePoints, ellipseVerts, outerEllipseVerts
def main2D_3D_2D_LR_Hair(): '''1. read origin obj and image''' objPath = '../../github/vrn-07231340/obj/trump-12.obj' objLines, vLines, fLines = readObj(objPath) imgPath = '../../github/vrn-07231340/examples/scaled/trump-12.jpg' img = cv2.imread(imgPath) imgHRPath = '../../github/vrn-07231340/examples/trump-12.jpg' imgHR = cv2.imread(imgHRPath) '''2. origin 2D landmark --> origin 3D landmark''' ''' BUT ONE 2D LANDMARK --> MORE THAN ONE 3D LANDMARK''' ''' HOW TO CHOOSE THE BEST ONE ?''' ''' OPTION 1: 选Z最大的3D landmark''' '''2.1 origin 2D landmark [x, y]''' originLandmark2DList = getLandmark2D(img) '''2.2 origin 3D landmark list [[[x, y], [(line1, vLine1), (line2, vLine2)]], ... ]''' originLandmark3DList = getLandmark3D(originLandmark2DList, vLines) '''2.3 OPTION 1: 选Z最大的3D landmark''' originLandmark3DList = optionChooseZMax(originLandmark3DList) '''3. nod''' nodAngle = 15 nodCenterMode = 'maxY' nodObjLines = nod(objPath, nodAngle, nodCenterMode) '''4. nod 3D landmark list''' '''nodLandmark3DList: [[x0, y0], [x0, y0, z0], [x1, y1], [x1, y1, z1], (line, vLine)]''' '''len(nodLandmark3DList): number of landmarks both on 2D and 3D''' '''[x0, y0]: original landmark on 2D img''' '''[x0, y0, z0]: original landmark on 3D model''' '''[x1, y1]: nod landmark on 2D img''' '''[x1, y1, z1]: nod landmark on 3D model''' '''(line, vLine): the line(th) nod vLine''' nodVLines = objLines2vLines(nodObjLines) nodLandmark3DList = getNodLandmark3D(nodVLines, originLandmark3DList) '''5. nod 3D landmark --> nod 2D landmark''' nodLandmark2D = [] for nodLandmark3D in nodLandmark3DList: nodLandmark2D.append(nodLandmark3D[2]) '''6. origin 2D landmarks''' originLandmark2D = [] for nodLandmark3D in nodLandmark3DList: originLandmark2D.append((nodLandmark3D[0][0], nodLandmark3D[0][1])) assert len(originLandmark2D) == len(nodLandmark2D) '''7. face 2D landmarks + hair 2D landmarks''' # hairLandmark2DList, nodHairLandmark2DList = main2D_3D_2D_ManuallySelectHair() hairLandmark2DList, nodHairLandmark2DList, edgePoints, ellipseVerts, outerEllipseVerts = main2D_3D_2D_LR_MeshHair( ) assert len(hairLandmark2DList) == len(nodHairLandmark2DList) '''7.1 origin face 2D landmarks + origin hair 2D landmarks''' originLandmark2D = originLandmark2D + hairLandmark2DList '''7.2 nod face 2D landmarks + nod hair 2D landmarks''' nodLandmark2D = nodLandmark2D + nodHairLandmark2DList '''7.3 save origin and nod 2D landmarks txt''' # originLandmark2DTxtPath = './originLandmark2D.txt' # originLandmark2DTxt = open(originLandmark2DTxtPath, 'w') # for i in originLandmark2D: # originLandmark2DTxt.write('{} {}\n'.format(i[0], i[1])) # originLandmark2DTxt.close() # nodLandmark2DTxtPath = './nodLandmark2D.txt' # nodLandmark2DTxt = open(nodLandmark2DTxtPath, 'w') # for i in nodLandmark2D: # nodLandmark2DTxt.write('{} {}\n'.format(i[0], i[1])) # nodLandmark2DTxt.close() '''8. add edge points''' '''8.1 OPTION1: 8 IMAGE EDGE POINTS''' ''' 如果在加头发关键点之前加8个edge points,第7个edge point(图像最上面那个)会被往下warp一点''' ''' 如果在加头发关键点之后加8个edge points,第7个edge point经过warp后仍在原处,形成一个小三角''' # originLandmark2D = addEdgeLandmark(originLandmark2D, img) # nodLandmark2D = addEdgeLandmark(nodLandmark2D, img) '''8.2 OPTION2: OUTER ELLIPSE''' # a = copy.copy(originLandmark2D) originLandmark2D = originLandmark2D + edgePoints nodLandmark2D = nodLandmark2D + edgePoints # drawPointsOnImg(originLandmark2D, img, 'b') # drawPointsOnImg(a, img, 'r') # drawPointsOnImg(nodLandmark2D, img, 'g') # drawPointsOnImg(ellipseVerts, img, 'g', cover=True) # drawPointsOnImg(outerEllipseVerts, img, 'r', cover=True) # drawPointsOnImg(originLandmark2D, img, 'g', cover=True) # drawPointsOnImg(edgePoints, img, 'b', radius=3) # drawPointsOnImg(nodLandmark2D, img, 'r') '''9. delaunay triangle for [origin landmarks (face + hair)] + (eight edge points)''' # triList = generateTriList(originLandmark2D, img) # '''9.1 save tri txt''' # triTxtPath = './nodTri.txt' # triTxt = open(triTxtPath, 'w') # for i in triList: # triTxt.write('{}\n'.format(i)) # triTxt.close() '''10. warp LR''' # imgMorph = morph_modify_for_2D3D2D_low_resolution( # originLandmark2D, nodLandmark2D, img, triList) # imshow('imgMorph', imgMorph) # cv2.imwrite('./nod.jpg', imgMorph) # return originLandmark2D, nodLandmark2D, triList '''LR to HR''' originLandmark2DHR = LR2HR(img, imgHR, originLandmark2D) for (x, y), (xmin, ymin) in zip(originLandmark2DHR, originLandmark2D): if float(x) >= 640 or float(y) >= 640: print(int(xmin), int(ymin)) cv2.circle(img, (int(xmin), int(ymin)), 3, (0, 255, 0), -1) imshow('img', img) nodLandmark2DHR = LR2HR(img, imgHR, nodLandmark2D) '''8. add 8 egde points''' # nodLandmark2DHR = addEdgeLandmark(nodLandmark2DHR, imgHR) # originLandmark2DHR = addEdgeLandmark(originLandmark2DHR, imgHR) # drawPointsOnImg(originLandmark2DHR, imgHR, 'g') # drawPointsOnImg(nodLandmark2DHR, imgHR, 'r') '''8.1 remove >= 640''' originLandmark2DHR = [(i[0], i[1]) for i in originLandmark2DHR if i[0] < imgHR.shape[0] and i[1] < imgHR.shape[1]] nodLandmark2DHR = [(i[0], i[1]) for i in nodLandmark2DHR if i[0] < imgHR.shape[0] and i[1] < imgHR.shape[1]] '''9. delaunay triangle for [origin landmarks (face + hair)] + (edge points)''' triList = generateTriList(originLandmark2DHR, imgHR) '''9.1 save tri txt''' triTxtPath = './nodTri.txt' triTxt = open(triTxtPath, 'w') for i in triList: triTxt.write('{}\n'.format(i)) triTxt.close() '''10. warp HR''' imgMorph = morph_modify_for_2D3D2D_low_resolution(originLandmark2DHR, nodLandmark2DHR, imgHR, triList) # imshow('imgMorph', imgMorph) # cv2.imwrite('./nod.jpg', imgMorph) return originLandmark2DHR, nodLandmark2DHR, triList
def mainMeshWarp(): '''1. read image and load new landmarks (nod)''' image = cv2.imread('../../github/vrn-07231340/examples/trump-12.jpg') imageTmp = copy.deepcopy(image) rows, cols = image.shape[0], image.shape[1] originLandmark2D = getLandmark2D(image) targetLandmark2DPath = '../../data/talkingphoto/IMG_2294/IMG_2294_26.png.txt' targetLandmark2D = readPoints(targetLandmark2DPath)[:68] triTxtPath = './meshTri_640_640(trump-12).txt' '''2. 新建source mesh''' gridSize = 50 src_rows = np.linspace(0, rows, gridSize) # 10 行 src_cols = np.linspace(0, cols, gridSize) # 20 列 src_rows, src_cols = np.meshgrid(src_rows, src_cols) src = np.dstack([src_cols.flat, src_rows.flat])[0] '''!!! IMPORTANT DEEPCOPY!!!''' dst = copy.deepcopy(src) '''3. SKIMAGE画一个椭圆, 并且得到椭圆内所有的网格点''' rr, cc = ellipse_perimeter(originLandmark2D[ELLIPSE_CENTER][1], originLandmark2D[ELLIPSE_CENTER][0], 200, 260, orientation=30) tmp = rr rr = cc cc = tmp # print rr.shape, cc.shape ellipseVerts = np.dstack([rr, cc])[0] # 椭圆边长上所有点 # print ellipseVerts mask = points_in_poly(src, ellipseVerts) pointsInEllipseList = [] indexInEllipseList = [] for i, (s, m) in enumerate(zip(src, mask)): if m == True: pointsInEllipseList.append(s) indexInEllipseList.append(i) # print len(indexInEllipseList) # x=pointsInEllipseList[i][1], y=pointsInEllipseList[i][0] pointsInEllipseArray = np.asarray(pointsInEllipseList) '''swap collums of pointsInEllipseList''' # x=pointsInEllipseList[i][0], y=pointsInEllipseList[i][1] # pointsInEllipseArray[:, [0, 1]] = pointsInEllipseArray[:, [1, 0]] # image[cc, rr] = 255 # draw ellipse perimeter on image drawPointsOnImg(pointsInEllipseArray, imageTmp, 'r') '''4. compute delta (68) of each new landmark X(Y) and old landmark X(Y)''' deltaAllLandmarksList = [] for i in range(len(targetLandmark2D)): deltaAllLandmarksList.append( [targetLandmark2D[i][0]-originLandmark2D[i][0], targetLandmark2D[i][1]-originLandmark2D[i][1]]) # deltaAllLandmarksArray = np.asarray(deltaAllLandmarksList) # print deltaAllLandmarksList '''5. compute delta of each point in ellipse''' radius = 5*rows/float(gridSize) targetPointsInEllipseList = [] for meshPoint in pointsInEllipseArray: _, _, landmarksInSmallGridArray, deltaInSmallGridArray = getLandmarksInSmallGrid( meshPoint, originLandmark2D, deltaAllLandmarksList, radius=radius) '''画smallGrid以及其内的所有landmarks的delta''' # imageTmp = copy.deepcopy(image) # cv2.circle(imageTmp, (int(meshPoint[0]), # int(meshPoint[1])), int(radius), (0, 255, 0), 2) # if landmarksInSmallGridArray.shape[0] != 0: # for (deltaX, deltaY), (landmarkX, landmarkY) in zip(deltaInSmallGridArray, landmarksInSmallGridArray): # cv2.line(imageTmp, (int(landmarkX), int(landmarkY)), (int( # landmarkX+deltaX), int(landmarkY+deltaY)), (255, 0, 0), 2) # cv2.circle(imageTmp, (int(landmarkX+deltaX), # int(landmarkY+deltaY)), 2, (0, 0, 255), -1) # cv2.imshow('imgTmp', imageTmp) # cv2.waitKey(50) deltaXOfMeshPoint = 0 deltaYOfMeshPoint = 0 if deltaInSmallGridArray.shape[0] != 0: for i, (deltaInSmallGrid, landmarkInSmallGrid) in enumerate(zip(deltaInSmallGridArray, landmarksInSmallGridArray)): deltaInSmallGridX = deltaInSmallGrid[0] deltaInSmallGridY = deltaInSmallGrid[1] deltaXOfMeshPoint = deltaXOfMeshPoint + deltaInSmallGridX / \ twoPointsDistance(meshPoint, landmarkInSmallGrid) deltaYOfMeshPoint = deltaYOfMeshPoint + deltaInSmallGridY / \ twoPointsDistance(meshPoint, landmarkInSmallGrid) targetMeshPointX = meshPoint[0]+deltaXOfMeshPoint targetMeshPointY = meshPoint[1]+deltaYOfMeshPoint '''画每一个meshPoint的起点和终点''' # cv2.line(imageTmp, (int(meshPoint[0]), int(meshPoint[1])), # (int(targetMeshPointX), int(targetMeshPointY)), (0, 255, 0), 2) # cv2.circle(imageTmp, (int(meshPoint[0]), int(meshPoint[1])), # 3, (0, 255, 0), -1) # cv2.circle(imageTmp, (int(targetMeshPointX), int(targetMeshPointY)), # 2, (0, 0, 255), -1) # cv2.imshow('imageTmp', imageTmp) # cv2.waitKey(0) targetPointsInEllipseList.append([targetMeshPointX, targetMeshPointY]) targetPointsInEllipseArray = np.asarray(targetPointsInEllipseList) # drawPointsOnImg(pointsInEllipseArray, imageTmp, 'b') # drawPointsOnImg(targetPointsInEllipseArray, imageTmp, 'r') '''6. compute final target mesh''' # print targetPointsInEllipseArray.shape drawPointsOnImg(targetPointsInEllipseArray, imageTmp, 'b') # targetPointsInEllipseArray[:, [0, 1] # ] = targetPointsInEllipseArray[:, [1, 0]] dst[indexInEllipseList] = targetPointsInEllipseArray '''draw src and dst mesh on image''' # drawPointsOnImg(src, imageTmp, 'g') # drawPointsOnImg(dst, imageTmp, 'b') cv2.imwrite('./tmp.png', imageTmp) '''7. PiecewiseAffineTransform from skimage''' # tform = PiecewiseAffineTransform() # tform.estimate(dst, src) # out = warp(image, tform) '''8. imshow and imwrite''' # imshow('out_pat', out) # cv2.imwrite('./out_pat.png', out*255) # cv2.imwrite('./ori.jpg', image) # # fig, ax = plt.subplots() # # ax.imshow(out) # # ax.scatter(pointsInEllipseArray[:, 0], pointsInEllipseArray[:, 1], # # marker='+', color='b', s=5) # # ax.plot(tform.inverse(src)[:, 0], tform.inverse(src)[:, 1], '.r') # # plt.show() '''7. mesh warp wirtten by meself''' '''draw dst triangle''' imgTmp = copy.copy(image) triTxt = open(triTxtPath, 'r') # for line in triTxt: # a, b, c = line.split() # a = int(a) # b = int(b) # c = int(c) # z1 = (int(dst[a][0]), int(dst[a][1])) # z2 = (int(dst[b][0]), int(dst[b][1])) # z3 = (int(dst[c][0]), int(dst[c][1])) # cv2.line(imgTmp, z1, z2, (255, 255, 255), 1) # cv2.line(imgTmp, z2, z3, (255, 255, 255), 1) # cv2.line(imgTmp, z3, z1, (255, 255, 255), 1) # cv2.imshow('tmp', imgTmp) # cv2.waitKey(1) imgMorph = morph_modify_for_meshwarp(src, dst, image, triTxtPath) '''8. imshow and imwrite''' imshow('imgMorph', imgMorph) cv2.imwrite('./out_wo.png', imgMorph)
def main2D_3D_2D_LR_Hair(): '''1. read origin obj and image''' objPath = '../../github/vrn-07231340/obj/trump-12.obj' objLines, vLines, fLines = readObj(objPath) imgPath = '../../github/vrn-07231340/examples/scaled/trump-12.jpg' img = cv2.imread(imgPath) '''2. origin 2D landmark --> origin 3D landmark''' ''' BUT ONE 2D LANDMARK --> MORE THAN ONE 3D LANDMARK''' ''' HOW TO CHOOSE THE BEST ONE ?''' ''' OPTION 1: 选Z最大的3D landmark''' '''2.1 origin 2D landmark [x, y]''' originLandmark2DList = getLandmark2D(img) '''2.2 origin 3D landmark list [[[x, y], [(line1, vLine1), (line2, vLine2)]], ... ]''' originLandmark3DList = getLandmark3D(originLandmark2DList, vLines) '''2.3 OPTION 1: 选Z最大的3D landmark''' originLandmark3DList = optionChooseZMax(originLandmark3DList) '''3. nod''' nodAngle = 15 nodCenterMode = 'maxY' nodObjLines = nod(objPath, nodAngle, nodCenterMode) '''4. nod 3D landmark list''' '''nodLandmark3DList: [[x0, y0], [x0, y0, z0], [x1, y1], [x1, y1, z1], (line, vLine)]''' '''len(nodLandmark3DList): number of landmarks both on 2D and 3D''' '''[x0, y0]: original landmark on 2D img''' '''[x0, y0, z0]: original landmark on 3D model''' '''[x1, y1]: nod landmark on 2D img''' '''[x1, y1, z1]: nod landmark on 3D model''' '''(line, vLine): the line(th) nod vLine''' nodVLines = objLines2vLines(nodObjLines) nodLandmark3DList = getNodLandmark3D(nodVLines, originLandmark3DList) '''5. nod 3D landmark --> nod 2D landmark + 8 edge points''' nodLandmark2D = [] for nodLandmark3D in nodLandmark3DList: nodLandmark2D.append(nodLandmark3D[2]) nodLandmark2D = addEdgeLandmark(nodLandmark2D, img) '''6. origin 2D landmarks + 8 edge points''' originLandmark2D = [] for nodLandmark3D in nodLandmark3DList: originLandmark2D.append((nodLandmark3D[0][0], nodLandmark3D[0][1])) originLandmark2D = addEdgeLandmark(originLandmark2D, img) assert len(originLandmark2D) == len(nodLandmark2D) '''7. face 2D landmarks + hair 2D landmarks''' # hairLandmark2DList, nodHairLandmark2DList = main2D_3D_2D_ManuallySelectHair() hairLandmark2DList, nodHairLandmark2DList = main2D_3D_2D_LR_MeshHair() '''7.1 origin face 2D landmarks + origin hair 2D landmarks''' originLandmark2D = originLandmark2D+hairLandmark2DList # drawPointsOnImg(originLandmark2D, img, 'g') '''7.2 nod face 2D landmarks + nod hair 2D landmarks''' nodLandmark2D = nodLandmark2D+nodHairLandmark2DList # drawPointsOnImg(nodLandmark2D, img, 'r') '''7.3 save origin and nod 2D landmarks txt''' originLandmark2DTxtPath = './originLandmark2D.txt' originLandmark2DTxt = open(originLandmark2DTxtPath, 'w') for i in originLandmark2D: originLandmark2DTxt.write('{} {}\n'.format(i[0], i[1])) originLandmark2DTxt.close() nodLandmark2DTxtPath = './nodLandmark2D.txt' nodLandmark2DTxt = open(nodLandmark2DTxtPath, 'w') for i in nodLandmark2D: nodLandmark2DTxt.write('{} {}\n'.format(i[0], i[1])) nodLandmark2DTxt.close() '''8. delaunay triangle for [origin landmarks (face + hair)] + (eight edge points)''' triList = generateTriList(originLandmark2D, img) # print triList '''8.1 save tri txt''' triTxtPath = './nodTri.txt' triTxt = open(triTxtPath, 'w') for i in triList: triTxt.write('{}\n'.format(i)) triTxt.close() '''9. warp''' imgMorph = morph_modify_for_2D3D2D_low_resolution( originLandmark2D, nodLandmark2D, img, triList) # imshow('imgMorph', imgMorph) cv2.imwrite('./nod.jpg', imgMorph)
def mainMeshWarp(): '''1. read image and load new landmarks (nod)''' tmp = cv2.imread('./tmp.png') image = cv2.imread('../../github/vrn-07231340/examples/trump-12.jpg') rows, cols = image.shape[0], image.shape[1] originLandmark2D = getLandmark2D(image) targetLandmark2DPath = '../../data/talkingphoto/IMG_2294/IMG_2294_26.png.txt' targetLandmark2D = readPoints(targetLandmark2DPath)[:68] '''2. 新建source mesh''' gridSize = 50 src_rows = np.linspace(0, rows, 10) # 10 行 src_cols = np.linspace(0, cols, gridSize) # 20 列 src_rows, src_cols = np.meshgrid(src_rows, src_cols) src = np.dstack([src_cols.flat, src_rows.flat])[0] '''!!! IMPORTANT DEEPCOPY!!!''' dst = copy.deepcopy(src) '''3. SKIMAGE画一个椭圆, 并且得到椭圆内所有的网格点''' rr, cc = ellipse_perimeter(originLandmark2D[NOSE_CENTER][1], originLandmark2D[NOSE_CENTER][0], 180, 300, orientation=30) # print rr.shape, cc.shape ellipseVerts = np.dstack([rr, cc])[0] # 椭圆边长上所有点 mask = points_in_poly(src, ellipseVerts) pointsInEllipseList = [] indexInEllipseList = [] for i, (s, m) in enumerate(zip(src, mask)): if m == True: pointsInEllipseList.append(s) indexInEllipseList.append(i) # print len(indexInEllipseList) # x=pointsInEllipseList[i][1], y=pointsInEllipseList[i][0] pointsInEllipseArray = np.asarray(pointsInEllipseList) '''swap collums of pointsInEllipseList''' # x=pointsInEllipseList[i][0], y=pointsInEllipseList[i][1] # pointsInEllipseArray[:, [0, 1]] = pointsInEllipseArray[:, [1, 0]] image[rr, cc] = 255 # draw ellipse perimeter on image # drawPointsOnImg(pointsInEllipseArray, image, 'r') '''4. compute delta (68) of each new landmark X(Y) and old landmark X(Y)''' deltaAllLandmarksList = [] for i in range(len(targetLandmark2D)): deltaAllLandmarksList.append( [targetLandmark2D[i][0]-originLandmark2D[i][0], targetLandmark2D[i][1]-originLandmark2D[i][1]]) # deltaAllLandmarksArray = np.asarray(deltaAllLandmarksList) # print deltaAllLandmarksList '''5. compute delta of each point in ellipse''' radius = 5*rows/float(gridSize) targetPointsInEllipseList = [] for meshPoint in pointsInEllipseArray: _, _, landmarksInSmallGridArray, deltaInSmallGridArray = getLandmarksInSmallGrid( meshPoint, originLandmark2D, deltaAllLandmarksList, radius=radius) '''画smallGrid以及其内的所有landmarks的delta''' # imageTmp = copy.deepcopy(image) # cv2.circle(imageTmp, (int(meshPoint[0]), # int(meshPoint[1])), int(radius), (0, 255, 0), 2) # if landmarksInSmallGridArray.shape[0] != 0: # for (deltaX, deltaY), (landmarkX, landmarkY) in zip(deltaInSmallGridArray, landmarksInSmallGridArray): # cv2.line(imageTmp, (int(landmarkX), int(landmarkY)), (int( # landmarkX+deltaX), int(landmarkY+deltaY)), (255, 0, 0), 2) # cv2.circle(imageTmp, (int(landmarkX+deltaX), # int(landmarkY+deltaY)), 2, (0, 0, 255), -1) # cv2.imshow('imgTmp', imageTmp) # cv2.waitKey(0) deltaXOfMeshPoint = 0 deltaYOfMeshPoint = 0 if deltaInSmallGridArray.shape[0] != 0: for i, (deltaInSmallGrid, landmarkInSmallGrid) in enumerate(zip(deltaInSmallGridArray, landmarksInSmallGridArray)): deltaInSmallGridX = deltaInSmallGrid[0] deltaInSmallGridY = deltaInSmallGrid[1] deltaXOfMeshPoint = deltaXOfMeshPoint + deltaInSmallGridX / \ twoPointsDistance(meshPoint, landmarkInSmallGrid) deltaYOfMeshPoint = deltaYOfMeshPoint + deltaInSmallGridY / \ twoPointsDistance(meshPoint, landmarkInSmallGrid) targetMeshPointX = meshPoint[0]+deltaXOfMeshPoint targetMeshPointY = meshPoint[1]+deltaYOfMeshPoint '''画每一个meshPoint的起点和终点''' # imageTmp = copy.deepcopy(tmp) # cv2.line(imageTmp, (int(meshPoint[0]), int(meshPoint[1])), # (int(targetMeshPointX), int(targetMeshPointY)), (0, 255, 0), 2) # cv2.circle(imageTmp, (int(meshPoint[0]), int(meshPoint[1])), # 3, (0, 255, 0), -1) # cv2.circle(imageTmp, (int(targetMeshPointX), int(targetMeshPointY)), # 2, (0, 0, 255), -1) # cv2.imshow('imageTmp', imageTmp) # cv2.waitKey(0) targetPointsInEllipseList.append([targetMeshPointX, targetMeshPointY]) targetPointsInEllipseArray = np.asarray(targetPointsInEllipseList) # drawPointsOnImg(pointsInEllipseArray, image, 'b') # drawPointsOnImg(targetPointsInEllipseArray, image, 'r') '''6. compute final target mesh''' print targetPointsInEllipseArray.shape drawPointsOnImg(targetPointsInEllipseArray, image, 'g') # targetPointsInEllipseArray[:, [0, 1] # ] = targetPointsInEllipseArray[:, [1, 0]] # dst[indexInEllipseList] = targetPointsInEllipseArray # dst[:, [0, 1]] = dst[:, [1, 0]] '''draw src and dst mesh on image''' drawPointsOnImg(src, image, 'b') drawPointsOnImg(dst, image, 'r') cv2.imwrite('./tmp.png', image) '''7. PiecewiseAffineTransform''' tform = PiecewiseAffineTransform() # for i, s, d in zip(indexInEllipseList, src, dst): # print src[i], dst[i] tform.estimate(src, dst) # image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB) out = warp(image, tform) cv2.imwrite('./out.jpg', out*255) cv2.imwrite('./ori.jpg', image)
def main2D_3D_2D_HR_NodHair(): '''1. read origin obj and image''' objPath = '../../github/vrn-07231340/obj/trump-12.obj' objLines, vLines, fLines = readObj(objPath) imgLRPath = '../../github/vrn-07231340/examples/scaled/trump-12.jpg' imgLR = cv2.imread(imgLRPath) imgHRPath = '../../github/vrn-07231340/examples/trump-12.jpg' imgHR = cv2.imread(imgHRPath) '''2. origin LR 2D landmark --> origin 3D landmark''' '''2.1 origin LR 2D landmark list [x, y]''' originLandmark2DLRList = getLandmark2D(imgLR) '''2.2 origin 3D landmark list [[[x, y], [(line1, vLine1), (line2, vLine2)]], ... ]''' originLandmark3DList = getLandmark3D(originLandmark2DLRList, vLines) '''2.3 OPTION 1: 相同(x, y)的情况下,选z最大的3D landmark''' originLandmark3DList = optionChooseZMax(originLandmark3DList) '''3. 3D model nod''' nodAngle = 10 nodCenterMode = 'maxY' nodObjLines = nod(objPath, nodAngle, nodCenterMode) '''4. nod 3D landmark list''' nodVLines = objLines2vLines(nodObjLines) nodLandmark3DList = getNodLandmark3D(nodVLines, originLandmark3DList) '''5. nod 3D landmark --> nod LR 2D landmark''' nodLandmark2DLR = [i[2] for i in nodLandmark3DList] '''6. origin 2D landmark''' originLandmark2DLR = [(i[0][0], i[0][1]) for i in nodLandmark3DList] assert len(originLandmark2DLR) == len(nodLandmark2DLR) '''7. LR --> HR: origin and nod face landmark''' originLandmark2DHR = lr2hr(imgLR, imgHR, originLandmark2DLR) nodLandmark2DHR = lr2hr(imgLR, imgHR, nodLandmark2DLR) '''8. HR: collar detection''' _, collarPoint = collarDetection(imgHR, minSize=1200, offset=15) '''9. HR: hair detection''' originHairPoints = hairDetection(imgHR, minSize=20000) '''10. HR: nod hair points''' '''??? nod center ???''' nodAngleHair = 18 # drawPointsOnImg(originHairPoints, imgHR, 'r', radius=3) # nodHairPoints = nodHairHR( # objLines, nodAngleHair, nodCenterMode, originHairPoints, minZ=100.0) nodHairPoints = nodHairHR_m( nodAngleHair, collarPoint, originHairPoints, minZ=100.0) # for (x, y), (i, j) in zip(nodHairPoints, nodHairPoints_m): # if x == i and y == j: # pass # else: # print 'yes' '''10. HR: all origin points and nod points''' originPoints = originLandmark2DHR+originHairPoints nodPoints = nodLandmark2DHR+nodHairPoints assert len(originPoints) == len(nodPoints) # drawPointsOnImg(originPoints, imgHR, 'g', radius=2) '''11. inner ellipse''' left = min(originPoints, key=lambda x: x[0])[0] right = max(originPoints, key=lambda x: x[0])[0] up = min(originPoints, key=lambda x: x[1])[1] bottom = max(originPoints, key=lambda x: x[1])[1] innerEllipseCenterX = int((left+right)/2.0) innerEllipseCenterY = int((up+bottom)/2.0) innerEllipseSemiX = int((right-left)/2.0) innerEllipseSemiY = int((bottom-up)/2.0) rr, cc = ellipse_perimeter(innerEllipseCenterX, innerEllipseCenterY, innerEllipseSemiY, innerEllipseSemiX, orientation=np.pi*1.5) innerEllipseVerts = np.dstack([rr, cc])[0] '''12. outer ellipse''' '''12.1 ratio = outer ellipse / inner ellipse''' ''' 椭圆心和衣领的线段 和 椭圆的交点''' minDistance = np.inf for pt in innerEllipseVerts: distance = twoPointsDistance(pt, collarPoint) if distance < minDistance: minDistance = distance ratio = twoPointsDistance((innerEllipseCenterX, innerEllipseCenterY), collarPoint) / \ twoPointsDistance( (innerEllipseCenterX, innerEllipseCenterY), pt) '''12.2 outer ellipse''' outerEllipseSemiX = int(ratio*innerEllipseSemiX) outerEllipseSemiY = int(ratio*innerEllipseSemiY) rr, cc = ellipse_perimeter(innerEllipseCenterX, innerEllipseCenterY, outerEllipseSemiY, outerEllipseSemiX, orientation=np.pi*1.5) outerEllipseVerts = np.dstack([rr, cc])[0] '''13. edge points on outer ellipse''' edgePoints = edgePointsOnOuterEllipse(outerEllipseVerts) '''14. final origin and nod HR points ''' originPointsFinal = originPoints+edgePoints nodPointsFinal = nodPoints+edgePoints assert len(originPointsFinal) == len(nodPointsFinal) drawPointsOnImg(originPointsFinal, imgHR, 'b', radius=2) drawPointsOnImg(originPoints, imgHR, 'r', radius=2) drawPointsOnImg(nodPointsFinal, imgHR, 'g', radius=2) '''15. tri.txt''' triList = generateTriList(originPointsFinal, imgHR, triTxtPath='./nodTri.txt') '''16. warp''' imgMorph = morph_modify_for_2D3D2D_low_resolution( originPointsFinal, nodPointsFinal, imgHR, triList) imshow('imgMorph', imgMorph) cv2.imwrite('./imgMorph_{}2.png'.format(nodAngleHair), imgMorph) return originPointsFinal, nodPointsFinal, triList