def main(args):
image_paths = read_list(args.image_list)
for path in image_paths:
im = open_image(path)
# resize for memory
width, height = im.size
if height > 800:
im = im.resize((int(800 * width / height), 800))
width, height = im.size
# use 2D-FAN detect landmarks
fa = FaceAlignment(LandmarksType._2D,
enable_cuda=True,
flip_input=False,
use_cnn_face_detector=True)
try:
landmarks = fa.get_landmarks(np.array(im))[-1]
landmarks[:, 1] = height - landmarks[:, 1]
except:
continue
# generate a contour curve with contour points
hull = ConvexHull(landmarks)
# draw landmarks
lm = np.array(im)
for i in range(landmarks.shape[0]):
rr, cc = draw.circle(height - landmarks[i, 1].astype('int32'),
landmarks[i, 0].astype('int32'), 5)
lm[rr, cc, :] = np.array((255, 0, 0))
# create mask
mask = np.zeros((height, width))
rr, cc = draw.polygon(height - landmarks[hull.vertices, 1],
landmarks[hull.vertices, 0], mask.shape)
mask[rr, cc] = 1
save = True if args.save == 'True' else False
path = path[:-1] if path[:-1] == '/' else path
image_name = path[path.rindex('/') + 1:-4] + '_contour_nocrf.png'
show_result(lm,
mask,
np.tile((mask != 0)[:, :, np.newaxis], (1, 1, 3)) * im,
save=save,
filename='images/' + image_name)
# add CRF
#prob = np.concatenate(((1-mask)[np.newaxis,:,:]*0.9 + mask[np.newaxis, :, :]*0.1, mask[np.newaxis, :, :]*0.9 + (1-mask)[np.newaxis, :, :]*0.1), axis=0)
prob = ndimage.gaussian_filter(mask * 1.0, sigma=5)
prob = np.concatenate(
((1 - prob)[np.newaxis, :, :], prob[np.newaxis, :, :]), axis=0)
map = CRF(prob, np.array(im))
image_name = path[path.rindex('/') + 1:-4] + '_contour_crf.png'
show_result(im,
map,
np.tile((map != 0)[:, :, np.newaxis], (1, 1, 3)) * im,
save=save,
filename='images/' + image_name)
import numpy as np
import os
import cv2
from FaceAligner import FaceAlign
from face_alignment.api import FaceAlignment, LandmarksType
source_dataset_path = '../datasets/fer2013/'
cant_crop_path = '../datasets/fer2013_HE_crop/cant_crop/'
more_face_path = '../datasets/fer2013_HE_crop/more_face/'
Coor_2D = FaceAlignment(LandmarksType._2D, flip_input = False)
Rotate = FaceAlign()
def HistEqualize(image_array):
equ = cv2.equalizeHist(image_array)
return equ
def cut_range(landmarks):
a = landmarks.max(axis = 0)
# print(a)
b = landmarks.min(axis = 0)
# print(b)
low_max = np.clip(a, 0, 100)
low_min = np.clip(b, 0, 100)
# width = low_max[0] - low_min[0]
# height = low_max[1] - low_min[1]
# x_min = low_min[0]
# y_min =
return low_min[0], low_min[1], low_max[0], low_max[1]
def main(args):
caffe.set_mode_gpu()
caffe.set_device(0)
# Load both networks
#net1 = caffe.Net('model/net_landmarks.prototxt', \
# 'model/params_landmarks.caffemodel', caffe.TEST)
net2 = caffe.Net('model/net_segmentation.prototxt', \
'model/params_segmentation.caffemodel', caffe.TEST)
palette = get_palette()
# We have a Gaussian to recover the output slightly - better results
f = scipy.io.loadmat('gaus.mat')['f']
# load image names
image_paths = read_list(args.image_list)
# segment and measure performance
for path in image_paths:
if path[-3:] == 'jpg' or path[-3:] == 'png':
imi = open_image(path)
# resize for memory
width, height = imi.size
if height > 800:
imi = imi.resize((int(800*width/height), 800))
else:
continue
# use 2D-FAN detect landmarks
fa = FaceAlignment(LandmarksType._2D, enable_cuda=True,
flip_input=False, use_cnn_face_detector=True)
try:
landmarks = fa.get_landmarks(np.array(imi))[-1]
landmarks = landmarks.astype('uint16')
except:
continue
if args.crop == 'middle':
imi, landmarks = crop_image_middle(landmarks, imi)
elif args.crop == 'min':
imi, landmarks = crop_image_min(landmarks, imi)
landmarks[:,0], landmarks[:,1] = landmarks[:,1].copy(), landmarks[:,0].copy()
# prepare the image, limit image size for memory
width, height = imi.size
if width > height:
if width > 450:
imi = imi.resize((450, int(450 * height/width)))
landmarks[:,0] = landmarks[:,0] * 450.0 / width
landmarks[:,1] = landmarks[:,1] * 450.0 / width
#elif height < 300:
# imi = imi.resize((int(300 * width/height), 300))
else:
if height > 450:
imi = imi.resize((int(450 * width/height), 450))
landmarks[:,0] = landmarks[:,0] * 450.0 / height
landmarks[:,1] = landmarks[:,1] * 450.0 / height
#elif width < 300:
# imi = imi.resize((300, int(300 * height/width)))
width, height = imi.size
im = np.array(imi, dtype=np.float32)
if len(im.shape) == 2:
im = np.reshape(im, im.shape+(1,))
im = np.concatenate((im,im,im), axis=2)
im = im[:,:,::-1] # RGB to BGR
# trained with different means (accidently)
segIm = im - np.array((87.86,101.92,133.01))
segIm = segIm.transpose((2,0,1))
# Do some recovery of the points
C = np.zeros((landmarks.shape[0], height, width), 'uint8') # cleaned up heatmaps
C = np.pad(C, ((0,0), (120,120), (120,120)), 'constant')
for k in range(0,68):
C[k,landmarks[k,0]+120-100:landmarks[k,0]+120+101,landmarks[k,1]+120-100:landmarks[k,1]+120+101] = f
C = C[:,120:-120,120:-120] * 0.5
# Forward through the segmentation network
D = np.concatenate((segIm, C))
net2.blobs['data'].reshape(1, *D.shape)
net2.blobs['data'].data[0,:,:,:] = D
net2.forward()
mask = net2.blobs['score'].data[0].argmax(axis=0)
S = Image.fromarray(mask.astype(np.uint8))
S.putpalette(palette)
print 'close figure to process next image'
# transfer score to probability with softmax for later unary term
score = net2.blobs['score'].data[0]
prob = np.exp(score) / np.sum(np.exp(score), 0) # (nlabels, height, width)
#prob_max = np.max(prob, 0) # (0.28, 1)
# CRF
map = CRF(prob, im) # final label
# show result
save = True if args.save == 'True' else False
path = path[:-1] if path[-1] == '/' else path
image_name = path[path.rindex('/')+1:-4] + '_part_nocrf_' + args.crop + '.png'
show_result(imi, mask, np.tile((mask!=0)[:,:,np.newaxis], (1,1,3)) * imi,
save=save, filename='images/'+image_name)
image_name = path[path.rindex('/')+1:-4] + '_part_crf_' + args.crop + '.png'
show_result(imi, map, np.tile((map!=0)[:,:,np.newaxis], (1,1,3)) * imi,
save=save, filename='images/'+image_name)
def main(args):
image_paths = read_list(args.image_list)
# init
caffe.set_device(0)
caffe.set_mode_gpu()
# load net
net = caffe.Net(args.prototxt, args.caffemodel, caffe.TEST)
for path in image_paths:
# load image, switch to BGR, subtract mean, and make dims C x H x W for Caffe
if path[-3:] == 'jpg' or path[-3:] == 'png':
imi = open_image(path)
# resize for memory
width, height = imi.size
if height > 800:
imi = imi.resize((int(800 * width / height), 800))
else:
continue
# use 2D-FAN detect landmarks
fa = FaceAlignment(LandmarksType._2D,
enable_cuda=True,
flip_input=False,
use_cnn_face_detector=True)
try:
landmarks = fa.get_landmarks(np.array(imi))[-1]
landmarks = landmarks.astype('uint16')
except:
continue
if args.crop == 'middle':
imi, landmarks = crop_image_middle(landmarks, imi)
elif args.crop == 'min':
imi, landmarks = crop_image_min(landmarks, imi)
if '300' in args.prototxt:
imi = imi.resize((300, 300))
else:
imi = imi.resize((500, 500))
im = np.array(imi, dtype=np.float32)
im = im[:, :, ::-1]
im -= np.array((104.00698793, 116.66876762, 122.67891434))
im = im.transpose((2, 0, 1))
# shape for input (data blob is N x C x H x W), set data
net.blobs['data'].reshape(1, *im.shape)
net.blobs['data'].data[...] = im
# run net and take argmax for prediction
net.forward()
mask = net.blobs['score'].data[0].argmax(axis=0)
im_seg = imi * np.tile((mask != 0)[:, :, np.newaxis], (1, 1, 3))
save = True if args.save == 'True' else False
path = path[:-1] if path[-1] == '/' else path
if '300' in args.prototxt:
image_name = path[path.rindex('/') +
1:-4] + '_yuval_300_nocrf_' + args.crop + '.png'
else:
image_name = path[path.rindex('/') +
1:-4] + '_yuval_nocrf_' + args.crop + '.png'
show_result(imi,
mask,
im_seg,
save=save,
filename='images/' + image_name)
# generate prob
#prob = np.concatenate(((1-mask)[np.newaxis,:,:]*0.9 + mask[np.newaxis,:,:]*0.1, mask[np.newaxis,:,:]*0.9+(1-mask)[np.newaxis,:,:]*0.1), axis=0)
prob = ndimage.gaussian_filter(mask * 1.0, sigma=5)
prob = np.concatenate(
((1 - prob)[np.newaxis, :, :], prob[np.newaxis, :, :]), axis=0)
# add CRF
map = CRF(prob, np.array(imi))
if '300' in args.prototxt:
image_name = path[path.rindex('/') +
1:-4] + '_yuval_300_crf_' + args.crop + '.png'
else:
image_name = path[path.rindex('/') +
1:-4] + '_yuval_crf_' + args.crop + '.png'
show_result(imi,
map,
np.tile((map != 0)[:, :, np.newaxis], (1, 1, 3)) * imi,
save=save,
filename='images/' + image_name)
import cv2
import sys
from face_detection.api import S3FD
from face_detection.bbox import nms
from face_alignment.api import FaceAlignment
from face_alignment.api import LandmarksType
s3fd = S3FD("models/s3fd_convert.pth")
face_alignment = FaceAlignment(LandmarksType._3D)
score_threshold = 0.5
nms_threshold = 0.3
image = cv2.imread(sys.argv[1])
boxes = s3fd.detect(image, score_threshold)
boxes = boxes[nms(boxes, nms_threshold)]
figure = image.copy()
for box in boxes:
box = box[0:4]
landmarks = face_alignment.get_landmarks(image, box)
for pt in landmarks:
pt = tuple(pt[0:2].astype(int))
cv2.circle(figure, pt, 3, (0, 255, 0))
cv2.imshow('', figure)
cv2.waitKey(0)
def main(args):
image_paths = read_list(args.image_list)
# init
caffe.set_device(0)
caffe.set_mode_gpu()
# load net
net = caffe.Net(args.prototxt, args.caffemodel, caffe.TEST)
for path in image_paths:
# load image, switch to BGR, subtract mean, and make dims C x H x W for Caffe
print path
if path[-3:] == 'jpg' or path[-3:] == 'png':
imi = open_image(path)
# resize for memory
width, height = imi.size
if height > 800:
imi = imi.resize((int(800*width/height), 800))
width, height = imi.size
else:
continue
dir = 'images/' + path[path.rindex('/', 0, path.rindex('/'))+1:path.rindex('/')] + '/'
if not os.path.exists(dir):
os.makedirs(dir)
txtfile = dir+path[path.rindex('/')+1:-3]+'txt'
# use 2D-FAN detect landmarks
fa = FaceAlignment(LandmarksType._2D, enable_cuda=True,
flip_input=False, use_cnn_face_detector=True)
try:
landmarks = fa.get_landmarks(np.array(imi))[-1]
save_landmarks(txtfile, landmarks)
landmarks = landmarks.astype('uint16')
except:
# no face deteced
continue
if args.crop == 'middle':
imi, landmarks = crop_image_middle(landmarks, imi)
elif args.crop == 'min':
imi, landmarks = crop_image_min(landmarks, imi)
width, height = imi.size
if '300' in args.prototxt:
imi = imi.resize((300, 300))
landmarks[:,0] = landmarks[:,0] * 300 / width
landmarks[:,1] = landmarks[:,1] * 300 / height
width = height = 300
else:
imi = imi.resize((500, 500))
landmarks[:,0] = landmarks[:,0] * 500 / width
landmarks[:,1] = landmarks[:,1] * 500 / height
width = height = 500
im = np.array(imi, dtype=np.float32)
im = im[:,:,::-1]
im -= np.array((104.00698793,116.66876762,122.67891434))
im = im.transpose((2,0,1))
# shape for input (data blob is N x C x H x W), set data
net.blobs['data'].reshape(1, *im.shape)
net.blobs['data'].data[...] = im
# run net and take argmax for prediction
net.forward()
mask = net.blobs['score'].data[0].argmax(axis=0)
save = True if args.save == 'True' else False
path = path[:-1] if path[-1] == '/' else path
if '300' in args.prototxt:
image_name = path[path.rindex('/')+1:-4] + '_yuvalcontour_300_nocrf_' + args.crop + '.png'
else:
image_name = path[path.rindex('/')+1:-4] + '_yuvalcontour_nocrf_' + args.crop + '.png'
# draw landmarks
lm = np.array(imi)
for i in range(landmarks.shape[0]):
rr, cc = draw.circle(landmarks[i,1].astype('int32'), landmarks[i,0].astype('int32'), 1)
lm[rr, cc, :] = np.array((255, 0, 0))
# create mask contour
hull = ConvexHull(landmarks)
mask_contour = np.zeros((height, width))
rr, cc = draw.polygon(landmarks[hull.vertices,1], landmarks[hull.vertices,0], mask_contour.shape)
mask_contour[rr,cc] = 1
mask = np.clip(mask + mask_contour, 0, 1)
im_seg = imi * np.tile((mask!=0)[:,:,np.newaxis], (1,1,3))
show_result(lm, mask, im_seg, save=save, filename=dir+image_name)
# generate prob
#prob = np.concatenate(((1-mask)[np.newaxis,:,:]*0.9 + mask[np.newaxis,:,:]*0.1, mask[np.newaxis,:,:]*0.9+(1-mask)[np.newaxis,:,:]*0.1), axis=0)
prob = ndimage.gaussian_filter(mask*1.0, sigma=5)
prob = np.concatenate(((1-prob)[np.newaxis,:,:], prob[np.newaxis,:,:]), axis=0)
# add CRF
map = CRF(prob, np.array(imi))
if '300' in args.prototxt:
image_name = path[path.rindex('/')+1:-4] + '_yuvalcontour_300_crf_' + args.crop + '.png'
else:
image_name = path[path.rindex('/')+1:-4] + '_yuvalcontour_crf_' + args.crop + '.png'
show_result(imi, map, np.tile((map!=0)[:,:,np.newaxis], (1,1,3)) * imi, save=save, filename=dir+image_name)
def remove_neck_landmarks(self, images_path, segs_path, root):
""" Removes neck segmentation according to face landmarks model of the FaceEngine module
Args:
images_path: list of pathes to images
segs_path: list of pathes to segmentation masks
Returns:
Segmentation masks without neck part and saves them as root/label/name.png
"""
# I. Set 2D landmarks detector and gender detector for lfw dataset
lm_detector = FaceAlignment(face_alignment.LandmarksType._2D, flip_input=False)
d = gdetect.Detector()
for i in tqdm(range(len(images_path))):
name = segs_path[i].split('/')[-1]
# II. Open IMAGE and segmentation MASK
source_img = cv.imread(images_path[i])
source_img = cv.cvtColor(source_img, cv.COLOR_BGR2RGB)
source_seg = Image.open(segs_path[i])
# III. Find LANDMARKS
landmarks = lm_detector.get_landmarks_from_image(source_img)[0]
# in order to remove ears as well
left_b = [landmarks[0][0], landmarks[0][1] - (landmarks[2][1] - landmarks[0][1])]
right_b = [landmarks[16][0], landmarks[16][1]-(landmarks[14][1] - landmarks[16][1])]
lm = landmarks[:17]
# IV. Get coordinates of points to create separation line (border)
h, w = source_seg.size
left_edge = [0, left_b[1]]
right_edge = [w-1, right_b[1]]
points_neck = left_edge + left_b + list(lm.ravel()) + right_b + right_edge
points_beard = left_edge + left_b + right_b + right_edge
# V. Get starting points for filler algorithm
left = [left_edge[0], left_edge[1]+1]
right = [right_edge[0], right_edge[1]+1]
# VI. Draw border line
source_seg = np.array(source_seg).astype(np.uint8)
if len(np.unique(source_seg)) != 3:
source_seg[source_seg==1] = 0
source_seg[source_seg==2] = 1
mask_neck = source_seg.copy()
# mask_neck[mask_neck == 2] = 0
mask_neck = Image.fromarray(mask_neck)
draw = ImageDraw.ImageDraw(mask_neck)
draw.line(points_neck, 255, 0)
# VII. Create a mask without neck
mask_neck = np.array(mask_neck)
mask_no_neck = fill_underline(mask_neck, h, w, left, right)
face_mask = source_seg.copy()
# VIII. Remove redundant features
first_name = name.split('_')[0]
sex = d.get_gender(first_name)
if len(np.unique(source_seg)) == 3:
if sex == 'male':
# Cut beards and mustache
mask_beard = source_seg.copy()
mask_beard = Image.fromarray(mask_beard)
# draw separation line
draw = ImageDraw.ImageDraw(mask_beard)
draw.line(points_beard, 255, 0)
# fill all the area with 1 label
mask_beard = np.array(mask_beard)
mask_no_beard = fill_underline(mask_beard, h, w, left, right, fill=1)
# remove mustache
face_mask[mask_no_beard == 1] = 1
# remove beard and neck
face_mask[mask_no_neck == 0] = 0
else:
# remove neck only for women
face_mask[mask_no_neck == 0] = 0
face_mask[source_seg == 2] = 2
else:
face_mask[mask_no_neck == 0] = 0
# IX. Add palette
mask_f = Image.fromarray(face_mask)
if len(np.unique(face_mask)) == 3:
mask_f = mask_f.convert('P', palette=Image.ADAPTIVE, colors=3)
reverse_colors = np.array(mask_f)
reverse_colors[reverse_colors == 0] = 3
reverse_colors[reverse_colors == 2] = 0
reverse_colors[reverse_colors == 3] = 2
else:
mask_f = mask_f.convert('P', palette=Image.ADAPTIVE, colors=2)
reverse_colors = np.array(mask_f)
reverse_colors[reverse_colors == 1] = 4
reverse_colors[reverse_colors == 0] = 1
reverse_colors[reverse_colors == 4] = 0
mask_f = Image.fromarray(reverse_colors, mode='P')
mask_f.putpalette([
0, 0, 0, # index 0 is black (background)
0, 255, 0, # index 1 is green (face)
255, 0, 0, # index 2 is red (hair)
])
# X. Save results
path = "{}/Masks/{}".format(root, name)
mask_f.save(path)
import csv
import numpy as np
import os
import cv2
from skimage import io
from face_alignment.api import FaceAlignment, LandmarksType
fa_2D = FaceAlignment(LandmarksType._2D, flip_input=False)
fa_3D = FaceAlignment(LandmarksType._3D, flip_input=False)
class DataTransform(object):
""""
class for get the landmarks of dataset and tranform
it to csv
"""
def __init__(self, dataset_name='fer2013', dataset_path=None, method='3D'):
self.dataset_name = dataset_name
self.dataset_path = dataset_path
self.method = method
# if self.method == '2D':
# self.landmark_detect = FaceAlignment(LandmarksType._2D, flip_input=False)
# elif self.method == '3D':
# self.landmark_detect = FaceAlignment(LandmarksType._3D, flip_input=False)
if self.dataset_path != None:
self.dataset_path = dataset_path
elif self.dataset_name == 'fer2013':
self.dataset_path = '../datasets/fer2013/'
elif self.dataset_name == 'KDEF':
self.dataset_path = '../datasets/KDEF/'