def test():
f = Face()
pic = "/home/gustavo/Documents/unb/das/joserobertoarruda.jpg"
img = cv2.imread(pic)
img = f.detect(img)
cv2.imshow('frame',img)
cv2.waitKey()
def demo_video(video_file):
import time
facedemo = Face(detector_method=DETECTOR, recognition_method=None)
cap = common.VideoStream(video_file, queueSize=4).start()
time.sleep(1)
total_t, counter = 0, 0
t = common.clock()
while not cap.stopped:
imgcv = cap.read()
if imgcv is not None:
counter += 1
detections = facedemo.detect(imgcv, upsamples=0)
ids = range(len(detections))
# temp = mtracker.update(imgcv, to_cvbox(detections))
# cvboxes, ids = [], []
# for tid,tracker in mtracker.trackers.items():
# if tracker.visible_count > 3 and tracker.consecutive_invisible_count<10:
# cvboxes.append(tracker.bbox)
# ids.append(tid)
# detections = to_bbox(cvboxes)
print(detections)
common.showImage(common.drawObjects(imgcv, detections, ids))
key = cv2.waitKey(1) & 0xFF
if key == 27:
break
t1 = common.clock()
dt = t1 - t
t = t1
total_t += dt
print(counter / total_t)
class Capture:
def __init__(self, label, clf):
self.frame = None
self.thread1 = None
self.thread2 = None
self.stopEvent1 = None
self.stopEvent2 = None
self.started = None
self.captured = None
self.label = label
self.vs = VideoStream().start()
self.fa = Face()
self.emotion = Emotion(clf)
self.outputPath = None
def start(self):
#setting started to true to set it is started in the train function.
#Because None type cannot be set
self.started = True
#Erasing stopEvent value (if any) to loop in the function
self.stopEvent1 = None
#checking if train thred is started
#if started close it
if (self.captured):
self.stopEvent2.set()
#initialize the thread
self.stopEvent1 = threading.Event()
self.thread1 = threading.Thread(target=self.videoLoop, args=())
self.thread1.start()
def close(self):
#initiallly checking if the threads have been started and then closing
#if started
if (self.started):
self.stopEvent1.set()
if (self.captured):
self.stopEvent2.set()
# self.vs.stop()
def exit(self):
if (self.started):
self.stopEvent1.set()
print "closing thread 1"
if (self.captured):
self.stopEvent2.set()
print "closing thread 2"
self.vs.stop()
VideoStream().stop()
def capture(self, emotion):
#setting the trained variable to true for same reason as started
self.captured = True
# self.emo = emotion
self.outputPath = "Dataset" + os.sep + emotion
if os.path.exists(self.outputPath):
pass
else:
os.mkdir(self.outputPath)
#Setting StopEvent to None to loop
self.stopEvent2 = None
#stopping started thread if running
if (self.started):
self.stopEvent1.set()
#initializing the train thred
self.stopEvent2 = threading.Event()
self.thread2 = threading.Thread(target=self.captureLoop, args=())
self.thread2.start()
def snapshot(self):
ts = datetime.datetime.now()
filename = "{}.jpg".format(ts.strftime("%Y-%m-%d_%H-%M-%S"))
output = os.path.sep.join((self.outputPath, filename))
cv2.imwrite(output, self.frame.copy())
self.showInfo()
def showInfo(self):
info.show()
def videoLoop(self):
try:
#loop until the stopEvent is not set
while not self.stopEvent1.is_set():
self.frame = self.vs.read()
gray = cv2.cvtColor(self.frame, cv2.COLOR_BGR2GRAY)
image = cv2.cvtColor(self.frame, cv2.COLOR_BGR2RGB)
faces = self.fa.detect(gray)
image = self.emotion.getLandmarks(image, faces)
image = Image.fromarray(image)
image = ImageTk.PhotoImage(image)
self.label.configure(image=image)
self.label.image = image
except RuntimeError, e:
print "Error"
class confusionMatrix:
def __init__(self):
self.predictor = dlib.shape_predictor("predictor.dat")
self.clf = SVC(kernel = 'linear', probability=True, tol=1e-3)
self.emotions = ['Angry','Happy','Neutral','Sad','Shocked']
self.face = Face()
def getconfusion(self):
# predictor = dlib.shape_predictor("predictor.dat")
# clf = SVC(kernel = 'linear', probability=True, tol=1e-3)
# emotions = ['Angry','Happy','Neutral','Sad','Shocked']
# face = Face()
mat = []
training_data,training_labels,prediction_data,prediction_labels = self.make_sets()
npar_train = np.array(training_data)
npar_pred = np.array(prediction_data)
self.clf.fit(npar_train,training_labels)
print "training..."
print len(prediction_data)
for index,pred in enumerate(prediction_data):
prediction = self.clf.predict(npar_pred[index].reshape(1,-1))
#print self.emotions[int(prediction)]
pred = int(prediction)
actual = int(prediction_labels[index])
if(pred != actual):
print "{} predicted as {}".format(self.emotions[actual],self.emotions[pred])
cell = "_"+str(actual)+str(pred)+"_"
mat.append(cell)
print "Successful"
print mat
def count_mat(mat):
new_mat = []
count = {}
for n in mat:
if n not in new_mat:
new_mat.append(n)
count[n] = 1
else:
count[n] += 1
return new_mat,count
New_mat, count = count_mat(mat)
for mats in New_mat:
print "{} : {} \n".format(mats,count[mats])
def get_images(self,emotion):
print "Getting {} images ".format(emotion)
files = glob.glob("Dataset/%s/*" %emotion)
random.shuffle(files)
training = files[:int(len(files)*0.7)]
prediction = files[-int(len(files)*0.3):]
print len(files)
return training,prediction
def get_landmarks(self,image):
image = modules.resize(image,width=400)
cv2.imshow("image",image)
cv2.waitKey(10)
faces = self.face.detect(image)
for x,y,w,h in faces:
roi = image[y:y+h,x:x+w]
#incresing the size of the ROI to give it to the shape predictor as image
#ROI = image[(y-10):(y+h+10),(x-10):(x+w+10)]
#resizing the roi
resized_roi = modules.resize(roi,width=800,inter=cv2.INTER_CUBIC)
#getting the bottom and the right corner of the resized image
bottom,right = resized_roi.shape[:2]
#converting cv2 rectangle to dlib rectangle
rect = dlib.rectangle(x,y,x+w,y+h)
resized_rect = dlib.rectangle(0,0,right,bottom)
shape = self.predictor(image,rect)
resized_shape = self.predictor(resized_roi,resized_rect)
xlist = []
ylist = []
landmarks_vectorised = []
for i in range(36,68):
xlist.append(float(resized_shape.part(i).x))
ylist.append(float(resized_shape.part(i).y))
xmean = np.mean(xlist)
ymean = np.mean(ylist)
xcentral = [(x-xmean) for x in xlist]
ycentral = [(y-ymean) for y in ylist]
for x,y,w,z in zip(xcentral,ycentral,xlist,ylist):
landmarks_vectorised.append(x)
landmarks_vectorised.append(y)
meannp = np.asarray((ymean,xmean))
coornp = np.asarray((z,w))
dist = np.linalg.norm(coornp-meannp)
landmarks_vectorised.append(dist)
if len(faces)<1:
landmarks_vectorised = "error"
return landmarks_vectorised
def make_sets(self):
training_data = []
training_labels = []
prediction_data= []
prediction_labels = []
for emotion in self.emotions:
training,prediction = self.get_images(emotion)
for item in training:
image = cv2.imread(item,0)
landmarks_vectorised = self.get_landmarks(image)
if landmarks_vectorised == "error":
pass
else:
training_data.append(landmarks_vectorised)
training_labels.append(self.emotions.index(emotion))
for item in prediction:
image = cv2.imread(item,0)
landmarks_vectorised = self.get_landmarks(image)
if landmarks_vectorised == "error":
pass
else:
prediction_data.append(landmarks_vectorised)
prediction_labels.append(self.emotions.index(emotion))
cv2.destroyAllWindows()
return training_data,training_labels,prediction_data,prediction_labels
class Demography(object):
def __init__(self, face_method='dlib', device='cpu', age_method='mobilenetv2',gender_method='mobilenetv2',gpu_config =0.3,model_file='age_gender_frozen_trt_mobilenetv2.pb'):
global config
if gpu_config is not None and config is not None:
config.gpu_options.per_process_gpu_memory_fraction = gpu_config
if age_method == 'mobilenetv2' and gender_method == "mobilenetv2":
self.gender_age_estimator = AgeGenderEstimate_mobilenetv2(gpu_frac=gpu_config, model_file=model_file)
self.mixed_model = True
else:
self.mixed_model = False
if age_method == 'inception':
self.age_estimator = AgeEstimate()
elif age_method == 'coral':
self.age_estimator = AgeEstimate_Coral()
elif age_method == "mobilenetv2":
self.age_estimator = AgeGenderEstimate_mobilenetv2(gpu_frac=gpu_config)
if gender_method == 'inception':
self.gender_estimator = GenderEstimate()
elif gender_method == 'mobilenetv2':
self.gender_estimator = AgeGenderEstimate_mobilenetv2(gpu_frac=gpu_config)
if face_method:
self.face_detect = Face(detector_method=face_method, recognition_method=None)
def run(self, imgcv, face_detect=False):
results = []
if face_detect:
faces = self.face_detect.detect(imgcv)
for face in faces:
results.append(self.run_face(imgcv, face['box']))
else:
results.append(self.run_face(imgcv, None))
return results
def run_face(self, imgcv, face_box):
if face_box is not None:
face_image = common.subImage(imgcv, face_box)
else:
face_image = imgcv
if self.mixed_model:
face_image = cv2.cvtColor(face_image, cv2.COLOR_BGR2GRAY)
gender, age = self.gender_age_estimator.run(face_image)
else:
if isinstance(self.gender_estimator, AgeGenderEstimate_mobilenetv2):
gender = self.gender_estimator.run(face_image)[0]
else:
gender = self.gender_estimator.run(face_image)
if isinstance(self.age_estimator, AgeGenderEstimate_mobilenetv2):
age = self.age_estimator.run(face_image)[1]
else:
if isinstance(self.age_estimator, AgeEstimate):
age = self.age_estimator.run(face_image)
elif isinstance(self.age_estimator, AgeEstimate_Coral):
face_image = common.subImage(imgcv, face_box, padding_type='coral')
age = self.age_estimator.run(face_image)
return self._format_results(face_box, age, gender)
def _format_results(self, face_box, age, gender):
out = {
'box': face_box,
'classes': [{'name': 'face',
'prob': None,
'meta': {'age': age,
'gender': gender
}
}]
}
return out
class dataCreator(Label):
def __init__(self, label, vs):
self.face = Face()
self.label = label
self.vs = vs
self.emotions = ['Angry', 'Happy', 'Neutral', 'Sad', 'Shocked']
self.target = "Dataset"
def create(self):
for emotion in self.emotions:
print("please look {}. Press Capture button when ready".format(
emotion))
start_time = time.time()
elapsed_time = time.time() - start_time
sample = 0
if os.path.exists(self.target + os.sep + emotion):
pass
else:
os.mkdir(self.target + os.sep + emotion)
while True:
while elapsed_time < 3.0:
image = self.vs.read()
image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)
elapsed_time = int(time.time() - start_time)
text = "Starting in {} seconds".format(3 - elapsed_time)
cv2.putText(image, str(text), (20, 40),
cv2.FONT_HERSHEY_SIMPLEX, 0.7, (0, 0, 255), 2)
cv2.waitKey(100)
img = Image.fromarray(image)
img = ImageTk.PhotoImage(img)
self.label.configure(image=img)
self.label.image = img
image = self.vs.read()
if (sample > 20):
break
else:
gray = cv2.cvtColor(image, cv2.COLOR_BGR2GRAY)
image_copy = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)
faces = self.face.detect(gray)
for x, y, w, h in faces:
sample += 1
#frame = image[y-10:y+h+10,x-10:x+w+10]
cv2.rectangle(image_copy, (x, y), (x + w, y + h),
(255, 0, 0), 1)
cv2.imwrite(
self.target + os.sep + emotion + os.sep +
str(time.time()) + ".jpg", image)
cv2.waitKey(100)
img = Image.fromarray(image_copy)
img = ImageTk.PhotoImage(img)
self.label.configure(image=img)
self.label.image = img
def demo_video(video_file):
facedemo = Face(detector_method=DETECTOR, recognition_method=None)
mtracker = MultiTracker(SingleTrackerType=CorrelationTracker)
# mtracker = MultiTracker(SingleTrackerType=CorrelationTracker,
# removalConfig=removalConfig)
# mtracker = MultiTracker(SingleTrackerType = cv2.TrackerKCF_create)
cap = common.VideoStream(video_file, queueSize=4).start()
cv2.waitKey(500)
Outcount, Incount = 0, 0
while not cap.stopped:
t = common.clock()
total_t, counter = 0, 0
imgcv = cap.read()
if imgcv is not None:
counter += 1
detections = facedemo.detect(imgcv, upsamples=0)
mtracker.update(imgcv, common.toCvbox(detections))
cvboxes, ids = [], []
for tid, tracker in mtracker.trackers.items():
if tracker.visible_count > 3 and tracker.consecutive_invisible_count < 10:
state_current = get_pos(tracker.bbox)
try:
if state_current != tracker.regionside:
tracker.statechange += 1
print state_current, tracker.regionside, tracker.statechange
if state_current == 'Positive':
if tracker.statechange % 2:
Incount += 1
else:
Outcount -= 1
else:
if tracker.statechange % 2:
Outcount += 1
else:
Incount -= 1
tracker.regionside = state_current
except AttributeError:
tracker.regionside = state_current
tracker.statechange = 0
cvboxes.append(tracker.bbox)
ids.append(tid)
detections = to_bbox(cvboxes)
print Incount, Outcount
cv2.line(imgcv, (LINE['x1'], LINE['y1']), (LINE['x2'], LINE['y2']),
(0, 0, 255), 4)
imgcv = common.drawLabel(imgcv,
"IN:%d OUT:%d" % (Incount, Outcount),
(10, 10),
color=(0, 0, 255))
common.showImage(common.drawObjects(imgcv, detections, ids))
key = cv2.waitKey(1) & 0xFF
if key == 27:
break
t1 = common.clock()
dt = t1 - t
t = t1
total_t += dt
print counter / total_t
class Trainer:
def __init__(self):
self.predictor = dlib.shape_predictor("predictor.dat")
self.clf = SVC(kernel='linear', probability=True, tol=1e-3)
self.emotions = ['Angry', 'Happy', 'Neutral', 'Sad', 'Shocked']
self.face = Face()
def train(self):
# predictor = dlib.shape_predictor("predictor.dat")
# clf = SVC(kernel = 'linear', probability=True, tol=1e-3)
# emotions = ['Angry','Happy','Neutral','Sad','Shocked']
# face = Face()
training_data, training_labels = self.make_sets()
npar_train = np.array(training_data)
self.clf.fit(npar_train, training_labels)
print "training"
joblib.dump(self.clf, 'CustomSVM.pkl')
print "SVM trained successfully"
mbox.showinfo("FER", "SVM training completed")
def accuracy(self):
test_data, test_labels = self.make_sets()
npar_test = np.array(test_data)
self.clf = joblib.load('Universal.pkl')
accuracy = self.clf.score(npar_test, test_labels)
print accuracy
def get_images(self, emotion):
print "Getting {} images ".format(emotion)
files = glob.glob("Dataset/%s/*" % emotion)
print len(files)
return files
def get_landmarks(self, image):
image = modules.resize(image, width=400)
cv2.imshow("SVM training", image)
cv2.waitKey(500)
faces = self.face.detect(image)
for x, y, w, h in faces:
roi = image[y:y + h, x:x + w]
#incresing the size of the ROI to give it to the shape predictor as image
#ROI = image[(y-10):(y+h+10),(x-10):(x+w+10)]
#resizing the roi
resized_roi = modules.resize(roi, width=800, inter=cv2.INTER_CUBIC)
#getting the bottom and the right corner of the resized image
bottom, right = resized_roi.shape[:2]
#converting cv2 rectangle to dlib rectangle
rect = dlib.rectangle(x, y, x + w, y + h)
resized_rect = dlib.rectangle(0, 0, right, bottom)
shape = self.predictor(image, rect)
resized_shape = self.predictor(resized_roi, resized_rect)
xlist = []
ylist = []
landmarks_vectorised = []
for i in range(36, 68):
xlist.append(float(resized_shape.part(i).x))
ylist.append(float(resized_shape.part(i).y))
xmean = np.mean(xlist)
ymean = np.mean(ylist)
xcentral = [(x - xmean) for x in xlist]
ycentral = [(y - ymean) for y in ylist]
for x, y, w, z in zip(xcentral, ycentral, xlist, ylist):
landmarks_vectorised.append(x)
landmarks_vectorised.append(y)
meannp = np.asarray((ymean, xmean))
coornp = np.asarray((z, w))
dist = np.linalg.norm(coornp - meannp)
landmarks_vectorised.append(dist)
if len(faces) < 1:
landmarks_vectorised = "error"
return landmarks_vectorised
def make_sets(self):
training_data = []
training_labels = []
for emotion in self.emotions:
training = self.get_images(emotion)
for item in training:
image = cv2.imread(item, 0)
landmarks_vectorised = self.get_landmarks(image)
if landmarks_vectorised == "error":
pass
else:
training_data.append(landmarks_vectorised)
training_labels.append(self.emotions.index(emotion))
cv2.destroyAllWindows()
return training_data, training_labels
