def predict(self,img, faces):
analysis = []
#getting input model shapes for inference
emotion_target_size = self.emotion_classifier.input_shape[1:3]
#rgb_image = load_image(img , grayscale=False)
rgb_image = cv2.cvtColor(img.copy(),cv2.cv2.COLOR_BGR2RGB)
#gray_image = load_image(img , grayscale=True)
gray_image = cv2.cvtColor(img.copy(),cv2.cv2.COLOR_BGR2GRAY)
gray_image = np.squeeze(gray_image)
gray_image = gray_image.astype('uint8')
for face_coordinates in faces:
x1, x2, y1, y2 = apply_offsets(face_coordinates, self.emotion_offsets)
gray_face = gray_image[y1:y2, x1:x2]
try:
gray_face = cv2.resize(gray_face, (emotion_target_size))
except:
continue
gray_face = preprocess_input(gray_face, True)
gray_face = np.expand_dims(gray_face, 0)
gray_face = np.expand_dims(gray_face, -1)
emotion_label_arg = np.argmax(self.emotion_classifier.predict(gray_face))
emotion_text = emotion_labels[emotion_label_arg]
color = (0, 0, 255)
draw_bounding_box(face_coordinates, rgb_image, color)
draw_text(face_coordinates, rgb_image, emotion_text, color, 0, -50, 1, 2)
#ouput
analysis.append([face_coordinates, emotion_text]) #Essa será a saida, externamente decidiremos para onde vai
bgr_image = cv2.cvtColor(rgb_image, cv2.COLOR_RGB2BGR)
return analysis, bgr_image # Aqui ele retorna lista de faces detectadas com os labels de genero e emoção
def process_image(image, filename, model_path):
try:
# parameters for loading data and images
detection_model_path = './trained_models/detection_models/haarcascade_frontalface_default.xml'
emotion_model_path = model_path + '/emotion_models/fer2013_mini_XCEPTION.102-0.66.hdf5'
gender_model_path = model_path + '/gender_models/simple_CNN.81-0.96.hdf5'
emotion_labels = get_labels('fer2013')
gender_labels = get_labels('imdb')
font = cv2.FONT_HERSHEY_SIMPLEX
# hyper-parameters for bounding boxes shape
gender_offsets = (30, 60)
gender_offsets = (10, 10)
emotion_offsets = (20, 40)
emotion_offsets = (0, 0)
# loading models
face_detection = load_detection_model(detection_model_path)
emotion_classifier = load_model(emotion_model_path, compile=False)
gender_classifier = load_model(gender_model_path, compile=False)
# getting input model shapes for inference
emotion_target_size = emotion_classifier.input_shape[1:3]
gender_target_size = gender_classifier.input_shape[1:3]
# loading images
image_array = np.fromstring(image, np.uint8)
unchanged_image = cv2.imdecode(image_array, cv2.IMREAD_UNCHANGED)
rgb_image = cv2.cvtColor(unchanged_image, cv2.COLOR_BGR2RGB)
gray_image = cv2.cvtColor(unchanged_image, cv2.COLOR_BGR2GRAY)
faces = detect_faces(face_detection, gray_image)
for face_coordinates in faces:
x1, x2, y1, y2 = apply_offsets(face_coordinates, gender_offsets)
rgb_face = rgb_image[y1:y2, x1:x2]
x1, x2, y1, y2 = apply_offsets(face_coordinates, emotion_offsets)
gray_face = gray_image[y1:y2, x1:x2]
try:
rgb_face = cv2.resize(rgb_face, (gender_target_size))
gray_face = cv2.resize(gray_face, (emotion_target_size))
except:
continue
rgb_face = preprocess_input(rgb_face, False)
rgb_face = np.expand_dims(rgb_face, 0)
gender_prediction = gender_classifier.predict(rgb_face)
gender_label_arg = np.argmax(gender_prediction)
gender_text = gender_labels[gender_label_arg]
gray_face = preprocess_input(gray_face, True)
gray_face = np.expand_dims(gray_face, 0)
gray_face = np.expand_dims(gray_face, -1)
emotion_label_arg = np.argmax(emotion_classifier.predict(gray_face))
emotion_text = emotion_labels[emotion_label_arg]
if gender_text == gender_labels[0]:
color = (0, 0, 255)
else:
color = (255, 0, 0)
draw_bounding_box(face_coordinates, rgb_image, color)
draw_text(face_coordinates, rgb_image, gender_text, color, 0, -20, 1, 2)
draw_text(face_coordinates, rgb_image, emotion_text, color, 0, -50, 1, 2)
except Exception as err:
logging.error('Error in emotion gender processor: "{0}"'.format(err))
bgr_image = cv2.cvtColor(rgb_image, cv2.COLOR_RGB2BGR)
dirname = 'result'
if not os.path.exists(dirname):
os.mkdir(dirname)
cv2.imwrite(os.path.join(dirname, filename), bgr_image)
elif emotion_text == 'sad':
color = emotion_probability * np.asarray((0, 0, 255))
elif emotion_text == 'happy':
color = emotion_probability * np.asarray((255, 255, 0))
elif emotion_text == 'surprise':
color = emotion_probability * np.asarray((0, 255, 255))
else:
color = emotion_probability * np.asarray((0, 255, 0))
color = color.astype(int)
color = color.tolist()
draw_bounding_box(face_coordinates, rgb_image, color)
draw_text(face_coordinates, rgb_image, emotion_mode,
color, 0, -45, 1, 1)
bgr_image = cv2.cvtColor(rgb_image, cv2.COLOR_RGB2BGR)
if(count<10000):
cv2.imwrite("pic_save/%d_%s" %(count,emotion_mode)+'.jpg',bgr_image)
cv2.imshow('window_frame', bgr_image)
count = count + 1
if cv2.waitKey(3) & 0xFF == ord('q'):
sess.close()
break
end=time.time();
print(str(end-start))
print("\n [INFO] Exiting program and cleanup stuff")
video_capture.release()
cv2.destroyAllWindows()
def image_emotion_gender(image):
bgr_image = image
rgb_image = cv2.cvtColor(bgr_image, cv2.COLOR_BGR2RGB)
gray_image = cv2.cvtColor(bgr_image, cv2.COLOR_BGR2GRAY)
detection_model_path = 'trained_models/detection_models/haarcascade_frontalface_default.xml'
emotion_model_path = 'trained_models/emotion_models/fer2013_mini_XCEPTION.102-0.66.hdf5'
gender_model_path = 'trained_models/gender_models/simple_CNN.81-0.96.hdf5'
emotion_labels = get_labels('fer2013')
gender_labels = get_labels('imdb')
font = cv2.FONT_HERSHEY_SIMPLEX
# hyper-parameters for bounding boxes shape
gender_offsets = (30, 60)
gender_offsets = (10, 10)
emotion_offsets = (20, 40)
emotion_offsets = (0, 0)
# loading models
face_detection = load_detection_model(detection_model_path)
emotion_classifier = load_model(emotion_model_path, compile=False)
gender_classifier = load_model(gender_model_path, compile=False)
# getting input model shapes for inference
emotion_target_size = emotion_classifier.input_shape[1:3]
gender_target_size = gender_classifier.input_shape[1:3]
faces = detect_faces(face_detection, gray_image)
for face_coordinates in faces:
x1, x2, y1, y2 = apply_offsets(face_coordinates, gender_offsets)
rgb_face = rgb_image[y1:y2, x1:x2]
x1, x2, y1, y2 = apply_offsets(face_coordinates, emotion_offsets)
gray_face = gray_image[y1:y2, x1:x2]
try:
rgb_face = cv2.resize(rgb_face, (gender_target_size))
gray_face = cv2.resize(gray_face, (emotion_target_size))
except:
continue
rgb_face = preprocess_input(rgb_face, False)
rgb_face = np.expand_dims(rgb_face, 0)
gender_prediction = gender_classifier.predict(rgb_face)
gender_label_arg = np.argmax(gender_prediction)
gender_text = gender_labels[gender_label_arg]
gray_face = preprocess_input(gray_face, True)
gray_face = np.expand_dims(gray_face, 0)
gray_face = np.expand_dims(gray_face, -1)
emotion_label_arg = np.argmax(emotion_classifier.predict(gray_face))
emotion_text = emotion_labels[emotion_label_arg]
if gender_text == gender_labels[0]:
color = (0, 0, 255)
else:
color = (255, 0, 0)
draw_bounding_box(face_coordinates, rgb_image, color)
draw_text(face_coordinates, rgb_image, gender_text, color, 0, -20, 1, 2)
draw_text(face_coordinates, rgb_image, emotion_text, color, 0, -50, 1, 2)
bgr_image = cv2.cvtColor(rgb_image, cv2.COLOR_RGB2BGR)
return bgr_image
def process_image(image):
K.clear_session()
# parameters for loading data and images
if sys.path[1] == '/app':
#load model for heroku
detection_model_path = sys.path[
1] + '/trained_models/detection_models/haarcascade_frontalface_default.xml'
emotion_model_path = sys.path[
1] + '/trained_models/emotion_models/fer2013_mini_XCEPTION.102-0.66.hdf5'
else:
detection_model_path = sys.path[
-1] + '/trained_models/detection_models/haarcascade_frontalface_default.xml'
emotion_model_path = sys.path[
-1] + '/trained_models/emotion_models/fer2013_mini_XCEPTION.102-0.66.hdf5'
emotion_labels = get_labels('fer2013')
font = cv2.FONT_HERSHEY_SIMPLEX
# hyper-parameters for bounding boxes shape
emotion_offsets = (20, 40)
emotion_offsets = (0, 0)
# loading models
face_detection = load_detection_model(detection_model_path)
emotion_classifier = load_model(emotion_model_path, compile=False)
# getting input model shapes for inference
emotion_target_size = emotion_classifier.input_shape[1:3]
# loading images
image_array = np.fromstring(image, np.uint8)
unchanged_image = cv2.imdecode(image_array, cv2.IMREAD_UNCHANGED)
rgb_image = cv2.cvtColor(unchanged_image, cv2.COLOR_BGR2RGB)
gray_image = cv2.cvtColor(unchanged_image, cv2.COLOR_BGR2GRAY)
gray_image = np.squeeze(gray_image)
gray_image = gray_image.astype('uint8')
faces = detect_faces(face_detection, gray_image)
emotion_text_arr = []
for face_coordinates in faces:
x1, x2, y1, y2 = apply_offsets(face_coordinates, emotion_offsets)
gray_face = gray_image[y1:y2, x1:x2]
try:
gray_face = cv2.resize(gray_face, (emotion_target_size))
except:
continue
gray_face = preprocess_input(gray_face, True)
gray_face = np.expand_dims(gray_face, 0)
gray_face = np.expand_dims(gray_face, -1)
emotion_label_arg = np.argmax(emotion_classifier.predict(gray_face))
emotion_text = emotion_labels[emotion_label_arg]
emotion_text_arr.append(emotion_text)
color = (255, 0, 0)
draw_bounding_box(face_coordinates, rgb_image, color)
draw_text(face_coordinates, rgb_image, emotion_text, color, 0, -50, 1,
2)
bgr_image = cv2.cvtColor(rgb_image, cv2.COLOR_RGB2BGR)
K.clear_session()
return (bgr_image, emotion_text_arr)
def race_emotion(image_path, save_path=None, task='save', faces):
image_path = image_path #'../test_images'#sys.argv[1]
detection_model_path = '../trained_models/detection_models/haarcascade_frontalface_default.xml'
emotion_model_path = '../trained_models/emotion_models/fer2013_mini_XCEPTION.102-0.66.hdf5'
gender_model_path = '../trained_models/gender_models/simple_CNN.81-0.96.hdf5'
emotion_labels = get_labels('../trained_models/fer2013')
gender_labels = get_labels('../trained_models/imdb')
font = cv2.FONT_HERSHEY_SIMPLEX
base = os.path.basename(image_path)
name = os.path.splitext(base)[0]
# hyper-parameters for bounding boxes shape
gender_offsets = (30, 60)
gender_offsets = (10, 10)
emotion_offsets = (20, 40)
emotion_offsets = (0, 0)
# loading models
face_detection = load_detection_model(detection_model_path)
emotion_classifier = load_model(emotion_model_path, compile=False)
gender_classifier = load_model(gender_model_path, compile=False)
# getting input model shapes for inference
emotion_target_size = emotion_classifier.input_shape[1:3]
gender_target_size = gender_classifier.input_shape[1:3]
# loading images
rgb_image = load_image(image_path, grayscale=False)
gray_image = load_image(image_path, grayscale=True)
gray_image = np.squeeze(gray_image)
gray_image = gray_image.astype('uint8')
#faces = detect_faces(face_detection, gray_image)
faces = faces
i = 0
for face_coordinates in faces:
x1, x2, y1, y2 = apply_offsets(face_coordinates, gender_offsets)
rgb_face = rgb_image[y1:y2, x1:x2]
x1, x2, y1, y2 = apply_offsets(face_coordinates, emotion_offsets)
gray_face = gray_image[y1:y2, x1:x2]
try:
rgb_face = cv2.resize(rgb_face, (gender_target_size))
gray_face = cv2.resize(gray_face, (emotion_target_size))
except:
print '=' * 10 + 'exception in resize'
continue
#print 'exception in resize' #continue
rgb_face = preprocess_input(rgb_face, False)
rgb_face = np.expand_dims(rgb_face, 0)
gender_prediction = gender_classifier.predict(rgb_face)
gender_label_arg = np.argmax(gender_prediction)
gender_text = gender_labels[gender_label_arg]
gray_face = preprocess_input(gray_face, True)
gray_face = np.expand_dims(gray_face, 0)
gray_face = np.expand_dims(gray_face, -1)
emotion_label_arg = np.argmax(emotion_classifier.predict(gray_face))
emotion_text = emotion_labels[emotion_label_arg]
if gender_text == gender_labels[0]:
color = (0, 0, 255)
else:
color = (255, 0, 0)
crop = rgb_image[y1:y2, x1:x2]
crop = cv2.cvtColor(crop, cv2.COLOR_RGB2BGR)
#race=find_race(crop,gender_text)
print('emotion and race are:', emotion_text, race)
#cv2.imwrite('images/'+str(i)+'.png', crop)
draw_bounding_box(face_coordinates, rgb_image, color)
draw_text(face_coordinates, rgb_image, color, 0, -20, 1, 2)
#draw_text(face_coordinates, rgb_image, gender_text, color, 0, -20, 1, 2)
draw_text(face_coordinates, rgb_image, emotion_text, color, 0, -50, 1,
2)
i = i + 1
if task == 'save':
bgr_image = cv2.cvtColor(rgb_image, cv2.COLOR_RGB2BGR)
cv2.imwrite('../OUT/' + name + '.png', bgr_image)
def main():
cv2.namedWindow('window_frame')
video_capture = cv2.VideoCapture(0)
while True:
bgr_image = video_capture.read()[1]
gray_image = cv2.cvtColor(bgr_image, cv2.COLOR_BGR2GRAY)
rgb_image = cv2.cvtColor(bgr_image, cv2.COLOR_BGR2RGB)
faces = detect_faces(face_detection, gray_image)
for face_coordinates in faces:
x1, x2, y1, y2 = apply_offsets(face_coordinates, emotion_offsets)
gray_face = gray_image[y1:y2, x1:x2]
try:
gray_face = cv2.resize(gray_face, (emotion_target_size))
except:
continue
gray_face = preprocess_input(gray_face, True)
gray_face = np.expand_dims(gray_face, 0)
gray_face = np.expand_dims(gray_face, -1)
emotion_prediction = emotion_classifier.predict(gray_face)
emotion_probability = np.max(emotion_prediction)
emotion_label_arg = np.argmax(emotion_prediction)
emotion_text = emotion_labels[emotion_label_arg]
emotion_window.append(emotion_text)
if len(emotion_window) > frame_window:
emotion_window.pop(0)
try:
emotion_mode = mode(emotion_window)
except:
continue
if emotion_text == 'angry':
color = emotion_probability * np.asarray((255, 0, 0))
print(emoji.emojize(emoji.demojize("😠")))
elif emotion_text == 'sad':
color = emotion_probability * np.asarray((0, 0, 255))
print(emoji.emojize(emoji.demojize("😰")))
elif emotion_text == 'happy':
color = emotion_probability * np.asarray((255, 255, 0))
print(emoji.emojize(emoji.demojize("😃")))
elif emotion_text == 'surprise':
color = emotion_probability * np.asarray((0, 255, 255))
print(emoji.emojize(emoji.demojize("😯")))
elif emotion_text == 'disgusted':
color = emotion_probability * np.asarray((0, 255, 255))
print(emoji.emojize(emoji.demojize("😣")))
elif emotion_text == 'fearful':
color = emotion_probability * np.asarray((0, 255, 255))
print(emoji.emojize(emoji.demojize("😟")))
else:
color = emotion_probability * np.asarray((0, 255, 0))
print(emotion_text)
color = color.astype(int)
color = color.tolist()
draw_bounding_box(face_coordinates, rgb_image, color)
draw_text(face_coordinates, rgb_image, emotion_mode, color, 0, -45,
1, 1)
bgr_image = cv2.cvtColor(rgb_image, cv2.COLOR_RGB2BGR)
cv2.imshow('window_frame', bgr_image)
if cv2.waitKey(1) & 0xFF == ord('q'):
break
for face_coordinates in faces:
x1, x2, y1, y2 = apply_offsets(face_coordinates, offsets)
gray_face = gray_image[y1:y2, x1:x2]
try:
gray_face = cv2.resize(gray_face, target_size)
except:
continue
gray_face = preprocess_input(gray_face, True)
gray_face = np.expand_dims(gray_face, 0)
gray_face = np.expand_dims(gray_face, -1)
guided_gradCAM = calculate_guided_gradient_CAM(gray_face,
gradient_function, saliency_function)
guided_gradCAM = cv2.resize(guided_gradCAM, (x2 - x1, y2 - y1))
try:
rgb_guided_gradCAM = np.repeat(guided_gradCAM[:, :, np.newaxis],
3, axis=2)
rgb_image[y1:y2, x1:x2, :] = rgb_guided_gradCAM
except:
continue
draw_bounding_box((x1, y1, x2 - x1, y2 - y1), rgb_image, color)
bgr_image = cv2.cvtColor(rgb_image, cv2.COLOR_RGB2BGR)
try:
cv2.imshow('window_frame', bgr_image)
except:
continue
if cv2.waitKey(1) & 0xFF == ord('q'):
break