def getFaceDetails(image_np):
gray_image = cv2.cvtColor(image_np, cv2.COLOR_RGB2GRAY) # the model requires a grayscale image
faces = detect_faces(face_detection_model, gray_image) # pass in the grayscale image
if len(faces) > 0: # make sure a face was found
draw_bounding_box(faces[0], image_np) # draws a square around the face, just for testing purposes
# doing preprocessing on grayscale image, found from the model creator's example
x1, x2, y1, y2 = apply_offsets(faces[0], emotion_offsets)
gray_face = gray_image[y1:y2, x1:x2]
gray_face = cv2.resize(gray_face, (emotion_target_size))
gray_face = preprocess_input(gray_face, True)
gray_face = np.expand_dims(gray_face, 0)
gray_face = np.expand_dims(gray_face, -1)
with graph.as_default(): # to prevent graph not found error
emotion_prediction = emotion_classifier.predict(gray_face) # predict the emotion
#emotion_probability = np.max(emotion_prediction) # uncomment this line to see the accuracy of the emotion
emotion_label_arg = np.argmax(emotion_prediction)
face_details = [emotion_labels[emotion_label_arg], int(faces[0][0]), int(faces[0][1]), int(faces[0][2]), int(faces[0][3])] # structure to return emotion and face coords
else:
face_details = [None, None, None, None, None] # in case no face was found
thread_storage['face'] = face_details # instead of returning, put it into dictionary
def detection(image_path):
# parameters for loading data and images
detection_model_path = 'C:\\Users\\l1f15bscs0049\\Desktop\\haarcascade_frontalface_default.xml'
font = cv2.FONT_HERSHEY_SIMPLEX
# hyper-parameters for bounding boxes shape
gender_offsets = (30, 60)
gender_offsets = (10, 10)
# loading models
face_detection = load_detection_model(detection_model_path)
# loading images
rgb_image = load_image(image_path, grayscale=False)
gray_image = load_image(image_path, grayscale=True)
gray_image = np.squeeze(gray_image)
gray_image = gray_image.astype('uint8')
count1 = 0
faces = detect_faces(face_detection, gray_image)
for face_coordinates in faces:
x1, x2, y1, y2 = apply_offsets(face_coordinates, gender_offsets)
rgb_face = rgb_image[y1:y2, x1:x2]
rgb_face = preprocess_input(rgb_face, False)
rgb_face = np.expand_dims(rgb_face, 0)
color = (0, 0, 255)
draw_bounding_box(face_coordinates, rgb_image, color)
count1 = count1 + 1
bgr_image = cv2.cvtColor(rgb_image, cv2.COLOR_RGB2BGR)
#cv2.imwrite('C:\\Users\\l1f15bscs0049\\Desktop\\test_cases\\testt2.png', bgr_image)
return count1
def process_image(image):
try:
# parameters for loading data and images
detection_model_path = './trained_models/detection_models/haarcascade_frontalface_default.xml'
emotion_model_path = './trained_models/emotion_models/fer2013_35a93d1a30cf52b8722837f3e096b7b1006949325de8d00f26595d5a418241d3.hdf5'
emotion_labels = get_labels('fer2013')
font = cv2.FONT_HERSHEY_SIMPLEX
# hyper-parameters for bounding boxes shape
emotion_offsets = (20, 40)
emotion_offsets = (0, 0)
# loading models
face_detection = load_detection_model(detection_model_path)
emotion_classifier = load_model(emotion_model_path, compile=False)
# getting input model shapes for inference
emotion_target_size = emotion_classifier.input_shape[1:3]
# loading images
image_array = np.fromstring(image, np.uint8)
unchanged_image = cv2.imdecode(image_array, cv2.IMREAD_UNCHANGED)
rgb_image = cv2.cvtColor(unchanged_image, cv2.COLOR_BGR2RGB)
gray_image = cv2.cvtColor(unchanged_image, cv2.COLOR_BGR2GRAY)
faces = detect_faces(face_detection, gray_image)
for face_coordinates in faces:
x1, x2, y1, y2 = apply_offsets(face_coordinates, emotion_offsets)
gray_face = gray_image[y1:y2, x1:x2]
try:
gray_face = cv2.resize(gray_face, (emotion_target_size))
except:
continue
rgb_face = preprocess_input(rgb_face, False)
rgb_face = np.expand_dims(rgb_face, 0)
gray_face = preprocess_input(gray_face, True)
gray_face = np.expand_dims(gray_face, 0)
gray_face = np.expand_dims(gray_face, -1)
emotion_label_arg = np.argmax(emotion_classifier.predict(gray_face))
emotion_text = emotion_labels[emotion_label_arg]
color = (255, 0, 0)
draw_bounding_box(face_coordinates, rgb_image, color)
draw_text(face_coordinates, rgb_image, emotion_text, color, 0, -50, 1, 2)
except Exception as err:
logging.error('Error in emotion gender processor: "{0}"'.format(err))
bgr_image = cv2.cvtColor(rgb_image, cv2.COLOR_RGB2BGR)
dirname = 'result'
if not os.path.exists(dirname):
os.mkdir(dirname)
cv2.imwrite(os.path.join(dirname, 'predicted_image.png'), bgr_image)
def final_ml_predict(bgr_image):
gray_image = cv2.cvtColor(bgr_image, cv2.COLOR_BGR2GRAY)
rgb_image = cv2.cvtColor(bgr_image, cv2.COLOR_BGR2RGB)
faces = face_cascade.detectMultiScale(gray_image,
scaleFactor=1.1,
minNeighbors=5,
minSize=(30, 30),
flags=cv2.CASCADE_SCALE_IMAGE)
for face_coordinates in faces:
x1, x2, y1, y2 = apply_offsets(face_coordinates, emotion_offsets)
gray_face = gray_image[y1:y2, x1:x2]
try:
gray_face = cv2.resize(gray_face, (emotion_target_size))
except:
continue
gray_face = preprocess_input(gray_face, True)
gray_face = np.expand_dims(gray_face, 0)
gray_face = np.expand_dims(gray_face, -1)
with graph.as_default():
emotion_prediction = emotion_classifier.predict(gray_face)
emotion_probability = np.max(emotion_prediction)
emotion_label_arg = np.argmax(emotion_prediction)
emotion_text = emotion_labels[emotion_label_arg]
emotion_window.append(emotion_text)
if len(emotion_window) > frame_window:
emotion_window.pop(0)
try:
emotion_mode = mode(emotion_window)
except:
continue
if emotion_text == 'angry':
color = emotion_probability * np.asarray((255, 0, 0))
elif emotion_text == 'sad':
color = emotion_probability * np.asarray((0, 0, 255))
elif emotion_text == 'happy':
color = emotion_probability * np.asarray((255, 255, 0))
elif emotion_text == 'surprise':
color = emotion_probability * np.asarray((0, 255, 255))
else:
color = emotion_probability * np.asarray((0, 255, 0))
color = color.astype(int)
color = color.tolist()
draw_bounding_box(face_coordinates, rgb_image, color)
draw_text(face_coordinates, rgb_image, emotion_mode, color, 0, -45, 1,
1)
bgr_image = cv2.cvtColor(rgb_image, cv2.COLOR_RGB2BGR)
return bgr_image
def recognition(f_2_s):
# parameters for loading data and images
#image_path = path_r
image_path = image_handler(f_2_s)
detection_model_path = '../trained_models/detection_models/haarcascade_frontalface_default.xml'
emotion_model_path = '../trained_models/emotion_models/fer2013_mini_XCEPTION.102-0.66.hdf5'
emotion_labels = get_labels('fer2013')
font = cv2.FONT_HERSHEY_SIMPLEX
# hyper-parameters for bounding boxes shape
emotion_offsets = (20, 40)
emotion_offsets = (0, 0)
# loading models
face_detection = load_detection_model(detection_model_path)
emotion_classifier = load_model(emotion_model_path, compile=False)
# getting input model shapes for inference
emotion_target_size = emotion_classifier.input_shape[1:3]
# loading images
rgb_image = load_image(image_path, grayscale=False)
gray_image = load_image(image_path, grayscale=True)
gray_image = np.squeeze(gray_image)
gray_image = gray_image.astype('uint8')
faces = detect_faces(face_detection, gray_image)
for face_coordinates in faces:
x1, x2, y1, y2 = apply_offsets(face_coordinates, emotion_offsets)
gray_face = gray_image[y1:y2, x1:x2]
try:
gray_face = cv2.resize(gray_face, (emotion_target_size))
except:
continue
gray_face = preprocess_input(gray_face, True)
gray_face = np.expand_dims(gray_face, 0)
gray_face = np.expand_dims(gray_face, -1)
emotion_label_arg = np.argmax(emotion_classifier.predict(gray_face))
emotion_text = ""
emotion_text = emotion_labels[emotion_label_arg]
color = (255, 0, 0) # 감정 정보 글씨 빨간색, 사각형도
draw_bounding_box(face_coordinates, rgb_image, color)
draw_text(face_coordinates, rgb_image, emotion_text, color, 0, -30,
1.5, 2)
#if(emotion_text == ""):
#recognition(f_2_s)
bgr_image = cv2.cvtColor(rgb_image, cv2.COLOR_RGB2BGR)
cv2.imwrite('../images/predicted_test_image.png', bgr_image) # 변수 활용
check_recoged_img('../images/predicted_test_image.png')
def classify_frame(bgr_image):
# bgr_image = video_capture.read()[1]
gray_image = cv2.cvtColor(bgr_image, cv2.COLOR_BGR2GRAY)
rgb = cv2.cvtColor(bgr_image, cv2.COLOR_BGR2RGB)
max_size = 512
aspect_ratio = rgb.shape[1] / rgb.shape[0]
scaling_factor = max([rgb.shape[1], rgb.shape[0]]) / max_size
new_width = int(rgb.shape[1] / scaling_factor)
new_height = int(rgb.shape[0] / scaling_factor)
rgb_image = cv2.resize(rgb, (new_width, new_height))
faces = detect_faces(face_detection, rgb_image)
for face_coordinates in faces:
x1, x2, y1, y2 = apply_offsets(face_coordinates, gender_offsets)
rgb_face = rgb_image[y1:y2, x1:x2]
gray_face = gray_image[y1:y2, x1:x2]
try:
rgb_face = cv2.resize(rgb_face, (gender_target_size))
except:
continue
gray_face = preprocess_input(gray_face, False)
gray_face = np.expand_dims(gray_face, 0)
gray_face = np.expand_dims(gray_face, -1)
rgb_face = np.expand_dims(rgb_face, 0)
rgb_face = preprocess_input(rgb_face, False)
gender_prediction = gender_classifier.predict(rgb_face)
gender_label_arg = np.argmax(gender_prediction)
gender_text = gender_labels[gender_label_arg]
gender_window.append(gender_text)
if len(gender_window) > frame_window:
gender_window.pop(0)
try:
gender_mode = mode(gender_window)
except:
continue
if gender_text == gender_labels[0]:
color = (0, 0, 255)
else:
color = (255, 0, 0)
draw_bounding_box(face_coordinates, rgb_image, color)
draw_text(face_coordinates, rgb_image, gender_mode,
color, 0, -20, 1, 1)
bgr_image = cv2.cvtColor(rgb_image, cv2.COLOR_RGB2BGR)
cv2.imshow('window_frame', bgr_image)
def select_mode(self):
""" Select a mode: Easy or Hard.
"""
self.tickcount + 1
if self.raspberry:
self.tickcount += 1
bgr_image = self.capture_frame()
# Draw "Easy" and "Hard".
# bgr_image = self.overlayUI(bgr_image)
easy_coord = (self.screenwidth // 8, (self.screenheight * 3) // 4)
draw_text(easy_coord, bgr_image, "Easy", font_scale=3)
hard_coord = (self.screenwidth // 2, (self.screenheight * 3) // 4)
draw_text(hard_coord, bgr_image, "Hard", font_scale=3)
# Listen for mode selection.
if self.currPosX and self.currPosX < self.screenwidth / 2:
cv2.rectangle(self.overlay, (0, 0),
(self.screenwidth // 2, int(self.screenheight)),
(211, 211, 211), -1)
else:
cv2.rectangle(self.overlay, (self.screenwidth // 2, 0),
(self.screenwidth, self.screenheight),
(211, 211, 211), -1)
if self.click_point_x: # If user clicks left mouse button.
# OPTIONAL: Positional mode selection
# self.easy_mode = True if self.click_point_x < self.screenwidth / 2
# else False
self.easy_mode = True
self.tickcount = 0
self.curr_level = 1
self.click_point_x = None
self.click_point_right_x = None
if self.click_point_right_x:
self.easy_mode = False
self.tickcount = 0
self.curr_level = 1
self.click_point_x = None
self.click_point_right_x = None
# Draw faces.
gray_image = cv2.cvtColor(bgr_image, cv2.COLOR_BGR2GRAY)
faces = detect_faces(face_detection, gray_image)
cv2.addWeighted(self.overlay, OPACITY, bgr_image, 1 - OPACITY, 0,
bgr_image)
if self.debug:
for face in faces:
draw_bounding_box(face, bgr_image, (255, 0, 0))
# Draw Christmas logo.
self.draw_hats(bgr_image, faces)
self.draw_christmas_logo(bgr_image) # Only for christmas
# Show image.
cv2.imshow('PartyPi', bgr_image)
def get_face(frame):
detection_model_path = './face.xml'
face_detection = load_detection_model(detection_model_path)
try:
gray_image = cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY)
faces = detect_faces(face_detection, gray_image)
except Exception as e:
print("Exception:", e)
return frame
for face in faces:
draw_bounding_box(face, frame, (255, 0, 0))
return frame
def run_classify(image_path):
emotion_text = ''
gender_text = ''
# loading images
rgb_image = load_image(image_path, grayscale=False)
gray_image = load_image(image_path, grayscale=True)
gray_image = np.squeeze(gray_image)
gray_image = gray_image.astype('uint8')
faces = detect_faces(face_detection, gray_image)
for face_coordinates in faces:
x1, x2, y1, y2 = apply_offsets(face_coordinates, gender_offsets)
rgb_face = rgb_image[y1:y2, x1:x2]
x1, x2, y1, y2 = apply_offsets(face_coordinates, emotion_offsets)
gray_face = gray_image[y1:y2, x1:x2]
try:
rgb_face = cv2.resize(rgb_face, (gender_target_size))
gray_face = cv2.resize(gray_face, (emotion_target_size))
except:
continue
rgb_face = preprocess_input(rgb_face, False)
rgb_face = np.expand_dims(rgb_face, 0)
gender_prediction = gender_classifier.predict(rgb_face)
gender_label_arg = np.argmax(gender_prediction)
gender_text = gender_labels[gender_label_arg]
gray_face = preprocess_input(gray_face, True)
gray_face = np.expand_dims(gray_face, 0)
gray_face = np.expand_dims(gray_face, -1)
emotion_label_arg = np.argmax(emotion_classifier.predict(gray_face))
emotion_text = emotion_labels[emotion_label_arg]
if gender_text == gender_labels[0]:
color = (0, 0, 255)
else:
color = (255, 0, 0)
draw_bounding_box(face_coordinates, rgb_image, color)
draw_text(face_coordinates, rgb_image, gender_text, color, 0, -20, 1,
2)
draw_text(face_coordinates, rgb_image, emotion_text, color, 0, -50, 1,
2)
bgr_image = cv2.cvtColor(rgb_image, cv2.COLOR_RGB2BGR)
cv2.imwrite(os.path.join(dir, '../images/predicted_test_image.png'),
bgr_image)
return ([gender_text, emotion_text])
def emo_Detect(self, bgr_image, face_coordinates):
gray_image = cv2.cvtColor(bgr_image, cv2.COLOR_BGR2GRAY)
x1, y1, x2, y2 = face_coordinates
face_coordinates = [int(x1), int(y1), int(x2 - x1), int(y2 - y1)]
x1, x2, y1, y2 = apply_offsets(face_coordinates, self.emotion_offsets)
gray_face = gray_image[y1:y2, x1:x2]
gray_face = cv2.resize(gray_face, (self.emotion_target_size))
gray_face = preprocess_input(gray_face, True)
gray_face = np.expand_dims(gray_face, 0)
gray_face = np.expand_dims(gray_face, -1)
emotion_prediction = self.emotion_classifier.predict(gray_face)
for idx, probability in enumerate(emotion_prediction[0]):
print(self.emotion_labels[idx], probability)
emotion_probability = np.max(emotion_prediction)
emotion_label_arg = np.argmax(emotion_prediction)
emotion_text = self.emotion_labels[emotion_label_arg]
self.emotion_window.append(emotion_text)
if len(self.emotion_window) > self.frame_window:
self.emotion_window.pop(0)
try:
emotion_mode = mode(self.emotion_window)
print("mode is " + emotion_mode)
except:
pass
if emotion_text == 'angry':
color = emotion_probability * np.asarray((0, 0, 255))
elif emotion_text == 'sad':
color = emotion_probability * np.asarray((255, 0, 0))
elif emotion_text == 'happy':
color = emotion_probability * np.asarray((0, 255, 255))
elif emotion_text == 'surprise':
color = emotion_probability * np.asarray((255, 255, 0))
else:
color = emotion_probability * np.asarray((0, 255, 0))
color = color.astype(int)
color = color.tolist()
draw_bounding_box(face_coordinates, bgr_image, color)
draw_text(face_coordinates, bgr_image, emotion_text, color, 0, -45, 1,
1)
return bgr_image, emotion_prediction[0]
def getFaceEmotion(image_path):
# parameters for loading data and images
base = 'C://Users/lenovo/Desktop/moodify/components/emotion-classification'
detection_model_path = base + '/trained_models/detection_models/haarcascade_frontalface_default.xml'
emotion_model_path = base + '/trained_models/emotion_models/fer2013_denseNet.59-0.68.hdf5'
emotion_labels = get_labels('fer2013')
font = cv2.FONT_HERSHEY_SIMPLEX
# hyper-parameters for bounding boxes shape
emotion_offsets = (0, 0)
# loading models
face_detection = load_detection_model(detection_model_path)
emotion_classifier = load_model(emotion_model_path, compile=False)
# getting input model shapes for inference
emotion_target_size = emotion_classifier.input_shape[1:3]
# loading images
rgb_image = load_image(image_path, grayscale=False)
gray_image = load_image(image_path, grayscale=True)
gray_image = np.squeeze(gray_image)
gray_image = gray_image.astype('uint8')
faces = detect_faces(face_detection, gray_image)
for face_coordinates in faces:
x1, x2, y1, y2 = apply_offsets(face_coordinates, emotion_offsets)
gray_face = gray_image[y1:y2, x1:x2]
try:
gray_face = cv2.resize(gray_face, (emotion_target_size))
except:
continue
gray_face = preprocess_input(gray_face, True)
gray_face = np.expand_dims(gray_face, 0)
gray_face = np.expand_dims(gray_face, -1)
emotion_label_arg = np.argmax(emotion_classifier.predict(gray_face))
emotion_text = emotion_labels[emotion_label_arg]
color = (255, 0, 0)
draw_bounding_box(face_coordinates, rgb_image, color)
draw_text(face_coordinates, rgb_image, emotion_text, color, 0, -10,
0.5, 2)
bgr_image = cv2.cvtColor(rgb_image, cv2.COLOR_RGB2BGR)
cv2.imwrite('predicted_test_image.png', bgr_image)
return emotion_text
def predict(image_path):
# loading images
start = time.time()
rgb_image = load_image(image_path, grayscale=False)
gray_image = load_image(image_path, grayscale=True)
gray_image = np.squeeze(gray_image)
gray_image = gray_image.astype('uint8')
with graph.as_default():
faces = detect_faces(face_detection, gray_image)
for face_coordinates in faces:
x1, x2, y1, y2 = apply_offsets(face_coordinates, gender_offsets)
rgb_face = rgb_image[y1:y2, x1:x2]
x1, x2, y1, y2 = apply_offsets(face_coordinates, emotion_offsets)
gray_face = gray_image[y1:y2, x1:x2]
try:
rgb_face = cv2.resize(rgb_face, (gender_target_size))
gray_face = cv2.resize(gray_face, (emotion_target_size))
except:
continue
rgb_face = preprocess_input(rgb_face, False)
rgb_face = np.expand_dims(rgb_face, 0)
gender_prediction = gender_classifier.predict(rgb_face)
gender_label_arg = np.argmax(gender_prediction)
gender_text = gender_labels[gender_label_arg]
gray_face = preprocess_input(gray_face, True)
gray_face = np.expand_dims(gray_face, 0)
gray_face = np.expand_dims(gray_face, -1)
emotion_label_arg = np.argmax(emotion_classifier.predict(gray_face))
emotion_text = emotion_labels[emotion_label_arg]
if gender_text == gender_labels[0]:
color = (0, 0, 255)
else:
color = (255, 0, 0)
draw_bounding_box(face_coordinates, rgb_image, color)
draw_text(face_coordinates, rgb_image, gender_text, color, 0, -10, 1, 2)
draw_text(face_coordinates, rgb_image, emotion_text, color, 0, -40, 1, 2)
bgr_image = cv2.cvtColor(rgb_image, cv2.COLOR_RGB2BGR)
resized_image = cv2.resize(bgr_image, (1024, 769))
file_path = 'result/' + image_path.split('/')[-1]
cv2.imwrite(file_path, resized_image)
end = time.time()
logging.info('[Time] for prediction(): {}'.format(end - start))
return file_path
def on_data(data):
f = open('current.jpg', 'wb')
f.write(base64.decodebytes(data))
f.close()
image_path = "current.jpg"
out = []
# loading images
rgb_image = load_image(image_path, grayscale=False)
gray_image = load_image(image_path, grayscale=True)
gray_image = np.squeeze(gray_image)
gray_image = gray_image.astype('uint8')
faces = detect_faces(face_detection, gray_image)
for face_coordinates in faces:
x1, x2, y1, y2 = apply_offsets(face_coordinates, emotion_offsets)
gray_face = gray_image[y1:y2, x1:x2]
try:
gray_face = cv2.resize(gray_face, (emotion_target_size))
except:
continue
gray_face = preprocess_input(gray_face, True)
gray_face = np.expand_dims(gray_face, 0)
gray_face = np.expand_dims(gray_face, -1)
emotion_label_arg = np.argmax(emotion_classifier.predict(gray_face))
emotion_text = emotion_labels[emotion_label_arg]
color = (0, 0, 255)
draw_bounding_box(face_coordinates, rgb_image, color)
draw_text(face_coordinates, rgb_image, emotion_text, color, 0, -50, 1,
2)
bgr_image = cv2.cvtColor(rgb_image, cv2.COLOR_RGB2BGR)
cv2.imwrite('predicted.png', bgr_image)
data = open('predicted.png', 'rb').read()
encoded = base64.encodebytes(data).decode('utf-8')
out.append({
'image': encoded,
'emotion': emotion_text,
})
return out
def process_image(image):
try:
# parameters for loading data and images
detection_model_path = './trained_models/detection_models/haarcascade_frontalface_default.xml'
emotion_model_path = './trained_models/emotion_models/fer2013_mini_XCEPTION.102-0.66.hdf5'
gender_model_path = './trained_models/gender_models/simple_CNN.81-0.96.hdf5'
emotion_labels = get_labels('fer2013')
gender_labels = get_labels('imdb')
# hyper-parameters for bounding boxes shape
gender_offsets = (10, 10)
emotion_offsets = (0, 0)
# loading models
face_detection = load_detection_model(detection_model_path)
emotion_classifier = load_model(emotion_model_path, compile=False)
gender_classifier = load_model(gender_model_path, compile=False)
# getting input model shapes for inference
emotion_target_size = emotion_classifier.input_shape[1:3]
gender_target_size = gender_classifier.input_shape[1:3]
# loading images
image_array = np.fromstring(image, np.uint8)
unchanged_image = cv2.imdecode(image_array, cv2.IMREAD_UNCHANGED)
rgb_image = cv2.cvtColor(unchanged_image, cv2.COLOR_BGR2RGB)
gray_image = cv2.cvtColor(unchanged_image, cv2.COLOR_BGR2GRAY)
faces = detect_faces(face_detection, gray_image)
for face_coordinates in faces:
x1, x2, y1, y2 = apply_offsets(face_coordinates, gender_offsets)
rgb_face = rgb_image[y1:y2, x1:x2]
x1, x2, y1, y2 = apply_offsets(face_coordinates, emotion_offsets)
gray_face = gray_image[y1:y2, x1:x2]
try:
rgb_face = cv2.resize(rgb_face, gender_target_size)
gray_face = cv2.resize(gray_face, emotion_target_size)
except:
continue
rgb_face = preprocess_input(rgb_face, False)
rgb_face = np.expand_dims(rgb_face, 0)
gender_prediction = gender_classifier.predict(rgb_face)
gender_label_arg = np.argmax(gender_prediction)
gender_text = gender_labels[gender_label_arg]
gray_face = preprocess_input(gray_face, True)
gray_face = np.expand_dims(gray_face, 0)
gray_face = np.expand_dims(gray_face, -1)
emotion_label_arg = np.argmax(
emotion_classifier.predict(gray_face))
emotion_text = emotion_labels[emotion_label_arg]
if gender_text == gender_labels[0]:
color = (0, 0, 255)
else:
color = (255, 0, 0)
draw_bounding_box(face_coordinates, rgb_image, color)
draw_text(face_coordinates, rgb_image, gender_text, color, 0, -20,
1, 2)
draw_text(face_coordinates, rgb_image, emotion_text, color, 0, -50,
1, 2)
except Exception as err:
logging.error('Error in emotion gender processor: "{0}"'.format(err))
bgr_image = cv2.cvtColor(rgb_image, cv2.COLOR_RGB2BGR)
dirname = 'result'
if not os.path.exists(dirname):
os.mkdir(dirname)
cv2.imwrite(os.path.join(dirname, 'predicted_image.png'), bgr_image)
def r8k9(fileName='dog.jpg', score=10, length=10):
# getting input model shapes for inference
emotion_target_size = emotion_classifier.input_shape[1:3]
# starting lists for calculating modes
emotion_window = []
# starting video streaming
cv2.namedWindow('window_frame')
video_capture = cv2.VideoCapture(0)
im = cv2.imread(fileName)
im_resized = cv2.resize(im, (224, 224), interpolation=cv2.INTER_LINEAR)
countdown = ["Ready?", "3", "2", "1"]
#plt.imshow(cv2.cvtColor(im_resized, cv2.COLOR_BGR2RGB))
#for i in range(4):
# print(countdown[i])
# time.sleep(1)
#print("GO")
#plt.show(block=False)
t = time.time()
td = []
while time.time() < t + length:
bgr_image = video_capture.read()[1]
try:
gray_image = cv2.cvtColor(bgr_image, cv2.COLOR_BGR2GRAY)
except:
time.sleep(.5)
gray_image = cv2.cvtColor(bgr_image, cv2.COLOR_BGR2GRAY)
rgb_image = cv2.cvtColor(bgr_image, cv2.COLOR_BGR2RGB)
faces = detect_faces(face_detection, gray_image)
for face_coordinates in faces:
x1, x2, y1, y2 = apply_offsets(face_coordinates, emotion_offsets)
gray_face = gray_image[y1:y2, x1:x2]
try:
gray_face = cv2.resize(gray_face, (emotion_target_size))
except:
continue
gray_face = preprocess_input(gray_face, True)
gray_face = np.expand_dims(gray_face, 0)
gray_face = np.expand_dims(gray_face, -1)
emotion_prediction = emotion_classifier.predict(gray_face)
#print(emotion_prediction)
if len(td) == 0:
data = emotion_prediction
td = [time.time() - t]
else:
data = np.concatenate((data, emotion_prediction), axis=0)
td = td + [time.time() - t]
emotion_probability = np.max(emotion_prediction)
emotion_label_arg = np.argmax(emotion_prediction)
emotion_text = emotion_labels[emotion_label_arg]
emotion_window.append(emotion_text)
if len(emotion_window) > frame_window:
emotion_window.pop(0)
try:
emotion_mode = mode(emotion_window)
except:
continue
if emotion_text == 'angry':
color = emotion_probability * np.asarray((255, 0, 0))
elif emotion_text == 'sad':
color = emotion_probability * np.asarray((0, 0, 255))
elif emotion_text == 'happy':
color = emotion_probability * np.asarray((255, 255, 0))
elif emotion_text == 'surprise':
color = emotion_probability * np.asarray((0, 255, 255))
else:
color = emotion_probability * np.asarray((0, 255, 0))
color = color.astype(int)
color = color.tolist()
draw_bounding_box(face_coordinates, rgb_image, color)
draw_text(face_coordinates, rgb_image, emotion_mode, color, 0, -45,
1, 1)
bgr_image = cv2.cvtColor(rgb_image, cv2.COLOR_RGB2BGR)
cv2.imshow('window_frame', bgr_image)
if cv2.waitKey(1) & 0xFF == ord('q'):
break
#print(np.transpose(data))
#print(emotion_labels)
plt.clf()
plt.plot(td, data)
plt.legend(
['angry', 'disgust', 'fear', 'happy', 'sad', 'surprise', 'neutral'],
loc=2)
plt.savefig('Graph.png')
#splt.show()
totals = [0, 0, 0, 0, 0, 0, 0]
for j in range(7):
for i in range(1, len(td)):
#print(time[i-1])
#print(data[i-1+l*(j+1)])
#print(time[i],time[i-1])
totals[j] += (data[i][j] + data[i - 1][j]) / 2 * (
td[i] - td[i - 1]) / (td[-1] - td[0])
np.append(totals, np.std(data))
print(totals)
#print(td[-1] - td[0])
#print(sum(totals))
#f = open("DogData.txt","a+")
#f.write((str(score) + ' ' + ' '.join(str(t) for t in totals) + "\n"))
print("Rating:", 15 - spot(np.array(totals)) / 10, "/10")
return (15 - spot(np.array(totals)) / 10)
def get_sim_percent(image_1, image_2, target_class):
# parameters for loading data and images
# image_path = sys.argv[1]
detection_model_path = 'trained_models/detection_models/haarcascade_frontalface_default.xml'
emotion_model_path = 'trained_models/fer2013_big_XCEPTION.54-0.66.hdf5'
gender_model_path = 'trained_models/gender_models/simple_CNN.81-0.96.hdf5'
emotion_labels = get_labels('fer2013')
gender_labels = get_labels('imdb')
font = cv2.FONT_HERSHEY_SIMPLEX
# hyper-parameters for bounding boxes shape
gender_offsets = (20, 40)
gender_offsets = (10, 10)
emotion_offsets = (10, 30)
emotion_offsets = (0, 0)
# loading models
face_detection = load_detection_model(detection_model_path)
emotion_classifier = load_model(emotion_model_path, compile=False)
gender_classifier = load_model(gender_model_path, compile=False)
# getting input model shapes for inference
emotion_target_size = emotion_classifier.input_shape[1:3]
gender_target_size = gender_classifier.input_shape[1:3]
target_emo = target_class
image_path = image_1
# loading images
rgb_image = load_image(image_path, grayscale=False)
gray_image = load_image(image_path, grayscale=True)
gray_image = np.squeeze(gray_image)
gray_image = gray_image.astype('uint8')
faces = detect_faces(face_detection, gray_image)
for face_coordinates in faces:
x1, x2, y1, y2 = apply_offsets(face_coordinates, gender_offsets)
rgb_face = rgb_image[y1:y2, x1:x2]
x1, x2, y1, y2 = apply_offsets(face_coordinates, emotion_offsets)
gray_face = gray_image[y1:y2, x1:x2]
try:
rgb_face = cv2.resize(rgb_face, (gender_target_size))
gray_face = cv2.resize(gray_face, (emotion_target_size))
except:
continue
rgb_face = preprocess_input(rgb_face, False)
rgb_face = np.expand_dims(rgb_face, 0)
gender_prediction = gender_classifier.predict(rgb_face)
gender_label_arg = np.argmax(gender_prediction)
gender_text = gender_labels[gender_label_arg]
gray_face = preprocess_input(gray_face, True)
gray_face = np.expand_dims(gray_face, 0)
gray_face = np.expand_dims(gray_face, -1)
emotion_label_arg = emotion_classifier.predict(gray_face)[0][target_emo]
emotion_percent = str(round(emotion_label_arg*100, 2))
emotion_text = emotion_labels[target_emo] + ' - '+ emotion_percent
# print(target_emo)
# print(emotion_text)
if gender_text == gender_labels[0]:
color = (0, 0, 255)
else:
color = (255, 0, 0)
draw_bounding_box(face_coordinates, rgb_image, color)
draw_text(face_coordinates, rgb_image, gender_text, color, -20, 30, 1, 2)
draw_text(face_coordinates, rgb_image, emotion_text, color, 0, 0, 1, 2)
bgr_image = cv2.cvtColor(rgb_image, cv2.COLOR_RGB2BGR)
cv2.imwrite('predicted_test_image.png', bgr_image)
return emotion_percent
x1, x2, y1, y2 = apply_offsets(face_coordinates, offsets)
gray_face = gray_image[y1:y2, x1:x2]
# processing input
try:
gray_face = cv2.resize(gray_face, (target_size))
except:
continue
gray_face = preprocess_input(gray_face, True)
gray_face = np.expand_dims(gray_face, 0)
gray_face = np.expand_dims(gray_face, -1)
# prediction
predicted_class = np.argmax(model.predict(gray_face))
label_text = labels[predicted_class]
gradient_function = compile_gradient_function(model,
predicted_class, 'conv2d_7')
register_gradient()
guided_model = modify_backprop(model, 'GuidedBackProp', task)
saliency_function = compile_saliency_function(guided_model, 'conv2d_7')
guided_gradCAM = calculate_guided_gradient_CAM(gray_face,
gradient_function, saliency_function)
guided_gradCAM = cv2.resize(guided_gradCAM, (x2-x1, y2-y1))
rgb_guided_gradCAM = np.repeat(guided_gradCAM[:, :, np.newaxis], 3, axis=2)
rgb_image[y1:y2, x1:x2, :] = rgb_guided_gradCAM
draw_bounding_box((x1, y1, x2 - x1, y2 - y1), rgb_image, color)
bgr_image = cv2.cvtColor(rgb_image, cv2.COLOR_RGB2BGR)
cv2.imwrite('../images/guided_gradCAM.png', bgr_image)
def callback(self,data):
#/////////////////////////////////////////////
#cv_image = bridge.imgmsg_to_cv2(image_msg, desired_encoding="passthrough")
#cv_image = cv2.resize(cv_image, target_size) # resize image
#np_image = np.asarray(cv_image) # read as np array
#np_image = np.expand_dims(np_image, axis=0) # Add another dimension for tensorflow
#np_image = np_image.astype(float) # preprocess needs float64 and img is uint8
#np_image = preprocess_input(np_image) # Regularize the data
#/////////////////////////////////////////////
if(not USE_LOCAL_CAMERA):
try:
frame = self.bridge.imgmsg_to_cv2(data, desired_encoding="passthrough")
except CvBridgeError as e:
print(e)
# Capture frame-by-frame
if(USE_LOCAL_CAMERA):
ret, frame1 = self.video_capture.read()
#/////////////////////////////////////////////
#print"--------"
#print('input_msg height : {}'.format(frame.height))
#print('input_msg width : {}'.format(frame.width))
#print('input_msg step : {}'.format(frame.step))
#print('input_msg encoding: {}'.format(frame.encoding))
#print('output dtype : {}'.format(frame.dtype))
#print('output shape : {}'.format(frame.shape))
#print"--------"
gray_image = cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY)
rgb_image = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)
faces = detect_faces(self.face_detection, gray_image)
for face_coordinates in faces:
print "inside the for loop!"
print face_coordinates
x1, x2, y1, y2 = apply_offsets(face_coordinates, self.emotion_offsets)
gray_face = gray_image[y1:y2, x1:x2]
#print len(frame)
#print gray_face.size
#print gray_face.shape
#print gray_face.dtype
try:
gray_face = cv2.resize(gray_face, (self.emotion_target_size))
except:
continue
gray_face = preprocess_input(gray_face, True)
gray_face = np.expand_dims(gray_face, 0)
gray_face = np.expand_dims(gray_face, -1)
#print"************"
#print('gray_face dtype : {}'.format(gray_face.dtype))
#print('gray_face shape : {}'.format(gray_face.shape))
#print('gray_face size : {}'.format(gray_face.size))
#print"************"
## This is a workaround for asynchronous execution using TF and ROS
# https://github.com/keras-team/keras/issues/2397
# http://projectsfromtech.blogspot.com/2017/10/visual-object-recognition-in-ros-using.html
#global self.graph
with self.graph.as_default():
emotion_prediction = self.emotion_classifier.predict(gray_face)
emotion_probability = np.max(emotion_prediction)
emotion_label_arg = np.argmax(emotion_prediction)
emotion_text = self.emotion_labels[emotion_label_arg]
print emotion_text
print(emotion_probability)
print('%')
self.emotion_window.append(emotion_text)
#self.emotion_msg.data = emotion_text
#self.emotion_publisher.publish(emotion_msg)
#self.speech_msg.data = 'I see that you are ' + emotion_text
#self.speech_publisher.publish(speech_msg)
if len(self.emotion_window) > self.frame_window:
self.emotion_window.pop(0)
try:
emotion_mode = mode(self.emotion_window)
except:
continue
if emotion_text == 'angry':
color = emotion_probability * np.asarray((255, 0, 0))
elif emotion_text == 'sad':
color = emotion_probability * np.asarray((0, 0, 255))
elif emotion_text == 'happy':
color = emotion_probability * np.asarray((255, 255, 0))
elif emotion_text == 'surprise':
color = emotion_probability * np.asarray((0, 255, 255))
else:
color = emotion_probability * np.asarray((0, 255, 0))
color = color.astype(int)
color = color.tolist()
draw_bounding_box(face_coordinates, rgb_image, color)
draw_text(face_coordinates, rgb_image, emotion_mode,
color, 0, -45, 1, 1)
try:
self.image_pub.publish(self.bridge.cv2_to_imgmsg(rgb_image, "bgr8"))
except CvBridgeError as e:
print(e)
if len(emotion_window) > frame_window:
emotion_window.pop(0)
try:
emotion_mode = mode(emotion_window)
except:
continue
if emotion_text == 'angry':
color = emotion_probability * np.asarray((255, 0, 0))
elif emotion_text == 'sad':
color = emotion_probability * np.asarray((0, 0, 255))
elif emotion_text == 'happy':
color = emotion_probability * np.asarray((255, 255, 0))
elif emotion_text == 'surprise':
color = emotion_probability * np.asarray((0, 255, 255))
else:
color = emotion_probability * np.asarray((0, 255, 0))
color = color.astype(int)
color = color.tolist()
draw_bounding_box(face_coordinates, rgb_image, color)
draw_text(face_coordinates, rgb_image, emotion_mode,
color, 0, -45, 1, 1)
bgr_image = cv2.cvtColor(rgb_image, cv2.COLOR_RGB2BGR)
cv2.imshow('window_frame', bgr_image)
if cv2.waitKey(1) & 0xFF == ord('q'):
break
def biaoqing(frame):
gray_image = cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY)
rgb_image = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)
dets = detector(gray_image, 1)
#print(dets)
#faces = detect_faces(face_detection, gray_image)#[[横，纵，宽，高]]
#print(faces)
max_face = 0
max_k = 0
emotion_text = 0
for k,d in enumerate(dets):
if (d.right()-d.left())*(d.bottom()-d.top()) > max_face:
max_face = (d.right()-d.left())*(d.bottom()-d.top())
max_k = k
for k,d in enumerate(dets):
if k != max_k: continue
face_coordinates=[]
face_coordinates.append(d.left())
face_coordinates.append(d.top())
face_coordinates.append(d.right()-d.left())
face_coordinates.append(d.bottom()-d.top())
x1, x2, y1, y2 = apply_offsets(face_coordinates, emotion_offsets)
gray_face = gray_image[y1:y2, x1:x2]
try:
gray_face = cv2.resize(gray_face, (emotion_target_size))
except:
continue
gray_face = preprocess_input(gray_face, True)
gray_face = np.expand_dims(gray_face, 0)
gray_face = np.expand_dims(gray_face, -1)
emotion_prediction = emotion_classifier.predict(gray_face)
emotion_probability = np.max(emotion_prediction)
emotion_label_arg = np.argmax(emotion_prediction)
emotion_text = emotion_labels[emotion_label_arg]
emotion_window.append(emotion_text)
if len(emotion_window) > frame_window:
emotion_window.pop(0)
try:
emotion_mode = mode(emotion_window)
except:
continue
if emotion_text == 'angry':
color = emotion_probability * np.asarray((255, 0, 0))
elif emotion_text == 'sad':
color = emotion_probability * np.asarray((0, 0, 255))
elif emotion_text == 'happy':
color = emotion_probability * np.asarray((255, 255, 0))
elif emotion_text == 'surprise':
color = emotion_probability * np.asarray((0, 255, 255))
else:
color = emotion_probability * np.asarray((0, 255, 0))
color = color.astype(int)
color = color.tolist()
draw_bounding_box(face_coordinates, rgb_image, color)
draw_text(face_coordinates, rgb_image, emotion_mode,
color, 0, -45, 1, 1)
bgr_image = cv2.cvtColor(rgb_image, cv2.COLOR_RGB2BGR)
# cv2.imshow('out', bgr_image)
if emotion_text:
return emotion_text, bgr_image
else:
return 0, frame
def processa():
# parameters for loading data and images
emotion_model_path = './models/emotion_model.hdf5' #Carica il modello delle emozioni , pre-addestrato
emotion_labels = get_labels('fer2013')
# hyper-parameters for bounding boxes shape
frame_window = 10
emotion_offsets = (20, 40)
# loading models #Rilevamento caratteristiche del viso
detector = dlib.get_frontal_face_detector(
) #Rilevamento caratteristiche del viso
emotion_classifier = load_model(emotion_model_path)
# predictor = dlib.shape_predictor("shape_predictor_68_face_landmarks.dat")
# getting input model shapes for inference
emotion_target_size = emotion_classifier.input_shape[
1:3] #Fa una stima parziale del modello di input
# starting lists for calculating modes
emotion_window = []
# Initialize some variables
face_locations = []
face_encodings = [] #Delimita i volti umani
face_names = []
process_this_frame = True
def face_compare(frame, process_this_frame):
# Resize frame of video to 1/4 size for faster face recognition processing
small_frame = cv2.resize(frame, (0, 0), fx=0.50, fy=0.50)
# Convert the image from BGR color (which OpenCV uses) to RGB color (which face_recognition uses)
rgb_small_frame = small_frame[:, :, ::-1]
# Only process every other frame of video to save time
if process_this_frame:
# Find all the faces and face encodings in the current frame of video(cerca i volti nel frame video)
face_locations = face_recognition.face_locations(
rgb_small_frame) #Delimitazione del volto umano
face_encodings = face_recognition.face_encodings(
rgb_small_frame, face_locations
) #apprende se c'è un volto umano tramite face location
# starting video streaming
cv2.namedWindow('window_frame')
cap = None
# Webcam source
cap = cv2.VideoCapture(0)
while cap.isOpened(): # True:
ret, frame = cap.read(
) #restituisce le immagini della webcam frame per frame
#frame = video_capture.read()[1]
# To print the facial landmarks
# landmrk = face_recognition.face_landmarks(frame)
# for l in landmrk:
# for key,val in l.items():
# for (x,y) in val:
# cv2.circle(frame, (x, y), 1, (255,0, 0), -1)
gray_image = cv2.cvtColor(
frame, cv2.COLOR_BGR2GRAY) #trasforma i frame in scala di grigio
rgb_image = cv2.cvtColor(frame,
cv2.COLOR_BGR2RGB) #trasforma i frame in rgb
faces = detector(rgb_image) #rileva caratteristiche del viso
# face_locations = face_recognition.face_locations(rgb_image)
# print (reversed(face_locations))
face_name = face_compare(rgb_image, process_this_frame)
for face_coordinates, fname in zip(faces, "Unknow"):
x1, x2, y1, y2 = apply_offsets(
face_utils.rect_to_bb(face_coordinates), emotion_offsets)
gray_face = gray_image[y1:y2, x1:x2]
try:
gray_face = cv2.resize(gray_face, (emotion_target_size))
except:
continue
gray_face = preprocess_input(gray_face, True)
gray_face = np.expand_dims(gray_face, 0)
gray_face = np.expand_dims(gray_face, -1)
emotion_prediction = emotion_classifier.predict(
gray_face) #fa delle predizioni
emotion_probability = np.max(
emotion_prediction) #prende la predizione con valore massimo
emotion_label_arg = np.argmax(
emotion_prediction
) #prende l'indice della predizione con valore massimo
emotion_text = emotion_labels[
emotion_label_arg] #inserisce nella variabile l'umore predetto
emotion_window.append(emotion_text)
if len(emotion_window) > frame_window:
emotion_window.pop(0)
try:
emotion_mode = mode(
emotion_window
) #restituisce il dato più comune nella lista
except:
continue
if emotion_text == 'Triste':
color = emotion_probability * np.asarray((0, 0, 255))
elif emotion_text == 'Felice':
color = emotion_probability * np.asarray((255, 255, 0))
else:
color = emotion_probability * np.asarray((0, 255, 0))
color = color.astype(int)
color = color.tolist() #converte un colore in un elenco del modulo
name = emotion_text
draw_bounding_box(face_utils.rect_to_bb(face_coordinates),
rgb_image, color) #crea un rettangolo colorato
draw_text(face_utils.rect_to_bb(face_coordinates), rgb_image, name,
color, 0, -45, 0.5,
1) #inserisce l'emozione predetta colorata
frame = cv2.cvtColor(rgb_image,
cv2.COLOR_RGB2BGR) #converte da rgb a bgr
cv2.imshow('window_frame',
frame) #mostra il frame nella finestra windows_frame
if cv2.waitKey(1) & 0xFF == ord(
'q'): #se c'è un ritardo di un milli secondo allora break
return emotion_text
cap.release()
cv2.destroyAllWindows()
def callback(self,data):
try:
bgr_image = self.bridge.imgmsg_to_cv2(data, "bgr8")
cv2.imshow("Image window", bgr_image)
except CvBridgeError as e:
print(e)
######################### Start the image processing
gray_image = cv2.cvtColor(bgr_image, cv2.COLOR_BGR2GRAY)
rgb_image = cv2.cvtColor(bgr_image, cv2.COLOR_BGR2RGB)
faces = detect_faces(self.face_detection, gray_image)
for face_coordinates in faces:
x1, x2, y1, y2 = apply_offsets(face_coordinates, self.emotion_offsets)
gray_face = gray_image[y1:y2, x1:x2]
try:
gray_face = cv2.resize(gray_face, (self.emotion_target_size))
except:
continue
gray_face = preprocess_input(gray_face, True)
gray_face = np.expand_dims(gray_face, 0)
gray_face = np.expand_dims(gray_face, -1)
print "start"
print len(gray_face)
print gray_face
print "end"
emotion_prediction = self.emotion_classifier.predict(gray_face)
emotion_probability = np.max(emotion_prediction)
emotion_label_arg = np.argmax(emotion_prediction)
emotion_text = emotion_labels[emotion_label_arg]
print emotion_text
print(emotion_probability)
print('%')
self.emotion_window.append(emotion_text)
if len(self.emotion_window) > self.frame_window:
self.emotion_window.pop(0)
try:
emotion_mode = mode(self.emotion_window)
except:
continue
if emotion_text == 'angry':
color = emotion_probability * np.asarray((255, 0, 0))
elif emotion_text == 'sad':
color = emotion_probability * np.asarray((0, 0, 255))
elif emotion_text == 'happy':
color = emotion_probability * np.asarray((255, 255, 0))
elif emotion_text == 'surprise':
color = emotion_probability * np.asarray((0, 255, 255))
else:
color = emotion_probability * np.asarray((0, 255, 0))
color = color.astype(int)
color = color.tolist()
draw_bounding_box(face_coordinates, rgb_image, color)
draw_text(face_coordinates, rgb_image, emotion_mode,
color, 0, -45, 1, 1)
bgr_image = cv2.cvtColor(rgb_image, cv2.COLOR_RGB2BGR)
######################### End the image processing
cv2.imshow("Image window", bgr_image)
cv2.waitKey(3)
try:
self.image_pub.publish(self.bridge.cv2_to_imgmsg(bgr_image, "bgr8"))
except CvBridgeError as e:
print(e)
def emotion_demo():
# parameters for loading data and images
detection_model_path = '../trained_models/detection_models/haarcascade_frontalface_default.xml'
emotion_model_path = '../trained_models/emotion_models/fer2013_mini_XCEPTION.102-0.66.hdf5'
emotion_labels = get_labels('fer2013')
# 目のカスケードファイル追加
lefteyecc__path = "../trained_models/detection_models/haarcascade_lefteye_2splits.xml"
righteyecc_path = "../trained_models/detection_models/haarcascade_righteye_2splits.xml"
nose_path = "../trained_models/detection_models/data_haarcascades_haarcascade_mcs_nose.xml"
lefteyecc = cv2.CascadeClassifier(lefteyecc__path)
righteyecc = cv2.CascadeClassifier(righteyecc_path)
nose = cv2.CascadeClassifier(nose_path)
lex = 0; ley = 0; lew = 0; leh = 0
rex = 0; rey = 0; rew = 0; reh = 0
nox = 0; noy = 0; now = 0; noh = 0
# dlib
dlib_ini()
# hyper-parameters for bounding boxes shape
frame_window = 10
emotion_offsets = (20, 40)
# loading models
face_detection = load_detection_model(detection_model_path)
emotion_classifier = load_model(emotion_model_path, compile=False)
# getting input model shapes for inference
emotion_target_size = emotion_classifier.input_shape[1:3]
# starting lists for calculating modes
emotion_window = []
global img, flag, slp_count
img = cv2.imread('../img/happy.png')
flag = 0
slp_count = 0
# dlib用グローバル変数
global gray_image, rgb_image, gray_face, mark_list
# starting video streaming
cv2.namedWindow('window_frame', cv2.WINDOW_NORMAL)
video_capture = cv2.VideoCapture(0) # 0は内蔵カメラ, 1はUSBカメラ
while True:
bgr_image = video_capture.read()[1]
gray_image = cv2.cvtColor(bgr_image, cv2.COLOR_BGR2GRAY)
rgb_image = cv2.cvtColor(bgr_image, cv2.COLOR_BGRA2RGBA)
faces = detect_faces(face_detection, gray_image)
for face_coordinates in faces:
# 目や鼻認識用
(x,y,w,h) = face_coordinates
video_face = gray_image[y:y+h,x:x+w]
x1, x2, y1, y2 = apply_offsets(face_coordinates, emotion_offsets)
gray_face = gray_image[y1:y2, x1:x2]
try:
gray_face = cv2.resize(gray_face, (emotion_target_size))
except:
continue
# ランドマーク検出
marks_list = marks_list_def(bgr_image, x, y, w, h)
print(marks_list)
gray_face = preprocess_input(gray_face, True)
gray_face = np.expand_dims(gray_face, 0)
gray_face = np.expand_dims(gray_face, -1)
emotion_prediction = emotion_classifier.predict(gray_face)
emotion_probability = np.max(emotion_prediction)
emotion_label_arg = np.argmax(emotion_prediction)
emotion_text = emotion_labels[emotion_label_arg]
emotion_window.append(emotion_text)
if len(emotion_window) > frame_window:
emotion_window.pop(0)
try:
emotion_mode = mode(emotion_window)
except:
continue
if flag == 0 or flag == 1:
if emotion_text == 'angry':
img = cv2.imread('../img/angry.png', -1)
color = emotion_probability * np.asarray((255, 0, 0))
elif emotion_text == 'sad':
img = cv2.imread('../img/sad.png', -1) # 関数にする
color = emotion_probability * np.asarray((0, 0, 255))
elif emotion_text == 'happy':
img = cv2.imread('../img/happy.png', -1)
color = emotion_probability * np.asarray((255, 255, 0))
elif emotion_text == 'surprise':
img = cv2.imread('../img/odoroki.png', -1)
color = emotion_probability * np.asarray((0, 255, 255))
else :
img = cv2.imread('../img/neutral.png', -1)
color = emotion_probability * np.asarray((0, 255, 0))
else:
if emotion_text == 'angry':
img = cv2.imread('../img/ikari.png', -1)
color = emotion_probability * np.asarray((255, 0, 0))
elif emotion_text == 'sad':
img = cv2.imread('../img/shock.png', -1)
color = emotion_probability * np.asarray((0, 0, 255))
elif emotion_text == 'happy':
img = cv2.imread('../img/kirakira.png', -1)
color = emotion_probability * np.asarray((255, 255, 0))
elif emotion_text == 'surprise':
img = cv2.imread('../img/bikkuri.png', -1)
color = emotion_probability * np.asarray((0, 255, 255))
else :
img = cv2.imread('../img/toumei.png', -1)
color = emotion_probability * np.asarray((0, 255, 0))
color = color.astype(int)
color = color.tolist()
if flag == 0:
draw_bounding_box(face_coordinates, rgb_image, color)
elif flag == 1:
rgb_image = draw_bounding_box2(face_coordinates, rgb_image, color, img, marks_list)
elif flag == 2:
overlay_pic = draw_bounding_box3(face_coordinates, rgb_image, color, img, marks_list)
rgb_image = overlay_pic
draw_text(face_coordinates, rgb_image, emotion_mode, color, 0, -45, 1, 1)
bgr_image = cv2.cvtColor(rgb_image, cv2.COLOR_RGB2BGR)
if flag == 0:
img = image_resize(img)
cv2.imshow('image', img)
cv2.destroyWindow('Window_frame')
elif flag == 1 or flag == 2:
cv2.destroyWindow('image')
cv2.imshow('window_frame', bgr_image)
cv2.waitKey(10)
# cv2.imshow('own_window', bgr_image)
if cv2.waitKey(1) & 0xFF == ord('z'):
flag = 0
elif cv2.waitKey(1) & 0xFF == ord('x'):
flag = 1
elif cv2.waitKey(1) & 0xFF == ord('c'):
flag = 2
elif cv2.waitKey(1) & 0xFF == ord('q'):
break
video_capture.release()
cv2.destroyAllWindows()
"input": {
"'neutral'": "1"
},
"robot": "Pepper"
}
print("neutral")
pub_emotion.publish(data)
color = 1 * np.asarray((0, 0, 0))
else:
draw = False
if draw:
color = color.astype(int)
color = color.tolist()
draw_bounding_box(box, rgb_image, color)
draw_text(box, rgb_image, emotion_mode, color, 0, -45, 1, 1)
if (show_image == 1):
bgr_image = cv2.cvtColor(rgb_image, cv2.COLOR_RGB2BGR)
cv2.imshow('window_frame', bgr_image)
if cv2.waitKey(1) & 0xFF == ord('q'):
break
except:
pass
else:
pass
def recognize_video(self, path=None, csv=True):
# starting video streaming
if self.show_frame:
cv2.namedWindow('window_frame')
# Select video or webcam feed
cap = None
if path is None:
cap = cv2.VideoCapture(0) # Webcam source
elif isinstance(path, str):
cap = cv2.VideoCapture(path) # Video file source
else:
raise Exception('path argument must be a string')
emotions_by_frames = [
] # container for recording emotion probabilities by frame
frame_counter = 0
while cap.isOpened(): # True:
try:
ret, bgr_image = cap.read()
if not ret:
break
#print('Frame {} is read'.format(frame_counter))
frame_counter += 1
rgb_image = cv2.cvtColor(bgr_image, cv2.COLOR_BGR2RGB)
start = default_timer()
faces = self.detector.detect_faces(rgb_image)
#print('faces', faces)
end = default_timer()
#print('face detection:', end - start)
if len(faces) == 0:
emotions_by_frames.append(np.array([0] * 7))
for face_coordinates in faces:
# face_coordinates: [x, y, w, h]
face_coordinates = list(map(int, face_coordinates))
x, y, w, h = face_coordinates
# expand to 20%
padding_size_x, padding_size_y = int(0.2 * x), int(0.2 * y)
w += padding_size_x
h += padding_size_y
x -= padding_size_x // 2
y -= padding_size_y // 2
face_image = rgb_image[y:y + h, x:x + w]
try:
face_image = cv2.resize(
face_image, (self.face_size, self.face_size))
except:
emotions_by_frames.append(
np.array([0] * len(self.emotion_labels)))
continue
face_image = self.prepare_face_image(face_image)
#print('Shape:', face_image.shape)
emotion_prediction = self.emotion_classifier.predict(
face_image, probs=True)[0].numpy()
#print('Prediction:', emotion_prediction)
emotion_probability = np.max(emotion_prediction)
if len(emotion_prediction) > 0:
emotions_by_frames.append(emotion_prediction)
else:
emotions_by_frames.append(
np.array([0] * len(self.emotion_labels)))
emotion_label_arg = np.argmax(emotion_prediction)
emotion_text = self.emotion_labels[emotion_label_arg]
if emotion_text == 'angry':
color = emotion_probability * np.asarray((255, 0, 0))
elif emotion_text == 'sad':
color = emotion_probability * np.asarray((0, 0, 255))
elif emotion_text == 'happy':
color = emotion_probability * np.asarray((255, 255, 0))
elif emotion_text == 'surprise':
color = emotion_probability * np.asarray((0, 255, 255))
else:
color = emotion_probability * np.asarray((0, 255, 0))
color = color.astype(int)
color = color.tolist()
draw_bounding_box(face_coordinates, bgr_image, color)
draw_text(face_coordinates, bgr_image, emotion_text, color,
0, -45, 1, 1)
#bgr_image = cv2.cvtColor(rgb_image, cv2.COLOR_RGB2BGR)
cv2.imshow('window_frame', bgr_image)
if (cv2.waitKey(1) & 0xFF == ord('q')) and self.show_frame:
break
except:
traceback.print_exc()
break
cap.release()
cv2.destroyAllWindows()
if csv:
return self.to_csv(emotions_by_frames)
return emotions_by_frames
elif emotion_text == 'sad':
color = emotion_probability * np.asarray((0, 0, 255))
elif emotion_text == 'happy':
color = emotion_probability * np.asarray((255, 255, 0))
elif emotion_text == 'surprise':
color = emotion_probability * np.asarray((0, 255, 255))
else:
color = emotion_probability * np.asarray((0, 255, 0))
color = color.astype(int)
color = color.tolist()
if fname == "Unknown":
name = emotion_text
else:
name = str(fname) + " is " + str(emotion_text)
draw_bounding_box(face_utils.rect_to_bb(face_coordinates), rgb_image, color)
draw_text(face_utils.rect_to_bb(face_coordinates), rgb_image, name,
color, 0, -45, 0.5, 1)
frame = cv2.cvtColor(rgb_image, cv2.COLOR_RGB2BGR)
cv2.imshow('window_frame', frame)
if cv2.waitKey(1) & 0xFF == ord('q'):
break
cap.release()
cv2.destroyAllWindows()
x1, x2, y1, y2 = apply_offsets(face_coordinates, offsets)
gray_face = gray_image[y1:y2, x1:x2]
try:
gray_face = cv2.resize(gray_face, (target_size))
except:
continue
gray_face = preprocess_input(gray_face, True)
gray_face = np.expand_dims(gray_face, 0)
gray_face = np.expand_dims(gray_face, -1)
guided_gradCAM = calculate_guided_gradient_CAM(gray_face,
gradient_function, saliency_function)
guided_gradCAM = cv2.resize(guided_gradCAM, (x2-x1, y2-y1))
try:
rgb_guided_gradCAM = np.repeat(guided_gradCAM[:, :, np.newaxis],
3, axis=2)
rgb_image[y1:y2, x1:x2, :] = rgb_guided_gradCAM
except:
continue
draw_bounding_box((x1, y1, x2 - x1, y2 - y1), rgb_image, color)
bgr_image = cv2.cvtColor(rgb_image, cv2.COLOR_RGB2BGR)
try:
cv2.imshow('window_frame', bgr_image)
except:
continue
if cv2.waitKey(1) & 0xFF == ord('q'):
break
def predict(image_folder_path, emotion_kind):
detection_model_path = '../trained_models/detection_models/haarcascade_frontalface_default.xml'
emotion_model_path = '../trained_models/emotion_models/fer2013_mini_XCEPTION.102-0.66.hdf5'
gender_model_path = '../trained_models/gender_models/simple_CNN.81-0.96.hdf5'
emotion_labels = get_labels('fer2013')
target_file = '../result/predicted_' + EmotionName[emotion_kind] + '.txt'
gender_labels = get_labels('imdb')
font = cv2.FONT_HERSHEY_SIMPLEX
# hyper-parameters for bounding boxes shape
gender_offsets = (30, 60)
gender_offsets = (10, 10)
emotion_offsets = (20, 40)
emotion_offsets = (0, 0)
# loading models
face_detection = load_detection_model(detection_model_path)
emotion_classifier = load_model(emotion_model_path, compile=False)
gender_classifier = load_model(gender_model_path, compile=False)
# getting input model shapes for inference
emotion_target_size = emotion_classifier.input_shape[1:3]
gender_target_size = gender_classifier.input_shape[1:3]
image_path = dir_data_folder(image_folder_path)
predicted_label = []
for num in range(len(image_path)):
# print the process info
if print_switch == 0:
print('deal with the ' + image_path[num])
# loading images
rgb_image = load_image(image_path[num], grayscale=False)
gray_image = load_image(image_path[num], grayscale=True)
gray_image = np.squeeze(gray_image)
gray_image = gray_image.astype('uint8')
# pdb.set_trace()
faces = detect_faces(face_detection, gray_image)
emotion_label_arg = -1
for face_coordinates in faces:
x1, x2, y1, y2 = apply_offsets(face_coordinates, gender_offsets)
rgb_face = rgb_image[y1:y2, x1:x2]
x1, x2, y1, y2 = apply_offsets(face_coordinates, emotion_offsets)
gray_face = gray_image[y1:y2, x1:x2]
try:
rgb_face = cv2.resize(rgb_face, (gender_target_size))
gray_face = cv2.resize(gray_face, (emotion_target_size))
except:
continue
rgb_face = preprocess_input(rgb_face, False)
rgb_face = np.expand_dims(rgb_face, 0)
gender_prediction = gender_classifier.predict(rgb_face)
gender_label_arg = np.argmax(gender_prediction)
gender_text = gender_labels[gender_label_arg]
gray_face = preprocess_input(gray_face, True)
gray_face = np.expand_dims(gray_face, 0)
gray_face = np.expand_dims(gray_face, -1)
emotion_label_arg = np.argmax(
emotion_classifier.predict(gray_face))
emotion_text = emotion_labels[emotion_label_arg]
if single_set_draw_switch == 0:
if gender_text == gender_labels[0]:
color = (0, 0, 255)
else:
color = (255, 0, 0)
draw_bounding_box(face_coordinates, rgb_image, color)
# draw_text(face_coordinates, rgb_image, gender_text, color, 0, -20, 1, 2)
draw_text(face_coordinates, rgb_image, emotion_text, color, 0,
20, 1, 2)
if single_set_draw_switch == 0:
bgr_image = cv2.cvtColor(rgb_image, cv2.COLOR_RGB2BGR)
filename = image_path[num].split('/')[-1]
filename = filename.split('.')[0]
newname = '../predicted_image/' + 'predicted_' + filename + '.png'
cv2.imwrite(newname, bgr_image)
# add the predicted label
predicted_label.append(emotion_label_arg)
unrecognized_cnt = predicted_label.count(-1)
true_cnt = predicted_label.count(emotion_kind)
total_cnt = len(image_path)
face_cnt = total_cnt - unrecognized_cnt
face_recog_ratio = face_cnt / float(total_cnt)
total_accuracy = true_cnt / float(total_cnt)
recog_accuracy = true_cnt / float(face_cnt)
f = open(target_file, 'w+')
#with open(target_file,'w') as f:
f.write(EmotionName[emotion_kind] + ' count = ' + str(true_cnt) + '\n')
f.write(EmotionName[emotion_kind] + ' total count = ' + str(total_cnt) +
'\n')
f.write(EmotionName[emotion_kind] + '_accuracy in total: ' +
str(total_accuracy) + '\n')
f.write(EmotionName[emotion_kind] + '_accuracy in recognized: ' +
str(recog_accuracy) + '\n')
f.write('face_recognize_count : ' + str(face_cnt) + '\n')
f.write('face_recognize_ratio : ' + str(face_recog_ratio) + '\n')
for label in predicted_label:
f.write(str(label) + '\n')
f.close()
return [
true_cnt, total_cnt, face_cnt, total_accuracy, face_recog_ratio,
recog_accuracy
]
rgb_image = cv2.cvtColor(bgr_image, cv2.COLOR_BGR2RGB)
faces = face_cascade.detectMultiScale(gray_image,
scaleFactor=1.1,
minNeighbors=5,
minSize=(30, 30),
flags=cv2.CASCADE_SCALE_IMAGE)
# for face_coordinates in faces:
if len(faces):
x1, x2, y1, y2 = apply_offsets(faces[0], emotion_offsets)
gray_face = gray_image[y1:y2, x1:x2]
gray_face = preprocess_input(gray_face, True)
gray_face = np.expand_dims(gray_face, 0)
gray_face = np.expand_dims(gray_face, -1)
color = np.asarray((255, 0, 0))
color = color.astype(int)
color = color.tolist()
draw_bounding_box(faces[0], rgb_image, color)
firebase(x1, x2, y1, y2)
bgr_image = cv2.cvtColor(rgb_image, cv2.COLOR_RGB2BGR)
cv2.imshow('window_frame', bgr_image)
if cv2.waitKey(1) & 0xFF == ord('q'):
break
cap.release()
cv2.destroyAllWindows()
def process_image(image):
try:
# parameters for loading data and images
detection_model_path = './trained_models/detection_models/haarcascade_frontalface_default.xml'
emotion_model_path = './trained_models/emotion_models/fer2013_mini_XCEPTION.102-0.66.hdf5'
gender_model_path = './trained_models/gender_models/simple_CNN.81-0.96.hdf5'
emotion_labels = get_labels('fer2013')
gender_labels = get_labels('imdb')
font = cv2.FONT_HERSHEY_SIMPLEX
# hyper-parameters for bounding boxes shape
gender_offsets = (30, 60)
gender_offsets = (10, 10)
emotion_offsets = (20, 40)
emotion_offsets = (0, 0)
# loading models
face_detection = load_detection_model(detection_model_path)
emotion_classifier = load_model(emotion_model_path, compile=False)
gender_classifier = load_model(gender_model_path, compile=False)
# getting input model shapes for inference
emotion_target_size = emotion_classifier.input_shape[1:3]
gender_target_size = gender_classifier.input_shape[1:3]
# loading images
image_array = np.fromstring(image, np.uint8)
unchanged_image = cv2.imdecode(image_array, cv2.IMREAD_UNCHANGED)
rgb_image = cv2.cvtColor(unchanged_image, cv2.COLOR_BGR2RGB)
gray_image = cv2.cvtColor(unchanged_image, cv2.COLOR_BGR2GRAY)
faces = detect_faces(face_detection, gray_image)
for face_coordinates in faces:
x1, x2, y1, y2 = apply_offsets(face_coordinates, gender_offsets)
rgb_face = rgb_image[y1:y2, x1:x2]
x1, x2, y1, y2 = apply_offsets(face_coordinates, emotion_offsets)
gray_face = gray_image[y1:y2, x1:x2]
try:
rgb_face = cv2.resize(rgb_face, (gender_target_size))
gray_face = cv2.resize(gray_face, (emotion_target_size))
except:
continue
rgb_face = preprocess_input(rgb_face, False)
rgb_face = np.expand_dims(rgb_face, 0)
gender_prediction = gender_classifier.predict(rgb_face)
gender_label_arg = np.argmax(gender_prediction)
gender_text = gender_labels[gender_label_arg]
gray_face = preprocess_input(gray_face, True)
gray_face = np.expand_dims(gray_face, 0)
gray_face = np.expand_dims(gray_face, -1)
emotion_label_arg = np.argmax(emotion_classifier.predict(gray_face))
emotion_text = emotion_labels[emotion_label_arg]
if gender_text == gender_labels[0]:
color = (0, 0, 255)
else:
color = (255, 0, 0)
draw_bounding_box(face_coordinates, rgb_image, color)
draw_text(face_coordinates, rgb_image, gender_text, color, 0, -20, 1, 2)
draw_text(face_coordinates, rgb_image, emotion_text, color, 0, -50, 1, 2)
except Exception as err:
logging.error('Error in emotion gender processor: "{0}"'.format(err))
bgr_image = cv2.cvtColor(rgb_image, cv2.COLOR_RGB2BGR)
dirname = 'result'
if not os.path.exists(dirname):
os.mkdir(dirname)
cv2.imwrite(os.path.join(dirname, 'predicted_image.png'), bgr_image)
def extract_emotions(self, bgr_image):
base_path = os.path.realpath(__file__)
base_path = base_path[:base_path.find('emotions')]
emotion_labels = get_labels('fer2013')
# hyper-parameters for bounding boxes shape
frame_window = 10
emotion_offsets = (20, 40)
# loading models
if self.emotion_classifier == None:
self.emotion_classifier = load_model(self.emotion_model_path)
# getting input model shapes for inference
emotion_target_size = self.emotion_classifier.input_shape[1:3]
# starting lists for calculating modes
emotion_window = []
gray_image = cv2.cvtColor(bgr_image, cv2.COLOR_BGR2GRAY)
rgb_image = cv2.cvtColor(bgr_image, cv2.COLOR_BGR2RGB)
faces = self.face_cascade.detectMultiScale(
gray_image,
scaleFactor=1.1,
minNeighbors=5,
minSize=(30, 30),
flags=cv2.CASCADE_SCALE_IMAGE)
for face_coordinates in faces:
x1, x2, y1, y2 = apply_offsets(face_coordinates, emotion_offsets)
gray_face = gray_image[y1:y2, x1:x2]
try:
gray_face = cv2.resize(gray_face, (emotion_target_size))
except:
continue
gray_face = preprocess_input(gray_face, True)
gray_face = np.expand_dims(gray_face, 0)
gray_face = np.expand_dims(gray_face, -1)
emotion_prediction = self.emotion_classifier.predict(gray_face)
emotion_probability = np.max(emotion_prediction)
emotion_label_arg = np.argmax(emotion_prediction)
emotion_text = emotion_labels[emotion_label_arg]
emotion_window.append(emotion_text)
if len(emotion_window) > frame_window:
emotion_window.pop(0)
try:
emotion_mode = mode(emotion_window)
except:
continue
if emotion_text == 'angry':
color = emotion_probability * np.asarray((255, 0, 0))
elif emotion_text == 'sad':
color = emotion_probability * np.asarray((0, 0, 255))
elif emotion_text == 'happy':
color = emotion_probability * np.asarray((255, 255, 0))
elif emotion_text == 'surprise':
color = emotion_probability * np.asarray((0, 255, 255))
elif emotion_text == 'fear':
color = emotion_probability * np.asarray((255, 0, 255))
elif emotion_text == 'disgust':
color = emotion_probability * np.asarray((255, 255, 255))
else:
color = emotion_probability * np.asarray((50, 200, 100))
color = color.astype(int)
color = color.tolist()
draw_bounding_box(face_coordinates, rgb_image, color)
draw_text(face_coordinates, rgb_image, emotion_mode, color, 0, -45,
1, 1)
bgr_image = cv2.cvtColor(rgb_image, cv2.COLOR_RGB2BGR)
cv2.imshow('window_frame', bgr_image)
if cv2.waitKey(1) & 0xFF == ord('q'):
break
def process_image(image):
detected_peoples = []
json_info = {}
try:
# parameters for loading data and images
emotion_labels = get_labels('fer2013')
gender_labels = get_labels('imdb')
# font = cv2.FONT_HERSHEY_SIMPLEX
# gender_keys = list(gender_labels.values())
# emotion_keys = list(emotion_labels.values())
# hyper-parameters for bounding boxes shape
gender_offsets = (30, 60)
gender_offsets = (10, 10)
emotion_offsets = (20, 40)
emotion_offsets = (0, 0)
# getting input model shapes for inference
gender_target_size = _GENDER_CLASSIFIER.input_shape[1:3]
emotion_target_size = _EMOTION_CLASSIFIER.input_shape[1:3]
# loading images
image_array = np.fromstring(image, np.uint8)
unchanged_image = cv2.imdecode(image_array, cv2.IMREAD_UNCHANGED)
rgb_image = cv2.cvtColor(unchanged_image, cv2.COLOR_BGR2RGB)
gray_image = cv2.cvtColor(unchanged_image, cv2.COLOR_BGR2GRAY)
faces = detect_faces(_FACE_DETECTION, gray_image)
for face_coordinates in faces:
x1, x2, y1, y2 = apply_offsets(face_coordinates, gender_offsets)
rgb_face = rgb_image[y1:y2, x1:x2]
x1, x2, y1, y2 = apply_offsets(face_coordinates, emotion_offsets)
gray_face = gray_image[y1:y2, x1:x2]
try:
rgb_face = cv2.resize(rgb_face, (gender_target_size))
gray_face = cv2.resize(gray_face, (emotion_target_size))
except:
continue
start_time = datetime.now()
with _GRAPH.as_default():
rgb_face = preprocess_input(rgb_face, False)
rgb_face = np.expand_dims(rgb_face, 0)
gender_prediction = _GENDER_CLASSIFIER.predict(rgb_face)
gender_label_arg = np.argmax(gender_prediction)
gender_text = gender_labels[gender_label_arg]
end_time = datetime.now()
delta1 = end_time - start_time
gray_face = preprocess_input(gray_face, True)
gray_face = np.expand_dims(gray_face, 0)
gray_face = np.expand_dims(gray_face, -1)
emotion_prediction = _EMOTION_CLASSIFIER.predict(gray_face)
emotion_label_arg = np.argmax(emotion_prediction)
emotion_text = emotion_labels[emotion_label_arg]
delta2 = datetime.now() - end_time
print("Delta for gender classificator {0}".format(
delta1.total_seconds() * 1000.0))
print("Delta for emotion classificator {0}".format(
delta2.total_seconds() * 1000.0))
json_info['gender'] = "{0}:{1}".format(
gender_text, gender_prediction.flat[gender_label_arg])
json_info['emotion'] = "{0}:{1}".format(
emotion_text, emotion_prediction.flat[emotion_label_arg])
json_info['face_bound'] = list(
map(lambda it: str(it),
list(face_coordinates.astype(np.int).flat)))
detected_peoples.append(json_info.copy())
print(detected_peoples)
if gender_text == gender_labels[0]:
color = (0, 0, 255)
else:
color = (255, 0, 0)
draw_bounding_box(face_coordinates, rgb_image, color)
draw_text(face_coordinates, rgb_image, gender_text, color, 0, -20,
1, 2)
draw_text(face_coordinates, rgb_image, emotion_text, color, 0, -10,
1, 2)
except Exception as err:
logging.error('Error in emotion gender processor: "{0}"'.format(err))
start_time = datetime.now()
bgr_image = cv2.cvtColor(rgb_image, cv2.COLOR_RGB2BGR)
if not os.path.exists(_SAVE_DIR):
os.mkdir(_SAVE_DIR)
recognition_datetime = str(datetime.now()).replace(' ', '_')
filepath = os.path.join(
_SAVE_DIR, 'predicted_image_{0}.png'.format(recognition_datetime))
cv2.imwrite(filepath, bgr_image)
delta = datetime.now() - start_time
print("delta for saving image", delta.total_seconds() * 1000.0)
return detected_peoples
def main():
warnings.filterwarnings("ignore")
logging.disable(logging.CRITICAL)
global emotion_cache #Vishal
emotion_cache = [] #Vishal
global stop_detecting
#Create server client for socket connection
port = opt.port
#host_name = socket.gethostname()
#host_ip = socket.gethostbyname(host_name)
host_ip = opt.ip
mySocket = socket.socket()
mySocket.bind((host_ip, port))
print ("Camera server listening on port {0} ".format(port)+ "and IP " + str(host_ip))
mySocket.listen(1)
chatbot_socket, adress = mySocket.accept()
print("Connection established to: " + str(adress))
#Start socket process to listen to chatbot requests
chatbot = Process(target = chatbot_process, args = (chatbot_socket, chatbot_request, chatbot_response, stop_camera))
chatbot.start()
signal.signal(signal.SIGTSTP, handler)
#face_recognizer = Face_Recognizer(True)# True to show video feed
# parameters for loading data and images
detection_model_path = './models/face_recognition/trained_models/haarcascade_frontalface_default.xml'
emotion_model_path = './models/face_recognition/trained_models/fer2013_mini_XCEPTION.102-0.66.hdf5'
emotion_labels = get_labels('fer2013')
# hyper-parameters for bounding boxes shape
frame_window = 10
emotion_offsets = (20, 40)
# loading models
face_detection = load_detection_model(detection_model_path)
emotion_classifier = load_model(emotion_model_path, compile=False)
# getting input model shapes for inference
emotion_target_size = emotion_classifier.input_shape[1:3]
# starting lists for calculating modes
emotion_window = []
emotion_text = 'Happy'
# starting video streaming
cv2.namedWindow('ODO_frame')
OdoSet='nvarguscamerasrc ! video/x-raw(memory:NVMM), width=3820, height=2464, format=NV12, framerate=21/1 ! nvvidconv flip-method='+str(flip)+' ! video/x-raw, width='+str(dispW)+', height='+str(dispH)+', format=BGRx ! videoconvert ! video/x-raw, format=BGR ! appsink'
video_capture = cv2.VideoCapture(OdoSet)
fps = video_capture.get(cv2.CAP_PROP_FPS)
print(f'FPS: {fps}')
num_frames = 21;
if video_capture.isOpened():
print("True")
else:
print("Error")
##################Vishal#####################
while stop_detecting == False:
#############################################
# Detect Emotion
#print("Check")
start = time.time() #starting FPS timer
bgr_image = video_capture.read()[1]
gray_image = cv2.cvtColor(bgr_image, cv2.COLOR_BGR2GRAY)
rgb_image = cv2.cvtColor(bgr_image, cv2.COLOR_BGR2RGB)
faces = detect_faces(face_detection, gray_image)
for face_coordinates in faces:
x1, x2, y1, y2 = apply_offsets(face_coordinates, emotion_offsets)
gray_face = gray_image[y1:y2, x1:x2]
try:
gray_face = cv2.resize(gray_face, (emotion_target_size))
except:
continue
gray_face = preprocess_input(gray_face, True)
gray_face = np.expand_dims(gray_face, 0)
gray_face = np.expand_dims(gray_face, -1)
emotion_prediction = emotion_classifier.predict(gray_face)
emotion_probability = np.max(emotion_prediction)
emotion_label_arg = np.argmax(emotion_prediction)
emotion_text = emotion_labels[emotion_label_arg]
emotion_window.append(emotion_text)
if len(emotion_window) > frame_window:
emotion_window.pop(0)
try:
emotion_mode = mode(emotion_window)
except:
continue
if emotion_text == 'angry':
color = emotion_probability * np.asarray((255, 0, 0))
elif emotion_text == 'sad':
color = emotion_probability * np.asarray((0, 0, 255))
elif emotion_text == 'happy':
color = emotion_probability * np.asarray((255, 255, 0))
elif emotion_text == 'surprise':
color = emotion_probability * np.asarray((0, 255, 255))
else:
color = emotion_probability * np.asarray((0, 255, 0))
emotion_cache.append(emotion_text)
with open('listfile.txt', 'w') as filehandle:
filehandle.writelines("%s\n" % emotions for emotions in emotion_cache)
if len(emotion_cache) > 500:
emotion_cache = emotion_cache[:-500]
##########################################
total_faces = len(faces)
current_time = time.time()
if chatbot_request.empty() == False:
request = chatbot_request.get()
data_request = request['request']
if data_request.lower() == 'send_emotion':
data = {}
data['emotion'] = emotion_detected
data['total_faces'] = total_faces
data['time'] = current_time
json_data = json.dumps(data)
chatbot_response.put({'response': json_data})
color = color.astype(int)
color = color.tolist()
#print(face_coordinates, rgb_image, color)
draw_bounding_box(face_coordinates, rgb_image, color)
draw_text(face_coordinates, rgb_image, emotion_mode,
color, 0, -45, 1, 1)
if stop_camera.empty() == False:
stop_detecting = True
end = time.time() #end time for FPS
bgr_image = cv2.cvtColor(rgb_image, cv2.COLOR_RGB2BGR)
seconds = end - start #FPS
fps = num_frames/seconds #FPS
fps_coords = (0,30)
framerate_str = 'Framerate: ' + str(int(fps))
cv2.putText(bgr_image, framerate_str, fps_coords, cv2.FONT_HERSHEY_SIMPLEX, 1, (255,255,200))
print(framerate_str)
cv2.imshow('ODO_frame', bgr_image)
if cv2.waitKey(1) & 0xFF == ord('q'):
break
video_capture.release()
cv2.destroyAllWindows()
print("Closing chatbot socket")
chatbot.terminate()
chatbot.join()
def run(self):
detection_graph, sess = detector_utils.load_inference_graph()
# parameters for loading data and images
detection_model_path = 'haarcascade_frontalface_default.xml'
emotion_model_path = 'fer2013_mini_XCEPTION.102-0.66.hdf5'
emotion_labels = get_labels('fer2013')
# hyper-parameters for bounding boxes shape
frame_window = 10
emotion_offsets = (20, 40)
# loading models
face_detection = load_detection_model(detection_model_path)
emotion_classifier = load_model(emotion_model_path, compile=False)
# getting input model shapes for inference
emotion_target_size = emotion_classifier.input_shape[1:3]
# starting lists for calculating modes
emotion_window = []
start_time = datetime.datetime.now()
im_width, im_height = (400, 350)
num_hands_detect = 2 # max number of hands we want to detect/track, can scale this up
min_threshold = 0.2
old_points = [None] * num_hands_detect
cv2.namedWindow('Single-Threaded Detection', cv2.WINDOW_NORMAL)
while True:
# Expand dimensions since the model expects images to have shape: [1, None, None, 3]
if self.image_queue.empty():
continue
img_data = base64.b64decode(str(self.image_queue.get()))
image_np = np.asarray(Image.open(io.BytesIO(img_data)))
image_np = cv2.flip(image_np, 1)
gray_image = cv2.cvtColor(image_np, cv2.COLOR_RGB2GRAY)
faces = detect_faces(face_detection, gray_image)
# Actual detection. Variable boxes contains the bounding box cordinates for hands detected,
# while scores contains the confidence for each of these boxes.
# Hint: If len(boxes) > 1 , you may assume you have found atleast one hand (within your score threshold)
boxes, scores = detector_utils.detect_objects(
image_np, detection_graph, sess)
valid_hands = 0
calc_displacement = False
for score in scores:
if score > min_threshold:
valid_hands += 1
if valid_hands == num_hands_detect:
calc_displacement = True #tell function to return new displacement
# draw bounding boxes on frame
self.total_displacement += detector_utils.draw_box_on_image(
num_hands_detect, min_threshold, scores, boxes, im_width,
im_height, image_np, old_points,
calc_displacement) #0.2 is the min threshold
# Calculate Frames per second (FPS)
self.num_frames += 1
elapsed_time = (datetime.datetime.now() -
start_time).total_seconds()
fps = self.num_frames / elapsed_time
if self.current_emotion in self.emotions:
self.emotions[self.current_emotion] += 1
else:
self.emotions[self.current_emotion] = 1
print(self.total_displacement / (10 * self.num_frames),
self.current_emotion, self.emotions)
for face_coordinates in faces:
x1, x2, y1, y2 = apply_offsets(face_coordinates,
emotion_offsets)
gray_face = gray_image[y1:y2, x1:x2]
try:
gray_face = cv2.resize(gray_face, (emotion_target_size))
except:
continue
gray_face = preprocess_input(gray_face, True)
gray_face = np.expand_dims(gray_face, 0)
gray_face = np.expand_dims(gray_face, -1)
emotion_prediction = emotion_classifier.predict(gray_face)
emotion_probability = np.max(emotion_prediction)
emotion_label_arg = np.argmax(emotion_prediction)
self.current_emotion = emotion_labels[emotion_label_arg]
emotion_window.append(self.current_emotion)
if len(emotion_window) > frame_window:
emotion_window.pop(0)
try:
emotion_mode = mode(emotion_window)
except:
continue
draw_bounding_box(face_coordinates, image_np)
# Display FPS on frame:
detector_utils.draw_fps_on_image("FPS : " + str(int(fps)),
image_np)
cv2.imshow('Single-Threaded Detection',
cv2.cvtColor(image_np, cv2.COLOR_RGB2BGR))
if cv2.waitKey(25) & 0xFF == ord('q'):
cv2.destroyAllWindows()
break
rgb_face = preprocess_input(rgb_face, False)
gender_prediction = gender_classifier.predict(rgb_face)
gender_label_arg = np.argmax(gender_prediction)
gender_text = gender_labels[gender_label_arg]
gender_window.append(gender_text)
if len(gender_window) > frame_window:
emotion_window.pop(0)
gender_window.pop(0)
try:
emotion_mode = mode(emotion_window)
gender_mode = mode(gender_window)
except:
continue
if gender_text == gender_labels[0]:
color = (0, 0, 255)
else:
color = (255, 0, 0)
draw_bounding_box(face_coordinates, rgb_image, color)
draw_text(face_coordinates, rgb_image, gender_mode, color, 0, -20, 1,
1)
draw_text(face_coordinates, rgb_image, emotion_mode, color, 0, -45, 1,
1)
bgr_image = cv2.cvtColor(rgb_image, cv2.COLOR_RGB2BGR)
cv2.imshow('window_frame', bgr_image)
if cv2.waitKey(1) & 0xFF == ord('q'):
break
def emotion_identify(img_url):
# parameters for loading data and images
detection_model_path = 'C:/Users/Admin/PycharmProjects/Emotion_Detection/trained_models/detection_models/haarcascade_frontalface_default.xml'
emotion_model_path = 'C:/Users/Admin/PycharmProjects/Emotion_Detection/trained_models/emotion_models/fer2013_mini_XCEPTION.102-0.66.hdf5'
gender_model_path = 'C:/Users/Admin/PycharmProjects/Emotion_Detection/trained_models/gender_models/simple_CNN.81-0.96.hdf5'
emotion_labels = get_labels('fer2013')
gender_labels = get_labels('imdb')
font = cv2.FONT_HERSHEY_SIMPLEX
# hyper-parameters for bounding boxes shape
gender_offsets = (30, 60)
gender_offsets = (10, 10)
emotion_offsets = (20, 40)
emotion_offsets = (0, 0)
# loading models
face_detection = load_detection_model(detection_model_path)
emotion_classifier = load_model(emotion_model_path, compile=False)
gender_classifier = load_model(gender_model_path, compile=False)
# getting input model shapes for inference
emotion_target_size = emotion_classifier.input_shape[1:3]
gender_target_size = gender_classifier.input_shape[1:3]
# loading images
image_path = img_url
rgb_image = load_image(image_path, grayscale=False)
gray_image = load_image(image_path, grayscale=True)
gray_image = np.squeeze(gray_image)
gray_image = gray_image.astype('uint8')
faces = detect_faces(face_detection, gray_image)
if len(faces) == 0:
print("No face")
K.clear_session()
return False
emotions = collections.defaultdict(int)
for face_coordinates in faces:
x1, x2, y1, y2 = apply_offsets(face_coordinates, gender_offsets)
rgb_face = rgb_image[y1:y2, x1:x2]
x1, x2, y1, y2 = apply_offsets(face_coordinates, emotion_offsets)
gray_face = gray_image[y1:y2, x1:x2]
try:
rgb_face = cv2.resize(rgb_face, (gender_target_size))
gray_face = cv2.resize(gray_face, (emotion_target_size))
except:
continue
rgb_face = preprocess_input(rgb_face, False)
rgb_face = np.expand_dims(rgb_face, 0)
gender_prediction = gender_classifier.predict(rgb_face)
gender_label_arg = np.argmax(gender_prediction)
gender_text = gender_labels[gender_label_arg]
gray_face = preprocess_input(gray_face, True)
gray_face = np.expand_dims(gray_face, 0)
gray_face = np.expand_dims(gray_face, -1)
emotion_label_arg = np.argmax(emotion_classifier.predict(gray_face))
emotion_text = emotion_labels[emotion_label_arg]
emotions[emotion_text] += 1
if gender_text == gender_labels[0]:
color = (0, 0, 255)
else:
color = (255, 0, 0)
draw_bounding_box(face_coordinates, rgb_image, color)
draw_text(face_coordinates, rgb_image, gender_text, color, 0, -20, 1, 2)
draw_text(face_coordinates, rgb_image, emotion_text, color, 0, -50, 1, 2)
max_num = 0
max_emotion = None
for key, value in emotions.items():
if value > max_num:
max_num = value
max_emotion = key
print("The emotion of this picture is: ", max_emotion)
bgr_image = cv2.cvtColor(rgb_image, cv2.COLOR_RGB2BGR)
cv2.imwrite('./result_images/predicted_test_image.png', bgr_image)
K.clear_session()
return max_emotion
