def process_image(image):
try:
# parameters for loading data and images
detection_model_path = './trained_models/detection_models/haarcascade_frontalface_default.xml'
emotion_model_path = './trained_models/emotion_models/fer2013_35a93d1a30cf52b8722837f3e096b7b1006949325de8d00f26595d5a418241d3.hdf5'
emotion_labels = get_labels('fer2013')
font = cv2.FONT_HERSHEY_SIMPLEX
# hyper-parameters for bounding boxes shape
emotion_offsets = (20, 40)
emotion_offsets = (0, 0)
# loading models
face_detection = load_detection_model(detection_model_path)
emotion_classifier = load_model(emotion_model_path, compile=False)
# getting input model shapes for inference
emotion_target_size = emotion_classifier.input_shape[1:3]
# loading images
image_array = np.fromstring(image, np.uint8)
unchanged_image = cv2.imdecode(image_array, cv2.IMREAD_UNCHANGED)
rgb_image = cv2.cvtColor(unchanged_image, cv2.COLOR_BGR2RGB)
gray_image = cv2.cvtColor(unchanged_image, cv2.COLOR_BGR2GRAY)
faces = detect_faces(face_detection, gray_image)
for face_coordinates in faces:
x1, x2, y1, y2 = apply_offsets(face_coordinates, emotion_offsets)
gray_face = gray_image[y1:y2, x1:x2]
try:
gray_face = cv2.resize(gray_face, (emotion_target_size))
except:
continue
rgb_face = preprocess_input(rgb_face, False)
rgb_face = np.expand_dims(rgb_face, 0)
gray_face = preprocess_input(gray_face, True)
gray_face = np.expand_dims(gray_face, 0)
gray_face = np.expand_dims(gray_face, -1)
emotion_label_arg = np.argmax(emotion_classifier.predict(gray_face))
emotion_text = emotion_labels[emotion_label_arg]
color = (255, 0, 0)
draw_bounding_box(face_coordinates, rgb_image, color)
draw_text(face_coordinates, rgb_image, emotion_text, color, 0, -50, 1, 2)
except Exception as err:
logging.error('Error in emotion gender processor: "{0}"'.format(err))
bgr_image = cv2.cvtColor(rgb_image, cv2.COLOR_RGB2BGR)
dirname = 'result'
if not os.path.exists(dirname):
os.mkdir(dirname)
cv2.imwrite(os.path.join(dirname, 'predicted_image.png'), bgr_image)
def save_biggest_emotion_images_cut(video_path, save_path, detector,
frame_interval_ms, start_ms, end_ms):
video_capture = cv2.VideoCapture(video_path)
if not video_capture.isOpened():
return
fps = video_capture.get(cv2.CAP_PROP_FPS)
start_frame_no = int(start_ms / 1000 * fps + 1)
end_frame_no = int(end_ms / 1000 * fps)
frame_count = video_capture.get(cv2.CAP_PROP_FRAME_COUNT)
if start_frame_no < 0 or end_frame_no > frame_count:
return
# 创建视频存储文件夹
if not os.path.exists(save_path):
os.mkdir(save_path)
frame_no = 1 # 抽取帧的序号
interval_frame_num = int(frame_interval_ms / 1000 * fps) # 间隔帧数
if interval_frame_num < 1:
interval_frame_num = 1 # 防止帧间隔为0的情况
while frame_no <= end_frame_no:
video_capture.set(cv2.CAP_PROP_POS_FRAMES, frame_no)
frame = video_capture.read()[1]
if frame is None or np.size(frame) is 0:
frame_no += interval_frame_num
continue
time = int(video_capture.get(cv2.CAP_PROP_POS_MSEC))
prediction, coord = detector.detect_biggest(frame)
if prediction is None:
frame_no += interval_frame_num
continue
emotion_probability = np.max(prediction)
frame_no += interval_frame_num
emotion_label_arg = np.argmax(prediction)
emotion_text = detector.labels[emotion_label_arg]
# 根据情绪选择颜色
if emotion_text == 'angry':
color = emotion_probability * np.asarray((255, 0, 0))
elif emotion_text == 'sad':
color = emotion_probability * np.asarray((0, 0, 255))
elif emotion_text == 'happy':
color = emotion_probability * np.asarray((255, 255, 0))
elif emotion_text == 'surprise':
color = emotion_probability * np.asarray((0, 255, 255))
else:
color = emotion_probability * np.asarray((0, 255, 0))
color = color.astype(int)
color = color.tolist()
text = emotion_text + ' ' + str(time / 1000) + 's'
cv2.rectangle(frame, (coord[0], coord[2]), (coord[1], coord[3]), color,
2)
draw_text(coord, frame, text, color, 0, -45, 1, 1)
image_path = save_path + '/' + str(frame_no) + '.png'
cv2.imwrite(image_path, frame)
video_capture.release()
return
def recognition(f_2_s):
# parameters for loading data and images
#image_path = path_r
image_path = image_handler(f_2_s)
detection_model_path = '../trained_models/detection_models/haarcascade_frontalface_default.xml'
emotion_model_path = '../trained_models/emotion_models/fer2013_mini_XCEPTION.102-0.66.hdf5'
emotion_labels = get_labels('fer2013')
font = cv2.FONT_HERSHEY_SIMPLEX
# hyper-parameters for bounding boxes shape
emotion_offsets = (20, 40)
emotion_offsets = (0, 0)
# loading models
face_detection = load_detection_model(detection_model_path)
emotion_classifier = load_model(emotion_model_path, compile=False)
# getting input model shapes for inference
emotion_target_size = emotion_classifier.input_shape[1:3]
# loading images
rgb_image = load_image(image_path, grayscale=False)
gray_image = load_image(image_path, grayscale=True)
gray_image = np.squeeze(gray_image)
gray_image = gray_image.astype('uint8')
faces = detect_faces(face_detection, gray_image)
for face_coordinates in faces:
x1, x2, y1, y2 = apply_offsets(face_coordinates, emotion_offsets)
gray_face = gray_image[y1:y2, x1:x2]
try:
gray_face = cv2.resize(gray_face, (emotion_target_size))
except:
continue
gray_face = preprocess_input(gray_face, True)
gray_face = np.expand_dims(gray_face, 0)
gray_face = np.expand_dims(gray_face, -1)
emotion_label_arg = np.argmax(emotion_classifier.predict(gray_face))
emotion_text = ""
emotion_text = emotion_labels[emotion_label_arg]
color = (255, 0, 0) # 감정 정보 글씨 빨간색, 사각형도
draw_bounding_box(face_coordinates, rgb_image, color)
draw_text(face_coordinates, rgb_image, emotion_text, color, 0, -30,
1.5, 2)
#if(emotion_text == ""):
#recognition(f_2_s)
bgr_image = cv2.cvtColor(rgb_image, cv2.COLOR_RGB2BGR)
cv2.imwrite('../images/predicted_test_image.png', bgr_image) # 변수 활용
check_recoged_img('../images/predicted_test_image.png')
def final_ml_predict(bgr_image):
gray_image = cv2.cvtColor(bgr_image, cv2.COLOR_BGR2GRAY)
rgb_image = cv2.cvtColor(bgr_image, cv2.COLOR_BGR2RGB)
faces = face_cascade.detectMultiScale(gray_image,
scaleFactor=1.1,
minNeighbors=5,
minSize=(30, 30),
flags=cv2.CASCADE_SCALE_IMAGE)
for face_coordinates in faces:
x1, x2, y1, y2 = apply_offsets(face_coordinates, emotion_offsets)
gray_face = gray_image[y1:y2, x1:x2]
try:
gray_face = cv2.resize(gray_face, (emotion_target_size))
except:
continue
gray_face = preprocess_input(gray_face, True)
gray_face = np.expand_dims(gray_face, 0)
gray_face = np.expand_dims(gray_face, -1)
with graph.as_default():
emotion_prediction = emotion_classifier.predict(gray_face)
emotion_probability = np.max(emotion_prediction)
emotion_label_arg = np.argmax(emotion_prediction)
emotion_text = emotion_labels[emotion_label_arg]
emotion_window.append(emotion_text)
if len(emotion_window) > frame_window:
emotion_window.pop(0)
try:
emotion_mode = mode(emotion_window)
except:
continue
if emotion_text == 'angry':
color = emotion_probability * np.asarray((255, 0, 0))
elif emotion_text == 'sad':
color = emotion_probability * np.asarray((0, 0, 255))
elif emotion_text == 'happy':
color = emotion_probability * np.asarray((255, 255, 0))
elif emotion_text == 'surprise':
color = emotion_probability * np.asarray((0, 255, 255))
else:
color = emotion_probability * np.asarray((0, 255, 0))
color = color.astype(int)
color = color.tolist()
draw_bounding_box(face_coordinates, rgb_image, color)
draw_text(face_coordinates, rgb_image, emotion_mode, color, 0, -45, 1,
1)
bgr_image = cv2.cvtColor(rgb_image, cv2.COLOR_RGB2BGR)
return bgr_image
def classify_frame(bgr_image):
# bgr_image = video_capture.read()[1]
gray_image = cv2.cvtColor(bgr_image, cv2.COLOR_BGR2GRAY)
rgb = cv2.cvtColor(bgr_image, cv2.COLOR_BGR2RGB)
max_size = 512
aspect_ratio = rgb.shape[1] / rgb.shape[0]
scaling_factor = max([rgb.shape[1], rgb.shape[0]]) / max_size
new_width = int(rgb.shape[1] / scaling_factor)
new_height = int(rgb.shape[0] / scaling_factor)
rgb_image = cv2.resize(rgb, (new_width, new_height))
faces = detect_faces(face_detection, rgb_image)
for face_coordinates in faces:
x1, x2, y1, y2 = apply_offsets(face_coordinates, gender_offsets)
rgb_face = rgb_image[y1:y2, x1:x2]
gray_face = gray_image[y1:y2, x1:x2]
try:
rgb_face = cv2.resize(rgb_face, (gender_target_size))
except:
continue
gray_face = preprocess_input(gray_face, False)
gray_face = np.expand_dims(gray_face, 0)
gray_face = np.expand_dims(gray_face, -1)
rgb_face = np.expand_dims(rgb_face, 0)
rgb_face = preprocess_input(rgb_face, False)
gender_prediction = gender_classifier.predict(rgb_face)
gender_label_arg = np.argmax(gender_prediction)
gender_text = gender_labels[gender_label_arg]
gender_window.append(gender_text)
if len(gender_window) > frame_window:
gender_window.pop(0)
try:
gender_mode = mode(gender_window)
except:
continue
if gender_text == gender_labels[0]:
color = (0, 0, 255)
else:
color = (255, 0, 0)
draw_bounding_box(face_coordinates, rgb_image, color)
draw_text(face_coordinates, rgb_image, gender_mode,
color, 0, -20, 1, 1)
bgr_image = cv2.cvtColor(rgb_image, cv2.COLOR_RGB2BGR)
cv2.imshow('window_frame', bgr_image)
def detect_emotion(frame,model,emotion_classifier,emotion_target_size,face_cascade,frame_window,emotion_offsets,emotion_labels):
gray_image = cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY)
faces = face_cascade.detectMultiScale(gray_image, scaleFactor=1.1, minNeighbors=5,minSize=(30, 30), flags=cv2.CASCADE_SCALE_IMAGE)
for face_coordinates in faces:
x1, x2, y1, y2 = apply_offsets(face_coordinates, emotion_offsets)
# gray_face = gray_image[y1:y2, x1:x2]
# gray_c = cv2.cvtColor(gray_image[y1:y2, x1:x2], cv2.COLOR_BGR2GRAY)
gray_c = gray_image[y1:y2, x1:x2]
try:
gray_face = cv2.resize(gray_c, (emotion_target_size))
except:
continue
gray_face = preprocess_input(gray_face, True)
gray_face = np.expand_dims(gray_face, 0)
gray_face = np.expand_dims(gray_face, -1)
# print(gray_face)
emotion_prediction = emotion_classifier.predict(gray_face)
emotion_probability =round( np.max(emotion_prediction),2)
emotion_label_arg = np.argmax(emotion_prediction)
emotion_text = emotion_labels[emotion_label_arg]
emotion_window.append(emotion_text)
# print(emotion_text)
# print(emotion_window)
if len(emotion_window) > frame_window:
emotion_window.pop(0)
try:
emotion_mode = mode(emotion_window)
except:
continue
# if emotion_text == 'angry':
# color = emotion_probability * np.asarray((255, 0, 0))
# elif emotion_text == 'sad':
# color = emotion_probability * np.asarray((0, 0, 255))
# elif emotion_text == 'happy':
# # print (emotion_probability)
# color = emotion_probability * np.asarray((255, 255, 0))
# elif emotion_text == 'surprise':
# color = emotion_probability * np.asarray((0, 255, 255))
# else:
# color = emotion_probability * np.asarray((0, 255, 0))
# color = color.astype(int)
# color = color.tolist()
color=(255,255,255)
draw_text(emotion_probability,face_coordinates, frame, emotion_mode,color, 0, -20, 0.5, 1)
def run_classify(image_path):
emotion_text = ''
gender_text = ''
# loading images
rgb_image = load_image(image_path, grayscale=False)
gray_image = load_image(image_path, grayscale=True)
gray_image = np.squeeze(gray_image)
gray_image = gray_image.astype('uint8')
faces = detect_faces(face_detection, gray_image)
for face_coordinates in faces:
x1, x2, y1, y2 = apply_offsets(face_coordinates, gender_offsets)
rgb_face = rgb_image[y1:y2, x1:x2]
x1, x2, y1, y2 = apply_offsets(face_coordinates, emotion_offsets)
gray_face = gray_image[y1:y2, x1:x2]
try:
rgb_face = cv2.resize(rgb_face, (gender_target_size))
gray_face = cv2.resize(gray_face, (emotion_target_size))
except:
continue
rgb_face = preprocess_input(rgb_face, False)
rgb_face = np.expand_dims(rgb_face, 0)
gender_prediction = gender_classifier.predict(rgb_face)
gender_label_arg = np.argmax(gender_prediction)
gender_text = gender_labels[gender_label_arg]
gray_face = preprocess_input(gray_face, True)
gray_face = np.expand_dims(gray_face, 0)
gray_face = np.expand_dims(gray_face, -1)
emotion_label_arg = np.argmax(emotion_classifier.predict(gray_face))
emotion_text = emotion_labels[emotion_label_arg]
if gender_text == gender_labels[0]:
color = (0, 0, 255)
else:
color = (255, 0, 0)
draw_bounding_box(face_coordinates, rgb_image, color)
draw_text(face_coordinates, rgb_image, gender_text, color, 0, -20, 1,
2)
draw_text(face_coordinates, rgb_image, emotion_text, color, 0, -50, 1,
2)
bgr_image = cv2.cvtColor(rgb_image, cv2.COLOR_RGB2BGR)
cv2.imwrite(os.path.join(dir, '../images/predicted_test_image.png'),
bgr_image)
return ([gender_text, emotion_text])
def emo_Detect(self, bgr_image, face_coordinates):
gray_image = cv2.cvtColor(bgr_image, cv2.COLOR_BGR2GRAY)
x1, y1, x2, y2 = face_coordinates
face_coordinates = [int(x1), int(y1), int(x2 - x1), int(y2 - y1)]
x1, x2, y1, y2 = apply_offsets(face_coordinates, self.emotion_offsets)
gray_face = gray_image[y1:y2, x1:x2]
gray_face = cv2.resize(gray_face, (self.emotion_target_size))
gray_face = preprocess_input(gray_face, True)
gray_face = np.expand_dims(gray_face, 0)
gray_face = np.expand_dims(gray_face, -1)
emotion_prediction = self.emotion_classifier.predict(gray_face)
for idx, probability in enumerate(emotion_prediction[0]):
print(self.emotion_labels[idx], probability)
emotion_probability = np.max(emotion_prediction)
emotion_label_arg = np.argmax(emotion_prediction)
emotion_text = self.emotion_labels[emotion_label_arg]
self.emotion_window.append(emotion_text)
if len(self.emotion_window) > self.frame_window:
self.emotion_window.pop(0)
try:
emotion_mode = mode(self.emotion_window)
print("mode is " + emotion_mode)
except:
pass
if emotion_text == 'angry':
color = emotion_probability * np.asarray((0, 0, 255))
elif emotion_text == 'sad':
color = emotion_probability * np.asarray((255, 0, 0))
elif emotion_text == 'happy':
color = emotion_probability * np.asarray((0, 255, 255))
elif emotion_text == 'surprise':
color = emotion_probability * np.asarray((255, 255, 0))
else:
color = emotion_probability * np.asarray((0, 255, 0))
color = color.astype(int)
color = color.tolist()
draw_bounding_box(face_coordinates, bgr_image, color)
draw_text(face_coordinates, bgr_image, emotion_text, color, 0, -45, 1,
1)
return bgr_image, emotion_prediction[0]
def getFaceEmotion(image_path):
# parameters for loading data and images
base = 'C://Users/lenovo/Desktop/moodify/components/emotion-classification'
detection_model_path = base + '/trained_models/detection_models/haarcascade_frontalface_default.xml'
emotion_model_path = base + '/trained_models/emotion_models/fer2013_denseNet.59-0.68.hdf5'
emotion_labels = get_labels('fer2013')
font = cv2.FONT_HERSHEY_SIMPLEX
# hyper-parameters for bounding boxes shape
emotion_offsets = (0, 0)
# loading models
face_detection = load_detection_model(detection_model_path)
emotion_classifier = load_model(emotion_model_path, compile=False)
# getting input model shapes for inference
emotion_target_size = emotion_classifier.input_shape[1:3]
# loading images
rgb_image = load_image(image_path, grayscale=False)
gray_image = load_image(image_path, grayscale=True)
gray_image = np.squeeze(gray_image)
gray_image = gray_image.astype('uint8')
faces = detect_faces(face_detection, gray_image)
for face_coordinates in faces:
x1, x2, y1, y2 = apply_offsets(face_coordinates, emotion_offsets)
gray_face = gray_image[y1:y2, x1:x2]
try:
gray_face = cv2.resize(gray_face, (emotion_target_size))
except:
continue
gray_face = preprocess_input(gray_face, True)
gray_face = np.expand_dims(gray_face, 0)
gray_face = np.expand_dims(gray_face, -1)
emotion_label_arg = np.argmax(emotion_classifier.predict(gray_face))
emotion_text = emotion_labels[emotion_label_arg]
color = (255, 0, 0)
draw_bounding_box(face_coordinates, rgb_image, color)
draw_text(face_coordinates, rgb_image, emotion_text, color, 0, -10,
0.5, 2)
bgr_image = cv2.cvtColor(rgb_image, cv2.COLOR_RGB2BGR)
cv2.imwrite('predicted_test_image.png', bgr_image)
return emotion_text
def predict(image_path):
# loading images
start = time.time()
rgb_image = load_image(image_path, grayscale=False)
gray_image = load_image(image_path, grayscale=True)
gray_image = np.squeeze(gray_image)
gray_image = gray_image.astype('uint8')
with graph.as_default():
faces = detect_faces(face_detection, gray_image)
for face_coordinates in faces:
x1, x2, y1, y2 = apply_offsets(face_coordinates, gender_offsets)
rgb_face = rgb_image[y1:y2, x1:x2]
x1, x2, y1, y2 = apply_offsets(face_coordinates, emotion_offsets)
gray_face = gray_image[y1:y2, x1:x2]
try:
rgb_face = cv2.resize(rgb_face, (gender_target_size))
gray_face = cv2.resize(gray_face, (emotion_target_size))
except:
continue
rgb_face = preprocess_input(rgb_face, False)
rgb_face = np.expand_dims(rgb_face, 0)
gender_prediction = gender_classifier.predict(rgb_face)
gender_label_arg = np.argmax(gender_prediction)
gender_text = gender_labels[gender_label_arg]
gray_face = preprocess_input(gray_face, True)
gray_face = np.expand_dims(gray_face, 0)
gray_face = np.expand_dims(gray_face, -1)
emotion_label_arg = np.argmax(emotion_classifier.predict(gray_face))
emotion_text = emotion_labels[emotion_label_arg]
if gender_text == gender_labels[0]:
color = (0, 0, 255)
else:
color = (255, 0, 0)
draw_bounding_box(face_coordinates, rgb_image, color)
draw_text(face_coordinates, rgb_image, gender_text, color, 0, -10, 1, 2)
draw_text(face_coordinates, rgb_image, emotion_text, color, 0, -40, 1, 2)
bgr_image = cv2.cvtColor(rgb_image, cv2.COLOR_RGB2BGR)
resized_image = cv2.resize(bgr_image, (1024, 769))
file_path = 'result/' + image_path.split('/')[-1]
cv2.imwrite(file_path, resized_image)
end = time.time()
logging.info('[Time] for prediction(): {}'.format(end - start))
return file_path
def on_data(data):
f = open('current.jpg', 'wb')
f.write(base64.decodebytes(data))
f.close()
image_path = "current.jpg"
out = []
# loading images
rgb_image = load_image(image_path, grayscale=False)
gray_image = load_image(image_path, grayscale=True)
gray_image = np.squeeze(gray_image)
gray_image = gray_image.astype('uint8')
faces = detect_faces(face_detection, gray_image)
for face_coordinates in faces:
x1, x2, y1, y2 = apply_offsets(face_coordinates, emotion_offsets)
gray_face = gray_image[y1:y2, x1:x2]
try:
gray_face = cv2.resize(gray_face, (emotion_target_size))
except:
continue
gray_face = preprocess_input(gray_face, True)
gray_face = np.expand_dims(gray_face, 0)
gray_face = np.expand_dims(gray_face, -1)
emotion_label_arg = np.argmax(emotion_classifier.predict(gray_face))
emotion_text = emotion_labels[emotion_label_arg]
color = (0, 0, 255)
draw_bounding_box(face_coordinates, rgb_image, color)
draw_text(face_coordinates, rgb_image, emotion_text, color, 0, -50, 1,
2)
bgr_image = cv2.cvtColor(rgb_image, cv2.COLOR_RGB2BGR)
cv2.imwrite('predicted.png', bgr_image)
data = open('predicted.png', 'rb').read()
encoded = base64.encodebytes(data).decode('utf-8')
out.append({
'image': encoded,
'emotion': emotion_text,
})
return out
emotion_mode = mode(emotion_window)
except:
continue
if emotion_text == 'angry':
color = emotion_probability * np.asarray((255, 0, 0))
elif emotion_text == 'sad':
color = emotion_probability * np.asarray((0, 0, 255))
elif emotion_text == 'happy':
color = emotion_probability * np.asarray((255, 255, 0))
elif emotion_text == 'surprise':
color = emotion_probability * np.asarray((0, 255, 255))
else:
color = emotion_probability * np.asarray((0, 255, 0))
color = color.astype(int)
color = color.tolist()
draw_bounding_box(face_utils.rect_to_bb(face_coordinates), rgb_image,
color)
draw_text(face_utils.rect_to_bb(face_coordinates), rgb_image,
emotion_mode, color, 0, -45, 1, 1)
frame = cv2.cvtColor(rgb_image, cv2.COLOR_RGB2BGR)
cv2.imshow('window_frame', frame)
if cv2.waitKey(1) & 0xFF == ord('q'):
break
cap.release()
cv2.destroyAllWindows()
def r8k9(fileName='dog.jpg', score=10, length=10):
# getting input model shapes for inference
emotion_target_size = emotion_classifier.input_shape[1:3]
# starting lists for calculating modes
emotion_window = []
# starting video streaming
cv2.namedWindow('window_frame')
video_capture = cv2.VideoCapture(0)
im = cv2.imread(fileName)
im_resized = cv2.resize(im, (224, 224), interpolation=cv2.INTER_LINEAR)
countdown = ["Ready?", "3", "2", "1"]
#plt.imshow(cv2.cvtColor(im_resized, cv2.COLOR_BGR2RGB))
#for i in range(4):
# print(countdown[i])
# time.sleep(1)
#print("GO")
#plt.show(block=False)
t = time.time()
td = []
while time.time() < t + length:
bgr_image = video_capture.read()[1]
try:
gray_image = cv2.cvtColor(bgr_image, cv2.COLOR_BGR2GRAY)
except:
time.sleep(.5)
gray_image = cv2.cvtColor(bgr_image, cv2.COLOR_BGR2GRAY)
rgb_image = cv2.cvtColor(bgr_image, cv2.COLOR_BGR2RGB)
faces = detect_faces(face_detection, gray_image)
for face_coordinates in faces:
x1, x2, y1, y2 = apply_offsets(face_coordinates, emotion_offsets)
gray_face = gray_image[y1:y2, x1:x2]
try:
gray_face = cv2.resize(gray_face, (emotion_target_size))
except:
continue
gray_face = preprocess_input(gray_face, True)
gray_face = np.expand_dims(gray_face, 0)
gray_face = np.expand_dims(gray_face, -1)
emotion_prediction = emotion_classifier.predict(gray_face)
#print(emotion_prediction)
if len(td) == 0:
data = emotion_prediction
td = [time.time() - t]
else:
data = np.concatenate((data, emotion_prediction), axis=0)
td = td + [time.time() - t]
emotion_probability = np.max(emotion_prediction)
emotion_label_arg = np.argmax(emotion_prediction)
emotion_text = emotion_labels[emotion_label_arg]
emotion_window.append(emotion_text)
if len(emotion_window) > frame_window:
emotion_window.pop(0)
try:
emotion_mode = mode(emotion_window)
except:
continue
if emotion_text == 'angry':
color = emotion_probability * np.asarray((255, 0, 0))
elif emotion_text == 'sad':
color = emotion_probability * np.asarray((0, 0, 255))
elif emotion_text == 'happy':
color = emotion_probability * np.asarray((255, 255, 0))
elif emotion_text == 'surprise':
color = emotion_probability * np.asarray((0, 255, 255))
else:
color = emotion_probability * np.asarray((0, 255, 0))
color = color.astype(int)
color = color.tolist()
draw_bounding_box(face_coordinates, rgb_image, color)
draw_text(face_coordinates, rgb_image, emotion_mode, color, 0, -45,
1, 1)
bgr_image = cv2.cvtColor(rgb_image, cv2.COLOR_RGB2BGR)
cv2.imshow('window_frame', bgr_image)
if cv2.waitKey(1) & 0xFF == ord('q'):
break
#print(np.transpose(data))
#print(emotion_labels)
plt.clf()
plt.plot(td, data)
plt.legend(
['angry', 'disgust', 'fear', 'happy', 'sad', 'surprise', 'neutral'],
loc=2)
plt.savefig('Graph.png')
#splt.show()
totals = [0, 0, 0, 0, 0, 0, 0]
for j in range(7):
for i in range(1, len(td)):
#print(time[i-1])
#print(data[i-1+l*(j+1)])
#print(time[i],time[i-1])
totals[j] += (data[i][j] + data[i - 1][j]) / 2 * (
td[i] - td[i - 1]) / (td[-1] - td[0])
np.append(totals, np.std(data))
print(totals)
#print(td[-1] - td[0])
#print(sum(totals))
#f = open("DogData.txt","a+")
#f.write((str(score) + ' ' + ' '.join(str(t) for t in totals) + "\n"))
print("Rating:", 15 - spot(np.array(totals)) / 10, "/10")
return (15 - spot(np.array(totals)) / 10)
def get_sim_percent(image_1, image_2, target_class):
# parameters for loading data and images
# image_path = sys.argv[1]
detection_model_path = 'trained_models/detection_models/haarcascade_frontalface_default.xml'
emotion_model_path = 'trained_models/fer2013_big_XCEPTION.54-0.66.hdf5'
gender_model_path = 'trained_models/gender_models/simple_CNN.81-0.96.hdf5'
emotion_labels = get_labels('fer2013')
gender_labels = get_labels('imdb')
font = cv2.FONT_HERSHEY_SIMPLEX
# hyper-parameters for bounding boxes shape
gender_offsets = (20, 40)
gender_offsets = (10, 10)
emotion_offsets = (10, 30)
emotion_offsets = (0, 0)
# loading models
face_detection = load_detection_model(detection_model_path)
emotion_classifier = load_model(emotion_model_path, compile=False)
gender_classifier = load_model(gender_model_path, compile=False)
# getting input model shapes for inference
emotion_target_size = emotion_classifier.input_shape[1:3]
gender_target_size = gender_classifier.input_shape[1:3]
target_emo = target_class
image_path = image_1
# loading images
rgb_image = load_image(image_path, grayscale=False)
gray_image = load_image(image_path, grayscale=True)
gray_image = np.squeeze(gray_image)
gray_image = gray_image.astype('uint8')
faces = detect_faces(face_detection, gray_image)
for face_coordinates in faces:
x1, x2, y1, y2 = apply_offsets(face_coordinates, gender_offsets)
rgb_face = rgb_image[y1:y2, x1:x2]
x1, x2, y1, y2 = apply_offsets(face_coordinates, emotion_offsets)
gray_face = gray_image[y1:y2, x1:x2]
try:
rgb_face = cv2.resize(rgb_face, (gender_target_size))
gray_face = cv2.resize(gray_face, (emotion_target_size))
except:
continue
rgb_face = preprocess_input(rgb_face, False)
rgb_face = np.expand_dims(rgb_face, 0)
gender_prediction = gender_classifier.predict(rgb_face)
gender_label_arg = np.argmax(gender_prediction)
gender_text = gender_labels[gender_label_arg]
gray_face = preprocess_input(gray_face, True)
gray_face = np.expand_dims(gray_face, 0)
gray_face = np.expand_dims(gray_face, -1)
emotion_label_arg = emotion_classifier.predict(gray_face)[0][target_emo]
emotion_percent = str(round(emotion_label_arg*100, 2))
emotion_text = emotion_labels[target_emo] + ' - '+ emotion_percent
# print(target_emo)
# print(emotion_text)
if gender_text == gender_labels[0]:
color = (0, 0, 255)
else:
color = (255, 0, 0)
draw_bounding_box(face_coordinates, rgb_image, color)
draw_text(face_coordinates, rgb_image, gender_text, color, -20, 30, 1, 2)
draw_text(face_coordinates, rgb_image, emotion_text, color, 0, 0, 1, 2)
bgr_image = cv2.cvtColor(rgb_image, cv2.COLOR_RGB2BGR)
cv2.imwrite('predicted_test_image.png', bgr_image)
return emotion_percent
rgb_face = np.expand_dims(rgb_face, 0)
gender_prediciton = gender_classifier.predict(rgb_face)
gender_label_arg = np.argmax(gender_prediciton)
gender_text = gender_labels[gender_label_arg]
gray_face = preprocess_input(gray_face, True)
gray_face = np.expand_dims(gray_face, 0)
gray_face = np.expand_dims(gray_face, -1)
emotion_label_arg = np.argmax(emotion_classifier.predict(gray_face))
emotion_text = emotion_labels[emotion_label_arg]
if gender_text == gender_labels[0]:
color = (0, 0, 255)
else:
color = (255, 0, 0)
draw_bounding_box(face_coordinates, rgb_image, color)
draw_text(face_coordinates, rgb_image, gender_text, color, 0, -20, 1, 2)
draw_text(face_coordinates, rgb_image, emotion_text, color, 0, -50, 1, 2)
end = time.time()
print('Used {} s'.format((end - start)))
bgr_image = cv2.cvtColor(rgb_image, cv2.COLOR_RGB2BGR)
cv2.imwrite('../images/test.png', bgr_image)
img = cv2.imread('../images/test.png')
cv2.namedWindow('predicted_img', cv2.WINDOW_NORMAL)
cv2.resizeWindow('predicted_img', 880, 680)
cv2.imshow('predicted_img', img)
cv2.waitKey(0)
cv2.destroyAllWindows()
elif emotion_text == 'sad':
color = emotion_probability * np.asarray((0, 0, 255))
elif emotion_text == 'happy':
color = emotion_probability * np.asarray((255, 255, 0))
elif emotion_text == 'surprise':
color = emotion_probability * np.asarray((0, 255, 255))
else:
color = emotion_probability * np.asarray((0, 255, 0))
color = color.astype(int)
color = color.tolist()
if fname == "Unknown":
name = emotion_text
else:
name = str(fname) + " is " + str(emotion_text)
draw_bounding_box(face_utils.rect_to_bb(face_coordinates), rgb_image, color)
draw_text(face_utils.rect_to_bb(face_coordinates), rgb_image, name,
color, 0, -45, 0.5, 1)
frame = cv2.cvtColor(rgb_image, cv2.COLOR_RGB2BGR)
cv2.imshow('window_frame', frame)
if cv2.waitKey(1) & 0xFF == ord('q'):
break
cap.release()
cv2.destroyAllWindows()
def extract_emotions(self, bgr_image):
base_path = os.path.realpath(__file__)
base_path = base_path[:base_path.find('emotions')]
emotion_labels = get_labels('fer2013')
# hyper-parameters for bounding boxes shape
frame_window = 10
emotion_offsets = (20, 40)
# loading models
if self.emotion_classifier == None:
self.emotion_classifier = load_model(self.emotion_model_path)
# getting input model shapes for inference
emotion_target_size = self.emotion_classifier.input_shape[1:3]
# starting lists for calculating modes
emotion_window = []
gray_image = cv2.cvtColor(bgr_image, cv2.COLOR_BGR2GRAY)
rgb_image = cv2.cvtColor(bgr_image, cv2.COLOR_BGR2RGB)
faces = self.face_cascade.detectMultiScale(
gray_image,
scaleFactor=1.1,
minNeighbors=5,
minSize=(30, 30),
flags=cv2.CASCADE_SCALE_IMAGE)
for face_coordinates in faces:
x1, x2, y1, y2 = apply_offsets(face_coordinates, emotion_offsets)
gray_face = gray_image[y1:y2, x1:x2]
try:
gray_face = cv2.resize(gray_face, (emotion_target_size))
except:
continue
gray_face = preprocess_input(gray_face, True)
gray_face = np.expand_dims(gray_face, 0)
gray_face = np.expand_dims(gray_face, -1)
emotion_prediction = self.emotion_classifier.predict(gray_face)
emotion_probability = np.max(emotion_prediction)
emotion_label_arg = np.argmax(emotion_prediction)
emotion_text = emotion_labels[emotion_label_arg]
emotion_window.append(emotion_text)
if len(emotion_window) > frame_window:
emotion_window.pop(0)
try:
emotion_mode = mode(emotion_window)
except:
continue
if emotion_text == 'angry':
color = emotion_probability * np.asarray((255, 0, 0))
elif emotion_text == 'sad':
color = emotion_probability * np.asarray((0, 0, 255))
elif emotion_text == 'happy':
color = emotion_probability * np.asarray((255, 255, 0))
elif emotion_text == 'surprise':
color = emotion_probability * np.asarray((0, 255, 255))
elif emotion_text == 'fear':
color = emotion_probability * np.asarray((255, 0, 255))
elif emotion_text == 'disgust':
color = emotion_probability * np.asarray((255, 255, 255))
else:
color = emotion_probability * np.asarray((50, 200, 100))
color = color.astype(int)
color = color.tolist()
draw_bounding_box(face_coordinates, rgb_image, color)
draw_text(face_coordinates, rgb_image, emotion_mode, color, 0, -45,
1, 1)
bgr_image = cv2.cvtColor(rgb_image, cv2.COLOR_RGB2BGR)
cv2.imshow('window_frame', bgr_image)
if cv2.waitKey(1) & 0xFF == ord('q'):
break
continue
rgb_face = preprocess_input(rgb_face, False)
rgb_face = np.expand_dims(rgb_face, 0)
gender_prediction = gender_classifier.predict(rgb_face)
gender_label_arg = np.argmax(gender_prediction)
gender_text = gender_labels[gender_label_arg]
gray_face = preprocess_input(gray_face, True)
gray_face = np.expand_dims(gray_face, 0)
gray_face = np.expand_dims(gray_face, -1)
emotion_label_arg = np.argmax(emotion_classifier.predict(gray_face))
emotion_text = emotion_labels[emotion_label_arg]
if gender_text == gender_labels[0]:
color = (0, 0, 255)
else:
color = (255, 0, 0)
draw_bounding_box(face_coordinates, rgb_image, color)
# draw_text(face_coordinates, rgb_image, gender_text, color, 0, -20, 1, 2)
draw_text(face_coordinates, rgb_image, emotion_text, color, 0, 20, 1,
2)
bgr_image = cv2.cvtColor(rgb_image, cv2.COLOR_RGB2BGR)
filename = image_path[num].split('/')[-1]
filename = filename.split('.')[0]
newname = '../predicted_image/' + 'predicted_' + filename + '.png'
cv2.imwrite(newname, bgr_image)
def process_image(image):
try:
# parameters for loading data and images
detection_model_path = './trained_models/detection_models/haarcascade_frontalface_default.xml'
emotion_model_path = './trained_models/emotion_models/fer2013_mini_XCEPTION.102-0.66.hdf5'
gender_model_path = './trained_models/gender_models/simple_CNN.81-0.96.hdf5'
emotion_labels = get_labels('fer2013')
gender_labels = get_labels('imdb')
# hyper-parameters for bounding boxes shape
gender_offsets = (10, 10)
emotion_offsets = (0, 0)
# loading models
face_detection = load_detection_model(detection_model_path)
emotion_classifier = load_model(emotion_model_path, compile=False)
gender_classifier = load_model(gender_model_path, compile=False)
# getting input model shapes for inference
emotion_target_size = emotion_classifier.input_shape[1:3]
gender_target_size = gender_classifier.input_shape[1:3]
# loading images
image_array = np.fromstring(image, np.uint8)
unchanged_image = cv2.imdecode(image_array, cv2.IMREAD_UNCHANGED)
rgb_image = cv2.cvtColor(unchanged_image, cv2.COLOR_BGR2RGB)
gray_image = cv2.cvtColor(unchanged_image, cv2.COLOR_BGR2GRAY)
faces = detect_faces(face_detection, gray_image)
for face_coordinates in faces:
x1, x2, y1, y2 = apply_offsets(face_coordinates, gender_offsets)
rgb_face = rgb_image[y1:y2, x1:x2]
x1, x2, y1, y2 = apply_offsets(face_coordinates, emotion_offsets)
gray_face = gray_image[y1:y2, x1:x2]
try:
rgb_face = cv2.resize(rgb_face, gender_target_size)
gray_face = cv2.resize(gray_face, emotion_target_size)
except:
continue
rgb_face = preprocess_input(rgb_face, False)
rgb_face = np.expand_dims(rgb_face, 0)
gender_prediction = gender_classifier.predict(rgb_face)
gender_label_arg = np.argmax(gender_prediction)
gender_text = gender_labels[gender_label_arg]
gray_face = preprocess_input(gray_face, True)
gray_face = np.expand_dims(gray_face, 0)
gray_face = np.expand_dims(gray_face, -1)
emotion_label_arg = np.argmax(
emotion_classifier.predict(gray_face))
emotion_text = emotion_labels[emotion_label_arg]
if gender_text == gender_labels[0]:
color = (0, 0, 255)
else:
color = (255, 0, 0)
draw_bounding_box(face_coordinates, rgb_image, color)
draw_text(face_coordinates, rgb_image, gender_text, color, 0, -20,
1, 2)
draw_text(face_coordinates, rgb_image, emotion_text, color, 0, -50,
1, 2)
except Exception as err:
logging.error('Error in emotion gender processor: "{0}"'.format(err))
bgr_image = cv2.cvtColor(rgb_image, cv2.COLOR_RGB2BGR)
dirname = 'result'
if not os.path.exists(dirname):
os.mkdir(dirname)
cv2.imwrite(os.path.join(dirname, 'predicted_image.png'), bgr_image)
try:
rgb_face = cv2.resize(rgb_face, (gender_target_size))
gray_face = cv2.resize(gray_face, (emotion_target_size))
except:
continue
rgb_face = preprocess_input(rgb_face, False)
rgb_face = np.expand_dims(rgb_face, 0)
gender_prediction = gender_classifier.predict(rgb_face)
gender_label_arg = np.argmax(gender_prediction)
gender_text = gender_labels[gender_label_arg]
gray_face = preprocess_input(gray_face, True)
gray_face = np.expand_dims(gray_face, 0)
gray_face = np.expand_dims(gray_face, -1)
emotion_label_arg = np.argmax(emotion_classifier.predict(gray_face))
emotion_text = emotion_labels[emotion_label_arg]
if gender_text == gender_labels[0]:
color = (0, 0, 255)
else:
color = (255, 0, 0)
draw_bounding_box(face_coordinates, rgb_image, color)
draw_text(face_coordinates, rgb_image, gender_text, color, 0, -20, 1, 2)
draw_text(face_coordinates, rgb_image, emotion_text, color, 0, -50, 1, 2)
bgr_image = cv2.cvtColor(rgb_image, cv2.COLOR_RGB2BGR)
cv2.imwrite('../images/predicted_test_image.png', bgr_image)
def callback(self,data):
try:
bgr_image = self.bridge.imgmsg_to_cv2(data, "bgr8")
cv2.imshow("Image window", bgr_image)
except CvBridgeError as e:
print(e)
######################### Start the image processing
gray_image = cv2.cvtColor(bgr_image, cv2.COLOR_BGR2GRAY)
rgb_image = cv2.cvtColor(bgr_image, cv2.COLOR_BGR2RGB)
faces = detect_faces(self.face_detection, gray_image)
for face_coordinates in faces:
x1, x2, y1, y2 = apply_offsets(face_coordinates, self.emotion_offsets)
gray_face = gray_image[y1:y2, x1:x2]
try:
gray_face = cv2.resize(gray_face, (self.emotion_target_size))
except:
continue
gray_face = preprocess_input(gray_face, True)
gray_face = np.expand_dims(gray_face, 0)
gray_face = np.expand_dims(gray_face, -1)
print "start"
print len(gray_face)
print gray_face
print "end"
emotion_prediction = self.emotion_classifier.predict(gray_face)
emotion_probability = np.max(emotion_prediction)
emotion_label_arg = np.argmax(emotion_prediction)
emotion_text = emotion_labels[emotion_label_arg]
print emotion_text
print(emotion_probability)
print('%')
self.emotion_window.append(emotion_text)
if len(self.emotion_window) > self.frame_window:
self.emotion_window.pop(0)
try:
emotion_mode = mode(self.emotion_window)
except:
continue
if emotion_text == 'angry':
color = emotion_probability * np.asarray((255, 0, 0))
elif emotion_text == 'sad':
color = emotion_probability * np.asarray((0, 0, 255))
elif emotion_text == 'happy':
color = emotion_probability * np.asarray((255, 255, 0))
elif emotion_text == 'surprise':
color = emotion_probability * np.asarray((0, 255, 255))
else:
color = emotion_probability * np.asarray((0, 255, 0))
color = color.astype(int)
color = color.tolist()
draw_bounding_box(face_coordinates, rgb_image, color)
draw_text(face_coordinates, rgb_image, emotion_mode,
color, 0, -45, 1, 1)
bgr_image = cv2.cvtColor(rgb_image, cv2.COLOR_RGB2BGR)
######################### End the image processing
cv2.imshow("Image window", bgr_image)
cv2.waitKey(3)
try:
self.image_pub.publish(self.bridge.cv2_to_imgmsg(bgr_image, "bgr8"))
except CvBridgeError as e:
print(e)
def callback(self,data):
#/////////////////////////////////////////////
#cv_image = bridge.imgmsg_to_cv2(image_msg, desired_encoding="passthrough")
#cv_image = cv2.resize(cv_image, target_size) # resize image
#np_image = np.asarray(cv_image) # read as np array
#np_image = np.expand_dims(np_image, axis=0) # Add another dimension for tensorflow
#np_image = np_image.astype(float) # preprocess needs float64 and img is uint8
#np_image = preprocess_input(np_image) # Regularize the data
#/////////////////////////////////////////////
if(not USE_LOCAL_CAMERA):
try:
frame = self.bridge.imgmsg_to_cv2(data, desired_encoding="passthrough")
except CvBridgeError as e:
print(e)
# Capture frame-by-frame
if(USE_LOCAL_CAMERA):
ret, frame1 = self.video_capture.read()
#/////////////////////////////////////////////
#print"--------"
#print('input_msg height : {}'.format(frame.height))
#print('input_msg width : {}'.format(frame.width))
#print('input_msg step : {}'.format(frame.step))
#print('input_msg encoding: {}'.format(frame.encoding))
#print('output dtype : {}'.format(frame.dtype))
#print('output shape : {}'.format(frame.shape))
#print"--------"
gray_image = cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY)
rgb_image = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)
faces = detect_faces(self.face_detection, gray_image)
for face_coordinates in faces:
print "inside the for loop!"
print face_coordinates
x1, x2, y1, y2 = apply_offsets(face_coordinates, self.emotion_offsets)
gray_face = gray_image[y1:y2, x1:x2]
#print len(frame)
#print gray_face.size
#print gray_face.shape
#print gray_face.dtype
try:
gray_face = cv2.resize(gray_face, (self.emotion_target_size))
except:
continue
gray_face = preprocess_input(gray_face, True)
gray_face = np.expand_dims(gray_face, 0)
gray_face = np.expand_dims(gray_face, -1)
#print"************"
#print('gray_face dtype : {}'.format(gray_face.dtype))
#print('gray_face shape : {}'.format(gray_face.shape))
#print('gray_face size : {}'.format(gray_face.size))
#print"************"
## This is a workaround for asynchronous execution using TF and ROS
# https://github.com/keras-team/keras/issues/2397
# http://projectsfromtech.blogspot.com/2017/10/visual-object-recognition-in-ros-using.html
#global self.graph
with self.graph.as_default():
emotion_prediction = self.emotion_classifier.predict(gray_face)
emotion_probability = np.max(emotion_prediction)
emotion_label_arg = np.argmax(emotion_prediction)
emotion_text = self.emotion_labels[emotion_label_arg]
print emotion_text
print(emotion_probability)
print('%')
self.emotion_window.append(emotion_text)
#self.emotion_msg.data = emotion_text
#self.emotion_publisher.publish(emotion_msg)
#self.speech_msg.data = 'I see that you are ' + emotion_text
#self.speech_publisher.publish(speech_msg)
if len(self.emotion_window) > self.frame_window:
self.emotion_window.pop(0)
try:
emotion_mode = mode(self.emotion_window)
except:
continue
if emotion_text == 'angry':
color = emotion_probability * np.asarray((255, 0, 0))
elif emotion_text == 'sad':
color = emotion_probability * np.asarray((0, 0, 255))
elif emotion_text == 'happy':
color = emotion_probability * np.asarray((255, 255, 0))
elif emotion_text == 'surprise':
color = emotion_probability * np.asarray((0, 255, 255))
else:
color = emotion_probability * np.asarray((0, 255, 0))
color = color.astype(int)
color = color.tolist()
draw_bounding_box(face_coordinates, rgb_image, color)
draw_text(face_coordinates, rgb_image, emotion_mode,
color, 0, -45, 1, 1)
try:
self.image_pub.publish(self.bridge.cv2_to_imgmsg(rgb_image, "bgr8"))
except CvBridgeError as e:
print(e)
def emotion_demo():
# parameters for loading data and images
detection_model_path = '../trained_models/detection_models/haarcascade_frontalface_default.xml'
emotion_model_path = '../trained_models/emotion_models/fer2013_mini_XCEPTION.102-0.66.hdf5'
emotion_labels = get_labels('fer2013')
# 目のカスケードファイル追加
lefteyecc__path = "../trained_models/detection_models/haarcascade_lefteye_2splits.xml"
righteyecc_path = "../trained_models/detection_models/haarcascade_righteye_2splits.xml"
nose_path = "../trained_models/detection_models/data_haarcascades_haarcascade_mcs_nose.xml"
lefteyecc = cv2.CascadeClassifier(lefteyecc__path)
righteyecc = cv2.CascadeClassifier(righteyecc_path)
nose = cv2.CascadeClassifier(nose_path)
lex = 0; ley = 0; lew = 0; leh = 0
rex = 0; rey = 0; rew = 0; reh = 0
nox = 0; noy = 0; now = 0; noh = 0
# dlib
dlib_ini()
# hyper-parameters for bounding boxes shape
frame_window = 10
emotion_offsets = (20, 40)
# loading models
face_detection = load_detection_model(detection_model_path)
emotion_classifier = load_model(emotion_model_path, compile=False)
# getting input model shapes for inference
emotion_target_size = emotion_classifier.input_shape[1:3]
# starting lists for calculating modes
emotion_window = []
global img, flag, slp_count
img = cv2.imread('../img/happy.png')
flag = 0
slp_count = 0
# dlib用グローバル変数
global gray_image, rgb_image, gray_face, mark_list
# starting video streaming
cv2.namedWindow('window_frame', cv2.WINDOW_NORMAL)
video_capture = cv2.VideoCapture(0) # 0は内蔵カメラ, 1はUSBカメラ
while True:
bgr_image = video_capture.read()[1]
gray_image = cv2.cvtColor(bgr_image, cv2.COLOR_BGR2GRAY)
rgb_image = cv2.cvtColor(bgr_image, cv2.COLOR_BGRA2RGBA)
faces = detect_faces(face_detection, gray_image)
for face_coordinates in faces:
# 目や鼻認識用
(x,y,w,h) = face_coordinates
video_face = gray_image[y:y+h,x:x+w]
x1, x2, y1, y2 = apply_offsets(face_coordinates, emotion_offsets)
gray_face = gray_image[y1:y2, x1:x2]
try:
gray_face = cv2.resize(gray_face, (emotion_target_size))
except:
continue
# ランドマーク検出
marks_list = marks_list_def(bgr_image, x, y, w, h)
print(marks_list)
gray_face = preprocess_input(gray_face, True)
gray_face = np.expand_dims(gray_face, 0)
gray_face = np.expand_dims(gray_face, -1)
emotion_prediction = emotion_classifier.predict(gray_face)
emotion_probability = np.max(emotion_prediction)
emotion_label_arg = np.argmax(emotion_prediction)
emotion_text = emotion_labels[emotion_label_arg]
emotion_window.append(emotion_text)
if len(emotion_window) > frame_window:
emotion_window.pop(0)
try:
emotion_mode = mode(emotion_window)
except:
continue
if flag == 0 or flag == 1:
if emotion_text == 'angry':
img = cv2.imread('../img/angry.png', -1)
color = emotion_probability * np.asarray((255, 0, 0))
elif emotion_text == 'sad':
img = cv2.imread('../img/sad.png', -1) # 関数にする
color = emotion_probability * np.asarray((0, 0, 255))
elif emotion_text == 'happy':
img = cv2.imread('../img/happy.png', -1)
color = emotion_probability * np.asarray((255, 255, 0))
elif emotion_text == 'surprise':
img = cv2.imread('../img/odoroki.png', -1)
color = emotion_probability * np.asarray((0, 255, 255))
else :
img = cv2.imread('../img/neutral.png', -1)
color = emotion_probability * np.asarray((0, 255, 0))
else:
if emotion_text == 'angry':
img = cv2.imread('../img/ikari.png', -1)
color = emotion_probability * np.asarray((255, 0, 0))
elif emotion_text == 'sad':
img = cv2.imread('../img/shock.png', -1)
color = emotion_probability * np.asarray((0, 0, 255))
elif emotion_text == 'happy':
img = cv2.imread('../img/kirakira.png', -1)
color = emotion_probability * np.asarray((255, 255, 0))
elif emotion_text == 'surprise':
img = cv2.imread('../img/bikkuri.png', -1)
color = emotion_probability * np.asarray((0, 255, 255))
else :
img = cv2.imread('../img/toumei.png', -1)
color = emotion_probability * np.asarray((0, 255, 0))
color = color.astype(int)
color = color.tolist()
if flag == 0:
draw_bounding_box(face_coordinates, rgb_image, color)
elif flag == 1:
rgb_image = draw_bounding_box2(face_coordinates, rgb_image, color, img, marks_list)
elif flag == 2:
overlay_pic = draw_bounding_box3(face_coordinates, rgb_image, color, img, marks_list)
rgb_image = overlay_pic
draw_text(face_coordinates, rgb_image, emotion_mode, color, 0, -45, 1, 1)
bgr_image = cv2.cvtColor(rgb_image, cv2.COLOR_RGB2BGR)
if flag == 0:
img = image_resize(img)
cv2.imshow('image', img)
cv2.destroyWindow('Window_frame')
elif flag == 1 or flag == 2:
cv2.destroyWindow('image')
cv2.imshow('window_frame', bgr_image)
cv2.waitKey(10)
# cv2.imshow('own_window', bgr_image)
if cv2.waitKey(1) & 0xFF == ord('z'):
flag = 0
elif cv2.waitKey(1) & 0xFF == ord('x'):
flag = 1
elif cv2.waitKey(1) & 0xFF == ord('c'):
flag = 2
elif cv2.waitKey(1) & 0xFF == ord('q'):
break
video_capture.release()
cv2.destroyAllWindows()
def recognize_video(self, path=None, csv=True):
# starting video streaming
if self.show_frame:
cv2.namedWindow('window_frame')
# Select video or webcam feed
cap = None
if path is None:
cap = cv2.VideoCapture(0) # Webcam source
elif isinstance(path, str):
cap = cv2.VideoCapture(path) # Video file source
else:
raise Exception('path argument must be a string')
emotions_by_frames = [
] # container for recording emotion probabilities by frame
frame_counter = 0
while cap.isOpened(): # True:
try:
ret, bgr_image = cap.read()
if not ret:
break
#print('Frame {} is read'.format(frame_counter))
frame_counter += 1
rgb_image = cv2.cvtColor(bgr_image, cv2.COLOR_BGR2RGB)
start = default_timer()
faces = self.detector.detect_faces(rgb_image)
#print('faces', faces)
end = default_timer()
#print('face detection:', end - start)
if len(faces) == 0:
emotions_by_frames.append(np.array([0] * 7))
for face_coordinates in faces:
# face_coordinates: [x, y, w, h]
face_coordinates = list(map(int, face_coordinates))
x, y, w, h = face_coordinates
# expand to 20%
padding_size_x, padding_size_y = int(0.2 * x), int(0.2 * y)
w += padding_size_x
h += padding_size_y
x -= padding_size_x // 2
y -= padding_size_y // 2
face_image = rgb_image[y:y + h, x:x + w]
try:
face_image = cv2.resize(
face_image, (self.face_size, self.face_size))
except:
emotions_by_frames.append(
np.array([0] * len(self.emotion_labels)))
continue
face_image = self.prepare_face_image(face_image)
#print('Shape:', face_image.shape)
emotion_prediction = self.emotion_classifier.predict(
face_image, probs=True)[0].numpy()
#print('Prediction:', emotion_prediction)
emotion_probability = np.max(emotion_prediction)
if len(emotion_prediction) > 0:
emotions_by_frames.append(emotion_prediction)
else:
emotions_by_frames.append(
np.array([0] * len(self.emotion_labels)))
emotion_label_arg = np.argmax(emotion_prediction)
emotion_text = self.emotion_labels[emotion_label_arg]
if emotion_text == 'angry':
color = emotion_probability * np.asarray((255, 0, 0))
elif emotion_text == 'sad':
color = emotion_probability * np.asarray((0, 0, 255))
elif emotion_text == 'happy':
color = emotion_probability * np.asarray((255, 255, 0))
elif emotion_text == 'surprise':
color = emotion_probability * np.asarray((0, 255, 255))
else:
color = emotion_probability * np.asarray((0, 255, 0))
color = color.astype(int)
color = color.tolist()
draw_bounding_box(face_coordinates, bgr_image, color)
draw_text(face_coordinates, bgr_image, emotion_text, color,
0, -45, 1, 1)
#bgr_image = cv2.cvtColor(rgb_image, cv2.COLOR_RGB2BGR)
cv2.imshow('window_frame', bgr_image)
if (cv2.waitKey(1) & 0xFF == ord('q')) and self.show_frame:
break
except:
traceback.print_exc()
break
cap.release()
cv2.destroyAllWindows()
if csv:
return self.to_csv(emotions_by_frames)
return emotions_by_frames
if len(emotion_window) > frame_window:
emotion_window.pop(0)
try:
emotion_mode = mode(emotion_window)
except:
continue
if emotion_text == 'angry':
color = emotion_probability * np.asarray((255, 0, 0))
elif emotion_text == 'sad':
color = emotion_probability * np.asarray((0, 0, 255))
elif emotion_text == 'happy':
color = emotion_probability * np.asarray((255, 255, 0))
elif emotion_text == 'surprise':
color = emotion_probability * np.asarray((0, 255, 255))
else:
color = emotion_probability * np.asarray((0, 255, 0))
color = color.astype(int)
color = color.tolist()
draw_bounding_box(face_coordinates, rgb_image, color)
draw_text(face_coordinates, rgb_image, emotion_mode,
color, 0, -45, 1, 1)
bgr_image = cv2.cvtColor(rgb_image, cv2.COLOR_RGB2BGR)
cv2.imshow('window_frame', bgr_image)
if cv2.waitKey(1) & 0xFF == ord('q'):
break
def biaoqing(frame):
gray_image = cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY)
rgb_image = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)
dets = detector(gray_image, 1)
#print(dets)
#faces = detect_faces(face_detection, gray_image)#[[横，纵，宽，高]]
#print(faces)
max_face = 0
max_k = 0
emotion_text = 0
for k,d in enumerate(dets):
if (d.right()-d.left())*(d.bottom()-d.top()) > max_face:
max_face = (d.right()-d.left())*(d.bottom()-d.top())
max_k = k
for k,d in enumerate(dets):
if k != max_k: continue
face_coordinates=[]
face_coordinates.append(d.left())
face_coordinates.append(d.top())
face_coordinates.append(d.right()-d.left())
face_coordinates.append(d.bottom()-d.top())
x1, x2, y1, y2 = apply_offsets(face_coordinates, emotion_offsets)
gray_face = gray_image[y1:y2, x1:x2]
try:
gray_face = cv2.resize(gray_face, (emotion_target_size))
except:
continue
gray_face = preprocess_input(gray_face, True)
gray_face = np.expand_dims(gray_face, 0)
gray_face = np.expand_dims(gray_face, -1)
emotion_prediction = emotion_classifier.predict(gray_face)
emotion_probability = np.max(emotion_prediction)
emotion_label_arg = np.argmax(emotion_prediction)
emotion_text = emotion_labels[emotion_label_arg]
emotion_window.append(emotion_text)
if len(emotion_window) > frame_window:
emotion_window.pop(0)
try:
emotion_mode = mode(emotion_window)
except:
continue
if emotion_text == 'angry':
color = emotion_probability * np.asarray((255, 0, 0))
elif emotion_text == 'sad':
color = emotion_probability * np.asarray((0, 0, 255))
elif emotion_text == 'happy':
color = emotion_probability * np.asarray((255, 255, 0))
elif emotion_text == 'surprise':
color = emotion_probability * np.asarray((0, 255, 255))
else:
color = emotion_probability * np.asarray((0, 255, 0))
color = color.astype(int)
color = color.tolist()
draw_bounding_box(face_coordinates, rgb_image, color)
draw_text(face_coordinates, rgb_image, emotion_mode,
color, 0, -45, 1, 1)
bgr_image = cv2.cvtColor(rgb_image, cv2.COLOR_RGB2BGR)
# cv2.imshow('out', bgr_image)
if emotion_text:
return emotion_text, bgr_image
else:
return 0, frame
def processa():
# parameters for loading data and images
emotion_model_path = './models/emotion_model.hdf5' #Carica il modello delle emozioni , pre-addestrato
emotion_labels = get_labels('fer2013')
# hyper-parameters for bounding boxes shape
frame_window = 10
emotion_offsets = (20, 40)
# loading models #Rilevamento caratteristiche del viso
detector = dlib.get_frontal_face_detector(
) #Rilevamento caratteristiche del viso
emotion_classifier = load_model(emotion_model_path)
# predictor = dlib.shape_predictor("shape_predictor_68_face_landmarks.dat")
# getting input model shapes for inference
emotion_target_size = emotion_classifier.input_shape[
1:3] #Fa una stima parziale del modello di input
# starting lists for calculating modes
emotion_window = []
# Initialize some variables
face_locations = []
face_encodings = [] #Delimita i volti umani
face_names = []
process_this_frame = True
def face_compare(frame, process_this_frame):
# Resize frame of video to 1/4 size for faster face recognition processing
small_frame = cv2.resize(frame, (0, 0), fx=0.50, fy=0.50)
# Convert the image from BGR color (which OpenCV uses) to RGB color (which face_recognition uses)
rgb_small_frame = small_frame[:, :, ::-1]
# Only process every other frame of video to save time
if process_this_frame:
# Find all the faces and face encodings in the current frame of video(cerca i volti nel frame video)
face_locations = face_recognition.face_locations(
rgb_small_frame) #Delimitazione del volto umano
face_encodings = face_recognition.face_encodings(
rgb_small_frame, face_locations
) #apprende se c'è un volto umano tramite face location
# starting video streaming
cv2.namedWindow('window_frame')
cap = None
# Webcam source
cap = cv2.VideoCapture(0)
while cap.isOpened(): # True:
ret, frame = cap.read(
) #restituisce le immagini della webcam frame per frame
#frame = video_capture.read()[1]
# To print the facial landmarks
# landmrk = face_recognition.face_landmarks(frame)
# for l in landmrk:
# for key,val in l.items():
# for (x,y) in val:
# cv2.circle(frame, (x, y), 1, (255,0, 0), -1)
gray_image = cv2.cvtColor(
frame, cv2.COLOR_BGR2GRAY) #trasforma i frame in scala di grigio
rgb_image = cv2.cvtColor(frame,
cv2.COLOR_BGR2RGB) #trasforma i frame in rgb
faces = detector(rgb_image) #rileva caratteristiche del viso
# face_locations = face_recognition.face_locations(rgb_image)
# print (reversed(face_locations))
face_name = face_compare(rgb_image, process_this_frame)
for face_coordinates, fname in zip(faces, "Unknow"):
x1, x2, y1, y2 = apply_offsets(
face_utils.rect_to_bb(face_coordinates), emotion_offsets)
gray_face = gray_image[y1:y2, x1:x2]
try:
gray_face = cv2.resize(gray_face, (emotion_target_size))
except:
continue
gray_face = preprocess_input(gray_face, True)
gray_face = np.expand_dims(gray_face, 0)
gray_face = np.expand_dims(gray_face, -1)
emotion_prediction = emotion_classifier.predict(
gray_face) #fa delle predizioni
emotion_probability = np.max(
emotion_prediction) #prende la predizione con valore massimo
emotion_label_arg = np.argmax(
emotion_prediction
) #prende l'indice della predizione con valore massimo
emotion_text = emotion_labels[
emotion_label_arg] #inserisce nella variabile l'umore predetto
emotion_window.append(emotion_text)
if len(emotion_window) > frame_window:
emotion_window.pop(0)
try:
emotion_mode = mode(
emotion_window
) #restituisce il dato più comune nella lista
except:
continue
if emotion_text == 'Triste':
color = emotion_probability * np.asarray((0, 0, 255))
elif emotion_text == 'Felice':
color = emotion_probability * np.asarray((255, 255, 0))
else:
color = emotion_probability * np.asarray((0, 255, 0))
color = color.astype(int)
color = color.tolist() #converte un colore in un elenco del modulo
name = emotion_text
draw_bounding_box(face_utils.rect_to_bb(face_coordinates),
rgb_image, color) #crea un rettangolo colorato
draw_text(face_utils.rect_to_bb(face_coordinates), rgb_image, name,
color, 0, -45, 0.5,
1) #inserisce l'emozione predetta colorata
frame = cv2.cvtColor(rgb_image,
cv2.COLOR_RGB2BGR) #converte da rgb a bgr
cv2.imshow('window_frame',
frame) #mostra il frame nella finestra windows_frame
if cv2.waitKey(1) & 0xFF == ord(
'q'): #se c'è un ritardo di un milli secondo allora break
return emotion_text
cap.release()
cv2.destroyAllWindows()
emotion_window.pop(0)
try:
emotion_mode = mode(emotion_window)
except:
continue
if emotion_text == 'angry':
color = emotion_probability * np.asarray((255, 0, 0))
elif emotion_text == 'sad':
color = emotion_probability * np.asarray((0, 0, 255))
elif emotion_text == 'happy':
color = emotion_probability * np.asarray((255, 255, 0))
elif emotion_text == 'surprise':
color = emotion_probability * np.asarray((0, 255, 255))
else:
color = emotion_probability * np.asarray((0, 255, 0))
color = color.astype(int)
color = color.tolist()
draw_bounding_box(face_coordinates, rgb_image, color)
draw_text(face_coordinates, rgb_image, emotion_mode, color, 0, -45, 1,
2)
bgr_image = cv2.cvtColor(rgb_image, cv2.COLOR_RGB2BGR)
cv2.imshow('window_frame', bgr_image)
if cv2.waitKey(1) & 0xFF == ord('q'):
break
video_capture.release()
cv2.destroyAllWindows()
"input": {
"'neutral'": "1"
},
"robot": "Pepper"
}
print("neutral")
pub_emotion.publish(data)
color = 1 * np.asarray((0, 0, 0))
else:
draw = False
if draw:
color = color.astype(int)
color = color.tolist()
draw_bounding_box(box, rgb_image, color)
draw_text(box, rgb_image, emotion_mode, color, 0, -45, 1, 1)
if (show_image == 1):
bgr_image = cv2.cvtColor(rgb_image, cv2.COLOR_RGB2BGR)
cv2.imshow('window_frame', bgr_image)
if cv2.waitKey(1) & 0xFF == ord('q'):
break
except:
pass
else:
pass
def process_image(image):
try:
# parameters for loading data and images
detection_model_path = './trained_models/detection_models/haarcascade_frontalface_default.xml'
emotion_model_path = './trained_models/emotion_models/fer2013_mini_XCEPTION.102-0.66.hdf5'
gender_model_path = './trained_models/gender_models/simple_CNN.81-0.96.hdf5'
emotion_labels = get_labels('fer2013')
gender_labels = get_labels('imdb')
font = cv2.FONT_HERSHEY_SIMPLEX
# hyper-parameters for bounding boxes shape
gender_offsets = (30, 60)
gender_offsets = (10, 10)
emotion_offsets = (20, 40)
emotion_offsets = (0, 0)
# loading models
face_detection = load_detection_model(detection_model_path)
emotion_classifier = load_model(emotion_model_path, compile=False)
gender_classifier = load_model(gender_model_path, compile=False)
# getting input model shapes for inference
emotion_target_size = emotion_classifier.input_shape[1:3]
gender_target_size = gender_classifier.input_shape[1:3]
# loading images
image_array = np.fromstring(image, np.uint8)
unchanged_image = cv2.imdecode(image_array, cv2.IMREAD_UNCHANGED)
rgb_image = cv2.cvtColor(unchanged_image, cv2.COLOR_BGR2RGB)
gray_image = cv2.cvtColor(unchanged_image, cv2.COLOR_BGR2GRAY)
faces = detect_faces(face_detection, gray_image)
for face_coordinates in faces:
x1, x2, y1, y2 = apply_offsets(face_coordinates, gender_offsets)
rgb_face = rgb_image[y1:y2, x1:x2]
x1, x2, y1, y2 = apply_offsets(face_coordinates, emotion_offsets)
gray_face = gray_image[y1:y2, x1:x2]
try:
rgb_face = cv2.resize(rgb_face, (gender_target_size))
gray_face = cv2.resize(gray_face, (emotion_target_size))
except:
continue
rgb_face = preprocess_input(rgb_face, False)
rgb_face = np.expand_dims(rgb_face, 0)
gender_prediction = gender_classifier.predict(rgb_face)
gender_label_arg = np.argmax(gender_prediction)
gender_text = gender_labels[gender_label_arg]
gray_face = preprocess_input(gray_face, True)
gray_face = np.expand_dims(gray_face, 0)
gray_face = np.expand_dims(gray_face, -1)
emotion_label_arg = np.argmax(emotion_classifier.predict(gray_face))
emotion_text = emotion_labels[emotion_label_arg]
if gender_text == gender_labels[0]:
color = (0, 0, 255)
else:
color = (255, 0, 0)
draw_bounding_box(face_coordinates, rgb_image, color)
draw_text(face_coordinates, rgb_image, gender_text, color, 0, -20, 1, 2)
draw_text(face_coordinates, rgb_image, emotion_text, color, 0, -50, 1, 2)
except Exception as err:
logging.error('Error in emotion gender processor: "{0}"'.format(err))
bgr_image = cv2.cvtColor(rgb_image, cv2.COLOR_RGB2BGR)
dirname = 'result'
if not os.path.exists(dirname):
os.mkdir(dirname)
cv2.imwrite(os.path.join(dirname, 'predicted_image.png'), bgr_image)
emotion_mode = mode(emotion_window)
except:
continue
if emotion_text == 'angry':
color = emotion_probability * np.asarray((255, 0, 0))
elif emotion_text == 'sad':
color = emotion_probability * np.asarray((0, 0, 255))
elif emotion_text == 'happy':
print (emotion_probability)
color = emotion_probability * np.asarray((255, 255, 0))
elif emotion_text == 'surprise':
color = emotion_probability * np.asarray((0, 255, 255))
else:
color = emotion_probability * np.asarray((0, 255, 0))
color = color.astype(int)
color = color.tolist()
draw_bounding_box(face_coordinates, rgb_image, color)
draw_text(emotion_probability,face_coordinates, rgb_image, emotion_mode,
color, 0, -45, 1, 1)
bgr_image = cv2.cvtColor(rgb_image, cv2.COLOR_RGB2BGR)
out.write(bgr_image)
cv2.imshow('window_frame', bgr_image)
if cv2.waitKey(1) & 0xFF == ord('q'):
break
cap.release()
cv2.destroyAllWindows()
if emotion_text == 'angry':
color = emotion_probability * np.asarray((255, 0, 0))
elif emotion_text == 'sad':
color = emotion_probability * np.asarray((0, 0, 255))
elif emotion_text == 'happy':
color = emotion_probability * np.asarray((255, 255, 0))
elif emotion_text == 'surprise':
color = emotion_probability * np.asarray((0, 255, 255))
else:
color = emotion_probability * np.asarray((0, 255, 0))
color = color.astype(int)
color = color.tolist()
draw_bounding_box(face_coordinates, rgb_image, color)
draw_text(face_coordinates, rgb_image, emotion_mode, color, 0, -45,
1, 1)
for i in range(len(emotion_prediction[0])):
print(face_coordinates)
x, y, w, h = face_coordinates
emotion_text_coordinates = (x, y + h + 20 * i)
emotion_text = emotion_labels[i]
emotion_score = "{}: {:.2f}".format(emotion_text,
emotion_prediction[0][i])
draw_text(emotion_text_coordinates, rgb_image, emotion_score,
(0, 255, 0), 0, 45, 0.5, 1)
except:
pass
bgr_image = cv2.cvtColor(rgb_image, cv2.COLOR_RGB2BGR)
# bgr_image = cv2.flip(bgr_image, )
cv2.imshow('window_frame', bgr_image)
def main():
warnings.filterwarnings("ignore")
logging.disable(logging.CRITICAL)
global emotion_cache #Vishal
emotion_cache = [] #Vishal
global stop_detecting
#Create server client for socket connection
port = opt.port
#host_name = socket.gethostname()
#host_ip = socket.gethostbyname(host_name)
host_ip = opt.ip
mySocket = socket.socket()
mySocket.bind((host_ip, port))
print ("Camera server listening on port {0} ".format(port)+ "and IP " + str(host_ip))
mySocket.listen(1)
chatbot_socket, adress = mySocket.accept()
print("Connection established to: " + str(adress))
#Start socket process to listen to chatbot requests
chatbot = Process(target = chatbot_process, args = (chatbot_socket, chatbot_request, chatbot_response, stop_camera))
chatbot.start()
signal.signal(signal.SIGTSTP, handler)
#face_recognizer = Face_Recognizer(True)# True to show video feed
# parameters for loading data and images
detection_model_path = './models/face_recognition/trained_models/haarcascade_frontalface_default.xml'
emotion_model_path = './models/face_recognition/trained_models/fer2013_mini_XCEPTION.102-0.66.hdf5'
emotion_labels = get_labels('fer2013')
# hyper-parameters for bounding boxes shape
frame_window = 10
emotion_offsets = (20, 40)
# loading models
face_detection = load_detection_model(detection_model_path)
emotion_classifier = load_model(emotion_model_path, compile=False)
# getting input model shapes for inference
emotion_target_size = emotion_classifier.input_shape[1:3]
# starting lists for calculating modes
emotion_window = []
emotion_text = 'Happy'
# starting video streaming
cv2.namedWindow('ODO_frame')
OdoSet='nvarguscamerasrc ! video/x-raw(memory:NVMM), width=3820, height=2464, format=NV12, framerate=21/1 ! nvvidconv flip-method='+str(flip)+' ! video/x-raw, width='+str(dispW)+', height='+str(dispH)+', format=BGRx ! videoconvert ! video/x-raw, format=BGR ! appsink'
video_capture = cv2.VideoCapture(OdoSet)
fps = video_capture.get(cv2.CAP_PROP_FPS)
print(f'FPS: {fps}')
num_frames = 21;
if video_capture.isOpened():
print("True")
else:
print("Error")
##################Vishal#####################
while stop_detecting == False:
#############################################
# Detect Emotion
#print("Check")
start = time.time() #starting FPS timer
bgr_image = video_capture.read()[1]
gray_image = cv2.cvtColor(bgr_image, cv2.COLOR_BGR2GRAY)
rgb_image = cv2.cvtColor(bgr_image, cv2.COLOR_BGR2RGB)
faces = detect_faces(face_detection, gray_image)
for face_coordinates in faces:
x1, x2, y1, y2 = apply_offsets(face_coordinates, emotion_offsets)
gray_face = gray_image[y1:y2, x1:x2]
try:
gray_face = cv2.resize(gray_face, (emotion_target_size))
except:
continue
gray_face = preprocess_input(gray_face, True)
gray_face = np.expand_dims(gray_face, 0)
gray_face = np.expand_dims(gray_face, -1)
emotion_prediction = emotion_classifier.predict(gray_face)
emotion_probability = np.max(emotion_prediction)
emotion_label_arg = np.argmax(emotion_prediction)
emotion_text = emotion_labels[emotion_label_arg]
emotion_window.append(emotion_text)
if len(emotion_window) > frame_window:
emotion_window.pop(0)
try:
emotion_mode = mode(emotion_window)
except:
continue
if emotion_text == 'angry':
color = emotion_probability * np.asarray((255, 0, 0))
elif emotion_text == 'sad':
color = emotion_probability * np.asarray((0, 0, 255))
elif emotion_text == 'happy':
color = emotion_probability * np.asarray((255, 255, 0))
elif emotion_text == 'surprise':
color = emotion_probability * np.asarray((0, 255, 255))
else:
color = emotion_probability * np.asarray((0, 255, 0))
emotion_cache.append(emotion_text)
with open('listfile.txt', 'w') as filehandle:
filehandle.writelines("%s\n" % emotions for emotions in emotion_cache)
if len(emotion_cache) > 500:
emotion_cache = emotion_cache[:-500]
##########################################
total_faces = len(faces)
current_time = time.time()
if chatbot_request.empty() == False:
request = chatbot_request.get()
data_request = request['request']
if data_request.lower() == 'send_emotion':
data = {}
data['emotion'] = emotion_detected
data['total_faces'] = total_faces
data['time'] = current_time
json_data = json.dumps(data)
chatbot_response.put({'response': json_data})
color = color.astype(int)
color = color.tolist()
#print(face_coordinates, rgb_image, color)
draw_bounding_box(face_coordinates, rgb_image, color)
draw_text(face_coordinates, rgb_image, emotion_mode,
color, 0, -45, 1, 1)
if stop_camera.empty() == False:
stop_detecting = True
end = time.time() #end time for FPS
bgr_image = cv2.cvtColor(rgb_image, cv2.COLOR_RGB2BGR)
seconds = end - start #FPS
fps = num_frames/seconds #FPS
fps_coords = (0,30)
framerate_str = 'Framerate: ' + str(int(fps))
cv2.putText(bgr_image, framerate_str, fps_coords, cv2.FONT_HERSHEY_SIMPLEX, 1, (255,255,200))
print(framerate_str)
cv2.imshow('ODO_frame', bgr_image)
if cv2.waitKey(1) & 0xFF == ord('q'):
break
video_capture.release()
cv2.destroyAllWindows()
print("Closing chatbot socket")
chatbot.terminate()
chatbot.join()
def process_image(image):
detected_peoples = []
json_info = {}
try:
# parameters for loading data and images
emotion_labels = get_labels('fer2013')
gender_labels = get_labels('imdb')
# font = cv2.FONT_HERSHEY_SIMPLEX
# gender_keys = list(gender_labels.values())
# emotion_keys = list(emotion_labels.values())
# hyper-parameters for bounding boxes shape
gender_offsets = (30, 60)
gender_offsets = (10, 10)
emotion_offsets = (20, 40)
emotion_offsets = (0, 0)
# getting input model shapes for inference
gender_target_size = _GENDER_CLASSIFIER.input_shape[1:3]
emotion_target_size = _EMOTION_CLASSIFIER.input_shape[1:3]
# loading images
image_array = np.fromstring(image, np.uint8)
unchanged_image = cv2.imdecode(image_array, cv2.IMREAD_UNCHANGED)
rgb_image = cv2.cvtColor(unchanged_image, cv2.COLOR_BGR2RGB)
gray_image = cv2.cvtColor(unchanged_image, cv2.COLOR_BGR2GRAY)
faces = detect_faces(_FACE_DETECTION, gray_image)
for face_coordinates in faces:
x1, x2, y1, y2 = apply_offsets(face_coordinates, gender_offsets)
rgb_face = rgb_image[y1:y2, x1:x2]
x1, x2, y1, y2 = apply_offsets(face_coordinates, emotion_offsets)
gray_face = gray_image[y1:y2, x1:x2]
try:
rgb_face = cv2.resize(rgb_face, (gender_target_size))
gray_face = cv2.resize(gray_face, (emotion_target_size))
except:
continue
start_time = datetime.now()
with _GRAPH.as_default():
rgb_face = preprocess_input(rgb_face, False)
rgb_face = np.expand_dims(rgb_face, 0)
gender_prediction = _GENDER_CLASSIFIER.predict(rgb_face)
gender_label_arg = np.argmax(gender_prediction)
gender_text = gender_labels[gender_label_arg]
end_time = datetime.now()
delta1 = end_time - start_time
gray_face = preprocess_input(gray_face, True)
gray_face = np.expand_dims(gray_face, 0)
gray_face = np.expand_dims(gray_face, -1)
emotion_prediction = _EMOTION_CLASSIFIER.predict(gray_face)
emotion_label_arg = np.argmax(emotion_prediction)
emotion_text = emotion_labels[emotion_label_arg]
delta2 = datetime.now() - end_time
print("Delta for gender classificator {0}".format(
delta1.total_seconds() * 1000.0))
print("Delta for emotion classificator {0}".format(
delta2.total_seconds() * 1000.0))
json_info['gender'] = "{0}:{1}".format(
gender_text, gender_prediction.flat[gender_label_arg])
json_info['emotion'] = "{0}:{1}".format(
emotion_text, emotion_prediction.flat[emotion_label_arg])
json_info['face_bound'] = list(
map(lambda it: str(it),
list(face_coordinates.astype(np.int).flat)))
detected_peoples.append(json_info.copy())
print(detected_peoples)
if gender_text == gender_labels[0]:
color = (0, 0, 255)
else:
color = (255, 0, 0)
draw_bounding_box(face_coordinates, rgb_image, color)
draw_text(face_coordinates, rgb_image, gender_text, color, 0, -20,
1, 2)
draw_text(face_coordinates, rgb_image, emotion_text, color, 0, -10,
1, 2)
except Exception as err:
logging.error('Error in emotion gender processor: "{0}"'.format(err))
start_time = datetime.now()
bgr_image = cv2.cvtColor(rgb_image, cv2.COLOR_RGB2BGR)
if not os.path.exists(_SAVE_DIR):
os.mkdir(_SAVE_DIR)
recognition_datetime = str(datetime.now()).replace(' ', '_')
filepath = os.path.join(
_SAVE_DIR, 'predicted_image_{0}.png'.format(recognition_datetime))
cv2.imwrite(filepath, bgr_image)
delta = datetime.now() - start_time
print("delta for saving image", delta.total_seconds() * 1000.0)
return detected_peoples
rgb_face = preprocess_input(rgb_face, False)
gender_prediction = gender_classifier.predict(rgb_face)
gender_label_arg = np.argmax(gender_prediction)
gender_text = gender_labels[gender_label_arg]
gender_window.append(gender_text)
if len(gender_window) > frame_window:
emotion_window.pop(0)
gender_window.pop(0)
try:
emotion_mode = mode(emotion_window)
gender_mode = mode(gender_window)
except:
continue
if gender_text == gender_labels[0]:
color = (0, 0, 255)
else:
color = (255, 0, 0)
draw_bounding_box(face_coordinates, rgb_image, color)
draw_text(face_coordinates, rgb_image, gender_mode, color, 0, -20, 1,
1)
draw_text(face_coordinates, rgb_image, emotion_mode, color, 0, -45, 1,
1)
bgr_image = cv2.cvtColor(rgb_image, cv2.COLOR_RGB2BGR)
cv2.imshow('window_frame', bgr_image)
if cv2.waitKey(1) & 0xFF == ord('q'):
break
def emotion_identify(img_url):
# parameters for loading data and images
detection_model_path = 'C:/Users/Admin/PycharmProjects/Emotion_Detection/trained_models/detection_models/haarcascade_frontalface_default.xml'
emotion_model_path = 'C:/Users/Admin/PycharmProjects/Emotion_Detection/trained_models/emotion_models/fer2013_mini_XCEPTION.102-0.66.hdf5'
gender_model_path = 'C:/Users/Admin/PycharmProjects/Emotion_Detection/trained_models/gender_models/simple_CNN.81-0.96.hdf5'
emotion_labels = get_labels('fer2013')
gender_labels = get_labels('imdb')
font = cv2.FONT_HERSHEY_SIMPLEX
# hyper-parameters for bounding boxes shape
gender_offsets = (30, 60)
gender_offsets = (10, 10)
emotion_offsets = (20, 40)
emotion_offsets = (0, 0)
# loading models
face_detection = load_detection_model(detection_model_path)
emotion_classifier = load_model(emotion_model_path, compile=False)
gender_classifier = load_model(gender_model_path, compile=False)
# getting input model shapes for inference
emotion_target_size = emotion_classifier.input_shape[1:3]
gender_target_size = gender_classifier.input_shape[1:3]
# loading images
image_path = img_url
rgb_image = load_image(image_path, grayscale=False)
gray_image = load_image(image_path, grayscale=True)
gray_image = np.squeeze(gray_image)
gray_image = gray_image.astype('uint8')
faces = detect_faces(face_detection, gray_image)
if len(faces) == 0:
print("No face")
K.clear_session()
return False
emotions = collections.defaultdict(int)
for face_coordinates in faces:
x1, x2, y1, y2 = apply_offsets(face_coordinates, gender_offsets)
rgb_face = rgb_image[y1:y2, x1:x2]
x1, x2, y1, y2 = apply_offsets(face_coordinates, emotion_offsets)
gray_face = gray_image[y1:y2, x1:x2]
try:
rgb_face = cv2.resize(rgb_face, (gender_target_size))
gray_face = cv2.resize(gray_face, (emotion_target_size))
except:
continue
rgb_face = preprocess_input(rgb_face, False)
rgb_face = np.expand_dims(rgb_face, 0)
gender_prediction = gender_classifier.predict(rgb_face)
gender_label_arg = np.argmax(gender_prediction)
gender_text = gender_labels[gender_label_arg]
gray_face = preprocess_input(gray_face, True)
gray_face = np.expand_dims(gray_face, 0)
gray_face = np.expand_dims(gray_face, -1)
emotion_label_arg = np.argmax(emotion_classifier.predict(gray_face))
emotion_text = emotion_labels[emotion_label_arg]
emotions[emotion_text] += 1
if gender_text == gender_labels[0]:
color = (0, 0, 255)
else:
color = (255, 0, 0)
draw_bounding_box(face_coordinates, rgb_image, color)
draw_text(face_coordinates, rgb_image, gender_text, color, 0, -20, 1, 2)
draw_text(face_coordinates, rgb_image, emotion_text, color, 0, -50, 1, 2)
max_num = 0
max_emotion = None
for key, value in emotions.items():
if value > max_num:
max_num = value
max_emotion = key
print("The emotion of this picture is: ", max_emotion)
bgr_image = cv2.cvtColor(rgb_image, cv2.COLOR_RGB2BGR)
cv2.imwrite('./result_images/predicted_test_image.png', bgr_image)
K.clear_session()
return max_emotion
