def process_image(image): try: # parameters for loading data and images detection_model_path = './trained_models/detection_models/haarcascade_frontalface_default.xml' emotion_model_path = './trained_models/emotion_models/fer2013_35a93d1a30cf52b8722837f3e096b7b1006949325de8d00f26595d5a418241d3.hdf5' emotion_labels = get_labels('fer2013') font = cv2.FONT_HERSHEY_SIMPLEX # hyper-parameters for bounding boxes shape emotion_offsets = (20, 40) emotion_offsets = (0, 0) # loading models face_detection = load_detection_model(detection_model_path) emotion_classifier = load_model(emotion_model_path, compile=False) # getting input model shapes for inference emotion_target_size = emotion_classifier.input_shape[1:3] # loading images image_array = np.fromstring(image, np.uint8) unchanged_image = cv2.imdecode(image_array, cv2.IMREAD_UNCHANGED) rgb_image = cv2.cvtColor(unchanged_image, cv2.COLOR_BGR2RGB) gray_image = cv2.cvtColor(unchanged_image, cv2.COLOR_BGR2GRAY) faces = detect_faces(face_detection, gray_image) for face_coordinates in faces: x1, x2, y1, y2 = apply_offsets(face_coordinates, emotion_offsets) gray_face = gray_image[y1:y2, x1:x2] try: gray_face = cv2.resize(gray_face, (emotion_target_size)) except: continue rgb_face = preprocess_input(rgb_face, False) rgb_face = np.expand_dims(rgb_face, 0) gray_face = preprocess_input(gray_face, True) gray_face = np.expand_dims(gray_face, 0) gray_face = np.expand_dims(gray_face, -1) emotion_label_arg = np.argmax(emotion_classifier.predict(gray_face)) emotion_text = emotion_labels[emotion_label_arg] color = (255, 0, 0) draw_bounding_box(face_coordinates, rgb_image, color) draw_text(face_coordinates, rgb_image, emotion_text, color, 0, -50, 1, 2) except Exception as err: logging.error('Error in emotion gender processor: "{0}"'.format(err)) bgr_image = cv2.cvtColor(rgb_image, cv2.COLOR_RGB2BGR) dirname = 'result' if not os.path.exists(dirname): os.mkdir(dirname) cv2.imwrite(os.path.join(dirname, 'predicted_image.png'), bgr_image)
def save_biggest_emotion_images_cut(video_path, save_path, detector, frame_interval_ms, start_ms, end_ms): video_capture = cv2.VideoCapture(video_path) if not video_capture.isOpened(): return fps = video_capture.get(cv2.CAP_PROP_FPS) start_frame_no = int(start_ms / 1000 * fps + 1) end_frame_no = int(end_ms / 1000 * fps) frame_count = video_capture.get(cv2.CAP_PROP_FRAME_COUNT) if start_frame_no < 0 or end_frame_no > frame_count: return # 创建视频存储文件夹 if not os.path.exists(save_path): os.mkdir(save_path) frame_no = 1 # 抽取帧的序号 interval_frame_num = int(frame_interval_ms / 1000 * fps) # 间隔帧数 if interval_frame_num < 1: interval_frame_num = 1 # 防止帧间隔为0的情况 while frame_no <= end_frame_no: video_capture.set(cv2.CAP_PROP_POS_FRAMES, frame_no) frame = video_capture.read()[1] if frame is None or np.size(frame) is 0: frame_no += interval_frame_num continue time = int(video_capture.get(cv2.CAP_PROP_POS_MSEC)) prediction, coord = detector.detect_biggest(frame) if prediction is None: frame_no += interval_frame_num continue emotion_probability = np.max(prediction) frame_no += interval_frame_num emotion_label_arg = np.argmax(prediction) emotion_text = detector.labels[emotion_label_arg] # 根据情绪选择颜色 if emotion_text == 'angry': color = emotion_probability * np.asarray((255, 0, 0)) elif emotion_text == 'sad': color = emotion_probability * np.asarray((0, 0, 255)) elif emotion_text == 'happy': color = emotion_probability * np.asarray((255, 255, 0)) elif emotion_text == 'surprise': color = emotion_probability * np.asarray((0, 255, 255)) else: color = emotion_probability * np.asarray((0, 255, 0)) color = color.astype(int) color = color.tolist() text = emotion_text + ' ' + str(time / 1000) + 's' cv2.rectangle(frame, (coord[0], coord[2]), (coord[1], coord[3]), color, 2) draw_text(coord, frame, text, color, 0, -45, 1, 1) image_path = save_path + '/' + str(frame_no) + '.png' cv2.imwrite(image_path, frame) video_capture.release() return
def recognition(f_2_s): # parameters for loading data and images #image_path = path_r image_path = image_handler(f_2_s) detection_model_path = '../trained_models/detection_models/haarcascade_frontalface_default.xml' emotion_model_path = '../trained_models/emotion_models/fer2013_mini_XCEPTION.102-0.66.hdf5' emotion_labels = get_labels('fer2013') font = cv2.FONT_HERSHEY_SIMPLEX # hyper-parameters for bounding boxes shape emotion_offsets = (20, 40) emotion_offsets = (0, 0) # loading models face_detection = load_detection_model(detection_model_path) emotion_classifier = load_model(emotion_model_path, compile=False) # getting input model shapes for inference emotion_target_size = emotion_classifier.input_shape[1:3] # loading images rgb_image = load_image(image_path, grayscale=False) gray_image = load_image(image_path, grayscale=True) gray_image = np.squeeze(gray_image) gray_image = gray_image.astype('uint8') faces = detect_faces(face_detection, gray_image) for face_coordinates in faces: x1, x2, y1, y2 = apply_offsets(face_coordinates, emotion_offsets) gray_face = gray_image[y1:y2, x1:x2] try: gray_face = cv2.resize(gray_face, (emotion_target_size)) except: continue gray_face = preprocess_input(gray_face, True) gray_face = np.expand_dims(gray_face, 0) gray_face = np.expand_dims(gray_face, -1) emotion_label_arg = np.argmax(emotion_classifier.predict(gray_face)) emotion_text = "" emotion_text = emotion_labels[emotion_label_arg] color = (255, 0, 0) # 감정 정보 글씨 빨간색, 사각형도 draw_bounding_box(face_coordinates, rgb_image, color) draw_text(face_coordinates, rgb_image, emotion_text, color, 0, -30, 1.5, 2) #if(emotion_text == ""): #recognition(f_2_s) bgr_image = cv2.cvtColor(rgb_image, cv2.COLOR_RGB2BGR) cv2.imwrite('../images/predicted_test_image.png', bgr_image) # 변수 활용 check_recoged_img('../images/predicted_test_image.png')
def final_ml_predict(bgr_image): gray_image = cv2.cvtColor(bgr_image, cv2.COLOR_BGR2GRAY) rgb_image = cv2.cvtColor(bgr_image, cv2.COLOR_BGR2RGB) faces = face_cascade.detectMultiScale(gray_image, scaleFactor=1.1, minNeighbors=5, minSize=(30, 30), flags=cv2.CASCADE_SCALE_IMAGE) for face_coordinates in faces: x1, x2, y1, y2 = apply_offsets(face_coordinates, emotion_offsets) gray_face = gray_image[y1:y2, x1:x2] try: gray_face = cv2.resize(gray_face, (emotion_target_size)) except: continue gray_face = preprocess_input(gray_face, True) gray_face = np.expand_dims(gray_face, 0) gray_face = np.expand_dims(gray_face, -1) with graph.as_default(): emotion_prediction = emotion_classifier.predict(gray_face) emotion_probability = np.max(emotion_prediction) emotion_label_arg = np.argmax(emotion_prediction) emotion_text = emotion_labels[emotion_label_arg] emotion_window.append(emotion_text) if len(emotion_window) > frame_window: emotion_window.pop(0) try: emotion_mode = mode(emotion_window) except: continue if emotion_text == 'angry': color = emotion_probability * np.asarray((255, 0, 0)) elif emotion_text == 'sad': color = emotion_probability * np.asarray((0, 0, 255)) elif emotion_text == 'happy': color = emotion_probability * np.asarray((255, 255, 0)) elif emotion_text == 'surprise': color = emotion_probability * np.asarray((0, 255, 255)) else: color = emotion_probability * np.asarray((0, 255, 0)) color = color.astype(int) color = color.tolist() draw_bounding_box(face_coordinates, rgb_image, color) draw_text(face_coordinates, rgb_image, emotion_mode, color, 0, -45, 1, 1) bgr_image = cv2.cvtColor(rgb_image, cv2.COLOR_RGB2BGR) return bgr_image
def classify_frame(bgr_image): # bgr_image = video_capture.read()[1] gray_image = cv2.cvtColor(bgr_image, cv2.COLOR_BGR2GRAY) rgb = cv2.cvtColor(bgr_image, cv2.COLOR_BGR2RGB) max_size = 512 aspect_ratio = rgb.shape[1] / rgb.shape[0] scaling_factor = max([rgb.shape[1], rgb.shape[0]]) / max_size new_width = int(rgb.shape[1] / scaling_factor) new_height = int(rgb.shape[0] / scaling_factor) rgb_image = cv2.resize(rgb, (new_width, new_height)) faces = detect_faces(face_detection, rgb_image) for face_coordinates in faces: x1, x2, y1, y2 = apply_offsets(face_coordinates, gender_offsets) rgb_face = rgb_image[y1:y2, x1:x2] gray_face = gray_image[y1:y2, x1:x2] try: rgb_face = cv2.resize(rgb_face, (gender_target_size)) except: continue gray_face = preprocess_input(gray_face, False) gray_face = np.expand_dims(gray_face, 0) gray_face = np.expand_dims(gray_face, -1) rgb_face = np.expand_dims(rgb_face, 0) rgb_face = preprocess_input(rgb_face, False) gender_prediction = gender_classifier.predict(rgb_face) gender_label_arg = np.argmax(gender_prediction) gender_text = gender_labels[gender_label_arg] gender_window.append(gender_text) if len(gender_window) > frame_window: gender_window.pop(0) try: gender_mode = mode(gender_window) except: continue if gender_text == gender_labels[0]: color = (0, 0, 255) else: color = (255, 0, 0) draw_bounding_box(face_coordinates, rgb_image, color) draw_text(face_coordinates, rgb_image, gender_mode, color, 0, -20, 1, 1) bgr_image = cv2.cvtColor(rgb_image, cv2.COLOR_RGB2BGR) cv2.imshow('window_frame', bgr_image)
def detect_emotion(frame,model,emotion_classifier,emotion_target_size,face_cascade,frame_window,emotion_offsets,emotion_labels): gray_image = cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY) faces = face_cascade.detectMultiScale(gray_image, scaleFactor=1.1, minNeighbors=5,minSize=(30, 30), flags=cv2.CASCADE_SCALE_IMAGE) for face_coordinates in faces: x1, x2, y1, y2 = apply_offsets(face_coordinates, emotion_offsets) # gray_face = gray_image[y1:y2, x1:x2] # gray_c = cv2.cvtColor(gray_image[y1:y2, x1:x2], cv2.COLOR_BGR2GRAY) gray_c = gray_image[y1:y2, x1:x2] try: gray_face = cv2.resize(gray_c, (emotion_target_size)) except: continue gray_face = preprocess_input(gray_face, True) gray_face = np.expand_dims(gray_face, 0) gray_face = np.expand_dims(gray_face, -1) # print(gray_face) emotion_prediction = emotion_classifier.predict(gray_face) emotion_probability =round( np.max(emotion_prediction),2) emotion_label_arg = np.argmax(emotion_prediction) emotion_text = emotion_labels[emotion_label_arg] emotion_window.append(emotion_text) # print(emotion_text) # print(emotion_window) if len(emotion_window) > frame_window: emotion_window.pop(0) try: emotion_mode = mode(emotion_window) except: continue # if emotion_text == 'angry': # color = emotion_probability * np.asarray((255, 0, 0)) # elif emotion_text == 'sad': # color = emotion_probability * np.asarray((0, 0, 255)) # elif emotion_text == 'happy': # # print (emotion_probability) # color = emotion_probability * np.asarray((255, 255, 0)) # elif emotion_text == 'surprise': # color = emotion_probability * np.asarray((0, 255, 255)) # else: # color = emotion_probability * np.asarray((0, 255, 0)) # color = color.astype(int) # color = color.tolist() color=(255,255,255) draw_text(emotion_probability,face_coordinates, frame, emotion_mode,color, 0, -20, 0.5, 1)
def run_classify(image_path): emotion_text = '' gender_text = '' # loading images rgb_image = load_image(image_path, grayscale=False) gray_image = load_image(image_path, grayscale=True) gray_image = np.squeeze(gray_image) gray_image = gray_image.astype('uint8') faces = detect_faces(face_detection, gray_image) for face_coordinates in faces: x1, x2, y1, y2 = apply_offsets(face_coordinates, gender_offsets) rgb_face = rgb_image[y1:y2, x1:x2] x1, x2, y1, y2 = apply_offsets(face_coordinates, emotion_offsets) gray_face = gray_image[y1:y2, x1:x2] try: rgb_face = cv2.resize(rgb_face, (gender_target_size)) gray_face = cv2.resize(gray_face, (emotion_target_size)) except: continue rgb_face = preprocess_input(rgb_face, False) rgb_face = np.expand_dims(rgb_face, 0) gender_prediction = gender_classifier.predict(rgb_face) gender_label_arg = np.argmax(gender_prediction) gender_text = gender_labels[gender_label_arg] gray_face = preprocess_input(gray_face, True) gray_face = np.expand_dims(gray_face, 0) gray_face = np.expand_dims(gray_face, -1) emotion_label_arg = np.argmax(emotion_classifier.predict(gray_face)) emotion_text = emotion_labels[emotion_label_arg] if gender_text == gender_labels[0]: color = (0, 0, 255) else: color = (255, 0, 0) draw_bounding_box(face_coordinates, rgb_image, color) draw_text(face_coordinates, rgb_image, gender_text, color, 0, -20, 1, 2) draw_text(face_coordinates, rgb_image, emotion_text, color, 0, -50, 1, 2) bgr_image = cv2.cvtColor(rgb_image, cv2.COLOR_RGB2BGR) cv2.imwrite(os.path.join(dir, '../images/predicted_test_image.png'), bgr_image) return ([gender_text, emotion_text])
def emo_Detect(self, bgr_image, face_coordinates): gray_image = cv2.cvtColor(bgr_image, cv2.COLOR_BGR2GRAY) x1, y1, x2, y2 = face_coordinates face_coordinates = [int(x1), int(y1), int(x2 - x1), int(y2 - y1)] x1, x2, y1, y2 = apply_offsets(face_coordinates, self.emotion_offsets) gray_face = gray_image[y1:y2, x1:x2] gray_face = cv2.resize(gray_face, (self.emotion_target_size)) gray_face = preprocess_input(gray_face, True) gray_face = np.expand_dims(gray_face, 0) gray_face = np.expand_dims(gray_face, -1) emotion_prediction = self.emotion_classifier.predict(gray_face) for idx, probability in enumerate(emotion_prediction[0]): print(self.emotion_labels[idx], probability) emotion_probability = np.max(emotion_prediction) emotion_label_arg = np.argmax(emotion_prediction) emotion_text = self.emotion_labels[emotion_label_arg] self.emotion_window.append(emotion_text) if len(self.emotion_window) > self.frame_window: self.emotion_window.pop(0) try: emotion_mode = mode(self.emotion_window) print("mode is " + emotion_mode) except: pass if emotion_text == 'angry': color = emotion_probability * np.asarray((0, 0, 255)) elif emotion_text == 'sad': color = emotion_probability * np.asarray((255, 0, 0)) elif emotion_text == 'happy': color = emotion_probability * np.asarray((0, 255, 255)) elif emotion_text == 'surprise': color = emotion_probability * np.asarray((255, 255, 0)) else: color = emotion_probability * np.asarray((0, 255, 0)) color = color.astype(int) color = color.tolist() draw_bounding_box(face_coordinates, bgr_image, color) draw_text(face_coordinates, bgr_image, emotion_text, color, 0, -45, 1, 1) return bgr_image, emotion_prediction[0]
def getFaceEmotion(image_path): # parameters for loading data and images base = 'C://Users/lenovo/Desktop/moodify/components/emotion-classification' detection_model_path = base + '/trained_models/detection_models/haarcascade_frontalface_default.xml' emotion_model_path = base + '/trained_models/emotion_models/fer2013_denseNet.59-0.68.hdf5' emotion_labels = get_labels('fer2013') font = cv2.FONT_HERSHEY_SIMPLEX # hyper-parameters for bounding boxes shape emotion_offsets = (0, 0) # loading models face_detection = load_detection_model(detection_model_path) emotion_classifier = load_model(emotion_model_path, compile=False) # getting input model shapes for inference emotion_target_size = emotion_classifier.input_shape[1:3] # loading images rgb_image = load_image(image_path, grayscale=False) gray_image = load_image(image_path, grayscale=True) gray_image = np.squeeze(gray_image) gray_image = gray_image.astype('uint8') faces = detect_faces(face_detection, gray_image) for face_coordinates in faces: x1, x2, y1, y2 = apply_offsets(face_coordinates, emotion_offsets) gray_face = gray_image[y1:y2, x1:x2] try: gray_face = cv2.resize(gray_face, (emotion_target_size)) except: continue gray_face = preprocess_input(gray_face, True) gray_face = np.expand_dims(gray_face, 0) gray_face = np.expand_dims(gray_face, -1) emotion_label_arg = np.argmax(emotion_classifier.predict(gray_face)) emotion_text = emotion_labels[emotion_label_arg] color = (255, 0, 0) draw_bounding_box(face_coordinates, rgb_image, color) draw_text(face_coordinates, rgb_image, emotion_text, color, 0, -10, 0.5, 2) bgr_image = cv2.cvtColor(rgb_image, cv2.COLOR_RGB2BGR) cv2.imwrite('predicted_test_image.png', bgr_image) return emotion_text
def predict(image_path): # loading images start = time.time() rgb_image = load_image(image_path, grayscale=False) gray_image = load_image(image_path, grayscale=True) gray_image = np.squeeze(gray_image) gray_image = gray_image.astype('uint8') with graph.as_default(): faces = detect_faces(face_detection, gray_image) for face_coordinates in faces: x1, x2, y1, y2 = apply_offsets(face_coordinates, gender_offsets) rgb_face = rgb_image[y1:y2, x1:x2] x1, x2, y1, y2 = apply_offsets(face_coordinates, emotion_offsets) gray_face = gray_image[y1:y2, x1:x2] try: rgb_face = cv2.resize(rgb_face, (gender_target_size)) gray_face = cv2.resize(gray_face, (emotion_target_size)) except: continue rgb_face = preprocess_input(rgb_face, False) rgb_face = np.expand_dims(rgb_face, 0) gender_prediction = gender_classifier.predict(rgb_face) gender_label_arg = np.argmax(gender_prediction) gender_text = gender_labels[gender_label_arg] gray_face = preprocess_input(gray_face, True) gray_face = np.expand_dims(gray_face, 0) gray_face = np.expand_dims(gray_face, -1) emotion_label_arg = np.argmax(emotion_classifier.predict(gray_face)) emotion_text = emotion_labels[emotion_label_arg] if gender_text == gender_labels[0]: color = (0, 0, 255) else: color = (255, 0, 0) draw_bounding_box(face_coordinates, rgb_image, color) draw_text(face_coordinates, rgb_image, gender_text, color, 0, -10, 1, 2) draw_text(face_coordinates, rgb_image, emotion_text, color, 0, -40, 1, 2) bgr_image = cv2.cvtColor(rgb_image, cv2.COLOR_RGB2BGR) resized_image = cv2.resize(bgr_image, (1024, 769)) file_path = 'result/' + image_path.split('/')[-1] cv2.imwrite(file_path, resized_image) end = time.time() logging.info('[Time] for prediction(): {}'.format(end - start)) return file_path
def on_data(data): f = open('current.jpg', 'wb') f.write(base64.decodebytes(data)) f.close() image_path = "current.jpg" out = [] # loading images rgb_image = load_image(image_path, grayscale=False) gray_image = load_image(image_path, grayscale=True) gray_image = np.squeeze(gray_image) gray_image = gray_image.astype('uint8') faces = detect_faces(face_detection, gray_image) for face_coordinates in faces: x1, x2, y1, y2 = apply_offsets(face_coordinates, emotion_offsets) gray_face = gray_image[y1:y2, x1:x2] try: gray_face = cv2.resize(gray_face, (emotion_target_size)) except: continue gray_face = preprocess_input(gray_face, True) gray_face = np.expand_dims(gray_face, 0) gray_face = np.expand_dims(gray_face, -1) emotion_label_arg = np.argmax(emotion_classifier.predict(gray_face)) emotion_text = emotion_labels[emotion_label_arg] color = (0, 0, 255) draw_bounding_box(face_coordinates, rgb_image, color) draw_text(face_coordinates, rgb_image, emotion_text, color, 0, -50, 1, 2) bgr_image = cv2.cvtColor(rgb_image, cv2.COLOR_RGB2BGR) cv2.imwrite('predicted.png', bgr_image) data = open('predicted.png', 'rb').read() encoded = base64.encodebytes(data).decode('utf-8') out.append({ 'image': encoded, 'emotion': emotion_text, }) return out
emotion_mode = mode(emotion_window) except: continue if emotion_text == 'angry': color = emotion_probability * np.asarray((255, 0, 0)) elif emotion_text == 'sad': color = emotion_probability * np.asarray((0, 0, 255)) elif emotion_text == 'happy': color = emotion_probability * np.asarray((255, 255, 0)) elif emotion_text == 'surprise': color = emotion_probability * np.asarray((0, 255, 255)) else: color = emotion_probability * np.asarray((0, 255, 0)) color = color.astype(int) color = color.tolist() draw_bounding_box(face_utils.rect_to_bb(face_coordinates), rgb_image, color) draw_text(face_utils.rect_to_bb(face_coordinates), rgb_image, emotion_mode, color, 0, -45, 1, 1) frame = cv2.cvtColor(rgb_image, cv2.COLOR_RGB2BGR) cv2.imshow('window_frame', frame) if cv2.waitKey(1) & 0xFF == ord('q'): break cap.release() cv2.destroyAllWindows()
def r8k9(fileName='dog.jpg', score=10, length=10): # getting input model shapes for inference emotion_target_size = emotion_classifier.input_shape[1:3] # starting lists for calculating modes emotion_window = [] # starting video streaming cv2.namedWindow('window_frame') video_capture = cv2.VideoCapture(0) im = cv2.imread(fileName) im_resized = cv2.resize(im, (224, 224), interpolation=cv2.INTER_LINEAR) countdown = ["Ready?", "3", "2", "1"] #plt.imshow(cv2.cvtColor(im_resized, cv2.COLOR_BGR2RGB)) #for i in range(4): # print(countdown[i]) # time.sleep(1) #print("GO") #plt.show(block=False) t = time.time() td = [] while time.time() < t + length: bgr_image = video_capture.read()[1] try: gray_image = cv2.cvtColor(bgr_image, cv2.COLOR_BGR2GRAY) except: time.sleep(.5) gray_image = cv2.cvtColor(bgr_image, cv2.COLOR_BGR2GRAY) rgb_image = cv2.cvtColor(bgr_image, cv2.COLOR_BGR2RGB) faces = detect_faces(face_detection, gray_image) for face_coordinates in faces: x1, x2, y1, y2 = apply_offsets(face_coordinates, emotion_offsets) gray_face = gray_image[y1:y2, x1:x2] try: gray_face = cv2.resize(gray_face, (emotion_target_size)) except: continue gray_face = preprocess_input(gray_face, True) gray_face = np.expand_dims(gray_face, 0) gray_face = np.expand_dims(gray_face, -1) emotion_prediction = emotion_classifier.predict(gray_face) #print(emotion_prediction) if len(td) == 0: data = emotion_prediction td = [time.time() - t] else: data = np.concatenate((data, emotion_prediction), axis=0) td = td + [time.time() - t] emotion_probability = np.max(emotion_prediction) emotion_label_arg = np.argmax(emotion_prediction) emotion_text = emotion_labels[emotion_label_arg] emotion_window.append(emotion_text) if len(emotion_window) > frame_window: emotion_window.pop(0) try: emotion_mode = mode(emotion_window) except: continue if emotion_text == 'angry': color = emotion_probability * np.asarray((255, 0, 0)) elif emotion_text == 'sad': color = emotion_probability * np.asarray((0, 0, 255)) elif emotion_text == 'happy': color = emotion_probability * np.asarray((255, 255, 0)) elif emotion_text == 'surprise': color = emotion_probability * np.asarray((0, 255, 255)) else: color = emotion_probability * np.asarray((0, 255, 0)) color = color.astype(int) color = color.tolist() draw_bounding_box(face_coordinates, rgb_image, color) draw_text(face_coordinates, rgb_image, emotion_mode, color, 0, -45, 1, 1) bgr_image = cv2.cvtColor(rgb_image, cv2.COLOR_RGB2BGR) cv2.imshow('window_frame', bgr_image) if cv2.waitKey(1) & 0xFF == ord('q'): break #print(np.transpose(data)) #print(emotion_labels) plt.clf() plt.plot(td, data) plt.legend( ['angry', 'disgust', 'fear', 'happy', 'sad', 'surprise', 'neutral'], loc=2) plt.savefig('Graph.png') #splt.show() totals = [0, 0, 0, 0, 0, 0, 0] for j in range(7): for i in range(1, len(td)): #print(time[i-1]) #print(data[i-1+l*(j+1)]) #print(time[i],time[i-1]) totals[j] += (data[i][j] + data[i - 1][j]) / 2 * ( td[i] - td[i - 1]) / (td[-1] - td[0]) np.append(totals, np.std(data)) print(totals) #print(td[-1] - td[0]) #print(sum(totals)) #f = open("DogData.txt","a+") #f.write((str(score) + ' ' + ' '.join(str(t) for t in totals) + "\n")) print("Rating:", 15 - spot(np.array(totals)) / 10, "/10") return (15 - spot(np.array(totals)) / 10)
def get_sim_percent(image_1, image_2, target_class): # parameters for loading data and images # image_path = sys.argv[1] detection_model_path = 'trained_models/detection_models/haarcascade_frontalface_default.xml' emotion_model_path = 'trained_models/fer2013_big_XCEPTION.54-0.66.hdf5' gender_model_path = 'trained_models/gender_models/simple_CNN.81-0.96.hdf5' emotion_labels = get_labels('fer2013') gender_labels = get_labels('imdb') font = cv2.FONT_HERSHEY_SIMPLEX # hyper-parameters for bounding boxes shape gender_offsets = (20, 40) gender_offsets = (10, 10) emotion_offsets = (10, 30) emotion_offsets = (0, 0) # loading models face_detection = load_detection_model(detection_model_path) emotion_classifier = load_model(emotion_model_path, compile=False) gender_classifier = load_model(gender_model_path, compile=False) # getting input model shapes for inference emotion_target_size = emotion_classifier.input_shape[1:3] gender_target_size = gender_classifier.input_shape[1:3] target_emo = target_class image_path = image_1 # loading images rgb_image = load_image(image_path, grayscale=False) gray_image = load_image(image_path, grayscale=True) gray_image = np.squeeze(gray_image) gray_image = gray_image.astype('uint8') faces = detect_faces(face_detection, gray_image) for face_coordinates in faces: x1, x2, y1, y2 = apply_offsets(face_coordinates, gender_offsets) rgb_face = rgb_image[y1:y2, x1:x2] x1, x2, y1, y2 = apply_offsets(face_coordinates, emotion_offsets) gray_face = gray_image[y1:y2, x1:x2] try: rgb_face = cv2.resize(rgb_face, (gender_target_size)) gray_face = cv2.resize(gray_face, (emotion_target_size)) except: continue rgb_face = preprocess_input(rgb_face, False) rgb_face = np.expand_dims(rgb_face, 0) gender_prediction = gender_classifier.predict(rgb_face) gender_label_arg = np.argmax(gender_prediction) gender_text = gender_labels[gender_label_arg] gray_face = preprocess_input(gray_face, True) gray_face = np.expand_dims(gray_face, 0) gray_face = np.expand_dims(gray_face, -1) emotion_label_arg = emotion_classifier.predict(gray_face)[0][target_emo] emotion_percent = str(round(emotion_label_arg*100, 2)) emotion_text = emotion_labels[target_emo] + ' - '+ emotion_percent # print(target_emo) # print(emotion_text) if gender_text == gender_labels[0]: color = (0, 0, 255) else: color = (255, 0, 0) draw_bounding_box(face_coordinates, rgb_image, color) draw_text(face_coordinates, rgb_image, gender_text, color, -20, 30, 1, 2) draw_text(face_coordinates, rgb_image, emotion_text, color, 0, 0, 1, 2) bgr_image = cv2.cvtColor(rgb_image, cv2.COLOR_RGB2BGR) cv2.imwrite('predicted_test_image.png', bgr_image) return emotion_percent
rgb_face = np.expand_dims(rgb_face, 0) gender_prediciton = gender_classifier.predict(rgb_face) gender_label_arg = np.argmax(gender_prediciton) gender_text = gender_labels[gender_label_arg] gray_face = preprocess_input(gray_face, True) gray_face = np.expand_dims(gray_face, 0) gray_face = np.expand_dims(gray_face, -1) emotion_label_arg = np.argmax(emotion_classifier.predict(gray_face)) emotion_text = emotion_labels[emotion_label_arg] if gender_text == gender_labels[0]: color = (0, 0, 255) else: color = (255, 0, 0) draw_bounding_box(face_coordinates, rgb_image, color) draw_text(face_coordinates, rgb_image, gender_text, color, 0, -20, 1, 2) draw_text(face_coordinates, rgb_image, emotion_text, color, 0, -50, 1, 2) end = time.time() print('Used {} s'.format((end - start))) bgr_image = cv2.cvtColor(rgb_image, cv2.COLOR_RGB2BGR) cv2.imwrite('../images/test.png', bgr_image) img = cv2.imread('../images/test.png') cv2.namedWindow('predicted_img', cv2.WINDOW_NORMAL) cv2.resizeWindow('predicted_img', 880, 680) cv2.imshow('predicted_img', img) cv2.waitKey(0) cv2.destroyAllWindows()
elif emotion_text == 'sad': color = emotion_probability * np.asarray((0, 0, 255)) elif emotion_text == 'happy': color = emotion_probability * np.asarray((255, 255, 0)) elif emotion_text == 'surprise': color = emotion_probability * np.asarray((0, 255, 255)) else: color = emotion_probability * np.asarray((0, 255, 0)) color = color.astype(int) color = color.tolist() if fname == "Unknown": name = emotion_text else: name = str(fname) + " is " + str(emotion_text) draw_bounding_box(face_utils.rect_to_bb(face_coordinates), rgb_image, color) draw_text(face_utils.rect_to_bb(face_coordinates), rgb_image, name, color, 0, -45, 0.5, 1) frame = cv2.cvtColor(rgb_image, cv2.COLOR_RGB2BGR) cv2.imshow('window_frame', frame) if cv2.waitKey(1) & 0xFF == ord('q'): break cap.release() cv2.destroyAllWindows()
def extract_emotions(self, bgr_image): base_path = os.path.realpath(__file__) base_path = base_path[:base_path.find('emotions')] emotion_labels = get_labels('fer2013') # hyper-parameters for bounding boxes shape frame_window = 10 emotion_offsets = (20, 40) # loading models if self.emotion_classifier == None: self.emotion_classifier = load_model(self.emotion_model_path) # getting input model shapes for inference emotion_target_size = self.emotion_classifier.input_shape[1:3] # starting lists for calculating modes emotion_window = [] gray_image = cv2.cvtColor(bgr_image, cv2.COLOR_BGR2GRAY) rgb_image = cv2.cvtColor(bgr_image, cv2.COLOR_BGR2RGB) faces = self.face_cascade.detectMultiScale( gray_image, scaleFactor=1.1, minNeighbors=5, minSize=(30, 30), flags=cv2.CASCADE_SCALE_IMAGE) for face_coordinates in faces: x1, x2, y1, y2 = apply_offsets(face_coordinates, emotion_offsets) gray_face = gray_image[y1:y2, x1:x2] try: gray_face = cv2.resize(gray_face, (emotion_target_size)) except: continue gray_face = preprocess_input(gray_face, True) gray_face = np.expand_dims(gray_face, 0) gray_face = np.expand_dims(gray_face, -1) emotion_prediction = self.emotion_classifier.predict(gray_face) emotion_probability = np.max(emotion_prediction) emotion_label_arg = np.argmax(emotion_prediction) emotion_text = emotion_labels[emotion_label_arg] emotion_window.append(emotion_text) if len(emotion_window) > frame_window: emotion_window.pop(0) try: emotion_mode = mode(emotion_window) except: continue if emotion_text == 'angry': color = emotion_probability * np.asarray((255, 0, 0)) elif emotion_text == 'sad': color = emotion_probability * np.asarray((0, 0, 255)) elif emotion_text == 'happy': color = emotion_probability * np.asarray((255, 255, 0)) elif emotion_text == 'surprise': color = emotion_probability * np.asarray((0, 255, 255)) elif emotion_text == 'fear': color = emotion_probability * np.asarray((255, 0, 255)) elif emotion_text == 'disgust': color = emotion_probability * np.asarray((255, 255, 255)) else: color = emotion_probability * np.asarray((50, 200, 100)) color = color.astype(int) color = color.tolist() draw_bounding_box(face_coordinates, rgb_image, color) draw_text(face_coordinates, rgb_image, emotion_mode, color, 0, -45, 1, 1) bgr_image = cv2.cvtColor(rgb_image, cv2.COLOR_RGB2BGR) cv2.imshow('window_frame', bgr_image) if cv2.waitKey(1) & 0xFF == ord('q'): break
continue rgb_face = preprocess_input(rgb_face, False) rgb_face = np.expand_dims(rgb_face, 0) gender_prediction = gender_classifier.predict(rgb_face) gender_label_arg = np.argmax(gender_prediction) gender_text = gender_labels[gender_label_arg] gray_face = preprocess_input(gray_face, True) gray_face = np.expand_dims(gray_face, 0) gray_face = np.expand_dims(gray_face, -1) emotion_label_arg = np.argmax(emotion_classifier.predict(gray_face)) emotion_text = emotion_labels[emotion_label_arg] if gender_text == gender_labels[0]: color = (0, 0, 255) else: color = (255, 0, 0) draw_bounding_box(face_coordinates, rgb_image, color) # draw_text(face_coordinates, rgb_image, gender_text, color, 0, -20, 1, 2) draw_text(face_coordinates, rgb_image, emotion_text, color, 0, 20, 1, 2) bgr_image = cv2.cvtColor(rgb_image, cv2.COLOR_RGB2BGR) filename = image_path[num].split('/')[-1] filename = filename.split('.')[0] newname = '../predicted_image/' + 'predicted_' + filename + '.png' cv2.imwrite(newname, bgr_image)
def process_image(image): try: # parameters for loading data and images detection_model_path = './trained_models/detection_models/haarcascade_frontalface_default.xml' emotion_model_path = './trained_models/emotion_models/fer2013_mini_XCEPTION.102-0.66.hdf5' gender_model_path = './trained_models/gender_models/simple_CNN.81-0.96.hdf5' emotion_labels = get_labels('fer2013') gender_labels = get_labels('imdb') # hyper-parameters for bounding boxes shape gender_offsets = (10, 10) emotion_offsets = (0, 0) # loading models face_detection = load_detection_model(detection_model_path) emotion_classifier = load_model(emotion_model_path, compile=False) gender_classifier = load_model(gender_model_path, compile=False) # getting input model shapes for inference emotion_target_size = emotion_classifier.input_shape[1:3] gender_target_size = gender_classifier.input_shape[1:3] # loading images image_array = np.fromstring(image, np.uint8) unchanged_image = cv2.imdecode(image_array, cv2.IMREAD_UNCHANGED) rgb_image = cv2.cvtColor(unchanged_image, cv2.COLOR_BGR2RGB) gray_image = cv2.cvtColor(unchanged_image, cv2.COLOR_BGR2GRAY) faces = detect_faces(face_detection, gray_image) for face_coordinates in faces: x1, x2, y1, y2 = apply_offsets(face_coordinates, gender_offsets) rgb_face = rgb_image[y1:y2, x1:x2] x1, x2, y1, y2 = apply_offsets(face_coordinates, emotion_offsets) gray_face = gray_image[y1:y2, x1:x2] try: rgb_face = cv2.resize(rgb_face, gender_target_size) gray_face = cv2.resize(gray_face, emotion_target_size) except: continue rgb_face = preprocess_input(rgb_face, False) rgb_face = np.expand_dims(rgb_face, 0) gender_prediction = gender_classifier.predict(rgb_face) gender_label_arg = np.argmax(gender_prediction) gender_text = gender_labels[gender_label_arg] gray_face = preprocess_input(gray_face, True) gray_face = np.expand_dims(gray_face, 0) gray_face = np.expand_dims(gray_face, -1) emotion_label_arg = np.argmax( emotion_classifier.predict(gray_face)) emotion_text = emotion_labels[emotion_label_arg] if gender_text == gender_labels[0]: color = (0, 0, 255) else: color = (255, 0, 0) draw_bounding_box(face_coordinates, rgb_image, color) draw_text(face_coordinates, rgb_image, gender_text, color, 0, -20, 1, 2) draw_text(face_coordinates, rgb_image, emotion_text, color, 0, -50, 1, 2) except Exception as err: logging.error('Error in emotion gender processor: "{0}"'.format(err)) bgr_image = cv2.cvtColor(rgb_image, cv2.COLOR_RGB2BGR) dirname = 'result' if not os.path.exists(dirname): os.mkdir(dirname) cv2.imwrite(os.path.join(dirname, 'predicted_image.png'), bgr_image)
try: rgb_face = cv2.resize(rgb_face, (gender_target_size)) gray_face = cv2.resize(gray_face, (emotion_target_size)) except: continue rgb_face = preprocess_input(rgb_face, False) rgb_face = np.expand_dims(rgb_face, 0) gender_prediction = gender_classifier.predict(rgb_face) gender_label_arg = np.argmax(gender_prediction) gender_text = gender_labels[gender_label_arg] gray_face = preprocess_input(gray_face, True) gray_face = np.expand_dims(gray_face, 0) gray_face = np.expand_dims(gray_face, -1) emotion_label_arg = np.argmax(emotion_classifier.predict(gray_face)) emotion_text = emotion_labels[emotion_label_arg] if gender_text == gender_labels[0]: color = (0, 0, 255) else: color = (255, 0, 0) draw_bounding_box(face_coordinates, rgb_image, color) draw_text(face_coordinates, rgb_image, gender_text, color, 0, -20, 1, 2) draw_text(face_coordinates, rgb_image, emotion_text, color, 0, -50, 1, 2) bgr_image = cv2.cvtColor(rgb_image, cv2.COLOR_RGB2BGR) cv2.imwrite('../images/predicted_test_image.png', bgr_image)
def callback(self,data): try: bgr_image = self.bridge.imgmsg_to_cv2(data, "bgr8") cv2.imshow("Image window", bgr_image) except CvBridgeError as e: print(e) ######################### Start the image processing gray_image = cv2.cvtColor(bgr_image, cv2.COLOR_BGR2GRAY) rgb_image = cv2.cvtColor(bgr_image, cv2.COLOR_BGR2RGB) faces = detect_faces(self.face_detection, gray_image) for face_coordinates in faces: x1, x2, y1, y2 = apply_offsets(face_coordinates, self.emotion_offsets) gray_face = gray_image[y1:y2, x1:x2] try: gray_face = cv2.resize(gray_face, (self.emotion_target_size)) except: continue gray_face = preprocess_input(gray_face, True) gray_face = np.expand_dims(gray_face, 0) gray_face = np.expand_dims(gray_face, -1) print "start" print len(gray_face) print gray_face print "end" emotion_prediction = self.emotion_classifier.predict(gray_face) emotion_probability = np.max(emotion_prediction) emotion_label_arg = np.argmax(emotion_prediction) emotion_text = emotion_labels[emotion_label_arg] print emotion_text print(emotion_probability) print('%') self.emotion_window.append(emotion_text) if len(self.emotion_window) > self.frame_window: self.emotion_window.pop(0) try: emotion_mode = mode(self.emotion_window) except: continue if emotion_text == 'angry': color = emotion_probability * np.asarray((255, 0, 0)) elif emotion_text == 'sad': color = emotion_probability * np.asarray((0, 0, 255)) elif emotion_text == 'happy': color = emotion_probability * np.asarray((255, 255, 0)) elif emotion_text == 'surprise': color = emotion_probability * np.asarray((0, 255, 255)) else: color = emotion_probability * np.asarray((0, 255, 0)) color = color.astype(int) color = color.tolist() draw_bounding_box(face_coordinates, rgb_image, color) draw_text(face_coordinates, rgb_image, emotion_mode, color, 0, -45, 1, 1) bgr_image = cv2.cvtColor(rgb_image, cv2.COLOR_RGB2BGR) ######################### End the image processing cv2.imshow("Image window", bgr_image) cv2.waitKey(3) try: self.image_pub.publish(self.bridge.cv2_to_imgmsg(bgr_image, "bgr8")) except CvBridgeError as e: print(e)
def callback(self,data): #///////////////////////////////////////////// #cv_image = bridge.imgmsg_to_cv2(image_msg, desired_encoding="passthrough") #cv_image = cv2.resize(cv_image, target_size) # resize image #np_image = np.asarray(cv_image) # read as np array #np_image = np.expand_dims(np_image, axis=0) # Add another dimension for tensorflow #np_image = np_image.astype(float) # preprocess needs float64 and img is uint8 #np_image = preprocess_input(np_image) # Regularize the data #///////////////////////////////////////////// if(not USE_LOCAL_CAMERA): try: frame = self.bridge.imgmsg_to_cv2(data, desired_encoding="passthrough") except CvBridgeError as e: print(e) # Capture frame-by-frame if(USE_LOCAL_CAMERA): ret, frame1 = self.video_capture.read() #///////////////////////////////////////////// #print"--------" #print('input_msg height : {}'.format(frame.height)) #print('input_msg width : {}'.format(frame.width)) #print('input_msg step : {}'.format(frame.step)) #print('input_msg encoding: {}'.format(frame.encoding)) #print('output dtype : {}'.format(frame.dtype)) #print('output shape : {}'.format(frame.shape)) #print"--------" gray_image = cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY) rgb_image = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB) faces = detect_faces(self.face_detection, gray_image) for face_coordinates in faces: print "inside the for loop!" print face_coordinates x1, x2, y1, y2 = apply_offsets(face_coordinates, self.emotion_offsets) gray_face = gray_image[y1:y2, x1:x2] #print len(frame) #print gray_face.size #print gray_face.shape #print gray_face.dtype try: gray_face = cv2.resize(gray_face, (self.emotion_target_size)) except: continue gray_face = preprocess_input(gray_face, True) gray_face = np.expand_dims(gray_face, 0) gray_face = np.expand_dims(gray_face, -1) #print"************" #print('gray_face dtype : {}'.format(gray_face.dtype)) #print('gray_face shape : {}'.format(gray_face.shape)) #print('gray_face size : {}'.format(gray_face.size)) #print"************" ## This is a workaround for asynchronous execution using TF and ROS # https://github.com/keras-team/keras/issues/2397 # http://projectsfromtech.blogspot.com/2017/10/visual-object-recognition-in-ros-using.html #global self.graph with self.graph.as_default(): emotion_prediction = self.emotion_classifier.predict(gray_face) emotion_probability = np.max(emotion_prediction) emotion_label_arg = np.argmax(emotion_prediction) emotion_text = self.emotion_labels[emotion_label_arg] print emotion_text print(emotion_probability) print('%') self.emotion_window.append(emotion_text) #self.emotion_msg.data = emotion_text #self.emotion_publisher.publish(emotion_msg) #self.speech_msg.data = 'I see that you are ' + emotion_text #self.speech_publisher.publish(speech_msg) if len(self.emotion_window) > self.frame_window: self.emotion_window.pop(0) try: emotion_mode = mode(self.emotion_window) except: continue if emotion_text == 'angry': color = emotion_probability * np.asarray((255, 0, 0)) elif emotion_text == 'sad': color = emotion_probability * np.asarray((0, 0, 255)) elif emotion_text == 'happy': color = emotion_probability * np.asarray((255, 255, 0)) elif emotion_text == 'surprise': color = emotion_probability * np.asarray((0, 255, 255)) else: color = emotion_probability * np.asarray((0, 255, 0)) color = color.astype(int) color = color.tolist() draw_bounding_box(face_coordinates, rgb_image, color) draw_text(face_coordinates, rgb_image, emotion_mode, color, 0, -45, 1, 1) try: self.image_pub.publish(self.bridge.cv2_to_imgmsg(rgb_image, "bgr8")) except CvBridgeError as e: print(e)
def emotion_demo(): # parameters for loading data and images detection_model_path = '../trained_models/detection_models/haarcascade_frontalface_default.xml' emotion_model_path = '../trained_models/emotion_models/fer2013_mini_XCEPTION.102-0.66.hdf5' emotion_labels = get_labels('fer2013') # 目のカスケードファイル追加 lefteyecc__path = "../trained_models/detection_models/haarcascade_lefteye_2splits.xml" righteyecc_path = "../trained_models/detection_models/haarcascade_righteye_2splits.xml" nose_path = "../trained_models/detection_models/data_haarcascades_haarcascade_mcs_nose.xml" lefteyecc = cv2.CascadeClassifier(lefteyecc__path) righteyecc = cv2.CascadeClassifier(righteyecc_path) nose = cv2.CascadeClassifier(nose_path) lex = 0; ley = 0; lew = 0; leh = 0 rex = 0; rey = 0; rew = 0; reh = 0 nox = 0; noy = 0; now = 0; noh = 0 # dlib dlib_ini() # hyper-parameters for bounding boxes shape frame_window = 10 emotion_offsets = (20, 40) # loading models face_detection = load_detection_model(detection_model_path) emotion_classifier = load_model(emotion_model_path, compile=False) # getting input model shapes for inference emotion_target_size = emotion_classifier.input_shape[1:3] # starting lists for calculating modes emotion_window = [] global img, flag, slp_count img = cv2.imread('../img/happy.png') flag = 0 slp_count = 0 # dlib用グローバル変数 global gray_image, rgb_image, gray_face, mark_list # starting video streaming cv2.namedWindow('window_frame', cv2.WINDOW_NORMAL) video_capture = cv2.VideoCapture(0) # 0は内蔵カメラ, 1はUSBカメラ while True: bgr_image = video_capture.read()[1] gray_image = cv2.cvtColor(bgr_image, cv2.COLOR_BGR2GRAY) rgb_image = cv2.cvtColor(bgr_image, cv2.COLOR_BGRA2RGBA) faces = detect_faces(face_detection, gray_image) for face_coordinates in faces: # 目や鼻認識用 (x,y,w,h) = face_coordinates video_face = gray_image[y:y+h,x:x+w] x1, x2, y1, y2 = apply_offsets(face_coordinates, emotion_offsets) gray_face = gray_image[y1:y2, x1:x2] try: gray_face = cv2.resize(gray_face, (emotion_target_size)) except: continue # ランドマーク検出 marks_list = marks_list_def(bgr_image, x, y, w, h) print(marks_list) gray_face = preprocess_input(gray_face, True) gray_face = np.expand_dims(gray_face, 0) gray_face = np.expand_dims(gray_face, -1) emotion_prediction = emotion_classifier.predict(gray_face) emotion_probability = np.max(emotion_prediction) emotion_label_arg = np.argmax(emotion_prediction) emotion_text = emotion_labels[emotion_label_arg] emotion_window.append(emotion_text) if len(emotion_window) > frame_window: emotion_window.pop(0) try: emotion_mode = mode(emotion_window) except: continue if flag == 0 or flag == 1: if emotion_text == 'angry': img = cv2.imread('../img/angry.png', -1) color = emotion_probability * np.asarray((255, 0, 0)) elif emotion_text == 'sad': img = cv2.imread('../img/sad.png', -1) # 関数にする color = emotion_probability * np.asarray((0, 0, 255)) elif emotion_text == 'happy': img = cv2.imread('../img/happy.png', -1) color = emotion_probability * np.asarray((255, 255, 0)) elif emotion_text == 'surprise': img = cv2.imread('../img/odoroki.png', -1) color = emotion_probability * np.asarray((0, 255, 255)) else : img = cv2.imread('../img/neutral.png', -1) color = emotion_probability * np.asarray((0, 255, 0)) else: if emotion_text == 'angry': img = cv2.imread('../img/ikari.png', -1) color = emotion_probability * np.asarray((255, 0, 0)) elif emotion_text == 'sad': img = cv2.imread('../img/shock.png', -1) color = emotion_probability * np.asarray((0, 0, 255)) elif emotion_text == 'happy': img = cv2.imread('../img/kirakira.png', -1) color = emotion_probability * np.asarray((255, 255, 0)) elif emotion_text == 'surprise': img = cv2.imread('../img/bikkuri.png', -1) color = emotion_probability * np.asarray((0, 255, 255)) else : img = cv2.imread('../img/toumei.png', -1) color = emotion_probability * np.asarray((0, 255, 0)) color = color.astype(int) color = color.tolist() if flag == 0: draw_bounding_box(face_coordinates, rgb_image, color) elif flag == 1: rgb_image = draw_bounding_box2(face_coordinates, rgb_image, color, img, marks_list) elif flag == 2: overlay_pic = draw_bounding_box3(face_coordinates, rgb_image, color, img, marks_list) rgb_image = overlay_pic draw_text(face_coordinates, rgb_image, emotion_mode, color, 0, -45, 1, 1) bgr_image = cv2.cvtColor(rgb_image, cv2.COLOR_RGB2BGR) if flag == 0: img = image_resize(img) cv2.imshow('image', img) cv2.destroyWindow('Window_frame') elif flag == 1 or flag == 2: cv2.destroyWindow('image') cv2.imshow('window_frame', bgr_image) cv2.waitKey(10) # cv2.imshow('own_window', bgr_image) if cv2.waitKey(1) & 0xFF == ord('z'): flag = 0 elif cv2.waitKey(1) & 0xFF == ord('x'): flag = 1 elif cv2.waitKey(1) & 0xFF == ord('c'): flag = 2 elif cv2.waitKey(1) & 0xFF == ord('q'): break video_capture.release() cv2.destroyAllWindows()
def recognize_video(self, path=None, csv=True): # starting video streaming if self.show_frame: cv2.namedWindow('window_frame') # Select video or webcam feed cap = None if path is None: cap = cv2.VideoCapture(0) # Webcam source elif isinstance(path, str): cap = cv2.VideoCapture(path) # Video file source else: raise Exception('path argument must be a string') emotions_by_frames = [ ] # container for recording emotion probabilities by frame frame_counter = 0 while cap.isOpened(): # True: try: ret, bgr_image = cap.read() if not ret: break #print('Frame {} is read'.format(frame_counter)) frame_counter += 1 rgb_image = cv2.cvtColor(bgr_image, cv2.COLOR_BGR2RGB) start = default_timer() faces = self.detector.detect_faces(rgb_image) #print('faces', faces) end = default_timer() #print('face detection:', end - start) if len(faces) == 0: emotions_by_frames.append(np.array([0] * 7)) for face_coordinates in faces: # face_coordinates: [x, y, w, h] face_coordinates = list(map(int, face_coordinates)) x, y, w, h = face_coordinates # expand to 20% padding_size_x, padding_size_y = int(0.2 * x), int(0.2 * y) w += padding_size_x h += padding_size_y x -= padding_size_x // 2 y -= padding_size_y // 2 face_image = rgb_image[y:y + h, x:x + w] try: face_image = cv2.resize( face_image, (self.face_size, self.face_size)) except: emotions_by_frames.append( np.array([0] * len(self.emotion_labels))) continue face_image = self.prepare_face_image(face_image) #print('Shape:', face_image.shape) emotion_prediction = self.emotion_classifier.predict( face_image, probs=True)[0].numpy() #print('Prediction:', emotion_prediction) emotion_probability = np.max(emotion_prediction) if len(emotion_prediction) > 0: emotions_by_frames.append(emotion_prediction) else: emotions_by_frames.append( np.array([0] * len(self.emotion_labels))) emotion_label_arg = np.argmax(emotion_prediction) emotion_text = self.emotion_labels[emotion_label_arg] if emotion_text == 'angry': color = emotion_probability * np.asarray((255, 0, 0)) elif emotion_text == 'sad': color = emotion_probability * np.asarray((0, 0, 255)) elif emotion_text == 'happy': color = emotion_probability * np.asarray((255, 255, 0)) elif emotion_text == 'surprise': color = emotion_probability * np.asarray((0, 255, 255)) else: color = emotion_probability * np.asarray((0, 255, 0)) color = color.astype(int) color = color.tolist() draw_bounding_box(face_coordinates, bgr_image, color) draw_text(face_coordinates, bgr_image, emotion_text, color, 0, -45, 1, 1) #bgr_image = cv2.cvtColor(rgb_image, cv2.COLOR_RGB2BGR) cv2.imshow('window_frame', bgr_image) if (cv2.waitKey(1) & 0xFF == ord('q')) and self.show_frame: break except: traceback.print_exc() break cap.release() cv2.destroyAllWindows() if csv: return self.to_csv(emotions_by_frames) return emotions_by_frames
if len(emotion_window) > frame_window: emotion_window.pop(0) try: emotion_mode = mode(emotion_window) except: continue if emotion_text == 'angry': color = emotion_probability * np.asarray((255, 0, 0)) elif emotion_text == 'sad': color = emotion_probability * np.asarray((0, 0, 255)) elif emotion_text == 'happy': color = emotion_probability * np.asarray((255, 255, 0)) elif emotion_text == 'surprise': color = emotion_probability * np.asarray((0, 255, 255)) else: color = emotion_probability * np.asarray((0, 255, 0)) color = color.astype(int) color = color.tolist() draw_bounding_box(face_coordinates, rgb_image, color) draw_text(face_coordinates, rgb_image, emotion_mode, color, 0, -45, 1, 1) bgr_image = cv2.cvtColor(rgb_image, cv2.COLOR_RGB2BGR) cv2.imshow('window_frame', bgr_image) if cv2.waitKey(1) & 0xFF == ord('q'): break
def biaoqing(frame): gray_image = cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY) rgb_image = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB) dets = detector(gray_image, 1) #print(dets) #faces = detect_faces(face_detection, gray_image)#[[横,纵,宽,高]] #print(faces) max_face = 0 max_k = 0 emotion_text = 0 for k,d in enumerate(dets): if (d.right()-d.left())*(d.bottom()-d.top()) > max_face: max_face = (d.right()-d.left())*(d.bottom()-d.top()) max_k = k for k,d in enumerate(dets): if k != max_k: continue face_coordinates=[] face_coordinates.append(d.left()) face_coordinates.append(d.top()) face_coordinates.append(d.right()-d.left()) face_coordinates.append(d.bottom()-d.top()) x1, x2, y1, y2 = apply_offsets(face_coordinates, emotion_offsets) gray_face = gray_image[y1:y2, x1:x2] try: gray_face = cv2.resize(gray_face, (emotion_target_size)) except: continue gray_face = preprocess_input(gray_face, True) gray_face = np.expand_dims(gray_face, 0) gray_face = np.expand_dims(gray_face, -1) emotion_prediction = emotion_classifier.predict(gray_face) emotion_probability = np.max(emotion_prediction) emotion_label_arg = np.argmax(emotion_prediction) emotion_text = emotion_labels[emotion_label_arg] emotion_window.append(emotion_text) if len(emotion_window) > frame_window: emotion_window.pop(0) try: emotion_mode = mode(emotion_window) except: continue if emotion_text == 'angry': color = emotion_probability * np.asarray((255, 0, 0)) elif emotion_text == 'sad': color = emotion_probability * np.asarray((0, 0, 255)) elif emotion_text == 'happy': color = emotion_probability * np.asarray((255, 255, 0)) elif emotion_text == 'surprise': color = emotion_probability * np.asarray((0, 255, 255)) else: color = emotion_probability * np.asarray((0, 255, 0)) color = color.astype(int) color = color.tolist() draw_bounding_box(face_coordinates, rgb_image, color) draw_text(face_coordinates, rgb_image, emotion_mode, color, 0, -45, 1, 1) bgr_image = cv2.cvtColor(rgb_image, cv2.COLOR_RGB2BGR) # cv2.imshow('out', bgr_image) if emotion_text: return emotion_text, bgr_image else: return 0, frame
def processa(): # parameters for loading data and images emotion_model_path = './models/emotion_model.hdf5' #Carica il modello delle emozioni , pre-addestrato emotion_labels = get_labels('fer2013') # hyper-parameters for bounding boxes shape frame_window = 10 emotion_offsets = (20, 40) # loading models #Rilevamento caratteristiche del viso detector = dlib.get_frontal_face_detector( ) #Rilevamento caratteristiche del viso emotion_classifier = load_model(emotion_model_path) # predictor = dlib.shape_predictor("shape_predictor_68_face_landmarks.dat") # getting input model shapes for inference emotion_target_size = emotion_classifier.input_shape[ 1:3] #Fa una stima parziale del modello di input # starting lists for calculating modes emotion_window = [] # Initialize some variables face_locations = [] face_encodings = [] #Delimita i volti umani face_names = [] process_this_frame = True def face_compare(frame, process_this_frame): # Resize frame of video to 1/4 size for faster face recognition processing small_frame = cv2.resize(frame, (0, 0), fx=0.50, fy=0.50) # Convert the image from BGR color (which OpenCV uses) to RGB color (which face_recognition uses) rgb_small_frame = small_frame[:, :, ::-1] # Only process every other frame of video to save time if process_this_frame: # Find all the faces and face encodings in the current frame of video(cerca i volti nel frame video) face_locations = face_recognition.face_locations( rgb_small_frame) #Delimitazione del volto umano face_encodings = face_recognition.face_encodings( rgb_small_frame, face_locations ) #apprende se c'è un volto umano tramite face location # starting video streaming cv2.namedWindow('window_frame') cap = None # Webcam source cap = cv2.VideoCapture(0) while cap.isOpened(): # True: ret, frame = cap.read( ) #restituisce le immagini della webcam frame per frame #frame = video_capture.read()[1] # To print the facial landmarks # landmrk = face_recognition.face_landmarks(frame) # for l in landmrk: # for key,val in l.items(): # for (x,y) in val: # cv2.circle(frame, (x, y), 1, (255,0, 0), -1) gray_image = cv2.cvtColor( frame, cv2.COLOR_BGR2GRAY) #trasforma i frame in scala di grigio rgb_image = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB) #trasforma i frame in rgb faces = detector(rgb_image) #rileva caratteristiche del viso # face_locations = face_recognition.face_locations(rgb_image) # print (reversed(face_locations)) face_name = face_compare(rgb_image, process_this_frame) for face_coordinates, fname in zip(faces, "Unknow"): x1, x2, y1, y2 = apply_offsets( face_utils.rect_to_bb(face_coordinates), emotion_offsets) gray_face = gray_image[y1:y2, x1:x2] try: gray_face = cv2.resize(gray_face, (emotion_target_size)) except: continue gray_face = preprocess_input(gray_face, True) gray_face = np.expand_dims(gray_face, 0) gray_face = np.expand_dims(gray_face, -1) emotion_prediction = emotion_classifier.predict( gray_face) #fa delle predizioni emotion_probability = np.max( emotion_prediction) #prende la predizione con valore massimo emotion_label_arg = np.argmax( emotion_prediction ) #prende l'indice della predizione con valore massimo emotion_text = emotion_labels[ emotion_label_arg] #inserisce nella variabile l'umore predetto emotion_window.append(emotion_text) if len(emotion_window) > frame_window: emotion_window.pop(0) try: emotion_mode = mode( emotion_window ) #restituisce il dato più comune nella lista except: continue if emotion_text == 'Triste': color = emotion_probability * np.asarray((0, 0, 255)) elif emotion_text == 'Felice': color = emotion_probability * np.asarray((255, 255, 0)) else: color = emotion_probability * np.asarray((0, 255, 0)) color = color.astype(int) color = color.tolist() #converte un colore in un elenco del modulo name = emotion_text draw_bounding_box(face_utils.rect_to_bb(face_coordinates), rgb_image, color) #crea un rettangolo colorato draw_text(face_utils.rect_to_bb(face_coordinates), rgb_image, name, color, 0, -45, 0.5, 1) #inserisce l'emozione predetta colorata frame = cv2.cvtColor(rgb_image, cv2.COLOR_RGB2BGR) #converte da rgb a bgr cv2.imshow('window_frame', frame) #mostra il frame nella finestra windows_frame if cv2.waitKey(1) & 0xFF == ord( 'q'): #se c'è un ritardo di un milli secondo allora break return emotion_text cap.release() cv2.destroyAllWindows()
emotion_window.pop(0) try: emotion_mode = mode(emotion_window) except: continue if emotion_text == 'angry': color = emotion_probability * np.asarray((255, 0, 0)) elif emotion_text == 'sad': color = emotion_probability * np.asarray((0, 0, 255)) elif emotion_text == 'happy': color = emotion_probability * np.asarray((255, 255, 0)) elif emotion_text == 'surprise': color = emotion_probability * np.asarray((0, 255, 255)) else: color = emotion_probability * np.asarray((0, 255, 0)) color = color.astype(int) color = color.tolist() draw_bounding_box(face_coordinates, rgb_image, color) draw_text(face_coordinates, rgb_image, emotion_mode, color, 0, -45, 1, 2) bgr_image = cv2.cvtColor(rgb_image, cv2.COLOR_RGB2BGR) cv2.imshow('window_frame', bgr_image) if cv2.waitKey(1) & 0xFF == ord('q'): break video_capture.release() cv2.destroyAllWindows()
"input": { "'neutral'": "1" }, "robot": "Pepper" } print("neutral") pub_emotion.publish(data) color = 1 * np.asarray((0, 0, 0)) else: draw = False if draw: color = color.astype(int) color = color.tolist() draw_bounding_box(box, rgb_image, color) draw_text(box, rgb_image, emotion_mode, color, 0, -45, 1, 1) if (show_image == 1): bgr_image = cv2.cvtColor(rgb_image, cv2.COLOR_RGB2BGR) cv2.imshow('window_frame', bgr_image) if cv2.waitKey(1) & 0xFF == ord('q'): break except: pass else: pass
def process_image(image): try: # parameters for loading data and images detection_model_path = './trained_models/detection_models/haarcascade_frontalface_default.xml' emotion_model_path = './trained_models/emotion_models/fer2013_mini_XCEPTION.102-0.66.hdf5' gender_model_path = './trained_models/gender_models/simple_CNN.81-0.96.hdf5' emotion_labels = get_labels('fer2013') gender_labels = get_labels('imdb') font = cv2.FONT_HERSHEY_SIMPLEX # hyper-parameters for bounding boxes shape gender_offsets = (30, 60) gender_offsets = (10, 10) emotion_offsets = (20, 40) emotion_offsets = (0, 0) # loading models face_detection = load_detection_model(detection_model_path) emotion_classifier = load_model(emotion_model_path, compile=False) gender_classifier = load_model(gender_model_path, compile=False) # getting input model shapes for inference emotion_target_size = emotion_classifier.input_shape[1:3] gender_target_size = gender_classifier.input_shape[1:3] # loading images image_array = np.fromstring(image, np.uint8) unchanged_image = cv2.imdecode(image_array, cv2.IMREAD_UNCHANGED) rgb_image = cv2.cvtColor(unchanged_image, cv2.COLOR_BGR2RGB) gray_image = cv2.cvtColor(unchanged_image, cv2.COLOR_BGR2GRAY) faces = detect_faces(face_detection, gray_image) for face_coordinates in faces: x1, x2, y1, y2 = apply_offsets(face_coordinates, gender_offsets) rgb_face = rgb_image[y1:y2, x1:x2] x1, x2, y1, y2 = apply_offsets(face_coordinates, emotion_offsets) gray_face = gray_image[y1:y2, x1:x2] try: rgb_face = cv2.resize(rgb_face, (gender_target_size)) gray_face = cv2.resize(gray_face, (emotion_target_size)) except: continue rgb_face = preprocess_input(rgb_face, False) rgb_face = np.expand_dims(rgb_face, 0) gender_prediction = gender_classifier.predict(rgb_face) gender_label_arg = np.argmax(gender_prediction) gender_text = gender_labels[gender_label_arg] gray_face = preprocess_input(gray_face, True) gray_face = np.expand_dims(gray_face, 0) gray_face = np.expand_dims(gray_face, -1) emotion_label_arg = np.argmax(emotion_classifier.predict(gray_face)) emotion_text = emotion_labels[emotion_label_arg] if gender_text == gender_labels[0]: color = (0, 0, 255) else: color = (255, 0, 0) draw_bounding_box(face_coordinates, rgb_image, color) draw_text(face_coordinates, rgb_image, gender_text, color, 0, -20, 1, 2) draw_text(face_coordinates, rgb_image, emotion_text, color, 0, -50, 1, 2) except Exception as err: logging.error('Error in emotion gender processor: "{0}"'.format(err)) bgr_image = cv2.cvtColor(rgb_image, cv2.COLOR_RGB2BGR) dirname = 'result' if not os.path.exists(dirname): os.mkdir(dirname) cv2.imwrite(os.path.join(dirname, 'predicted_image.png'), bgr_image)
emotion_mode = mode(emotion_window) except: continue if emotion_text == 'angry': color = emotion_probability * np.asarray((255, 0, 0)) elif emotion_text == 'sad': color = emotion_probability * np.asarray((0, 0, 255)) elif emotion_text == 'happy': print (emotion_probability) color = emotion_probability * np.asarray((255, 255, 0)) elif emotion_text == 'surprise': color = emotion_probability * np.asarray((0, 255, 255)) else: color = emotion_probability * np.asarray((0, 255, 0)) color = color.astype(int) color = color.tolist() draw_bounding_box(face_coordinates, rgb_image, color) draw_text(emotion_probability,face_coordinates, rgb_image, emotion_mode, color, 0, -45, 1, 1) bgr_image = cv2.cvtColor(rgb_image, cv2.COLOR_RGB2BGR) out.write(bgr_image) cv2.imshow('window_frame', bgr_image) if cv2.waitKey(1) & 0xFF == ord('q'): break cap.release() cv2.destroyAllWindows()
if emotion_text == 'angry': color = emotion_probability * np.asarray((255, 0, 0)) elif emotion_text == 'sad': color = emotion_probability * np.asarray((0, 0, 255)) elif emotion_text == 'happy': color = emotion_probability * np.asarray((255, 255, 0)) elif emotion_text == 'surprise': color = emotion_probability * np.asarray((0, 255, 255)) else: color = emotion_probability * np.asarray((0, 255, 0)) color = color.astype(int) color = color.tolist() draw_bounding_box(face_coordinates, rgb_image, color) draw_text(face_coordinates, rgb_image, emotion_mode, color, 0, -45, 1, 1) for i in range(len(emotion_prediction[0])): print(face_coordinates) x, y, w, h = face_coordinates emotion_text_coordinates = (x, y + h + 20 * i) emotion_text = emotion_labels[i] emotion_score = "{}: {:.2f}".format(emotion_text, emotion_prediction[0][i]) draw_text(emotion_text_coordinates, rgb_image, emotion_score, (0, 255, 0), 0, 45, 0.5, 1) except: pass bgr_image = cv2.cvtColor(rgb_image, cv2.COLOR_RGB2BGR) # bgr_image = cv2.flip(bgr_image, ) cv2.imshow('window_frame', bgr_image)
def main(): warnings.filterwarnings("ignore") logging.disable(logging.CRITICAL) global emotion_cache #Vishal emotion_cache = [] #Vishal global stop_detecting #Create server client for socket connection port = opt.port #host_name = socket.gethostname() #host_ip = socket.gethostbyname(host_name) host_ip = opt.ip mySocket = socket.socket() mySocket.bind((host_ip, port)) print ("Camera server listening on port {0} ".format(port)+ "and IP " + str(host_ip)) mySocket.listen(1) chatbot_socket, adress = mySocket.accept() print("Connection established to: " + str(adress)) #Start socket process to listen to chatbot requests chatbot = Process(target = chatbot_process, args = (chatbot_socket, chatbot_request, chatbot_response, stop_camera)) chatbot.start() signal.signal(signal.SIGTSTP, handler) #face_recognizer = Face_Recognizer(True)# True to show video feed # parameters for loading data and images detection_model_path = './models/face_recognition/trained_models/haarcascade_frontalface_default.xml' emotion_model_path = './models/face_recognition/trained_models/fer2013_mini_XCEPTION.102-0.66.hdf5' emotion_labels = get_labels('fer2013') # hyper-parameters for bounding boxes shape frame_window = 10 emotion_offsets = (20, 40) # loading models face_detection = load_detection_model(detection_model_path) emotion_classifier = load_model(emotion_model_path, compile=False) # getting input model shapes for inference emotion_target_size = emotion_classifier.input_shape[1:3] # starting lists for calculating modes emotion_window = [] emotion_text = 'Happy' # starting video streaming cv2.namedWindow('ODO_frame') OdoSet='nvarguscamerasrc ! video/x-raw(memory:NVMM), width=3820, height=2464, format=NV12, framerate=21/1 ! nvvidconv flip-method='+str(flip)+' ! video/x-raw, width='+str(dispW)+', height='+str(dispH)+', format=BGRx ! videoconvert ! video/x-raw, format=BGR ! appsink' video_capture = cv2.VideoCapture(OdoSet) fps = video_capture.get(cv2.CAP_PROP_FPS) print(f'FPS: {fps}') num_frames = 21; if video_capture.isOpened(): print("True") else: print("Error") ##################Vishal##################### while stop_detecting == False: ############################################# # Detect Emotion #print("Check") start = time.time() #starting FPS timer bgr_image = video_capture.read()[1] gray_image = cv2.cvtColor(bgr_image, cv2.COLOR_BGR2GRAY) rgb_image = cv2.cvtColor(bgr_image, cv2.COLOR_BGR2RGB) faces = detect_faces(face_detection, gray_image) for face_coordinates in faces: x1, x2, y1, y2 = apply_offsets(face_coordinates, emotion_offsets) gray_face = gray_image[y1:y2, x1:x2] try: gray_face = cv2.resize(gray_face, (emotion_target_size)) except: continue gray_face = preprocess_input(gray_face, True) gray_face = np.expand_dims(gray_face, 0) gray_face = np.expand_dims(gray_face, -1) emotion_prediction = emotion_classifier.predict(gray_face) emotion_probability = np.max(emotion_prediction) emotion_label_arg = np.argmax(emotion_prediction) emotion_text = emotion_labels[emotion_label_arg] emotion_window.append(emotion_text) if len(emotion_window) > frame_window: emotion_window.pop(0) try: emotion_mode = mode(emotion_window) except: continue if emotion_text == 'angry': color = emotion_probability * np.asarray((255, 0, 0)) elif emotion_text == 'sad': color = emotion_probability * np.asarray((0, 0, 255)) elif emotion_text == 'happy': color = emotion_probability * np.asarray((255, 255, 0)) elif emotion_text == 'surprise': color = emotion_probability * np.asarray((0, 255, 255)) else: color = emotion_probability * np.asarray((0, 255, 0)) emotion_cache.append(emotion_text) with open('listfile.txt', 'w') as filehandle: filehandle.writelines("%s\n" % emotions for emotions in emotion_cache) if len(emotion_cache) > 500: emotion_cache = emotion_cache[:-500] ########################################## total_faces = len(faces) current_time = time.time() if chatbot_request.empty() == False: request = chatbot_request.get() data_request = request['request'] if data_request.lower() == 'send_emotion': data = {} data['emotion'] = emotion_detected data['total_faces'] = total_faces data['time'] = current_time json_data = json.dumps(data) chatbot_response.put({'response': json_data}) color = color.astype(int) color = color.tolist() #print(face_coordinates, rgb_image, color) draw_bounding_box(face_coordinates, rgb_image, color) draw_text(face_coordinates, rgb_image, emotion_mode, color, 0, -45, 1, 1) if stop_camera.empty() == False: stop_detecting = True end = time.time() #end time for FPS bgr_image = cv2.cvtColor(rgb_image, cv2.COLOR_RGB2BGR) seconds = end - start #FPS fps = num_frames/seconds #FPS fps_coords = (0,30) framerate_str = 'Framerate: ' + str(int(fps)) cv2.putText(bgr_image, framerate_str, fps_coords, cv2.FONT_HERSHEY_SIMPLEX, 1, (255,255,200)) print(framerate_str) cv2.imshow('ODO_frame', bgr_image) if cv2.waitKey(1) & 0xFF == ord('q'): break video_capture.release() cv2.destroyAllWindows() print("Closing chatbot socket") chatbot.terminate() chatbot.join()
def process_image(image): detected_peoples = [] json_info = {} try: # parameters for loading data and images emotion_labels = get_labels('fer2013') gender_labels = get_labels('imdb') # font = cv2.FONT_HERSHEY_SIMPLEX # gender_keys = list(gender_labels.values()) # emotion_keys = list(emotion_labels.values()) # hyper-parameters for bounding boxes shape gender_offsets = (30, 60) gender_offsets = (10, 10) emotion_offsets = (20, 40) emotion_offsets = (0, 0) # getting input model shapes for inference gender_target_size = _GENDER_CLASSIFIER.input_shape[1:3] emotion_target_size = _EMOTION_CLASSIFIER.input_shape[1:3] # loading images image_array = np.fromstring(image, np.uint8) unchanged_image = cv2.imdecode(image_array, cv2.IMREAD_UNCHANGED) rgb_image = cv2.cvtColor(unchanged_image, cv2.COLOR_BGR2RGB) gray_image = cv2.cvtColor(unchanged_image, cv2.COLOR_BGR2GRAY) faces = detect_faces(_FACE_DETECTION, gray_image) for face_coordinates in faces: x1, x2, y1, y2 = apply_offsets(face_coordinates, gender_offsets) rgb_face = rgb_image[y1:y2, x1:x2] x1, x2, y1, y2 = apply_offsets(face_coordinates, emotion_offsets) gray_face = gray_image[y1:y2, x1:x2] try: rgb_face = cv2.resize(rgb_face, (gender_target_size)) gray_face = cv2.resize(gray_face, (emotion_target_size)) except: continue start_time = datetime.now() with _GRAPH.as_default(): rgb_face = preprocess_input(rgb_face, False) rgb_face = np.expand_dims(rgb_face, 0) gender_prediction = _GENDER_CLASSIFIER.predict(rgb_face) gender_label_arg = np.argmax(gender_prediction) gender_text = gender_labels[gender_label_arg] end_time = datetime.now() delta1 = end_time - start_time gray_face = preprocess_input(gray_face, True) gray_face = np.expand_dims(gray_face, 0) gray_face = np.expand_dims(gray_face, -1) emotion_prediction = _EMOTION_CLASSIFIER.predict(gray_face) emotion_label_arg = np.argmax(emotion_prediction) emotion_text = emotion_labels[emotion_label_arg] delta2 = datetime.now() - end_time print("Delta for gender classificator {0}".format( delta1.total_seconds() * 1000.0)) print("Delta for emotion classificator {0}".format( delta2.total_seconds() * 1000.0)) json_info['gender'] = "{0}:{1}".format( gender_text, gender_prediction.flat[gender_label_arg]) json_info['emotion'] = "{0}:{1}".format( emotion_text, emotion_prediction.flat[emotion_label_arg]) json_info['face_bound'] = list( map(lambda it: str(it), list(face_coordinates.astype(np.int).flat))) detected_peoples.append(json_info.copy()) print(detected_peoples) if gender_text == gender_labels[0]: color = (0, 0, 255) else: color = (255, 0, 0) draw_bounding_box(face_coordinates, rgb_image, color) draw_text(face_coordinates, rgb_image, gender_text, color, 0, -20, 1, 2) draw_text(face_coordinates, rgb_image, emotion_text, color, 0, -10, 1, 2) except Exception as err: logging.error('Error in emotion gender processor: "{0}"'.format(err)) start_time = datetime.now() bgr_image = cv2.cvtColor(rgb_image, cv2.COLOR_RGB2BGR) if not os.path.exists(_SAVE_DIR): os.mkdir(_SAVE_DIR) recognition_datetime = str(datetime.now()).replace(' ', '_') filepath = os.path.join( _SAVE_DIR, 'predicted_image_{0}.png'.format(recognition_datetime)) cv2.imwrite(filepath, bgr_image) delta = datetime.now() - start_time print("delta for saving image", delta.total_seconds() * 1000.0) return detected_peoples
rgb_face = preprocess_input(rgb_face, False) gender_prediction = gender_classifier.predict(rgb_face) gender_label_arg = np.argmax(gender_prediction) gender_text = gender_labels[gender_label_arg] gender_window.append(gender_text) if len(gender_window) > frame_window: emotion_window.pop(0) gender_window.pop(0) try: emotion_mode = mode(emotion_window) gender_mode = mode(gender_window) except: continue if gender_text == gender_labels[0]: color = (0, 0, 255) else: color = (255, 0, 0) draw_bounding_box(face_coordinates, rgb_image, color) draw_text(face_coordinates, rgb_image, gender_mode, color, 0, -20, 1, 1) draw_text(face_coordinates, rgb_image, emotion_mode, color, 0, -45, 1, 1) bgr_image = cv2.cvtColor(rgb_image, cv2.COLOR_RGB2BGR) cv2.imshow('window_frame', bgr_image) if cv2.waitKey(1) & 0xFF == ord('q'): break
def emotion_identify(img_url): # parameters for loading data and images detection_model_path = 'C:/Users/Admin/PycharmProjects/Emotion_Detection/trained_models/detection_models/haarcascade_frontalface_default.xml' emotion_model_path = 'C:/Users/Admin/PycharmProjects/Emotion_Detection/trained_models/emotion_models/fer2013_mini_XCEPTION.102-0.66.hdf5' gender_model_path = 'C:/Users/Admin/PycharmProjects/Emotion_Detection/trained_models/gender_models/simple_CNN.81-0.96.hdf5' emotion_labels = get_labels('fer2013') gender_labels = get_labels('imdb') font = cv2.FONT_HERSHEY_SIMPLEX # hyper-parameters for bounding boxes shape gender_offsets = (30, 60) gender_offsets = (10, 10) emotion_offsets = (20, 40) emotion_offsets = (0, 0) # loading models face_detection = load_detection_model(detection_model_path) emotion_classifier = load_model(emotion_model_path, compile=False) gender_classifier = load_model(gender_model_path, compile=False) # getting input model shapes for inference emotion_target_size = emotion_classifier.input_shape[1:3] gender_target_size = gender_classifier.input_shape[1:3] # loading images image_path = img_url rgb_image = load_image(image_path, grayscale=False) gray_image = load_image(image_path, grayscale=True) gray_image = np.squeeze(gray_image) gray_image = gray_image.astype('uint8') faces = detect_faces(face_detection, gray_image) if len(faces) == 0: print("No face") K.clear_session() return False emotions = collections.defaultdict(int) for face_coordinates in faces: x1, x2, y1, y2 = apply_offsets(face_coordinates, gender_offsets) rgb_face = rgb_image[y1:y2, x1:x2] x1, x2, y1, y2 = apply_offsets(face_coordinates, emotion_offsets) gray_face = gray_image[y1:y2, x1:x2] try: rgb_face = cv2.resize(rgb_face, (gender_target_size)) gray_face = cv2.resize(gray_face, (emotion_target_size)) except: continue rgb_face = preprocess_input(rgb_face, False) rgb_face = np.expand_dims(rgb_face, 0) gender_prediction = gender_classifier.predict(rgb_face) gender_label_arg = np.argmax(gender_prediction) gender_text = gender_labels[gender_label_arg] gray_face = preprocess_input(gray_face, True) gray_face = np.expand_dims(gray_face, 0) gray_face = np.expand_dims(gray_face, -1) emotion_label_arg = np.argmax(emotion_classifier.predict(gray_face)) emotion_text = emotion_labels[emotion_label_arg] emotions[emotion_text] += 1 if gender_text == gender_labels[0]: color = (0, 0, 255) else: color = (255, 0, 0) draw_bounding_box(face_coordinates, rgb_image, color) draw_text(face_coordinates, rgb_image, gender_text, color, 0, -20, 1, 2) draw_text(face_coordinates, rgb_image, emotion_text, color, 0, -50, 1, 2) max_num = 0 max_emotion = None for key, value in emotions.items(): if value > max_num: max_num = value max_emotion = key print("The emotion of this picture is: ", max_emotion) bgr_image = cv2.cvtColor(rgb_image, cv2.COLOR_RGB2BGR) cv2.imwrite('./result_images/predicted_test_image.png', bgr_image) K.clear_session() return max_emotion