def process_image(image): try: # parameters for loading data and images detection_model_path = './trained_models/detection_models/haarcascade_frontalface_default.xml' emotion_model_path = './trained_models/emotion_models/fer2013_35a93d1a30cf52b8722837f3e096b7b1006949325de8d00f26595d5a418241d3.hdf5' emotion_labels = get_labels('fer2013') font = cv2.FONT_HERSHEY_SIMPLEX # hyper-parameters for bounding boxes shape emotion_offsets = (20, 40) emotion_offsets = (0, 0) # loading models face_detection = load_detection_model(detection_model_path) emotion_classifier = load_model(emotion_model_path, compile=False) # getting input model shapes for inference emotion_target_size = emotion_classifier.input_shape[1:3] # loading images image_array = np.fromstring(image, np.uint8) unchanged_image = cv2.imdecode(image_array, cv2.IMREAD_UNCHANGED) rgb_image = cv2.cvtColor(unchanged_image, cv2.COLOR_BGR2RGB) gray_image = cv2.cvtColor(unchanged_image, cv2.COLOR_BGR2GRAY) faces = detect_faces(face_detection, gray_image) for face_coordinates in faces: x1, x2, y1, y2 = apply_offsets(face_coordinates, emotion_offsets) gray_face = gray_image[y1:y2, x1:x2] try: gray_face = cv2.resize(gray_face, (emotion_target_size)) except: continue rgb_face = preprocess_input(rgb_face, False) rgb_face = np.expand_dims(rgb_face, 0) gray_face = preprocess_input(gray_face, True) gray_face = np.expand_dims(gray_face, 0) gray_face = np.expand_dims(gray_face, -1) emotion_label_arg = np.argmax(emotion_classifier.predict(gray_face)) emotion_text = emotion_labels[emotion_label_arg] color = (255, 0, 0) draw_bounding_box(face_coordinates, rgb_image, color) draw_text(face_coordinates, rgb_image, emotion_text, color, 0, -50, 1, 2) except Exception as err: logging.error('Error in emotion gender processor: "{0}"'.format(err)) bgr_image = cv2.cvtColor(rgb_image, cv2.COLOR_RGB2BGR) dirname = 'result' if not os.path.exists(dirname): os.mkdir(dirname) cv2.imwrite(os.path.join(dirname, 'predicted_image.png'), bgr_image)
def get_emotion(self, image_path_, face_detection, emotion_classifier, gender_classifier): #print(face_detection) emotion_labels = get_labels('fer2013') gender_labels = get_labels('imdb') font = cv2.FONT_HERSHEY_SIMPLEX # hyper-parameters for bounding boxes shape # change for variance gender_offsets = (30, 60) gender_offsets = (10, 10) emotion_offsets = (20, 40) emotion_offsets = (0, 0) # loading models # getting input model shapes for inference emotion_target_size = emotion_classifier.input_shape[1:3] gender_target_size = gender_classifier.input_shape[1:3] #By default, set emotion to "Neutral" and gender to "Unknown" emotion_text = "happy" gender_text = "Unknown" # loading images rgb_image = load_image(image_path_, grayscale=False) gray_image = load_image(image_path_, grayscale=True) gray_image = np.squeeze(gray_image) gray_image = gray_image.astype('uint8') faces = detect_faces(face_detection, gray_image) for face_coordinates in faces: x1, x2, y1, y2 = apply_offsets(face_coordinates, gender_offsets) rgb_face = rgb_image[y1:y2, x1:x2] x1, x2, y1, y2 = apply_offsets(face_coordinates, emotion_offsets) gray_face = gray_image[y1:y2, x1:x2] try: rgb_face = cv2.resize(rgb_face, (gender_target_size)) gray_face = cv2.resize(gray_face, (emotion_target_size)) except: continue rgb_face = preprocess_input(rgb_face, False) rgb_face = np.expand_dims(rgb_face, 0) gender_prediction = gender_classifier.predict(rgb_face) gender_label_arg = np.argmax(gender_prediction) gender_text = gender_labels[gender_label_arg] gray_face = preprocess_input(gray_face, True) gray_face = np.expand_dims(gray_face, 0) gray_face = np.expand_dims(gray_face, -1) emotion_label_arg = np.argmax( emotion_classifier.predict(gray_face)) emotion_text = emotion_labels[emotion_label_arg] return emotion_text, gender_text
def detect(self, image_path): # loading images rgb_image = load_image(image_path, grayscale=False) gray_image = load_image(image_path, grayscale=True) gray_image = np.squeeze(gray_image) gray_image = gray_image.astype('uint8') res = [] faces = detect_faces(self.face_detection, gray_image) for face_coordinates in faces: x1, x2, y1, y2 = apply_offsets(face_coordinates, self.gender_offsets) rgb_face = rgb_image[y1:y2, x1:x2] x1, x2, y1, y2 = apply_offsets(face_coordinates, self.emotion_offsets) gray_face = gray_image[y1:y2, x1:x2] try: rgb_face = cv2.resize(rgb_face, (self.gender_target_size)) gray_face = cv2.resize(gray_face, (self.emotion_target_size)) except: continue rgb_face = preprocess_input(rgb_face, False) rgb_face = np.expand_dims(rgb_face, 0) gender_prediction = self.gender_classifier.predict(rgb_face) gender_label_arg = np.argmax(gender_prediction) gender_text = self.gender_labels[gender_label_arg] gray_face = preprocess_input(gray_face, True) gray_face = np.expand_dims(gray_face, 0) gray_face = np.expand_dims(gray_face, -1) emotion_label_arg = np.argmax( self.emotion_classifier.predict(gray_face)) emotion_text = self.emotion_labels[emotion_label_arg] if gender_text == self.gender_labels[0]: color = (0, 0, 255) else: color = (255, 0, 0) res.append({ 'left': x1, 'top': y1, 'right': x2, 'bottom': y2, 'male': gender_text, 'emotion': emotion_text }) # print(gender_text, emotion_text) # draw = ImageDraw.Draw(img) # draw.rectangle(((x1, y1), (x2,y2)), outline='red') # draw.text((0, 0), "something123") return res
def classify_frame(bgr_image): # bgr_image = video_capture.read()[1] gray_image = cv2.cvtColor(bgr_image, cv2.COLOR_BGR2GRAY) rgb = cv2.cvtColor(bgr_image, cv2.COLOR_BGR2RGB) max_size = 512 aspect_ratio = rgb.shape[1] / rgb.shape[0] scaling_factor = max([rgb.shape[1], rgb.shape[0]]) / max_size new_width = int(rgb.shape[1] / scaling_factor) new_height = int(rgb.shape[0] / scaling_factor) rgb_image = cv2.resize(rgb, (new_width, new_height)) faces = detect_faces(face_detection, rgb_image) for face_coordinates in faces: x1, x2, y1, y2 = apply_offsets(face_coordinates, gender_offsets) rgb_face = rgb_image[y1:y2, x1:x2] gray_face = gray_image[y1:y2, x1:x2] try: rgb_face = cv2.resize(rgb_face, (gender_target_size)) except: continue gray_face = preprocess_input(gray_face, False) gray_face = np.expand_dims(gray_face, 0) gray_face = np.expand_dims(gray_face, -1) rgb_face = np.expand_dims(rgb_face, 0) rgb_face = preprocess_input(rgb_face, False) gender_prediction = gender_classifier.predict(rgb_face) gender_label_arg = np.argmax(gender_prediction) gender_text = gender_labels[gender_label_arg] gender_window.append(gender_text) if len(gender_window) > frame_window: gender_window.pop(0) try: gender_mode = mode(gender_window) except: continue if gender_text == gender_labels[0]: color = (0, 0, 255) else: color = (255, 0, 0) draw_bounding_box(face_coordinates, rgb_image, color) draw_text(face_coordinates, rgb_image, gender_mode, color, 0, -20, 1, 1) bgr_image = cv2.cvtColor(rgb_image, cv2.COLOR_RGB2BGR) cv2.imshow('window_frame', bgr_image)
def run_classify(image_path): emotion_text = '' gender_text = '' # loading images rgb_image = load_image(image_path, grayscale=False) gray_image = load_image(image_path, grayscale=True) gray_image = np.squeeze(gray_image) gray_image = gray_image.astype('uint8') faces = detect_faces(face_detection, gray_image) for face_coordinates in faces: x1, x2, y1, y2 = apply_offsets(face_coordinates, gender_offsets) rgb_face = rgb_image[y1:y2, x1:x2] x1, x2, y1, y2 = apply_offsets(face_coordinates, emotion_offsets) gray_face = gray_image[y1:y2, x1:x2] try: rgb_face = cv2.resize(rgb_face, (gender_target_size)) gray_face = cv2.resize(gray_face, (emotion_target_size)) except: continue rgb_face = preprocess_input(rgb_face, False) rgb_face = np.expand_dims(rgb_face, 0) gender_prediction = gender_classifier.predict(rgb_face) gender_label_arg = np.argmax(gender_prediction) gender_text = gender_labels[gender_label_arg] gray_face = preprocess_input(gray_face, True) gray_face = np.expand_dims(gray_face, 0) gray_face = np.expand_dims(gray_face, -1) emotion_label_arg = np.argmax(emotion_classifier.predict(gray_face)) emotion_text = emotion_labels[emotion_label_arg] if gender_text == gender_labels[0]: color = (0, 0, 255) else: color = (255, 0, 0) draw_bounding_box(face_coordinates, rgb_image, color) draw_text(face_coordinates, rgb_image, gender_text, color, 0, -20, 1, 2) draw_text(face_coordinates, rgb_image, emotion_text, color, 0, -50, 1, 2) bgr_image = cv2.cvtColor(rgb_image, cv2.COLOR_RGB2BGR) cv2.imwrite(os.path.join(dir, '../images/predicted_test_image.png'), bgr_image) return ([gender_text, emotion_text])
def make_label(image_path): # loading models face_detection = load_detection_model(detection_model_path) emotion_classifier = load_model(emotion_model_path, compile=False) gender_classifier = load_model(gender_model_path, compile=False) # getting input model shapes for inference emotion_target_size = emotion_classifier.input_shape[1:3] gender_target_size = gender_classifier.input_shape[1:3] # loading images rgb_image = load_image(image_path, grayscale=False) gray_image = load_image(image_path, grayscale=True) gray_image = np.squeeze(gray_image) gray_image = gray_image.astype('uint8') faces = detect_faces(face_detection, gray_image) for face_coordinates in faces: x1, x2, y1, y2 = apply_offsets(face_coordinates, gender_offsets) rgb_face = rgb_image[y1:y2, x1:x2] x1, x2, y1, y2 = apply_offsets(face_coordinates, emotion_offsets) gray_face = gray_image[y1:y2, x1:x2] try: rgb_face = cv2.resize(rgb_face, (gender_target_size)) gray_face = cv2.resize(gray_face, (emotion_target_size)) except: continue rgb_face = preprocess_input(rgb_face, False) rgb_face = np.expand_dims(rgb_face, 0) gender_prediction = gender_classifier.predict(rgb_face) gender_label_arg = np.argmax(gender_prediction) gender_text = gender_labels[gender_label_arg] gray_face = preprocess_input(gray_face, True) gray_face = np.expand_dims(gray_face, 0) gray_face = np.expand_dims(gray_face, -1) emotion_label_arg = np.argmax(emotion_classifier.predict(gray_face)) emotion_text = emotion_labels[emotion_label_arg] if gender_text == gender_labels[0]: color = (0, 0, 255) else: color = (255, 0, 0) return emotion_text
def predict_emotion_gender(rgb_face, gray_face): rgb_face = preprocess_input(rgb_face, False) rgb_face = np.expand_dims(rgb_face, 0) gender_prediction = gender_classifier.predict(rgb_face) gender_label_arg = np.argmax(gender_prediction) gender_text = gender_labels[gender_label_arg] gray_face = preprocess_input(gray_face, True) gray_face = np.expand_dims(gray_face, 0) gray_face = np.expand_dims(gray_face, -1) emotion_label_arg = np.argmax(emotion_classifier.predict(gray_face)) emotion_text = emotion_labels[emotion_label_arg] return (emotion_text, gender_text)
def det_emo_gen_reco(self, frame): self.res = [] self.emotion = [] self.gender = [] bgr_image = frame gray_image = cv2.cvtColor(bgr_image, cv2.COLOR_BGR2GRAY) rgb_image = cv2.cvtColor(bgr_image, cv2.COLOR_BGR2RGB) bounding_boxes = detect_faces(face_detection, gray_image) for face_coordinates in bounding_boxes: emo_gen = [] res_dic = {} x1, x2, y1, y2 = apply_offsets(face_coordinates, gender_offsets) rgb_face = rgb_image[y1:y2, x1:x2] x1, x2, y1, y2 = apply_offsets(face_coordinates, emotion_offsets) gray_face = gray_image[y1:y2, x1:x2] try: rgb_face = cv2.resize(rgb_face, (gender_target_size)) gray_face = cv2.resize(gray_face, (emotion_target_size)) except: continue gray_face = preprocess_input(gray_face, False) gray_face = np.expand_dims(gray_face, 0) gray_face = np.expand_dims(gray_face, -1) emotion_label_arg = np.argmax( emotion_classifier.predict(gray_face)) emotion_text = emotion_labels[emotion_label_arg] self.emotion.append(emotion_text) emo_gen.append(emotion_text) rgb_face = np.expand_dims(rgb_face, 0) rgb_face = preprocess_input(rgb_face, False) gender_prediction = gender_classifier.predict(rgb_face) gender_label_arg = np.argmax(gender_prediction) gender_text = gender_labels[gender_label_arg] self.gender.append(gender_text) emo_gen.append(gender_text) #print (face_coordinates) #res_dic[emotion_text] = face_coordinates res_dic[emotion_text + ',' + gender_text] = (int(x1), int(y1), int(x2), int(y2)) #res_dic[emotion_text+','+gender_text] = (int(face_coordinates[0]),int(face_coordinates[1]),int(face_coordinates[2]),int(face_coordinates[3])) self.res.append(res_dic) print(self.res) print(type(self.res)) return self.res
def emotion_recogination(image_path): frame = cv2.imread(image_path) gray_image = cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY) rgb_image = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB) faces = detector(rgb_image) for face_coordinates in faces: x1, x2, y1, y2 = apply_offsets(face_utils.rect_to_bb(face_coordinates), emotion_offsets) gray_face = gray_image[y1:y2, x1:x2] try: gray_face = cv2.resize(gray_face, (emotion_target_size)) except: continue gray_face = preprocess_input(gray_face, True) gray_face = np.expand_dims(gray_face, 0) gray_face = np.expand_dims(gray_face, -1) emotion_prediction = emotion_classifier.predict(gray_face) emotion_probability = np.max(emotion_prediction) emotion_label_arg = np.argmax(emotion_prediction) emotion_text = emotion_labels[emotion_label_arg] print(emotion_text) return emotion_text
def predict(self,image): gray_image = cv2.cvtColor(image, cv2.COLOR_BGR2GRAY) rgb_image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB) faces = self.face_cascade.detectMultiScale(gray_image, scaleFactor=1.1, minNeighbors=5,minSize=(30, 30), flags=cv2.CASCADE_SCALE_IMAGE) emotion_window = [] for face_coordinates in faces: x1, x2, y1, y2 = apply_offsets(face_coordinates, self.emotion_offsets) gray_face = gray_image[y1:y2, x1:x2] try: gray_face = cv2.resize(gray_face, (self.emotion_target_size)) except: continue gray_face = preprocess_input(gray_face, True) gray_face = np.expand_dims(gray_face, 0) gray_face = np.expand_dims(gray_face, -1) emotion_prediction = self.emotion_classifier.predict(gray_face) emotion_probability = np.max(emotion_prediction) emotion_label_arg = np.argmax(emotion_prediction) emotion_text = self.emotion_labels[emotion_label_arg] return emotion_text,emotion_label_arg return 'null',6
def emotion_face(bgr_image_base64): bgr_image = data_uri_to_cv2_img(bgr_image_base64) gray_image = cv2.cvtColor(bgr_image, cv2.COLOR_BGR2GRAY) rgb_image = cv2.cvtColor(bgr_image, cv2.COLOR_BGR2RGB) faces = face_cascade.detectMultiScale(gray_image, scaleFactor=1.1, minNeighbors=5, minSize=(30, 30), flags=cv2.CASCADE_SCALE_IMAGE) for face_coordinates in faces: x1, x2, y1, y2 = apply_offsets(face_coordinates, emotion_offsets) gray_face = gray_image[y1:y2, x1:x2] try: gray_face = cv2.resize(gray_face, (emotion_target_size)) except: continue gray_face = preprocess_input(gray_face, True) gray_face = np.expand_dims(gray_face, 0) gray_face = np.expand_dims(gray_face, -1) with graph.as_default(): emotion_prediction = emotion_classifier.predict(gray_face) emotion_probability = np.max(emotion_prediction) emotion_label_arg = np.argmax(emotion_prediction) emotion_text = emotion_labels[emotion_label_arg] print("emotion_text", emotion_text) return emotion_text
def main(): true = 0 emotion_model_path = '../trained_models/emotion_models/RAF/RAF_mini_XCEPTION.76-0.82.hdf5' emotion_classifier = load_model(emotion_model_path, compile=False) np.set_printoptions(precision=2) # loading dataset data_loader = DataManager(dataset_name, image_size=input_shape[:2]) train_faces, train_emotions, test_faces, test_emotions = data_loader.get_data( ) test_faces = preprocess_input(test_faces) num_samples, num_classes = test_emotions.shape predictions = emotion_classifier.predict(test_faces) predictions = np.argmax(predictions, axis=1) real_emotion = np.argmax(test_emotions, axis=1) # for i in range(num_samples): # if predictions[i] == real_emotion[i]: # true += 1 # acc = 100. * true / num_samples conf_mat = confusion_matrix(real_emotion, predictions) plt.figure(figsize=(5, 4)) plot_confusion_matrix( conf_mat, classes=['Sur', 'Fea', 'Dis', 'Hap', 'Sad', 'Ang', 'Neu'], normalize=True, ) # title='Normalized Confusion Matrix' plt.savefig(os.path.join(save_path, dataset_name + 'Confusion Matrix8-2.pdf'), bbox_inches='tight', dpi=600) plt.show() plt.close()
def detect_face(self): face_cascade = cv2.CascadeClassifier(self.CASE_PATH) # 0 means the default video capture device in OS video_capture = cv2.VideoCapture(0) # infinite loop, break by key ESC while True: if not video_capture.isOpened(): sleep(5) # Capture frame-by-frame ret, frame = video_capture.read() gray = cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY) faces = face_cascade.detectMultiScale(gray, scaleFactor=1.2, minNeighbors=10, minSize=(self.face_size, self.face_size)) # placeholder for cropped faces face_imgs = np.empty( (len(faces), self.face_size, self.face_size, 3)) for i, face in enumerate(faces): face_img, cropped = self.crop_face(frame, face, margin=40, size=self.face_size) (x, y, w, h) = cropped cv2.rectangle(frame, (x, y), (x + w, y + h), (0, 128, 0), 2) face_imgs[i, :, :, :] = face_img x1, x2, y1, y2 = apply_offsets(face, emotion_offsets) gray_face = gray[y1:y2, x1:x2] try: gray_face = cv2.resize(gray_face, (emotion_target_size)) except: continue #cv2.imshow("gray-face",gray_face) if len(face_imgs) > 0: results = self.model.predict(face_imgs) predicted_genders = results[0] ages = np.arange(0, 101).reshape(101, 1) predicted_ages = results[1].dot(ages).flatten() gray_face = preprocess_input(gray_face, False) gray_face = np.expand_dims(gray_face, 0) gray_face = np.expand_dims(gray_face, -1) emotion_label_arg = np.argmax( emotion_classifier.predict(gray_face)) emotion_text = emotion_labels[emotion_label_arg] # draw results for i, face in enumerate(faces): label = "{}, {}".format( int(predicted_ages[i]), "F" if predicted_genders[i][0] > 0.5 else "M") label = label + ", " + emotion_text.upper() self.draw_label(frame, (face[0], face[1]), label) cv2.imshow('Detection', frame) if cv2.waitKey(5) == 27: # ESC key press break # When everything is done, release the capture video_capture.release() cv2.destroyAllWindows()
def main(): emotion_model_path = '../trained_models/emotion_models/fer2013/fer2013_mini_XCEPTION.102-0.66.hdf5' emotion_classifier = load_model(emotion_model_path, compile=False) np.set_printoptions(precision=2) # loading dataset data_loader = DataManager(dataset_name, image_size=input_shape[:2]) faces, emotions = data_loader.get_data() faces = preprocess_input(faces) num_samples, num_classes = emotions.shape train_data, val_data = split_data(faces, emotions, validation_split) val_faces, val_emotions = val_data predictions = emotion_classifier.predict(val_faces) predictions = np.argmax(predictions, axis=1) real_emotions = np.argmax(val_emotions, axis=1) conf_mat = confusion_matrix(real_emotions, predictions) plt.figure(figsize=(10, 8)) plot_confusion_matrix( conf_mat, classes=['Ang', 'Dis', 'Fea', 'Hap', 'Sad', 'Sur', 'Neu'], normalize=True) # plot_confusion_matrix(conf_mat, classes=['AN', 'DI', 'AF', # 'HA', 'SA', 'SU', 'NE'], # normalize=True) plt.show()
def getFaceDetails(image_np): gray_image = cv2.cvtColor(image_np, cv2.COLOR_RGB2GRAY) # the model requires a grayscale image faces = detect_faces(face_detection_model, gray_image) # pass in the grayscale image if len(faces) > 0: # make sure a face was found draw_bounding_box(faces[0], image_np) # draws a square around the face, just for testing purposes # doing preprocessing on grayscale image, found from the model creator's example x1, x2, y1, y2 = apply_offsets(faces[0], emotion_offsets) gray_face = gray_image[y1:y2, x1:x2] gray_face = cv2.resize(gray_face, (emotion_target_size)) gray_face = preprocess_input(gray_face, True) gray_face = np.expand_dims(gray_face, 0) gray_face = np.expand_dims(gray_face, -1) with graph.as_default(): # to prevent graph not found error emotion_prediction = emotion_classifier.predict(gray_face) # predict the emotion #emotion_probability = np.max(emotion_prediction) # uncomment this line to see the accuracy of the emotion emotion_label_arg = np.argmax(emotion_prediction) face_details = [emotion_labels[emotion_label_arg], int(faces[0][0]), int(faces[0][1]), int(faces[0][2]), int(faces[0][3])] # structure to return emotion and face coords else: face_details = [None, None, None, None, None] # in case no face was found thread_storage['face'] = face_details # instead of returning, put it into dictionary
def detection(image_path): # parameters for loading data and images detection_model_path = 'C:\\Users\\l1f15bscs0049\\Desktop\\haarcascade_frontalface_default.xml' font = cv2.FONT_HERSHEY_SIMPLEX # hyper-parameters for bounding boxes shape gender_offsets = (30, 60) gender_offsets = (10, 10) # loading models face_detection = load_detection_model(detection_model_path) # loading images rgb_image = load_image(image_path, grayscale=False) gray_image = load_image(image_path, grayscale=True) gray_image = np.squeeze(gray_image) gray_image = gray_image.astype('uint8') count1 = 0 faces = detect_faces(face_detection, gray_image) for face_coordinates in faces: x1, x2, y1, y2 = apply_offsets(face_coordinates, gender_offsets) rgb_face = rgb_image[y1:y2, x1:x2] rgb_face = preprocess_input(rgb_face, False) rgb_face = np.expand_dims(rgb_face, 0) color = (0, 0, 255) draw_bounding_box(face_coordinates, rgb_image, color) count1 = count1 + 1 bgr_image = cv2.cvtColor(rgb_image, cv2.COLOR_RGB2BGR) #cv2.imwrite('C:\\Users\\l1f15bscs0049\\Desktop\\test_cases\\testt2.png', bgr_image) return count1
def process_image(image): results = [] image_array = np.fromstring(image, np.uint8) unchanged_image = cv2.imdecode(image_array, cv2.IMREAD_UNCHANGED) rgb_image = cv2.cvtColor(unchanged_image, cv2.COLOR_BGR2RGB) gray_image = cv2.cvtColor(unchanged_image, cv2.COLOR_BGR2GRAY) faces = detect_faces(face_detection, gray_image) for face_coordinates in faces: x1, x2, y1, y2 = apply_offsets(face_coordinates, emotion_offsets) gray_face = gray_image[y1:y2, x1:x2] try: gray_face = cv2.resize(gray_face, (emotion_target_size)) except: continue gray_face = preprocess_input(gray_face, True) gray_face = np.expand_dims(gray_face, 0) gray_face = np.expand_dims(gray_face, -1) with graph.as_default(): emotion_label_arg = np.argmax( emotion_classifier.predict(gray_face)) emotion_text = emotion_labels[emotion_label_arg] results.append(emotion_text) dic_result = {"emotions": results, "number face": len(faces)} return dic_result
def process_image(image): results = [] try: # loading images image_array = np.fromstring(image, np.uint8) unchanged_image = cv2.imdecode(image_array, cv2.IMREAD_UNCHANGED) rgb_image = cv2.cvtColor(unchanged_image, cv2.COLOR_BGR2RGB) gray_image = cv2.cvtColor(unchanged_image, cv2.COLOR_BGR2GRAY) faces = detect_faces(face_detection, gray_image) for face_coordinates in faces: x1, x2, y1, y2 = apply_offsets(face_coordinates, gender_offsets) rgb_face = rgb_image[y1:y2, x1:x2] x1, x2, y1, y2 = apply_offsets(face_coordinates, emotion_offsets) gray_face = gray_image[y1:y2, x1:x2] try: rgb_face = cv2.resize(rgb_face, (gender_target_size)) gray_face = cv2.resize(gray_face, (emotion_target_size)) except: continue rgb_face = preprocess_input(rgb_face, False) rgb_face = np.expand_dims(rgb_face, 0) gender_prediction = gender_classifier.predict(rgb_face) gender_label_arg = np.argmax(gender_prediction) gender_text = gender_labels[gender_label_arg] gray_face = preprocess_input(gray_face, True) gray_face = np.expand_dims(gray_face, 0) gray_face = np.expand_dims(gray_face, -1) emotion_label_arg = np.argmax( emotion_classifier.predict(gray_face)) emotion_text = emotion_labels[emotion_label_arg] results.append({ "coords": face_coordinates.tolist(), "gender": gender_text, "emotion": emotion_text }) except Exception as err: logging.error('Error in emotion gender processor: "{0}"'.format(err)) return results
def predict(image_path): # loading images start = time.time() rgb_image = load_image(image_path, grayscale=False) gray_image = load_image(image_path, grayscale=True) gray_image = np.squeeze(gray_image) gray_image = gray_image.astype('uint8') with graph.as_default(): faces = detect_faces(face_detection, gray_image) for face_coordinates in faces: x1, x2, y1, y2 = apply_offsets(face_coordinates, gender_offsets) rgb_face = rgb_image[y1:y2, x1:x2] x1, x2, y1, y2 = apply_offsets(face_coordinates, emotion_offsets) gray_face = gray_image[y1:y2, x1:x2] try: rgb_face = cv2.resize(rgb_face, (gender_target_size)) gray_face = cv2.resize(gray_face, (emotion_target_size)) except: continue rgb_face = preprocess_input(rgb_face, False) rgb_face = np.expand_dims(rgb_face, 0) gender_prediction = gender_classifier.predict(rgb_face) gender_label_arg = np.argmax(gender_prediction) gender_text = gender_labels[gender_label_arg] gray_face = preprocess_input(gray_face, True) gray_face = np.expand_dims(gray_face, 0) gray_face = np.expand_dims(gray_face, -1) emotion_label_arg = np.argmax(emotion_classifier.predict(gray_face)) emotion_text = emotion_labels[emotion_label_arg] if gender_text == gender_labels[0]: color = (0, 0, 255) else: color = (255, 0, 0) draw_bounding_box(face_coordinates, rgb_image, color) draw_text(face_coordinates, rgb_image, gender_text, color, 0, -10, 1, 2) draw_text(face_coordinates, rgb_image, emotion_text, color, 0, -40, 1, 2) bgr_image = cv2.cvtColor(rgb_image, cv2.COLOR_RGB2BGR) resized_image = cv2.resize(bgr_image, (1024, 769)) file_path = 'result/' + image_path.split('/')[-1] cv2.imwrite(file_path, resized_image) end = time.time() logging.info('[Time] for prediction(): {}'.format(end - start)) return file_path
def get_gender(self, image_path_, face_detection, emotion_classifier, gender_classifier): gender_labels = get_labels('imdb') # hyper-parameters for bounding boxes shape # change for variance gender_offsets = (30, 60) gender_offsets = (10, 10) emotion_offsets = (20, 40) emotion_offsets = (0, 0) # loading models # getting input model shapes for inference gender_target_size = gender_classifier.input_shape[1:3] # loading images rgb_image = load_image(image_path_, grayscale=False) gray_image = load_image(image_path_, grayscale=True) gray_image = np.squeeze(gray_image) gray_image = gray_image.astype('uint8') faces = detect_faces(face_detection, gray_image) for face_coordinates in faces: x1, x2, y1, y2 = apply_offsets(face_coordinates, emotion_offsets) gray_face = gray_image[y1:y2, x1:x2] rgb_face = rgb_image[y1:y2, x1:x2] try: rgb_face = cv2.resize(rgb_face, (gender_target_size)) gray_face = cv2.resize(gray_face, (emotion_target_size)) except: continue rgb_face = preprocess_input(rgb_face, False) rgb_face = np.expand_dims(rgb_face, 0) gender_prediction = gender_classifier.predict(rgb_face) gender_label_arg = np.argmax(gender_prediction) gender_text = gender_labels[gender_label_arg] gray_face = preprocess_input(gray_face, True) gray_face = np.expand_dims(gray_face, 0) gray_face = np.expand_dims(gray_face, -1) return gender_text
def emotion_classificator(image): detected_peoples = [] json_info = {} try: # parameters for loading data and images emotion_labels = get_labels('fer2013') # hyper-parameters for bounding boxes shape emotion_offsets = (20, 40) emotion_offsets = (0, 0) # getting input model shapes for inference emotion_target_size = _EMOTION_CLASSIFIER.input_shape[1:3] start_time_preprocess = datetime.now() gray_image = preprocess_image(image, grascale=True) faces = detect_faces(_FACE_DETECTION, gray_image) delta_preprocess = datetime.now() - start_time_preprocess print("delta for preprocess {0}".format( delta_preprocess.total_seconds() * 1000.0)) for face_coordinates in faces: x1, x2, y1, y2 = apply_offsets(face_coordinates, emotion_offsets) gray_face = gray_image[y1:y2, x1:x2] try: gray_face = cv2.resize(gray_face, (emotion_target_size)) except: continue start_time = datetime.now() with _GRAPH.as_default(): gray_face = preprocess_input(gray_face, True) gray_face = np.expand_dims(gray_face, 0) gray_face = np.expand_dims(gray_face, -1) emotion_prediction = _EMOTION_CLASSIFIER.predict(gray_face) emotion_label_arg = np.argmax(emotion_prediction) emotion_text = emotion_labels[emotion_label_arg] delta = datetime.now() - start_time print("Delta for emotion classificator {0}".format( delta.total_seconds() * 1000.0)) json_info['emotion'] = "{0}:{1}".format( emotion_text, emotion_prediction.flat[emotion_label_arg]) json_info['face_bound'] = list( map(lambda it: str(it), list(face_coordinates.astype(np.int).flat))) detected_peoples.append(json_info.copy()) except Exception as err: logging.error('Error in emotion processor: "{0}"'.format(err)) return detected_peoples
def emotionRecog(self, record=False): self.emotionRecord = np.zeros(7) time_start = time.time() time_cost = 0 while True: ret, frame = self.camera.read() grayFrame = cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY) # 首先检测人脸,返回的是框住人脸的矩形框 faces = self.faceCascade.detectMultiScale( grayFrame, scaleFactor=1.1, minNeighbors=5, minSize=(30, 30), flags=cv2.CASCADE_SCALE_IMAGE ) # 画出每一个人脸,提取出人脸所在区域 for (x, y, w, h) in faces: grayFace = grayFrame[y:y+h, x:x+w] try: grayFace = cv2.resize(grayFace, (self.emotionTargetSize)) except: continue grayFace = preprocess_input(grayFace, True) grayFace = np.expand_dims(grayFace, 0) grayFace = np.expand_dims(grayFace, -1) emotionPrediction = self.emotionClassifier.predict(grayFace) emotionProbability = np.max(emotionPrediction) emotionLabelArg = np.argmax(emotionPrediction) emotionText = self.emotionLabels[emotionLabelArg] if emotionText == 'angry': color = emotionProbability * np.asarray((255, 0, 0)) elif emotionText == 'sad': color = emotionProbability * np.asarray((0, 0, 255)) elif emotionText == 'happy': color = emotionProbability * np.asarray((255, 255, 0)) elif emotionText == 'surprise': color = emotionProbability * np.asarray((0, 255, 255)) else: color = emotionProbability * np.asarray((0, 255, 0)) # 标明心情 框出人脸 cv2.rectangle(frame, (x, y), (x+w, y+h), color, 2) if record: self.emotionRecord[emotionLabelArg] += 1 time_cost = time.time() - time_start cv2.putText(frame, emotionText+' %ds' % (5-time_cost+1), (x, y - 7), 3, 1.2, color, 2, cv2.LINE_AA) else: cv2.putText(frame, emotionText, (x, y - 7), 3, 1.2, color, 2, cv2.LINE_AA) if cv2.waitKey(85) & 0xff == ord('q') | self.runCommand != self.RUN_EMOTION_RECOG: break if time_cost > 5: break self.display(frame)
def recognition(f_2_s): # parameters for loading data and images #image_path = path_r image_path = image_handler(f_2_s) detection_model_path = '../trained_models/detection_models/haarcascade_frontalface_default.xml' emotion_model_path = '../trained_models/emotion_models/fer2013_mini_XCEPTION.102-0.66.hdf5' emotion_labels = get_labels('fer2013') font = cv2.FONT_HERSHEY_SIMPLEX # hyper-parameters for bounding boxes shape emotion_offsets = (20, 40) emotion_offsets = (0, 0) # loading models face_detection = load_detection_model(detection_model_path) emotion_classifier = load_model(emotion_model_path, compile=False) # getting input model shapes for inference emotion_target_size = emotion_classifier.input_shape[1:3] # loading images rgb_image = load_image(image_path, grayscale=False) gray_image = load_image(image_path, grayscale=True) gray_image = np.squeeze(gray_image) gray_image = gray_image.astype('uint8') faces = detect_faces(face_detection, gray_image) for face_coordinates in faces: x1, x2, y1, y2 = apply_offsets(face_coordinates, emotion_offsets) gray_face = gray_image[y1:y2, x1:x2] try: gray_face = cv2.resize(gray_face, (emotion_target_size)) except: continue gray_face = preprocess_input(gray_face, True) gray_face = np.expand_dims(gray_face, 0) gray_face = np.expand_dims(gray_face, -1) emotion_label_arg = np.argmax(emotion_classifier.predict(gray_face)) emotion_text = "" emotion_text = emotion_labels[emotion_label_arg] color = (255, 0, 0) # 감정 정보 글씨 빨간색, 사각형도 draw_bounding_box(face_coordinates, rgb_image, color) draw_text(face_coordinates, rgb_image, emotion_text, color, 0, -30, 1.5, 2) #if(emotion_text == ""): #recognition(f_2_s) bgr_image = cv2.cvtColor(rgb_image, cv2.COLOR_RGB2BGR) cv2.imwrite('../images/predicted_test_image.png', bgr_image) # 변수 활용 check_recoged_img('../images/predicted_test_image.png')
def final_ml_predict(bgr_image): gray_image = cv2.cvtColor(bgr_image, cv2.COLOR_BGR2GRAY) rgb_image = cv2.cvtColor(bgr_image, cv2.COLOR_BGR2RGB) faces = face_cascade.detectMultiScale(gray_image, scaleFactor=1.1, minNeighbors=5, minSize=(30, 30), flags=cv2.CASCADE_SCALE_IMAGE) for face_coordinates in faces: x1, x2, y1, y2 = apply_offsets(face_coordinates, emotion_offsets) gray_face = gray_image[y1:y2, x1:x2] try: gray_face = cv2.resize(gray_face, (emotion_target_size)) except: continue gray_face = preprocess_input(gray_face, True) gray_face = np.expand_dims(gray_face, 0) gray_face = np.expand_dims(gray_face, -1) with graph.as_default(): emotion_prediction = emotion_classifier.predict(gray_face) emotion_probability = np.max(emotion_prediction) emotion_label_arg = np.argmax(emotion_prediction) emotion_text = emotion_labels[emotion_label_arg] emotion_window.append(emotion_text) if len(emotion_window) > frame_window: emotion_window.pop(0) try: emotion_mode = mode(emotion_window) except: continue if emotion_text == 'angry': color = emotion_probability * np.asarray((255, 0, 0)) elif emotion_text == 'sad': color = emotion_probability * np.asarray((0, 0, 255)) elif emotion_text == 'happy': color = emotion_probability * np.asarray((255, 255, 0)) elif emotion_text == 'surprise': color = emotion_probability * np.asarray((0, 255, 255)) else: color = emotion_probability * np.asarray((0, 255, 0)) color = color.astype(int) color = color.tolist() draw_bounding_box(face_coordinates, rgb_image, color) draw_text(face_coordinates, rgb_image, emotion_mode, color, 0, -45, 1, 1) bgr_image = cv2.cvtColor(rgb_image, cv2.COLOR_RGB2BGR) return bgr_image
def emotionimg(img): # bgr_image = video_capture.read()[1] gray_face = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY) gray_face = cv2.resize(gray_face, (emotion_target_size)) gray_face = preprocess_input(gray_face, True) gray_face = np.expand_dims(gray_face, 0) gray_face = np.expand_dims(gray_face, -1) emotion_prediction = emotion_classifier.predict(gray_face) emotion_label_arg = np.argmax(emotion_prediction) emotion_text = emotion_labels[emotion_label_arg] return emotion_text, emotion_prediction
def detect_emotion(frame,model,emotion_classifier,emotion_target_size,face_cascade,frame_window,emotion_offsets,emotion_labels): gray_image = cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY) faces = face_cascade.detectMultiScale(gray_image, scaleFactor=1.1, minNeighbors=5,minSize=(30, 30), flags=cv2.CASCADE_SCALE_IMAGE) for face_coordinates in faces: x1, x2, y1, y2 = apply_offsets(face_coordinates, emotion_offsets) # gray_face = gray_image[y1:y2, x1:x2] # gray_c = cv2.cvtColor(gray_image[y1:y2, x1:x2], cv2.COLOR_BGR2GRAY) gray_c = gray_image[y1:y2, x1:x2] try: gray_face = cv2.resize(gray_c, (emotion_target_size)) except: continue gray_face = preprocess_input(gray_face, True) gray_face = np.expand_dims(gray_face, 0) gray_face = np.expand_dims(gray_face, -1) # print(gray_face) emotion_prediction = emotion_classifier.predict(gray_face) emotion_probability =round( np.max(emotion_prediction),2) emotion_label_arg = np.argmax(emotion_prediction) emotion_text = emotion_labels[emotion_label_arg] emotion_window.append(emotion_text) # print(emotion_text) # print(emotion_window) if len(emotion_window) > frame_window: emotion_window.pop(0) try: emotion_mode = mode(emotion_window) except: continue # if emotion_text == 'angry': # color = emotion_probability * np.asarray((255, 0, 0)) # elif emotion_text == 'sad': # color = emotion_probability * np.asarray((0, 0, 255)) # elif emotion_text == 'happy': # # print (emotion_probability) # color = emotion_probability * np.asarray((255, 255, 0)) # elif emotion_text == 'surprise': # color = emotion_probability * np.asarray((0, 255, 255)) # else: # color = emotion_probability * np.asarray((0, 255, 0)) # color = color.astype(int) # color = color.tolist() color=(255,255,255) draw_text(emotion_probability,face_coordinates, frame, emotion_mode,color, 0, -20, 0.5, 1)
def getEmotion(x1, x2, y1, y2, gray): gray_image = gray.copy() gray_face = gray_image[y1:y2, x1:x2] gray_face = cv2.resize(gray_face, (emotion_target_size)) gray_face = preprocess_input(gray_face, True) gray_face = np.expand_dims(gray_face, 0) gray_face = np.expand_dims(gray_face, -1) emotion_prediction = emotion_classifier.predict(gray_face) emotion_probability = np.max(emotion_prediction) emotion_label_arg = np.argmax(emotion_prediction) emotion_text = emotion_labels[emotion_label_arg] return emotion_text
def emotion(): detection_model_path = '../trained_models/detection_models/haarcascade_frontalface_default.xml' emotion_model_path = '../trained_models/emotion_models/fer2013_big_XCEPTION.54-0.66.hdf5' emotion_labels = get_labels('fer2013') # hyper-parameters for bounding boxes shape frame_window = 10 emotion_offsets = (20, 40) # loading models face_detection = load_detection_model(detection_model_path) emotion_classifier = load_model(emotion_model_path, compile=False) # getting input model shapes for inference emotion_target_size = emotion_classifier.input_shape[1:3] # starting lists for calculating modes emotion_window = [] emotion_count_array = [0 for x in range(7)]; # starting video streaming #cv2.namedWindow('window_frame') video_capture = cv2.VideoCapture(0) duration = 10; start = time.time(); while (time.time() - start) < duration: bgr_image = video_capture.read()[1]; gray_image = cv2.cvtColor(bgr_image, cv2.COLOR_BGR2GRAY) rgb_image = cv2.cvtColor(bgr_image, cv2.COLOR_BGR2RGB) faces = detect_faces(face_detection, gray_image) for face_coordinates in faces: x1, x2, y1, y2 = apply_offsets(face_coordinates, emotion_offsets) gray_face = gray_image[y1:y2, x1:x2] try: gray_face = cv2.resize(gray_face, (emotion_target_size)) except: continue gray_face = preprocess_input(gray_face, True) gray_face = np.expand_dims(gray_face, 0) gray_face = np.expand_dims(gray_face, -1) emotion_prediction = emotion_classifier.predict(gray_face) emotion_probability = np.max(emotion_prediction) emotion_label_arg = np.argmax(emotion_prediction) emotion_text = emotion_labels[emotion_label_arg] emotion_count_array[emotion_label_arg] = emotion_count_array[emotion_label_arg] + 1; chosen = np.argmax(emotion_count_array); return emotion_labels[chosen]
def analyse_image(image_path): print(image_path) print(emotion_labels) image = cv2.imread(image_path) gray_image = cv2.cvtColor(image, cv2.COLOR_BGR2GRAY) rgb_image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB) faces = face_cascade.detectMultiScale(gray_image, scaleFactor=1.3, minNeighbors=5, minSize=(30, 30), flags=cv2.CASCADE_SCALE_IMAGE) if len(faces) == 0: return {'error': 'No faces found!'} print('faces found:' + str(len(faces))) for face_coordinates in faces: try: x1, x2, y1, y2 = apply_offsets(face_coordinates, emotion_offsets) gray_face = gray_image[y1:y2, x1:x2] try: gray_face = cv2.resize(gray_face, (emotion_target_size)) except: continue gray_face = preprocess_input(gray_face, True) gray_face = np.expand_dims(gray_face, 0) gray_face = np.expand_dims(gray_face, -1) emotion_prediction = emotion_classifier.predict(gray_face) # custom_prediction = emotion_prediction[0] # prediction_dict = {} # for i in range(0, len(custom_prediction)): # prediction_dict[emotion_labels[i]] = custom_prediction[i] # emotion_probability = np.max(emotion_prediction) # emotion_label_arg = np.argmax(emotion_prediction) # emotion_text = emotion_labels[emotion_label_arg] if len(emotion_prediction) > 0: return {'result': ','.join(map(str, emotion_prediction[0]))} else: return {'error': 'no predictions!'} except: # danger stuff return {'error': 'unknown error'}
def gender_classificator(image): detected_peoples = [] json_info = {} try: # parameters for loading data and images gender_labels = get_labels('imdb') # hyper-parameters for bounding boxes shape gender_offsets = (30, 60) gender_offsets = (10, 10) # getting input model shapes for inference gender_target_size = _GENDER_CLASSIFIER.input_shape[1:3] rgb_image = preprocess_image(image) gray_image = preprocess_image(image, grascale=True) faces = detect_faces(_FACE_DETECTION, gray_image) for face_coordinates in faces: x1, x2, y1, y2 = apply_offsets(face_coordinates, gender_offsets) rgb_face = rgb_image[y1:y2, x1:x2] try: rgb_face = cv2.resize(rgb_face, (gender_target_size)) except: continue start_time = datetime.now() with _GRAPH.as_default(): rgb_face = preprocess_input(rgb_face, False) rgb_face = np.expand_dims(rgb_face, 0) gender_prediction = _GENDER_CLASSIFIER.predict(rgb_face) gender_label_arg = np.argmax(gender_prediction) gender_text = gender_labels[gender_label_arg] delta = datetime.now() - start_time print("Delta for gender classificator {0}".format( delta.total_seconds() * 1000.0)) json_info['gender'] = "{0}:{1}".format( gender_text, gender_prediction.flat[gender_label_arg]) json_info['face_bound'] = list( map(lambda it: str(it), list(face_coordinates.astype(np.int).flat))) detected_peoples.append(json_info.copy()) except Exception as err: logging.error('Error in gender processor: "{0}"'.format(err)) return detected_peoples
def process_image(image): try: # parameters for loading data and images detection_model_path = './trained_models/detection_models/haarcascade_frontalface_default.xml' emotion_model_path = './trained_models/emotion_models/fer2013_mini_XCEPTION.102-0.66.hdf5' emotion_labels = get_labels('fer2013') # hyper-parameters for bounding boxes shape emotion_offsets = (0, 0) # loading models face_detection = load_detection_model(detection_model_path) emotion_classifier = load_model(emotion_model_path, compile=False) # getting input model shapes for inference emotion_target_size = emotion_classifier.input_shape[1:3] # loading images image_array = np.fromstring(image, np.uint8) unchanged_image = cv2.imdecode(image_array, cv2.IMREAD_UNCHANGED) gray_image = cv2.cvtColor(unchanged_image, cv2.COLOR_BGR2GRAY) faces = detect_faces(face_detection, gray_image) detected_emotions = [] for face_coordinates in faces: x1, x2, y1, y2 = apply_offsets(face_coordinates, emotion_offsets) gray_face = gray_image[y1:y2, x1:x2] try: gray_face = cv2.resize(gray_face, emotion_target_size) except: continue gray_face = preprocess_input(gray_face, True) gray_face = np.expand_dims(gray_face, 0) gray_face = np.expand_dims(gray_face, -1) emotion_label_arg = np.argmax(emotion_classifier.predict(gray_face)) detected_emotions.append({ "coordinates": [str(y1), str(x2), str(y2), str(x1)], "emotion": emotion_labels[emotion_label_arg]}) return detected_emotions except Exception as err: logging.error('Error in emotion gender processor: "{0}"'.format(err))
datasets = ['fer2013'] for dataset_name in datasets: print('Training dataset:', dataset_name) # callbacks log_file_path = base_path + dataset_name + '_emotion_training.log' csv_logger = CSVLogger(log_file_path, append=False) early_stop = EarlyStopping('val_loss', patience=patience) reduce_lr = ReduceLROnPlateau('val_loss', factor=0.1, patience=int(patience/4), verbose=1) trained_models_path = base_path + dataset_name + '_mini_XCEPTION' model_names = trained_models_path + '.{epoch:02d}-{val_acc:.2f}.hdf5' model_checkpoint = ModelCheckpoint(model_names, 'val_loss', verbose=1, save_best_only=True) callbacks = [model_checkpoint, csv_logger, early_stop, reduce_lr] # loading dataset data_loader = DataManager(dataset_name, image_size=input_shape[:2]) faces, emotions = data_loader.get_data() faces = preprocess_input(faces) num_samples, num_classes = emotions.shape train_data, val_data = split_data(faces, emotions, validation_split) train_faces, train_emotions = train_data model.fit_generator(data_generator.flow(train_faces, train_emotions, batch_size), steps_per_epoch=len(train_faces) / batch_size, epochs=num_epochs, verbose=1, callbacks=callbacks, validation_data=val_data)
def callback(self,data): #///////////////////////////////////////////// #cv_image = bridge.imgmsg_to_cv2(image_msg, desired_encoding="passthrough") #cv_image = cv2.resize(cv_image, target_size) # resize image #np_image = np.asarray(cv_image) # read as np array #np_image = np.expand_dims(np_image, axis=0) # Add another dimension for tensorflow #np_image = np_image.astype(float) # preprocess needs float64 and img is uint8 #np_image = preprocess_input(np_image) # Regularize the data #///////////////////////////////////////////// if(not USE_LOCAL_CAMERA): try: frame = self.bridge.imgmsg_to_cv2(data, desired_encoding="passthrough") except CvBridgeError as e: print(e) # Capture frame-by-frame if(USE_LOCAL_CAMERA): ret, frame1 = self.video_capture.read() #///////////////////////////////////////////// #print"--------" #print('input_msg height : {}'.format(frame.height)) #print('input_msg width : {}'.format(frame.width)) #print('input_msg step : {}'.format(frame.step)) #print('input_msg encoding: {}'.format(frame.encoding)) #print('output dtype : {}'.format(frame.dtype)) #print('output shape : {}'.format(frame.shape)) #print"--------" gray_image = cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY) rgb_image = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB) faces = detect_faces(self.face_detection, gray_image) for face_coordinates in faces: print "inside the for loop!" print face_coordinates x1, x2, y1, y2 = apply_offsets(face_coordinates, self.emotion_offsets) gray_face = gray_image[y1:y2, x1:x2] #print len(frame) #print gray_face.size #print gray_face.shape #print gray_face.dtype try: gray_face = cv2.resize(gray_face, (self.emotion_target_size)) except: continue gray_face = preprocess_input(gray_face, True) gray_face = np.expand_dims(gray_face, 0) gray_face = np.expand_dims(gray_face, -1) #print"************" #print('gray_face dtype : {}'.format(gray_face.dtype)) #print('gray_face shape : {}'.format(gray_face.shape)) #print('gray_face size : {}'.format(gray_face.size)) #print"************" ## This is a workaround for asynchronous execution using TF and ROS # https://github.com/keras-team/keras/issues/2397 # http://projectsfromtech.blogspot.com/2017/10/visual-object-recognition-in-ros-using.html #global self.graph with self.graph.as_default(): emotion_prediction = self.emotion_classifier.predict(gray_face) emotion_probability = np.max(emotion_prediction) emotion_label_arg = np.argmax(emotion_prediction) emotion_text = self.emotion_labels[emotion_label_arg] print emotion_text print(emotion_probability) print('%') self.emotion_window.append(emotion_text) #self.emotion_msg.data = emotion_text #self.emotion_publisher.publish(emotion_msg) #self.speech_msg.data = 'I see that you are ' + emotion_text #self.speech_publisher.publish(speech_msg) if len(self.emotion_window) > self.frame_window: self.emotion_window.pop(0) try: emotion_mode = mode(self.emotion_window) except: continue if emotion_text == 'angry': color = emotion_probability * np.asarray((255, 0, 0)) elif emotion_text == 'sad': color = emotion_probability * np.asarray((0, 0, 255)) elif emotion_text == 'happy': color = emotion_probability * np.asarray((255, 255, 0)) elif emotion_text == 'surprise': color = emotion_probability * np.asarray((0, 255, 255)) else: color = emotion_probability * np.asarray((0, 255, 0)) color = color.astype(int) color = color.tolist() draw_bounding_box(face_coordinates, rgb_image, color) draw_text(face_coordinates, rgb_image, emotion_mode, color, 0, -45, 1, 1) try: self.image_pub.publish(self.bridge.cv2_to_imgmsg(rgb_image, "bgr8")) except CvBridgeError as e: print(e)
def process_image(image): try: # parameters for loading data and images detection_model_path = './trained_models/detection_models/haarcascade_frontalface_default.xml' emotion_model_path = './trained_models/emotion_models/fer2013_mini_XCEPTION.102-0.66.hdf5' gender_model_path = './trained_models/gender_models/simple_CNN.81-0.96.hdf5' emotion_labels = get_labels('fer2013') gender_labels = get_labels('imdb') font = cv2.FONT_HERSHEY_SIMPLEX # hyper-parameters for bounding boxes shape gender_offsets = (30, 60) gender_offsets = (10, 10) emotion_offsets = (20, 40) emotion_offsets = (0, 0) # loading models face_detection = load_detection_model(detection_model_path) emotion_classifier = load_model(emotion_model_path, compile=False) gender_classifier = load_model(gender_model_path, compile=False) # getting input model shapes for inference emotion_target_size = emotion_classifier.input_shape[1:3] gender_target_size = gender_classifier.input_shape[1:3] # loading images image_array = np.fromstring(image, np.uint8) unchanged_image = cv2.imdecode(image_array, cv2.IMREAD_UNCHANGED) rgb_image = cv2.cvtColor(unchanged_image, cv2.COLOR_BGR2RGB) gray_image = cv2.cvtColor(unchanged_image, cv2.COLOR_BGR2GRAY) faces = detect_faces(face_detection, gray_image) for face_coordinates in faces: x1, x2, y1, y2 = apply_offsets(face_coordinates, gender_offsets) rgb_face = rgb_image[y1:y2, x1:x2] x1, x2, y1, y2 = apply_offsets(face_coordinates, emotion_offsets) gray_face = gray_image[y1:y2, x1:x2] try: rgb_face = cv2.resize(rgb_face, (gender_target_size)) gray_face = cv2.resize(gray_face, (emotion_target_size)) except: continue rgb_face = preprocess_input(rgb_face, False) rgb_face = np.expand_dims(rgb_face, 0) gender_prediction = gender_classifier.predict(rgb_face) gender_label_arg = np.argmax(gender_prediction) gender_text = gender_labels[gender_label_arg] gray_face = preprocess_input(gray_face, True) gray_face = np.expand_dims(gray_face, 0) gray_face = np.expand_dims(gray_face, -1) emotion_label_arg = np.argmax(emotion_classifier.predict(gray_face)) emotion_text = emotion_labels[emotion_label_arg] if gender_text == gender_labels[0]: color = (0, 0, 255) else: color = (255, 0, 0) draw_bounding_box(face_coordinates, rgb_image, color) draw_text(face_coordinates, rgb_image, gender_text, color, 0, -20, 1, 2) draw_text(face_coordinates, rgb_image, emotion_text, color, 0, -50, 1, 2) except Exception as err: logging.error('Error in emotion gender processor: "{0}"'.format(err)) bgr_image = cv2.cvtColor(rgb_image, cv2.COLOR_RGB2BGR) dirname = 'result' if not os.path.exists(dirname): os.mkdir(dirname) cv2.imwrite(os.path.join(dirname, 'predicted_image.png'), bgr_image)
def callback(self,data): try: bgr_image = self.bridge.imgmsg_to_cv2(data, "bgr8") cv2.imshow("Image window", bgr_image) except CvBridgeError as e: print(e) ######################### Start the image processing gray_image = cv2.cvtColor(bgr_image, cv2.COLOR_BGR2GRAY) rgb_image = cv2.cvtColor(bgr_image, cv2.COLOR_BGR2RGB) faces = detect_faces(self.face_detection, gray_image) for face_coordinates in faces: x1, x2, y1, y2 = apply_offsets(face_coordinates, self.emotion_offsets) gray_face = gray_image[y1:y2, x1:x2] try: gray_face = cv2.resize(gray_face, (self.emotion_target_size)) except: continue gray_face = preprocess_input(gray_face, True) gray_face = np.expand_dims(gray_face, 0) gray_face = np.expand_dims(gray_face, -1) print "start" print len(gray_face) print gray_face print "end" emotion_prediction = self.emotion_classifier.predict(gray_face) emotion_probability = np.max(emotion_prediction) emotion_label_arg = np.argmax(emotion_prediction) emotion_text = emotion_labels[emotion_label_arg] print emotion_text print(emotion_probability) print('%') self.emotion_window.append(emotion_text) if len(self.emotion_window) > self.frame_window: self.emotion_window.pop(0) try: emotion_mode = mode(self.emotion_window) except: continue if emotion_text == 'angry': color = emotion_probability * np.asarray((255, 0, 0)) elif emotion_text == 'sad': color = emotion_probability * np.asarray((0, 0, 255)) elif emotion_text == 'happy': color = emotion_probability * np.asarray((255, 255, 0)) elif emotion_text == 'surprise': color = emotion_probability * np.asarray((0, 255, 255)) else: color = emotion_probability * np.asarray((0, 255, 0)) color = color.astype(int) color = color.tolist() draw_bounding_box(face_coordinates, rgb_image, color) draw_text(face_coordinates, rgb_image, emotion_mode, color, 0, -45, 1, 1) bgr_image = cv2.cvtColor(rgb_image, cv2.COLOR_RGB2BGR) ######################### End the image processing cv2.imshow("Image window", bgr_image) cv2.waitKey(3) try: self.image_pub.publish(self.bridge.cv2_to_imgmsg(bgr_image, "bgr8")) except CvBridgeError as e: print(e)
def emotion_identify(img_url): # parameters for loading data and images detection_model_path = 'C:/Users/Admin/PycharmProjects/Emotion_Detection/trained_models/detection_models/haarcascade_frontalface_default.xml' emotion_model_path = 'C:/Users/Admin/PycharmProjects/Emotion_Detection/trained_models/emotion_models/fer2013_mini_XCEPTION.102-0.66.hdf5' gender_model_path = 'C:/Users/Admin/PycharmProjects/Emotion_Detection/trained_models/gender_models/simple_CNN.81-0.96.hdf5' emotion_labels = get_labels('fer2013') gender_labels = get_labels('imdb') font = cv2.FONT_HERSHEY_SIMPLEX # hyper-parameters for bounding boxes shape gender_offsets = (30, 60) gender_offsets = (10, 10) emotion_offsets = (20, 40) emotion_offsets = (0, 0) # loading models face_detection = load_detection_model(detection_model_path) emotion_classifier = load_model(emotion_model_path, compile=False) gender_classifier = load_model(gender_model_path, compile=False) # getting input model shapes for inference emotion_target_size = emotion_classifier.input_shape[1:3] gender_target_size = gender_classifier.input_shape[1:3] # loading images image_path = img_url rgb_image = load_image(image_path, grayscale=False) gray_image = load_image(image_path, grayscale=True) gray_image = np.squeeze(gray_image) gray_image = gray_image.astype('uint8') faces = detect_faces(face_detection, gray_image) if len(faces) == 0: print("No face") K.clear_session() return False emotions = collections.defaultdict(int) for face_coordinates in faces: x1, x2, y1, y2 = apply_offsets(face_coordinates, gender_offsets) rgb_face = rgb_image[y1:y2, x1:x2] x1, x2, y1, y2 = apply_offsets(face_coordinates, emotion_offsets) gray_face = gray_image[y1:y2, x1:x2] try: rgb_face = cv2.resize(rgb_face, (gender_target_size)) gray_face = cv2.resize(gray_face, (emotion_target_size)) except: continue rgb_face = preprocess_input(rgb_face, False) rgb_face = np.expand_dims(rgb_face, 0) gender_prediction = gender_classifier.predict(rgb_face) gender_label_arg = np.argmax(gender_prediction) gender_text = gender_labels[gender_label_arg] gray_face = preprocess_input(gray_face, True) gray_face = np.expand_dims(gray_face, 0) gray_face = np.expand_dims(gray_face, -1) emotion_label_arg = np.argmax(emotion_classifier.predict(gray_face)) emotion_text = emotion_labels[emotion_label_arg] emotions[emotion_text] += 1 if gender_text == gender_labels[0]: color = (0, 0, 255) else: color = (255, 0, 0) draw_bounding_box(face_coordinates, rgb_image, color) draw_text(face_coordinates, rgb_image, gender_text, color, 0, -20, 1, 2) draw_text(face_coordinates, rgb_image, emotion_text, color, 0, -50, 1, 2) max_num = 0 max_emotion = None for key, value in emotions.items(): if value > max_num: max_num = value max_emotion = key print("The emotion of this picture is: ", max_emotion) bgr_image = cv2.cvtColor(rgb_image, cv2.COLOR_RGB2BGR) cv2.imwrite('./result_images/predicted_test_image.png', bgr_image) K.clear_session() return max_emotion
while True: bgr_image = video_capture.read()[1] gray_image = cv2.cvtColor(bgr_image, cv2.COLOR_BGR2GRAY) rgb_image = cv2.cvtColor(bgr_image, cv2.COLOR_BGR2RGB) faces = detect_faces(face_detection, gray_image) for face_coordinates in faces: x1, x2, y1, y2 = apply_offsets(face_coordinates, offsets) gray_face = gray_image[y1:y2, x1:x2] try: gray_face = cv2.resize(gray_face, (target_size)) except: continue gray_face = preprocess_input(gray_face, True) gray_face = np.expand_dims(gray_face, 0) gray_face = np.expand_dims(gray_face, -1) guided_gradCAM = calculate_guided_gradient_CAM(gray_face, gradient_function, saliency_function) guided_gradCAM = cv2.resize(guided_gradCAM, (x2-x1, y2-y1)) try: rgb_guided_gradCAM = np.repeat(guided_gradCAM[:, :, np.newaxis], 3, axis=2) rgb_image[y1:y2, x1:x2, :] = rgb_guided_gradCAM except: continue draw_bounding_box((x1, y1, x2 - x1, y2 - y1), rgb_image, color) bgr_image = cv2.cvtColor(rgb_image, cv2.COLOR_RGB2BGR) try: cv2.imshow('window_frame', bgr_image)
faces = detect_faces(face_detection, gray_image) for face_coordinates in faces: x1, x2, y1, y2 = apply_offsets(face_coordinates, gender_offsets) rgb_face = rgb_image[y1:y2, x1:x2] x1, x2, y1, y2 = apply_offsets(face_coordinates, emotion_offsets) gray_face = gray_image[y1:y2, x1:x2] try: rgb_face = cv2.resize(rgb_face, (gender_target_size)) gray_face = cv2.resize(gray_face, (emotion_target_size)) except: continue rgb_face = preprocess_input(rgb_face, False) rgb_face = np.expand_dims(rgb_face, 0) gender_prediction = gender_classifier.predict(rgb_face) gender_label_arg = np.argmax(gender_prediction) gender_text = gender_labels[gender_label_arg] gray_face = preprocess_input(gray_face, True) gray_face = np.expand_dims(gray_face, 0) gray_face = np.expand_dims(gray_face, -1) emotion_label_arg = np.argmax(emotion_classifier.predict(gray_face)) emotion_text = emotion_labels[emotion_label_arg] if gender_text == gender_labels[0]: color = (0, 0, 255) else: color = (255, 0, 0)