Beispiel #1
0
def getFaceDetails(image_np):
    gray_image = cv2.cvtColor(image_np, cv2.COLOR_RGB2GRAY) # the model requires a grayscale image

    faces = detect_faces(face_detection_model, gray_image) # pass in the grayscale image

    if len(faces) > 0: # make sure a face was found
        draw_bounding_box(faces[0], image_np) # draws a square around the face, just for testing purposes
        
        # doing preprocessing on grayscale image, found from the model creator's example
        x1, x2, y1, y2 = apply_offsets(faces[0], emotion_offsets) 
        gray_face = gray_image[y1:y2, x1:x2]
        gray_face = cv2.resize(gray_face, (emotion_target_size))
        gray_face = preprocess_input(gray_face, True)
        gray_face = np.expand_dims(gray_face, 0)
        gray_face = np.expand_dims(gray_face, -1)
        
        with graph.as_default(): # to prevent graph not found error
            emotion_prediction = emotion_classifier.predict(gray_face) # predict the emotion

        #emotion_probability = np.max(emotion_prediction) # uncomment this line to see the accuracy of the emotion
        
        emotion_label_arg = np.argmax(emotion_prediction)
    
        face_details = [emotion_labels[emotion_label_arg], int(faces[0][0]), int(faces[0][1]), int(faces[0][2]), int(faces[0][3])] # structure to return emotion and face coords
    else:
        face_details = [None, None, None, None, None] # in case no face was found

    thread_storage['face'] = face_details # instead of returning, put it into dictionary
Beispiel #2
0
def detection(image_path):

    # parameters for loading data and images
    detection_model_path = 'C:\\Users\\l1f15bscs0049\\Desktop\\haarcascade_frontalface_default.xml'

    font = cv2.FONT_HERSHEY_SIMPLEX

    # hyper-parameters for bounding boxes shape
    gender_offsets = (30, 60)
    gender_offsets = (10, 10)

    # loading models
    face_detection = load_detection_model(detection_model_path)

    # loading images
    rgb_image = load_image(image_path, grayscale=False)
    gray_image = load_image(image_path, grayscale=True)
    gray_image = np.squeeze(gray_image)
    gray_image = gray_image.astype('uint8')
    count1 = 0
    faces = detect_faces(face_detection, gray_image)
    for face_coordinates in faces:
        x1, x2, y1, y2 = apply_offsets(face_coordinates, gender_offsets)
        rgb_face = rgb_image[y1:y2, x1:x2]

        rgb_face = preprocess_input(rgb_face, False)
        rgb_face = np.expand_dims(rgb_face, 0)

        color = (0, 0, 255)
        draw_bounding_box(face_coordinates, rgb_image, color)
        count1 = count1 + 1

    bgr_image = cv2.cvtColor(rgb_image, cv2.COLOR_RGB2BGR)
    #cv2.imwrite('C:\\Users\\l1f15bscs0049\\Desktop\\test_cases\\testt2.png', bgr_image)
    return count1
Beispiel #3
0
def process_image(image):

    try:
        # parameters for loading data and images
        detection_model_path = './trained_models/detection_models/haarcascade_frontalface_default.xml'
        emotion_model_path = './trained_models/emotion_models/fer2013_35a93d1a30cf52b8722837f3e096b7b1006949325de8d00f26595d5a418241d3.hdf5'
        emotion_labels = get_labels('fer2013')
        font = cv2.FONT_HERSHEY_SIMPLEX

        # hyper-parameters for bounding boxes shape
        emotion_offsets = (20, 40)
        emotion_offsets = (0, 0)

        # loading models
        face_detection = load_detection_model(detection_model_path)
        emotion_classifier = load_model(emotion_model_path, compile=False)

        # getting input model shapes for inference
        emotion_target_size = emotion_classifier.input_shape[1:3]

        # loading images
        image_array = np.fromstring(image, np.uint8)
        unchanged_image = cv2.imdecode(image_array, cv2.IMREAD_UNCHANGED)

        rgb_image = cv2.cvtColor(unchanged_image, cv2.COLOR_BGR2RGB)
        gray_image = cv2.cvtColor(unchanged_image, cv2.COLOR_BGR2GRAY)

        faces = detect_faces(face_detection, gray_image)
        for face_coordinates in faces:
            x1, x2, y1, y2 = apply_offsets(face_coordinates, emotion_offsets)
            gray_face = gray_image[y1:y2, x1:x2]

            try:
                gray_face = cv2.resize(gray_face, (emotion_target_size))
            except:
                continue

            rgb_face = preprocess_input(rgb_face, False)
            rgb_face = np.expand_dims(rgb_face, 0)

            gray_face = preprocess_input(gray_face, True)
            gray_face = np.expand_dims(gray_face, 0)
            gray_face = np.expand_dims(gray_face, -1)
            emotion_label_arg = np.argmax(emotion_classifier.predict(gray_face))
            emotion_text = emotion_labels[emotion_label_arg]

            color = (255, 0, 0)

            draw_bounding_box(face_coordinates, rgb_image, color)
            draw_text(face_coordinates, rgb_image, emotion_text, color, 0, -50, 1, 2)
    except Exception as err:
        logging.error('Error in emotion gender processor: "{0}"'.format(err))

    bgr_image = cv2.cvtColor(rgb_image, cv2.COLOR_RGB2BGR)

    dirname = 'result'
    if not os.path.exists(dirname):
        os.mkdir(dirname)

    cv2.imwrite(os.path.join(dirname, 'predicted_image.png'), bgr_image)
def final_ml_predict(bgr_image):
    gray_image = cv2.cvtColor(bgr_image, cv2.COLOR_BGR2GRAY)
    rgb_image = cv2.cvtColor(bgr_image, cv2.COLOR_BGR2RGB)

    faces = face_cascade.detectMultiScale(gray_image,
                                          scaleFactor=1.1,
                                          minNeighbors=5,
                                          minSize=(30, 30),
                                          flags=cv2.CASCADE_SCALE_IMAGE)

    for face_coordinates in faces:

        x1, x2, y1, y2 = apply_offsets(face_coordinates, emotion_offsets)
        gray_face = gray_image[y1:y2, x1:x2]
        try:
            gray_face = cv2.resize(gray_face, (emotion_target_size))
        except:
            continue

        gray_face = preprocess_input(gray_face, True)
        gray_face = np.expand_dims(gray_face, 0)
        gray_face = np.expand_dims(gray_face, -1)
        with graph.as_default():
            emotion_prediction = emotion_classifier.predict(gray_face)
        emotion_probability = np.max(emotion_prediction)
        emotion_label_arg = np.argmax(emotion_prediction)
        emotion_text = emotion_labels[emotion_label_arg]
        emotion_window.append(emotion_text)

        if len(emotion_window) > frame_window:
            emotion_window.pop(0)
        try:
            emotion_mode = mode(emotion_window)
        except:
            continue

        if emotion_text == 'angry':
            color = emotion_probability * np.asarray((255, 0, 0))
        elif emotion_text == 'sad':
            color = emotion_probability * np.asarray((0, 0, 255))
        elif emotion_text == 'happy':
            color = emotion_probability * np.asarray((255, 255, 0))
        elif emotion_text == 'surprise':
            color = emotion_probability * np.asarray((0, 255, 255))
        else:
            color = emotion_probability * np.asarray((0, 255, 0))

        color = color.astype(int)
        color = color.tolist()

        draw_bounding_box(face_coordinates, rgb_image, color)
        draw_text(face_coordinates, rgb_image, emotion_mode, color, 0, -45, 1,
                  1)

    bgr_image = cv2.cvtColor(rgb_image, cv2.COLOR_RGB2BGR)
    return bgr_image
Beispiel #5
0
def recognition(f_2_s):

    # parameters for loading data and images
    #image_path = path_r
    image_path = image_handler(f_2_s)
    detection_model_path = '../trained_models/detection_models/haarcascade_frontalface_default.xml'
    emotion_model_path = '../trained_models/emotion_models/fer2013_mini_XCEPTION.102-0.66.hdf5'
    emotion_labels = get_labels('fer2013')
    font = cv2.FONT_HERSHEY_SIMPLEX

    # hyper-parameters for bounding boxes shape
    emotion_offsets = (20, 40)
    emotion_offsets = (0, 0)

    # loading models
    face_detection = load_detection_model(detection_model_path)
    emotion_classifier = load_model(emotion_model_path, compile=False)

    # getting input model shapes for inference
    emotion_target_size = emotion_classifier.input_shape[1:3]

    # loading images
    rgb_image = load_image(image_path, grayscale=False)
    gray_image = load_image(image_path, grayscale=True)
    gray_image = np.squeeze(gray_image)
    gray_image = gray_image.astype('uint8')

    faces = detect_faces(face_detection, gray_image)
    for face_coordinates in faces:

        x1, x2, y1, y2 = apply_offsets(face_coordinates, emotion_offsets)
        gray_face = gray_image[y1:y2, x1:x2]

        try:
            gray_face = cv2.resize(gray_face, (emotion_target_size))
        except:
            continue

        gray_face = preprocess_input(gray_face, True)
        gray_face = np.expand_dims(gray_face, 0)
        gray_face = np.expand_dims(gray_face, -1)
        emotion_label_arg = np.argmax(emotion_classifier.predict(gray_face))
        emotion_text = ""
        emotion_text = emotion_labels[emotion_label_arg]

        color = (255, 0, 0)  # 감정 정보 글씨 빨간색, 사각형도

        draw_bounding_box(face_coordinates, rgb_image, color)
        draw_text(face_coordinates, rgb_image, emotion_text, color, 0, -30,
                  1.5, 2)
    #if(emotion_text == ""):
    #recognition(f_2_s)

    bgr_image = cv2.cvtColor(rgb_image, cv2.COLOR_RGB2BGR)
    cv2.imwrite('../images/predicted_test_image.png', bgr_image)  # 변수 활용
    check_recoged_img('../images/predicted_test_image.png')
Beispiel #6
0
def classify_frame(bgr_image):
    # bgr_image = video_capture.read()[1]
    gray_image = cv2.cvtColor(bgr_image, cv2.COLOR_BGR2GRAY)
    rgb = cv2.cvtColor(bgr_image, cv2.COLOR_BGR2RGB)

    max_size = 512 

    aspect_ratio = rgb.shape[1] / rgb.shape[0]
    scaling_factor = max([rgb.shape[1], rgb.shape[0]]) / max_size

    new_width = int(rgb.shape[1] / scaling_factor)
    new_height = int(rgb.shape[0] / scaling_factor)

    rgb_image = cv2.resize(rgb, (new_width, new_height))

    faces = detect_faces(face_detection, rgb_image)

    for face_coordinates in faces:

        x1, x2, y1, y2 = apply_offsets(face_coordinates, gender_offsets)
        rgb_face = rgb_image[y1:y2, x1:x2]

        gray_face = gray_image[y1:y2, x1:x2]
        try:
            rgb_face = cv2.resize(rgb_face, (gender_target_size))
        except:
            continue
        gray_face = preprocess_input(gray_face, False)
        gray_face = np.expand_dims(gray_face, 0)
        gray_face = np.expand_dims(gray_face, -1)

        rgb_face = np.expand_dims(rgb_face, 0)
        rgb_face = preprocess_input(rgb_face, False)
        gender_prediction = gender_classifier.predict(rgb_face)
        gender_label_arg = np.argmax(gender_prediction)
        gender_text = gender_labels[gender_label_arg]
        gender_window.append(gender_text)

        if len(gender_window) > frame_window:
            gender_window.pop(0)
        try:
            gender_mode = mode(gender_window)
        except:
            continue

        if gender_text == gender_labels[0]:
            color = (0, 0, 255)
        else:
            color = (255, 0, 0)

        draw_bounding_box(face_coordinates, rgb_image, color)
        draw_text(face_coordinates, rgb_image, gender_mode,
                  color, 0, -20, 1, 1)
    
    bgr_image = cv2.cvtColor(rgb_image, cv2.COLOR_RGB2BGR)
    cv2.imshow('window_frame', bgr_image)
Beispiel #7
0
    def select_mode(self):
        """ Select a mode: Easy or Hard.

        """
        self.tickcount + 1

        if self.raspberry:
            self.tickcount += 1
        bgr_image = self.capture_frame()
        # Draw "Easy" and "Hard".
        # bgr_image = self.overlayUI(bgr_image)
        easy_coord = (self.screenwidth // 8, (self.screenheight * 3) // 4)
        draw_text(easy_coord, bgr_image, "Easy", font_scale=3)
        hard_coord = (self.screenwidth // 2, (self.screenheight * 3) // 4)
        draw_text(hard_coord, bgr_image, "Hard", font_scale=3)

        # Listen for mode selection.
        if self.currPosX and self.currPosX < self.screenwidth / 2:
            cv2.rectangle(self.overlay, (0, 0),
                          (self.screenwidth // 2, int(self.screenheight)),
                          (211, 211, 211), -1)
        else:
            cv2.rectangle(self.overlay, (self.screenwidth // 2, 0),
                          (self.screenwidth, self.screenheight),
                          (211, 211, 211), -1)
        if self.click_point_x:  # If user clicks left mouse button.
            # OPTIONAL: Positional mode selection
            # self.easy_mode = True if self.click_point_x < self.screenwidth / 2
            # else False
            self.easy_mode = True
            self.tickcount = 0
            self.curr_level = 1
            self.click_point_x = None
            self.click_point_right_x = None
        if self.click_point_right_x:
            self.easy_mode = False
            self.tickcount = 0
            self.curr_level = 1
            self.click_point_x = None
            self.click_point_right_x = None

        # Draw faces.
        gray_image = cv2.cvtColor(bgr_image, cv2.COLOR_BGR2GRAY)
        faces = detect_faces(face_detection, gray_image)
        cv2.addWeighted(self.overlay, OPACITY, bgr_image, 1 - OPACITY, 0,
                        bgr_image)

        if self.debug:
            for face in faces:
                draw_bounding_box(face, bgr_image, (255, 0, 0))
        # Draw Christmas logo.
        self.draw_hats(bgr_image, faces)
        self.draw_christmas_logo(bgr_image)  # Only for christmas
        # Show image.
        cv2.imshow('PartyPi', bgr_image)
Beispiel #8
0
def get_face(frame):
    detection_model_path = './face.xml'
    face_detection = load_detection_model(detection_model_path)
    try:
        gray_image = cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY)
        faces = detect_faces(face_detection, gray_image)
    except Exception as e:
        print("Exception:", e)
        return frame
    for face in faces:
        draw_bounding_box(face, frame, (255, 0, 0))
    return frame
def run_classify(image_path):
    emotion_text = ''
    gender_text = ''
    # loading images
    rgb_image = load_image(image_path, grayscale=False)
    gray_image = load_image(image_path, grayscale=True)
    gray_image = np.squeeze(gray_image)
    gray_image = gray_image.astype('uint8')

    faces = detect_faces(face_detection, gray_image)
    for face_coordinates in faces:
        x1, x2, y1, y2 = apply_offsets(face_coordinates, gender_offsets)
        rgb_face = rgb_image[y1:y2, x1:x2]

        x1, x2, y1, y2 = apply_offsets(face_coordinates, emotion_offsets)
        gray_face = gray_image[y1:y2, x1:x2]

        try:
            rgb_face = cv2.resize(rgb_face, (gender_target_size))
            gray_face = cv2.resize(gray_face, (emotion_target_size))
        except:
            continue

        rgb_face = preprocess_input(rgb_face, False)
        rgb_face = np.expand_dims(rgb_face, 0)
        gender_prediction = gender_classifier.predict(rgb_face)
        gender_label_arg = np.argmax(gender_prediction)
        gender_text = gender_labels[gender_label_arg]

        gray_face = preprocess_input(gray_face, True)
        gray_face = np.expand_dims(gray_face, 0)
        gray_face = np.expand_dims(gray_face, -1)
        emotion_label_arg = np.argmax(emotion_classifier.predict(gray_face))
        emotion_text = emotion_labels[emotion_label_arg]

        if gender_text == gender_labels[0]:
            color = (0, 0, 255)
        else:
            color = (255, 0, 0)

        draw_bounding_box(face_coordinates, rgb_image, color)
        draw_text(face_coordinates, rgb_image, gender_text, color, 0, -20, 1,
                  2)
        draw_text(face_coordinates, rgb_image, emotion_text, color, 0, -50, 1,
                  2)

    bgr_image = cv2.cvtColor(rgb_image, cv2.COLOR_RGB2BGR)
    cv2.imwrite(os.path.join(dir, '../images/predicted_test_image.png'),
                bgr_image)

    return ([gender_text, emotion_text])
    def emo_Detect(self, bgr_image, face_coordinates):

        gray_image = cv2.cvtColor(bgr_image, cv2.COLOR_BGR2GRAY)

        x1, y1, x2, y2 = face_coordinates
        face_coordinates = [int(x1), int(y1), int(x2 - x1), int(y2 - y1)]
        x1, x2, y1, y2 = apply_offsets(face_coordinates, self.emotion_offsets)
        gray_face = gray_image[y1:y2, x1:x2]

        gray_face = cv2.resize(gray_face, (self.emotion_target_size))

        gray_face = preprocess_input(gray_face, True)
        gray_face = np.expand_dims(gray_face, 0)
        gray_face = np.expand_dims(gray_face, -1)
        emotion_prediction = self.emotion_classifier.predict(gray_face)
        for idx, probability in enumerate(emotion_prediction[0]):
            print(self.emotion_labels[idx], probability)
        emotion_probability = np.max(emotion_prediction)
        emotion_label_arg = np.argmax(emotion_prediction)
        emotion_text = self.emotion_labels[emotion_label_arg]
        self.emotion_window.append(emotion_text)

        if len(self.emotion_window) > self.frame_window:
            self.emotion_window.pop(0)
        try:
            emotion_mode = mode(self.emotion_window)
            print("mode is " + emotion_mode)
        except:
            pass

        if emotion_text == 'angry':
            color = emotion_probability * np.asarray((0, 0, 255))
        elif emotion_text == 'sad':
            color = emotion_probability * np.asarray((255, 0, 0))
        elif emotion_text == 'happy':
            color = emotion_probability * np.asarray((0, 255, 255))
        elif emotion_text == 'surprise':
            color = emotion_probability * np.asarray((255, 255, 0))
        else:
            color = emotion_probability * np.asarray((0, 255, 0))

        color = color.astype(int)
        color = color.tolist()

        draw_bounding_box(face_coordinates, bgr_image, color)
        draw_text(face_coordinates, bgr_image, emotion_text, color, 0, -45, 1,
                  1)

        return bgr_image, emotion_prediction[0]
Beispiel #11
0
def getFaceEmotion(image_path):
    # parameters for loading data and images
    base = 'C://Users/lenovo/Desktop/moodify/components/emotion-classification'
    detection_model_path = base + '/trained_models/detection_models/haarcascade_frontalface_default.xml'
    emotion_model_path = base + '/trained_models/emotion_models/fer2013_denseNet.59-0.68.hdf5'
    emotion_labels = get_labels('fer2013')
    font = cv2.FONT_HERSHEY_SIMPLEX

    # hyper-parameters for bounding boxes shape
    emotion_offsets = (0, 0)

    # loading models
    face_detection = load_detection_model(detection_model_path)
    emotion_classifier = load_model(emotion_model_path, compile=False)

    # getting input model shapes for inference
    emotion_target_size = emotion_classifier.input_shape[1:3]

    # loading images
    rgb_image = load_image(image_path, grayscale=False)
    gray_image = load_image(image_path, grayscale=True)
    gray_image = np.squeeze(gray_image)
    gray_image = gray_image.astype('uint8')

    faces = detect_faces(face_detection, gray_image)
    for face_coordinates in faces:
        x1, x2, y1, y2 = apply_offsets(face_coordinates, emotion_offsets)
        gray_face = gray_image[y1:y2, x1:x2]

        try:
            gray_face = cv2.resize(gray_face, (emotion_target_size))
        except:
            continue

        gray_face = preprocess_input(gray_face, True)
        gray_face = np.expand_dims(gray_face, 0)
        gray_face = np.expand_dims(gray_face, -1)
        emotion_label_arg = np.argmax(emotion_classifier.predict(gray_face))
        emotion_text = emotion_labels[emotion_label_arg]

        color = (255, 0, 0)
        draw_bounding_box(face_coordinates, rgb_image, color)
        draw_text(face_coordinates, rgb_image, emotion_text, color, 0, -10,
                  0.5, 2)

    bgr_image = cv2.cvtColor(rgb_image, cv2.COLOR_RGB2BGR)
    cv2.imwrite('predicted_test_image.png', bgr_image)
    return emotion_text
def predict(image_path):
    # loading images
    start = time.time()
    rgb_image = load_image(image_path, grayscale=False)
    gray_image = load_image(image_path, grayscale=True)
    gray_image = np.squeeze(gray_image)
    gray_image = gray_image.astype('uint8')
    with graph.as_default():
        faces = detect_faces(face_detection, gray_image)
        for face_coordinates in faces:
            x1, x2, y1, y2 = apply_offsets(face_coordinates, gender_offsets)
            rgb_face = rgb_image[y1:y2, x1:x2]
            x1, x2, y1, y2 = apply_offsets(face_coordinates, emotion_offsets)
            gray_face = gray_image[y1:y2, x1:x2]
            try:
                rgb_face = cv2.resize(rgb_face, (gender_target_size))
                gray_face = cv2.resize(gray_face, (emotion_target_size))
            except:
                continue
            rgb_face = preprocess_input(rgb_face, False)
            rgb_face = np.expand_dims(rgb_face, 0)
            gender_prediction = gender_classifier.predict(rgb_face)
            gender_label_arg = np.argmax(gender_prediction)
            gender_text = gender_labels[gender_label_arg]
            gray_face = preprocess_input(gray_face, True)
            gray_face = np.expand_dims(gray_face, 0)
            gray_face = np.expand_dims(gray_face, -1)
            emotion_label_arg = np.argmax(emotion_classifier.predict(gray_face))
            emotion_text = emotion_labels[emotion_label_arg]

            if gender_text == gender_labels[0]:
                color = (0, 0, 255)
            else:
                color = (255, 0, 0)

            draw_bounding_box(face_coordinates, rgb_image, color)
            draw_text(face_coordinates, rgb_image, gender_text, color, 0, -10, 1, 2)
            draw_text(face_coordinates, rgb_image, emotion_text, color, 0, -40, 1, 2)

        bgr_image = cv2.cvtColor(rgb_image, cv2.COLOR_RGB2BGR)
        resized_image = cv2.resize(bgr_image, (1024, 769))
    file_path = 'result/' + image_path.split('/')[-1]
    cv2.imwrite(file_path, resized_image)
    end = time.time()
    logging.info('[Time] for prediction(): {}'.format(end - start))
    return file_path
Beispiel #13
0
def on_data(data):
    f = open('current.jpg', 'wb')
    f.write(base64.decodebytes(data))
    f.close()
    image_path = "current.jpg"

    out = []
    # loading images
    rgb_image = load_image(image_path, grayscale=False)
    gray_image = load_image(image_path, grayscale=True)
    gray_image = np.squeeze(gray_image)
    gray_image = gray_image.astype('uint8')

    faces = detect_faces(face_detection, gray_image)
    for face_coordinates in faces:
        x1, x2, y1, y2 = apply_offsets(face_coordinates, emotion_offsets)
        gray_face = gray_image[y1:y2, x1:x2]

        try:
            gray_face = cv2.resize(gray_face, (emotion_target_size))
        except:
            continue

        gray_face = preprocess_input(gray_face, True)
        gray_face = np.expand_dims(gray_face, 0)
        gray_face = np.expand_dims(gray_face, -1)
        emotion_label_arg = np.argmax(emotion_classifier.predict(gray_face))
        emotion_text = emotion_labels[emotion_label_arg]
        color = (0, 0, 255)

        draw_bounding_box(face_coordinates, rgb_image, color)
        draw_text(face_coordinates, rgb_image, emotion_text, color, 0, -50, 1,
                  2)
        bgr_image = cv2.cvtColor(rgb_image, cv2.COLOR_RGB2BGR)

        cv2.imwrite('predicted.png', bgr_image)
        data = open('predicted.png', 'rb').read()
        encoded = base64.encodebytes(data).decode('utf-8')
        out.append({
            'image': encoded,
            'emotion': emotion_text,
        })

    return out
Beispiel #14
0
def process_image(image):
    try:
        # parameters for loading data and images
        detection_model_path = './trained_models/detection_models/haarcascade_frontalface_default.xml'
        emotion_model_path = './trained_models/emotion_models/fer2013_mini_XCEPTION.102-0.66.hdf5'
        gender_model_path = './trained_models/gender_models/simple_CNN.81-0.96.hdf5'
        emotion_labels = get_labels('fer2013')
        gender_labels = get_labels('imdb')

        # hyper-parameters for bounding boxes shape
        gender_offsets = (10, 10)
        emotion_offsets = (0, 0)

        # loading models
        face_detection = load_detection_model(detection_model_path)
        emotion_classifier = load_model(emotion_model_path, compile=False)
        gender_classifier = load_model(gender_model_path, compile=False)

        # getting input model shapes for inference
        emotion_target_size = emotion_classifier.input_shape[1:3]
        gender_target_size = gender_classifier.input_shape[1:3]

        # loading images
        image_array = np.fromstring(image, np.uint8)
        unchanged_image = cv2.imdecode(image_array, cv2.IMREAD_UNCHANGED)

        rgb_image = cv2.cvtColor(unchanged_image, cv2.COLOR_BGR2RGB)
        gray_image = cv2.cvtColor(unchanged_image, cv2.COLOR_BGR2GRAY)

        faces = detect_faces(face_detection, gray_image)
        for face_coordinates in faces:
            x1, x2, y1, y2 = apply_offsets(face_coordinates, gender_offsets)
            rgb_face = rgb_image[y1:y2, x1:x2]

            x1, x2, y1, y2 = apply_offsets(face_coordinates, emotion_offsets)
            gray_face = gray_image[y1:y2, x1:x2]

            try:
                rgb_face = cv2.resize(rgb_face, gender_target_size)
                gray_face = cv2.resize(gray_face, emotion_target_size)
            except:
                continue

            rgb_face = preprocess_input(rgb_face, False)
            rgb_face = np.expand_dims(rgb_face, 0)
            gender_prediction = gender_classifier.predict(rgb_face)
            gender_label_arg = np.argmax(gender_prediction)
            gender_text = gender_labels[gender_label_arg]

            gray_face = preprocess_input(gray_face, True)
            gray_face = np.expand_dims(gray_face, 0)
            gray_face = np.expand_dims(gray_face, -1)
            emotion_label_arg = np.argmax(
                emotion_classifier.predict(gray_face))
            emotion_text = emotion_labels[emotion_label_arg]

            if gender_text == gender_labels[0]:
                color = (0, 0, 255)
            else:
                color = (255, 0, 0)

            draw_bounding_box(face_coordinates, rgb_image, color)
            draw_text(face_coordinates, rgb_image, gender_text, color, 0, -20,
                      1, 2)
            draw_text(face_coordinates, rgb_image, emotion_text, color, 0, -50,
                      1, 2)
    except Exception as err:
        logging.error('Error in emotion gender processor: "{0}"'.format(err))

    bgr_image = cv2.cvtColor(rgb_image, cv2.COLOR_RGB2BGR)

    dirname = 'result'
    if not os.path.exists(dirname):
        os.mkdir(dirname)

    cv2.imwrite(os.path.join(dirname, 'predicted_image.png'), bgr_image)
Beispiel #15
0
def r8k9(fileName='dog.jpg', score=10, length=10):
    # getting input model shapes for inference
    emotion_target_size = emotion_classifier.input_shape[1:3]

    # starting lists for calculating modes
    emotion_window = []

    # starting video streaming
    cv2.namedWindow('window_frame')
    video_capture = cv2.VideoCapture(0)

    im = cv2.imread(fileName)
    im_resized = cv2.resize(im, (224, 224), interpolation=cv2.INTER_LINEAR)
    countdown = ["Ready?", "3", "2", "1"]
    #plt.imshow(cv2.cvtColor(im_resized, cv2.COLOR_BGR2RGB))
    #for i in range(4):
    #	print(countdown[i])
    #	time.sleep(1)
    #print("GO")
    #plt.show(block=False)
    t = time.time()
    td = []
    while time.time() < t + length:
        bgr_image = video_capture.read()[1]
        try:
            gray_image = cv2.cvtColor(bgr_image, cv2.COLOR_BGR2GRAY)
        except:
            time.sleep(.5)
            gray_image = cv2.cvtColor(bgr_image, cv2.COLOR_BGR2GRAY)
        rgb_image = cv2.cvtColor(bgr_image, cv2.COLOR_BGR2RGB)
        faces = detect_faces(face_detection, gray_image)

        for face_coordinates in faces:

            x1, x2, y1, y2 = apply_offsets(face_coordinates, emotion_offsets)
            gray_face = gray_image[y1:y2, x1:x2]
            try:
                gray_face = cv2.resize(gray_face, (emotion_target_size))
            except:
                continue

            gray_face = preprocess_input(gray_face, True)
            gray_face = np.expand_dims(gray_face, 0)
            gray_face = np.expand_dims(gray_face, -1)
            emotion_prediction = emotion_classifier.predict(gray_face)
            #print(emotion_prediction)
            if len(td) == 0:
                data = emotion_prediction
                td = [time.time() - t]
            else:
                data = np.concatenate((data, emotion_prediction), axis=0)
                td = td + [time.time() - t]

            emotion_probability = np.max(emotion_prediction)
            emotion_label_arg = np.argmax(emotion_prediction)
            emotion_text = emotion_labels[emotion_label_arg]
            emotion_window.append(emotion_text)

            if len(emotion_window) > frame_window:
                emotion_window.pop(0)
            try:
                emotion_mode = mode(emotion_window)
            except:
                continue

            if emotion_text == 'angry':
                color = emotion_probability * np.asarray((255, 0, 0))
            elif emotion_text == 'sad':
                color = emotion_probability * np.asarray((0, 0, 255))
            elif emotion_text == 'happy':
                color = emotion_probability * np.asarray((255, 255, 0))
            elif emotion_text == 'surprise':
                color = emotion_probability * np.asarray((0, 255, 255))
            else:
                color = emotion_probability * np.asarray((0, 255, 0))

            color = color.astype(int)
            color = color.tolist()

            draw_bounding_box(face_coordinates, rgb_image, color)
            draw_text(face_coordinates, rgb_image, emotion_mode, color, 0, -45,
                      1, 1)

        bgr_image = cv2.cvtColor(rgb_image, cv2.COLOR_RGB2BGR)
        cv2.imshow('window_frame', bgr_image)
        if cv2.waitKey(1) & 0xFF == ord('q'):
            break

    #print(np.transpose(data))
    #print(emotion_labels)
    plt.clf()
    plt.plot(td, data)

    plt.legend(
        ['angry', 'disgust', 'fear', 'happy', 'sad', 'surprise', 'neutral'],
        loc=2)
    plt.savefig('Graph.png')
    #splt.show()
    totals = [0, 0, 0, 0, 0, 0, 0]
    for j in range(7):
        for i in range(1, len(td)):
            #print(time[i-1])
            #print(data[i-1+l*(j+1)])
            #print(time[i],time[i-1])
            totals[j] += (data[i][j] + data[i - 1][j]) / 2 * (
                td[i] - td[i - 1]) / (td[-1] - td[0])
    np.append(totals, np.std(data))
    print(totals)
    #print(td[-1] - td[0])
    #print(sum(totals))
    #f = open("DogData.txt","a+")
    #f.write((str(score) + ' ' + ' '.join(str(t) for t in totals) + "\n"))
    print("Rating:", 15 - spot(np.array(totals)) / 10, "/10")
    return (15 - spot(np.array(totals)) / 10)
def get_sim_percent(image_1, image_2, target_class):

    # parameters for loading data and images
    # image_path = sys.argv[1]
    detection_model_path = 'trained_models/detection_models/haarcascade_frontalface_default.xml'
    emotion_model_path = 'trained_models/fer2013_big_XCEPTION.54-0.66.hdf5'
    gender_model_path = 'trained_models/gender_models/simple_CNN.81-0.96.hdf5'
    emotion_labels = get_labels('fer2013')
    gender_labels = get_labels('imdb')
    font = cv2.FONT_HERSHEY_SIMPLEX

    # hyper-parameters for bounding boxes shape
    gender_offsets = (20, 40)
    gender_offsets = (10, 10)
    emotion_offsets = (10, 30)
    emotion_offsets = (0, 0)

    # loading models
    face_detection = load_detection_model(detection_model_path)
    emotion_classifier = load_model(emotion_model_path, compile=False)
    gender_classifier = load_model(gender_model_path, compile=False)

    # getting input model shapes for inference
    emotion_target_size = emotion_classifier.input_shape[1:3]
    gender_target_size = gender_classifier.input_shape[1:3]

    target_emo = target_class

    image_path = image_1
    # loading images
    rgb_image = load_image(image_path, grayscale=False)
    gray_image = load_image(image_path, grayscale=True)
    gray_image = np.squeeze(gray_image)
    gray_image = gray_image.astype('uint8')

    faces = detect_faces(face_detection, gray_image)
    for face_coordinates in faces:
        x1, x2, y1, y2 = apply_offsets(face_coordinates, gender_offsets)
        rgb_face = rgb_image[y1:y2, x1:x2]

        x1, x2, y1, y2 = apply_offsets(face_coordinates, emotion_offsets)
        gray_face = gray_image[y1:y2, x1:x2]

        try:
            rgb_face = cv2.resize(rgb_face, (gender_target_size))
            gray_face = cv2.resize(gray_face, (emotion_target_size))
        except:
            continue

        rgb_face = preprocess_input(rgb_face, False)
        rgb_face = np.expand_dims(rgb_face, 0)
        gender_prediction = gender_classifier.predict(rgb_face)
        gender_label_arg = np.argmax(gender_prediction)
        gender_text = gender_labels[gender_label_arg]

        gray_face = preprocess_input(gray_face, True)
        gray_face = np.expand_dims(gray_face, 0)
        gray_face = np.expand_dims(gray_face, -1)
        emotion_label_arg = emotion_classifier.predict(gray_face)[0][target_emo]
        emotion_percent = str(round(emotion_label_arg*100, 2))
        emotion_text = emotion_labels[target_emo] + ' - '+ emotion_percent
        # print(target_emo)
        # print(emotion_text)

        if gender_text == gender_labels[0]:
            color = (0, 0, 255)
        else:
            color = (255, 0, 0)

        draw_bounding_box(face_coordinates, rgb_image, color)
        draw_text(face_coordinates, rgb_image, gender_text, color, -20, 30, 1, 2)
        draw_text(face_coordinates, rgb_image, emotion_text, color, 0, 0, 1, 2)

    bgr_image = cv2.cvtColor(rgb_image, cv2.COLOR_RGB2BGR)
    cv2.imwrite('predicted_test_image.png', bgr_image)

    return emotion_percent
Beispiel #17
0
    x1, x2, y1, y2 = apply_offsets(face_coordinates, offsets)
    gray_face = gray_image[y1:y2, x1:x2]

    # processing input
    try:
        gray_face = cv2.resize(gray_face, (target_size))
    except:
        continue
    gray_face = preprocess_input(gray_face, True)
    gray_face = np.expand_dims(gray_face, 0)
    gray_face = np.expand_dims(gray_face, -1)

    # prediction
    predicted_class = np.argmax(model.predict(gray_face))
    label_text = labels[predicted_class]

    gradient_function = compile_gradient_function(model,
                            predicted_class, 'conv2d_7')
    register_gradient()
    guided_model = modify_backprop(model, 'GuidedBackProp', task)
    saliency_function = compile_saliency_function(guided_model, 'conv2d_7')

    guided_gradCAM = calculate_guided_gradient_CAM(gray_face,
                        gradient_function, saliency_function)
    guided_gradCAM = cv2.resize(guided_gradCAM, (x2-x1, y2-y1))
    rgb_guided_gradCAM = np.repeat(guided_gradCAM[:, :, np.newaxis], 3, axis=2)
    rgb_image[y1:y2, x1:x2, :] = rgb_guided_gradCAM
    draw_bounding_box((x1, y1, x2 - x1, y2 - y1), rgb_image, color)
bgr_image = cv2.cvtColor(rgb_image, cv2.COLOR_RGB2BGR)
cv2.imwrite('../images/guided_gradCAM.png', bgr_image)
        def callback(self,data):
                    #/////////////////////////////////////////////
                    #cv_image = bridge.imgmsg_to_cv2(image_msg, desired_encoding="passthrough")
                    #cv_image = cv2.resize(cv_image, target_size)  # resize image
                    #np_image = np.asarray(cv_image)               # read as np array
                    #np_image = np.expand_dims(np_image, axis=0)   # Add another dimension for tensorflow
                    #np_image = np_image.astype(float)  # preprocess needs float64 and img is uint8
                    #np_image = preprocess_input(np_image)         # Regularize the data
                    #/////////////////////////////////////////////
                    if(not USE_LOCAL_CAMERA):                                                    
                        try:
                                frame = self.bridge.imgmsg_to_cv2(data, desired_encoding="passthrough")
                        except CvBridgeError as e:
                                print(e)

                    # Capture frame-by-frame
                    if(USE_LOCAL_CAMERA):
                        ret, frame1 = self.video_capture.read()
                    #/////////////////////////////////////////////    
                    
                    
                    
                    #print"--------"
                    #print('input_msg height  : {}'.format(frame.height))
                    #print('input_msg width   : {}'.format(frame.width))
                    #print('input_msg step    : {}'.format(frame.step))
                    #print('input_msg encoding: {}'.format(frame.encoding))
                    #print('output dtype      : {}'.format(frame.dtype))
                    #print('output shape      : {}'.format(frame.shape))
                    #print"--------"
                   
                    
                    gray_image = cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY)
                    rgb_image = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)
                    faces = detect_faces(self.face_detection, gray_image)

                    for face_coordinates in faces:
                            print "inside the for loop!"
                            print face_coordinates

                            x1, x2, y1, y2 = apply_offsets(face_coordinates, self.emotion_offsets)
                            gray_face = gray_image[y1:y2, x1:x2]
                            #print len(frame)
                            #print gray_face.size
                            #print gray_face.shape
                            #print gray_face.dtype

                            try:
                                    gray_face = cv2.resize(gray_face, (self.emotion_target_size))
                            except:
                                    continue

                    
                            gray_face = preprocess_input(gray_face, True)
                            gray_face = np.expand_dims(gray_face, 0)
                            gray_face = np.expand_dims(gray_face, -1)
                            
                            #print"************"
                            #print('gray_face dtype      : {}'.format(gray_face.dtype))
                            #print('gray_face shape      : {}'.format(gray_face.shape))
                            #print('gray_face size      : {}'.format(gray_face.size))
                            #print"************"


                            ## This is a workaround for asynchronous execution using TF and ROS
                            # https://github.com/keras-team/keras/issues/2397
                            # http://projectsfromtech.blogspot.com/2017/10/visual-object-recognition-in-ros-using.html
                            #global self.graph                                  
                            with self.graph.as_default():
                                emotion_prediction = self.emotion_classifier.predict(gray_face)
                                emotion_probability = np.max(emotion_prediction)
                                emotion_label_arg = np.argmax(emotion_prediction)
                                emotion_text = self.emotion_labels[emotion_label_arg]	
                                print emotion_text
                                print(emotion_probability)
                                print('%')
                                self.emotion_window.append(emotion_text)

                
                                #self.emotion_msg.data = emotion_text
                                #self.emotion_publisher.publish(emotion_msg)
                                #self.speech_msg.data = 'I see that you are ' + emotion_text
                                #self.speech_publisher.publish(speech_msg)

                                if len(self.emotion_window) > self.frame_window:
                                    self.emotion_window.pop(0)
                                try:
                                    emotion_mode = mode(self.emotion_window)
                                except:
                                    continue

                                if emotion_text == 'angry':
                                    color = emotion_probability * np.asarray((255, 0, 0))
                                elif emotion_text == 'sad':
                                    color = emotion_probability * np.asarray((0, 0, 255))
                                elif emotion_text == 'happy':
                                    color = emotion_probability * np.asarray((255, 255, 0))
                                elif emotion_text == 'surprise':
                                    color = emotion_probability * np.asarray((0, 255, 255))
                                else:
                                    color = emotion_probability * np.asarray((0, 255, 0))

                                color = color.astype(int)
                                color = color.tolist()

                                draw_bounding_box(face_coordinates, rgb_image, color)
                                draw_text(face_coordinates, rgb_image, emotion_mode,
                                        color, 0, -45, 1, 1)
                         
                    try:
                        self.image_pub.publish(self.bridge.cv2_to_imgmsg(rgb_image, "bgr8"))
                    except CvBridgeError as e:
                        print(e)
        if len(emotion_window) > frame_window:
            emotion_window.pop(0)
        try:
            emotion_mode = mode(emotion_window)
        except:
            continue

        if emotion_text == 'angry':
            color = emotion_probability * np.asarray((255, 0, 0))
        elif emotion_text == 'sad':
            color = emotion_probability * np.asarray((0, 0, 255))
        elif emotion_text == 'happy':
            color = emotion_probability * np.asarray((255, 255, 0))
        elif emotion_text == 'surprise':
            color = emotion_probability * np.asarray((0, 255, 255))
        else:
            color = emotion_probability * np.asarray((0, 255, 0))

        color = color.astype(int)
        color = color.tolist()

        draw_bounding_box(face_coordinates, rgb_image, color)
        draw_text(face_coordinates, rgb_image, emotion_mode,
                  color, 0, -45, 1, 1)

    bgr_image = cv2.cvtColor(rgb_image, cv2.COLOR_RGB2BGR)
    cv2.imshow('window_frame', bgr_image)
    if cv2.waitKey(1) & 0xFF == ord('q'):
        break
Beispiel #20
0
def biaoqing(frame):
    gray_image = cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY)
    rgb_image = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)
    dets = detector(gray_image, 1)
    #print(dets)
    #faces = detect_faces(face_detection, gray_image)#[[横,纵,宽,高]]
    #print(faces)

    max_face = 0
    max_k = 0
    emotion_text = 0
    for k,d in enumerate(dets):
        if (d.right()-d.left())*(d.bottom()-d.top()) > max_face:
            max_face = (d.right()-d.left())*(d.bottom()-d.top())
            max_k = k
    for k,d in enumerate(dets):
        if k != max_k: continue
        face_coordinates=[]
        face_coordinates.append(d.left())
        face_coordinates.append(d.top())
        face_coordinates.append(d.right()-d.left())
        face_coordinates.append(d.bottom()-d.top())
        
        x1, x2, y1, y2 = apply_offsets(face_coordinates, emotion_offsets)
        gray_face = gray_image[y1:y2, x1:x2]
        try:
            gray_face = cv2.resize(gray_face, (emotion_target_size))
        except:
            continue

        gray_face = preprocess_input(gray_face, True)
        gray_face = np.expand_dims(gray_face, 0)
        gray_face = np.expand_dims(gray_face, -1)
        emotion_prediction = emotion_classifier.predict(gray_face)
        emotion_probability = np.max(emotion_prediction)
        emotion_label_arg = np.argmax(emotion_prediction)
        emotion_text = emotion_labels[emotion_label_arg]
        emotion_window.append(emotion_text)

        if len(emotion_window) > frame_window:
            emotion_window.pop(0)
        try:
            emotion_mode = mode(emotion_window)
        except:
            continue

        if emotion_text == 'angry':
            color = emotion_probability * np.asarray((255, 0, 0))
        elif emotion_text == 'sad':
            color = emotion_probability * np.asarray((0, 0, 255))
        elif emotion_text == 'happy':
            color = emotion_probability * np.asarray((255, 255, 0))
        elif emotion_text == 'surprise':
            color = emotion_probability * np.asarray((0, 255, 255))
        else:
            color = emotion_probability * np.asarray((0, 255, 0))

        color = color.astype(int)
        color = color.tolist()

        draw_bounding_box(face_coordinates, rgb_image, color)
        draw_text(face_coordinates, rgb_image, emotion_mode,
                    color, 0, -45, 1, 1)

    bgr_image = cv2.cvtColor(rgb_image, cv2.COLOR_RGB2BGR)
    # cv2.imshow('out', bgr_image)
    if emotion_text:
        return emotion_text, bgr_image
    else:
        return 0, frame
Beispiel #21
0
def processa():
    # parameters for loading data and images
    emotion_model_path = './models/emotion_model.hdf5'  #Carica il modello delle emozioni , pre-addestrato
    emotion_labels = get_labels('fer2013')

    # hyper-parameters for bounding boxes shape
    frame_window = 10
    emotion_offsets = (20, 40)

    # loading models #Rilevamento caratteristiche del viso
    detector = dlib.get_frontal_face_detector(
    )  #Rilevamento caratteristiche del viso
    emotion_classifier = load_model(emotion_model_path)

    # predictor = dlib.shape_predictor("shape_predictor_68_face_landmarks.dat")

    # getting input model shapes for inference
    emotion_target_size = emotion_classifier.input_shape[
        1:3]  #Fa una stima parziale del modello di input

    # starting lists for calculating modes
    emotion_window = []

    # Initialize some variables
    face_locations = []
    face_encodings = []  #Delimita i volti umani
    face_names = []
    process_this_frame = True

    def face_compare(frame, process_this_frame):
        # Resize frame of video to 1/4 size for faster face recognition processing
        small_frame = cv2.resize(frame, (0, 0), fx=0.50, fy=0.50)

        # Convert the image from BGR color (which OpenCV uses) to RGB color (which face_recognition uses)
        rgb_small_frame = small_frame[:, :, ::-1]

        # Only process every other frame of video to save time
        if process_this_frame:
            # Find all the faces and face encodings in the current frame of video(cerca i volti nel frame video)
            face_locations = face_recognition.face_locations(
                rgb_small_frame)  #Delimitazione del volto umano
            face_encodings = face_recognition.face_encodings(
                rgb_small_frame, face_locations
            )  #apprende se c'è un volto umano tramite face location

    # starting video streaming
    cv2.namedWindow('window_frame')

    cap = None
    # Webcam source
    cap = cv2.VideoCapture(0)

    while cap.isOpened():  # True:
        ret, frame = cap.read(
        )  #restituisce le immagini della webcam frame per frame

        #frame = video_capture.read()[1]

        # To print the facial landmarks
        # landmrk = face_recognition.face_landmarks(frame)
        # for l in landmrk:
        #     for key,val in l.items():
        #         for (x,y) in val:
        #             cv2.circle(frame, (x, y), 1, (255,0, 0), -1)

        gray_image = cv2.cvtColor(
            frame, cv2.COLOR_BGR2GRAY)  #trasforma i frame in scala di grigio
        rgb_image = cv2.cvtColor(frame,
                                 cv2.COLOR_BGR2RGB)  #trasforma i frame in rgb

        faces = detector(rgb_image)  #rileva caratteristiche del viso
        # face_locations = face_recognition.face_locations(rgb_image)
        # print (reversed(face_locations))
        face_name = face_compare(rgb_image, process_this_frame)
        for face_coordinates, fname in zip(faces, "Unknow"):
            x1, x2, y1, y2 = apply_offsets(
                face_utils.rect_to_bb(face_coordinates), emotion_offsets)
            gray_face = gray_image[y1:y2, x1:x2]
            try:
                gray_face = cv2.resize(gray_face, (emotion_target_size))
            except:
                continue

            gray_face = preprocess_input(gray_face, True)
            gray_face = np.expand_dims(gray_face, 0)
            gray_face = np.expand_dims(gray_face, -1)
            emotion_prediction = emotion_classifier.predict(
                gray_face)  #fa delle predizioni
            emotion_probability = np.max(
                emotion_prediction)  #prende la predizione con valore massimo
            emotion_label_arg = np.argmax(
                emotion_prediction
            )  #prende l'indice della predizione con valore massimo
            emotion_text = emotion_labels[
                emotion_label_arg]  #inserisce nella variabile l'umore predetto
            emotion_window.append(emotion_text)

            if len(emotion_window) > frame_window:
                emotion_window.pop(0)
            try:
                emotion_mode = mode(
                    emotion_window
                )  #restituisce il dato più comune nella lista
            except:
                continue

            if emotion_text == 'Triste':
                color = emotion_probability * np.asarray((0, 0, 255))
            elif emotion_text == 'Felice':
                color = emotion_probability * np.asarray((255, 255, 0))
            else:
                color = emotion_probability * np.asarray((0, 255, 0))

            color = color.astype(int)
            color = color.tolist()  #converte un colore in un elenco del modulo

            name = emotion_text

            draw_bounding_box(face_utils.rect_to_bb(face_coordinates),
                              rgb_image, color)  #crea un rettangolo colorato
            draw_text(face_utils.rect_to_bb(face_coordinates), rgb_image, name,
                      color, 0, -45, 0.5,
                      1)  #inserisce l'emozione predetta colorata

        frame = cv2.cvtColor(rgb_image,
                             cv2.COLOR_RGB2BGR)  #converte da rgb a bgr
        cv2.imshow('window_frame',
                   frame)  #mostra il frame nella finestra windows_frame
        if cv2.waitKey(1) & 0xFF == ord(
                'q'):  #se c'è un ritardo di un milli secondo allora break
            return emotion_text

    cap.release()
    cv2.destroyAllWindows()
  def callback(self,data):
    try:
      bgr_image = self.bridge.imgmsg_to_cv2(data, "bgr8")
      cv2.imshow("Image window", bgr_image)
    except CvBridgeError as e:
      print(e)



######################### Start the image processing
    gray_image = cv2.cvtColor(bgr_image, cv2.COLOR_BGR2GRAY)
    rgb_image = cv2.cvtColor(bgr_image, cv2.COLOR_BGR2RGB)
    faces = detect_faces(self.face_detection, gray_image)

    for face_coordinates in faces:

        x1, x2, y1, y2 = apply_offsets(face_coordinates, self.emotion_offsets)
        gray_face = gray_image[y1:y2, x1:x2]
	
        try:
            gray_face = cv2.resize(gray_face, (self.emotion_target_size))
        except:
            continue

        gray_face = preprocess_input(gray_face, True)
        gray_face = np.expand_dims(gray_face, 0)
        gray_face = np.expand_dims(gray_face, -1)
        print "start"
        print len(gray_face)
        
        print gray_face
        print "end"

        emotion_prediction = self.emotion_classifier.predict(gray_face)
        emotion_probability = np.max(emotion_prediction)
        emotion_label_arg = np.argmax(emotion_prediction)
        emotion_text = emotion_labels[emotion_label_arg]	
	print emotion_text
	print(emotion_probability)
	print('%')
        self.emotion_window.append(emotion_text)

	


        if len(self.emotion_window) > self.frame_window:
            self.emotion_window.pop(0)
        try:
            emotion_mode = mode(self.emotion_window)
        except:
            continue

        if emotion_text == 'angry':
            color = emotion_probability * np.asarray((255, 0, 0))
        elif emotion_text == 'sad':
            color = emotion_probability * np.asarray((0, 0, 255))
        elif emotion_text == 'happy':
            color = emotion_probability * np.asarray((255, 255, 0))
        elif emotion_text == 'surprise':
            color = emotion_probability * np.asarray((0, 255, 255))
        else:
            color = emotion_probability * np.asarray((0, 255, 0))

        color = color.astype(int)
        color = color.tolist()

        draw_bounding_box(face_coordinates, rgb_image, color)
        draw_text(face_coordinates, rgb_image, emotion_mode,
                  color, 0, -45, 1, 1)

    bgr_image = cv2.cvtColor(rgb_image, cv2.COLOR_RGB2BGR)
    
######################### End the image processing	



    cv2.imshow("Image window", bgr_image)
    cv2.waitKey(3)

    try:
      self.image_pub.publish(self.bridge.cv2_to_imgmsg(bgr_image, "bgr8"))
    except CvBridgeError as e:
      print(e)
def emotion_demo():
    # parameters for loading data and images
    detection_model_path = '../trained_models/detection_models/haarcascade_frontalface_default.xml'
    emotion_model_path = '../trained_models/emotion_models/fer2013_mini_XCEPTION.102-0.66.hdf5'
    emotion_labels = get_labels('fer2013')

    # 目のカスケードファイル追加
    lefteyecc__path = "../trained_models/detection_models/haarcascade_lefteye_2splits.xml"
    righteyecc_path = "../trained_models/detection_models/haarcascade_righteye_2splits.xml"
    nose_path = "../trained_models/detection_models/data_haarcascades_haarcascade_mcs_nose.xml"
    lefteyecc = cv2.CascadeClassifier(lefteyecc__path)
    righteyecc = cv2.CascadeClassifier(righteyecc_path)
    nose = cv2.CascadeClassifier(nose_path)
    lex = 0; ley = 0; lew = 0; leh = 0
    rex = 0; rey = 0; rew = 0; reh = 0
    nox = 0; noy = 0; now = 0; noh = 0

    # dlib
    dlib_ini()

    # hyper-parameters for bounding boxes shape
    frame_window = 10
    emotion_offsets = (20, 40)

    # loading models
    face_detection = load_detection_model(detection_model_path)
    emotion_classifier = load_model(emotion_model_path, compile=False)

    # getting input model shapes for inference
    emotion_target_size = emotion_classifier.input_shape[1:3]

    # starting lists for calculating modes
    emotion_window = []

    global img, flag, slp_count
    img = cv2.imread('../img/happy.png')
    flag = 0
    slp_count = 0

    # dlib用グローバル変数
    global gray_image, rgb_image, gray_face, mark_list

    # starting video streaming
    cv2.namedWindow('window_frame', cv2.WINDOW_NORMAL)
    video_capture = cv2.VideoCapture(0) # 0は内蔵カメラ, 1はUSBカメラ
    
    while True:
        bgr_image = video_capture.read()[1]
        gray_image = cv2.cvtColor(bgr_image, cv2.COLOR_BGR2GRAY)
        rgb_image = cv2.cvtColor(bgr_image, cv2.COLOR_BGRA2RGBA)
        faces = detect_faces(face_detection, gray_image)
          

        for face_coordinates in faces:
            # 目や鼻認識用
            (x,y,w,h) = face_coordinates
            video_face = gray_image[y:y+h,x:x+w]
            x1, x2, y1, y2 = apply_offsets(face_coordinates, emotion_offsets)
            gray_face = gray_image[y1:y2, x1:x2]
            try:
                gray_face = cv2.resize(gray_face, (emotion_target_size))
            except:
                continue

            # ランドマーク検出
            marks_list = marks_list_def(bgr_image, x, y, w, h)
            print(marks_list)


            gray_face = preprocess_input(gray_face, True)
            gray_face = np.expand_dims(gray_face, 0)
            gray_face = np.expand_dims(gray_face, -1)
            emotion_prediction = emotion_classifier.predict(gray_face)
            emotion_probability = np.max(emotion_prediction)
            emotion_label_arg = np.argmax(emotion_prediction)
            emotion_text = emotion_labels[emotion_label_arg]
            emotion_window.append(emotion_text)

            if len(emotion_window) > frame_window:
                emotion_window.pop(0)
            try:
                emotion_mode = mode(emotion_window)
            except:
                continue

            if flag == 0 or flag == 1:
                if emotion_text == 'angry':
                    img = cv2.imread('../img/angry.png', -1)
                    color = emotion_probability * np.asarray((255, 0, 0))
                elif emotion_text == 'sad':
                    img = cv2.imread('../img/sad.png', -1) # 関数にする
                    color = emotion_probability * np.asarray((0, 0, 255))
                elif emotion_text == 'happy':
                    img = cv2.imread('../img/happy.png', -1)
                    color = emotion_probability * np.asarray((255, 255, 0))
                elif emotion_text == 'surprise':
                    img = cv2.imread('../img/odoroki.png', -1)
                    color = emotion_probability * np.asarray((0, 255, 255))
                else :
                    img = cv2.imread('../img/neutral.png', -1)
                    color = emotion_probability * np.asarray((0, 255, 0))
            else:    
                if emotion_text == 'angry':
                    img = cv2.imread('../img/ikari.png', -1)
                    color = emotion_probability * np.asarray((255, 0, 0))
                elif emotion_text == 'sad':
                    img = cv2.imread('../img/shock.png', -1) 
                    color = emotion_probability * np.asarray((0, 0, 255))
                elif emotion_text == 'happy':
                    img = cv2.imread('../img/kirakira.png', -1)
                    color = emotion_probability * np.asarray((255, 255, 0))
                elif emotion_text == 'surprise':
                    img = cv2.imread('../img/bikkuri.png', -1)
                    color = emotion_probability * np.asarray((0, 255, 255))
                else :
                    img = cv2.imread('../img/toumei.png', -1)
                    color = emotion_probability * np.asarray((0, 255, 0))
                

            color = color.astype(int)
            color = color.tolist()

            if flag == 0:
                draw_bounding_box(face_coordinates, rgb_image, color)
            elif flag == 1:
                rgb_image = draw_bounding_box2(face_coordinates, rgb_image, color, img, marks_list)              
            elif flag == 2:
                overlay_pic = draw_bounding_box3(face_coordinates, rgb_image, color, img, marks_list)
                rgb_image = overlay_pic       

            draw_text(face_coordinates, rgb_image, emotion_mode, color, 0, -45, 1, 1)
            bgr_image = cv2.cvtColor(rgb_image, cv2.COLOR_RGB2BGR)  
        
        
        if flag == 0:
            img = image_resize(img)
            cv2.imshow('image', img)
            cv2.destroyWindow('Window_frame')
        elif flag == 1 or flag == 2:
            cv2.destroyWindow('image')
            cv2.imshow('window_frame', bgr_image)
        cv2.waitKey(10)

        
        # cv2.imshow('own_window', bgr_image)

        if cv2.waitKey(1) & 0xFF == ord('z'):
            flag = 0
        elif cv2.waitKey(1) & 0xFF == ord('x'):
            flag = 1
        elif cv2.waitKey(1) & 0xFF == ord('c'):
            flag = 2
        elif cv2.waitKey(1) & 0xFF == ord('q'):
            break


    video_capture.release()
    cv2.destroyAllWindows()
                    "input": {
                        "'neutral'": "1"
                    },
                    "robot": "Pepper"
                }
                print("neutral")
                pub_emotion.publish(data)
                color = 1 * np.asarray((0, 0, 0))
            else:
                draw = False

            if draw:

                color = color.astype(int)
                color = color.tolist()

                draw_bounding_box(box, rgb_image, color)
                draw_text(box, rgb_image, emotion_mode, color, 0, -45, 1, 1)

            if (show_image == 1):
                bgr_image = cv2.cvtColor(rgb_image, cv2.COLOR_RGB2BGR)
                cv2.imshow('window_frame', bgr_image)
                if cv2.waitKey(1) & 0xFF == ord('q'):
                    break

        except:
            pass

    else:
        pass
Beispiel #25
0
    def recognize_video(self, path=None, csv=True):
        # starting video streaming

        if self.show_frame:
            cv2.namedWindow('window_frame')

        # Select video or webcam feed
        cap = None
        if path is None:
            cap = cv2.VideoCapture(0)  # Webcam source
        elif isinstance(path, str):
            cap = cv2.VideoCapture(path)  # Video file source
        else:
            raise Exception('path argument must be a string')

        emotions_by_frames = [
        ]  # container for recording emotion probabilities by frame
        frame_counter = 0
        while cap.isOpened():  # True:
            try:

                ret, bgr_image = cap.read()
                if not ret:
                    break
                #print('Frame {} is read'.format(frame_counter))
                frame_counter += 1

                rgb_image = cv2.cvtColor(bgr_image, cv2.COLOR_BGR2RGB)

                start = default_timer()
                faces = self.detector.detect_faces(rgb_image)
                #print('faces', faces)
                end = default_timer()
                #print('face detection:', end - start)

                if len(faces) == 0:
                    emotions_by_frames.append(np.array([0] * 7))

                for face_coordinates in faces:
                    # face_coordinates: [x, y, w, h]
                    face_coordinates = list(map(int, face_coordinates))
                    x, y, w, h = face_coordinates

                    # expand to 20%
                    padding_size_x, padding_size_y = int(0.2 * x), int(0.2 * y)
                    w += padding_size_x
                    h += padding_size_y
                    x -= padding_size_x // 2
                    y -= padding_size_y // 2

                    face_image = rgb_image[y:y + h, x:x + w]
                    try:
                        face_image = cv2.resize(
                            face_image, (self.face_size, self.face_size))
                    except:
                        emotions_by_frames.append(
                            np.array([0] * len(self.emotion_labels)))
                        continue

                    face_image = self.prepare_face_image(face_image)
                    #print('Shape:', face_image.shape)

                    emotion_prediction = self.emotion_classifier.predict(
                        face_image, probs=True)[0].numpy()
                    #print('Prediction:', emotion_prediction)
                    emotion_probability = np.max(emotion_prediction)
                    if len(emotion_prediction) > 0:
                        emotions_by_frames.append(emotion_prediction)
                    else:
                        emotions_by_frames.append(
                            np.array([0] * len(self.emotion_labels)))

                    emotion_label_arg = np.argmax(emotion_prediction)
                    emotion_text = self.emotion_labels[emotion_label_arg]

                    if emotion_text == 'angry':
                        color = emotion_probability * np.asarray((255, 0, 0))
                    elif emotion_text == 'sad':
                        color = emotion_probability * np.asarray((0, 0, 255))
                    elif emotion_text == 'happy':
                        color = emotion_probability * np.asarray((255, 255, 0))
                    elif emotion_text == 'surprise':
                        color = emotion_probability * np.asarray((0, 255, 255))
                    else:
                        color = emotion_probability * np.asarray((0, 255, 0))

                    color = color.astype(int)
                    color = color.tolist()

                    draw_bounding_box(face_coordinates, bgr_image, color)
                    draw_text(face_coordinates, bgr_image, emotion_text, color,
                              0, -45, 1, 1)

                #bgr_image = cv2.cvtColor(rgb_image, cv2.COLOR_RGB2BGR)
                cv2.imshow('window_frame', bgr_image)
                if (cv2.waitKey(1) & 0xFF == ord('q')) and self.show_frame:
                    break
            except:
                traceback.print_exc()
                break

        cap.release()
        cv2.destroyAllWindows()
        if csv:
            return self.to_csv(emotions_by_frames)
        return emotions_by_frames
Beispiel #26
0
        elif emotion_text == 'sad':
            color = emotion_probability * np.asarray((0, 0, 255))
        elif emotion_text == 'happy':
            color = emotion_probability * np.asarray((255, 255, 0))
        elif emotion_text == 'surprise':
            color = emotion_probability * np.asarray((0, 255, 255))
        else:
            color = emotion_probability * np.asarray((0, 255, 0))

        color = color.astype(int)
        color = color.tolist()

        if fname == "Unknown":
            name = emotion_text
        else:
            name = str(fname) + " is " + str(emotion_text)
        
        draw_bounding_box(face_utils.rect_to_bb(face_coordinates), rgb_image, color)
        draw_text(face_utils.rect_to_bb(face_coordinates), rgb_image, name,
                  color, 0, -45, 0.5, 1)


    frame = cv2.cvtColor(rgb_image, cv2.COLOR_RGB2BGR)
    cv2.imshow('window_frame', frame)
    if cv2.waitKey(1) & 0xFF == ord('q'):
        break

cap.release()
cv2.destroyAllWindows()

        x1, x2, y1, y2 = apply_offsets(face_coordinates, offsets)
        gray_face = gray_image[y1:y2, x1:x2]
        try:
            gray_face = cv2.resize(gray_face, (target_size))
        except:
            continue

        gray_face = preprocess_input(gray_face, True)
        gray_face = np.expand_dims(gray_face, 0)
        gray_face = np.expand_dims(gray_face, -1)
        guided_gradCAM = calculate_guided_gradient_CAM(gray_face,
                            gradient_function, saliency_function)
        guided_gradCAM = cv2.resize(guided_gradCAM, (x2-x1, y2-y1))
        try:
            rgb_guided_gradCAM = np.repeat(guided_gradCAM[:, :, np.newaxis],
                                                                3, axis=2)
            rgb_image[y1:y2, x1:x2, :] = rgb_guided_gradCAM
        except:
            continue
        draw_bounding_box((x1, y1, x2 - x1, y2 - y1), rgb_image, color)
    bgr_image = cv2.cvtColor(rgb_image, cv2.COLOR_RGB2BGR)
    try:
        cv2.imshow('window_frame', bgr_image)
    except:
        continue
    if cv2.waitKey(1) & 0xFF == ord('q'):
        break


def predict(image_folder_path, emotion_kind):

    detection_model_path = '../trained_models/detection_models/haarcascade_frontalface_default.xml'
    emotion_model_path = '../trained_models/emotion_models/fer2013_mini_XCEPTION.102-0.66.hdf5'
    gender_model_path = '../trained_models/gender_models/simple_CNN.81-0.96.hdf5'
    emotion_labels = get_labels('fer2013')

    target_file = '../result/predicted_' + EmotionName[emotion_kind] + '.txt'

    gender_labels = get_labels('imdb')
    font = cv2.FONT_HERSHEY_SIMPLEX

    # hyper-parameters for bounding boxes shape
    gender_offsets = (30, 60)
    gender_offsets = (10, 10)
    emotion_offsets = (20, 40)
    emotion_offsets = (0, 0)

    # loading models
    face_detection = load_detection_model(detection_model_path)
    emotion_classifier = load_model(emotion_model_path, compile=False)
    gender_classifier = load_model(gender_model_path, compile=False)

    # getting input model shapes for inference
    emotion_target_size = emotion_classifier.input_shape[1:3]
    gender_target_size = gender_classifier.input_shape[1:3]

    image_path = dir_data_folder(image_folder_path)

    predicted_label = []

    for num in range(len(image_path)):
        # print the process info
        if print_switch == 0:
            print('deal with the ' + image_path[num])

        # loading images
        rgb_image = load_image(image_path[num], grayscale=False)
        gray_image = load_image(image_path[num], grayscale=True)
        gray_image = np.squeeze(gray_image)
        gray_image = gray_image.astype('uint8')

        # pdb.set_trace()

        faces = detect_faces(face_detection, gray_image)
        emotion_label_arg = -1
        for face_coordinates in faces:
            x1, x2, y1, y2 = apply_offsets(face_coordinates, gender_offsets)
            rgb_face = rgb_image[y1:y2, x1:x2]

            x1, x2, y1, y2 = apply_offsets(face_coordinates, emotion_offsets)
            gray_face = gray_image[y1:y2, x1:x2]

            try:
                rgb_face = cv2.resize(rgb_face, (gender_target_size))
                gray_face = cv2.resize(gray_face, (emotion_target_size))
            except:
                continue

            rgb_face = preprocess_input(rgb_face, False)
            rgb_face = np.expand_dims(rgb_face, 0)
            gender_prediction = gender_classifier.predict(rgb_face)
            gender_label_arg = np.argmax(gender_prediction)
            gender_text = gender_labels[gender_label_arg]

            gray_face = preprocess_input(gray_face, True)
            gray_face = np.expand_dims(gray_face, 0)
            gray_face = np.expand_dims(gray_face, -1)
            emotion_label_arg = np.argmax(
                emotion_classifier.predict(gray_face))
            emotion_text = emotion_labels[emotion_label_arg]

            if single_set_draw_switch == 0:
                if gender_text == gender_labels[0]:
                    color = (0, 0, 255)
                else:
                    color = (255, 0, 0)

                draw_bounding_box(face_coordinates, rgb_image, color)
                # draw_text(face_coordinates, rgb_image, gender_text, color, 0, -20, 1, 2)
                draw_text(face_coordinates, rgb_image, emotion_text, color, 0,
                          20, 1, 2)

        if single_set_draw_switch == 0:
            bgr_image = cv2.cvtColor(rgb_image, cv2.COLOR_RGB2BGR)

            filename = image_path[num].split('/')[-1]
            filename = filename.split('.')[0]
            newname = '../predicted_image/' + 'predicted_' + filename + '.png'
            cv2.imwrite(newname, bgr_image)

        # add the predicted label
        predicted_label.append(emotion_label_arg)

    unrecognized_cnt = predicted_label.count(-1)
    true_cnt = predicted_label.count(emotion_kind)
    total_cnt = len(image_path)
    face_cnt = total_cnt - unrecognized_cnt
    face_recog_ratio = face_cnt / float(total_cnt)
    total_accuracy = true_cnt / float(total_cnt)
    recog_accuracy = true_cnt / float(face_cnt)

    f = open(target_file, 'w+')
    #with open(target_file,'w') as f:
    f.write(EmotionName[emotion_kind] + ' count = ' + str(true_cnt) + '\n')
    f.write(EmotionName[emotion_kind] + ' total count = ' + str(total_cnt) +
            '\n')
    f.write(EmotionName[emotion_kind] + '_accuracy in total: ' +
            str(total_accuracy) + '\n')
    f.write(EmotionName[emotion_kind] + '_accuracy in recognized: ' +
            str(recog_accuracy) + '\n')
    f.write('face_recognize_count : ' + str(face_cnt) + '\n')
    f.write('face_recognize_ratio : ' + str(face_recog_ratio) + '\n')
    for label in predicted_label:
        f.write(str(label) + '\n')
    f.close()
    return [
        true_cnt, total_cnt, face_cnt, total_accuracy, face_recog_ratio,
        recog_accuracy
    ]
Beispiel #29
0
    rgb_image = cv2.cvtColor(bgr_image, cv2.COLOR_BGR2RGB)

    faces = face_cascade.detectMultiScale(gray_image,
                                          scaleFactor=1.1,
                                          minNeighbors=5,
                                          minSize=(30, 30),
                                          flags=cv2.CASCADE_SCALE_IMAGE)

    # for face_coordinates in faces:
    if len(faces):
        x1, x2, y1, y2 = apply_offsets(faces[0], emotion_offsets)
        gray_face = gray_image[y1:y2, x1:x2]
        gray_face = preprocess_input(gray_face, True)
        gray_face = np.expand_dims(gray_face, 0)
        gray_face = np.expand_dims(gray_face, -1)

        color = np.asarray((255, 0, 0))
        color = color.astype(int)
        color = color.tolist()

        draw_bounding_box(faces[0], rgb_image, color)
        firebase(x1, x2, y1, y2)

    bgr_image = cv2.cvtColor(rgb_image, cv2.COLOR_RGB2BGR)
    cv2.imshow('window_frame', bgr_image)
    if cv2.waitKey(1) & 0xFF == ord('q'):
        break

cap.release()
cv2.destroyAllWindows()
def process_image(image):

    try:
        # parameters for loading data and images
        detection_model_path = './trained_models/detection_models/haarcascade_frontalface_default.xml'
        emotion_model_path = './trained_models/emotion_models/fer2013_mini_XCEPTION.102-0.66.hdf5'
        gender_model_path = './trained_models/gender_models/simple_CNN.81-0.96.hdf5'
        emotion_labels = get_labels('fer2013')
        gender_labels = get_labels('imdb')
        font = cv2.FONT_HERSHEY_SIMPLEX

        # hyper-parameters for bounding boxes shape
        gender_offsets = (30, 60)
        gender_offsets = (10, 10)
        emotion_offsets = (20, 40)
        emotion_offsets = (0, 0)

        # loading models
        face_detection = load_detection_model(detection_model_path)
        emotion_classifier = load_model(emotion_model_path, compile=False)
        gender_classifier = load_model(gender_model_path, compile=False)

        # getting input model shapes for inference
        emotion_target_size = emotion_classifier.input_shape[1:3]
        gender_target_size = gender_classifier.input_shape[1:3]

        # loading images
        image_array = np.fromstring(image, np.uint8)
        unchanged_image = cv2.imdecode(image_array, cv2.IMREAD_UNCHANGED)

        rgb_image = cv2.cvtColor(unchanged_image, cv2.COLOR_BGR2RGB)
        gray_image = cv2.cvtColor(unchanged_image, cv2.COLOR_BGR2GRAY)

        faces = detect_faces(face_detection, gray_image)
        for face_coordinates in faces:
            x1, x2, y1, y2 = apply_offsets(face_coordinates, gender_offsets)
            rgb_face = rgb_image[y1:y2, x1:x2]

            x1, x2, y1, y2 = apply_offsets(face_coordinates, emotion_offsets)
            gray_face = gray_image[y1:y2, x1:x2]

            try:
                rgb_face = cv2.resize(rgb_face, (gender_target_size))
                gray_face = cv2.resize(gray_face, (emotion_target_size))
            except:
                continue

            rgb_face = preprocess_input(rgb_face, False)
            rgb_face = np.expand_dims(rgb_face, 0)
            gender_prediction = gender_classifier.predict(rgb_face)
            gender_label_arg = np.argmax(gender_prediction)
            gender_text = gender_labels[gender_label_arg]

            gray_face = preprocess_input(gray_face, True)
            gray_face = np.expand_dims(gray_face, 0)
            gray_face = np.expand_dims(gray_face, -1)
            emotion_label_arg = np.argmax(emotion_classifier.predict(gray_face))
            emotion_text = emotion_labels[emotion_label_arg]

            if gender_text == gender_labels[0]:
                color = (0, 0, 255)
            else:
                color = (255, 0, 0)

            draw_bounding_box(face_coordinates, rgb_image, color)
            draw_text(face_coordinates, rgb_image, gender_text, color, 0, -20, 1, 2)
            draw_text(face_coordinates, rgb_image, emotion_text, color, 0, -50, 1, 2)
    except Exception as err:
        logging.error('Error in emotion gender processor: "{0}"'.format(err))

    bgr_image = cv2.cvtColor(rgb_image, cv2.COLOR_RGB2BGR)

    dirname = 'result'
    if not os.path.exists(dirname):
        os.mkdir(dirname)

    cv2.imwrite(os.path.join(dirname, 'predicted_image.png'), bgr_image)
Beispiel #31
0
    def extract_emotions(self, bgr_image):
        base_path = os.path.realpath(__file__)
        base_path = base_path[:base_path.find('emotions')]

        emotion_labels = get_labels('fer2013')
        # hyper-parameters for bounding boxes shape
        frame_window = 10
        emotion_offsets = (20, 40)
        # loading models
        if self.emotion_classifier == None:
            self.emotion_classifier = load_model(self.emotion_model_path)

        # getting input model shapes for inference
        emotion_target_size = self.emotion_classifier.input_shape[1:3]
        # starting lists for calculating modes
        emotion_window = []

        gray_image = cv2.cvtColor(bgr_image, cv2.COLOR_BGR2GRAY)
        rgb_image = cv2.cvtColor(bgr_image, cv2.COLOR_BGR2RGB)

        faces = self.face_cascade.detectMultiScale(
            gray_image,
            scaleFactor=1.1,
            minNeighbors=5,
            minSize=(30, 30),
            flags=cv2.CASCADE_SCALE_IMAGE)

        for face_coordinates in faces:

            x1, x2, y1, y2 = apply_offsets(face_coordinates, emotion_offsets)
            gray_face = gray_image[y1:y2, x1:x2]
            try:
                gray_face = cv2.resize(gray_face, (emotion_target_size))
            except:
                continue

            gray_face = preprocess_input(gray_face, True)
            gray_face = np.expand_dims(gray_face, 0)
            gray_face = np.expand_dims(gray_face, -1)
            emotion_prediction = self.emotion_classifier.predict(gray_face)
            emotion_probability = np.max(emotion_prediction)
            emotion_label_arg = np.argmax(emotion_prediction)
            emotion_text = emotion_labels[emotion_label_arg]
            emotion_window.append(emotion_text)

            if len(emotion_window) > frame_window:
                emotion_window.pop(0)
            try:
                emotion_mode = mode(emotion_window)
            except:
                continue

            if emotion_text == 'angry':
                color = emotion_probability * np.asarray((255, 0, 0))
            elif emotion_text == 'sad':
                color = emotion_probability * np.asarray((0, 0, 255))
            elif emotion_text == 'happy':
                color = emotion_probability * np.asarray((255, 255, 0))
            elif emotion_text == 'surprise':
                color = emotion_probability * np.asarray((0, 255, 255))
            elif emotion_text == 'fear':
                color = emotion_probability * np.asarray((255, 0, 255))
            elif emotion_text == 'disgust':
                color = emotion_probability * np.asarray((255, 255, 255))
            else:
                color = emotion_probability * np.asarray((50, 200, 100))

            color = color.astype(int)
            color = color.tolist()

            draw_bounding_box(face_coordinates, rgb_image, color)
            draw_text(face_coordinates, rgb_image, emotion_mode, color, 0, -45,
                      1, 1)

            bgr_image = cv2.cvtColor(rgb_image, cv2.COLOR_RGB2BGR)
            cv2.imshow('window_frame', bgr_image)
            if cv2.waitKey(1) & 0xFF == ord('q'):
                break
def process_image(image):
    detected_peoples = []
    json_info = {}
    try:
        # parameters for loading data and images
        emotion_labels = get_labels('fer2013')
        gender_labels = get_labels('imdb')
        # font = cv2.FONT_HERSHEY_SIMPLEX

        # gender_keys = list(gender_labels.values())
        # emotion_keys = list(emotion_labels.values())

        # hyper-parameters for bounding boxes shape
        gender_offsets = (30, 60)
        gender_offsets = (10, 10)
        emotion_offsets = (20, 40)
        emotion_offsets = (0, 0)

        # getting input model shapes for inference
        gender_target_size = _GENDER_CLASSIFIER.input_shape[1:3]
        emotion_target_size = _EMOTION_CLASSIFIER.input_shape[1:3]

        # loading images
        image_array = np.fromstring(image, np.uint8)
        unchanged_image = cv2.imdecode(image_array, cv2.IMREAD_UNCHANGED)

        rgb_image = cv2.cvtColor(unchanged_image, cv2.COLOR_BGR2RGB)
        gray_image = cv2.cvtColor(unchanged_image, cv2.COLOR_BGR2GRAY)

        faces = detect_faces(_FACE_DETECTION, gray_image)
        for face_coordinates in faces:
            x1, x2, y1, y2 = apply_offsets(face_coordinates, gender_offsets)
            rgb_face = rgb_image[y1:y2, x1:x2]

            x1, x2, y1, y2 = apply_offsets(face_coordinates, emotion_offsets)
            gray_face = gray_image[y1:y2, x1:x2]

            try:
                rgb_face = cv2.resize(rgb_face, (gender_target_size))
                gray_face = cv2.resize(gray_face, (emotion_target_size))
            except:
                continue

            start_time = datetime.now()

            with _GRAPH.as_default():
                rgb_face = preprocess_input(rgb_face, False)
                rgb_face = np.expand_dims(rgb_face, 0)
                gender_prediction = _GENDER_CLASSIFIER.predict(rgb_face)
                gender_label_arg = np.argmax(gender_prediction)
                gender_text = gender_labels[gender_label_arg]

                end_time = datetime.now()
                delta1 = end_time - start_time

                gray_face = preprocess_input(gray_face, True)
                gray_face = np.expand_dims(gray_face, 0)
                gray_face = np.expand_dims(gray_face, -1)
                emotion_prediction = _EMOTION_CLASSIFIER.predict(gray_face)
                emotion_label_arg = np.argmax(emotion_prediction)
                emotion_text = emotion_labels[emotion_label_arg]

            delta2 = datetime.now() - end_time
            print("Delta for gender classificator {0}".format(
                delta1.total_seconds() * 1000.0))
            print("Delta for emotion classificator {0}".format(
                delta2.total_seconds() * 1000.0))

            json_info['gender'] = "{0}:{1}".format(
                gender_text, gender_prediction.flat[gender_label_arg])

            json_info['emotion'] = "{0}:{1}".format(
                emotion_text, emotion_prediction.flat[emotion_label_arg])

            json_info['face_bound'] = list(
                map(lambda it: str(it),
                    list(face_coordinates.astype(np.int).flat)))

            detected_peoples.append(json_info.copy())

            print(detected_peoples)

            if gender_text == gender_labels[0]:
                color = (0, 0, 255)
            else:
                color = (255, 0, 0)

            draw_bounding_box(face_coordinates, rgb_image, color)
            draw_text(face_coordinates, rgb_image, gender_text, color, 0, -20,
                      1, 2)
            draw_text(face_coordinates, rgb_image, emotion_text, color, 0, -10,
                      1, 2)
    except Exception as err:
        logging.error('Error in emotion gender processor: "{0}"'.format(err))

    start_time = datetime.now()
    bgr_image = cv2.cvtColor(rgb_image, cv2.COLOR_RGB2BGR)

    if not os.path.exists(_SAVE_DIR):
        os.mkdir(_SAVE_DIR)

    recognition_datetime = str(datetime.now()).replace(' ', '_')
    filepath = os.path.join(
        _SAVE_DIR, 'predicted_image_{0}.png'.format(recognition_datetime))
    cv2.imwrite(filepath, bgr_image)
    delta = datetime.now() - start_time
    print("delta for saving image", delta.total_seconds() * 1000.0)

    return detected_peoples
Beispiel #33
0
def main():
    warnings.filterwarnings("ignore")
    logging.disable(logging.CRITICAL)
    global emotion_cache #Vishal
    emotion_cache = [] #Vishal
    global stop_detecting

    #Create server client for socket connection
    port = opt.port
    #host_name = socket.gethostname() 
    #host_ip = socket.gethostbyname(host_name) 
    host_ip = opt.ip
    mySocket = socket.socket()
    mySocket.bind((host_ip, port))

    print ("Camera server listening on port {0} ".format(port)+ "and IP " + str(host_ip))
    
    mySocket.listen(1)

    chatbot_socket, adress = mySocket.accept()
    print("Connection established to: " + str(adress))

    #Start socket process to listen to chatbot requests 
    chatbot = Process(target = chatbot_process, args = (chatbot_socket, chatbot_request, chatbot_response, stop_camera))
    chatbot.start()

    signal.signal(signal.SIGTSTP, handler)

    #face_recognizer = Face_Recognizer(True)# True to show video feed
    # parameters for loading data and images

    detection_model_path = './models/face_recognition/trained_models/haarcascade_frontalface_default.xml'
    emotion_model_path = './models/face_recognition/trained_models/fer2013_mini_XCEPTION.102-0.66.hdf5'
    emotion_labels = get_labels('fer2013')

    # hyper-parameters for bounding boxes shape
    frame_window = 10
    emotion_offsets = (20, 40)

    # loading models
    face_detection = load_detection_model(detection_model_path)
    emotion_classifier = load_model(emotion_model_path, compile=False)

    # getting input model shapes for inference
    emotion_target_size = emotion_classifier.input_shape[1:3]

    # starting lists for calculating modes
    emotion_window = []
    emotion_text = 'Happy'	
    # starting video streaming
    cv2.namedWindow('ODO_frame')
    OdoSet='nvarguscamerasrc !  video/x-raw(memory:NVMM), width=3820, height=2464, format=NV12,  framerate=21/1 ! nvvidconv flip-method='+str(flip)+' ! video/x-raw, width='+str(dispW)+', height='+str(dispH)+', format=BGRx ! videoconvert ! video/x-raw, format=BGR ! appsink'

    video_capture = cv2.VideoCapture(OdoSet)

    fps = video_capture.get(cv2.CAP_PROP_FPS)
    print(f'FPS: {fps}')
    num_frames = 21;



    if video_capture.isOpened():
        print("True")
    else:
        print("Error")
    ##################Vishal#####################
    while stop_detecting == False:
    #############################################
        # Detect Emotion
        #print("Check")

        start = time.time() #starting FPS timer

        bgr_image = video_capture.read()[1]
        gray_image = cv2.cvtColor(bgr_image, cv2.COLOR_BGR2GRAY)
        rgb_image = cv2.cvtColor(bgr_image, cv2.COLOR_BGR2RGB)
        faces = detect_faces(face_detection, gray_image)

        for face_coordinates in faces:

            x1, x2, y1, y2 = apply_offsets(face_coordinates, emotion_offsets)
            gray_face = gray_image[y1:y2, x1:x2]
            try:
                gray_face = cv2.resize(gray_face, (emotion_target_size))
            except:
                continue

            gray_face = preprocess_input(gray_face, True)
            gray_face = np.expand_dims(gray_face, 0)
            gray_face = np.expand_dims(gray_face, -1)

            emotion_prediction = emotion_classifier.predict(gray_face)

            emotion_probability = np.max(emotion_prediction)
            emotion_label_arg = np.argmax(emotion_prediction)
            emotion_text = emotion_labels[emotion_label_arg]
            emotion_window.append(emotion_text)

            if len(emotion_window) > frame_window:
                emotion_window.pop(0)
            try:
                emotion_mode = mode(emotion_window)
            except:
               continue

            if emotion_text == 'angry':
                color = emotion_probability * np.asarray((255, 0, 0))
            elif emotion_text == 'sad':
                color = emotion_probability * np.asarray((0, 0, 255))
            elif emotion_text == 'happy':
                color = emotion_probability * np.asarray((255, 255, 0))
            elif emotion_text == 'surprise':
                color = emotion_probability * np.asarray((0, 255, 255))
            else:
                color = emotion_probability * np.asarray((0, 255, 0))

            emotion_cache.append(emotion_text)
            with open('listfile.txt', 'w') as filehandle:
                filehandle.writelines("%s\n" % emotions for emotions in emotion_cache)
            if len(emotion_cache) > 500:
                emotion_cache = emotion_cache[:-500]
            ##########################################
            total_faces = len(faces)
            current_time = time.time()
            if chatbot_request.empty() == False:
                request = chatbot_request.get()
                data_request = request['request']
                if data_request.lower() == 'send_emotion':
                    data = {}
                    data['emotion'] = emotion_detected
                    data['total_faces'] = total_faces
                    data['time'] = current_time
                    json_data = json.dumps(data)
                    chatbot_response.put({'response': json_data})

            
            color = color.astype(int)
            color = color.tolist()
            #print(face_coordinates, rgb_image, color)
            draw_bounding_box(face_coordinates, rgb_image, color)
            draw_text(face_coordinates, rgb_image, emotion_mode,
                  color, 0, -45, 1, 1)

        if stop_camera.empty() == False:
            stop_detecting = True
        end = time.time() #end time for FPS
        bgr_image = cv2.cvtColor(rgb_image, cv2.COLOR_RGB2BGR)
        seconds = end - start #FPS
        fps = num_frames/seconds #FPS
        fps_coords = (0,30)
        framerate_str = 'Framerate: ' + str(int(fps))
        cv2.putText(bgr_image, framerate_str, fps_coords, cv2.FONT_HERSHEY_SIMPLEX, 1, (255,255,200))
        print(framerate_str)
        cv2.imshow('ODO_frame', bgr_image)
        if cv2.waitKey(1) & 0xFF == ord('q'):
            break
        

    video_capture.release()
    cv2.destroyAllWindows()
    print("Closing chatbot socket")
    chatbot.terminate()
    chatbot.join()
Beispiel #34
0
    def run(self):
        detection_graph, sess = detector_utils.load_inference_graph()

        # parameters for loading data and images
        detection_model_path = 'haarcascade_frontalface_default.xml'
        emotion_model_path = 'fer2013_mini_XCEPTION.102-0.66.hdf5'
        emotion_labels = get_labels('fer2013')

        # hyper-parameters for bounding boxes shape
        frame_window = 10
        emotion_offsets = (20, 40)

        # loading models
        face_detection = load_detection_model(detection_model_path)
        emotion_classifier = load_model(emotion_model_path, compile=False)

        # getting input model shapes for inference
        emotion_target_size = emotion_classifier.input_shape[1:3]

        # starting lists for calculating modes
        emotion_window = []

        start_time = datetime.datetime.now()
        im_width, im_height = (400, 350)
        num_hands_detect = 2  # max number of hands we want to detect/track, can scale this up
        min_threshold = 0.2

        old_points = [None] * num_hands_detect
        cv2.namedWindow('Single-Threaded Detection', cv2.WINDOW_NORMAL)

        while True:
            # Expand dimensions since the model expects images to have shape: [1, None, None, 3]
            if self.image_queue.empty():
                continue
            img_data = base64.b64decode(str(self.image_queue.get()))
            image_np = np.asarray(Image.open(io.BytesIO(img_data)))
            image_np = cv2.flip(image_np, 1)
            gray_image = cv2.cvtColor(image_np, cv2.COLOR_RGB2GRAY)

            faces = detect_faces(face_detection, gray_image)

            # Actual detection. Variable boxes contains the bounding box cordinates for hands detected,
            # while scores contains the confidence for each of these boxes.
            # Hint: If len(boxes) > 1 , you may assume you have found atleast one hand (within your score threshold)

            boxes, scores = detector_utils.detect_objects(
                image_np, detection_graph, sess)

            valid_hands = 0
            calc_displacement = False
            for score in scores:
                if score > min_threshold:
                    valid_hands += 1
            if valid_hands == num_hands_detect:
                calc_displacement = True  #tell function to return new displacement

            # draw bounding boxes on frame
            self.total_displacement += detector_utils.draw_box_on_image(
                num_hands_detect, min_threshold, scores, boxes, im_width,
                im_height, image_np, old_points,
                calc_displacement)  #0.2 is the min threshold
            # Calculate Frames per second (FPS)
            self.num_frames += 1
            elapsed_time = (datetime.datetime.now() -
                            start_time).total_seconds()
            fps = self.num_frames / elapsed_time

            if self.current_emotion in self.emotions:
                self.emotions[self.current_emotion] += 1
            else:
                self.emotions[self.current_emotion] = 1

            print(self.total_displacement / (10 * self.num_frames),
                  self.current_emotion, self.emotions)

            for face_coordinates in faces:
                x1, x2, y1, y2 = apply_offsets(face_coordinates,
                                               emotion_offsets)
                gray_face = gray_image[y1:y2, x1:x2]
                try:
                    gray_face = cv2.resize(gray_face, (emotion_target_size))
                except:
                    continue

                gray_face = preprocess_input(gray_face, True)
                gray_face = np.expand_dims(gray_face, 0)
                gray_face = np.expand_dims(gray_face, -1)
                emotion_prediction = emotion_classifier.predict(gray_face)
                emotion_probability = np.max(emotion_prediction)
                emotion_label_arg = np.argmax(emotion_prediction)
                self.current_emotion = emotion_labels[emotion_label_arg]
                emotion_window.append(self.current_emotion)

                if len(emotion_window) > frame_window:
                    emotion_window.pop(0)
                try:
                    emotion_mode = mode(emotion_window)
                except:
                    continue

                draw_bounding_box(face_coordinates, image_np)

            # Display FPS on frame:
            detector_utils.draw_fps_on_image("FPS : " + str(int(fps)),
                                             image_np)

            cv2.imshow('Single-Threaded Detection',
                       cv2.cvtColor(image_np, cv2.COLOR_RGB2BGR))

            if cv2.waitKey(25) & 0xFF == ord('q'):
                cv2.destroyAllWindows()
                break
        rgb_face = preprocess_input(rgb_face, False)
        gender_prediction = gender_classifier.predict(rgb_face)
        gender_label_arg = np.argmax(gender_prediction)
        gender_text = gender_labels[gender_label_arg]
        gender_window.append(gender_text)

        if len(gender_window) > frame_window:
            emotion_window.pop(0)
            gender_window.pop(0)
        try:
            emotion_mode = mode(emotion_window)
            gender_mode = mode(gender_window)
        except:
            continue

        if gender_text == gender_labels[0]:
            color = (0, 0, 255)
        else:
            color = (255, 0, 0)

        draw_bounding_box(face_coordinates, rgb_image, color)
        draw_text(face_coordinates, rgb_image, gender_mode, color, 0, -20, 1,
                  1)
        draw_text(face_coordinates, rgb_image, emotion_mode, color, 0, -45, 1,
                  1)

    bgr_image = cv2.cvtColor(rgb_image, cv2.COLOR_RGB2BGR)
    cv2.imshow('window_frame', bgr_image)
    if cv2.waitKey(1) & 0xFF == ord('q'):
        break
def emotion_identify(img_url):
    # parameters for loading data and images

    detection_model_path = 'C:/Users/Admin/PycharmProjects/Emotion_Detection/trained_models/detection_models/haarcascade_frontalface_default.xml'
    emotion_model_path = 'C:/Users/Admin/PycharmProjects/Emotion_Detection/trained_models/emotion_models/fer2013_mini_XCEPTION.102-0.66.hdf5'
    gender_model_path = 'C:/Users/Admin/PycharmProjects/Emotion_Detection/trained_models/gender_models/simple_CNN.81-0.96.hdf5'
    emotion_labels = get_labels('fer2013')
    gender_labels = get_labels('imdb')
    font = cv2.FONT_HERSHEY_SIMPLEX

    # hyper-parameters for bounding boxes shape
    gender_offsets = (30, 60)
    gender_offsets = (10, 10)
    emotion_offsets = (20, 40)
    emotion_offsets = (0, 0)

    # loading models
    face_detection = load_detection_model(detection_model_path)
    emotion_classifier = load_model(emotion_model_path, compile=False)
    gender_classifier = load_model(gender_model_path, compile=False)

    # getting input model shapes for inference
    emotion_target_size = emotion_classifier.input_shape[1:3]
    gender_target_size = gender_classifier.input_shape[1:3]
    # loading images
    image_path = img_url
    rgb_image = load_image(image_path, grayscale=False)
    gray_image = load_image(image_path, grayscale=True)
    gray_image = np.squeeze(gray_image)
    gray_image = gray_image.astype('uint8')

    faces = detect_faces(face_detection, gray_image)
    if len(faces) == 0:
        print("No face")
        K.clear_session()
        return False

    emotions = collections.defaultdict(int)
    for face_coordinates in faces:
        x1, x2, y1, y2 = apply_offsets(face_coordinates, gender_offsets)
        rgb_face = rgb_image[y1:y2, x1:x2]

        x1, x2, y1, y2 = apply_offsets(face_coordinates, emotion_offsets)
        gray_face = gray_image[y1:y2, x1:x2]

        try:
            rgb_face = cv2.resize(rgb_face, (gender_target_size))
            gray_face = cv2.resize(gray_face, (emotion_target_size))
        except:
            continue

        rgb_face = preprocess_input(rgb_face, False)
        rgb_face = np.expand_dims(rgb_face, 0)
        gender_prediction = gender_classifier.predict(rgb_face)
        gender_label_arg = np.argmax(gender_prediction)
        gender_text = gender_labels[gender_label_arg]

        gray_face = preprocess_input(gray_face, True)
        gray_face = np.expand_dims(gray_face, 0)
        gray_face = np.expand_dims(gray_face, -1)
        emotion_label_arg = np.argmax(emotion_classifier.predict(gray_face))
        emotion_text = emotion_labels[emotion_label_arg]
        emotions[emotion_text] += 1
        if gender_text == gender_labels[0]:
            color = (0, 0, 255)
        else:
            color = (255, 0, 0)

        draw_bounding_box(face_coordinates, rgb_image, color)
        draw_text(face_coordinates, rgb_image, gender_text, color, 0, -20, 1, 2)
        draw_text(face_coordinates, rgb_image, emotion_text, color, 0, -50, 1, 2)
    max_num = 0
    max_emotion = None
    for key, value in emotions.items():
        if value > max_num:
            max_num = value
            max_emotion = key
    print("The emotion of this picture is: ", max_emotion)
    bgr_image = cv2.cvtColor(rgb_image, cv2.COLOR_RGB2BGR)
    cv2.imwrite('./result_images/predicted_test_image.png', bgr_image)
    K.clear_session()
    return max_emotion