def detection(image_path):
# parameters for loading data and images
detection_model_path = 'C:\\Users\\l1f15bscs0049\\Desktop\\haarcascade_frontalface_default.xml'
font = cv2.FONT_HERSHEY_SIMPLEX
# hyper-parameters for bounding boxes shape
gender_offsets = (30, 60)
gender_offsets = (10, 10)
# loading models
face_detection = load_detection_model(detection_model_path)
# loading images
rgb_image = load_image(image_path, grayscale=False)
gray_image = load_image(image_path, grayscale=True)
gray_image = np.squeeze(gray_image)
gray_image = gray_image.astype('uint8')
count1 = 0
faces = detect_faces(face_detection, gray_image)
for face_coordinates in faces:
x1, x2, y1, y2 = apply_offsets(face_coordinates, gender_offsets)
rgb_face = rgb_image[y1:y2, x1:x2]
rgb_face = preprocess_input(rgb_face, False)
rgb_face = np.expand_dims(rgb_face, 0)
color = (0, 0, 255)
draw_bounding_box(face_coordinates, rgb_image, color)
count1 = count1 + 1
bgr_image = cv2.cvtColor(rgb_image, cv2.COLOR_RGB2BGR)
#cv2.imwrite('C:\\Users\\l1f15bscs0049\\Desktop\\test_cases\\testt2.png', bgr_image)
return count1
def __init__(self):
# parameters for loading data and images
self.detection_model_path = '../trained_models/detection_models/haarcascade_frontalface_default.xml'
self.emotion_model_path = '../trained_models/emotion_models/fer2013_mini_XCEPTION.110-0.65.hdf5'
self.emotion_labels = get_labels('fer2013')
# hyper-parameters for bounding boxes shape
self.frame_window = 10
self.emotion_offsets = (20, 40)
# loading models
self.face_detection = load_detection_model(self.detection_model_path)
self.emotion_classifier = load_model(self.emotion_model_path, compile=False)
# getting input model shapes for inference
self.emotion_target_size = self.emotion_classifier.input_shape[1:3]
# starting lists for calculating modes
self.emotion_window = []
# Initialize the node with rosp
rospy.init_node('publisher_node')
self.emotion_publisher = rospy.Publisher("/qt_face/setEmotion",String,queue_size=10)
self.speech_publisher = rospy.Publisher("/speaker",String,queue_size=10)
self.emotion_msg = String()
self.speech_msg = String()
# starting video streaming
cv2.namedWindow('window_frame')
#video_capture = cv2.VideoCapture(0)
print "init!"
rospy.Subscriber("/naoqi_driver/camera/front/image_raw/compressed", CompressedImage, self.callback, queue_size = 1)
def process_image(image):
try:
# parameters for loading data and images
detection_model_path = './trained_models/detection_models/haarcascade_frontalface_default.xml'
emotion_model_path = './trained_models/emotion_models/fer2013_35a93d1a30cf52b8722837f3e096b7b1006949325de8d00f26595d5a418241d3.hdf5'
emotion_labels = get_labels('fer2013')
font = cv2.FONT_HERSHEY_SIMPLEX
# hyper-parameters for bounding boxes shape
emotion_offsets = (20, 40)
emotion_offsets = (0, 0)
# loading models
face_detection = load_detection_model(detection_model_path)
emotion_classifier = load_model(emotion_model_path, compile=False)
# getting input model shapes for inference
emotion_target_size = emotion_classifier.input_shape[1:3]
# loading images
image_array = np.fromstring(image, np.uint8)
unchanged_image = cv2.imdecode(image_array, cv2.IMREAD_UNCHANGED)
rgb_image = cv2.cvtColor(unchanged_image, cv2.COLOR_BGR2RGB)
gray_image = cv2.cvtColor(unchanged_image, cv2.COLOR_BGR2GRAY)
faces = detect_faces(face_detection, gray_image)
for face_coordinates in faces:
x1, x2, y1, y2 = apply_offsets(face_coordinates, emotion_offsets)
gray_face = gray_image[y1:y2, x1:x2]
try:
gray_face = cv2.resize(gray_face, (emotion_target_size))
except:
continue
rgb_face = preprocess_input(rgb_face, False)
rgb_face = np.expand_dims(rgb_face, 0)
gray_face = preprocess_input(gray_face, True)
gray_face = np.expand_dims(gray_face, 0)
gray_face = np.expand_dims(gray_face, -1)
emotion_label_arg = np.argmax(emotion_classifier.predict(gray_face))
emotion_text = emotion_labels[emotion_label_arg]
color = (255, 0, 0)
draw_bounding_box(face_coordinates, rgb_image, color)
draw_text(face_coordinates, rgb_image, emotion_text, color, 0, -50, 1, 2)
except Exception as err:
logging.error('Error in emotion gender processor: "{0}"'.format(err))
bgr_image = cv2.cvtColor(rgb_image, cv2.COLOR_RGB2BGR)
dirname = 'result'
if not os.path.exists(dirname):
os.mkdir(dirname)
cv2.imwrite(os.path.join(dirname, 'predicted_image.png'), bgr_image)
def emotion_model(self):
os.environ['TF_CPP_MIN_LOG_LEVEL']='2'
# Expect an image through terminal
# parameters for loading data and images
image_path = sys.argv[1]
detection_model_path = '../trained_models/detection_models/haarcascade_frontalface_default.xml'
emotion_model_path = '../trained_models/emotion_models/fer2013_mini_XCEPTION.102-0.66.hdf5'
gender_model_path = '../trained_models/gender_models/simple_CNN.81-0.96.hdf5'
emotion_labels = get_labels('fer2013')
gender_labels = get_labels('imdb')
font = cv2.FONT_HERSHEY_SIMPLEX
# hyper-parameters for bounding boxes shape
# change for variance
gender_offsets = (30, 60)
gender_offsets = (10, 10)
emotion_offsets = (20, 40)
emotion_offsets = (0, 0)
# loading models
face_detection = load_detection_model(detection_model_path)
print("444444")
emotion_classifier = load_model(emotion_model_path, compile=False)
print(emotion_classifier)
print("55555")
gender_classifier = load_model(gender_model_path, compile=False)
return emotion_classifier
def get_Insights(self, path_to_file):
#image_path = '/Users/adelwang/Documents/Hackery/Gender-Age-Expression/GenderExpression/images/joe-biden.jpg'
#path_to_file = '/Users/adelwang/Documents/Hackery/Gender-Age-Expression/GenderExpression/images'
#image_path = '../images/joe-biden.jpg'
#path_to_file = '../GenderExpression/images'
person = Person_Input(path_to_file)
face_detection = load_detection_model(detection_model_path)
emotion_classifier = load_model(emotion_model_path, compile=False)
gender_classifier = load_model(gender_model_path, compile=False)
#file_pi = open('filename_pi.obj', 'w')
#pickle.dump(emotion_classifier, file_pi)
#pickle.dump(emotion_classifier, file_pi)
#pickle.dump(gender_classifier, file_pi)
#image_path = os.listdir(path_to_file)
count_image = 0
image_to_align_ = None
for f in listdir(path_to_file):
if isfile(join(path_to_file,
f)) and not f.startswith('.') and count_image is 0:
image_to_align_ = join(path_to_file, f)
if image_to_align_ is None:
return None
aligned_image, image, rect_nums, XY = person.load_image(
image_to_align_, shape_detector)
#store the data from each of the 5 photos in array of "jsons" called five_insights
five_insights = [None] * 5
count = 0
#**For this demo, only look at the first photo. Need to decide how to average data for 5 photos**
for f in listdir(path_to_file):
if isfile(join(path_to_file,
f)) and not f.startswith('.') and count is 0:
image_path_ = join(path_to_file, f)
emotion, gender = person.get_emotion(image_path_,
face_detection,
emotion_classifier,
gender_classifier)
age = person.get_age(aligned_image, shape_detector)
one_insight = {
'age': int(age),
'gender': gender,
'emotion': emotion
}
five_insights[count] = one_insight
return five_insights[0]
def recognition(f_2_s):
# parameters for loading data and images
#image_path = path_r
image_path = image_handler(f_2_s)
detection_model_path = '../trained_models/detection_models/haarcascade_frontalface_default.xml'
emotion_model_path = '../trained_models/emotion_models/fer2013_mini_XCEPTION.102-0.66.hdf5'
emotion_labels = get_labels('fer2013')
font = cv2.FONT_HERSHEY_SIMPLEX
# hyper-parameters for bounding boxes shape
emotion_offsets = (20, 40)
emotion_offsets = (0, 0)
# loading models
face_detection = load_detection_model(detection_model_path)
emotion_classifier = load_model(emotion_model_path, compile=False)
# getting input model shapes for inference
emotion_target_size = emotion_classifier.input_shape[1:3]
# loading images
rgb_image = load_image(image_path, grayscale=False)
gray_image = load_image(image_path, grayscale=True)
gray_image = np.squeeze(gray_image)
gray_image = gray_image.astype('uint8')
faces = detect_faces(face_detection, gray_image)
for face_coordinates in faces:
x1, x2, y1, y2 = apply_offsets(face_coordinates, emotion_offsets)
gray_face = gray_image[y1:y2, x1:x2]
try:
gray_face = cv2.resize(gray_face, (emotion_target_size))
except:
continue
gray_face = preprocess_input(gray_face, True)
gray_face = np.expand_dims(gray_face, 0)
gray_face = np.expand_dims(gray_face, -1)
emotion_label_arg = np.argmax(emotion_classifier.predict(gray_face))
emotion_text = ""
emotion_text = emotion_labels[emotion_label_arg]
color = (255, 0, 0) # 감정 정보 글씨 빨간색, 사각형도
draw_bounding_box(face_coordinates, rgb_image, color)
draw_text(face_coordinates, rgb_image, emotion_text, color, 0, -30,
1.5, 2)
#if(emotion_text == ""):
#recognition(f_2_s)
bgr_image = cv2.cvtColor(rgb_image, cv2.COLOR_RGB2BGR)
cv2.imwrite('../images/predicted_test_image.png', bgr_image) # 변수 활용
check_recoged_img('../images/predicted_test_image.png')
def get_Insights(image_path):
face_detection = load_detection_model(detection_model_path)
emotion_classifier = load_model(emotion_model_path, compile=False)
gender_classifier = load_model(gender_model_path, compile=False)
emotion, gender_two = get_emotion(image_path, face_detection,
emotion_classifier, gender_classifier)
return emotion
'''
def set_Emotion(self):
self.emotion_model_path = '../trained_models/emotion_models/fer2013_mini_XCEPTION.102-0.66.hdf5'
self.emotion_labels = get_labels('fer2013')
self.emotion_classifier = load_model(self.emotion_model_path,
compile=False)
self.emotion_target_size = self.emotion_classifier.input_shape[1:3]
self.emotion_offsets = (20, 40)
# Faces
self.detection_model_path = '../trained_models/detection_models/haarcascade_frontalface_default.xml'
self.face_detection = load_detection_model(self.detection_model_path)
return
def get_face(frame):
detection_model_path = './face.xml'
face_detection = load_detection_model(detection_model_path)
try:
gray_image = cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY)
faces = detect_faces(face_detection, gray_image)
except Exception as e:
print("Exception:", e)
return frame
for face in faces:
draw_bounding_box(face, frame, (255, 0, 0))
return frame
def emotion():
detection_model_path = '../trained_models/detection_models/haarcascade_frontalface_default.xml'
emotion_model_path = '../trained_models/emotion_models/fer2013_big_XCEPTION.54-0.66.hdf5'
emotion_labels = get_labels('fer2013')
# hyper-parameters for bounding boxes shape
frame_window = 10
emotion_offsets = (20, 40)
# loading models
face_detection = load_detection_model(detection_model_path)
emotion_classifier = load_model(emotion_model_path, compile=False)
# getting input model shapes for inference
emotion_target_size = emotion_classifier.input_shape[1:3]
# starting lists for calculating modes
emotion_window = []
emotion_count_array = [0 for x in range(7)];
# starting video streaming
#cv2.namedWindow('window_frame')
video_capture = cv2.VideoCapture(0)
duration = 10;
start = time.time();
while (time.time() - start) < duration:
bgr_image = video_capture.read()[1];
gray_image = cv2.cvtColor(bgr_image, cv2.COLOR_BGR2GRAY)
rgb_image = cv2.cvtColor(bgr_image, cv2.COLOR_BGR2RGB)
faces = detect_faces(face_detection, gray_image)
for face_coordinates in faces:
x1, x2, y1, y2 = apply_offsets(face_coordinates, emotion_offsets)
gray_face = gray_image[y1:y2, x1:x2]
try:
gray_face = cv2.resize(gray_face, (emotion_target_size))
except:
continue
gray_face = preprocess_input(gray_face, True)
gray_face = np.expand_dims(gray_face, 0)
gray_face = np.expand_dims(gray_face, -1)
emotion_prediction = emotion_classifier.predict(gray_face)
emotion_probability = np.max(emotion_prediction)
emotion_label_arg = np.argmax(emotion_prediction)
emotion_text = emotion_labels[emotion_label_arg]
emotion_count_array[emotion_label_arg] = emotion_count_array[emotion_label_arg] + 1;
chosen = np.argmax(emotion_count_array);
return emotion_labels[chosen]
def modeling():
# load the pre-trained Keras model (here we are using a model
# pre-trained on ImageNet and provided by Keras, but you can
# substitute in your own networks just as easily)
global face_detection
face_detection = load_detection_model(detection_model_path)
global emotion_classifier
emotion_classifier = load_model(emotion_model_path, compile=False)
global graph
graph = tf.get_default_graph()
global emotion_target_size
emotion_target_size = emotion_classifier.input_shape[1:3]
def getFacialInsights(self, path_to_file):
if path_to_file is None:
return 0
image_path = '/Users/adelwang/Documents/Hackery/Gender-Age-Expression/GenderExpression2/images/obama1.jpg'
path_to_file = '/Users/adelwang/Documents/Hackery/Gender-Age-Expression/GenderExpression2/images'
shape_detector = "shape_predictor_68_face_landmarks.dat"
model_path = "./models/"
person = Person_Input(path_to_file)
face_detection = load_detection_model(detection_model_path)
emotion_classifier = load_model(emotion_model_path, compile=False)
gender_classifier = load_model(gender_model_path, compile=False)
#aligned_image, image, rect_nums, XY = person.load_image(image_path, shape_detector)
five_insights = [None] * 5
count = 0
sess, age_, gender_, train_mode, images_pl = person.load_network(
"./models")
for f in listdir(path_to_file):
if isfile(join(path_to_file, f)) and not f.startswith('.'):
image_path_ = join(path_to_file, f)
emotion = person.get_emotion(image_path_, face_detection,
emotion_classifier, gender_classifier)
gender = person.get_gender(image_path_, face_detection,
emotion_classifier, gender_classifier)
#Another way to get age.
#sess, age, gender_, train_mode,images_pl = person.load_network("./models")
#aligned_image, image, rect_nums, XY = person.load_image(image_path_, shape_detector)
#age = person.get_age(aligned_image, shape_detector)
age = person.get_age2(sess, age_, gender_, train_mode, images_pl,
image_path_)
result = person.predict_ethnic(image_path)
ethnicity = ETHNIC[np.argmax(result)]
#one_person_insight_ = {'age':int(age), 'gender':gender, 'expression':emotion, 'ethnicity': ethnicity}
one_person_insight_ = {
'age': int(age),
'gender': gender,
'expression': emotion
}
one_person_insight = json.dumps(one_person_insight_)
five_insights[count] = one_person_insight
count += 1
#print(five_insights)
return five_insights
def __init__(self):
rospy.init_node('image_converter', anonymous=True)
rospy.loginfo("recognizer started")
print "1................................................"
self.bridge = CvBridge()
self._image_topic_output = "~image_topic_output"
print rospy.has_param(self._image_topic_output)
if rospy.has_param(self._image_topic_output):
self.image_topic_output = rospy.get_param(self._image_topic_output)
self.image_pub = rospy.Publisher(self.image_topic_output,
Image,
queue_size=10)
self._image_topic_input = "~image_topic_input"
print rospy.has_param(self._image_topic_input)
if rospy.has_param(self._image_topic_input):
self.image_topic_input = rospy.get_param(self._image_topic_input)
self.image_sub = rospy.Subscriber(self.image_topic_input, Image,
self.callback)
self._detection_model_path = "~detection_model_path"
print rospy.has_param(self._detection_model_path)
if rospy.has_param(self._detection_model_path):
self.detection_model_path = rospy.get_param(
self._detection_model_path)
self._emotion_model_path = "~emotion_model_path"
print rospy.has_param(self._emotion_model_path)
if rospy.has_param(self._emotion_model_path):
self.emotion_model_path = rospy.get_param(self._emotion_model_path)
self.emotion_labels = get_labels('fer2013')
# hyper-parameters for bounding boxes shape
self.frame_window = 10
self.emotion_offsets = (20, 40)
# loading models
self.face_detection = load_detection_model(self.detection_model_path)
self.emotion_classifier = load_model(self.emotion_model_path,
compile=False)
# getting input model shapes for inference
self.emotion_target_size = self.emotion_classifier.input_shape[1:3]
# starting lists for calculating modes
self.emotion_window = []
def make_label(image_path):
# loading models
face_detection = load_detection_model(detection_model_path)
emotion_classifier = load_model(emotion_model_path, compile=False)
gender_classifier = load_model(gender_model_path, compile=False)
# getting input model shapes for inference
emotion_target_size = emotion_classifier.input_shape[1:3]
gender_target_size = gender_classifier.input_shape[1:3]
# loading images
rgb_image = load_image(image_path, grayscale=False)
gray_image = load_image(image_path, grayscale=True)
gray_image = np.squeeze(gray_image)
gray_image = gray_image.astype('uint8')
faces = detect_faces(face_detection, gray_image)
for face_coordinates in faces:
x1, x2, y1, y2 = apply_offsets(face_coordinates, gender_offsets)
rgb_face = rgb_image[y1:y2, x1:x2]
x1, x2, y1, y2 = apply_offsets(face_coordinates, emotion_offsets)
gray_face = gray_image[y1:y2, x1:x2]
try:
rgb_face = cv2.resize(rgb_face, (gender_target_size))
gray_face = cv2.resize(gray_face, (emotion_target_size))
except:
continue
rgb_face = preprocess_input(rgb_face, False)
rgb_face = np.expand_dims(rgb_face, 0)
gender_prediction = gender_classifier.predict(rgb_face)
gender_label_arg = np.argmax(gender_prediction)
gender_text = gender_labels[gender_label_arg]
gray_face = preprocess_input(gray_face, True)
gray_face = np.expand_dims(gray_face, 0)
gray_face = np.expand_dims(gray_face, -1)
emotion_label_arg = np.argmax(emotion_classifier.predict(gray_face))
emotion_text = emotion_labels[emotion_label_arg]
if gender_text == gender_labels[0]:
color = (0, 0, 255)
else:
color = (255, 0, 0)
return emotion_text
def getFaceEmotion(image_path):
# parameters for loading data and images
base = 'C://Users/lenovo/Desktop/moodify/components/emotion-classification'
detection_model_path = base + '/trained_models/detection_models/haarcascade_frontalface_default.xml'
emotion_model_path = base + '/trained_models/emotion_models/fer2013_denseNet.59-0.68.hdf5'
emotion_labels = get_labels('fer2013')
font = cv2.FONT_HERSHEY_SIMPLEX
# hyper-parameters for bounding boxes shape
emotion_offsets = (0, 0)
# loading models
face_detection = load_detection_model(detection_model_path)
emotion_classifier = load_model(emotion_model_path, compile=False)
# getting input model shapes for inference
emotion_target_size = emotion_classifier.input_shape[1:3]
# loading images
rgb_image = load_image(image_path, grayscale=False)
gray_image = load_image(image_path, grayscale=True)
gray_image = np.squeeze(gray_image)
gray_image = gray_image.astype('uint8')
faces = detect_faces(face_detection, gray_image)
for face_coordinates in faces:
x1, x2, y1, y2 = apply_offsets(face_coordinates, emotion_offsets)
gray_face = gray_image[y1:y2, x1:x2]
try:
gray_face = cv2.resize(gray_face, (emotion_target_size))
except:
continue
gray_face = preprocess_input(gray_face, True)
gray_face = np.expand_dims(gray_face, 0)
gray_face = np.expand_dims(gray_face, -1)
emotion_label_arg = np.argmax(emotion_classifier.predict(gray_face))
emotion_text = emotion_labels[emotion_label_arg]
color = (255, 0, 0)
draw_bounding_box(face_coordinates, rgb_image, color)
draw_text(face_coordinates, rgb_image, emotion_text, color, 0, -10,
0.5, 2)
bgr_image = cv2.cvtColor(rgb_image, cv2.COLOR_RGB2BGR)
cv2.imwrite('predicted_test_image.png', bgr_image)
return emotion_text
def __init__(self):
rospy.init_node('image_converter', anonymous=True)
rospy.loginfo("recognizer started")
print "1................................................"
self.bridge = CvBridge()
self._image_topic_output = "~image_topic_output"
print rospy.has_param(self._image_topic_output)
if rospy.has_param(self._image_topic_output):
self.image_topic_output = rospy.get_param(self._image_topic_output)
self.image_pub = rospy.Publisher(self.image_topic_output, Image, queue_size=10)
self._image_topic_input = "~image_topic_input"
print rospy.has_param(self._image_topic_input)
if rospy.has_param(self._image_topic_input):
self.image_topic_input = rospy.get_param(self._image_topic_input)
self.image_sub = rospy.Subscriber(self.image_topic_input, Image, self.callback)
self._detection_model_path = "~detection_model_path"
print rospy.has_param(self._detection_model_path)
if rospy.has_param(self._detection_model_path):
self.detection_model_path = rospy.get_param(self._detection_model_path)
self._emotion_model_path = "~emotion_model_path"
print rospy.has_param(self._emotion_model_path)
if rospy.has_param(self._emotion_model_path):
self.emotion_model_path = rospy.get_param(self._emotion_model_path)
self.emotion_labels = get_labels('fer2013')
# hyper-parameters for bounding boxes shape
self.frame_window = 10
self.emotion_offsets = (20, 40)
# loading models
self.face_detection = load_detection_model(self.detection_model_path)
self.emotion_classifier = load_model(self.emotion_model_path, compile=False)
# getting input model shapes for inference
self.emotion_target_size = self.emotion_classifier.input_shape[1:3]
# starting lists for calculating modes
self.emotion_window = []
def process_image(image):
try:
# parameters for loading data and images
detection_model_path = './trained_models/detection_models/haarcascade_frontalface_default.xml'
emotion_model_path = './trained_models/emotion_models/fer2013_mini_XCEPTION.102-0.66.hdf5'
emotion_labels = get_labels('fer2013')
# hyper-parameters for bounding boxes shape
emotion_offsets = (0, 0)
# loading models
face_detection = load_detection_model(detection_model_path)
emotion_classifier = load_model(emotion_model_path, compile=False)
# getting input model shapes for inference
emotion_target_size = emotion_classifier.input_shape[1:3]
# loading images
image_array = np.fromstring(image, np.uint8)
unchanged_image = cv2.imdecode(image_array, cv2.IMREAD_UNCHANGED)
gray_image = cv2.cvtColor(unchanged_image, cv2.COLOR_BGR2GRAY)
faces = detect_faces(face_detection, gray_image)
detected_emotions = []
for face_coordinates in faces:
x1, x2, y1, y2 = apply_offsets(face_coordinates, emotion_offsets)
gray_face = gray_image[y1:y2, x1:x2]
try:
gray_face = cv2.resize(gray_face, emotion_target_size)
except:
continue
gray_face = preprocess_input(gray_face, True)
gray_face = np.expand_dims(gray_face, 0)
gray_face = np.expand_dims(gray_face, -1)
emotion_label_arg = np.argmax(emotion_classifier.predict(gray_face))
detected_emotions.append({
"coordinates": [str(y1), str(x2), str(y2), str(x1)],
"emotion": emotion_labels[emotion_label_arg]})
return detected_emotions
except Exception as err:
logging.error('Error in emotion gender processor: "{0}"'.format(err))
def __init__(self):
detection_model_path = BASE_DIR + '/../trained_models/detection_models/haarcascade_frontalface_default.xml'
emotion_model_path = BASE_DIR + '/../trained_models/emotion_models/fer2013_mini_XCEPTION.102-0.66.hdf5'
gender_model_path = BASE_DIR + '/../trained_models/gender_models/simple_CNN.81-0.96.hdf5'
self.emotion_labels = get_labels('fer2013')
self.gender_labels = get_labels('imdb')
self.font = cv2.FONT_HERSHEY_SIMPLEX
# loading models
self.face_detection = load_detection_model(detection_model_path)
self.emotion_classifier = load_model(emotion_model_path, compile=False)
self.gender_classifier = load_model(gender_model_path, compile=False)
# getting input model shapes for inference
self.emotion_target_size = self.emotion_classifier.input_shape[1:3]
self.gender_target_size = self.gender_classifier.input_shape[1:3]
# hyper-parameters for bounding boxes shape
self.gender_offsets = (30, 60)
self.gender_offsets = (10, 10)
self.emotion_offsets = (20, 40)
self.emotion_offsets = (0, 0)
def __init__(self, img=None, use_haar_cascade=False):
self.emotion_window = []
self.use_haar_cascade = use_haar_cascade
if self.use_haar_cascade:
self.face_detection_model = load_detection_model(emotion_recognition.haar_cascade_path)
# print('[INFO] Haar-cascade loaded')
# else:
# # Use MTCNN face detector
# self.min_face_size = 50
# self.threshold = [0.6, 0.7, 0.7]
# self.scale_factor = 0.709
# with tf.Session().as_default() as sess:
# self.pnet, self.rnet, self.onet = detect_face.create_mtcnn(sess=sess, model_path=None)
if img is not None:
self.img = img
self.emotion_labels = emotion_recognition.emotion_labels
global emotion_classifier
emotion_classifier = tf.keras.models.load_model(emotion_recognition.emotion_model_path, compile=False)
print('[Emotions] FER loaded')
def main():
warnings.filterwarnings("ignore")
logging.disable(logging.CRITICAL)
global emotion_cache #Vishal
emotion_cache = [] #Vishal
global stop_detecting
#Create server client for socket connection
port = opt.port
#host_name = socket.gethostname()
#host_ip = socket.gethostbyname(host_name)
host_ip = opt.ip
mySocket = socket.socket()
mySocket.bind((host_ip, port))
print ("Camera server listening on port {0} ".format(port)+ "and IP " + str(host_ip))
mySocket.listen(1)
chatbot_socket, adress = mySocket.accept()
print("Connection established to: " + str(adress))
#Start socket process to listen to chatbot requests
chatbot = Process(target = chatbot_process, args = (chatbot_socket, chatbot_request, chatbot_response, stop_camera))
chatbot.start()
signal.signal(signal.SIGTSTP, handler)
#face_recognizer = Face_Recognizer(True)# True to show video feed
# parameters for loading data and images
detection_model_path = './models/face_recognition/trained_models/haarcascade_frontalface_default.xml'
emotion_model_path = './models/face_recognition/trained_models/fer2013_mini_XCEPTION.102-0.66.hdf5'
emotion_labels = get_labels('fer2013')
# hyper-parameters for bounding boxes shape
frame_window = 10
emotion_offsets = (20, 40)
# loading models
face_detection = load_detection_model(detection_model_path)
emotion_classifier = load_model(emotion_model_path, compile=False)
# getting input model shapes for inference
emotion_target_size = emotion_classifier.input_shape[1:3]
# starting lists for calculating modes
emotion_window = []
emotion_text = 'Happy'
# starting video streaming
cv2.namedWindow('ODO_frame')
OdoSet='nvarguscamerasrc ! video/x-raw(memory:NVMM), width=3820, height=2464, format=NV12, framerate=21/1 ! nvvidconv flip-method='+str(flip)+' ! video/x-raw, width='+str(dispW)+', height='+str(dispH)+', format=BGRx ! videoconvert ! video/x-raw, format=BGR ! appsink'
video_capture = cv2.VideoCapture(OdoSet)
fps = video_capture.get(cv2.CAP_PROP_FPS)
print(f'FPS: {fps}')
num_frames = 21;
if video_capture.isOpened():
print("True")
else:
print("Error")
##################Vishal#####################
while stop_detecting == False:
#############################################
# Detect Emotion
#print("Check")
start = time.time() #starting FPS timer
bgr_image = video_capture.read()[1]
gray_image = cv2.cvtColor(bgr_image, cv2.COLOR_BGR2GRAY)
rgb_image = cv2.cvtColor(bgr_image, cv2.COLOR_BGR2RGB)
faces = detect_faces(face_detection, gray_image)
for face_coordinates in faces:
x1, x2, y1, y2 = apply_offsets(face_coordinates, emotion_offsets)
gray_face = gray_image[y1:y2, x1:x2]
try:
gray_face = cv2.resize(gray_face, (emotion_target_size))
except:
continue
gray_face = preprocess_input(gray_face, True)
gray_face = np.expand_dims(gray_face, 0)
gray_face = np.expand_dims(gray_face, -1)
emotion_prediction = emotion_classifier.predict(gray_face)
emotion_probability = np.max(emotion_prediction)
emotion_label_arg = np.argmax(emotion_prediction)
emotion_text = emotion_labels[emotion_label_arg]
emotion_window.append(emotion_text)
if len(emotion_window) > frame_window:
emotion_window.pop(0)
try:
emotion_mode = mode(emotion_window)
except:
continue
if emotion_text == 'angry':
color = emotion_probability * np.asarray((255, 0, 0))
elif emotion_text == 'sad':
color = emotion_probability * np.asarray((0, 0, 255))
elif emotion_text == 'happy':
color = emotion_probability * np.asarray((255, 255, 0))
elif emotion_text == 'surprise':
color = emotion_probability * np.asarray((0, 255, 255))
else:
color = emotion_probability * np.asarray((0, 255, 0))
emotion_cache.append(emotion_text)
with open('listfile.txt', 'w') as filehandle:
filehandle.writelines("%s\n" % emotions for emotions in emotion_cache)
if len(emotion_cache) > 500:
emotion_cache = emotion_cache[:-500]
##########################################
total_faces = len(faces)
current_time = time.time()
if chatbot_request.empty() == False:
request = chatbot_request.get()
data_request = request['request']
if data_request.lower() == 'send_emotion':
data = {}
data['emotion'] = emotion_detected
data['total_faces'] = total_faces
data['time'] = current_time
json_data = json.dumps(data)
chatbot_response.put({'response': json_data})
color = color.astype(int)
color = color.tolist()
#print(face_coordinates, rgb_image, color)
draw_bounding_box(face_coordinates, rgb_image, color)
draw_text(face_coordinates, rgb_image, emotion_mode,
color, 0, -45, 1, 1)
if stop_camera.empty() == False:
stop_detecting = True
end = time.time() #end time for FPS
bgr_image = cv2.cvtColor(rgb_image, cv2.COLOR_RGB2BGR)
seconds = end - start #FPS
fps = num_frames/seconds #FPS
fps_coords = (0,30)
framerate_str = 'Framerate: ' + str(int(fps))
cv2.putText(bgr_image, framerate_str, fps_coords, cv2.FONT_HERSHEY_SIMPLEX, 1, (255,255,200))
print(framerate_str)
cv2.imshow('ODO_frame', bgr_image)
if cv2.waitKey(1) & 0xFF == ord('q'):
break
video_capture.release()
cv2.destroyAllWindows()
print("Closing chatbot socket")
chatbot.terminate()
chatbot.join()
model_filename = '../trained_models/emotion_models/fer2013_mini_XCEPTION.102-0.66.hdf5'
# model_filename = '../trained_models/fer2013_big_XCEPTION.54-0.66.hdf5'
class_to_arg = get_class_to_arg('fer2013')
# predicted_class = class_to_arg[class_name]
predicted_class = 0
offsets = (0, 0)
model = load_model(model_filename, compile=False)
gradient_function = compile_gradient_function(model, predicted_class, 'conv2d_7')
register_gradient()
guided_model = modify_backprop(model, 'GuidedBackProp', task)
saliency_function = compile_saliency_function(guided_model, 'conv2d_7')
# parameters for loading data and images
detection_model_path = '../trained_models/detection_models/haarcascade_frontalface_default.xml'
face_detection = load_detection_model(detection_model_path)
color = (0, 255, 0)
# getting input model shapes for inference
target_size = model.input_shape[1:3]
# starting lists for calculating modes
emotion_window = []
# starting video streaming
cv2.namedWindow('window_frame')
video_capture = cv2.VideoCapture(0)
while True:
bgr_image = video_capture.read()[1]
gray_image = cv2.cvtColor(bgr_image, cv2.COLOR_BGR2GRAY)
rgb_image = cv2.cvtColor(bgr_image, cv2.COLOR_BGR2RGB)
def emotion_demo():
# parameters for loading data and images
detection_model_path = '../trained_models/detection_models/haarcascade_frontalface_default.xml'
emotion_model_path = '../trained_models/emotion_models/fer2013_mini_XCEPTION.102-0.66.hdf5'
emotion_labels = get_labels('fer2013')
# 目のカスケードファイル追加
lefteyecc__path = "../trained_models/detection_models/haarcascade_lefteye_2splits.xml"
righteyecc_path = "../trained_models/detection_models/haarcascade_righteye_2splits.xml"
nose_path = "../trained_models/detection_models/data_haarcascades_haarcascade_mcs_nose.xml"
lefteyecc = cv2.CascadeClassifier(lefteyecc__path)
righteyecc = cv2.CascadeClassifier(righteyecc_path)
nose = cv2.CascadeClassifier(nose_path)
lex = 0; ley = 0; lew = 0; leh = 0
rex = 0; rey = 0; rew = 0; reh = 0
nox = 0; noy = 0; now = 0; noh = 0
# dlib
dlib_ini()
# hyper-parameters for bounding boxes shape
frame_window = 10
emotion_offsets = (20, 40)
# loading models
face_detection = load_detection_model(detection_model_path)
emotion_classifier = load_model(emotion_model_path, compile=False)
# getting input model shapes for inference
emotion_target_size = emotion_classifier.input_shape[1:3]
# starting lists for calculating modes
emotion_window = []
global img, flag, slp_count
img = cv2.imread('../img/happy.png')
flag = 0
slp_count = 0
# dlib用グローバル変数
global gray_image, rgb_image, gray_face, mark_list
# starting video streaming
cv2.namedWindow('window_frame', cv2.WINDOW_NORMAL)
video_capture = cv2.VideoCapture(0) # 0は内蔵カメラ, 1はUSBカメラ
while True:
bgr_image = video_capture.read()[1]
gray_image = cv2.cvtColor(bgr_image, cv2.COLOR_BGR2GRAY)
rgb_image = cv2.cvtColor(bgr_image, cv2.COLOR_BGRA2RGBA)
faces = detect_faces(face_detection, gray_image)
for face_coordinates in faces:
# 目や鼻認識用
(x,y,w,h) = face_coordinates
video_face = gray_image[y:y+h,x:x+w]
x1, x2, y1, y2 = apply_offsets(face_coordinates, emotion_offsets)
gray_face = gray_image[y1:y2, x1:x2]
try:
gray_face = cv2.resize(gray_face, (emotion_target_size))
except:
continue
# ランドマーク検出
marks_list = marks_list_def(bgr_image, x, y, w, h)
print(marks_list)
gray_face = preprocess_input(gray_face, True)
gray_face = np.expand_dims(gray_face, 0)
gray_face = np.expand_dims(gray_face, -1)
emotion_prediction = emotion_classifier.predict(gray_face)
emotion_probability = np.max(emotion_prediction)
emotion_label_arg = np.argmax(emotion_prediction)
emotion_text = emotion_labels[emotion_label_arg]
emotion_window.append(emotion_text)
if len(emotion_window) > frame_window:
emotion_window.pop(0)
try:
emotion_mode = mode(emotion_window)
except:
continue
if flag == 0 or flag == 1:
if emotion_text == 'angry':
img = cv2.imread('../img/angry.png', -1)
color = emotion_probability * np.asarray((255, 0, 0))
elif emotion_text == 'sad':
img = cv2.imread('../img/sad.png', -1) # 関数にする
color = emotion_probability * np.asarray((0, 0, 255))
elif emotion_text == 'happy':
img = cv2.imread('../img/happy.png', -1)
color = emotion_probability * np.asarray((255, 255, 0))
elif emotion_text == 'surprise':
img = cv2.imread('../img/odoroki.png', -1)
color = emotion_probability * np.asarray((0, 255, 255))
else :
img = cv2.imread('../img/neutral.png', -1)
color = emotion_probability * np.asarray((0, 255, 0))
else:
if emotion_text == 'angry':
img = cv2.imread('../img/ikari.png', -1)
color = emotion_probability * np.asarray((255, 0, 0))
elif emotion_text == 'sad':
img = cv2.imread('../img/shock.png', -1)
color = emotion_probability * np.asarray((0, 0, 255))
elif emotion_text == 'happy':
img = cv2.imread('../img/kirakira.png', -1)
color = emotion_probability * np.asarray((255, 255, 0))
elif emotion_text == 'surprise':
img = cv2.imread('../img/bikkuri.png', -1)
color = emotion_probability * np.asarray((0, 255, 255))
else :
img = cv2.imread('../img/toumei.png', -1)
color = emotion_probability * np.asarray((0, 255, 0))
color = color.astype(int)
color = color.tolist()
if flag == 0:
draw_bounding_box(face_coordinates, rgb_image, color)
elif flag == 1:
rgb_image = draw_bounding_box2(face_coordinates, rgb_image, color, img, marks_list)
elif flag == 2:
overlay_pic = draw_bounding_box3(face_coordinates, rgb_image, color, img, marks_list)
rgb_image = overlay_pic
draw_text(face_coordinates, rgb_image, emotion_mode, color, 0, -45, 1, 1)
bgr_image = cv2.cvtColor(rgb_image, cv2.COLOR_RGB2BGR)
if flag == 0:
img = image_resize(img)
cv2.imshow('image', img)
cv2.destroyWindow('Window_frame')
elif flag == 1 or flag == 2:
cv2.destroyWindow('image')
cv2.imshow('window_frame', bgr_image)
cv2.waitKey(10)
# cv2.imshow('own_window', bgr_image)
if cv2.waitKey(1) & 0xFF == ord('z'):
flag = 0
elif cv2.waitKey(1) & 0xFF == ord('x'):
flag = 1
elif cv2.waitKey(1) & 0xFF == ord('c'):
flag = 2
elif cv2.waitKey(1) & 0xFF == ord('q'):
break
video_capture.release()
cv2.destroyAllWindows()
def process_image(image):
try:
# parameters for loading data and images
detection_model_path = './trained_models/detection_models/haarcascade_frontalface_default.xml'
emotion_model_path = './trained_models/emotion_models/fer2013_mini_XCEPTION.102-0.66.hdf5'
gender_model_path = './trained_models/gender_models/simple_CNN.81-0.96.hdf5'
emotion_labels = get_labels('fer2013')
gender_labels = get_labels('imdb')
# hyper-parameters for bounding boxes shape
gender_offsets = (10, 10)
emotion_offsets = (0, 0)
# loading models
face_detection = load_detection_model(detection_model_path)
emotion_classifier = load_model(emotion_model_path, compile=False)
gender_classifier = load_model(gender_model_path, compile=False)
# getting input model shapes for inference
emotion_target_size = emotion_classifier.input_shape[1:3]
gender_target_size = gender_classifier.input_shape[1:3]
# loading images
image_array = np.fromstring(image, np.uint8)
unchanged_image = cv2.imdecode(image_array, cv2.IMREAD_UNCHANGED)
rgb_image = cv2.cvtColor(unchanged_image, cv2.COLOR_BGR2RGB)
gray_image = cv2.cvtColor(unchanged_image, cv2.COLOR_BGR2GRAY)
faces = detect_faces(face_detection, gray_image)
for face_coordinates in faces:
x1, x2, y1, y2 = apply_offsets(face_coordinates, gender_offsets)
rgb_face = rgb_image[y1:y2, x1:x2]
x1, x2, y1, y2 = apply_offsets(face_coordinates, emotion_offsets)
gray_face = gray_image[y1:y2, x1:x2]
try:
rgb_face = cv2.resize(rgb_face, gender_target_size)
gray_face = cv2.resize(gray_face, emotion_target_size)
except:
continue
rgb_face = preprocess_input(rgb_face, False)
rgb_face = np.expand_dims(rgb_face, 0)
gender_prediction = gender_classifier.predict(rgb_face)
gender_label_arg = np.argmax(gender_prediction)
gender_text = gender_labels[gender_label_arg]
gray_face = preprocess_input(gray_face, True)
gray_face = np.expand_dims(gray_face, 0)
gray_face = np.expand_dims(gray_face, -1)
emotion_label_arg = np.argmax(
emotion_classifier.predict(gray_face))
emotion_text = emotion_labels[emotion_label_arg]
if gender_text == gender_labels[0]:
color = (0, 0, 255)
else:
color = (255, 0, 0)
draw_bounding_box(face_coordinates, rgb_image, color)
draw_text(face_coordinates, rgb_image, gender_text, color, 0, -20,
1, 2)
draw_text(face_coordinates, rgb_image, emotion_text, color, 0, -50,
1, 2)
except Exception as err:
logging.error('Error in emotion gender processor: "{0}"'.format(err))
bgr_image = cv2.cvtColor(rgb_image, cv2.COLOR_RGB2BGR)
dirname = 'result'
if not os.path.exists(dirname):
os.mkdir(dirname)
cv2.imwrite(os.path.join(dirname, 'predicted_image.png'), bgr_image)
def get_sim_percent(image_1, image_2, target_class):
# parameters for loading data and images
# image_path = sys.argv[1]
detection_model_path = 'trained_models/detection_models/haarcascade_frontalface_default.xml'
emotion_model_path = 'trained_models/fer2013_big_XCEPTION.54-0.66.hdf5'
gender_model_path = 'trained_models/gender_models/simple_CNN.81-0.96.hdf5'
emotion_labels = get_labels('fer2013')
gender_labels = get_labels('imdb')
font = cv2.FONT_HERSHEY_SIMPLEX
# hyper-parameters for bounding boxes shape
gender_offsets = (20, 40)
gender_offsets = (10, 10)
emotion_offsets = (10, 30)
emotion_offsets = (0, 0)
# loading models
face_detection = load_detection_model(detection_model_path)
emotion_classifier = load_model(emotion_model_path, compile=False)
gender_classifier = load_model(gender_model_path, compile=False)
# getting input model shapes for inference
emotion_target_size = emotion_classifier.input_shape[1:3]
gender_target_size = gender_classifier.input_shape[1:3]
target_emo = target_class
image_path = image_1
# loading images
rgb_image = load_image(image_path, grayscale=False)
gray_image = load_image(image_path, grayscale=True)
gray_image = np.squeeze(gray_image)
gray_image = gray_image.astype('uint8')
faces = detect_faces(face_detection, gray_image)
for face_coordinates in faces:
x1, x2, y1, y2 = apply_offsets(face_coordinates, gender_offsets)
rgb_face = rgb_image[y1:y2, x1:x2]
x1, x2, y1, y2 = apply_offsets(face_coordinates, emotion_offsets)
gray_face = gray_image[y1:y2, x1:x2]
try:
rgb_face = cv2.resize(rgb_face, (gender_target_size))
gray_face = cv2.resize(gray_face, (emotion_target_size))
except:
continue
rgb_face = preprocess_input(rgb_face, False)
rgb_face = np.expand_dims(rgb_face, 0)
gender_prediction = gender_classifier.predict(rgb_face)
gender_label_arg = np.argmax(gender_prediction)
gender_text = gender_labels[gender_label_arg]
gray_face = preprocess_input(gray_face, True)
gray_face = np.expand_dims(gray_face, 0)
gray_face = np.expand_dims(gray_face, -1)
emotion_label_arg = emotion_classifier.predict(gray_face)[0][target_emo]
emotion_percent = str(round(emotion_label_arg*100, 2))
emotion_text = emotion_labels[target_emo] + ' - '+ emotion_percent
# print(target_emo)
# print(emotion_text)
if gender_text == gender_labels[0]:
color = (0, 0, 255)
else:
color = (255, 0, 0)
draw_bounding_box(face_coordinates, rgb_image, color)
draw_text(face_coordinates, rgb_image, gender_text, color, -20, 30, 1, 2)
draw_text(face_coordinates, rgb_image, emotion_text, color, 0, 0, 1, 2)
bgr_image = cv2.cvtColor(rgb_image, cv2.COLOR_RGB2BGR)
cv2.imwrite('predicted_test_image.png', bgr_image)
return emotion_percent
def run(self):
detection_graph, sess = detector_utils.load_inference_graph()
# parameters for loading data and images
detection_model_path = 'haarcascade_frontalface_default.xml'
emotion_model_path = 'fer2013_mini_XCEPTION.102-0.66.hdf5'
emotion_labels = get_labels('fer2013')
# hyper-parameters for bounding boxes shape
frame_window = 10
emotion_offsets = (20, 40)
# loading models
face_detection = load_detection_model(detection_model_path)
emotion_classifier = load_model(emotion_model_path, compile=False)
# getting input model shapes for inference
emotion_target_size = emotion_classifier.input_shape[1:3]
# starting lists for calculating modes
emotion_window = []
start_time = datetime.datetime.now()
im_width, im_height = (400, 350)
num_hands_detect = 2 # max number of hands we want to detect/track, can scale this up
min_threshold = 0.2
old_points = [None] * num_hands_detect
cv2.namedWindow('Single-Threaded Detection', cv2.WINDOW_NORMAL)
while True:
# Expand dimensions since the model expects images to have shape: [1, None, None, 3]
if self.image_queue.empty():
continue
img_data = base64.b64decode(str(self.image_queue.get()))
image_np = np.asarray(Image.open(io.BytesIO(img_data)))
image_np = cv2.flip(image_np, 1)
gray_image = cv2.cvtColor(image_np, cv2.COLOR_RGB2GRAY)
faces = detect_faces(face_detection, gray_image)
# Actual detection. Variable boxes contains the bounding box cordinates for hands detected,
# while scores contains the confidence for each of these boxes.
# Hint: If len(boxes) > 1 , you may assume you have found atleast one hand (within your score threshold)
boxes, scores = detector_utils.detect_objects(
image_np, detection_graph, sess)
valid_hands = 0
calc_displacement = False
for score in scores:
if score > min_threshold:
valid_hands += 1
if valid_hands == num_hands_detect:
calc_displacement = True #tell function to return new displacement
# draw bounding boxes on frame
self.total_displacement += detector_utils.draw_box_on_image(
num_hands_detect, min_threshold, scores, boxes, im_width,
im_height, image_np, old_points,
calc_displacement) #0.2 is the min threshold
# Calculate Frames per second (FPS)
self.num_frames += 1
elapsed_time = (datetime.datetime.now() -
start_time).total_seconds()
fps = self.num_frames / elapsed_time
if self.current_emotion in self.emotions:
self.emotions[self.current_emotion] += 1
else:
self.emotions[self.current_emotion] = 1
print(self.total_displacement / (10 * self.num_frames),
self.current_emotion, self.emotions)
for face_coordinates in faces:
x1, x2, y1, y2 = apply_offsets(face_coordinates,
emotion_offsets)
gray_face = gray_image[y1:y2, x1:x2]
try:
gray_face = cv2.resize(gray_face, (emotion_target_size))
except:
continue
gray_face = preprocess_input(gray_face, True)
gray_face = np.expand_dims(gray_face, 0)
gray_face = np.expand_dims(gray_face, -1)
emotion_prediction = emotion_classifier.predict(gray_face)
emotion_probability = np.max(emotion_prediction)
emotion_label_arg = np.argmax(emotion_prediction)
self.current_emotion = emotion_labels[emotion_label_arg]
emotion_window.append(self.current_emotion)
if len(emotion_window) > frame_window:
emotion_window.pop(0)
try:
emotion_mode = mode(emotion_window)
except:
continue
draw_bounding_box(face_coordinates, image_np)
# Display FPS on frame:
detector_utils.draw_fps_on_image("FPS : " + str(int(fps)),
image_np)
cv2.imshow('Single-Threaded Detection',
cv2.cvtColor(image_np, cv2.COLOR_RGB2BGR))
if cv2.waitKey(25) & 0xFF == ord('q'):
cv2.destroyAllWindows()
break
from utils.datasets import get_labels
from utils.inference import detect_faces
from utils.inference import draw_text
from utils.inference import draw_bounding_box
from utils.inference import apply_offsets
from utils.inference import load_detection_model
from utils.inference import load_image
from utils.inference import preprocess_image
from utils.preprocessor import preprocess_input
backend.clear_session()
_SAVE_DIR = 'static/result'
_DETECTION_MODEL_PATH = './trained_models/detection_models/haarcascade_frontalface_default.xml'
_GENDER_MODEL_PATH = './trained_models/gender_models/simple_CNN.81-0.96.hdf5'
_EMOTION_MODEL_PATH = './trained_models/emotion_models/fer2013_mini_XCEPTION.102-0.66.hdf5'
_FACE_DETECTION = load_detection_model(_DETECTION_MODEL_PATH)
_GENDER_CLASSIFIER = load_model(_GENDER_MODEL_PATH, compile=False)
_EMOTION_CLASSIFIER = load_model(_EMOTION_MODEL_PATH, compile=False)
_GRAPH = tf.get_default_graph()
def emotion_classificator(image):
detected_peoples = []
json_info = {}
try:
# parameters for loading data and images
emotion_labels = get_labels('fer2013')
# hyper-parameters for bounding boxes shape
emotion_offsets = (20, 40)
emotion_offsets = (0, 0)
def fun(in_path, out_image_path,
out_info_path, in_finished_path,
model_path, image_resolution):
"""
>>> fun(/fer_input, /fer_output, /fer_result, /fer_finished, /fer_model, image_resolution)
.jpg files in the fer_intput folder will move to fer_finished folder.
Processed .jpg files will be saved in fer_output folder.
.csv files will be saved in fer_result folder.
only process the image that its resolution is 720p and above(image_resolution = 720, can be adjusted)
"""
global model, F
detect_emo = True
#save config
save_image = True
save_info = True
show_image = False
#config = tf.ConfigProto()
# config.gpu_options.per_process_gpu_memory_fraction = 0.7
# config.gpu_options.allow_growth = True
# session = InteractiveSession(config=config)
#%%
# parameters for loading data and images
detection_model_path = model_path + '/haarcascade_frontalface_default.xml'
if detect_emo:
emotion_model_path = model_path + '/googlenet__googlenetwei__2020Aug29_16.21'
emotion_labels = get_labels('fer2013')
print(emotion_labels)
emotion_offsets = (20, 40)
# loading models
model = getattr(model, 'googlenet')
model = model(in_channels=3, num_classes=7)
#print(torch.cuda.is_available())
#print(torch.cuda.device_count())
state = torch.load(emotion_model_path, map_location='cpu')
model.load_state_dict(state['net'])
#model.cuda()
model.eval()
# getting input model shapes for inference
emotion_target_size = (224,224)
# starting lists for calculating modes
emotion_window = []
# hyper-parameters for bounding boxes shape
frame_window = 10
emotion_offsets = (20, 40)
# loading models
face_detection = load_detection_model(detection_model_path)
info_name = ['file name', 'face_x', 'face_y', 'face_w', 'face_h', 'emotion', 'angry_prob', 'disgust_prob', 'fear_prob', 'happy_prob', 'sad_prob', 'surprise_prob', 'neutral_prob']
input_image_root = in_path
output_image_root = out_image_path
output_info_root = out_info_path
#covert png to jpg
for image_path in glob.glob(input_image_root+'/**/*.png', recursive=True):
img_path_no_PNG = image_path.split('.png')[0]
image_name = image_path.split('/')[-1].split('.png')[0]
no_root_path = image_path[len(input_image_root):].replace(image_path.split('/')[-1], '')
im = Image.open(image_path)
rgb_im = im.convert('RGB')
rgb_im.save(img_path_no_PNG + '.jpg')
#delete PNG file
os.remove(image_path)
for image_path in glob.glob(input_image_root+'/**/*.jpg', recursive=True):
print(image_path)
no_root_path = image_path[len(input_image_root):].replace(image_path.split('/')[-1], '')
spec_csv_name = ('/' + (image_path[len("feri_input/"):].replace(image_path.split('/')[-1], ''))).split('/')[-2]
if spec_csv_name =="":
spec_csv_name = "result.csv"
else:
spec_csv_name = spec_csv_name + '.csv'
image_capture = cv2.imread(image_path)
image_cap_ori = image_capture
image_name = image_path.split('/')[-1].split('.jpg')[0]
ori_image_name = image_path.split('/')[-1]
size = (round(image_capture.shape[0]), round(image_capture.shape[1])) # float
ori_size = size
reduce_resolution = 0
first_construct_csv = False
scaling_factor_x = 1
scaling_factor_y = 1
if image_resolution == "720p" and size[0] > 720 and size[1] > 1280:
#need to reduce resolution to 720p
reduce_resolution = 1
out_path = input_image_root + no_root_path+'resize_to_720p_'+image_path.split('/')[-1]
image_capture = cv2.resize(image_capture, (1280, 720), interpolation=cv2.INTER_CUBIC)
cv2.imwrite(out_path, image_capture)
scaling_factor_y = size[0]/720
scaling_factor_x = size[1]/1280
#original resolution image move to fer_finished dir
src = image_path
dst = in_finished_path + no_root_path + image_name + ".jpg"
os.makedirs(os.path.dirname(in_finished_path + no_root_path), exist_ok=True)
shutil.move(src, dst)
#capture ori resolution image to draw bounding box
image_cap_ori = cv2.imread(dst)
#capture reducing resolution image to construct csv file
image_path = out_path
image_capture = cv2.imread(image_path)
image_name = image_path.split('/')[-1].split('.jpg')[0]
size = (round(image_capture.shape[0]), round(image_capture.shape[1])) # float
if size[0] == 720 and size[1] == 1280:
if save_image:
os.makedirs(os.path.dirname(output_image_root + no_root_path), exist_ok=True)
out_path = output_image_root+no_root_path+ori_image_name
if save_info:
os.makedirs(os.path.dirname(output_info_root + no_root_path), exist_ok=True)
if os.path.isfile(output_info_root+no_root_path+spec_csv_name) == False:
first_construct_csv = True
csv_info = codecs.open(
output_info_root+no_root_path+spec_csv_name, 'a', encoding="utf_8_sig"
)
csv_writer = csv.writer(csv_info)
if first_construct_csv == True:
csv_writer.writerow(info_name)
st_time = time.time()
gray_image = cv2.cvtColor(image_capture, cv2.COLOR_BGR2GRAY)
faces = detect_faces(face_detection, gray_image)
if not isinstance(faces, tuple):
faces = faces[faces[:,0].argsort()]
faces = faces[faces[:,1].argsort()]
faces = faces[faces[:,2].argsort()]
faces = faces[faces[:,3].argsort()]
for face_coordinates in faces:
x_1, x_2, y_1, y_2 = apply_offsets(face_coordinates, emotion_offsets)
if detect_emo:
gray_face = gray_image[y_1:y_2, x_1:x_2]
try:
gray_face = cv2.resize(gray_face, (emotion_target_size))
except:
continue
gray_face = np.dstack([gray_face] * 3)
gray_face = transforms.Compose([ transforms.ToPILImage(),transforms.ToTensor(),])(np.uint8(gray_face))
gray_face = torch.stack([gray_face], 0)
#gray_face = gray_face.cuda(non_blocking=True)
outputs = model(gray_face).cpu()
outputs = F.softmax(outputs, 1)
emotion_prediction = torch.sum(outputs, 0).cpu().detach().numpy() # outputs.shape [tta_size, 7]
emotion_probability = np.max(emotion_prediction)
emotion_label_arg = np.argmax(emotion_prediction)
emotion_text = emotion_labels[emotion_label_arg]
emotion_window.append(emotion_text)
if len(emotion_window) > frame_window:
emotion_window.pop(0)
'''
try:
emotion_mode = mode(emotion_window)
except:
continue
'''
x = int(float(face_coordinates[0]*scaling_factor_x))
y = int(float(face_coordinates[1]*scaling_factor_y))
w = int(float(face_coordinates[2]*scaling_factor_x))
h = int(float(face_coordinates[3]*scaling_factor_y))
if emotion_text == 'angry':
cv2.rectangle(image_cap_ori, (x, y), (x+w, y+h), (255,0,0), 4)
cv2.putText(image_cap_ori, 'angry', (int(float(x+w/2-43)), y-10), cv2.FONT_HERSHEY_DUPLEX, 1, (255, 0, 0), 1, cv2.LINE_AA)
elif emotion_text == 'sad':
cv2.rectangle(image_cap_ori, (x, y), (x+w, y+h), (0,0,255), 4)
cv2.putText(image_cap_ori, 'sad', (int(float(x+w/2-43)), y-10), cv2.FONT_HERSHEY_DUPLEX, 1, (0,0,255), 1, cv2.LINE_AA)
elif emotion_text == 'happy':
cv2.rectangle(image_cap_ori, (x, y), (x+w, y+h), (255,255,0), 4)
cv2.putText(image_cap_ori, 'happy', (int(float(x+w/2-43)), y-10), cv2.FONT_HERSHEY_DUPLEX, 1, (255,255,0), 1, cv2.LINE_AA)
elif emotion_text == 'surprise':
cv2.rectangle(image_cap_ori, (x, y), (x+w, y+h), (0,255,255), 4)
cv2.putText(image_cap_ori, 'surprise', (int(float(x+w/2-43)), y-10), cv2.FONT_HERSHEY_DUPLEX, 1, (0,255,255), 1, cv2.LINE_AA)
elif emotion_text == 'disgust':
cv2.rectangle(image_cap_ori, (x, y), (x+w, y+h), (0,0,0), 4)
cv2.putText(image_cap_ori, 'disgust', (int(float(x+w/2-43)), y-10), cv2.FONT_HERSHEY_DUPLEX, 1, (0,0,0), 1, cv2.LINE_AA)
elif emotion_text == 'fear':
cv2.rectangle(image_cap_ori, (x, y), (x+w, y+h), (255,0,255), 4)
cv2.putText(image_cap_ori, 'fear', (int(float(x+w/2-43)), y-10), cv2.FONT_HERSHEY_DUPLEX, 1, (255,0,255), 1, cv2.LINE_AA)
else:
cv2.rectangle(image_cap_ori, (x, y), (x+w, y+h), (0,255,0), 4)
cv2.putText(image_cap_ori, 'neutral', (int(float(x+w/2-43)), y-10), cv2.FONT_HERSHEY_DUPLEX, 1, (0,255,0), 1, cv2.LINE_AA)
if save_info:
op_info_list = [ori_image_name,
face_coordinates[0]*scaling_factor_x, face_coordinates[1]*scaling_factor_y,
face_coordinates[2]*scaling_factor_x, face_coordinates[3]*scaling_factor_y]
for i in range(len(op_info_list)):
op_info_list[i] = str(op_info_list[i])
if detect_emo:
op_info_list.append(emotion_text)
for prob in emotion_prediction:
op_info_list.append(prob)
csv_writer.writerow(op_info_list)
if save_image:
cv2.imwrite(output_image_root + no_root_path + image_name + ".jpg", image_cap_ori)
if save_info:
csv_info.close()
print(image_path+' DONE!!\tSpend Time: '+str(time.time()-st_time)+'(s)')
else:
os.makedirs(os.path.dirname(output_info_root + no_root_path), exist_ok=True)
if os.path.isfile(output_info_root+no_root_path+spec_csv_name) == False:
first_construct_csv = True
csv_info = codecs.open(output_info_root+no_root_path+spec_csv_name,
'a', encoding="utf_8_sig")
csv_writer = csv.writer(csv_info)
if first_construct_csv == True:
csv_writer.writerow(info_name)
err_msg = "The resolution of " + image_name + ".jpg is lower than 720p."
csv_writer.writerow([err_msg])
csv_info.close()
src = image_path
dst = in_finished_path + no_root_path + image_name + ".jpg"
os.makedirs(os.path.dirname(in_finished_path + no_root_path), exist_ok=True)
shutil.move(src, dst)
if reduce_resolution == 1:
image_ori_name = image_name[15:]
os.remove(dst)
os.rename(output_image_root+no_root_path+image_name+'.jpg', output_image_root+no_root_path+image_ori_name+'.jpg')
shutil.rmtree(input_image_root, ignore_errors=True)
def process_image(image):
try:
# parameters for loading data and images
detection_model_path = './trained_models/detection_models/haarcascade_frontalface_default.xml'
emotion_model_path = './trained_models/emotion_models/fer2013_mini_XCEPTION.102-0.66.hdf5'
gender_model_path = './trained_models/gender_models/simple_CNN.81-0.96.hdf5'
emotion_labels = get_labels('fer2013')
gender_labels = get_labels('imdb')
font = cv2.FONT_HERSHEY_SIMPLEX
# hyper-parameters for bounding boxes shape
gender_offsets = (30, 60)
gender_offsets = (10, 10)
emotion_offsets = (20, 40)
emotion_offsets = (0, 0)
# loading models
face_detection = load_detection_model(detection_model_path)
emotion_classifier = load_model(emotion_model_path, compile=False)
gender_classifier = load_model(gender_model_path, compile=False)
# getting input model shapes for inference
emotion_target_size = emotion_classifier.input_shape[1:3]
gender_target_size = gender_classifier.input_shape[1:3]
# loading images
image_array = np.fromstring(image, np.uint8)
unchanged_image = cv2.imdecode(image_array, cv2.IMREAD_UNCHANGED)
rgb_image = cv2.cvtColor(unchanged_image, cv2.COLOR_BGR2RGB)
gray_image = cv2.cvtColor(unchanged_image, cv2.COLOR_BGR2GRAY)
faces = detect_faces(face_detection, gray_image)
for face_coordinates in faces:
x1, x2, y1, y2 = apply_offsets(face_coordinates, gender_offsets)
rgb_face = rgb_image[y1:y2, x1:x2]
x1, x2, y1, y2 = apply_offsets(face_coordinates, emotion_offsets)
gray_face = gray_image[y1:y2, x1:x2]
try:
rgb_face = cv2.resize(rgb_face, (gender_target_size))
gray_face = cv2.resize(gray_face, (emotion_target_size))
except:
continue
rgb_face = preprocess_input(rgb_face, False)
rgb_face = np.expand_dims(rgb_face, 0)
gender_prediction = gender_classifier.predict(rgb_face)
gender_label_arg = np.argmax(gender_prediction)
gender_text = gender_labels[gender_label_arg]
gray_face = preprocess_input(gray_face, True)
gray_face = np.expand_dims(gray_face, 0)
gray_face = np.expand_dims(gray_face, -1)
emotion_label_arg = np.argmax(emotion_classifier.predict(gray_face))
emotion_text = emotion_labels[emotion_label_arg]
if gender_text == gender_labels[0]:
color = (0, 0, 255)
else:
color = (255, 0, 0)
draw_bounding_box(face_coordinates, rgb_image, color)
draw_text(face_coordinates, rgb_image, gender_text, color, 0, -20, 1, 2)
draw_text(face_coordinates, rgb_image, emotion_text, color, 0, -50, 1, 2)
except Exception as err:
logging.error('Error in emotion gender processor: "{0}"'.format(err))
bgr_image = cv2.cvtColor(rgb_image, cv2.COLOR_RGB2BGR)
dirname = 'result'
if not os.path.exists(dirname):
os.mkdir(dirname)
cv2.imwrite(os.path.join(dirname, 'predicted_image.png'), bgr_image)
def predict(image_folder_path, emotion_kind):
detection_model_path = '../trained_models/detection_models/haarcascade_frontalface_default.xml'
emotion_model_path = '../trained_models/emotion_models/fer2013_mini_XCEPTION.102-0.66.hdf5'
gender_model_path = '../trained_models/gender_models/simple_CNN.81-0.96.hdf5'
emotion_labels = get_labels('fer2013')
target_file = '../result/predicted_' + EmotionName[emotion_kind] + '.txt'
gender_labels = get_labels('imdb')
font = cv2.FONT_HERSHEY_SIMPLEX
# hyper-parameters for bounding boxes shape
gender_offsets = (30, 60)
gender_offsets = (10, 10)
emotion_offsets = (20, 40)
emotion_offsets = (0, 0)
# loading models
face_detection = load_detection_model(detection_model_path)
emotion_classifier = load_model(emotion_model_path, compile=False)
gender_classifier = load_model(gender_model_path, compile=False)
# getting input model shapes for inference
emotion_target_size = emotion_classifier.input_shape[1:3]
gender_target_size = gender_classifier.input_shape[1:3]
image_path = dir_data_folder(image_folder_path)
predicted_label = []
for num in range(len(image_path)):
# print the process info
if print_switch == 0:
print('deal with the ' + image_path[num])
# loading images
rgb_image = load_image(image_path[num], grayscale=False)
gray_image = load_image(image_path[num], grayscale=True)
gray_image = np.squeeze(gray_image)
gray_image = gray_image.astype('uint8')
# pdb.set_trace()
faces = detect_faces(face_detection, gray_image)
emotion_label_arg = -1
for face_coordinates in faces:
x1, x2, y1, y2 = apply_offsets(face_coordinates, gender_offsets)
rgb_face = rgb_image[y1:y2, x1:x2]
x1, x2, y1, y2 = apply_offsets(face_coordinates, emotion_offsets)
gray_face = gray_image[y1:y2, x1:x2]
try:
rgb_face = cv2.resize(rgb_face, (gender_target_size))
gray_face = cv2.resize(gray_face, (emotion_target_size))
except:
continue
rgb_face = preprocess_input(rgb_face, False)
rgb_face = np.expand_dims(rgb_face, 0)
gender_prediction = gender_classifier.predict(rgb_face)
gender_label_arg = np.argmax(gender_prediction)
gender_text = gender_labels[gender_label_arg]
gray_face = preprocess_input(gray_face, True)
gray_face = np.expand_dims(gray_face, 0)
gray_face = np.expand_dims(gray_face, -1)
emotion_label_arg = np.argmax(
emotion_classifier.predict(gray_face))
emotion_text = emotion_labels[emotion_label_arg]
if single_set_draw_switch == 0:
if gender_text == gender_labels[0]:
color = (0, 0, 255)
else:
color = (255, 0, 0)
draw_bounding_box(face_coordinates, rgb_image, color)
# draw_text(face_coordinates, rgb_image, gender_text, color, 0, -20, 1, 2)
draw_text(face_coordinates, rgb_image, emotion_text, color, 0,
20, 1, 2)
if single_set_draw_switch == 0:
bgr_image = cv2.cvtColor(rgb_image, cv2.COLOR_RGB2BGR)
filename = image_path[num].split('/')[-1]
filename = filename.split('.')[0]
newname = '../predicted_image/' + 'predicted_' + filename + '.png'
cv2.imwrite(newname, bgr_image)
# add the predicted label
predicted_label.append(emotion_label_arg)
unrecognized_cnt = predicted_label.count(-1)
true_cnt = predicted_label.count(emotion_kind)
total_cnt = len(image_path)
face_cnt = total_cnt - unrecognized_cnt
face_recog_ratio = face_cnt / float(total_cnt)
total_accuracy = true_cnt / float(total_cnt)
recog_accuracy = true_cnt / float(face_cnt)
f = open(target_file, 'w+')
#with open(target_file,'w') as f:
f.write(EmotionName[emotion_kind] + ' count = ' + str(true_cnt) + '\n')
f.write(EmotionName[emotion_kind] + ' total count = ' + str(total_cnt) +
'\n')
f.write(EmotionName[emotion_kind] + '_accuracy in total: ' +
str(total_accuracy) + '\n')
f.write(EmotionName[emotion_kind] + '_accuracy in recognized: ' +
str(recog_accuracy) + '\n')
f.write('face_recognize_count : ' + str(face_cnt) + '\n')
f.write('face_recognize_ratio : ' + str(face_recog_ratio) + '\n')
for label in predicted_label:
f.write(str(label) + '\n')
f.close()
return [
true_cnt, total_cnt, face_cnt, total_accuracy, face_recog_ratio,
recog_accuracy
]
def recognition(f_2_s):
tt = False #이중 조건문을 탈출하기 위해서, goto문이 불가능하다 파이썬은..
emotion_c = ""
#recap == False
# parameters for loading data and images
#image_path = path_r
image_path = image_handler(f_2_s)
detection_model_path = '../trained_models/detection_models/haarcascade_frontalface_default.xml'
emotion_model_path = '../trained_models/emotion_models/fer2013_mini_XCEPTION.102-0.66.hdf5'
emotion_labels = get_labels('fer2013')
font = cv2.FONT_HERSHEY_SIMPLEX
# hyper-parameters for bounding boxes shape
emotion_offsets = (20, 40)
emotion_offsets = (0, 0)
# loading models
face_detection = load_detection_model(detection_model_path)
emotion_classifier = load_model(emotion_model_path, compile=False)
# getting input model shapes for inference
emotion_target_size = emotion_classifier.input_shape[1:3]
# loading images
rgb_image = load_image(image_path, grayscale=False)
gray_image = load_image(image_path, grayscale=True)
gray_image = np.squeeze(gray_image)
gray_image = gray_image.astype('uint8')
faces = detect_faces(face_detection, gray_image)
for face_coordinates in faces:
x1, x2, y1, y2 = apply_offsets(face_coordinates, emotion_offsets)
gray_face = gray_image[y1:y2, x1:x2]
try:
gray_face = cv2.resize(gray_face, (emotion_target_size))
except:
#pyautogui.confirm(text='one more', title='test', buttons=['ok', 'exit'])
#recap = True
continue
gray_face = preprocess_input(gray_face, True)
gray_face = np.expand_dims(gray_face, 0)
gray_face = np.expand_dims(gray_face, -1)
emotion_label_arg = np.argmax(emotion_classifier.predict(gray_face))
emotion_text = ""
emotion_text = emotion_labels[emotion_label_arg]
#감정인식이 성공되면 감정 상태를 물어보고, 감정 확인 후 저장 or 탈출
tof = pyautogui.confirm(text='Are you ' + emotion_text + '?',
title=emotion_text,
buttons=['yes', 'no'])
if (tof == 'yes'):
tt = True
emotion_c = emotion_text
color = (255, 0, 0) # 감정 정보 글씨 빨간색, 사각형도
draw_bounding_box(face_coordinates, rgb_image, color)
draw_text(face_coordinates, rgb_image, emotion_text, color, 0, -30,
1.5, 2)
elif (tof == 'no'):
tt = False
break
#color = (255, 0, 0) # 감정 정보 글씨 빨간색, 사각형도
#draw_bounding_box(face_coordinates, rgb_image, color)
#draw_text(face_coordinates, rgb_image, emotion_text, color, 0, -30, 1.5, 2)
if (tt == True):
bgr_image = cv2.cvtColor(rgb_image, cv2.COLOR_RGB2BGR)
cv2.imwrite('../src/' + emotion_text + '.jpg', bgr_image) # 변수 활용
check_recoged_img('../src/' + emotion_text + '.jpg')
else:
pyautogui.alert(text='no emtion captured', title='error', button='OK')
def emotion_identify(img_url):
# parameters for loading data and images
detection_model_path = 'C:/Users/Admin/PycharmProjects/Emotion_Detection/trained_models/detection_models/haarcascade_frontalface_default.xml'
emotion_model_path = 'C:/Users/Admin/PycharmProjects/Emotion_Detection/trained_models/emotion_models/fer2013_mini_XCEPTION.102-0.66.hdf5'
gender_model_path = 'C:/Users/Admin/PycharmProjects/Emotion_Detection/trained_models/gender_models/simple_CNN.81-0.96.hdf5'
emotion_labels = get_labels('fer2013')
gender_labels = get_labels('imdb')
font = cv2.FONT_HERSHEY_SIMPLEX
# hyper-parameters for bounding boxes shape
gender_offsets = (30, 60)
gender_offsets = (10, 10)
emotion_offsets = (20, 40)
emotion_offsets = (0, 0)
# loading models
face_detection = load_detection_model(detection_model_path)
emotion_classifier = load_model(emotion_model_path, compile=False)
gender_classifier = load_model(gender_model_path, compile=False)
# getting input model shapes for inference
emotion_target_size = emotion_classifier.input_shape[1:3]
gender_target_size = gender_classifier.input_shape[1:3]
# loading images
image_path = img_url
rgb_image = load_image(image_path, grayscale=False)
gray_image = load_image(image_path, grayscale=True)
gray_image = np.squeeze(gray_image)
gray_image = gray_image.astype('uint8')
faces = detect_faces(face_detection, gray_image)
if len(faces) == 0:
print("No face")
K.clear_session()
return False
emotions = collections.defaultdict(int)
for face_coordinates in faces:
x1, x2, y1, y2 = apply_offsets(face_coordinates, gender_offsets)
rgb_face = rgb_image[y1:y2, x1:x2]
x1, x2, y1, y2 = apply_offsets(face_coordinates, emotion_offsets)
gray_face = gray_image[y1:y2, x1:x2]
try:
rgb_face = cv2.resize(rgb_face, (gender_target_size))
gray_face = cv2.resize(gray_face, (emotion_target_size))
except:
continue
rgb_face = preprocess_input(rgb_face, False)
rgb_face = np.expand_dims(rgb_face, 0)
gender_prediction = gender_classifier.predict(rgb_face)
gender_label_arg = np.argmax(gender_prediction)
gender_text = gender_labels[gender_label_arg]
gray_face = preprocess_input(gray_face, True)
gray_face = np.expand_dims(gray_face, 0)
gray_face = np.expand_dims(gray_face, -1)
emotion_label_arg = np.argmax(emotion_classifier.predict(gray_face))
emotion_text = emotion_labels[emotion_label_arg]
emotions[emotion_text] += 1
if gender_text == gender_labels[0]:
color = (0, 0, 255)
else:
color = (255, 0, 0)
draw_bounding_box(face_coordinates, rgb_image, color)
draw_text(face_coordinates, rgb_image, gender_text, color, 0, -20, 1, 2)
draw_text(face_coordinates, rgb_image, emotion_text, color, 0, -50, 1, 2)
max_num = 0
max_emotion = None
for key, value in emotions.items():
if value > max_num:
max_num = value
max_emotion = key
print("The emotion of this picture is: ", max_emotion)
bgr_image = cv2.cvtColor(rgb_image, cv2.COLOR_RGB2BGR)
cv2.imwrite('./result_images/predicted_test_image.png', bgr_image)
K.clear_session()
return max_emotion
response["error"] = "Unable to recognize speech"
return response
# parameters for loading data and images
detection_model_path = '../trained_models/detection_models/haarcascade_frontalface_default.xml'
emotion_model_path = '../trained_models/emotion_models/fer2013_mini_XCEPTION.102-0.66.hdf5'
emotion_labels = get_labels('fer2013')
# hyper-parameters for bounding boxes shape
frame_window = 10
emotion_offsets = (20, 40)
# loading models
face_detection = load_detection_model(detection_model_path)
emotion_classifier = load_model(emotion_model_path, compile=False)
# getting input model shapes for inference
emotion_target_size = emotion_classifier.input_shape[1:3]
# starting lists for calculating modes
emotion_window = []
# starting video streaming
cv2.namedWindow('window_frame')
#creating voice recognizer and microphone
recognizer = sr.Recognizer()
#use sr.Microphone.list_microphone_names() to find the device index
#if device_index arg is omitted, it will use the default microphone
def main(yolo):
t = datetime.datetime.now().replace(microsecond=0).isoformat()
# write only if values are not empty
graphInputs = ['1', '8', 'Emotion', '2018-10-23T14:02:29', 'MALE', '2']
with open(r'templates/test2.csv', 'a') as f:
writer = csv.writer(f)
writer.writerow(graphInputs)
# parameters for loading data and images
detection_model_path = 'trained_models/detection_models/haarcascade_frontalface_default.xml'
emotion_model_path = 'trained_models/emotion_models/fer2013_mini_XCEPTION.102-0.66.hdf5'
gender_model_path = 'trained_models/gender_models/simple_CNN.81-0.96.hdf5'
emotion_labels = get_labels('fer2013')
gender_labels = get_labels('imdb')
font = cv2.FONT_HERSHEY_SIMPLEX
# hyper-parameters for bounding boxes shape
frame_window = 10
gender_offsets = (30, 60)
emotion_offsets = (20, 40)
# loading models
face_detection = load_detection_model(detection_model_path)
emotion_classifier = load_model(emotion_model_path, compile=False)
gender_classifier = load_model(gender_model_path, compile=False)
# getting input model shapes for inference
emotion_target_size = emotion_classifier.input_shape[1:3]
gender_target_size = gender_classifier.input_shape[1:3]
# starting lists for calculating modes
gender_window = []
emotion_window = []
# Definition of the parameters
max_cosine_distance = 0.3
nn_budget = None
nms_max_overlap = 1.0
# deep_sort
model_filename = 'model_data/mars-small128.pb'
encoder = gdet.create_box_encoder(model_filename, batch_size=1)
metric = nn_matching.NearestNeighborDistanceMetric("cosine",
max_cosine_distance,
nn_budget)
tracker = Tracker(metric)
writeVideo_flag = False
resetCounter = 0
amountOfFramesPerScan = 10
peopleInFrameList = []
# video_capture = cv2.VideoCapture('demo/dinner.mp4')
video_capture = cv2.VideoCapture('demo/MOT1712.mp4')
# video_capture = cv2.VideoCapture(0)
if writeVideo_flag:
# Define the codec and create VideoWriter object
w = int(video_capture.get(3))
h = int(video_capture.get(4))
fourcc = cv2.VideoWriter_fourcc(*'MJPG')
out = cv2.VideoWriter('output.avi', fourcc, 15, (w, h))
list_file = open('detection.txt', 'w')
frame_index = -1
fps = 0.0
while True:
ret, frame = video_capture.read() # frame shape 640*480*3
if ret != True:
break
# -------------------START EMOTION CODE
# cv2.imshow('window_frame', frame) SHOWS EMOTION FRAME SEPERATE
# -------------------------------- END EMOTION CODE
t1 = time.time()
currentPeopleInFrame = 0
image = Image.fromarray(frame)
boxs = yolo.detect_image(image)
# print("box_num",len(boxs))
features = encoder(frame, boxs)
# score to 1.0 here).
detections = [
Detection(bbox, 1.0, feature)
for bbox, feature in zip(boxs, features)
]
# Run non-maxima suppression.
boxes = np.array([d.tlwh for d in detections])
scores = np.array([d.confidence for d in detections])
indices = preprocessing.non_max_suppression(boxes, nms_max_overlap,
scores)
detections = [detections[i] for i in indices]
# Imagelist is a list of all the images within the tracked bounding boxes of our tracker.
imageList = []
# Call the tracker
tracker.predict()
tracker.update(detections)
trackerIDs = []
for track in tracker.tracks:
if track.is_confirmed() and track.time_since_update > 1:
continue
# Gets the location of the BBOx coordinates within the tracker.
bbox = track.to_tlbr()
# Put rectangle and text on the image
currentPeopleInFrame += 1
# print(int(bbox[0]), int(bbox[1]), int(bbox[2]), int(bbox[3]))
# Check if bounding box 3 isn't out of bounds before creating image
if int(bbox[2]) <= 640:
numpArr = np.array(frame[int((bbox[1])):int(bbox[1] + bbox[3]),
int(bbox[0]):int(bbox[0] + bbox[2])])
else:
numpArr = np.array(frame[int((bbox[1])):int(bbox[1] + bbox[3]),
int(bbox[0]):(int(bbox[0]) + 640)])
imageList.append(numpArr)
# cv2.destroyAllWindows()
trackerIDs.append(track.track_id)
i = 0
cv2.rectangle(frame, (int(bbox[0]), int(bbox[1])),
(int(bbox[2]), int(bbox[3])), (255, 255, 255), 2)
cv2.putText(frame, str(track.track_id),
(int(bbox[0]), int(bbox[1])), 0, 5e-3 * 200,
(0, 255, 0), 2)
print("Found tracked human")
for item in (imageList):
print("Scanning human")
gray_image = cv2.cvtColor(item, cv2.COLOR_BGR2GRAY)
rgb_image = cv2.cvtColor(item, cv2.COLOR_BGR2RGB)
faces = detect_faces(face_detection, gray_image)
graphInputs[0] = '%d' % trackerIDs[i]
i += 1
graphInputs[3] = '%s' % datetime.datetime.now().replace(
microsecond=0).isoformat()
for face_coordinates in faces:
print("printje KOMT 1")
x1, x2, y1, y2 = apply_offsets(face_coordinates,
gender_offsets)
rgb_face = rgb_image[y1:y2, x1:x2]
x1, x2, y1, y2 = apply_offsets(face_coordinates,
emotion_offsets)
gray_face = gray_image[y1:y2, x1:x2]
try:
rgb_face = cv2.resize(rgb_face, (gender_target_size))
gray_face = cv2.resize(gray_face, (emotion_target_size))
except:
continue
print("printje KOMT HIER2")
gray_face = preprocess_input(gray_face, True)
gray_face = np.expand_dims(gray_face, 0)
gray_face = np.expand_dims(gray_face, -1)
emotion_label_arg = np.argmax(
emotion_classifier.predict(gray_face))
emotion_text = emotion_labels[emotion_label_arg]
emotion_window.append(emotion_text)
rgb_face = np.expand_dims(rgb_face, 0)
rgb_face = preprocess_input(rgb_face, False)
gender_prediction = gender_classifier.predict(rgb_face)
gender_label_arg = np.argmax(gender_prediction)
gender_text = gender_labels[gender_label_arg]
graphInputs[4] = gender_text
gender_window.append(gender_text)
#overwrite emotion label met leeg als er niks gevonden wordt of laat hele persoon weg
#Als er niks gevonden wordt, niks schrijven
print("printje KOMT HIER 3 %s" %
emotion_labels[emotion_label_arg])
graphInputs[2] = ('%s' % emotion_labels[emotion_label_arg])
if gender_text == gender_labels[0]:
color = (0, 0, 255)
else:
color = (255, 0, 0)
# draw_bounding_box(face_coordinates, rgb_image, color)
# draw_text(face_coordinates, rgb_image, gender_mode,
# color, 0, -20, 1, 1)
# draw_text(face_coordinates, rgb_image, emotion_mode,
# color, 0, -45, 1, 1)
# gray_image = cv2.cvtColor(item, cv2.COLOR_BGR2GRAY)
# rgb_image = cv2.cvtColor(item, cv2.COLOR_BGR2RGB)
#
# faces = face_cascade.detectMultiScale(gray_image, scaleFactor=1.1, minNeighbors=5,
# minSize=(0, 0), flags=cv2.CASCADE_SCALE_IMAGE)
# #PersonID Set
# graphInputs[0] = '%d'%trackerIDs[i]
# # print("trackerID:", trackerIDs[i])
# i += 1
# graphInputs[3] = '%s'%datetime.datetime.now().replace(microsecond=0).isoformat()
#
# for face_coordinates in faces:
#
# x1, x2, y1, y2 = apply_offsets(face_coordinates, emotion_offsets)
# gray_face = gray_image[y1:y2, x1:x2]
# try:
# gray_face = cv2.resize(gray_face, (emotion_target_size))
# except:
# continue
#
# gray_face = preprocess_input(gray_face, True)
# gray_face = np.expand_dims(gray_face, 0)
# gray_face = np.expand_dims(gray_face, -1)
# emotion_prediction = emotion_classifier.predict(gray_face)
# emotion_probability = np.max(emotion_prediction)
# emotion_label_arg = np.argmax(emotion_prediction)
# emotion_text = emotion_labels[emotion_label_arg]
# emotion_window.append(emotion_text)
# if len(emotion_window) > frame_window:
# emotion_window.pop(0)
# try:
# emotion_mode = mode(emotion_window)
# except:
# continue
#
# #Emotion set
# if emotion_text == 'angry':
# color = emotion_probability * np.asarray((255, 0, 0))
# print("angry", i)
# graphInputs[2] = 'ANGRY'
# elif emotion_text == 'sad':
# color = emotion_probability * np.asarray((0, 0, 255))
# print("sad", i)
# graphInputs[2] = 'SAD'
# elif emotion_text == "happy":
# color = emotion_probability * np.asarray((255, 255, 0))
# print("happy", i)
# graphInputs[2] = 'HAPPY'
# elif emotion_text == 'surprise':
# color = emotion_probability * np.asarray((0, 255, 255))
# print("surprise", i)
# graphInputs[2] = 'SURPRISED'
# else:
# color = emotion_probability * np.asarray((0, 255, 0))
# print("neutral", i)
# graphInputs[2] = 'NEUTRAL'
# # color = color.astype(int)
# # color = color.tolist()
#
# # -------------------------------------
#
# draw_bounding_box(face_coordinates, rgb_image, color)
# draw_text(face_coordinates, rgb_image, emotion_mode,
# color, 0, -45, 1, 1)
print(graphInputs)
with open(r'templates/test2.csv', 'a') as f:
writer = csv.writer(f)
writer.writerow(graphInputs)
# cv2.imshow('jaja', frame[int((bbox[1])):int(bbox[1] + bbox[3]), int(bbox[0]):int(bbox[0] + bbox[2])])
# cv2.imshow('ajaja', imageList[0])
cv2.imshow('FilteredImage', frame)
if resetCounter >= amountOfFramesPerScan:
peopleInFrameList.append(currentPeopleInFrame)
print("Total amount of people %d" % (currentPeopleInFrame))
# Print
# for x in range(len(peopleInFrameList)):
# print("listie %d" % (peopleInFrameList[x]))
print(peopleInFrameList)
resetCounter = 0
else:
resetCounter += 1
print("Geen print of add deze keer %d" % (resetCounter))
if resetCounter >= amountOfFramesPerScan:
peopleInFrameList.append(currentPeopleInFrame)
print("Total amount of people %d" % (currentPeopleInFrame))
# for x in range(len(peopleInFrameList)):
# print("listie %d" % (peopleInFrameList[x]))
print(peopleInFrameList)
resetCounter = 0
else:
resetCounter += 1
print("Geen print of add deze keer %d" % (resetCounter))
if writeVideo_flag:
# save a frame
out.write(frame)
frame_index = frame_index + 1
list_file.write(str(frame_index) + '')
if len(boxs) != 0:
for i in range(0, len(boxs)):
list_file.write(
str(boxs[i][0]) + ' ' + str(boxs[i][1]) + ' ' +
str(boxs[i][2]) + ' ' + str(boxs[i][3]) + '')
list_file.write('\n')
fps = (fps + (1. / (time.time() - t1))) / 2
print("fps= %f" % (fps))
# Press Q to stop!
if cv2.waitKey(1) & 0xFF == ord('q'):
break
video_capture.release()
if writeVideo_flag:
out.release()
list_file.close()
cv2.destroyAllWindows()
def __init__(self):
# Initialize the node with rosp
rospy.init_node('emotion_recognizer_node', anonymous=True)
rospy.loginfo("recognizer started")
print "1................................................"
if(USE_LOCAL_CAMERA):
self.video_capture = cv2.VideoCapture(0)
self._detection_models = "~detection_models"
if rospy.has_param(self._detection_models):
self.detection_model_path = rospy.get_param(self._detection_models)
else:
rospy.logwarn("parameters need to be set to start recognizer.")
return
self.emotion_models = "~emotion_models"
if rospy.has_param(self.emotion_models):
self.emotion_model_path = rospy.get_param(self.emotion_models)
else:
rospy.logwarn("parameters need to be set to start recognizer.")
return
self.bridge = CvBridge()
# parameters for loading data and images
#self.detection_model_path = '../trained_models/detection_models/haarcascade_frontalface_default.xml'
#s elf.emotion_model_path = '../trained_models/emotion_models/fer2013_mini_XCEPTION.110-0.65.hdf5'
self.emotion_labels = get_labels('fer2013')
# hyper-parameters for bounding boxes shape
self.frame_window = 10
self.emotion_offsets = (20, 40)
# loading models
self.face_detection = load_detection_model(self.detection_model_path)
self.emotion_classifier = load_model(self.emotion_model_path, compile=False)
# getting input model shapes for inference
self.emotion_target_size = self.emotion_classifier.input_shape[1:3]
# starting lists for calculating modes
self.emotion_window = []
self.emotion_publisher = rospy.Publisher("/qt_face/setEmotion",String,queue_size=10)
self.speech_publisher = rospy.Publisher("/speaker",String,queue_size=10)
self.emotion_msg = String()
self.speech_msg = String()
#Where to publish
self._output_image_topic = "~image_topic_output"
print rospy.has_param(self._output_image_topic)
if rospy.has_param(self._output_image_topic):
output_image_topic = rospy.get_param(self._output_image_topic)
self.image_pub = rospy.Publisher(output_image_topic,Image, queue_size=10)
# Scaling factor for face recognition image
self.scaling_factor = 0.50
#Where to subscribe
self._input_image_topic = "~image_topic_input"
print rospy.has_param(self._input_image_topic)
if rospy.has_param(self._input_image_topic):
input_image_topic = rospy.get_param(self._input_image_topic)
if(not USE_LOCAL_CAMERA):
self.image_sub = rospy.Subscriber(input_image_topic, Image, self.callback)
self.graph = tf.get_default_graph()
from utils.preprocessor import preprocess_input
# parameters for loading data and images
emotion_model_path = '../trained_models/emotion_models/fer2013_mini_XCEPTION.102-0.66.hdf5'
gender_model_path = '../trained_models/gender_models/simple_CNN.81-0.96.hdf5'
emotion_labels = get_labels('fer2013')
gender_labels = get_labels('imdb')
font = cv2.FONT_HERSHEY_SIMPLEX
# hyper-parameters for bounding boxes shape
frame_window = 10
gender_offsets = (30, 60)
emotion_offsets = (20, 40)
# loading models
face_detection = load_detection_model()
emotion_classifier = load_model(emotion_model_path, compile=False)
gender_classifier = load_model(gender_model_path, compile=False)
# getting input model shapes for inference
emotion_target_size = emotion_classifier.input_shape[1:3]
gender_target_size = gender_classifier.input_shape[1:3]
# starting lists for calculating modes
gender_window = []
emotion_window = []
# starting video streaming
cv2.namedWindow('window_frame')
video_capture = cv2.VideoCapture(0)
while True:
def _load_face_detection(self):
self._face_detection = load_detection_model(self._detection_model)
