def process_image(image):
try:
# parameters for loading data and images
detection_model_path = './trained_models/detection_models/haarcascade_frontalface_default.xml'
emotion_model_path = './trained_models/emotion_models/fer2013_mini_XCEPTION.102-0.66.hdf5'
emotion_labels = get_labels('fer2013')
# hyper-parameters for bounding boxes shape
emotion_offsets = (0, 0)
# loading models
face_detection = load_detection_model(detection_model_path)
emotion_classifier = load_model(emotion_model_path, compile=False)
# getting input model shapes for inference
emotion_target_size = emotion_classifier.input_shape[1:3]
# loading images
image_array = np.fromstring(image, np.uint8)
unchanged_image = cv2.imdecode(image_array, cv2.IMREAD_UNCHANGED)
gray_image = cv2.cvtColor(unchanged_image, cv2.COLOR_BGR2GRAY)
faces = detect_faces(face_detection, gray_image)
detected_emotions = []
for face_coordinates in faces:
x1, x2, y1, y2 = apply_offsets(face_coordinates, emotion_offsets)
gray_face = gray_image[y1:y2, x1:x2]
try:
gray_face = cv2.resize(gray_face, emotion_target_size)
except:
continue
gray_face = preprocess_input(gray_face, True)
gray_face = np.expand_dims(gray_face, 0)
gray_face = np.expand_dims(gray_face, -1)
emotion_label_arg = np.argmax(emotion_classifier.predict(gray_face))
detected_emotions.append({
"coordinates": [str(y1), str(x2), str(y2), str(x1)],
"emotion": emotion_labels[emotion_label_arg]})
return detected_emotions
except Exception as err:
logging.error('Error in emotion gender processor: "{0}"'.format(err))
detection_model_path = '../trained_models/detection_models/haarcascade_frontalface_default.xml'
model = load_model(model_filename, compile=False)
target_size = model.input_shape[1:3]
face_detection = load_detection_model(detection_model_path)
# loading images
rgb_image = load_image(image_path, grayscale=False)
gray_image = load_image(image_path, grayscale=True)
gray_image = np.squeeze(gray_image)
gray_image = gray_image.astype('uint8')
faces = detect_faces(face_detection, gray_image)
# start prediction for every image
for face_coordinates in faces:
x1, x2, y1, y2 = apply_offsets(face_coordinates, offsets)
rgb_face = rgb_image[y1:y2, x1:x2]
x1, x2, y1, y2 = apply_offsets(face_coordinates, offsets)
gray_face = gray_image[y1:y2, x1:x2]
# processing input
try:
gray_face = cv2.resize(gray_face, (target_size))
except:
continue
gray_face = preprocess_input(gray_face, True)
gray_face = np.expand_dims(gray_face, 0)
gray_face = np.expand_dims(gray_face, -1)
# prediction
def gender_detection(image_path):
# parameters for loading data and images
detection_model_path = 'C:\\Users\\l1f15bscs0049\\Desktop\\haarcascade_frontalface_default.xml'
gender_model_path = 'C:\\Users\\l1f15bscs0049\\Desktop\\simple_CNN.81-0.96.hdf5'
gender_labels = get_labels('imdb')
font = cv2.FONT_HERSHEY_SIMPLEX
# hyper-parameters for bounding boxes shape
gender_offsets = (30, 60)
gender_offsets = (10, 10)
# loading models
face_detection = load_detection_model(detection_model_path)
gender_classifier = load_model(gender_model_path, compile=False)
# getting input model shapes for inference
gender_target_size = gender_classifier.input_shape[1:3]
# loading images
rgb_image = load_image(image_path, grayscale=False)
gray_image = load_image(image_path, grayscale=True)
gray_image = np.squeeze(gray_image)
gray_image = gray_image.astype('uint8')
#creating a file
save_path = 'C:\\Users\\l1f15bscs0049\\Desktop'
completeName = os.path.join(save_path, "hellojee.txt")
file = open(completeName, "a")
faces = detect_faces(face_detection, gray_image)
for face_coordinates in faces:
x1, x2, y1, y2 = apply_offsets(face_coordinates, gender_offsets)
rgb_face = rgb_image[y1:y2, x1:x2]
try:
rgb_face = cv2.resize(rgb_face, (gender_target_size))
except:
continue
rgb_face = preprocess_input(rgb_face, False)
rgb_face = np.expand_dims(rgb_face, 0)
gender_prediction = gender_classifier.predict(rgb_face)
gender_label_arg = np.argmax(gender_prediction)
gender_text = gender_labels[gender_label_arg]
#print(gender_label_arg)
file.write(str(gender_label_arg))
file.write("\n")
if gender_text == gender_labels[0]:
color = (0, 0, 255)
else:
color = (255, 0, 0)
draw_bounding_box(face_coordinates, rgb_image, color)
draw_text(face_coordinates, rgb_image, gender_text, color, 0, -20, 1,
2)
#print(gender_label_arg)
bgr_image = cv2.cvtColor(rgb_image, cv2.COLOR_RGB2BGR)
#cv2.imwrite('C:\\Users\\l1f15bscs0049\\Desktop\\a.png', bgr_image)
print('\n\tGender Detection Done')
file.close()
#check men women count
from collections import Counter
with open(completeName, "r") as f:
cd = Counter(int(line.split(None, 1)[0]) for line in f)
#print(cd)
women_count = cd[0]
men_count = cd[1]
# print(women_count)
#print(men_count)
#print(cd[0])
#print(cd[1])
os.remove(completeName)
print("file removed")
#call a wrapper function
if (women_count > men_count):
print("Women detected")
Wrapper_func(0)
elif (men_count > women_count):
print("men detected")
Wrapper_func(1)
else:
print("no Detection\n Random Ad's playing\n")
random_ads()
file.close()
def process_image(image):
try:
# parameters for loading data and images
detection_model_path = './trained_models/detection_models/haarcascade_frontalface_default.xml'
emotion_model_path = './trained_models/emotion_models/fer2013_mini_XCEPTION.102-0.66.hdf5'
gender_model_path = './trained_models/gender_models/simple_CNN.81-0.96.hdf5'
emotion_labels = get_labels('fer2013')
gender_labels = get_labels('imdb')
font = cv2.FONT_HERSHEY_SIMPLEX
# hyper-parameters for bounding boxes shape
gender_offsets = (30, 60)
gender_offsets = (10, 10)
emotion_offsets = (20, 40)
emotion_offsets = (0, 0)
# loading models
face_detection = load_detection_model(detection_model_path)
emotion_classifier = load_model(emotion_model_path, compile=False)
gender_classifier = load_model(gender_model_path, compile=False)
# getting input model shapes for inference
emotion_target_size = emotion_classifier.input_shape[1:3]
gender_target_size = gender_classifier.input_shape[1:3]
# loading images
image_array = np.fromstring(image, np.uint8)
unchanged_image = cv2.imdecode(image_array, cv2.IMREAD_UNCHANGED)
rgb_image = cv2.cvtColor(unchanged_image, cv2.COLOR_BGR2RGB)
gray_image = cv2.cvtColor(unchanged_image, cv2.COLOR_BGR2GRAY)
faces = detect_faces(face_detection, gray_image)
for face_coordinates in faces:
x1, x2, y1, y2 = apply_offsets(face_coordinates, gender_offsets)
rgb_face = rgb_image[y1:y2, x1:x2]
x1, x2, y1, y2 = apply_offsets(face_coordinates, emotion_offsets)
gray_face = gray_image[y1:y2, x1:x2]
try:
rgb_face = cv2.resize(rgb_face, (gender_target_size))
gray_face = cv2.resize(gray_face, (emotion_target_size))
except:
continue
rgb_face = preprocess_input(rgb_face, False)
rgb_face = np.expand_dims(rgb_face, 0)
gender_prediction = gender_classifier.predict(rgb_face)
gender_label_arg = np.argmax(gender_prediction)
gender_text = gender_labels[gender_label_arg]
gray_face = preprocess_input(gray_face, True)
gray_face = np.expand_dims(gray_face, 0)
gray_face = np.expand_dims(gray_face, -1)
emotion_label_arg = np.argmax(emotion_classifier.predict(gray_face))
emotion_text = emotion_labels[emotion_label_arg]
return emotion_text
# if gender_text == gender_labels[0]:
# color = (0, 0, 255)
# else:
# color = (255, 0, 0)
# draw_bounding_box(face_coordinates, rgb_image, color)
# draw_text(face_coordinates, rgb_image, gender_text, color, 0, -20, 1, 2)
# draw_text(face_coordinates, rgb_image, emotion_text, color, 0, -50, 1, 2)
except Exception as err:
logging.error('Error in emotion gender processor: "{0}"'.format(err))
def exe_c(path):
image_path = path #!!!获取图片路径的赋值到image_path
# parameters for loading data and images 加载数据和图片的参数
#sys.argv[1]
img_i=1
detection_model_path = '../trained_models/detection_models/haarcascade_frontalface_default.xml'
emotion_model_path = '../trained_models/emotion_models/fer2013_mini_XCEPTION.102-0.66.hdf5'
gender_model_path = '../trained_models/gender_models/simple_CNN.81-0.96.hdf5'
emotion_labels = get_labels('fer2013')
gender_labels = get_labels('imdb')
font = cv2.FONT_HERSHEY_SIMPLEX
# hyper-parameters for bounding boxes shape 边框大小——的超参数
#gender_offsets = (18, 25)
gender_offsets = (10, 10)
#emotion_offsets = (18, 25)
emotion_offsets = (0, 0)
# loading models 加载模型__face_detection,emotion_classifier,gender_classifier
face_detection = load_detection_model(detection_model_path)
emotion_classifier = load_model(emotion_model_path, compile=False)
gender_classifier = load_model(gender_model_path, compile=False)
# getting input model shapes for inference 获取输入模型的引用
emotion_target_size = emotion_classifier.input_shape[1:3]
gender_target_size = gender_classifier.input_shape[1:3]
# loading images 加载图片
rgb_image = load_image(image_path, grayscale=False)
gray_image = load_image(image_path, grayscale=True)
gray_image = np.squeeze(gray_image)
gray_image = gray_image.astype('uint8')
faces = detect_faces(face_detection, gray_image) #检测脸（face_detection，灰度图像？）
Jsons = []
for inx, face_coordinates in enumerate(faces): #enumerate 循环获取出inx值
x1, x2, y1, y2 = apply_offsets(face_coordinates, gender_offsets)
rgb_face = rgb_image[y1:y2, x1:x2]
x1, x2, y1, y2 = apply_offsets(face_coordinates, emotion_offsets)
gray_face = gray_image[y1:y2, x1:x2]
try:
rgb_face = cv2.resize(rgb_face, (gender_target_size))
gray_face = cv2.resize(gray_face, (emotion_target_size))
except:
continue
rgb_face = preprocess_input(rgb_face, False)
rgb_face = np.expand_dims(rgb_face, 0)
gender_prediction = gender_classifier.predict(rgb_face)
gender_label_arg = np.argmax(gender_prediction)
gender_text = gender_labels[gender_label_arg]
#jsonp['gender_text'] = gender_text #添加jsonp的gender标签
gray_face = preprocess_input(gray_face, True)
gray_face = np.expand_dims(gray_face, 0)
gray_face = np.expand_dims(gray_face, -1)
emotion_label_arg = np.argmax(emotion_classifier.predict(gray_face))
emotion_text = emotion_labels[emotion_label_arg]
# jsonp['emotion_text'] = emotion_text #添加jsonp的emotion标签
if gender_text == gender_labels[0]:
color = (0, 0, 255)
else:
color = (255, 0, 0)
draw_bounding_box(face_coordinates, rgb_image, color)
draw_text(face_coordinates, rgb_image, gender_text, color, 0, -20, 1, 2)
draw_text(face_coordinates, rgb_image, emotion_text, color, 0, -50, 1, 2)
#以下是bound列表转换成字典的方法
bound_key=['x','y','w','h']
c_x,c_y,c_w,c_h=face_coordinates
bound_xy=[str(c_x),str(c_y),str(c_w),str(c_h)]
bound_box = dict(zip(bound_key, bound_xy))
print(gender_text)
Jsons.append({'person':inx,"gender":gender_text,"emotion":emotion_text,'bound_box':bound_box}) #循环添加jsonp字典到jsons列表
bgr_image = cv2.cvtColor(rgb_image, cv2.COLOR_RGB2BGR)
img_i+=1
cv2.imwrite('../images/upload/result/predict_img'+str(img_i)+'.png', bgr_image) #将opencv2写出图像到指定路径
#{'gender':gender_text,'emotion':emotion_text}
return Jsons
def get_frame(self):
success, bgr_image = self.video.read()
# We are using Motion JPEG, but OpenCV defaults to capture raw images,
# so we must encode it into JPEG in order to correctly display the
# video stream.
# faces = face_cascade.detectMultiScale(image, 1.3, 5)
# for (x, y, w, h) in faces:
# cv2.rectangle(image, (x, y), (x + w, y + h), (255, 0, 0), 2)
# crop = image[y:y + h, x:x + w]
# cv2.imwrite(str(random.random()) + 'some_image.png', crop)
gray_image = cv2.cvtColor(bgr_image, cv2.COLOR_BGR2GRAY)
rgb_image = cv2.cvtColor(bgr_image, cv2.COLOR_BGR2RGB)
faces = self.face_cascade.detectMultiScale(
gray_image,
scaleFactor=1.1,
minNeighbors=5,
minSize=(30, 30),
flags=cv2.CASCADE_SCALE_IMAGE)
for face_coordinates in faces:
x1, x2, y1, y2 = apply_offsets(face_coordinates,
self.emotion_offsets)
gray_face = gray_image[y1:y2, x1:x2]
try:
gray_face = cv2.resize(gray_face, (self.emotion_target_size))
except:
continue
gray_face = preprocess_input(gray_face, True)
gray_face = np.expand_dims(gray_face, 0)
gray_face = np.expand_dims(gray_face, -1)
print(x1, x2, y1, y2)
emotion_prediction = self.emotion_classifier.predict(gray_face)
emotion_probability = np.max(emotion_prediction)
emotion_label_arg = np.argmax(emotion_prediction)
print(date_diff_in_Seconds(datetime.now(), self.log_date))
if date_diff_in_Seconds(datetime.now(), self.log_date) > 3:
self.log_date = datetime.now()
print("emotion log done")
self.current_emotion = Emotion.select().where(
Emotion.id == emotion_label_arg).get()
emotionlog = ChangeLogRecords(
patient_id=self.current_patient,
current_emotion_id=self.current_emotion,
change_date=datetime.now())
emotionlog.save()
emotion_text = self.emotion_labels[emotion_label_arg]
self.emotion_window.append(emotion_text)
if len(self.emotion_window) > self.frame_window:
self.emotion_window.pop(0)
try:
emotion_mode = mode(self.emotion_window)
except:
continue
if emotion_text == 'angry':
color = emotion_probability * np.asarray((255, 0, 0))
elif emotion_text == 'sad':
color = emotion_probability * np.asarray((0, 0, 255))
elif emotion_text == 'happy':
color = emotion_probability * np.asarray((255, 255, 0))
elif emotion_text == 'surprise':
color = emotion_probability * np.asarray((0, 255, 255))
else:
color = emotion_probability * np.asarray((0, 255, 0))
color = color.astype(int)
color = color.tolist()
draw_bounding_box(face_coordinates, rgb_image, color)
draw_text(face_coordinates, rgb_image, emotion_mode, color, 0, -45,
1, 1)
bgr_image = cv2.cvtColor(rgb_image, cv2.COLOR_RGB2BGR)
ret, jpeg = cv2.imencode('.jpg', bgr_image)
return jpeg.tobytes()
def start():
os.system("ls > a")
f = open("a", "r").readlines()
foldername = 'newpics' + str(len(f) + 1)
os.system("mkdir " + foldername)
USE_WEBCAM = True # If false, loads video file source
# parameters for loading data and images
emotion_model_path = './models/emotion_model.hdf5'
emotion_labels = get_labels('fer2013')
# hyper-parameters for bounding boxes shape
frame_window = 10
emotion_offsets = (20, 40)
# loading models
face_cascade = cv2.CascadeClassifier(
'./models/haarcascade_frontalface_default.xml')
emotion_classifier = load_model(emotion_model_path)
# getting input model shapes for inference
emotion_target_size = emotion_classifier.input_shape[1:3]
# starting lists for calculating modes
emotion_window = []
l = time.time()
# starting video streaming
cv2.namedWindow('window_frame')
#video_capture = cv2.VideoCapture(0)c
# Select video or webcam feed
cap = None
if (USE_WEBCAM == True):
cap = cv2.VideoCapture(0) # Webcam source
else:
cap = cv2.VideoCapture('./demo/dinner.mp4') # Video file source
i = 0
while cap.isOpened(): # True:
flag = 1
ret, bgr_image = cap.read()
#bgr_image = video_capture.read()[1]
m = 0
gray_image = cv2.cvtColor(bgr_image, cv2.COLOR_BGR2GRAY)
rgb_image = cv2.cvtColor(bgr_image, cv2.COLOR_BGR2RGB)
o = bgr_image
faces = face_cascade.detectMultiScale(gray_image,
scaleFactor=1.1,
minNeighbors=5,
minSize=(30, 30),
flags=cv2.CASCADE_SCALE_IMAGE)
for face_coordinates in faces:
m = 1
x1, x2, y1, y2 = apply_offsets(face_coordinates, emotion_offsets)
gray_face = gray_image[y1:y2, x1:x2]
try:
gray_face = cv2.resize(gray_face, (emotion_target_size))
except:
continue
gray_face = preprocess_input(gray_face, True)
gray_face = np.expand_dims(gray_face, 0)
gray_face = np.expand_dims(gray_face, -1)
emotion_prediction = emotion_classifier.predict(gray_face)
emotion_probability = np.max(emotion_prediction)
emotion_label_arg = np.argmax(emotion_prediction)
emotion_text = emotion_labels[emotion_label_arg]
emotion_window.append(emotion_text)
if len(emotion_window) > frame_window:
emotion_window.pop(0)
try:
emotion_mode = mode(emotion_window)
except:
continue
#o = cv2.cvtColor(rgb_image, cv2.COLOR_RGB2BGR)
print(emotion_probability * 100),
if emotion_text == 'angry':
color = emotion_probability * np.asarray((255, 0, 0))
## l = time.time()
# cv2.imwrite('foldername/pic'+str(i)+'.png',o)
# i+=1
flag = 0
break
elif emotion_text == 'sad':
color = emotion_probability * np.asarray((0, 0, 255))
#if emotion_probability*100 >=90 and time.time() - l >= 3:
# l = time.time()
# i+=1
flag = 0
break
elif emotion_text == 'happy':
color = emotion_probability * np.asarray((255, 255, 0))
if flag == 1 and emotion_probability * 100 >= 60:
flag = 1
else:
flag = 0
break
elif emotion_text == 'surprise':
color = emotion_probability * np.asarray((0, 255, 255))
if flag == 1 and emotion_probability * 100 >= 60:
flag = 1
else:
flag = 0
break
else:
flag = 0
break
color = emotion_probability * np.asarray((0, 255, 0))
color = color.astype(int)
color = color.tolist()
draw_bounding_box(face_coordinates, rgb_image, color)
draw_text(face_coordinates, rgb_image, emotion_mode, color, 0, -45,
1, 1)
print time.time() - l
if time.time() - l >= 3 and flag == 1 and m == 1:
cv2.imwrite(foldername + '/pic' + str(i) + '.png', o)
i += 1
l = time.time()
print ';'
bgr_image = cv2.cvtColor(rgb_image, cv2.COLOR_RGB2BGR)
cv2.imshow('window_frame', bgr_image)
cv2.waitKey(1)
if KEY == 1:
break
cap.release()
cv2.destroyAllWindows()
# starting lists for calculating modes
emotion_window = []
# starting video streaming
cv2.namedWindow('window_frame')
video_capture = cv2.VideoCapture(0)
while True:
bgr_image = video_capture.read()[1]
gray_image = cv2.cvtColor(bgr_image, cv2.COLOR_BGR2GRAY)
rgb_image = cv2.cvtColor(bgr_image, cv2.COLOR_BGR2RGB)
faces = detect_faces(face_detection, gray_image)
for face_coordinates in faces:
x1, x2, y1, y2 = apply_offsets(face_coordinates, emotion_offsets)
gray_face = gray_image[y1:y2, x1:x2]
try:
gray_face = cv2.resize(gray_face, (emotion_target_size))
except:
continue
gray_face = preprocess_input(gray_face, True)
gray_face = np.expand_dims(gray_face, 0)
gray_face = np.expand_dims(gray_face, -1)
emotion_prediction = emotion_classifier.predict(gray_face)
emotion_probability = np.max(emotion_prediction)
emotion_label_arg = np.argmax(emotion_prediction)
emotion_text = emotion_labels[emotion_label_arg]
print emotion_text
print(emotion_probability)
def fun(in_path, out_video_path, out_info_path, in_finished_path, model_path,
video_resolution):
"""
>>> fun(/fer_input, /fer_output, /fer_result, /fer_finished, /fer_model, video_resolution)
.mp4 files in the fer_intput folder will move to fer_finished folder.
Processed .mp4 files will be saved in fer_output folder.
.csv files will be saved in fer_result folder.
only process the video that its resolution is 720p and above(video_resolution = 720, can be adjusted)
"""
global model, F
detect_emo = True
#save config
save_video = True
save_info = True
show_video = False
#config = tf.ConfigProto()
# config.gpu_options.per_process_gpu_memory_fraction = 0.7
# config.gpu_options.allow_growth = True
# session = InteractiveSession(config=config)
#%%
# parameters for loading data and images
detection_model_path = model_path + '/haarcascade_frontalface_default.xml'
if detect_emo:
emotion_model_path = model_path + '/googlenet__googlenetwei__2020Aug29_16.21'
emotion_labels = get_labels('fer2013')
emotion_offsets = (20, 40)
# loading models
model = getattr(model, 'googlenet')
model = model(in_channels=3, num_classes=7)
#print(torch.cuda.is_available())
#print(torch.cuda.device_count())
state = torch.load(emotion_model_path, map_location='cpu')
model.load_state_dict(state['net'])
#model.cuda()
model.eval()
# getting input model shapes for inference
emotion_target_size = (224, 224)
# starting lists for calculating modes
emotion_window = []
# hyper-parameters for bounding boxes shape
frame_window = 10
emotion_offsets = (20, 40)
# loading models
face_detection = load_detection_model(detection_model_path)
info_name = [
'time', 'frame', 'face_x', 'face_y', 'face_w', 'face_h', 'emotion',
'angry_prob', 'disgust_prob', 'fear_prob', 'happy_prob', 'sad_prob',
'surprise_prob', 'neutral_prob'
]
input_video_root = in_path
output_video_root = out_video_path
output_info_root = out_info_path
for video_path in glob.glob(input_video_root + '/**/*.mp4',
recursive=True):
print(video_path)
no_root_path = video_path[len(input_video_root):].replace(
video_path.split('/')[-1], '')
video_capture = cv2.VideoCapture(video_path)
video_cap_ori = video_capture
video_name = video_path.split('/')[-1].split('.mp4')[0]
ori_video_name = video_path.split('/')[-1]
fps_float = video_capture.get(cv2.CAP_PROP_FPS)
fps = round(video_capture.get(cv2.CAP_PROP_FPS))
size = (round(video_capture.get(3)), round(video_capture.get(4))
) # float
ori_size = size
reduce_resolution = 0
scaling_factor_x = 1
scaling_factor_y = 1
if video_resolution == "720p" and size[0] > 1280 and size[1] > 720:
#need to reduce resolution to 720p
reduce_resolution = 1
out_path = input_video_root + no_root_path + 'resize_to_720p_' + video_path.split(
'/')[-1]
fourcc = cv2.VideoWriter_fourcc(*'MP4V')
out = cv2.VideoWriter(out_path, fourcc, fps, (1280, 720))
while True:
ret, frame = video_capture.read()
if ret == True:
b = cv2.resize(frame, (1280, 720),
fx=0,
fy=0,
interpolation=cv2.INTER_CUBIC)
out.write(b)
else:
break
video_capture.release()
out.release()
scaling_factor_x = size[0] / 1280
scaling_factor_y = size[1] / 720
#original resolution video move to fer_finished dir
src = video_path
dst = in_finished_path + no_root_path + video_name + ".mp4"
os.makedirs(os.path.dirname(in_finished_path + no_root_path),
exist_ok=True)
shutil.move(src, dst)
#capture ori resolution video to draw bounding box
video_cap_ori = cv2.VideoCapture(dst)
#capture reducing resolution video to construct csv file
video_path = out_path
video_capture = cv2.VideoCapture(video_path)
video_name = video_path.split('/')[-1].split('.mp4')[0]
fps_float = video_capture.get(cv2.CAP_PROP_FPS)
fps = round(video_capture.get(cv2.CAP_PROP_FPS))
size = (round(video_capture.get(3)), round(video_capture.get(4))
) # float
if size[0] == 1280 and size[1] == 720:
if save_video:
os.makedirs(os.path.dirname(output_video_root + no_root_path),
exist_ok=True)
out_path = output_video_root + no_root_path + ori_video_name
fourcc = cv2.VideoWriter_fourcc(*'MP4V')
out = cv2.VideoWriter(out_path, fourcc, fps, ori_size)
if save_info:
os.makedirs(os.path.dirname(output_info_root + no_root_path),
exist_ok=True)
csv_info = codecs.open(output_info_root + no_root_path +
video_name + '_info.csv',
'w',
encoding="utf_8_sig")
csv_writer = csv.writer(csv_info)
csv_writer.writerow(info_name)
frame_idx = 0
st_time = time.time()
while (video_cap_ori.isOpened()):
if frame_idx % 10 == 0:
print('Processing frame: ' + str(frame_idx) + ' ......')
video_flag_ori, bgr_image_ori = video_cap_ori.read(
) #ori image
video_flag, bgr_image = video_capture.read() #downscale image
if video_flag_ori:
frame_idx += 1
gray_image = cv2.cvtColor(bgr_image, cv2.COLOR_BGR2GRAY)
#rgb_image = cv2.cvtColor(bgr_image, cv2.COLOR_BGR2RGB)
gray_image_ori = cv2.cvtColor(bgr_image_ori,
cv2.COLOR_BGR2GRAY)
rgb_image_ori = cv2.cvtColor(bgr_image_ori,
cv2.COLOR_BGR2RGB)
faces = detect_faces(face_detection, gray_image)
if not isinstance(faces, tuple):
faces = faces[faces[:, 0].argsort()]
faces = faces[faces[:, 1].argsort()]
faces = faces[faces[:, 2].argsort()]
faces = faces[faces[:, 3].argsort()]
for face_coordinates in faces:
x_1, x_2, y_1, y_2 = apply_offsets(
face_coordinates, emotion_offsets)
if detect_emo:
gray_face = gray_image[y_1:y_2, x_1:x_2]
try:
gray_face = cv2.resize(gray_face,
(emotion_target_size))
except:
continue
gray_face = np.dstack([gray_face] * 3)
gray_face = transforms.Compose([
transforms.ToPILImage(),
transforms.ToTensor(),
])(np.uint8(gray_face))
gray_face = torch.stack([gray_face], 0)
#gray_face = gray_face.cuda(non_blocking=True)
outputs = model(gray_face).cpu()
outputs = F.softmax(outputs, 1)
emotion_prediction = torch.sum(outputs, 0).cpu(
).detach().numpy() # outputs.shape [tta_size, 7]
emotion_probability = np.max(emotion_prediction)
emotion_label_arg = np.argmax(emotion_prediction)
emotion_text = emotion_labels[emotion_label_arg]
emotion_window.append(emotion_text)
if len(emotion_window) > frame_window:
emotion_window.pop(0)
try:
emotion_mode = mode(emotion_window)
except:
continue
x = int(
float(face_coordinates[0] * scaling_factor_x))
y = int(
float(face_coordinates[1] * scaling_factor_y))
w = int(
float(face_coordinates[2] * scaling_factor_x))
h = int(
float(face_coordinates[3] * scaling_factor_y))
if emotion_text == 'angry':
cv2.rectangle(bgr_image_ori, (x, y),
(x + w, y + h), (255, 0, 0), 4)
cv2.putText(bgr_image_ori, 'angry', (int(
float(x + w / 2 - 43)), y - 10),
cv2.FONT_HERSHEY_DUPLEX, 1,
(255, 0, 0), 1, cv2.LINE_AA)
elif emotion_text == 'sad':
cv2.rectangle(bgr_image_ori, (x, y),
(x + w, y + h), (0, 0, 255), 4)
cv2.putText(bgr_image_ori, 'sad', (int(
float(x + w / 2 - 43)), y - 10),
cv2.FONT_HERSHEY_DUPLEX, 1,
(0, 0, 255), 1, cv2.LINE_AA)
elif emotion_text == 'happy':
cv2.rectangle(bgr_image_ori, (x, y),
(x + w, y + h), (255, 255, 0), 4)
cv2.putText(bgr_image_ori, 'happy', (int(
float(x + w / 2 - 43)), y - 10),
cv2.FONT_HERSHEY_DUPLEX, 1,
(255, 255, 0), 1, cv2.LINE_AA)
elif emotion_text == 'surprise':
cv2.rectangle(bgr_image_ori, (x, y),
(x + w, y + h), (0, 255, 255), 4)
cv2.putText(bgr_image_ori, 'surprise', (int(
float(x + w / 2 - 43)), y - 10),
cv2.FONT_HERSHEY_DUPLEX, 1,
(0, 255, 255), 1, cv2.LINE_AA)
elif emotion_text == 'disgust':
cv2.rectangle(bgr_image_ori, (x, y),
(x + w, y + h), (0, 0, 0), 4)
cv2.putText(bgr_image_ori, 'disgust', (int(
float(x + w / 2 - 43)), y - 10),
cv2.FONT_HERSHEY_DUPLEX, 1,
(0, 0, 0), 1, cv2.LINE_AA)
elif emotion_text == 'fear':
cv2.rectangle(bgr_image_ori, (x, y),
(x + w, y + h), (255, 0, 255), 4)
cv2.putText(bgr_image_ori, 'fear', (int(
float(x + w / 2 - 43)), y - 10),
cv2.FONT_HERSHEY_DUPLEX, 1,
(255, 0, 255), 1, cv2.LINE_AA)
else:
cv2.rectangle(bgr_image_ori, (x, y),
(x + w, y + h), (0, 255, 0), 4)
cv2.putText(bgr_image_ori, 'neutral', (int(
float(x + w / 2 - 43)), y - 10),
cv2.FONT_HERSHEY_DUPLEX, 1,
(0, 255, 0), 1, cv2.LINE_AA)
if not detect_emo:
color = np.asarray((0, 0, 0))
color = color.astype(int)
color = color.tolist()
draw_bounding_box(face_coordinates, rgb_image,
color)
if save_info:
op_info_list = [
round(frame_idx / fps_float, 3), frame_idx,
face_coordinates[0] * scaling_factor_x,
face_coordinates[1] * scaling_factor_y,
face_coordinates[2] * scaling_factor_x,
face_coordinates[3] * scaling_factor_y
]
for i in range(len(op_info_list)):
op_info_list[i] = str(op_info_list[i])
if detect_emo:
op_info_list.append(emotion_text)
for prob in emotion_prediction:
op_info_list.append(prob)
csv_writer.writerow(op_info_list)
#bgr_image = cv2.cvtColor(rgb_image, cv2.COLOR_RGB2BGR)
if save_video:
out.write(bgr_image_ori)
if show_video:
cv2.imshow('window_frame', bgr_image)
if cv2.waitKey(1) & 0xFF == ord('q'):
break
else:
break
if save_video:
out.release()
if save_info:
csv_info.close()
print(video_path + ' DONE!!\tSpend Time: ' +
str(time.time() - st_time) + '(s)')
video_capture.release()
video_cap_ori.release()
if show_video:
cv2.destroyAllWindows()
else:
os.makedirs(os.path.dirname(output_info_root + no_root_path),
exist_ok=True)
csv_info = codecs.open(output_info_root + no_root_path +
video_name + '_info.csv',
'w',
encoding="utf_8_sig")
csv_writer = csv.writer(csv_info)
err_msg = "The resolution of " + video_name + ".mp4 is lower than 720p."
csv_writer.writerow([err_msg])
csv_info.close()
src = video_path
dst = in_finished_path + no_root_path + video_name + ".mp4"
os.makedirs(os.path.dirname(in_finished_path + no_root_path),
exist_ok=True)
shutil.move(src, dst)
if reduce_resolution == 1:
video_ori_name = video_name[15:]
csv_path_rename = output_info_root + no_root_path + video_name + '_info.csv'
os.remove(dst)
os.rename(
output_info_root + no_root_path + video_name + '_info.csv',
output_info_root + no_root_path + video_ori_name + '_info.csv')
shutil.rmtree(input_video_root, ignore_errors=True)
def emotion_analyse(path):
print('Emotion Analysing...........................')
with graph.as_default():
emotion_analyse_data = {'angry': 0, 'sad': 0, 'happy': 0, 'surprise': 0, 'fear': 0, 'angry': 0}
# video_capture = cv2.VideoCapture(path)
vs = VideoStream(path).start()
# length = int(vs.get(cv2.CAP_PROP_FRAME_COUNT))
while True:
# bgr_image = video_capture.read()[1]
bgr_image = vs.read()
if bgr_image is None:
break
gray_image = cv2.cvtColor(bgr_image, cv2.COLOR_BGR2GRAY)
rgb_image = cv2.cvtColor(bgr_image, cv2.COLOR_BGR2RGB)
faces = detect_faces(face_detection, gray_image)
for face_coordinates in faces:
x1, x2, y1, y2 = apply_offsets(face_coordinates, emotion_offsets)
gray_face = gray_image[y1:y2, x1:x2]
try:
gray_face = cv2.resize(gray_face, (emotion_target_size))
except:
continue
gray_face = preprocess_input(gray_face, True)
gray_face = np.expand_dims(gray_face, 0)
gray_face = np.expand_dims(gray_face, -1)
emotion_prediction = emotion_classifier.predict(gray_face)
emotion_probability = np.max(emotion_prediction)
emotion_label_arg = np.argmax(emotion_prediction)
emotion_text = emotion_labels[emotion_label_arg]
emotion_window.append(emotion_text)
if len(emotion_window) > frame_window:
emotion_window.pop(0)
try:
emotion_mode = mode(emotion_window)
except:
continue
if emotion_text == 'angry':
color = emotion_probability * np.asarray((255, 0, 0))
emotion_analyse_data['angry'] = emotion_analyse_data.pop('angry') + 1
elif emotion_text == 'sad':
color = emotion_probability * np.asarray((0, 0, 255))
emotion_analyse_data['sad'] = emotion_analyse_data.pop('sad') + 1
elif emotion_text == 'happy':
color = emotion_probability * np.asarray((255, 255, 0))
emotion_analyse_data['happy'] = emotion_analyse_data.pop('happy') + 1
elif emotion_text == 'surprise':
color = emotion_probability * np.asarray((0, 255, 255))
emotion_analyse_data['surprise'] = emotion_analyse_data.pop('surprise') + 1
else:
color = emotion_probability * np.asarray((0, 255, 0))
emotion_analyse_data['fear'] = emotion_analyse_data.pop('fear') + 1
color = color.astype(int)
color = color.tolist()
# print(emotion_text)
draw_bounding_box(face_coordinates, rgb_image, color)
draw_text(face_coordinates, rgb_image, emotion_mode,
color, 0, -45, 1, 1)
print('Emotion Analysed...........................')
return emotion_analyse_data
def main():
cpt = 0
RESEND_LIMIT = 15 # about half a second lol
context = zmq.Context()
socket = context.socket(zmq.REQ)
socket.connect('tcp://127.0.0.1:5555')
# parameters for loading data and images
detection_model_path = '../trained_models/detection_models/haarcascade_frontalface_default.xml'
emotion_model_path = '../trained_models/emotion_models/fer2013_mini_XCEPTION.102-0.66.hdf5'
emotion_labels = get_labels('fer2013')
# hyper-parameters for bounding boxes shape
frame_window = 10
emotion_offsets = (20, 40)
# loading models
face_detection = load_detection_model(detection_model_path)
emotion_classifier = load_model(emotion_model_path, compile=False)
# getting input model shapes for inference
emotion_target_size = emotion_classifier.input_shape[1:3]
# starting lists for calculating modes
emotion_window = []
# starting video streaming
cv2.namedWindow('window_frame')
video_capture = cv2.VideoCapture(0)
while True:
bgr_image = video_capture.read()[1]
gray_image = cv2.cvtColor(bgr_image, cv2.COLOR_BGR2GRAY)
rgb_image = cv2.cvtColor(bgr_image, cv2.COLOR_BGR2RGB)
faces = detect_faces(face_detection, gray_image)
for face_coordinates in faces:
x1, x2, y1, y2 = apply_offsets(face_coordinates, emotion_offsets)
gray_face = gray_image[y1:y2, x1:x2]
try:
gray_face = cv2.resize(gray_face, (emotion_target_size))
except:
continue
gray_face = preprocess_input(gray_face, True)
gray_face = np.expand_dims(gray_face, 0)
gray_face = np.expand_dims(gray_face, -1)
emotion_prediction = emotion_classifier.predict(gray_face)
emotion_probability = np.max(emotion_prediction)
emotion_label_arg = np.argmax(emotion_prediction)
emotion_text = emotion_labels[emotion_label_arg]
emotion_window.append(emotion_text)
if len(emotion_window) > frame_window:
emotion_window.pop(0)
try:
emotion_mode = mode(emotion_window)
except:
continue
cpt += 1
if (cpt > RESEND_LIMIT):
socket.send_string(emotion_text)
msg = socket.recv()
cpt = 0
if emotion_text == 'angry':
color = emotion_probability * np.asarray((255, 0, 0))
elif emotion_text == 'sad':
color = emotion_probability * np.asarray((0, 0, 255))
elif emotion_text == 'happy':
color = emotion_probability * np.asarray((255, 255, 0))
elif emotion_text == 'surprise':
color = emotion_probability * np.asarray((0, 255, 255))
else:
color = emotion_probability * np.asarray((0, 255, 0))
color = color.astype(int)
color = color.tolist()
draw_bounding_box(face_coordinates, rgb_image, color)
draw_text(face_coordinates, rgb_image, emotion_mode, color, 0, -45,
1, 1)
bgr_image = cv2.cvtColor(rgb_image, cv2.COLOR_RGB2BGR)
cv2.imshow('window_frame', bgr_image)
if cv2.waitKey(1) & 0xFF == ord('q'):
break
video_capture.release()
cv2.destroyAllWindows()
def face_Identification(self):
flag, bgr_image = self.cap.read()
if (self.status != "1"):
return None
if flag:
gray_image = cv2.cvtColor(bgr_image, cv2.COLOR_BGR2GRAY)
rgb_image = cv2.cvtColor(bgr_image, cv2.COLOR_BGR2RGB)
faces = detect_faces(face_detection, gray_image)
img_h, img_w, _ = np.shape(rgb_image)
a = 0
for face_coordinates in faces:
# Output face number
print("---- Print face numbers ----")
print(faces.size / 4)
x1, x2, y1, y2 = apply_offsets(face_coordinates,
gender_offsets)
rgb_face = rgb_image[y1:y2, x1:x2]
x1, x2, y1, y2 = apply_offsets(face_coordinates,
emotion_offsets)
gray_face = gray_image[y1:y2, x1:x2]
try:
rgb_face = cv2.resize(rgb_face, (gender_target_size))
gray_face = cv2.resize(gray_face, (emotion_target_size))
except:
continue
# run_thread(bgr_image)
gray_face = preprocess_input(gray_face, False)
gray_face = np.expand_dims(gray_face, 0)
gray_face = np.expand_dims(gray_face, -1)
emotion_label_arg = np.argmax(
emotion_classifier.predict(gray_face))
emotion_text = emotion_labels[emotion_label_arg]
# emotion_window.append(English_2_chinese_emotion(emotion_text))
rgb_face = np.expand_dims(rgb_face, 0)
rgb_face = preprocess_input(rgb_face, False)
gender_prediction = gender_classifier.predict(rgb_face)
gender_label_arg = np.argmax(gender_prediction)
gender_text = gender_labels[gender_label_arg]
# gender_window.append(English_2_chinese_gender(gender_text))
set_icon = gender_text + "_" + emotion_text
print("----- emotion + gender -----")
print(set_icon)
words_img = words_dict[set_icon]
###################
if gender_text == gender_labels[0]:
color = (255, 0, 0)
else:
color = (0, 0, 255)
if (self.frq % 15 == 0):
# detect faces using dlib detector
detected = detector(rgb_image, 1)
faces_age = np.empty(
(len(detected), img_size, img_size, 3))
if len(detected) > 0:
for i, d in enumerate(detected):
x1, y1, x2, y2, w, h = d.left(), d.top(), d.right(
) + 1, d.bottom() + 1, d.width(), d.height()
xw1 = max(int(x1 - margin * w), 0)
yw1 = max(int(y1 - margin * h), 0)
xw2 = min(int(x2 + margin * w), img_w - 1)
yw2 = min(int(y2 + margin * h), img_h - 1)
cv2.rectangle(rgb_image, (x1, y1), (x2, y2),
(255, 0, 0), 2)
# cv2.rectangle(img, (xw1, yw1), (xw2, yw2), (255, 0, 0), 2)
faces_age[i, :, :, :] = cv2.resize(
rgb_image[yw1:yw2 + 1, xw1:xw2 + 1, :],
(img_size, img_size))
faces_age[i, :, :, :] = cv2.resize(
rgb_image[y1:y2, x1:x2, :],
(img_size, img_size))
# predict ages and genders of the detected faces
results = model.predict(faces_age)
ages = np.arange(0, 101).reshape(101, 1)
predicted_ages = results[1].dot(ages).flatten()
self.age_position = [
"30", "28", "25", "20", "35", "19", "30", "21",
"23", "26"
]
print("----- age -----")
print(predicted_ages)
for i, d in enumerate(detected):
print("-----every ages " + str(i) + "-----")
print(str(int(predicted_ages[i])))
if (int(predicted_ages[i] >= 25)):
self.age_position[i] = str(25 + int(
(predicted_ages[i] - 25) * 0.65))
else:
self.age_position[i] = str(25 - int(
(25 - predicted_ages[i]) * 0.65))
# self.age_position[i] = (
# str(int(predicted_ages[i] * 0.65)))
# if((face_coordinates[0] - face_coordinates[2]) > 50 and (face_coordinates[0] - face_coordinates[2]) < 180 and (face_coordinates[1]-80) > 20):
if (face_coordinates[1] - 80) > 20:
print("---- draw -----")
print(a, self.age_position[a])
print("---- draw -----")
solid_box = draw_solid_box(face_coordinates, rgb_image)
draw_bounding_box(face_coordinates, rgb_image, color)
words_img = cv2.resize(words_img, (180, 65))
solid_box = Addemotion_word(face_coordinates, solid_box,
words_img)
draw_text(face_coordinates, solid_box,
self.age_position[a], (255, 255, 255), 0, -20, 1,
1)
a += 1
self.frq += 1
bgr_image = cv2.cvtColor(rgb_image, cv2.COLOR_RGB2BGR)
show = cv2.resize(rgb_image, (1080, 960))
showImage = QtGui.QImage(show, show.shape[1], show.shape[0],
QtGui.QImage.Format_RGB888)
self.label_show_camera.setPixmap(
QtGui.QPixmap.fromImage(showImage))
def detect_face_emotions_and_draw(frame):
# parameters for loading data and images
detection_model_path = './trained_models/detection_models/haarcascade_frontalface_default.xml'
emotion_model_path = './trained_models/emotion_models/fer2013_mini_XCEPTION.102-0.66.hdf5'
emotion_labels = get_labels('fer2013')
# hyper-parameters for bounding boxes shape
frame_window = 10
emotion_offsets = (15, 35)
# loading models
face_detection = load_detection_model(detection_model_path)
emotion_classifier = load_model(emotion_model_path, compile=False)
# getting input model shapes for inference
emotion_target_size = emotion_classifier.input_shape[1:3]
# starting lists for calculating modes
emotion_window = []
bgr_image = frame
gray_image = cv2.cvtColor(bgr_image, cv2.COLOR_BGR2GRAY)
rgb_image = cv2.cvtColor(bgr_image, cv2.COLOR_BGR2RGB)
cv2.imwrite('temp.bmp', frame)
rect_image, faces = detect_faces('temp.bmp','./crops/')
cv2.imwrite('temp_rect.bmp', rect_image)
for face_coordinates in faces:
x1, x2, y1, y2 = apply_offsets(face_coordinates, emotion_offsets)
print(x1, x2, y1, y2, emotion_target_size)
gray_face = gray_image[y1:y2, x1:x2]
try:
gray_face = cv2.resize(gray_face, (emotion_target_size))
except:
print('exception ignored while resize')
continue
gray_face = preprocess_input(gray_face, True)
gray_face = np.expand_dims(gray_face, 0)
gray_face = np.expand_dims(gray_face, -1)
emotion_prediction = emotion_classifier.predict(gray_face)
emotion_probability = np.max(emotion_prediction)
emotion_label_arg = np.argmax(emotion_prediction)
emotion_text = emotion_labels[emotion_label_arg]
emotion_window.append(emotion_text)
if len(emotion_window) > frame_window:
emotion_window.pop(0)
#try:
#emotion_mode = mode(emotion_window)
#except:
# print('exception ignored for emotion mode')
# continue
if emotion_text == 'angry':
color = emotion_probability * np.asarray((255, 0, 0))
elif emotion_text == 'sad':
color = emotion_probability * np.asarray((0, 0, 255))
elif emotion_text == 'happy':
color = emotion_probability * np.asarray((255, 255, 0))
elif emotion_text == 'surprise':
color = emotion_probability * np.asarray((0, 255, 255))
else:
color = emotion_probability * np.asarray((0, 255, 0))
color = color.astype(int)
color = color.tolist()
print(emotion_text)
draw_bounding_box(face_coordinates, rgb_image, color)
draw_text(face_coordinates, rgb_image, emotion_text,
color, 0, 50, 1, 1)
bgr_image = cv2.cvtColor(rgb_image, cv2.COLOR_RGB2BGR)
return rgb_image
def callback(self,data):
try:
bgr_image = self.bridge.imgmsg_to_cv2(data, "bgr8")
cv2.imshow("Image window", bgr_image)
except CvBridgeError as e:
print(e)
######################### Start the image processing
gray_image = cv2.cvtColor(bgr_image, cv2.COLOR_BGR2GRAY)
rgb_image = cv2.cvtColor(bgr_image, cv2.COLOR_BGR2RGB)
faces = detect_faces(self.face_detection, gray_image)
for face_coordinates in faces:
x1, x2, y1, y2 = apply_offsets(face_coordinates, self.emotion_offsets)
gray_face = gray_image[y1:y2, x1:x2]
try:
gray_face = cv2.resize(gray_face, (self.emotion_target_size))
except:
continue
gray_face = preprocess_input(gray_face, True)
gray_face = np.expand_dims(gray_face, 0)
gray_face = np.expand_dims(gray_face, -1)
print "start"
print len(gray_face)
print gray_face
print "end"
emotion_prediction = self.emotion_classifier.predict(gray_face)
emotion_probability = np.max(emotion_prediction)
emotion_label_arg = np.argmax(emotion_prediction)
emotion_text = emotion_labels[emotion_label_arg]
print emotion_text
print(emotion_probability)
print('%')
self.emotion_window.append(emotion_text)
if len(self.emotion_window) > self.frame_window:
self.emotion_window.pop(0)
try:
emotion_mode = mode(self.emotion_window)
except:
continue
if emotion_text == 'angry':
color = emotion_probability * np.asarray((255, 0, 0))
elif emotion_text == 'sad':
color = emotion_probability * np.asarray((0, 0, 255))
elif emotion_text == 'happy':
color = emotion_probability * np.asarray((255, 255, 0))
elif emotion_text == 'surprise':
color = emotion_probability * np.asarray((0, 255, 255))
else:
color = emotion_probability * np.asarray((0, 255, 0))
color = color.astype(int)
color = color.tolist()
draw_bounding_box(face_coordinates, rgb_image, color)
draw_text(face_coordinates, rgb_image, emotion_mode,
color, 0, -45, 1, 1)
bgr_image = cv2.cvtColor(rgb_image, cv2.COLOR_RGB2BGR)
######################### End the image processing
cv2.imshow("Image window", bgr_image)
cv2.waitKey(3)
try:
self.image_pub.publish(self.bridge.cv2_to_imgmsg(bgr_image, "bgr8"))
except CvBridgeError as e:
print(e)
def main():
protoFile = "C:\\Users\\asus\\Documents\\major_project\\Project\\openpose-master\\openpose-master\\models\\pose\\coco\\pose_deploy_linevec.prototxt"
weightsFile = "C:\\Users\\asus\\Documents\\major_project\\Project\\openpose-master\\openpose-master\\models\\pose\\coco\\pose_iter_440000.caffemodel"
nPoints = 18
# COCO Output Format
keypointsMapping = [
'Nose', 'Neck', 'R-Sho', 'R-Elb', 'R-Wr', 'L-Sho', 'L-Elb', 'L-Wr',
'R-Hip', 'R-Knee', 'R-Ank', 'L-Hip', 'L-Knee', 'L-Ank', 'R-Eye',
'L-Eye', 'R-Ear', 'L-Ear'
]
POSE_PAIRS = [[1, 2], [1, 5], [2, 3], [3, 4], [5, 6], [6, 7], [1,
8], [8, 9],
[9, 10], [1, 11], [11, 12], [12, 13], [1, 0], [0, 14],
[14, 16], [0, 15], [15, 17], [2, 17], [5, 16]]
# index of pafs correspoding to the POSE_PAIRS
# e.g for POSE_PAIR(1,2), the PAFs are located at indices (31,32) of output, Similarly, (1,5) -> (39,40) and so on.
mapIdx = [[31, 32], [39, 40], [33, 34], [35, 36], [41, 42], [43, 44],
[19, 20], [21, 22], [23, 24], [25, 26], [27, 28], [29, 30],
[47, 48], [49, 50], [53, 54], [51, 52], [55, 56], [37, 38],
[45, 46]]
colors = [[0, 100, 255], [0, 100, 255], [0, 255, 255], [0, 100, 255],
[0, 255, 255], [0, 100, 255], [0, 255, 0], [255, 200, 100],
[255, 0, 255], [0, 255, 0], [255, 200, 100], [255, 0, 255],
[0, 0, 255], [255, 0, 0], [200, 200, 0], [255, 0, 0],
[200, 200, 0], [0, 0, 0]]
detection_model_path = 'C:\\Users\\asus\\Documents\\major_project\\Project\\face_classification-master\\face_classification-master\\trained_models\\detection_models\\haarcascade_frontalface_default.xml'
emotion_model_path = 'C:\\Users\\asus\\Documents\\major_project\\Project\\face_classification-master\\face_classification-master\\trained_models\\emotion_models\\fer2013_mini_XCEPTION.102-0.66.hdf5'
emotion_labels = get_labels('fer2013')
font = cv2.FONT_HERSHEY_SIMPLEX
# hyper-parameters for bounding boxes shape
emotion_offsets = (20, 40)
emotion_offsets = (0, 0)
# loading models
face_detection = load_detection_model(detection_model_path)
emotion_classifier = load_model(emotion_model_path, compile=False)
# getting input model shapes for inference
emotion_target_size = emotion_classifier.input_shape[1:3]
while True:
poll()
print(image_queue)
# parameters for loading data and images
image_name = image_queue.get()[0]
print(image_name)
image_path = get_image(image_name) #sys.argv[1]
# loading images
rgb_image = load_image(image_path, grayscale=False)
gray_image = load_image(image_path, grayscale=True)
gray_image = np.squeeze(gray_image)
gray_image = gray_image.astype('uint8')
faces = detect_faces(face_detection, gray_image)
cat_count = {
'angry': 0,
'disgust': 0,
'fear': 0,
'happy': 0,
'sad': 0,
'surprise': 0,
'neutral': 0
}
total_count = 0
for face_coordinates in faces:
x1, x2, y1, y2 = apply_offsets(face_coordinates, emotion_offsets)
gray_face = gray_image[y1:y2, x1:x2]
try:
gray_face = cv2.resize(gray_face, (emotion_target_size))
except:
continue
gray_face = preprocess_input(gray_face, True)
gray_face = np.expand_dims(gray_face, 0)
gray_face = np.expand_dims(gray_face, -1)
emotion_label_arg = np.argmax(
emotion_classifier.predict(gray_face))
emotion_text = emotion_labels[emotion_label_arg]
cat_count[emotion_text] = cat_count[emotion_text] + 1
total_count = total_count + 1
color = (255, 0, 0)
draw_bounding_box(face_coordinates, rgb_image, color)
draw_text(face_coordinates, rgb_image, emotion_text, color, 0, -50,
1, 2)
cv2.imwrite("../pose_images/" + image_name, rgb_image)
upload_file("../pose_images/" + image_name,
'major-project-processed-images', image_name)
connection = pymysql.connect(host,
user=user,
port=port,
passwd=password,
db=dbname)
cursor = connection.cursor()
cursor.execute(
'''INSERT INTO `expressions`(`name`, `happy`, `angry`, `sad`, `suprised`, `fear`,`neutral`,`disgust`,`total`)
VALUES (%s,
%s,
%s,
%s,
%s,
%s,
%s,
%s,
%s)''',
(image_name, cat_count['happy'], cat_count['angry'],
cat_count['sad'], cat_count['surprise'], cat_count['fear'],
cat_count['neutral'], cat_count['disgust'], total_count))
cursor.execute("commit")
#bgr_image = cv2.cvtColor(rgb_image, cv2.COLOR_RGB2BGR)
#cv2.imwrite('../images/predicted_test_image.png', bgr_image)
#pose estimation code........
image1 = cv2.imread(image_path)
frameWidth = image1.shape[1]
frameHeight = image1.shape[0]
#t = time.time()
net = cv2.dnn.readNetFromCaffe(protoFile, weightsFile)
# Fix the input Height and get the width according to the Aspect Ratio
inHeight = 368
inWidth = int((inHeight / frameHeight) * frameWidth)
inpBlob = cv2.dnn.blobFromImage(image1,
1.0 / 255, (inWidth, inHeight),
(0, 0, 0),
swapRB=False,
crop=False)
net.setInput(inpBlob)
output = net.forward()
#print("Time Taken in forward pass = {}".format(time.time() - t))
detected_keypoints = []
keypoints_list = np.zeros((0, 3))
keypoint_id = 0
threshold = 0.1
keypoint_location = {}
for part in range(nPoints):
probMap = output[0, part, :, :]
probMap = cv2.resize(probMap, (image1.shape[1], image1.shape[0]))
keypoints = getKeypoints(probMap, threshold)
print("Keypoints - {} : {}".format(keypointsMapping[part],
keypoints))
keypoints_with_id = []
for i in range(len(keypoints)):
keypoints_with_id.append(keypoints[i] + (keypoint_id, ))
keypoints_list = np.vstack([keypoints_list, keypoints[i]])
keypoint_location[keypoint_id] = keypoints[i]
keypoint_id += 1
detected_keypoints.append(keypoints_with_id)
frameClone = image1.copy()
for i in range(nPoints):
for j in range(len(detected_keypoints[i])):
cv2.circle(frameClone, detected_keypoints[i][j][0:2], 5,
colors[i], -1, cv2.LINE_AA)
#cv2.imshow("Keypoints",frameClone)
valid_pairs, invalid_pairs = getValidPairs(output, frameWidth,
frameHeight, mapIdx,
detected_keypoints,
POSE_PAIRS)
personwiseKeypoints = getPersonwiseKeypoints(valid_pairs,
invalid_pairs,
keypoints_list, mapIdx,
POSE_PAIRS)
for i in range(17):
for n in range(len(personwiseKeypoints)):
index = personwiseKeypoints[n][np.array(POSE_PAIRS[i])]
if -1 in index:
continue
B = np.int32(keypoints_list[index.astype(int), 0])
A = np.int32(keypoints_list[index.astype(int), 1])
cv2.line(frameClone, (B[0], A[0]), (B[1], A[1]), colors[i], 3,
cv2.LINE_AA)
cv2.imwrite("../pose_images/image0.jpg", frameClone)
class_status = classStatus(personwiseKeypoints, keypoint_location)
print(class_status)
cursor.execute(
'''INSERT INTO `gestures`(`name`,`left_turned`, `right_turned`, `back_turned`, `raised_hands`, `total`)
VALUES (%s,
%s,
%s,
%s,
%s,
%s
)''',
(image_name, class_status['turnedleft'],
class_status['turnedright'], class_status['turnedback'],
class_status['raisedhands'], class_status['classtotal']))
cursor.execute("commit")
#end of pose estimation code......................
cursor.execute(
''' UPDATE `images` SET `isprocessed`=1 WHERE `name`=%s''',
(image_name))
cursor.execute("commit")
cursor.close()
connection.close()
process_this_frame = not process_this_frame
# Emotion Detection
gray_image = cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY)
rgb_image = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)
faces = face_cascade.detectMultiScale(gray_image,
scaleFactor=1.1,
minNeighbors=5,
minSize=(30, 30),
flags=cv2.CASCADE_SCALE_IMAGE)
emotion_text = None
for face_coordinates in faces:
x1, x2, y1, y2 = apply_offsets(face_coordinates, emotion_offsets)
gray_face = gray_image[y1:y2, x1:x2]
try:
gray_face = cv2.resize(gray_face, (emotion_target_size))
except:
continue
gray_face = preprocess_input(gray_face, True)
gray_face = np.expand_dims(gray_face, 0)
gray_face = np.expand_dims(gray_face, -1)
emotion_prediction = emotion_classifier.predict(gray_face)
emotion_probability = np.max(emotion_prediction)
emotion_label_arg = np.argmax(emotion_prediction)
emotion_text = emotion_labels[emotion_label_arg]
# print(emotion_text)
# print(face_detection_name)
def func(video_path):
# file to store metadata
metaData = open(
'C:/Users/ASUS/Desktop/Face Recognition/trial1/Face Detection and Emotion Analysis/src/final1.csv',
'a')
writer = csv.writer(metaData)
# parameters for loading data and images
detection_model_path = '../trained_models/detection_models/haarcascade_frontalface_default.xml'
emotion_model_path = '../trained_models/emotion_models/fer2013_mini_XCEPTION.102-0.66.hdf5'
emotion_labels = get_labels('fer2013')
# hyper-parameters for bounding boxes shape
frame_window = 10
emotion_offsets = (20, 40)
# loading models
face_detection = load_detection_model(detection_model_path)
emotion_classifier = load_model(emotion_model_path, compile=False)
# getting input model shapes for inference
emotion_target_size = emotion_classifier.input_shape[1:3]
# starting lists for calculating modes
emotion_window = []
toc = time.time()
# starting video streaming
cv2.namedWindow('window_frame')
#video_capture = cv2.VideoCapture(sys.argv[1])
video_capture = cv2.VideoCapture(video_path)
#video_capture = cv2.VideoCapture('videoplayback.mp4')
while True:
bgr_image = video_capture.read()[1]
gray_image = cv2.cvtColor(bgr_image, cv2.COLOR_BGR2GRAY)
rgb_image = cv2.cvtColor(bgr_image, cv2.COLOR_BGR2RGB)
faces = detect_faces(face_detection, gray_image)
frame_count = int(video_capture.get(cv2.CAP_PROP_POS_FRAMES))
tic = time.time()
for face_coordinates in faces:
x1, x2, y1, y2 = apply_offsets(face_coordinates, emotion_offsets)
gray_face = gray_image[y1:y2, x1:x2]
try:
gray_face = cv2.resize(gray_face, (emotion_target_size))
except:
continue
actor_face = cv2.resize(gray_face, (128, 128))
cv2.imwrite(
"E:/tensorflow-master/tensorflow/examples/image_retraining/face.jpg",
actor_face)
video_capture.set(1, int(frame_count))
ret, frame = video_capture.read()
cv2.imwrite(
"E:/Object Detection/models-master/tutorials/image/imagenet/object.jpg",
gray_image)
gray_face = preprocess_input(gray_face, True)
gray_face = np.expand_dims(gray_face, 0)
gray_face = np.expand_dims(gray_face, -1)
emotion_prediction = emotion_classifier.predict(gray_face)
emotion_probability = np.max(emotion_prediction)
emotion_label_arg = np.argmax(emotion_prediction)
emotion_text = emotion_labels[emotion_label_arg]
emotion_window.append(emotion_text)
s2_out = subprocess.check_output([
sys.executable,
"E:/tensorflow-master/tensorflow/examples/label_image/label_image.py",
"--graph=E:/tmp/output_graph.pb",
"--labels=E:/tmp/output_labels.txt", "--input_layer=Mul",
"--output_layer=final_result", "--input_mean=128",
"--input_std=128",
"--image=E:/tensorflow-master/tensorflow/examples/image_retraining/face.jpg"
])
actor_confidence = s2_out.split()[1]
if (float(actor_confidence) > 0.5):
actor = s2_out.split()[0]
else:
actor = ""
print(s2_out)
s3_out = subprocess.check_output([
sys.executable,
"E:/Object Detection/models-master/tutorials/image/imagenet/classify_image.py",
"--image_file=E:/Object Detection/models-master/tutorials/image/imagenet/object.jpg"
])
object1 = s3_out.split()[0]
print(s3_out)
writer.writerows([[(tic - toc), frame_count, emotion_text,
emotion_probability, actor, actor_confidence,
face_coordinates, object1]])
if len(emotion_window) > frame_window:
emotion_window.pop(0)
try:
emotion_mode = mode(emotion_window)
except:
continue
if emotion_text == 'angry':
color = emotion_probability * np.asarray((255, 0, 0))
elif emotion_text == 'sad':
color = emotion_probability * np.asarray((0, 0, 255))
elif emotion_text == 'happy':
color = emotion_probability * np.asarray((255, 255, 0))
elif emotion_text == 'surprise':
color = emotion_probability * np.asarray((0, 255, 255))
else:
color = emotion_probability * np.asarray((0, 255, 0))
color = color.astype(int)
color = color.tolist()
draw_bounding_box(face_coordinates, rgb_image, color)
draw_text(face_coordinates, rgb_image, emotion_mode, color, 0, -20,
1, 1)
draw_text(face_coordinates, rgb_image, actor, color, 0, -45, 1, 1)
bgr_image = cv2.cvtColor(rgb_image, cv2.COLOR_RGB2BGR)
cv2.imshow('window_frame', bgr_image)
if cv2.waitKey(1) & 0xFF == ord('q'):
break
image = cv2.imread('image.PNG')
video_capture = cv2.VideoCapture(0)
video_capture.set(4, 150)
video_capture.set(3, 150)
while video_capture.isOpened():
reopenWindow = 0
_, bgr_image = video_capture.read()
gray_image = cv2.cvtColor(bgr_image, cv2.COLOR_BGR2GRAY)
rgb_image = cv2.cvtColor(bgr_image, cv2.COLOR_BGR2RGB)
faces = detect_faces(face_detection, gray_image)
if len(faces):
x1, x2, y1, y2 = apply_offsets(faces[0], emotion_offsets)
gray_face = gray_image[y1:y2, x1:x2]
try:
gray_face = cv2.resize(gray_face, (emotion_target_size))
except:
continue
gray_face = preprocess_input(gray_face, True)
gray_face = np.expand_dims(gray_face, 0)
gray_face = np.expand_dims(gray_face, -1)
emotion_prediction = emotion_classifier.predict(gray_face)
emotion_probability = np.max(emotion_prediction)
emotion_label_arg = np.argmax(emotion_prediction)
emotion_text = emotion_labels[emotion_label_arg]
emotion_window.append(emotion_text)
def main_thread(self):
if self.data_bridge.processing_chosen_by_radio_butten == 'img':
flag=0
detection_model_path = '../trained_models/detection_models/haarcascade_frontalface_default.xml'
emotion_model_path = '../trained_models/emotion_models/fer2013_mini_XCEPTION.102-0.66.hdf5'
gender_model_path = '../trained_models/gender_models/simple_CNN.81-0.96.hdf5'
emotion_labels = get_labels('fer2013')
gender_labels = get_labels('imdb')
face_detection = load_detection_model(detection_model_path)
emotion_classifier = load_model(emotion_model_path, compile=False)
gender_classifier = load_model(gender_model_path, compile=False)
emotion_target_size = emotion_classifier.input_shape[1:3]
gender_target_size = gender_classifier.input_shape[1:3]
while self.data_bridge.start_process_manager and flag==0:
flag=1
image_path = self.data_bridge.selected_video_file_path
font = cv2.FONT_HERSHEY_SIMPLEX
# hyper-parameters for bounding boxes shape
gender_offsets = (30, 60)
gender_offsets = (10, 10)
emotion_offsets = (20, 40)
emotion_offsets = (0, 0)
rgb_image = load_image(image_path, grayscale=False)
gray_image = load_image(image_path, grayscale=True)
gray_image = np.squeeze(gray_image)
gray_image = gray_image.astype('uint8')
faces = detect_faces(face_detection, gray_image)
for face_coordinates in faces:
x1, x2, y1, y2 = apply_offsets(face_coordinates, gender_offsets)
rgb_face = rgb_image[y1:y2, x1:x2]
x1, x2, y1, y2 = apply_offsets(face_coordinates, emotion_offsets)
gray_face = gray_image[y1:y2, x1:x2]
try:
rgb_face = cv2.resize(rgb_face, (gender_target_size))
gray_face = cv2.resize(gray_face, (emotion_target_size))
except:
continue
rgb_face = preprocess_input(rgb_face, False)
rgb_face = np.expand_dims(rgb_face, 0)
gender_prediction = gender_classifier.predict(rgb_face)
gender_label_arg = np.argmax(gender_prediction)
gender_text = gender_labels[gender_label_arg]
gray_face = preprocess_input(gray_face, True)
gray_face = np.expand_dims(gray_face, 0)
gray_face = np.expand_dims(gray_face, -1)
emotion_label_arg = np.argmax(emotion_classifier.predict(gray_face))
emotion_text = emotion_labels[emotion_label_arg]
if gender_text == gender_labels[0]:
color = (0, 0, 255)
else:
color = (255, 0, 0)
draw_bounding_box(face_coordinates, rgb_image, color)
draw_text(face_coordinates, rgb_image, gender_text, color, 0, -20, 1, 2)
draw_text(face_coordinates, rgb_image, emotion_text, color, 0, -50, 1, 2)
bgr_image = cv2.cvtColor(rgb_image, cv2.COLOR_RGB2BGR)
cv2.imwrite('../images/predicted_test_image.png', bgr_image)
print("File has been stored in Images folder")
print("Press stop processing to exit")
self.gui_root.update()
if( (self.data_bridge.processing_chosen_by_radio_butten == 'vid') or (self.data_bridge.processing_chosen_by_radio_butten=='web')):
detection_model_path = '../trained_models/detection_models/haarcascade_frontalface_default.xml'
emotion_model_path = '../trained_models/emotion_models/fer2013_mini_XCEPTION.102-0.66.hdf5'
gender_model_path = '../trained_models/gender_models/simple_CNN.81-0.96.hdf5'
emotion_labels = get_labels('fer2013')
gender_labels = get_labels('imdb')
# Models
face_detection = load_detection_model(detection_model_path)
emotion_classifier = load_model(emotion_model_path, compile=False)
gender_classifier = load_model(gender_model_path, compile=False)
emotion_target_size = emotion_classifier.input_shape[1:3]
gender_target_size = gender_classifier.input_shape[1:3]
while self.data_bridge.start_process_manager:
font = cv2.FONT_HERSHEY_SIMPLEX
frame_window = 10
gender_offsets = (30, 60)
emotion_offsets = (20, 40)
gender_window = []
emotion_window = []
# starting video streaming
cv2.namedWindow('Window_frame')
if self.data_bridge.processing_chosen_by_radio_butten=='vid':
self.cap=cv2.VideoCapture(self.data_bridge.selected_video_file_path)
else:
self.cap = cv2.VideoCapture(0)
fourcc = cv2.VideoWriter_fourcc(*'XVID')
out = cv2.VideoWriter('Save.avi', fourcc, 20.0, (720, 480))
while self.data_bridge.start_process_manager:
ret, bgr_image = self.cap.read()
gray_image = cv2.cvtColor(bgr_image, cv2.COLOR_BGR2GRAY)
rgb_image = cv2.cvtColor(bgr_image, cv2.COLOR_BGR2RGB)
faces = detect_faces(face_detection, gray_image)
for face_coordinates in faces:
x1, x2, y1, y2 = apply_offsets(face_coordinates, gender_offsets)
rgb_face = rgb_image[y1:y2, x1:x2]
x1, x2, y1, y2 = apply_offsets(face_coordinates, emotion_offsets)
gray_face = gray_image[y1:y2, x1:x2]
try:
rgb_face = cv2.resize(rgb_face, (gender_target_size))
gray_face = cv2.resize(gray_face, (emotion_target_size))
except:
continue
gray_face = preprocess_input(gray_face, False)
gray_face = np.expand_dims(gray_face, 0)
gray_face = np.expand_dims(gray_face, -1)
emotion_label_arg = np.argmax(emotion_classifier.predict(gray_face))
emotion_text = emotion_labels[emotion_label_arg]
emotion_window.append(emotion_text)
rgb_face = np.expand_dims(rgb_face, 0)
rgb_face = preprocess_input(rgb_face, False)
gender_prediction = gender_classifier.predict(rgb_face)
gender_label_arg = np.argmax(gender_prediction)
gender_text = gender_labels[gender_label_arg]
gender_window.append(gender_text)
if len(gender_window) > frame_window:
emotion_window.pop(0)
gender_window.pop(0)
try:
emotion_mode = mode(emotion_window)
gender_mode = mode(gender_window)
except:
continue
if gender_text == gender_labels[0]:
color = (0, 0, 0)
else:
color = (0, 0, 0)
draw_bounding_box(face_coordinates, rgb_image, color)
draw_text(face_coordinates, rgb_image, gender_mode,
color, 0, -20, 1, 1)
draw_text(face_coordinates, rgb_image, emotion_mode,
color, 0, -45, 1, 1)
bgr_image = cv2.cvtColor(rgb_image, cv2.COLOR_RGB2BGR)
cv2.imshow('Window_frame', bgr_image)
self.gui_root.update()
self.cap.release()
cv2.destroyAllWindows()
def job1():
global count1
global frame
use_gpu = True
# 显存分配。
if use_gpu:
os.environ["CUDA_VISIBLE_DEVICES"] = "0"
else:
os.environ["CUDA_VISIBLE_DEVICES"] = "-1"
from keras.backend.tensorflow_backend import set_session
config = tf.ConfigProto()
config.gpu_options.per_process_gpu_memory_fraction = 0.8
set_session(tf.Session(config=config))
facesDetect = {'classes_path': 'data/voc_classes.txt', 'model_path': './weights/best_model.h5'}
emotion_model_path = 'model/fer2013_mini_XCEPTION.102-0.66.hdf5'
emotion_labels = get_labels('fer2013')
# hyper-parameters for bounding boxes shape
frame_window = 10
emotion_offsets = (20, 40)
# loading models
face_detector = Decode(facesDetect['classes_path'], facesDetect['model_path'])
emotion_classifier = load_model(emotion_model_path, compile=False)
# getting input model shapes for inference
emotion_target_size = emotion_classifier.input_shape[1:3]
emotion_window1 = []
record1 = {'angry':[0], 'disgust':[0], 'fear':[0], 'happy':[0], 'sad':[0], 'surprise':[0], 'neutral':[0]}
#record_diff = {'angry':[], 'disgust':[], 'fear':[], 'happy':[], 'sad':[], 'surprise':[], 'neutral':[]}'''
emo_record1 = []
capture = cv2.VideoCapture(0)
while True:
if True:
bgr_image = capture.read()[1]
gray_image = cv2.cvtColor(bgr_image, cv2.COLOR_BGR2GRAY)
rgb_image = cv2.cvtColor(bgr_image, cv2.COLOR_BGR2RGB)
faces = face_detector.detect_image(bgr_image)[1]
if faces is None:
faces = ()
# print(faces)
for face_coordinates in faces:
x1, y1, x2, y2 = face_coordinates
face_coordinates = [int(x1), int(y1), int(x2-x1), int(y2-y1)]
x1, x2, y1, y2 = apply_offsets(face_coordinates, emotion_offsets)
gray_face = gray_image[y1:y2, x1:x2]
try:
gray_face = cv2.resize(gray_face, (emotion_target_size))
except:
continue
gray_face = preprocess_input(gray_face, True)
gray_face = np.expand_dims(gray_face, 0)
gray_face = np.expand_dims(gray_face, -1)
emotion_prediction = emotion_classifier.predict(gray_face)
###############移動平均公式################
'''for idx, probability in enumerate(emotion_prediction[0]):
alpha = 0.5
record[emotion_labels[idx]].append(record[emotion_labels[idx]][-1] + alpha * (round(probability*100, 2)-record[emotion_labels[idx]][-1]))
emotion_prediction[0][idx] = record[emotion_labels[idx]][-1]
if len(record[emotion_labels[idx]])>10:
record[emotion_labels[idx]].pop(0)
#print(record)
#print()'''
#########################################
#################自創權重##############
emotion_prediction[0] = weights_change(emo_record1, emotion_prediction[0])
data=[]
for idx, probability in enumerate(emotion_prediction[0]):
data.append((emotion_labels[idx], probability))
rd1.append(data)
count1+=1
emo_record1.append(np.argmax(emotion_prediction))
if len(emo_record1)>10:
emo_record1.pop(0)
#######################################
emotion_probability = np.max(emotion_prediction)
emotion_label_arg = np.argmax(emotion_prediction)
emotion_text = emotion_labels[emotion_label_arg]
emotion_window1.append(emotion_text)
if len(emotion_window1) > frame_window:
emotion_window1.pop(0)
try:
emotion_mode1 = mode(emotion_window1)
except:
continue
if emotion_text == 'angry':
color = emotion_probability * np.asarray((255, 0, 0))
elif emotion_text == 'sad':
color = emotion_probability * np.asarray((0, 0, 255))
elif emotion_text == 'happy':
color = emotion_probability * np.asarray((255, 255, 0))
elif emotion_text == 'surprise':
color = emotion_probability * np.asarray((0, 255, 255))
else:
color = emotion_probability * np.asarray((0, 255, 0))
color = color.astype(int)
color = color.tolist()
draw_bounding_box(face_coordinates, rgb_image, color)
draw_text(face_coordinates, rgb_image, emotion_mode1,
color, 0, -45, 1, 1)
bgr_image = cv2.cvtColor(rgb_image, cv2.COLOR_RGB2BGR)
cv2.imshow('window_frame1', bgr_image)
if cv2.waitKey(1) & 0xFF == 27:
break
cv2.destroyAllWindows()
# starting video streaming
cv2.namedWindow('window_frame')
video_capture = cv2.VideoCapture(
0) #r"F:\movies\hollywood\videoplayback_3.mp4")
while True:
bgr_image = video_capture.read()[1]
gray_image = cv2.cvtColor(bgr_image, cv2.COLOR_BGR2GRAY)
rgb_image = cv2.cvtColor(bgr_image, cv2.COLOR_BGR2RGB)
faces = detect_faces(face_detection, gray_image)
# plt.imshow(gray_image)
# plt.show()
count = 0
for face_coordinates in faces:
x1, x2, y1, y2 = apply_offsets(face_coordinates, (20, 40))
gray_face = gray_image[y1:y2, x1:x2]
scale = np.array([x2 - x1, y2 - y1]) / 96
try:
gray_face = cv2.resize(gray_face, keypoint_target_size)
except:
continue
#gray_face = preprocess_input(gray_face, True)
gray_face = np.expand_dims(gray_face, 0)
gray_face = np.expand_dims(gray_face, -1)
keypoints = keypoint_detector.predict(gray_face)
# cv2.imshow("face:-{0}".format(count),face_pt_plotter(np.resize(gray_face,[96,96]),keypoints[0],0,0,1,[1,1]))
# count=count+1
gray_image = face_pt_plotter(gray_image, keypoints[0], x1, y1, 2,
gender_window = []
cv2.namedWindow('window_frame')
video_capture = cv2.VideoCapture('x.MP4')
fourcc = cv2.VideoWriter_fourcc(*'XVID')
out = cv2.VideoWriter('Save.avi', fourcc, 25.0, (1280, 720))
while True:
bgr_image = video_capture.read()[1]
rgb_image = cv2.cvtColor(bgr_image, cv2.COLOR_BGR2RGB)
faces = detect_faces(face_detection, bgr_image) #gray_image)
for face_coordinates in faces:
x1, x2, y1, y2 = apply_offsets(face_coordinates, gender_offsets)
rgb_face = rgb_image[y1:y2, x1:x2]
try:
rgb_face = cv2.resize(rgb_face, (gender_target_size))
except:
continue
rgb_face = np.expand_dims(rgb_face, 0)
rgb_face = preprocess_input(rgb_face, False)
gender_prediction = gender_classifier.predict(rgb_face)
gender_label_arg = np.argmax(gender_prediction)
gender_text = gender_labels[gender_label_arg]
gender_window.append(gender_text)
if len(gender_window) > frame_window:
gender_window.pop(0)
def main(argv):
# parameters for loading data and images
detection_model_path = '../trained_models/detection_models/haarcascade_frontalface_default.xml'
emotion_model_path = '../trained_models/emotion_models/fer2013_mini_XCEPTION.102-0.66.hdf5'
gender_model_path = '../trained_models/gender_models/simple_CNN.81-0.96.hdf5'
emotion_labels = get_labels('fer2013')
gender_labels = get_labels('imdb')
font = cv2.FONT_HERSHEY_SIMPLEX
# hyper-parameters for bounding boxes shape
frame_window = 10
gender_offsets = (30, 60)
emotion_offsets = (20, 40)
# loading models
face_detection = load_detection_model(detection_model_path)
emotion_classifier = load_model(emotion_model_path, compile=False)
gender_classifier = load_model(gender_model_path, compile=False)
# getting input model shapes for inference
emotion_target_size = emotion_classifier.input_shape[1:3]
gender_target_size = gender_classifier.input_shape[1:3]
# starting lists for calculating modes
gender_window = []
emotion_window = []
# starting video streaming
cv2.namedWindow('window_frame')
video_capture = cv2.VideoCapture(0)
try:
opts, args = getopt.getopt(argv, "hs:", ["subject="])
except getopt.GetoptError:
print("emotion.py -s <subject>")
sys.exit(2)
subject = Subject()
for opt, arg in opts:
if opt == '-h':
print("emotion.py - s <subject>")
sys.exit()
elif opt in ("-s", "--subject"):
subject.name = arg
print(subject.name)
subject.start()
while True:
bgr_image = video_capture.read()[1]
gray_image = cv2.cvtColor(bgr_image, cv2.COLOR_BGR2GRAY)
rgb_image = cv2.cvtColor(bgr_image, cv2.COLOR_BGR2RGB)
faces = detect_faces(face_detection, gray_image)
for face_coordinates in faces:
x1, x2, y1, y2 = apply_offsets(face_coordinates, gender_offsets)
rgb_face = rgb_image[y1:y2, x1:x2]
x1, x2, y1, y2 = apply_offsets(face_coordinates, emotion_offsets)
gray_face = gray_image[y1:y2, x1:x2]
try:
rgb_face = cv2.resize(rgb_face, (gender_target_size))
gray_face = cv2.resize(gray_face, (emotion_target_size))
except:
continue
gray_face = preprocess_input(gray_face, False)
gray_face = np.expand_dims(gray_face, 0)
gray_face = np.expand_dims(gray_face, -1)
emotion_label_arg = np.argmax(
emotion_classifier.predict(gray_face))
emotion_text = emotion_labels[emotion_label_arg]
emotion_window.append(emotion_text)
rgb_face = np.expand_dims(rgb_face, 0)
rgb_face = preprocess_input(rgb_face, False)
gender_prediction = gender_classifier.predict(rgb_face)
gender_label_arg = np.argmax(gender_prediction)
gender_text = gender_labels[gender_label_arg]
gender_window.append(gender_text)
subject.addMood(emotion_text, gender_text)
if len(gender_window) > frame_window:
emotion_window.pop(0)
gender_window.pop(0)
try:
emotion_mode = mode(emotion_window)
gender_mode = mode(gender_window)
except:
continue
if gender_text == gender_labels[0]:
color = (0, 0, 255)
else:
color = (255, 0, 0)
draw_bounding_box(face_coordinates, rgb_image, color)
draw_text(face_coordinates, rgb_image, gender_mode, color, 0, -20,
1, 1)
draw_text(face_coordinates, rgb_image, emotion_mode, color, 0, -45,
1, 1)
bgr_image = cv2.cvtColor(rgb_image, cv2.COLOR_RGB2BGR)
cv2.imshow('window_frame', bgr_image)
if cv2.waitKey(1) & 0xFF == ord('q'):
break
subject.stop()
def show_face_information(self):
# bgr_image = img
gray_image = cv2.cvtColor(self.bgr_image, cv2.COLOR_BGR2GRAY)
rgb_image = cv2.cvtColor(self.bgr_image, cv2.COLOR_BGR2RGB)
faces = detect_faces(face_detection, gray_image)
img_h, img_w, _ = np.shape(rgb_image)
for face_coordinates in faces:
x1, x2, y1, y2 = apply_offsets(face_coordinates, gender_offsets)
rgb_face = rgb_image[y1:y2, x1:x2]
x1, x2, y1, y2 = apply_offsets(face_coordinates, emotion_offsets)
gray_face = gray_image[y1:y2, x1:x2]
try:
rgb_face = cv2.resize(rgb_face, (gender_target_size))
gray_face = cv2.resize(gray_face, (emotion_target_size))
except:
continue
# run_thread(bgr_image)
gray_face = preprocess_input(gray_face, False)
gray_face = np.expand_dims(gray_face, 0)
gray_face = np.expand_dims(gray_face, -1)
emotion_label_arg = np.argmax(
emotion_classifier.predict(gray_face))
emotion_text = emotion_labels[emotion_label_arg]
emotion_window.append(emotion_text)
# emotion_window.append(English_2_chinese_emotion(emotion_text))
rgb_face = np.expand_dims(rgb_face, 0)
rgb_face = preprocess_input(rgb_face, False)
gender_prediction = gender_classifier.predict(rgb_face)
gender_label_arg = np.argmax(gender_prediction)
gender_text = gender_labels[gender_label_arg]
# gender_window.append(English_2_chinese_gender(gender_text))
set_icon = emotion_text + "_" + gender_text
print(set_icon)
icon_img = self.icon_dict[set_icon]
words_img = self.words_dict[set_icon]
# if len(gender_window) > frame_window:
# emotion_window.pop(0)
# gender_window.pop(0)
# try:
# emotion_mode = mode(emotion_window)
# gender_mode = mode(gender_window)
# except:
# continue
if gender_text == gender_labels[0]:
color = (0, 0, 255)
else:
color = (255, 0, 0)
###################
if (self.frq % 60 == 0):
# detect faces using dlib detector
detected = self.detector(rgb_image, 1)
print(detected)
faces_age = np.empty((len(detected), img_size, img_size, 3))
if len(detected) > 0:
for i, d in enumerate(detected):
x1, y1, x2, y2, w, h = d.left(), d.top(), d.right(
) + 1, d.bottom() + 1, d.width(), d.height()
xw1 = max(int(x1 - self.margin * w), 0)
yw1 = max(int(y1 - self.margin * h), 0)
xw2 = min(int(x2 + self.margin * w), img_w - 1)
yw2 = min(int(y2 + self.margin * h), img_h - 1)
cv2.rectangle(img, (x1, y1), (x2, y2), (255, 0, 0), 2)
# cv2.rectangle(img, (xw1, yw1), (xw2, yw2), (255, 0, 0), 2)
faces_age[i, :, :, :] = cv2.resize(
img[yw1:yw2 + 1, xw1:xw2 + 1, :],
(img_size, img_size))
# predict ages and genders of the detected faces
results = self.model.predict(faces_age)
ages = np.arange(0, 101).reshape(101, 1)
predicted_ages = results[1].dot(ages).flatten()
print(predicted_ages)
###################
self.frq += 1
if ((face_coordinates[0] - face_coordinates[2]) > 50
and (face_coordinates[0] - face_coordinates[2]) < 180
and (face_coordinates[1] - 80) > 20):
solid_box = draw_solid_box(face_coordinates, rgb_image)
draw_bounding_box(face_coordinates, rgb_image, color)
solid_box = Addemotion(face_coordinates, solid_box, icon_img)
solid_box = Addemotion_word(face_coordinates, solid_box,
words_img)
draw_text(face_coordinates, rgb_image,
str(int(predicted_ages)), (255, 255, 255), 0, -20, 1,
1)
return rgb_image
for (top, right, bottom, left) in boxes:
# rescale the face coordinates
rect = [top, right, bottom, left]
top = int(top * r)
right = int(right * r)
bottom = int(bottom * r)
left = int(left * r)
# top = top*4
# right = right*4
# bottom = bottom*4
# left = left*4
y = top - 2 if top - 15 > 15 else top + 15
##emotion
x1, x2, y1, y2 = apply_offsets(rect, gender_offsets)
rgb_face = rgb_image[y1:y2, x1:x2]
x1, x2, y1, y2 = apply_offsets(rect, emotion_offsets)
gray_face = grayframe[y1:y2, x1:x2]
try:
rgb_face = cv2.resize(rgb_face, (gender_target_size))
gray_face = cv2.resize(gray_face, (emotion_target_size))
gray_face = preprocess_input(gray_face, False)
gray_face = np.expand_dims(gray_face, 0)
gray_face = np.expand_dims(gray_face, -1)
emotion_label_arg = np.argmax(
emotion_classifier.predict(gray_face))
emotion_max = np.max(emotion_classifier.predict(gray_face))
emotion_text = emotion_labels[emotion_label_arg]
#emotion_window.append(emotion_text)
def detection_FER():
# parameters for loading data and images
detection_model_path = '../FER_CNN/models/haarcascade_frontalface_default.xml'
emotion_model_path = '../FER_CNN/models/cnn_model.hdf5'
emotion_labels = {
0: 'Angry',
1: 'Disgust',
2: 'Fear',
3: 'Happy',
4: 'Sad',
5: 'Surprise',
6: 'Neutral'
}
# hyper-parameters for bounding boxes shape
frame_window = 10
emotion_offsets = (20, 40)
# loading models
face_detection = load_detection_model(detection_model_path)
emotion_classifier = load_model(emotion_model_path, compile=False)
# getting input model shapes for inference
emotion_target_size = emotion_classifier.input_shape[1:3]
# starting lists for calculating modes
emotion_window = []
# starting video streaming
cv2.namedWindow('Face Expression Recognition')
video_capture = cv2.VideoCapture(0)
while True:
bgr_image = video_capture.read()[1]
gray_image = cv2.cvtColor(bgr_image, cv2.COLOR_BGR2GRAY)
rgb_image = cv2.cvtColor(bgr_image, cv2.COLOR_BGR2RGB)
faces = detect_faces(face_detection, gray_image)
for face_coordinates in faces:
x1, x2, y1, y2 = apply_offsets(face_coordinates, emotion_offsets)
gray_face = gray_image[y1:y2, x1:x2]
try:
gray_face = cv2.resize(gray_face, (emotion_target_size))
except:
continue
gray_face = preprocess_input(gray_face, True)
gray_face = np.expand_dims(gray_face, 0)
gray_face = np.expand_dims(gray_face, -1)
emotion_prediction = emotion_classifier.predict(gray_face)
emotion_probability = np.max(emotion_prediction)
emotion_label_arg = np.argmax(emotion_prediction)
emotion_text = emotion_labels[emotion_label_arg]
emotion_window.append(emotion_text)
if len(emotion_window) > frame_window:
emotion_window.pop(0)
try:
emotion_mode = mode(emotion_window)
except:
continue
if emotion_text == 'Angry':
color = emotion_probability * np.asarray((255, 0, 0))
elif emotion_text == 'Sad':
color = emotion_probability * np.asarray((0, 0, 255))
elif emotion_text == 'Happy':
color = emotion_probability * np.asarray((255, 255, 0))
elif emotion_text == 'Surprise':
color = emotion_probability * np.asarray((0, 255, 255))
elif emotion_text == 'Neutral':
color = emotion_probability * np.asarray((0, 255, 255))
elif emotion_text == 'Fear':
color = emotion_probability * np.asarray((0, 255, 255))
else:
color = emotion_probability * np.asarray((0, 255, 0))
color = color.astype(int)
color = color.tolist()
draw_bounding_box(face_coordinates, rgb_image, color)
draw_text(face_coordinates, rgb_image, emotion_mode, color, 30,
290, 1, 1)
bgr_image = cv2.cvtColor(rgb_image, cv2.COLOR_RGB2BGR)
cv2.imshow('Face Expression Recognition', bgr_image)
if cv2.waitKey(1) & 0xFF == ord('q'):
break
# if __name__=="__main__":
# detection_FER()
gray_image = cv2.cvtColor(bgr_image,
cv2.COLOR_BGR2GRAY) # gray image for HOGs
rgb_image = cv2.cvtColor(bgr_image,
cv2.COLOR_BGR2RGB) # color image for video frame
faces = face_cascade.detectMultiScale(
gray_image,
scaleFactor=1.1,
minNeighbors=5, # image scaling
minSize=(30, 30),
flags=cv2.CASCADE_SCALE_IMAGE)
for face_coordinates in faces:
x1, x2, y1, y2 = apply_offsets(
face_coordinates,
emotion_offsets) # extracting faces from gray image
gray_face = gray_image[y1:y2, x1:x2] # storing faces into gray_face
try:
gray_face = cv2.resize(
gray_face,
(emotion_target_size)) # resizing for emotion detection
except:
continue
gray_face = preprocess_input(
gray_face, True) #converting image into a float 32bit Array
gray_face = np.expand_dims(
gray_face, 0) #adding 0 axis to the facial float 32-bit array
gray_face = np.expand_dims(
def callback(self,data):
#/////////////////////////////////////////////
#cv_image = bridge.imgmsg_to_cv2(image_msg, desired_encoding="passthrough")
#cv_image = cv2.resize(cv_image, target_size) # resize image
#np_image = np.asarray(cv_image) # read as np array
#np_image = np.expand_dims(np_image, axis=0) # Add another dimension for tensorflow
#np_image = np_image.astype(float) # preprocess needs float64 and img is uint8
#np_image = preprocess_input(np_image) # Regularize the data
#/////////////////////////////////////////////
if(not USE_LOCAL_CAMERA):
try:
frame = self.bridge.imgmsg_to_cv2(data, desired_encoding="passthrough")
except CvBridgeError as e:
print(e)
# Capture frame-by-frame
if(USE_LOCAL_CAMERA):
ret, frame1 = self.video_capture.read()
#/////////////////////////////////////////////
#print"--------"
#print('input_msg height : {}'.format(frame.height))
#print('input_msg width : {}'.format(frame.width))
#print('input_msg step : {}'.format(frame.step))
#print('input_msg encoding: {}'.format(frame.encoding))
#print('output dtype : {}'.format(frame.dtype))
#print('output shape : {}'.format(frame.shape))
#print"--------"
gray_image = cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY)
rgb_image = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)
faces = detect_faces(self.face_detection, gray_image)
for face_coordinates in faces:
print "inside the for loop!"
print face_coordinates
x1, x2, y1, y2 = apply_offsets(face_coordinates, self.emotion_offsets)
gray_face = gray_image[y1:y2, x1:x2]
#print len(frame)
#print gray_face.size
#print gray_face.shape
#print gray_face.dtype
try:
gray_face = cv2.resize(gray_face, (self.emotion_target_size))
except:
continue
gray_face = preprocess_input(gray_face, True)
gray_face = np.expand_dims(gray_face, 0)
gray_face = np.expand_dims(gray_face, -1)
#print"************"
#print('gray_face dtype : {}'.format(gray_face.dtype))
#print('gray_face shape : {}'.format(gray_face.shape))
#print('gray_face size : {}'.format(gray_face.size))
#print"************"
## This is a workaround for asynchronous execution using TF and ROS
# https://github.com/keras-team/keras/issues/2397
# http://projectsfromtech.blogspot.com/2017/10/visual-object-recognition-in-ros-using.html
#global self.graph
with self.graph.as_default():
emotion_prediction = self.emotion_classifier.predict(gray_face)
emotion_probability = np.max(emotion_prediction)
emotion_label_arg = np.argmax(emotion_prediction)
emotion_text = self.emotion_labels[emotion_label_arg]
print emotion_text
print(emotion_probability)
print('%')
self.emotion_window.append(emotion_text)
#self.emotion_msg.data = emotion_text
#self.emotion_publisher.publish(emotion_msg)
#self.speech_msg.data = 'I see that you are ' + emotion_text
#self.speech_publisher.publish(speech_msg)
if len(self.emotion_window) > self.frame_window:
self.emotion_window.pop(0)
try:
emotion_mode = mode(self.emotion_window)
except:
continue
if emotion_text == 'angry':
color = emotion_probability * np.asarray((255, 0, 0))
elif emotion_text == 'sad':
color = emotion_probability * np.asarray((0, 0, 255))
elif emotion_text == 'happy':
color = emotion_probability * np.asarray((255, 255, 0))
elif emotion_text == 'surprise':
color = emotion_probability * np.asarray((0, 255, 255))
else:
color = emotion_probability * np.asarray((0, 255, 0))
color = color.astype(int)
color = color.tolist()
draw_bounding_box(face_coordinates, rgb_image, color)
draw_text(face_coordinates, rgb_image, emotion_mode,
color, 0, -45, 1, 1)
try:
self.image_pub.publish(self.bridge.cv2_to_imgmsg(rgb_image, "bgr8"))
except CvBridgeError as e:
print(e)
def get_frame(self):
global out
if self.video.isOpened():
ret, image = self.video.read()
if ret:
# Write the frame into the file 'output.avi'
if is_recording:
out.write(image)
# Default resolutions of the frame are obtained.The default resolutions are system dependent.
# We convert the resolutions from float to integer.
gray = cv2.cvtColor(image, cv2.COLOR_BGR2GRAY)
rgb_image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)
# determine the facial landmarks for the face region, then
# convert the facial landmark (x, y)-coordinates to a NumPy
# array
faces = face_cascade.detectMultiScale(gray, scaleFactor=1.1, minNeighbors=5, minSize=(30, 30),
flags=cv2.CASCADE_SCALE_IMAGE)
emotion_text = 'NEUTRAL'
for face_coordinates in faces:
x1, x2, y1, y2 = apply_offsets(
face_coordinates, emotion_offsets)
gray_face = gray[y1:y2, x1:x2]
try:
gray_face = cv2.resize(gray_face, self.emotion_target_size)
except:
continue
gray_face = preprocess_input(gray_face, True)
gray_face = np.expand_dims(gray_face, 0)
gray_face = np.expand_dims(gray_face, -1)
with self.graph.as_default():
emotion_prediction = self.emotion_classifier.predict(
gray_face)
emotion_probability = np.max(emotion_prediction)
emotion_label_arg = np.argmax(emotion_prediction)
emotion_text = emotion_labels[emotion_label_arg]
emotion_window.append(emotion_text)
if len(emotion_window) > frame_window:
emotion_window.pop(0)
try:
emotion_mode = mode(emotion_window)
except:
continue
if emotion_text == "ANGRY":
color = emotion_probability * np.asarray((255, 0, 0))
elif emotion_text == "SAD":
color = emotion_probability * np.asarray((0, 0, 255))
elif emotion_text == "HAPPY":
color = emotion_probability * np.asarray((255, 255, 0))
elif emotion_text == "SURPRISE":
color = emotion_probability * np.asarray((0, 255, 255))
else:
color = emotion_probability * np.asarray((0, 255, 0))
App.setEmotionResults(self, emotion_text, emotion_probability)
color = color.astype(int)
color = color.tolist()
draw_bounding_box(face_coordinates, rgb_image, color)
draw_text(face_coordinates, rgb_image, emotion_mode,
color, 0, -45, 1, 1)
bgr_image = cv2.cvtColor(rgb_image, cv2.COLOR_RGB2BGR)
data_emotion = np.concatenate((image, bgr_image), axis=1)
if ret:
return (ret, cv2.cvtColor(bgr_image, cv2.COLOR_BGR2RGB))
else:
return (ret, None)
else:
return (ret, None)
else:
cap = cv2.VideoCapture('./test/testvdo.mp4') # Video file source
while cap.isOpened(): # True:
ret, bgr_image = cap.read()
#bgr_image = video_capture.read()[1]
gray_image = cv2.cvtColor(bgr_image, cv2.COLOR_BGR2GRAY)
rgb_image = cv2.cvtColor(bgr_image, cv2.COLOR_BGR2RGB)
faces = detector(rgb_image)
for face_coordinates in faces:
x1, x2, y1, y2 = apply_offsets(face_utils.rect_to_bb(face_coordinates), emotion_offsets)
gray_face = gray_image[y1:y2, x1:x2]
try:
gray_face = cv2.resize(gray_face, (emotion_target_size))
except:
continue
gray_face = preprocess_input(gray_face, True)
gray_face = np.expand_dims(gray_face, 0)
gray_face = np.expand_dims(gray_face, -1)
emotion_prediction = emotion_classifier.predict(gray_face)
emotion_probability = np.max(emotion_prediction)
emotion_label_arg = np.argmax(emotion_prediction)
emotion_text = emotion_labels[emotion_label_arg]
emotion_window.append(emotion_text)
def process_image(image):
with graph.as_default():
# Raw data
#arr = []
try:
# loading images
image_array = np.fromstring(image, np.uint8)
unchanged_image = cv2.imdecode(image_array, cv2.IMREAD_UNCHANGED)
#rgb_image = cv2.cvtColor(unchanged_image, cv2.COLOR_BGR2RGB)
gray_image = cv2.cvtColor(unchanged_image, cv2.COLOR_BGR2GRAY)
faces = detect_faces(face_detection, gray_image)
for face_coordinates in faces:
#x1, x2, y1, y2 = apply_offsets(face_coordinates, gender_offsets)
#rgb_face = rgb_image[y1:y2, x1:x2]
x1, x2, y1, y2 = apply_offsets(face_coordinates,
emotion_offsets)
gray_face = gray_image[y1:y2, x1:x2]
try:
#rgb_face = cv2.resize(rgb_face, (gender_target_size))
gray_face = cv2.resize(gray_face, (emotion_target_size))
except:
continue
#rgb_face = preprocess_input(rgb_face, False)
#rgb_face = np.expand_dims(rgb_face, 0)
#gender_prediction = gender_classifier.predict(rgb_face)
#gender_label_arg = np.argmax(gender_prediction)
#gender_text = gender_labels[gender_label_arg]
gray_face = preprocess_input(gray_face, True)
gray_face = np.expand_dims(gray_face, 0)
gray_face = np.expand_dims(gray_face, -1)
# Return raw data
#arr.append(gender_prediction)
#return emotion_classifier.predict(gray_face)
return emotion_labels[np.argmax(
emotion_classifier.predict(gray_face))]
continue
emotion_label_arg = np.argmax(
emotion_classifier.predict(gray_face))
emotion_text = emotion_labels[emotion_label_arg]
if gender_text == gender_labels[0]:
color = (0, 0, 255)
else:
color = (255, 0, 0)
draw_bounding_box(face_coordinates, rgb_image, color)
draw_text(face_coordinates, rgb_image, gender_text, color, 0,
-20, 1, 2)
draw_text(face_coordinates, rgb_image, emotion_text, color, 0,
-50, 1, 2)
except Exception as err:
logging.error(
'Error in emotion gender processor: "{0}"'.format(err))
# Return array of raw data
#return arr
return 'no face'
bgr_image = cv2.cvtColor(rgb_image, cv2.COLOR_RGB2BGR)
dirname = 'result'
if not os.path.exists(dirname):
os.mkdir(dirname)
cv2.imwrite(os.path.join(dirname, 'predicted_image.png'), bgr_image)
def generateResults(rgb_image, runNumber):
# parameters for loading data and images
#image_path = path
detection_model_path = '../trained_models/detection_models/haarcascade_frontalface_default.xml'
emotion_model_path = '../trained_models/emotion_models/fer2013_mini_XCEPTION.102-0.66.hdf5'
gender_model_path = '../trained_models/gender_models/simple_CNN.81-0.96.hdf5'
emotion_labels = get_labels('fer2013')
gender_labels = get_labels('imdb')
font = cv2.FONT_HERSHEY_SIMPLEX
# hyper-parameters for bounding boxes shape
gender_offsets = (30, 60)
gender_offsets = (10, 10)
emotion_offsets = (20, 40)
emotion_offsets = (0, 0)
# loading models
face_detection = load_detection_model(detection_model_path)
emotion_classifier = load_model(emotion_model_path, compile=False)
gender_classifier = load_model(gender_model_path, compile=False)
# getting input model shapes for inference
emotion_target_size = emotion_classifier.input_shape[1:3]
gender_target_size = gender_classifier.input_shape[1:3]
# loading images
#rgb_image = load_image(image_path, grayscale=False)
#gray_image = load_image(image_path, grayscale=True)
gray_image = cv2.cvtColor(rgb_image, cv2.COLOR_BGR2GRAY)
gray_image = np.squeeze(gray_image)
gray_image = gray_image.astype('uint8')
result = []
faces = detect_faces(face_detection, gray_image)
for face_coordinates in faces:
x1, x2, y1, y2 = apply_offsets(face_coordinates, gender_offsets)
rgb_face = rgb_image[y1:y2, x1:x2]
x1, x2, y1, y2 = apply_offsets(face_coordinates, emotion_offsets)
gray_face = gray_image[y1:y2, x1:x2]
try:
rgb_face = cv2.resize(rgb_face, (gender_target_size))
gray_face = cv2.resize(gray_face, (emotion_target_size))
except:
continue
rgb_face = preprocess_input(rgb_face, False)
rgb_face = np.expand_dims(rgb_face, 0)
gender_prediction = gender_classifier.predict(rgb_face)
gender_label_arg = np.argmax(gender_prediction)
gender_text = gender_labels[gender_label_arg]
print(gender_labels)
print(gender_prediction)
print(gender_label_arg)
print(emotion_labels)
gray_face = preprocess_input(gray_face, True)
gray_face = np.expand_dims(gray_face, 0)
gray_face = np.expand_dims(gray_face, -1)
emotion_label_arg = np.argmax(emotion_classifier.predict(gray_face))
emotion_text = emotion_labels[emotion_label_arg]
print(emotion_label_arg)
result.append((gender_label_arg, emotion_label_arg))
if gender_text == gender_labels[0]:
color = (0, 0, 255)
else:
color = (255, 0, 0)
draw_bounding_box(face_coordinates, rgb_image, color)
draw_text(face_coordinates, rgb_image, gender_text, color, 0, -20, 1, 2)
draw_text(face_coordinates, rgb_image, emotion_text, color, 0, -50, 1, 2)
bgr_image = cv2.cvtColor(rgb_image, cv2.COLOR_RGB2BGR)
cv2.imwrite('../images/predicted_test_image' + str(runNumber) + '.png' , bgr_image)
cv2.imwrite('../images/source_test_image' + str(runNumber) + '.png' , rgb_image)
return result
#generateResults(sys.argv[1])
def emotion_demo():
# parameters for loading data and images
detection_model_path = '../trained_models/detection_models/haarcascade_frontalface_default.xml'
emotion_model_path = '../trained_models/emotion_models/fer2013_mini_XCEPTION.102-0.66.hdf5'
emotion_labels = get_labels('fer2013')
# 目のカスケードファイル追加
lefteyecc__path = "../trained_models/detection_models/haarcascade_lefteye_2splits.xml"
righteyecc_path = "../trained_models/detection_models/haarcascade_righteye_2splits.xml"
nose_path = "../trained_models/detection_models/data_haarcascades_haarcascade_mcs_nose.xml"
lefteyecc = cv2.CascadeClassifier(lefteyecc__path)
righteyecc = cv2.CascadeClassifier(righteyecc_path)
nose = cv2.CascadeClassifier(nose_path)
lex = 0
ley = 0
lew = 0
leh = 0
rex = 0
rey = 0
rew = 0
reh = 0
nox = 0
noy = 0
now = 0
noh = 0
# dlib
dlib_ini()
# hyper-parameters for bounding boxes shape
frame_window = 10
emotion_offsets = (20, 40)
# loading models
face_detection = load_detection_model(detection_model_path)
emotion_classifier = load_model(emotion_model_path, compile=False)
# getting input model shapes for inference
emotion_target_size = emotion_classifier.input_shape[1:3]
# starting lists for calculating modes
emotion_window = []
global img, flag, slp_count
img = cv2.imread('../img/happy.png')
flag = 0
slp_count = 0
# dlib用グローバル変数
global gray_image, rgb_image, gray_face, mark_list
# starting video streaming
cv2.namedWindow('window_frame', cv2.WINDOW_NORMAL)
video_capture = cv2.VideoCapture(0) # 0は内蔵カメラ, 1はUSBカメラ
while True:
bgr_image = video_capture.read()[1]
gray_image = cv2.cvtColor(bgr_image, cv2.COLOR_BGR2GRAY)
rgb_image = cv2.cvtColor(bgr_image, cv2.COLOR_BGRA2RGBA)
faces = detect_faces(face_detection, gray_image)
try:
for face_coordinates in faces:
# 目や鼻認識用
(x, y, w, h) = face_coordinates
video_face = gray_image[y:y + h, x:x + w]
x1, x2, y1, y2 = apply_offsets(face_coordinates,
emotion_offsets)
gray_face = gray_image[y1:y2, x1:x2]
try:
gray_face = cv2.resize(gray_face, (emotion_target_size))
except:
continue
# ランドマーク検出
marks_list = marks_list_def(bgr_image, x, y, w, h)
# print(marks_list)
gray_face = preprocess_input(gray_face, True)
gray_face = np.expand_dims(gray_face, 0)
gray_face = np.expand_dims(gray_face, -1)
emotion_prediction = emotion_classifier.predict(gray_face)
emotion_probability = np.max(emotion_prediction)
emotion_label_arg = np.argmax(emotion_prediction)
emotion_text = emotion_labels[emotion_label_arg]
emotion_window.append(emotion_text)
if len(emotion_window) > frame_window:
emotion_window.pop(0)
try:
emotion_mode = mode(emotion_window)
except:
continue
if flag == 0 or flag == 1:
if emotion_text == 'angry':
img = cv2.imread('../img/angry.png', -1)
color = emotion_probability * np.asarray((255, 0, 0))
elif emotion_text == 'sad':
img = cv2.imread('../img/sad.png', -1) # 関数にする
color = emotion_probability * np.asarray((0, 0, 255))
elif emotion_text == 'happy':
img = cv2.imread('../img/happy.png', -1)
color = emotion_probability * np.asarray((255, 255, 0))
elif emotion_text == 'surprise':
img = cv2.imread('../img/odoroki.png', -1)
color = emotion_probability * np.asarray((0, 255, 255))
else:
img = cv2.imread('../img/neutral.png', -1)
color = emotion_probability * np.asarray((0, 255, 0))
else:
if emotion_text == 'angry':
img = cv2.imread('../img/ikari.png', -1)
color = emotion_probability * np.asarray((255, 0, 0))
elif emotion_text == 'sad':
img = cv2.imread('../img/shock.png', -1)
color = emotion_probability * np.asarray((0, 0, 255))
elif emotion_text == 'happy':
img = cv2.imread('../img/kirakira.png', -1)
color = emotion_probability * np.asarray((255, 255, 0))
elif emotion_text == 'surprise':
img = cv2.imread('../img/bikkuri.png', -1)
color = emotion_probability * np.asarray((0, 255, 255))
else:
img = cv2.imread('../img/toumei.png', -1)
color = emotion_probability * np.asarray((0, 255, 0))
color = color.astype(int)
color = color.tolist()
if flag == 0:
draw_bounding_box(face_coordinates, rgb_image, color)
elif flag == 1:
rgb_image = draw_bounding_box2(face_coordinates, rgb_image,
color, img, marks_list)
elif flag == 2:
overlay_pic = draw_bounding_box3(face_coordinates, rgb_image,
color, img, marks_list)
rgb_image = overlay_pic
draw_text(face_coordinates, rgb_image, emotion_mode, color, 0, -45,
1, 1)
bgr_image = cv2.cvtColor(rgb_image, cv2.COLOR_RGB2BGR)
except:
flag = 0
if flag == 0:
img = image_resize(img)
cv2.imshow('image', img)
cv2.destroyWindow('window_frame')
elif flag == 1 or flag == 2:
cv2.destroyWindow('image')
cv2.imshow('window_frame', bgr_image)
cv2.waitKey(10)
# cv2.imshow('own_window', bgr_image)
if cv2.waitKey(1) & 0xFF == ord('z'):
flag = 0
elif cv2.waitKey(1) & 0xFF == ord('x'):
flag = 1
elif cv2.waitKey(1) & 0xFF == ord('c'):
flag = 2
elif cv2.waitKey(1) & 0xFF == ord('q'):
break
video_capture.release()
cv2.destroyAllWindows()
def process_image(image):
try:
# parameters for loading data and images
detection_model_path = './trained_models/detection_models/haarcascade_frontalface_default.xml'
emotion_model_path = './trained_models/emotion_models/fer2013_mini_XCEPTION.102-0.66.hdf5'
gender_model_path = './trained_models/gender_models/simple_CNN.81-0.96.hdf5'
emotion_labels = get_labels('fer2013')
gender_labels = get_labels('imdb')
font = cv2.FONT_HERSHEY_SIMPLEX
# hyper-parameters for bounding boxes shape
gender_offsets = (30, 60)
gender_offsets = (10, 10)
emotion_offsets = (20, 40)
emotion_offsets = (0, 0)
# loading models
face_detection = load_detection_model(detection_model_path)
emotion_classifier = load_model(emotion_model_path, compile=False)
gender_classifier = load_model(gender_model_path, compile=False)
# getting input model shapes for inference
emotion_target_size = emotion_classifier.input_shape[1:3]
gender_target_size = gender_classifier.input_shape[1:3]
# loading images
image_array = np.fromstring(image, np.uint8)
unchanged_image = cv2.imdecode(image_array, cv2.IMREAD_UNCHANGED)
rgb_image = cv2.cvtColor(unchanged_image, cv2.COLOR_BGR2RGB)
gray_image = cv2.cvtColor(unchanged_image, cv2.COLOR_BGR2GRAY)
faces = detect_faces(face_detection, gray_image)
for face_coordinates in faces:
x1, x2, y1, y2 = apply_offsets(face_coordinates, gender_offsets)
rgb_face = rgb_image[y1:y2, x1:x2]
x1, x2, y1, y2 = apply_offsets(face_coordinates, emotion_offsets)
gray_face = gray_image[y1:y2, x1:x2]
try:
rgb_face = cv2.resize(rgb_face, (gender_target_size))
gray_face = cv2.resize(gray_face, (emotion_target_size))
except:
continue
rgb_face = preprocess_input(rgb_face, False)
rgb_face = np.expand_dims(rgb_face, 0)
gender_prediction = gender_classifier.predict(rgb_face)
gender_label_arg = np.argmax(gender_prediction)
gender_text = gender_labels[gender_label_arg]
gray_face = preprocess_input(gray_face, True)
gray_face = np.expand_dims(gray_face, 0)
gray_face = np.expand_dims(gray_face, -1)
emotion_label_arg = np.argmax(emotion_classifier.predict(gray_face))
emotion_text = emotion_labels[emotion_label_arg]
if gender_text == gender_labels[0]:
color = (0, 0, 255)
else:
color = (255, 0, 0)
draw_bounding_box(face_coordinates, rgb_image, color)
draw_text(face_coordinates, rgb_image, gender_text, color, 0, -20, 1, 2)
draw_text(face_coordinates, rgb_image, emotion_text, color, 0, -50, 1, 2)
except Exception as err:
logging.error('Error in emotion gender processor: "{0}"'.format(err))
bgr_image = cv2.cvtColor(rgb_image, cv2.COLOR_RGB2BGR)
dirname = 'result'
if not os.path.exists(dirname):
os.mkdir(dirname)
cv2.imwrite(os.path.join(dirname, 'predicted_image.png'), bgr_image)
def process_image(image):
try:
# hyper-parameters for bounding boxes shape
gender_offsets = (30, 60)
gender_offsets = (10, 10)
emotion_offsets = (20, 40)
emotion_offsets = (0, 0)
emotion_classifier = load_model(emotion_model_path, compile=False)
gender_classifier = load_model(gender_model_path, compile=False)
# getting input model shapes for inference
emotion_target_size = emotion_classifier.input_shape[1:3]
gender_target_size = gender_classifier.input_shape[1:3]
# loading images
image_array = np.fromstring(image, np.uint8)
unchanged_image = cv2.imdecode(image_array, cv2.IMREAD_UNCHANGED)
rgb_image = cv2.cvtColor(unchanged_image, cv2.COLOR_BGR2RGB)
gray_image = cv2.cvtColor(unchanged_image, cv2.COLOR_BGR2GRAY)
faces = detect_faces(face_detection, gray_image)
features = []
for face_coordinates in faces:
x1, x2, y1, y2 = apply_offsets(face_coordinates, gender_offsets)
rgb_face = rgb_image[y1:y2, x1:x2]
x1, x2, y1, y2 = apply_offsets(face_coordinates, emotion_offsets)
gray_face = gray_image[y1:y2, x1:x2]
try:
rgb_face = cv2.resize(rgb_face, (gender_target_size))
gray_face = cv2.resize(gray_face, (emotion_target_size))
except:
continue
rgb_face = preprocess_input(rgb_face, False)
rgb_face = np.expand_dims(rgb_face, 0)
gender_prediction = gender_classifier.predict(rgb_face)
gray_face = preprocess_input(gray_face, True)
gray_face = np.expand_dims(gray_face, 0)
gray_face = np.expand_dims(gray_face, -1)
emotion_prediction = emotion_classifier.predict(gray_face)
features.append({'gender_prediction_female':str(gender_prediction[0][0]),
'gender_prediction_male':str(gender_prediction[0][1]),
'emotion_prediction0':str(emotion_prediction[0][0]),
'emotion_prediction1':str(emotion_prediction[0][1]),
'emotion_prediction2':str(emotion_prediction[0][2]),
'emotion_prediction3':str(emotion_prediction[0][3]),
'emotion_prediction4':str(emotion_prediction[0][4]),
'emotion_prediction5':str(emotion_prediction[0][5]),
'emotion_prediction6':str(emotion_prediction[0][6]),
'x1':str(x1),
'x2':str(x2),
'y1':str(y1),
'y2':str(y2)
})
except Exception as err:
logging.error('Error in emotion gender processor: "{0}"'.format(err))
K.clear_session()
return features
# starting lists for calculating modes
emotion_window = []
# starting video streaming
cv2.namedWindow('window_frame')
video_capture = cv2.VideoCapture(0)
while True:
bgr_image = video_capture.read()[1]
gray_image = cv2.cvtColor(bgr_image, cv2.COLOR_BGR2GRAY)
rgb_image = cv2.cvtColor(bgr_image, cv2.COLOR_BGR2RGB)
faces = detect_faces(face_detection, gray_image)
for face_coordinates in faces:
x1, x2, y1, y2 = apply_offsets(face_coordinates, offsets)
gray_face = gray_image[y1:y2, x1:x2]
try:
gray_face = cv2.resize(gray_face, (target_size))
except:
continue
gray_face = preprocess_input(gray_face, True)
gray_face = np.expand_dims(gray_face, 0)
gray_face = np.expand_dims(gray_face, -1)
guided_gradCAM = calculate_guided_gradient_CAM(gray_face,
gradient_function, saliency_function)
guided_gradCAM = cv2.resize(guided_gradCAM, (x2-x1, y2-y1))
try:
rgb_guided_gradCAM = np.repeat(guided_gradCAM[:, :, np.newaxis],
3, axis=2)
def emotion_identify(img_url):
# parameters for loading data and images
detection_model_path = 'C:/Users/Admin/PycharmProjects/Emotion_Detection/trained_models/detection_models/haarcascade_frontalface_default.xml'
emotion_model_path = 'C:/Users/Admin/PycharmProjects/Emotion_Detection/trained_models/emotion_models/fer2013_mini_XCEPTION.102-0.66.hdf5'
gender_model_path = 'C:/Users/Admin/PycharmProjects/Emotion_Detection/trained_models/gender_models/simple_CNN.81-0.96.hdf5'
emotion_labels = get_labels('fer2013')
gender_labels = get_labels('imdb')
font = cv2.FONT_HERSHEY_SIMPLEX
# hyper-parameters for bounding boxes shape
gender_offsets = (30, 60)
gender_offsets = (10, 10)
emotion_offsets = (20, 40)
emotion_offsets = (0, 0)
# loading models
face_detection = load_detection_model(detection_model_path)
emotion_classifier = load_model(emotion_model_path, compile=False)
gender_classifier = load_model(gender_model_path, compile=False)
# getting input model shapes for inference
emotion_target_size = emotion_classifier.input_shape[1:3]
gender_target_size = gender_classifier.input_shape[1:3]
# loading images
image_path = img_url
rgb_image = load_image(image_path, grayscale=False)
gray_image = load_image(image_path, grayscale=True)
gray_image = np.squeeze(gray_image)
gray_image = gray_image.astype('uint8')
faces = detect_faces(face_detection, gray_image)
if len(faces) == 0:
print("No face")
K.clear_session()
return False
emotions = collections.defaultdict(int)
for face_coordinates in faces:
x1, x2, y1, y2 = apply_offsets(face_coordinates, gender_offsets)
rgb_face = rgb_image[y1:y2, x1:x2]
x1, x2, y1, y2 = apply_offsets(face_coordinates, emotion_offsets)
gray_face = gray_image[y1:y2, x1:x2]
try:
rgb_face = cv2.resize(rgb_face, (gender_target_size))
gray_face = cv2.resize(gray_face, (emotion_target_size))
except:
continue
rgb_face = preprocess_input(rgb_face, False)
rgb_face = np.expand_dims(rgb_face, 0)
gender_prediction = gender_classifier.predict(rgb_face)
gender_label_arg = np.argmax(gender_prediction)
gender_text = gender_labels[gender_label_arg]
gray_face = preprocess_input(gray_face, True)
gray_face = np.expand_dims(gray_face, 0)
gray_face = np.expand_dims(gray_face, -1)
emotion_label_arg = np.argmax(emotion_classifier.predict(gray_face))
emotion_text = emotion_labels[emotion_label_arg]
emotions[emotion_text] += 1
if gender_text == gender_labels[0]:
color = (0, 0, 255)
else:
color = (255, 0, 0)
draw_bounding_box(face_coordinates, rgb_image, color)
draw_text(face_coordinates, rgb_image, gender_text, color, 0, -20, 1, 2)
draw_text(face_coordinates, rgb_image, emotion_text, color, 0, -50, 1, 2)
max_num = 0
max_emotion = None
for key, value in emotions.items():
if value > max_num:
max_num = value
max_emotion = key
print("The emotion of this picture is: ", max_emotion)
bgr_image = cv2.cvtColor(rgb_image, cv2.COLOR_RGB2BGR)
cv2.imwrite('./result_images/predicted_test_image.png', bgr_image)
K.clear_session()
return max_emotion
