class EmotionShard(Shard):
name = 'emotions'
def __init__(self, model_path, logfile=None, **kwargs):
"""Initialize a Shard to analyze emotions from the webcam stream.
kwargs are passed to emotion_recognition.EmotionRecognition."""
self.recognition = EmotionRecognition(model_path=model_path, **kwargs)
self.classes = self.recognition.emotions
if logfile is not None:
self.memory = CSVMemory(logfile=logfile)
else:
logging.warning("Memory not activated for Shard '%s'." % self.name)
self.memory = None
self.state = None
def reflect(self, rays):
if 'faces' in rays:
results = []
for face in rays['faces']:
result = self.recognition.run_on_face(face['image'])
result['position'] = face['bounding_box']
results.append(result)
self.state = results
return self.state
if 'webcam' in rays:
results = self.recognition.run(rays['webcam'])
self.state = results
return self.state
else:
raise Exception("Need webcam capture to detect emotions!")
def infer():
#Load model
network = EmotionRecognition()
network.build_network()
#Predict
result = network.predict(format_image_norm(image))
label = np.argmax(result[0])
print("Emotion: " + str(EMOTIONS[np.argmax(result[0])]))
return label
def __init__(self, model_path, logfile=None, **kwargs):
"""Initialize a Shard to analyze emotions from the webcam stream.
kwargs are passed to emotion_recognition.EmotionRecognition."""
self.recognition = EmotionRecognition(model_path=model_path, **kwargs)
self.classes = self.recognition.emotions
if logfile is not None:
self.memory = CSVMemory(logfile=logfile)
else:
logging.warning("Memory not activated for Shard '%s'." % self.name)
self.memory = None
self.state = None
def train():
# Load Model
network = EmotionRecognition()
# network.model.summary()
# images, labels, images_testing, labels_testing = loadData()
images, labels = loadDate('./emotionDataset/train')
history = network.start_training(images, labels)
network.save_model()
showHistory(history=history)
print('train end!')
# network.build_network()
# network.load_model2()
# dataset = DatasetLoader()
# dataset.load_from_save()
print('[+] Dataset found and loaded')
def test():
# images, labels, images_testing, labels_testing = loadData()
images_testing, labels_testing = loadDate('./emotionDataset/test')
network = EmotionRecognition()
# network.build_network()
network.load_model2()
print('[+] Testing load model')
result = network.predict(images_testing)
# num = 0
y_true = []
y_pred = []
for label in range(7):
for ii in range(len(result)):
pre = list(result[ii]).index(np.max(result[ii]))
gt = list(labels_testing[ii]).index(np.max(labels_testing[ii]))
# y_true.append(gt)
# y_pred.append(pre)
# target_names = ['class 0', 'class 1', 'class 2', 'class 3', 'class 4', 'class 5', 'class 6']
# print(classification_report(y_true, y_pred, target_names=target_names))
if gt == label:
y_true.append(1)
else:
y_true.append(0)
if pre == label:
y_pred.append(1)
else:
y_pred.append(0)
recall = recall_score(y_true, y_pred)
precision = precision_score(y_true, y_pred)
accuracy = accuracy_score(y_true, y_pred)
f1_measure = f1_score(y_true, y_pred)
similarity = jaccard_similarity_score(y_true, y_pred)
print('###label :%d' % label)
print(
'recall:%s precision:%s similarity:%s F1_measure:%s accuracy:%s'
% (recall, precision, similarity, f1_measure, accuracy))
print(
"========================================================================="
)
y_true = []
y_pred = []
face = max_area_face
image = image[face[1]:(face[1] + face[2]), face[0]:(face[0] + face[3])]
# Resize image to network size
try:
image = cv2.resize(image, (SIZE_FACE, SIZE_FACE),
interpolation=cv2.INTER_CUBIC) / 255.
except Exception:
print("[+] Problem during resize")
return None
# cv2.imshow("Lol", image)
# cv2.waitKey(0)
return image
# Load Model
network = EmotionRecognition()
network.build_network()
video_capture = cv2.VideoCapture(0)
font = cv2.FONT_HERSHEY_SIMPLEX
feelings_faces = []
for index, emotion in enumerate(EMOTIONS):
feelings_faces.append(cv2.imread('./emojis/' + emotion + '.png', -1))
while True:
# Capture frame-by-frame
ret, frame = video_capture.read()
# Predict result with network
result = network.predict(format_image(frame))
face = max_area_face
image = image[face[1]:(face[1] + face[2]), face[0]:(face[0] + face[3])]
# Resize image to network size
try:
image = cv2.resize(image, (SIZE_FACE, SIZE_FACE),
interpolation=cv2.INTER_CUBIC) / 255.
except Exception:
print("[+] Problem during resize")
return None, None
# cv2.imshow("Lol", image)
# cv2.waitKey(0)
return image, faces
# Load Model
network = EmotionRecognition()
network.build_network()
video_capture = cv2.VideoCapture(0)
font = cv2.FONT_HERSHEY_SIMPLEX
feelings_faces = []
for index, emotion in enumerate(EMOTIONS):
feelings_faces.append(cv2.imread('./emojis/' + emotion + '.png', -1))
while True:
# Capture frame-by-frame
ret, frame = video_capture.read()
img, faces = format_image(frame)
# Predict result with network
if __name__ == "__main__":
input_sentence = ""
current_emotion = ""
meeting_emotion = ""
top_words = []
# Enter a Mircosoft Speech token into the SpeechRecognition constructor
speech = SpeechRecognition("")
user_interface = UserInterface(window)
context = ContextRecognition()
context.load_corpus("corpus/")
context.load_model()
emotion = EmotionRecognition(user_interface)
emotion.start()
# The chatbot will start listening to the user after they say "computer"
# The bot will then read your emotion via webcam
while input_sentence != "computer":
user_interface.update_sprites("Listening...", "Emotion: None", "User: Unknown", "Primary Topics: ")
user_interface.render()
emotion.run()
input_sentence = speech.recognize_speech()
print "You said: ", input_sentence
meeting_emotion = emotion.get_emotion()
print "Emotion read: ", meeting_emotion
max_area_face = face
# Chop image to face
face = max_area_face
image = image[face[1]:(face[1] + face[2]), face[0]:(face[0] + face[3])]
# Resize image to network size
try:
image = cv2.resize(image, (SIZE_FACE, SIZE_FACE), interpolation = cv2.INTER_CUBIC) / 255.
except Exception:
print("[+] Problem during resize")
return None
# cv2.imshow("Lol", image)
# cv2.waitKey(0)
return image
# Load Model
network = EmotionRecognition()
network.build_network()
video_capture = cv2.VideoCapture(0)
font = cv2.FONT_HERSHEY_SIMPLEX
feelings_faces = []
for index, emotion in enumerate(EMOTIONS):
feelings_faces.append(cv2.imread('./emojis/' + emotion + '.png', -1))
while True:
# Capture frame-by-frame
ret, frame = video_capture.read()
# Predict result with network
result = network.predict(format_image(frame))
camera_view = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)
camera_view = Image.fromarray(camera_view)
camera_view = ImageTk.PhotoImage(image=camera_view)
self.camera_view_canvas.create_image(camera_view.width() / 2 + 100,
camera_view.height() / 2,
image=camera_view)
self.update_idletasks()
self.update()
# When everything is done, release the capture
video_capture.release()
cv2.destroyAllWindows()
if __name__ == '__main__':
# Load Model
network = EmotionRecognition()
network.build_network() # build AlexNet, a kind of CNN
musicplayer = musicPlayer.musicPlayer()
camera_lock = threading.Lock()
font = cv2.FONT_HERSHEY_SIMPLEX
#load the emotion icon
feelings_faces = []
for index, emotion in enumerate(EMOTIONS):
feelings_faces.append(cv2.imread('./emojis/' + emotion + '.png', -1))
obj = SmartBudy()
obj.mainloop()
face = max_area_face
image = image[face[1]:(face[1] + face[2]), face[0]:(face[0] + face[3])]
# Resize image to network size
try:
image = cv2.resize(image, (SIZE_FACE, SIZE_FACE),
interpolation=cv2.INTER_CUBIC) / 255.
except Exception:
print("[+] Problem during resize")
return None
# cv2.imshow("Lol", image)
# cv2.waitKey(0)
return image
# Load Model
network = EmotionRecognition()
network.load_model()
video_capture = cv2.VideoCapture(0)
font = cv2.FONT_HERSHEY_SIMPLEX
feelings_faces = []
for index, emotion in enumerate(EMOTIONS):
feelings_faces.append(cv2.imread('./emojis/' + emotion + '.png', -1))
while True:
# Capture frame-by-frame
ret, frame = video_capture.read()
# Predict result with network
result = network.predict(format_image(frame))
import socket
import pickle
import cv2
import numpy
import preprocessing
from emotion_recognition import EmotionRecognition
HOST = '127.0.0.1' # The server's hostname or IP address
PORT = 65436 # The port used by the server
display = True
recvSocket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
recvSocket.bind((HOST, PORT))
recvSocket.listen()
network = EmotionRecognition()
network.build_network()
def result_to_emotion(result, default):
if result is None:
return default
if result[0].tolist().index(max(result[0])) == 0:
return "Smile"
elif result[0].tolist().index(max(result[0])) == 1:
return "Calm"
else:
return "Unhappy"
# Proof-of-concept import cv2 import sys from constants import * from emotion_recognition import EmotionRecognition import numpy as np cascade_classifier = cv2.CascadeClassifier(CASC_PATH) network = EmotionRecognition() network.load_model()
def detect():
video_capture = cv2.VideoCapture(0)
font = cv2.FONT_HERSHEY_SIMPLEX
feelings_faces = []
for index, emotion in enumerate(EMOTIONS):
feelings_faces.append(cv2.imread('./emojis/' + emotion + '.png', -1))
voting_1 = []
cal_1 = []
voting_2 = []
cal_2 = []
count = 0
network = EmotionRecognition()
# network.build_network()
network.load_model2()
print('[+] Testing load model')
while True:
# Capture frame-by-frame
ret, frame = video_capture.read()
#cv2.imshow('Video', frame)
cv2.imwrite("%05d.jpg", frame)
# Predict result with network
result = network.predict(format_image(frame))
# Draw face in frame
# for (x,y,w,h) in faces:
# cv2.rectangle(frame, (x,y), (x+w,y+h), (255,0,0), 2)
# Write results in frame
if result is not None:
for index, emotion in enumerate(
EMOTIONS): # 3 顯示每一frame之偵測資訊(文字、直方圖)
cv2.putText(frame, emotion, (10, index * 20 + 20),
cv2.FONT_HERSHEY_PLAIN, 0.5, (0, 255, 0), 1)
cv2.rectangle(frame, (130, index * 20 + 10),
(130 + int(result[0][index] * 100),
(index + 1) * 20 + 4), (255, 0, 0), -1)
count += 1
print(result[0].tolist().index(max(result[0])))
voting_1.append(result[0].tolist().index(max(result[0])))
if len(voting_1) == 60:
for i in range(7):
cal_1.append(voting_1.count(i))
maximum_face_times = np.max(cal_1)
maximum_face = cal_1.index(maximum_face_times)
# print(maximum_face, maximum_face_times)
voting_2.append(maximum_face)
voting_1.clear()
cal_1.clear()
if (
len(voting_2) == 7
): # 之後改次數--!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
for i in range(7):
cal_2.append(voting_2.count(i))
maximum_face_times_2 = np.max(cal_2)
maximum_face_2 = cal_2.index(maximum_face_times_2)
print("Voting 結果︰第", maximum_face_2, "類")
face_image = feelings_faces[maximum_face_2]
# emotion = ['angry', 'disgusted', 'fearful', 'happy', 'sad', 'surprised', 'neutral']
for c in range(0, 3):
frame[200:320, 10:130, c] = face_image[:, :, c] * (
face_image[:, :, 3] /
255.0) + frame[200:320, 10:130,
c] * (1.0 - face_image[:, :, 3] / 255.0)
voting_2.clear()
cal_2.clear()
###response
from pygame import mixer
import random
mixer.init()
mixer.music.load(
join('./response/', str(maximum_face_2)) + '/' +
str(random.randint(0, 3)) + '.mp3')
mixer.music.play()
insert_mongo(int(maximum_face_2))
# connection = MongoClient('140.138.145.77', 27017)
# connection.admin.authenticate("bigmms", "bigmms1413b")
# tdb = connection.musicky
# post = tdb.test
#
# # for i in tdb.test.find({"emotion": EMOTIONS[maximum_face_2]}): print(i)
# pipeline = [{"$match": {"emotion": EMOTIONS[maximum_face_2]}},
# {"$sample": {"size": 1}}] # 隨機取出一個"emotion":"sad"的資料
# data = list(tdb.test.aggregate(pipeline))
# print(data)
#
# a = str(data)
#
# delete = ["[", "{", "}", "]", "\'"] # 刪除不必要的符號
# for i in range(len(delete)):
# a = a.replace(delete[i], "")
#
# replace = [": ", ", "] # 替換不必要的符號
# for j in range(len(replace)):
# a = a.replace(replace[j], ",")
#
# a = a.split(",") # 以逗號區分不同字串
# rand_keyword = a[a.index("keyword") + 1] # 根據不同的情緒，抓出所要使用的keyword
# print(rand_keyword)
# keyword = rand_keyword
# keyword = urllib.parse.quote(rand_keyword)
# url = "https://www.youtube.com/results?search_query=" + keyword
# crawler(url)
if (
count >= 420
): # and (count%180)<=20): # Ugly transparent fix # 表情圖片顯示於螢幕 (停留20 frames)
for c in range(0, 3):
frame[200:320, 10:130, c] = face_image[:, :, c] * (
face_image[:, :, 3] /
255.0) + frame[200:320, 10:130,
c] * (1.0 - face_image[:, :, 3] / 255.0)
# # Display the resulting frame
cv2.imshow('Video', frame)
if cv2.waitKey(1) & 0xFF == ord('q'):
break
video_capture.release()
cv2.destroyAllWindows()
# -*- coding: utf-8 -*-
import matplotlib.pyplot as plt
import numpy as np
from constants import *
from emotion_recognition import EmotionRecognition
from dataset_loader import DatasetLoader
# Load Model
network = EmotionRecognition()
network.build_network(loadModel=True)
data = DatasetLoader()
data.load_from_save(data_source='fer2013')
images = data.images_test
labels = data.labels_test
print('[+] Loading Data')
data = np.zeros((len(EMOTIONS), len(EMOTIONS)))
for i in range(images.shape[0]):
if i % 1000 == 0:
print("Progress: {}/{} {:.2f}%".format(i, images.shape[0],
i * 100.0 / images.shape[0]))
result = network.predict(images[i])
data[np.argmax(labels[i]), np.argmax(result[0])] += 1
print("Accuracy: %f" % (np.sum(data.diagonal()) / np.sum(data)))
# Take % by column
