class EmotionShard(Shard): name = 'emotions' def __init__(self, model_path, logfile=None, **kwargs): """Initialize a Shard to analyze emotions from the webcam stream. kwargs are passed to emotion_recognition.EmotionRecognition.""" self.recognition = EmotionRecognition(model_path=model_path, **kwargs) self.classes = self.recognition.emotions if logfile is not None: self.memory = CSVMemory(logfile=logfile) else: logging.warning("Memory not activated for Shard '%s'." % self.name) self.memory = None self.state = None def reflect(self, rays): if 'faces' in rays: results = [] for face in rays['faces']: result = self.recognition.run_on_face(face['image']) result['position'] = face['bounding_box'] results.append(result) self.state = results return self.state if 'webcam' in rays: results = self.recognition.run(rays['webcam']) self.state = results return self.state else: raise Exception("Need webcam capture to detect emotions!")
def infer(): #Load model network = EmotionRecognition() network.build_network() #Predict result = network.predict(format_image_norm(image)) label = np.argmax(result[0]) print("Emotion: " + str(EMOTIONS[np.argmax(result[0])])) return label
def __init__(self, model_path, logfile=None, **kwargs): """Initialize a Shard to analyze emotions from the webcam stream. kwargs are passed to emotion_recognition.EmotionRecognition.""" self.recognition = EmotionRecognition(model_path=model_path, **kwargs) self.classes = self.recognition.emotions if logfile is not None: self.memory = CSVMemory(logfile=logfile) else: logging.warning("Memory not activated for Shard '%s'." % self.name) self.memory = None self.state = None
def train(): # Load Model network = EmotionRecognition() # network.model.summary() # images, labels, images_testing, labels_testing = loadData() images, labels = loadDate('./emotionDataset/train') history = network.start_training(images, labels) network.save_model() showHistory(history=history) print('train end!') # network.build_network() # network.load_model2() # dataset = DatasetLoader() # dataset.load_from_save() print('[+] Dataset found and loaded')
def test(): # images, labels, images_testing, labels_testing = loadData() images_testing, labels_testing = loadDate('./emotionDataset/test') network = EmotionRecognition() # network.build_network() network.load_model2() print('[+] Testing load model') result = network.predict(images_testing) # num = 0 y_true = [] y_pred = [] for label in range(7): for ii in range(len(result)): pre = list(result[ii]).index(np.max(result[ii])) gt = list(labels_testing[ii]).index(np.max(labels_testing[ii])) # y_true.append(gt) # y_pred.append(pre) # target_names = ['class 0', 'class 1', 'class 2', 'class 3', 'class 4', 'class 5', 'class 6'] # print(classification_report(y_true, y_pred, target_names=target_names)) if gt == label: y_true.append(1) else: y_true.append(0) if pre == label: y_pred.append(1) else: y_pred.append(0) recall = recall_score(y_true, y_pred) precision = precision_score(y_true, y_pred) accuracy = accuracy_score(y_true, y_pred) f1_measure = f1_score(y_true, y_pred) similarity = jaccard_similarity_score(y_true, y_pred) print('###label :%d' % label) print( 'recall:%s precision:%s similarity:%s F1_measure:%s accuracy:%s' % (recall, precision, similarity, f1_measure, accuracy)) print( "=========================================================================" ) y_true = [] y_pred = []
face = max_area_face image = image[face[1]:(face[1] + face[2]), face[0]:(face[0] + face[3])] # Resize image to network size try: image = cv2.resize(image, (SIZE_FACE, SIZE_FACE), interpolation=cv2.INTER_CUBIC) / 255. except Exception: print("[+] Problem during resize") return None # cv2.imshow("Lol", image) # cv2.waitKey(0) return image # Load Model network = EmotionRecognition() network.build_network() video_capture = cv2.VideoCapture(0) font = cv2.FONT_HERSHEY_SIMPLEX feelings_faces = [] for index, emotion in enumerate(EMOTIONS): feelings_faces.append(cv2.imread('./emojis/' + emotion + '.png', -1)) while True: # Capture frame-by-frame ret, frame = video_capture.read() # Predict result with network result = network.predict(format_image(frame))
face = max_area_face image = image[face[1]:(face[1] + face[2]), face[0]:(face[0] + face[3])] # Resize image to network size try: image = cv2.resize(image, (SIZE_FACE, SIZE_FACE), interpolation=cv2.INTER_CUBIC) / 255. except Exception: print("[+] Problem during resize") return None, None # cv2.imshow("Lol", image) # cv2.waitKey(0) return image, faces # Load Model network = EmotionRecognition() network.build_network() video_capture = cv2.VideoCapture(0) font = cv2.FONT_HERSHEY_SIMPLEX feelings_faces = [] for index, emotion in enumerate(EMOTIONS): feelings_faces.append(cv2.imread('./emojis/' + emotion + '.png', -1)) while True: # Capture frame-by-frame ret, frame = video_capture.read() img, faces = format_image(frame) # Predict result with network
if __name__ == "__main__": input_sentence = "" current_emotion = "" meeting_emotion = "" top_words = [] # Enter a Mircosoft Speech token into the SpeechRecognition constructor speech = SpeechRecognition("") user_interface = UserInterface(window) context = ContextRecognition() context.load_corpus("corpus/") context.load_model() emotion = EmotionRecognition(user_interface) emotion.start() # The chatbot will start listening to the user after they say "computer" # The bot will then read your emotion via webcam while input_sentence != "computer": user_interface.update_sprites("Listening...", "Emotion: None", "User: Unknown", "Primary Topics: ") user_interface.render() emotion.run() input_sentence = speech.recognize_speech() print "You said: ", input_sentence meeting_emotion = emotion.get_emotion() print "Emotion read: ", meeting_emotion
max_area_face = face # Chop image to face face = max_area_face image = image[face[1]:(face[1] + face[2]), face[0]:(face[0] + face[3])] # Resize image to network size try: image = cv2.resize(image, (SIZE_FACE, SIZE_FACE), interpolation = cv2.INTER_CUBIC) / 255. except Exception: print("[+] Problem during resize") return None # cv2.imshow("Lol", image) # cv2.waitKey(0) return image # Load Model network = EmotionRecognition() network.build_network() video_capture = cv2.VideoCapture(0) font = cv2.FONT_HERSHEY_SIMPLEX feelings_faces = [] for index, emotion in enumerate(EMOTIONS): feelings_faces.append(cv2.imread('./emojis/' + emotion + '.png', -1)) while True: # Capture frame-by-frame ret, frame = video_capture.read() # Predict result with network result = network.predict(format_image(frame))
camera_view = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB) camera_view = Image.fromarray(camera_view) camera_view = ImageTk.PhotoImage(image=camera_view) self.camera_view_canvas.create_image(camera_view.width() / 2 + 100, camera_view.height() / 2, image=camera_view) self.update_idletasks() self.update() # When everything is done, release the capture video_capture.release() cv2.destroyAllWindows() if __name__ == '__main__': # Load Model network = EmotionRecognition() network.build_network() # build AlexNet, a kind of CNN musicplayer = musicPlayer.musicPlayer() camera_lock = threading.Lock() font = cv2.FONT_HERSHEY_SIMPLEX #load the emotion icon feelings_faces = [] for index, emotion in enumerate(EMOTIONS): feelings_faces.append(cv2.imread('./emojis/' + emotion + '.png', -1)) obj = SmartBudy() obj.mainloop()
face = max_area_face image = image[face[1]:(face[1] + face[2]), face[0]:(face[0] + face[3])] # Resize image to network size try: image = cv2.resize(image, (SIZE_FACE, SIZE_FACE), interpolation=cv2.INTER_CUBIC) / 255. except Exception: print("[+] Problem during resize") return None # cv2.imshow("Lol", image) # cv2.waitKey(0) return image # Load Model network = EmotionRecognition() network.load_model() video_capture = cv2.VideoCapture(0) font = cv2.FONT_HERSHEY_SIMPLEX feelings_faces = [] for index, emotion in enumerate(EMOTIONS): feelings_faces.append(cv2.imread('./emojis/' + emotion + '.png', -1)) while True: # Capture frame-by-frame ret, frame = video_capture.read() # Predict result with network result = network.predict(format_image(frame))
import socket import pickle import cv2 import numpy import preprocessing from emotion_recognition import EmotionRecognition HOST = '127.0.0.1' # The server's hostname or IP address PORT = 65436 # The port used by the server display = True recvSocket = socket.socket(socket.AF_INET, socket.SOCK_STREAM) recvSocket.bind((HOST, PORT)) recvSocket.listen() network = EmotionRecognition() network.build_network() def result_to_emotion(result, default): if result is None: return default if result[0].tolist().index(max(result[0])) == 0: return "Smile" elif result[0].tolist().index(max(result[0])) == 1: return "Calm" else: return "Unhappy"
# Proof-of-concept import cv2 import sys from constants import * from emotion_recognition import EmotionRecognition import numpy as np cascade_classifier = cv2.CascadeClassifier(CASC_PATH) network = EmotionRecognition() network.load_model()
def detect(): video_capture = cv2.VideoCapture(0) font = cv2.FONT_HERSHEY_SIMPLEX feelings_faces = [] for index, emotion in enumerate(EMOTIONS): feelings_faces.append(cv2.imread('./emojis/' + emotion + '.png', -1)) voting_1 = [] cal_1 = [] voting_2 = [] cal_2 = [] count = 0 network = EmotionRecognition() # network.build_network() network.load_model2() print('[+] Testing load model') while True: # Capture frame-by-frame ret, frame = video_capture.read() #cv2.imshow('Video', frame) cv2.imwrite("%05d.jpg", frame) # Predict result with network result = network.predict(format_image(frame)) # Draw face in frame # for (x,y,w,h) in faces: # cv2.rectangle(frame, (x,y), (x+w,y+h), (255,0,0), 2) # Write results in frame if result is not None: for index, emotion in enumerate( EMOTIONS): # 3 顯示每一frame之偵測資訊(文字、直方圖) cv2.putText(frame, emotion, (10, index * 20 + 20), cv2.FONT_HERSHEY_PLAIN, 0.5, (0, 255, 0), 1) cv2.rectangle(frame, (130, index * 20 + 10), (130 + int(result[0][index] * 100), (index + 1) * 20 + 4), (255, 0, 0), -1) count += 1 print(result[0].tolist().index(max(result[0]))) voting_1.append(result[0].tolist().index(max(result[0]))) if len(voting_1) == 60: for i in range(7): cal_1.append(voting_1.count(i)) maximum_face_times = np.max(cal_1) maximum_face = cal_1.index(maximum_face_times) # print(maximum_face, maximum_face_times) voting_2.append(maximum_face) voting_1.clear() cal_1.clear() if ( len(voting_2) == 7 ): # 之後改次數--!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! for i in range(7): cal_2.append(voting_2.count(i)) maximum_face_times_2 = np.max(cal_2) maximum_face_2 = cal_2.index(maximum_face_times_2) print("Voting 結果︰第", maximum_face_2, "類") face_image = feelings_faces[maximum_face_2] # emotion = ['angry', 'disgusted', 'fearful', 'happy', 'sad', 'surprised', 'neutral'] for c in range(0, 3): frame[200:320, 10:130, c] = face_image[:, :, c] * ( face_image[:, :, 3] / 255.0) + frame[200:320, 10:130, c] * (1.0 - face_image[:, :, 3] / 255.0) voting_2.clear() cal_2.clear() ###response from pygame import mixer import random mixer.init() mixer.music.load( join('./response/', str(maximum_face_2)) + '/' + str(random.randint(0, 3)) + '.mp3') mixer.music.play() insert_mongo(int(maximum_face_2)) # connection = MongoClient('140.138.145.77', 27017) # connection.admin.authenticate("bigmms", "bigmms1413b") # tdb = connection.musicky # post = tdb.test # # # for i in tdb.test.find({"emotion": EMOTIONS[maximum_face_2]}): print(i) # pipeline = [{"$match": {"emotion": EMOTIONS[maximum_face_2]}}, # {"$sample": {"size": 1}}] # 隨機取出一個"emotion":"sad"的資料 # data = list(tdb.test.aggregate(pipeline)) # print(data) # # a = str(data) # # delete = ["[", "{", "}", "]", "\'"] # 刪除不必要的符號 # for i in range(len(delete)): # a = a.replace(delete[i], "") # # replace = [": ", ", "] # 替換不必要的符號 # for j in range(len(replace)): # a = a.replace(replace[j], ",") # # a = a.split(",") # 以逗號區分不同字串 # rand_keyword = a[a.index("keyword") + 1] # 根據不同的情緒,抓出所要使用的keyword # print(rand_keyword) # keyword = rand_keyword # keyword = urllib.parse.quote(rand_keyword) # url = "https://www.youtube.com/results?search_query=" + keyword # crawler(url) if ( count >= 420 ): # and (count%180)<=20): # Ugly transparent fix # 表情圖片顯示於螢幕 (停留20 frames) for c in range(0, 3): frame[200:320, 10:130, c] = face_image[:, :, c] * ( face_image[:, :, 3] / 255.0) + frame[200:320, 10:130, c] * (1.0 - face_image[:, :, 3] / 255.0) # # Display the resulting frame cv2.imshow('Video', frame) if cv2.waitKey(1) & 0xFF == ord('q'): break video_capture.release() cv2.destroyAllWindows()
# -*- coding: utf-8 -*- import matplotlib.pyplot as plt import numpy as np from constants import * from emotion_recognition import EmotionRecognition from dataset_loader import DatasetLoader # Load Model network = EmotionRecognition() network.build_network(loadModel=True) data = DatasetLoader() data.load_from_save(data_source='fer2013') images = data.images_test labels = data.labels_test print('[+] Loading Data') data = np.zeros((len(EMOTIONS), len(EMOTIONS))) for i in range(images.shape[0]): if i % 1000 == 0: print("Progress: {}/{} {:.2f}%".format(i, images.shape[0], i * 100.0 / images.shape[0])) result = network.predict(images[i]) data[np.argmax(labels[i]), np.argmax(result[0])] += 1 print("Accuracy: %f" % (np.sum(data.diagonal()) / np.sum(data))) # Take % by column