def __init__(self):
self.output_video_file_name = '/home/algernon/samba/video_queue/SalienceRecognizer/videos/processed/output2.mkv'
self.emotion_recognizer = EmotionRecognizer(self.EMOTION_PROB_THRESH)
self.segmentator = None
# self.clothes_detector = ClothesDetector("yolo/df2cfg/yolov3-df2.cfg", "yolo/weights/yolov3-df2_15000.weights", "yolo/df2cfg/df2.names")
self.video_file_name = '/home/algernon/Videos/source_videos/interview_anna.webm'
self.captioner = Captioner()
self.video_reader = VideoReader(self.video_file_name)
self.joke_picker = JokePicker('joke_picker/shortjokes.csv',
'joke_picker/joke_picker.fse')
self.video_writer = cv2.VideoWriter(
self.output_video_file_name, cv2.VideoWriter_fourcc(*"XVID"),
self.video_reader.fps,
(self.video_reader.width, self.video_reader.height))
self.face_recognizer = FaceRecognizer()
self.segmentator = SceneSegmentator(self.video_reader.fps *
self.DELAY_TO_DETECT_IN_SECS)
self.object_detection_reader = DetectionReader('detections.json')
self.object_detector = ObjectDetector(
'./configs/COCO-Detection/faster_rcnn_R_101_FPN_3x.yaml',
'https://dl.fbaipublicfiles.com/detectron2/COCO-Detection/faster_rcnn_R_101_FPN_3x/137851257/model_final_f6e8b1.pkl'
)
self.age_gender_predictor = AgeGenderPredictor()
self.beauty_estimator = BeautyEstimator(
'/home/algernon/DNNS/isBeauty/weights/epoch_50.pkl')
self.main_person_definer = MainPersonDefiner()
self.context_generator = ContextGenerator(self.object_detector.classes)
self.speech_recognizer = SpeechRecognizer(self.video_file_name)
def investigate_dataset_size(db, feature_extraction_methods, args, proportions,
best_params, config_path, size_invest_path):
config_dict = {}
for feat_extr_meth, par, best_param in zip(feature_extraction_methods,
args, best_params):
arg_dict = {list(par.keys())[0]: best_param}
if par["const_args"] is not None:
arg_dict.update(par["const_args"])
accuracy_values = []
for proportion in proportions:
X_train, y_train, X_test, y_test = train_and_test_split(
db, proportion)
recognizer = FaceRecognizer(feat_extr_meth, arg_dict)
recognizer.fit(X_train, y_train)
accuracy = find_accuracy(X_test, y_test, recognizer)
accuracy_values.append(accuracy)
method_name = feat_extr_meth.__name__
method_dict = {
method_name: {
'best_param': proportions[np.argmax(accuracy_values)],
'best_accuracy': max(accuracy_values)
}
}
config_dict.update(method_dict)
with open(config_path, 'w') as f:
yaml.dump(config_dict, f)
train_imgs_count = list(map(lambda x: int(x * 10), proportions))
file_path = os.path.join(size_invest_path, method_name)
make_plot(train_imgs_count, accuracy_values, file_path, method_name,
"Dataset size investigation")
print(f"Dataset size investigation for {method_name} is done")
def __init__(self):
self.detection_reader = DetectionReader('detections.json')
self.project_file_name = '/home/algernon/andro2'
self.video_file_name = ''
self.db_name = ''
self.data_base = None
self.video_maker = None
self.db_user_name = 'root'
self.db_user_pass = '******'
self.db_host = 'localhost'
self.commands = []
self.output_video_file_name = 'output.mkv'
self.video_reader = None
self.video_writer = None
self.emotion_detection_reader = DetectionReader('emotion_results/er.json')
self.emotion_recognizer = EmotionRecognizer(self.EMOTION_PROB_THRESH)
self.captioner = Captioner('/home/algernon/a-PyTorch-Tutorial-to-Image-Captioning/weights/BEST_checkpoint_coco_5_cap_per_img_5_min_word_freq.pth.tar',
'/home/algernon/a-PyTorch-Tutorial-to-Image-Captioning/weights/WORDMAP_coco_5_cap_per_img_5_min_word_freq.json')
self.segmentator = None
self.clothes_detector = ClothesDetector("yolo/df2cfg/yolov3-df2.cfg", "yolo/weights/yolov3-df2_15000.weights", "yolo/df2cfg/df2.names")
self.face_recognizer = FaceRecognizer()
self.open_project()
self.recognizer = Recognizer(
'/home/algernon/PycharmProjects/AIVlog/mmdetection/configs/pascal_voc/faster_rcnn_r50_fpn_1x_voc0712.py',
'/home/algernon/PycharmProjects/AIVlog/mmdetection/work_dirs/faster_rcnn_r50_fpn_1x_voc0712/epoch_10.pth')
def __init__(self):
'''
This class does not take any arguments
'''
self.fr = FaceRecognizer()
self.bridge = CvBridge()
rospy.Subscriber('/odom', Odometry, self.__odom_callback)
# Subscribe on the received images
rospy.Subscriber('/custom/compute_faces_and_positions', Image,
self.__request_callback)
# Pubish the computed information
self.recognition_pub = rospy.Publisher('/custom/people_simple',
Float32MultiArray,
queue_size=1)
self.position_publisher = rospy.Publisher('/custom/people_positions',
PoseStamped,
queue_size=1,
latch=True)
# Contineously keep track of the current odometry to calculate target location
# in correspondance to odom
self.odom = Odometry()
# Keep lists of the recognized ids, locations and times
# lists are easy to remove a target id from all lists after a
# while and to check if id is present, ...
self.recognized_ids = []
self.recognized_locs = []
self.recognized_times = []
def __init__(self):
self.fr = FaceRecognizer()
self.bridge = CvBridge()
self.odom_sub = rospy.Subscriber('/odom', Pose, self.__odom_callback)
self.odom = Pose()
self.recognition_pub = rospy.Publisher('/custom/people_simple',
Float32MultiArray,
queue_size=1)
def trainToFV(self):
recognizer = FaceRecognizer()
for i in self.pathClasses:
imgClass = [cv2.imread(imgPath,0) for imgPath in i]
# for j in range(len(imgClass)):
# cv2.imshow(str(i) + str(j), imgClass[j])
fvClass = [recognizer.extractFeatures(img) for img in imgClass]
self.fvClasses.append(fvClass)
self.fvClassNames.append(i[0])
pass
def recognize_face(): ''' Runs the face recognition algorithm on the appropriate frames. ''' # Saman: add face recognizer here # Hannah: Done! recognizer = FaceRecognizer() name = recognizer.result() #name = ":D" return name
def get_image_tensor(self):
image_path = self.data_dict['path']
face_finder = FaceRecognizer()
face_finder.new_image(image_path)
np_tensor = np.zeros(DataInterpreter.num_pixels)
try:
np_tensor = face_finder.get_face_1D_numpy()
except (AssertionError, FileNotFoundError) as e:
pass
return torch.from_numpy(np_tensor).double()
def get_image_and_name_tensor(self):
np_tensor = np.zeros(DataInterpreter.num_pixels + 6)
image_path = self.data_dict['path']
face_finder = FaceRecognizer()
face_finder.new_image(image_path)
np_face_tensor = np.zeros(DataInterpreter.num_pixels)
try:
np_face_tensor = face_finder.get_face_1D_numpy()
except AssertionError:
pass
assert np_face_tensor.size < np_tensor.size
ctr = 0
for i in range(np_face_tensor.size):
np_tensor[i] = np_face_tensor[i]
ctr += 1
name = ""
try:
name = self.data_dict['name']
if name == None or name == "":
raise NameError
if " " in name:
name = name[:name.index(" ")]
first_letter = DataInterpreter.hash_str(name[0])
first_two = DataInterpreter.hash_str(name[1])
first_three = DataInterpreter.hash_str(name[2])
last_three = DataInterpreter.hash_str(name[-3])
last_two = DataInterpreter.hash_str(name[-2])
last_letter = DataInterpreter.hash_str(name[-1])
np_tensor[ctr] = first_letter
np_tensor[ctr + 1] = first_two
np_tensor[ctr + 2] = first_three
np_tensor[ctr + 3] = last_three
np_tensor[ctr + 4] = last_two
np_tensor[ctr + 5] = last_letter
except NameError:
np_tensor[ctr] = 0
np_tensor[ctr + 1] = 0
np_tensor[ctr + 2] = 0
np_tensor[ctr + 3] = 0
np_tensor[ctr + 4] = 0
np_tensor[ctr + 5] = 0
return torch.from_numpy(np_tensor).double()
def run():
'''Does all the dirty work, so we can make a thread.'''
img_queue = Queue(maxsize=5) ## Spawns threads which capture
start_still_cap = StillStarter(
1, cv2, img_queue) ## images, putting in img_queue
c = cv2.VideoCapture(0)
_, f = c.read() ## store raw cam data in f
avg = np.float32(f) ## declare memory for math ops
dif = np.float32(f)
recognizer = FaceRecognizer(
img_queue, return_queue) ## Initialize facial recognition
recognizer.start_thread() ## thread
done = 1 ## Loop watches for movement,
while done != None: ## searching for faces if there
## is any
try:
done = inqueue.get(block=False)
except Empty:
pass
_, f = c.read()
cv2.accumulateWeighted(f, avg, 0.3) ## Put results of average over
## time into avg
dif = f - avg ## Difference between f and avg
## to find movement
res3 = cv2.convertScaleAbs(dif)
move_value = np.sum(dif)
## Uncomment to show webcam video
## cv2.imshow('img',f)
## Uncomment to show the difference between f and avg
## cv.ShowImage('Dif',cv.fromarray(dif))
if move_value > 1000000:
#print "Movement!", move_value
start_still_cap(1, c)
k = cv2.waitKey(20) ## Poll for close event
if k == 27 or done == None:
img_queue.put(None)
break
cv2.destroyAllWindows() ## Close windows when loop
c.release() ## terminates
def index(request):
if request.method == 'GET':
print request.GET
fc = FaceRecognizer()
res = fc.getClassification()
return JsonResponse({'foo':res})
#return HttpResponse("Hello face")
else:
print ("get post")
print (request.FILES)
f = request.FILES['file']
im = Image.open(f)
im.save('photo.png')
return JsonResponse({'foo':'bar'})
def run():
"""Does all the dirty work, so we can make a thread."""
img_queue = Queue(maxsize=5) ## Spawns threads which capture
start_still_cap = StillStarter(1, cv2, img_queue) ## images, putting in img_queue
c = cv2.VideoCapture(0)
_, f = c.read() ## store raw cam data in f
avg = np.float32(f) ## declare memory for math ops
dif = np.float32(f)
recognizer = FaceRecognizer(img_queue, return_queue) ## Initialize facial recognition
recognizer.start_thread() ## thread
done = 1 ## Loop watches for movement,
while done != None: ## searching for faces if there
## is any
try:
done = inqueue.get(block=False)
except Empty:
pass
_, f = c.read()
cv2.accumulateWeighted(f, avg, 0.3) ## Put results of average over
## time into avg
dif = f - avg ## Difference between f and avg
## to find movement
res3 = cv2.convertScaleAbs(dif)
move_value = np.sum(dif)
## Uncomment to show webcam video
## cv2.imshow('img',f)
## Uncomment to show the difference between f and avg
## cv.ShowImage('Dif',cv.fromarray(dif))
if move_value > 1000000:
# print "Movement!", move_value
start_still_cap(1, c)
k = cv2.waitKey(20) ## Poll for close event
if k == 27 or done == None:
img_queue.put(None)
break
cv2.destroyAllWindows() ## Close windows when loop
c.release() ## terminates
def person_recognition_example(db, proportion, feature_extraction_methods,
args, images_path):
X_train, y_train, X_test, y_test = train_and_test_split(db, proportion)
[img_width, img_height] = X_test[0].shape
new_width, new_height = img_width * 3, img_height * 3
resized_img = cv2.resize(X_test[0], (new_width, new_height))
show_image(resized_img, "Original image")
cv2.imwrite(os.path.join(images_path, "Original image.png"), X_test[0])
for feat_extr_meth, par in zip(feature_extraction_methods, args):
recognizer = FaceRecognizer(feat_extr_meth, par)
recognizer.fit(X_train, y_train)
prediction, pred_number = recognizer.predict(X_test[0])
method_name = feat_extr_meth.__name__
resized_img = cv2.resize(X_train[pred_number], (new_width, new_height))
show_image(resized_img, method_name)
cv2.imwrite(os.path.join(images_path, f"{method_name}.png"),
X_train[pred_number])
def get_data_file(dataset, thread_num):
detector = FaceRecognizer()
temp_dataset = []
ctr = 0
file_ctr = 0
gc.collect()
while len(dataset) > 0:
datum = dataset.pop(0)
print("File ", thread_num, file_ctr, "datapoint", ctr)
detector.new_image(datum['path'])
try:
a = detector.get_face_1D_numpy().tolist()
datum["as_tensor"] = a
datum["landmarks"] = detector.get_facial_landmarks(thread_num)
temp_dataset.append(datum)
gc.collect()
except (LookupError, AssertionError, NameError,
FileNotFoundError) as e:
gc.collect()
pass
ctr += 1
file_name = 'FinalDataset/FinalDataset' + str(thread_num) + "t_" + str(
file_ctr) + "f_" + '.json'
file_ctr += 1
with open(file_name, 'w') as outfile:
json.dump(temp_dataset, outfile)
del temp_dataset
gc.collect()
print("Thread Num: ", thread_num, " Done")
def investigate_params(X_train, y_train, X_test, y_test,
feature_extraction_methods, args, param_graphs_path,
config_path):
best_params = []
config_dict = {}
for feat_extr_meth, par in zip(feature_extraction_methods, args):
accuracy_values = []
for arg in list(par.values())[0]:
arg_dict = {list(par.keys())[0]: arg}
if par["const_args"] is not None:
arg_dict.update(par["const_args"])
recognizer = FaceRecognizer(feat_extr_meth, arg_dict)
recognizer.fit(X_train, y_train)
accuracy = find_accuracy(X_test, y_test, recognizer)
accuracy_values.append(accuracy)
best_param = list(par.values())[0][np.argmax(accuracy_values)]
best_params.append(best_param)
method_name = feat_extr_meth.__name__
method_dict = {
method_name: {
'best_param': best_param,
'best_accuracy': max(accuracy_values)
}
}
config_dict.update(method_dict)
with open(config_path, 'w') as f:
yaml.dump(config_dict, f)
paameters = list(par.values())[0]
file_path = os.path.join(param_graphs_path, method_name)
make_plot(paameters, accuracy_values, file_path, method_name,
"Parameters investigation")
print(f"Parameters investigation for {method_name} is done")
return best_params
def main(argv):
faceClassManager = None
dir = "/Users/admin/Desktop/Computer Vision/Projects/Final Project HBD/Final Project Github/Input/"
inputImagePaths = [os.path.join(dir,f) for f in os.listdir(dir)]
inputImages = [cv2.imread(i,0) for i in inputImagePaths]
if (len(argv) == 0):
faceClassManager = FaceClassManager("./DataSets/")
# TODO: inputimages default
# inputImages.append(cv2.imread(ex,0))
elif len(argv) == 1:
#TODO: datasets default
#TODO: input images from commands
pass
elif len(argv) == 2:
#TODO: input images and datasets from commands
pass
else:
#TODO Errors
pass
pp = pprint.PrettyPrinter(indent=4)
pp.pprint(faceClassManager.pathClasses)
faceClassManager.trainToFV()
recognizer = FaceRecognizer()
recognizer.recognize(inputImages, faceClassManager)
pp.pprint(faceClassManager.fvClassNames)
cv2.imshow("Hello", cv2.imread(faceClassManager.fvClassNames[2],0))
cv2.waitKey()
margin = argument_namespace.margin
distance_method = argument_namespace.distance_method
distance_method_list = ['euclidean', 'cosine']
assert distance_method in distance_method_list, 'distance_method must be in %s' % (
' or'.join(distance_method_list))
# 实例化相机对象
cameraIndex = 0
camera = cv2.VideoCapture(cameraIndex)
windowName = "faceDetection_demo"
is_successful, image_3d_array = camera.read()
if is_successful:
# 导入代码文件FaceRecognizer.py中的类FaceRecognizer
from FaceRecognizer import FaceRecognizer
print('成功加载人脸识别类FaceRecognizer')
face_detector = FaceDetector()
face_recognizer = FaceRecognizer(distance_method=distance_method)
print('成功实例化人脸检测对象、人脸识别对象')
else:
print('未成功调用相机，请检查相机是否已连接电脑')
sys.exit()
# 实例视频流写入对象
if video_aviFilePath != None:
fourcc = cv2.VideoWriter_fourcc('M', 'P', '4', '2')
image_height, image_width, _ = image_3d_array.shape
image_size = image_width, image_height
videoWriter = cv2.VideoWriter(video_aviFilePath, fourcc, 6, image_size)
while is_successful:
startTime = time.time()
# 使用cv2库从相机读取的图像数据均为blue、green、red通道顺序，与rgb顺序相反
is_successful, bgr_3d_array = camera.read()
rgb_3d_array = cv2.cvtColor(bgr_3d_array, cv2.COLOR_BGR2RGB)
class RecognitionSubscriber(object):
'''
Class that subscribes on compute_faces_and_position
receives images and computes the face id and target location from this
and publishes this on the people_simple topic
'''
def __init__(self):
'''
This class does not take any arguments
'''
self.fr = FaceRecognizer()
self.bridge = CvBridge()
rospy.Subscriber('/odom', Odometry, self.__odom_callback)
# Subscribe on the received images
rospy.Subscriber('/custom/compute_faces_and_positions', Image,
self.__request_callback)
# Pubish the computed information
self.recognition_pub = rospy.Publisher('/custom/people_simple',
Float32MultiArray,
queue_size=1)
self.position_publisher = rospy.Publisher('/custom/people_positions',
PoseStamped,
queue_size=1,
latch=True)
# Contineously keep track of the current odometry to calculate target location
# in correspondance to odom
self.odom = Odometry()
# Keep lists of the recognized ids, locations and times
# lists are easy to remove a target id from all lists after a
# while and to check if id is present, ...
self.recognized_ids = []
self.recognized_locs = []
self.recognized_times = []
def publish_position(self, x, y, frame_id):
'''
Publish the position of a detected face on the map
arguments: x, y, frame_id (reference)
'''
pose = PoseStamped()
pose.header.frame_id = frame_id
pose.header.stamp = rospy.Time.now()
pose.pose.position.x = x
pose.pose.position.y = y
self.position_publisher.publish(pose)
def __request_callback(self, msg):
'''
Callback if image is received, faces are detected and
added to the global array of people
A message is sent with the ids, locations and face widths
in a multiarray.
'''
cv_image = self.bridge.imgmsg_to_cv2(msg, 'rgb8')
if SHOW_RECEIVED_IMAGE:
cv2.imwrite('rec_img.png', cv_image)
ret = self.fr.recognize_people(cv_image)
simple_msg = Float32MultiArray()
if ret:
print('new face detected')
locs, ids = ret
for i, loc in zip(ids, locs):
if i in self.recognized_ids:
index = self.recognized_ids.index(i)
self.recognized_locs[index] = loc
self.recognized_times[index] = datetime.datetime.now()
else:
self.recognized_ids.append(i)
self.recognized_locs.append(loc)
self.recognized_times.append(datetime.datetime.now())
else:
print('No faces detected')
# Publish the computed info on the simple_people topic
# as a multiarray. The structure is defined as [id, xpos, ypos, width]
# in a repeated fashion for multiple faces.
# The width might be used as a metric for how far the face is away
dim = MultiArrayDimension()
dim.label = 'id_and_pos'
dim.size = 3
dim.stride = 0
dim2 = MultiArrayDimension()
dim2.label = 'length_faces'
dim2.size = len(self.recognized_ids)
dim2.stride = 0
simple_msg.layout.dim = [dim, dim2]
simple_msg.layout.data_offset = 0
data = []
# Add the recognized faces around the table to the message
for i, loc in zip(self.recognized_ids, self.recognized_locs):
width = self.__get_width(loc)
x, y, valid = self.__calculate_people_positions(width)
data = np.concatenate((data, [i, x, y, width]), axis=0)
simple_msg.data = data
print(simple_msg)
# Publish the message
self.recognition_pub.publish(simple_msg)
# Check if person is seen in the last TIME_BEFORE_FORGOTTEN seconds and otherwise remove
for index, time in enumerate(self.recognized_times):
if abs((time - datetime.datetime.now()
).total_seconds()) > TIME_BEFORE_FORGOTTEN:
del self.recognized_ids[index]
del self.recognized_locs[index]
del self.recognized_times[index]
def __odom_callback(self, msg):
'''
Keep the odometry up to date
'''
self.odom = msg
def __get_width(self, location):
'''
Helper function to calculate the width of a face
'''
return abs(location[3] - location[1])
def __get_height(self, location):
'''
Helper function to calculate the height of a face
'''
return abs(location[0] - location[2])
def __get_center(self, location):
'''
Helper function to calculate the center of a face
'''
def get_top_left(location):
return location[3], location[0]
def get_bottom_right(location):
return location[1], location[2]
# Calculate the x, y position of the found person
x_l, y_l = get_top_left(location)
x_r, y_r = get_bottom_right(location)
point = Point()
point.x = (x_l + x_r) / 2.0
point.y = (y_l + y_r) / 2.0
point.z = 0
return point
def __calculate_people_positions(self, width):
'''
Calculate the peoples position based on the current odometry
Assume that the face is straight ahead and the distance is
dependent on the face width
'''
_, _, yaw = tf.transformations.euler_from_quaternion([
self.odom.pose.pose.orientation.x,
self.odom.pose.pose.orientation.y,
self.odom.pose.pose.orientation.w,
self.odom.pose.pose.orientation.z,
])
print(yaw)
print(self.odom)
target_pose = copy.deepcopy(self.odom)
#fac = 1.0/width
offset = 0.10
fac = max(-1.36 * np.log10(width) + 3.63, 0)
#fac = 0.3
print(fac)
x = target_pose.pose.pose.position.x + fac - offset #np.cos(yaw) * fac * width
y = target_pose.pose.pose.position.y #- np.sin(yaw) * fac * width
#self.publish_position(x, y, 'base_link')
listener = tf.TransformListener()
listener.waitForTransform("/base_link", "/odom", rospy.Time(0),
rospy.Duration(4.0))
laser_point = PointStamped()
laser_point.header.frame_id = "base_link"
laser_point.header.stamp = rospy.Time(0)
laser_point.point.x = x
laser_point.point.y = y
laser_point.point.z = 0.0
p = listener.transformPoint("odom", laser_point)
x = p.point.x
y = p.point.y
valid = fac < 1.6
#new_vec = transform(np.asarray([x,y,0]), 'base_link', 'odom')
#x = new_vec[0]
#y = new_vec[1]
self.publish_position(x, y, 'odom')
return x, y, valid
argument_namespace = parse_args()
show_keypoints = argument_namespace.show_keypoints
show_personQuantity = argument_namespace.show_personQuantity
video_aviFilePath = argument_namespace.video_aviFilePath
margin = argument_namespace.margin
# 实例化相机对象
cameraIndex = 0
camera = cv2.VideoCapture(cameraIndex)
windowName = "faceDetection_demo"
is_successful, image_3d_array = camera.read()
if is_successful:
# 导入代码文件FaceRecognizer.py中的类FaceRecognizer
from FaceRecognizer import FaceRecognizer
print('成功加载人脸识别类FaceRecognizer')
face_detector = FaceDetector()
face_recognizer = FaceRecognizer()
print('成功实例化人脸检测对象、人脸识别对象')
else:
print('未成功调用相机，请检查相机是否已连接电脑')
sys.exit()
if video_aviFilePath != None:
fourcc = cv2.VideoWriter_fourcc('M', 'P', '4', '2')
image_height, image_width, _ = image_3d_array.shape
image_size = image_width, image_height
videoWriter = cv2.VideoWriter(video_aviFilePath, fourcc, 6, image_size)
while is_successful:
startTime = time.time()
# 使用cv2库从相机读取的图像数据均为blue、green、red通道顺序，与rgb顺序相反
is_successful, bgr_3d_array = camera.read()
rgb_3d_array = cv2.cvtColor(bgr_3d_array, cv2.COLOR_BGR2RGB)
box_2d_array, point_2d_array = face_detector.detect_image(
class InteractionMaker:
EMOTION_PROB_THRESH = 0
def __init__(self):
self.detection_reader = DetectionReader('detections.json')
self.project_file_name = '/home/algernon/andro2'
self.video_file_name = ''
self.db_name = ''
self.data_base = None
self.video_maker = None
self.db_user_name = 'root'
self.db_user_pass = '******'
self.db_host = 'localhost'
self.commands = []
self.output_video_file_name = 'output.mkv'
self.video_reader = None
self.video_writer = None
self.emotion_detection_reader = DetectionReader('emotion_results/er.json')
self.emotion_recognizer = EmotionRecognizer(self.EMOTION_PROB_THRESH)
self.captioner = Captioner('/home/algernon/a-PyTorch-Tutorial-to-Image-Captioning/weights/BEST_checkpoint_coco_5_cap_per_img_5_min_word_freq.pth.tar',
'/home/algernon/a-PyTorch-Tutorial-to-Image-Captioning/weights/WORDMAP_coco_5_cap_per_img_5_min_word_freq.json')
self.segmentator = None
self.clothes_detector = ClothesDetector("yolo/df2cfg/yolov3-df2.cfg", "yolo/weights/yolov3-df2_15000.weights", "yolo/df2cfg/df2.names")
self.face_recognizer = FaceRecognizer()
self.open_project()
self.recognizer = Recognizer(
'/home/algernon/PycharmProjects/AIVlog/mmdetection/configs/pascal_voc/faster_rcnn_r50_fpn_1x_voc0712.py',
'/home/algernon/PycharmProjects/AIVlog/mmdetection/work_dirs/faster_rcnn_r50_fpn_1x_voc0712/epoch_10.pth')
def open_project(self):
with open(self.project_file_name, 'r') as project_file:
self.video_file_name = project_file.readline().strip()
self.db_name = project_file.readline().strip()
self.data_base = DB(self.db_host, self.db_user_name, self.db_user_pass, self.db_name)
self.video_reader = VideoReader(self.video_file_name)
self.video_writer = cv2.VideoWriter(self.output_video_file_name, cv2.VideoWriter_fourcc(*"XVID"),
self.video_reader.fps,
(self.video_reader.width, self.video_reader.height))
self.segmentator = SceneSegmentator(self.video_reader.fps * 5)
self.load_commands_from_db()
def load_commands_from_db(self):
# upload commands
cursor = self.data_base.exec_query("SELECT * FROM Command")
while cursor.rownumber < cursor.rowcount:
command_response = cursor.fetchone()
query = "SELECT name FROM Labels WHERE label_id=%s"
attached_character_class = \
self.data_base.exec_template_query(query, [command_response['attached_character_class']]).fetchone()[
'name']
relation_class = ''
if command_response['relation_class'] is not None:
relation_class = \
self.data_base.exec_template_query(query, [command_response['relation_class']]).fetchone()[
'name']
media_response = self.data_base.exec_query(
f"SELECT * FROM Media WHERE media_id={command_response['media_id']}").fetchone()
media = Media(media_response['file_name'], media_response['type'], media_response['duration'])
trigger_cmd_name = ''
trigger_cmd_id = command_response['trigger_event_id']
if trigger_cmd_id is not None:
trigger_cmd_name = \
self.data_base.exec_query(f"SELECT name FROM Command WHERE command_id={trigger_cmd_id}").fetchone()[
'name']
delay = command_response['delay']
emotion = ''
emotion_id = command_response['expected_emotion_id']
if emotion_id is not None:
emotion = \
self.data_base.exec_query(f"SELECT name FROM Emotion WHERE emotion_id={emotion_id}").fetchone()['name']
command = Command(command_response['name'], command_response['centered'],
command_response['trigger_event_id'],
attached_character_class, relation_class,
CommandType(command_response['command_type_id']),
trigger_cmd_name, media, command_response['duration'], delay, emotion)
self.commands.append(command)
def process_commands(self):
for _ in trange(self.video_reader.frame_count):
frame = self.video_reader.get_next_frame()
cur_frame_num = self.video_reader.cur_frame_num
#emotion_detections = self.detect_emotions_on_frame(frame)
emotion_detections = []
#self.segmentator.push_frame(frame)
clothes_detections = self.clothes_detector.detect_clothes(frame)
self.draw_clothes(frame, clothes_detections)
emotions_per_frame = []
for emotion_pos, emotion in emotion_detections:
emotions_per_frame.append((emotion_pos, emotion))
self.draw_emotion_box(frame, emotion_pos, emotion)
#_, object_detections_per_frame = self.recognizer.inference(frame)
object_detections_per_frame = []
draw_det_boxes(frame, object_detections_per_frame)
labels_per_frame = [detection[0] for detection in object_detections_per_frame]
states_needed_to_be_checked_on_event = [Command.State.WAITING, Command.State.EXECUTING,
Command.State.AFTER_DELAYING]
commands_needed_to_be_checked_on_event = [cmd for cmd in self.commands if
cmd.cur_state in states_needed_to_be_checked_on_event]
for command in commands_needed_to_be_checked_on_event:
self.update_commands(command, object_detections_per_frame, emotions_per_frame, labels_per_frame)
executing_commands = [cmd for cmd in self.commands if cmd.cur_state == cmd.State.EXECUTING]
for active_cmd in executing_commands:
active_cmd.exec(frame)
delaying_commands = [cmd for cmd in self.commands if cmd.cur_state == cmd.State.DELAYING]
for delaying_command in delaying_commands:
if delaying_command.wait_out_delay():
delaying_command.set_as_after_delay()
#self.show_caption(frame)
cv2.imshow('frame', frame)
self.video_writer.write(frame)
cv2.waitKey(1)
def show_caption(self, frame):
most_clear_img = self.segmentator.get_most_clear_frame()
caption = self.captioner.caption_img(most_clear_img)
cv2.putText(frame, caption, (0, 30), cv2.FONT_HERSHEY_SIMPLEX, 1, Color.GOLD, 2)
def draw_clothes(self, frame, clothes_detections):
# clothes_detections: [[label: str, ((x1, y1), (x2, y2)), prob], ..]
font = cv2.FONT_HERSHEY_SIMPLEX
color = Color.BLACK
for label, ((x1, y1), (x2, y2)), prob in clothes_detections:
text = f'{label} ({prob}%)'
cv2.rectangle(frame, (x1, y1), (x2, y2), color, 3)
cv2.rectangle(frame, (x1 - 2, y1 - 25), (x1 + 8.5 * len(text), y1), color, -1)
cv2.putText(frame, text, (x1, y1 - 5), font, 0.5, (255, 255, 255), 1, cv2.LINE_AA)
def detect_emotions_on_frame(self, frame):
#return list of items of the following format: ((lt_point: tuple, rb_point: tuple), (emotion: str, prob: int))
detected_faces = self.face_recognizer.recognize_faces_on_image(frame)
emotions = []
for face_pos in detected_faces:
(l, t), (r, b) = face_pos
face_img = frame[t:b, l:r]
emotion = self.emotion_recognizer.recognize_emotion_by_face(face_img)
if emotion:
emotions.append((face_pos, emotion))
return emotions
def draw_emotion_box(self, frame, emotion_pos, emotion: list):
cv2.rectangle(frame, *emotion_pos, Color.GOLD, 2)
font = cv2.FONT_HERSHEY_SIMPLEX
cv2.putText(frame, f'{emotion[0]} - {emotion[1]}', (emotion_pos[0][0], emotion_pos[0][1] - 5), font, 1,
Color.YELLOW, 3)
def update_commands(self, command, detections_per_frame, emotions_per_frame, labels_per_frame):
if command.command_type == CommandType.OBJECTS_TRIGGER:
self.check_object_on_the_screen_event(command, detections_per_frame, labels_per_frame)
elif command.command_type == CommandType.REACTION_CHAIN_TRIGGER:
self.check_reactions_chain_event(command, detections_per_frame, labels_per_frame)
elif command.command_type == CommandType.EMOTION_TRIGGER:
self.check_emotion_event(command, detections_per_frame, emotions_per_frame, labels_per_frame)
def check_emotion_event(self, command: Command, objects_detections, emotion_detections, labels_per_frame):
# emotions_per_frame format - [((start_point, end_point), (emotion, prob)), ...]
# check whether there's main object
if command.attached_character_class in labels_per_frame:
# check whether there's expected emotion
expected_emotions = [emotion for emotion in emotion_detections if emotion[1][0] == command.emotion]
# check whether an emotion box is inside main object
main_object_box = self.get_coords(command, objects_detections, labels_per_frame)
main_object_box = (main_object_box[:2]), (main_object_box[2:])
emotion = [emotion for emotion in expected_emotions if self.is_rect_inside_rect((emotion[0][0], emotion[0][1]), main_object_box)]
assert len(emotion) <= 1
if emotion:
print(emotion)
coords = *emotion[0][0][0], *emotion[0][0][1]
self.update_state(True, command, objects_detections, labels_per_frame, coords=coords)
def is_rect_inside_rect(self, in_rect: tuple, out_rect: tuple):
lt_in_box_point_inside_out_box = all([out_rect[0][i] <= in_rect[0][i] <= out_rect[1][i] for i in range(2)])
rb_in_box_point_inside_out_box = all([out_rect[0][i] <= in_rect[0][i] <= out_rect[1][i] for i in range(2)])
return lt_in_box_point_inside_out_box and rb_in_box_point_inside_out_box
def check_reactions_chain_event(self, command: Command, detections_per_frame, labels_per_frame):
# there's main object
if command.attached_character_class in labels_per_frame:
# check whether triggered command is active
active_command_names = [command.name for command in self.commands if
command.cur_state == command.State.EXECUTING]
event_happened = command.trigger_cmd_name in active_command_names
self.update_state(event_happened, command, detections_per_frame, labels_per_frame)
def check_object_on_the_screen_event(self, command: Command, detections_per_frame, labels_per_frame):
desired_classes = {command.attached_character_class, command.relation_class}
# we found desired labels
event_happened = desired_classes.issubset(labels_per_frame)
self.update_state(event_happened, command, detections_per_frame, labels_per_frame)
def update_state(self, event_happened, command, detections_per_frame, labels_per_frame, coords=None):
if event_happened:
if command.cur_state == command.State.WAITING:
command.set_as_delaying(self.video_reader.one_frame_duration)
return
coords = self.get_coords(command, detections_per_frame, labels_per_frame) if not coords else coords
if command.cur_state == command.State.EXECUTING:
command.overlay.set_coords(coords)
# extract later this part from update_commands method
if command.cur_state == command.State.AFTER_DELAYING:
if command.media.type == MediaType.VIDEO:
command.overlay = self.generate_video_overlay(command, coords)
elif command.media.type == MediaType.IMAGE:
command.overlay = self.generate_image_overlay_object(command, coords)
elif command.media.type == MediaType.TEXT:
command.overlay = self.generate_text_overlay_object(command, coords)
command.set_as_executing()
elif command.cur_state == command.cur_state.AFTER_DELAYING:
command.set_as_waiting()
@staticmethod
def get_coords(command: Command, detections_per_frame, labels_per_frame):
main_box = detections_per_frame[labels_per_frame.index(command.attached_character_class)][1]
coords = main_box
if command.centered:
secondary_box = detections_per_frame[labels_per_frame.index(command.relation_class)][1]
main_box_center = [(main_box[i + 2] + main_box[i]) // 2 for i in range(2)]
secondary_box_center = [(secondary_box[i + 2] + secondary_box[i]) // 2 for i in range(2)]
boxes_center = [(main_box_center[i] + secondary_box_center[i]) // 2 for i in range(2)]
coords = boxes_center
return coords
def generate_video_overlay(self, command: Command, coords: tuple):
video_cap = cv2.VideoCapture(command.media.file_name)
duration = command.media.duration if command.duration == 0 else command.duration
return VideoOverlay(video_cap, duration, coords, self.video_reader.one_frame_duration)
def generate_image_overlay_object(self, command: Command, coords: tuple):
image = cv2.imread(command.media.file_name)
return ImageOverlay(image, command.duration, coords, self.video_reader.one_frame_duration)
def generate_text_overlay_object(self, command: Command, coords: tuple):
texts = self.read_text_from_file(command.media.file_name)
ellipse, text_rect = generate_thought_balloon_by_text(texts)
return TextOverlay((ellipse, text_rect), command.duration, coords, self.video_reader.one_frame_duration)
def read_text_from_file(self, txt_file):
with open(txt_file) as txt:
texts = txt.readlines()
return texts
def close(self):
if self.video_reader:
self.video_reader.close()
if self.video_writer:
self.video_writer.release()
from FaceRecognizer import FaceRecognizer
from DBAccess import DBAccess
from time import sleep
cap = cv2.VideoCapture(0)
dba = DBAccess()
sleep(1)
while True:
flag = True
ret, frame = cap.read()
gray = cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY)
faceDetector = FaceDetector()
cv2.imshow('attendance', frame)
status, gray = faceDetector.detect(gray)
if status:
fr = FaceRecognizer()
flag, person = fr.recognize(gray)
print "attendance : accept '%s'" % (person)
if flag:
s = raw_input("press y for yes n for no")
if s == 'y' or s == 'n':
if s == 'y':
dba.update(person, 1)
flag = False
break
else:
break
else:
print "please enter the correct option"
if cv2.waitKey(1) & 0xFF == ord('q'):
break
def forward_img(self, img_filepath):
fr = FaceRecognizer()
fr.new_image(img_filepath)
np_face_arr = fr.get_face_as_numpy()
input_tensor = torch.from_numpy(np_face_arr)
return self.forward(input_tensor)
class SalienceRecognizer:
EMOTION_PROB_THRESH = 0
DELAY_TO_DETECT_IN_SECS = 5
FONT = cv2.FONT_HERSHEY_SIMPLEX
def __init__(self):
self.output_video_file_name = '/home/algernon/samba/video_queue/SalienceRecognizer/videos/processed/output2.mkv'
self.emotion_recognizer = EmotionRecognizer(self.EMOTION_PROB_THRESH)
self.segmentator = None
# self.clothes_detector = ClothesDetector("yolo/df2cfg/yolov3-df2.cfg", "yolo/weights/yolov3-df2_15000.weights", "yolo/df2cfg/df2.names")
self.video_file_name = '/home/algernon/Videos/source_videos/interview_anna.webm'
self.captioner = Captioner()
self.video_reader = VideoReader(self.video_file_name)
self.joke_picker = JokePicker('joke_picker/shortjokes.csv',
'joke_picker/joke_picker.fse')
self.video_writer = cv2.VideoWriter(
self.output_video_file_name, cv2.VideoWriter_fourcc(*"XVID"),
self.video_reader.fps,
(self.video_reader.width, self.video_reader.height))
self.face_recognizer = FaceRecognizer()
self.segmentator = SceneSegmentator(self.video_reader.fps *
self.DELAY_TO_DETECT_IN_SECS)
self.object_detection_reader = DetectionReader('detections.json')
self.object_detector = ObjectDetector(
'./configs/COCO-Detection/faster_rcnn_R_101_FPN_3x.yaml',
'https://dl.fbaipublicfiles.com/detectron2/COCO-Detection/faster_rcnn_R_101_FPN_3x/137851257/model_final_f6e8b1.pkl'
)
self.age_gender_predictor = AgeGenderPredictor()
self.beauty_estimator = BeautyEstimator(
'/home/algernon/DNNS/isBeauty/weights/epoch_50.pkl')
self.main_person_definer = MainPersonDefiner()
self.context_generator = ContextGenerator(self.object_detector.classes)
self.speech_recognizer = SpeechRecognizer(self.video_file_name)
def launch(self):
for _ in trange(self.video_reader.frame_count):
frame = self.video_reader.get_next_frame()
frame_for_detection = self.get_clearest_frame(frame)
use_prev_detects = True if frame_for_detection is None else False
speech = self.speech_recognizer.rewind_audio(
self.video_reader.get_cur_timestamp())
if not use_prev_detects:
# faces
detected_faces = self.face_recognizer.recognize_faces_on_image(
frame_for_detection, get_prev=use_prev_detects)
# main person
main_person_index = self.main_person_definer.get_main_person_by_face_size(
detected_faces, get_prev=use_prev_detects)
# emotions
emotion_detections = self.emotion_recognizer.detect_emotions_on_frame(
frame_for_detection, detected_faces, get_prev=use_prev_detects)
emotion = [emotion_detections[main_person_index]
] if main_person_index is not None else []
# age gender
age_gender_predictions = self.age_gender_predictor.detect_age_dender_by_faces(
frame_for_detection, detected_faces, get_prev=use_prev_detects)
age_gender_prediction = [
age_gender_predictions[main_person_index]
] if main_person_index is not None else []
# beauty
beauty_scores = self.beauty_estimator.estimate_beauty_by_face(
frame_for_detection, detected_faces, get_prev=use_prev_detects)
beauty_score = [beauty_scores[main_person_index]
] if main_person_index is not None else []
# clothes
# clothes_detections = self.clothes_detector.detect_clothes(frame)
# clothes_detections = []
# self.draw_clothes(frame, clothes_detections)
# caption
# caption = self.captioner.caption_img(frame_for_detection, get_prev=use_prev_detects)
# objects
object_detections = self.object_detector.forward(
frame_for_detection, get_prev=use_prev_detects)
# object_detections = self.object_detection_reader.get_detections_per_specified_frame(cur_frame_num)
# context = self.context_generator.generate_context(object_detections, caption, emotion, age_gender_prediction,
# beauty_score, get_prev=use_prev_detects)
# cv2.putText(frame, 'Context:', (0, 360), cv2.FONT_HERSHEY_SIMPLEX, 0.8, Color.GOLD, 2)
# cv2.putText(frame, context, (0, 400), cv2.FONT_HERSHEY_SIMPLEX, 0.8, Color.GOLD, 2)
# jokes = self.joke_picker.pick_jokes_by_context(context, get_prev=use_prev_detects)
# self.apply_jokes_on_frame(jokes, frame)
self.apply_emotions_on_frame(frame, emotion_detections)
self.apply_beauty_scores_on_frame(frame, detected_faces,
beauty_scores)
self.apply_age_gender_on_frame(frame, detected_faces,
age_gender_predictions)
# self.apply_caption_on_frame(frame, caption)
frame = self.object_detector.draw_boxes(frame, object_detections)
cv2.imshow('frame', frame)
self.video_writer.write(frame)
cv2.waitKey(1)
def apply_age_gender_on_frame(self, frame, faces, age_gender_predictions):
for i, face in enumerate(faces):
tl_x, tl_y = face[i]
cv2.putText(frame, f'{age_gender_predictions[i]}',
(tl_x - 5, tl_y + 60), self.FONT, 1, Color.BLACK, 2)
def apply_beauty_scores_on_frame(self, frame, faces, beauty_scores):
for i, face in enumerate(faces):
tl_x, tl_y = face[i]
cv2.putText(
frame,
f'{ContextGenerator.beauty_score2desc(beauty_scores[i])} ({beauty_scores[i]})',
(tl_x - 5, tl_y + 30), self.FONT, 1, Color.BLACK, 2)
def apply_jokes_on_frame(self, jokes, frame):
height = frame.shape[0]
joke_height = 40
joke_y = height - joke_height * len(jokes)
for joke in jokes:
cv2.putText(frame, joke, (0, joke_y), cv2.FONT_HERSHEY_SIMPLEX,
0.8, Color.GOLD, 2)
joke_y += joke_height
def apply_emotions_on_frame(self, frame, emotion_detections):
for emotion_pos, emotion in emotion_detections:
self.draw_emotion_box(frame, emotion_pos, emotion)
def get_clearest_frame(self, frame):
self.segmentator.push_frame(frame)
most_clear_img = self.segmentator.get_most_clear_frame()
return most_clear_img
def apply_caption_on_frame(self, frame, caption):
cv2.putText(frame, caption, (0, 30), cv2.FONT_HERSHEY_SIMPLEX, 1,
Color.BLUE, 2)
def draw_clothes(self, frame, clothes_detections):
# clothes_detections: [[label: str, ((x1, y1), (x2, y2)), prob], ..]
color = Color.BLACK
for label, ((x1, y1), (x2, y2)), prob in clothes_detections:
text = f'{label} ({prob}%)'
cv2.rectangle(frame, (x1, y1), (x2, y2), color, 3)
cv2.rectangle(frame, (x1 - 2, y1 - 25), (x1 + 8.5 * len(text), y1),
color, -1)
cv2.putText(frame, text, (x1, y1 - 5), self.FONT, 0.5, color.WHITE,
1, cv2.LINE_AA)
def draw_emotion_box(self, frame, emotion_pos, emotion: list):
cv2.rectangle(frame, *emotion_pos, Color.GOLD, 2)
cv2.putText(frame, f'{emotion[0]} - {emotion[1]}',
(emotion_pos[0][0], emotion_pos[0][1] - 5), self.FONT, 1,
Color.BLACK, 3)
def is_rect_inside_rect(self, in_rect: tuple, out_rect: tuple):
lt_in_box_point_inside_out_box = all([
out_rect[0][i] <= in_rect[0][i] <= out_rect[1][i] for i in range(2)
])
rb_in_box_point_inside_out_box = all([
out_rect[0][i] <= in_rect[0][i] <= out_rect[1][i] for i in range(2)
])
return lt_in_box_point_inside_out_box and rb_in_box_point_inside_out_box
# def generate_video_overlay(self, command: Command, coords: tuple):
# video_cap = cv2.VideoCapture(command.media.file_name)
# duration = command.media.duration if command.duration == 0 else command.duration
# return VideoOverlay(video_cap, duration, coords, self.video_reader.one_frame_duration)
#
# def generate_image_overlay_object(self, command: Command, coords: tuple):
# image = cv2.imread(command.media.file_name)
# return ImageOverlay(image, command.duration, coords, self.video_reader.one_frame_duration)
#
# def generate_text_overlay_object(self, command: Command, coords: tuple):
# texts = self.read_text_from_file(command.media.file_name)
# ellipse, text_rect = generate_thought_balloon_by_text(texts)
# return TextOverlay((ellipse, text_rect), command.duration, coords, self.video_reader.one_frame_duration)
# def read_text_from_file(self, txt_file):
# with open(txt_file) as txt:
# texts = txt.readlines()
# return texts
def close(self):
if self.video_reader:
self.video_reader.close()
if self.video_writer:
self.video_writer.release()
import cv2
from Trainer import Trainer
from Button import Button
from Rectangle import Rectangle
from Point import Point
from FaceRecognizer import FaceRecognizer
from HandRecognizer import HandRecognizer
from Database import Database
import numpy as np
cap = cv2.VideoCapture(0)
cv2.namedWindow('img')
trainer = Trainer()
faceRecognizer = FaceRecognizer()
handRecognizer = HandRecognizer(.65, .8)
db = Database()
x = 0
y = 0
def clickHandler(event, x_click, y_click, flags, param):
global x, y
if event == cv2.EVENT_LBUTTONDOWN:
x = x_click
y = y_click
elif event == cv2.EVENT_LBUTTONUP:
x = 0
y = 0
import argparse
import json
import cv2
import numpy as np
import picamera
import time
import datetime
import os
import os.path
# Dictionary of known people. Add more people here if required.
known_people = { 1 : "Ambar"}
# create instance of FaceRecognizer class to get training data, train data, and predict person
FaceRecognizer = FaceRecognizer()
faceCascade = cv2.CascadeClassifier("haarcascade_frontalface_default.xml")
# set up command line argument parser for JSON file path
argParser = argparse.ArgumentParser()
argParser.add_argument("-c", "--conf", required = True, help = "Path to the JSON config file")
args = vars(argParser.parse_args())
conf = json.load(open(args["conf"]))
dropboxClient = None
if conf["use_dropbox"]:
# connecting to dropbox
flow = DropboxOAuth2FlowNoRedirect(conf["dropbox_key"], conf["dropbox_secret"])
print "[INFO] Authorize this application: {}".format(flow.start())
class RecognitionServer(object):
'''
In class so that the same face recognizer class is used
for all images (Ids and face embeddings are stored there)
# --
When receiving a request, it publishes a simple solution on '/custom/people_simple'
'''
def __init__(self):
self.fr = FaceRecognizer()
self.bridge = CvBridge()
self.odom_sub = rospy.Subscriber('/odom', Pose, self.__odom_callback)
self.odom = Pose()
self.recognition_pub = rospy.Publisher('/custom/people_simple',
Float32MultiArray,
queue_size=1)
def __odom_callback(self, msg):
self.odom = msg
def __get_center(self, location):
def get_top_left(location):
return location[3], location[0]
def get_bottom_right(location):
return location[1], location[2]
# Calculate the x, y position of the found person
x_l, y_l = get_top_left(location)
x_r, y_r = get_bottom_right(location)
point = Point()
point.x = (x_l + x_r) / 2.0
point.y = (y_l + y_r) / 2.0
point.z = 0
return point
def __calculate_people_positions(self):
_, _, yaw = tf.transformations.euler_from_quaternion([
self.odom.pose.orientation.x,
self.odom.pose.orientation.y,
self.odom.pose.orientation.w,
self.odom.pose.orientation.z,
])
target_pose = copy.deepcopy(self.odom)
target_pose.position.x += np.sin(yaw)
target_pose.position.x += np.cos(yaw)
return target_pose.x, target_pose.y
def handle_service_request(self, request):
cv_image = self.bridge.imgmsg_to_cv2(request.img, 'bgr8')
ret = self.fr.recognize_people(cv_image)
if ret:
locs, ids = ret
# Publish simple info on the simple_people topic
simple_msg = Float32MultiArray()
dim = MultiArrayDimension()
dim.label = 'id_and_pos'
dim.size = 3
dim.stride = 0
dim2 = MultiArrayDimension()
dim2.label = 'length_faces'
dim2.size = len(ids)
dim2.stride = 0
simple_msg.layout.dim.label = [dim, dim2]
simple_msg.data_offset = 0
data = []
for i in ids:
x, y = self.__calculate_people_positions()
np.concatenate(data, [i, x, y])
simple_msg.data = data
print(len(data), 3 * len(ids))
print(simple_msg)
# publish the simple message
self.recognition_pub(simple_msg)
# Handle service request
message = DetectFaceResponse()
message.ids = ids
message.positions = [self.__get_center(loc) for loc in locs]
return message
else:
return DetectFaceResponse()
import cv2
import base64
import numpy as np
import os
import io
from PIL import Image
from Utilities import Base64
from FaceDetector import FaceDetector
from FaceRecognizer import FaceRecognizer
app = Flask(__name__)
classes = ['dad', 'mom', 'son', 'daughter']
faceDetector = FaceDetector('FACE_ALT')
faceRecognizer = FaceRecognizer(classes, faceDetector)
faceRecognizer.load_model('src/models/cnn_model_3of4')
@app.route('/')
def index():
return render_template('index.html')
@app.route('/detection', methods=["GET", "POST"])
def detection():
if request.method == "GET":
return render_template('detection.html')
# get base64 image from request
raw_image = request.get_json()['image']
import matplotlib.pyplot as plt
from FaceRecognizer import FaceRecognizer
# FLAGS
parser = argparse.ArgumentParser(description='Parsing args for generate database')
parser.add_argument("--images", type=str, help="face image path organized by names", required=True)
parser.add_argument("--db_dir", type=str, help="database output directory", default='.')
args = parser.parse_args()
if __name__ == '__main__':
# model path
REC_MODEL_PATH_TPU = "pretrained_model/mobilefacenet_edgetpu_cocompiled.tflite"
recognizer = FaceRecognizer(REC_MODEL_PATH_TPU)
# buffer
label = []
db = []
for file in os.listdir(args.images):
try:
image = plt.imread(os.path.join(args.images, file))
embedding = recognizer.face_recognize(image)
label.append(file.split('.')[0])
db.append(embedding)
except:
continue
# write to file
import cv2
from FaceRecognizer import FaceRecognizer
from PIL import Image
import numpy as np
from subprocess import Popen, PIPE
if __name__ == '__main__':
video_capture = cv2.VideoCapture(0)
p = Popen([
'ffmpeg', '-y', '-i', '-', '-pix_fmt', 'yuyv422', '-f', 'v4l2',
'/dev/video2'
],
stdin=PIPE)
facerecognizer = FaceRecognizer()
while True:
ret, frame = video_capture.read()
facerecognizer.run(frame)
im = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)
im = Image.fromarray(np.uint8(im))
im.save(p.stdin, 'JPEG')
show_keypoints = argument_namespace.show_keypoints # 是否显示人脸关键点
show_personQuantity = argument_namespace.show_personQuantity # 人脸分数
video_aviFilePath = argument_namespace.video_aviFilePath # video_path
margin = argument_namespace.margin # 图像边缘阈值
# 实例化相机对象
cameraIndex = 0
camera = cv2.VideoCapture(cameraIndex)
windowName = "faceDetection_demo"
# 采集一帧图像
is_successful, image_3d_array = camera.read()
if is_successful:
# 导入代码文件FaceRecognizer.py中的类FaceRecognizer
from FaceRecognizer import FaceRecognizer
print('成功加载人脸识别类FaceRecognizer')
face_detector = FaceDetector() # 人脸检测类
face_recognizer = FaceRecognizer() # 人脸识别类--这里会计算注册库所有样本
print('成功实例化人脸检测对象、人脸识别对象')
else:
print('未成功调用相机，请检查相机是否已连接电脑')
sys.exit()
if video_aviFilePath != None:
fourcc = cv2.VideoWriter_fourcc('M', 'P', '4', '2')
image_height, image_width, _ = image_3d_array.shape
image_size = image_width, image_height
videoWriter = cv2.VideoWriter(video_aviFilePath, fourcc, 6, image_size)
while is_successful:
startTime = time.time()
# 使用cv2库从相机读取的图像数据均为blue、green、red通道顺序，与rgb顺序相反
is_successful, bgr_3d_array = camera.read() # 抓图,图片格式转换
