def do_some_stuff(self, world_proxy, eye_0_proxy, common_data_proxy):
logger.info('Starting Do_Stuff...')
while True:
world = world_proxy.get_values()
pupil_0 = eye_0_proxy.get_values()
if world[0] is None or pupil_0[0] is None or self.last_frame_processed == world[2]:
continue
# logger.info("Frame - {}, Timestamp - {}".format(world[2], world[1]))
# logger.info(
# "Eye_Id - {}, Norm_Pos - {}, Confidence - {}, Timestamp - {}".format(pupil_0[2], pupil_0[3], pupil_0[4],
# pupil_0[1]))
pupil_loc = pupil_0[3]
try:
detections = self.object_detect.perform_detect(world[3])
detections = denormalize_detections(detections, pupil_0[4])
except Exception as e:
raise e
if self.debug:
self.display_image(detections, pupil_loc, world[3])
self.last_frame_processed = world[2]
run_length_output = self.perform_run_length(detections, pupil_loc)
common_data_proxy.set_values(self.glass_id, world[1], world[2], run_length_output[0], run_length_output[1])
def __init__(self, current_state, max_vel=20000):
"""
kalman_filter: type of kalman filter to use
"""
self.prev_state = np.array([current_state[0], current_state[1]])
self.cur_time = current_state[2]
self.eps_max = 0.6
self.scale = 10
self.dumb = 0
self.kalman = self.tracker_4dof()
self.max_vel = max_vel
self.old_Q = self.kalman.Q
self.kalman.predict()
self.kalman.update(self.prev_state)
self.cur_state = (self.kalman.x[0], self.kalman.x[1])
res = self.kalman.y
eps = np.dot(res.T, inv(self.kalman.S)).dot(res)
if eps > self.eps_max:
self.kalman.Q = self.old_Q * self.scale
self.dumb += 1
elif self.dumb > 0:
self.kalman.Q = self.old_Q
self.dumb -= 1
logger.info("Initialised Kalman")
def world_receiver(self, world_proxy):
logger.info("Starting World Frames Listener...")
ctx = zmq.Context()
requester = ctx.socket(zmq.REQ)
ip = 'localhost'
requester.connect('tcp://%s:%s' % (ip, self.port))
requester.send_string('SUB_PORT')
sub_port = requester.recv_string()
logger.info("Connecting to port {}".format(sub_port))
subscriber = ctx.socket(zmq.SUB)
subscriber.connect('tcp://%s:%s' % (ip, sub_port))
subscriber.setsockopt_string(zmq.SUBSCRIBE, 'frame.world')
try:
while True:
topic = subscriber.recv_string()
info = serializer.unpackb(subscriber.recv(), encoding='utf-8')
# logger.info("Received Topic - {} Frame - {}, Timestamp - {}".format(topic, info['index'], info['timestamp']))
frame = []
while subscriber.get(zmq.RCVMORE):
frame.append(subscriber.recv())
frame_data = np.frombuffer(frame[0], dtype=np.uint8).reshape(
info['height'], info['width'], 3)
world_proxy.set_values(self.glass_id, info['timestamp'],
info['index'], frame_data)
except:
requester.close()
subscriber.close()
ctx.term()
logger.warn('Listener is shut down successfully')
raise
def main():
# Start glass 1 processes
world_receiver_1, eye_0_receiver_1, eye_1_receiver_1, do_some_stuff_1 = start_process(1, port_glass_1,
common_data_proxy_1,
world_proxy_glass_1,
eye_0_proxy_glass_1,
eye_1_proxy_glass_1,
object_detect_proxy_glass_1)
world_receiver_1.start()
eye_0_receiver_1.start()
eye_1_receiver_1.start()
do_some_stuff_1.start()
# Start glass 2 processes
world_receiver_2, eye_0_receiver_2, eye_1_receiver_2, do_some_stuff_2 = start_process(2, port_glass_2,
common_data_proxy_2,
world_proxy_glass_2,
eye_0_proxy_glass_2,
eye_1_proxy_glass_2,
object_detect_proxy_glass_2)
world_receiver_2.start()
eye_0_receiver_2.start()
eye_1_receiver_2.start()
do_some_stuff_2.start()
# Start final combined process
stuff_together = Process(target=do_stuff_together.do_some_stuff_together,
args=(common_data_proxy_1, common_data_proxy_2), name="do_stuff_together")
stuff_together.start()
time.sleep(1)
logger.info("Application started successfully")
world_receiver_1.join()
eye_0_receiver_1.join()
eye_1_receiver_1.join()
do_some_stuff_1.join()
world_receiver_2.join()
eye_0_receiver_2.join()
eye_1_receiver_2.join()
do_some_stuff_2.join()
stuff_together.join()
def do_some_stuff_together(self, common_data_proxy_1, common_data_proxy_2):
logger.info("Starting Do_Stuff_Together...")
while True:
common_data_1 = common_data_proxy_1.get_values()
common_data_2 = common_data_proxy_2.get_values()
if common_data_1[0] is None or common_data_2[0] is None or (
common_data_1[2] == self.last_frame_index_1
and common_data_2[2] == self.last_frame_index_2):
continue
# logger.info("Glass_1 Frame - {}, Glass_2 Frame - {}, Glass_1 Timestamp - {}, Glass_2 Timestamp - {}".format(
# common_data_1[2], common_data_2[2], common_data_1[1], common_data_2[1]))
# logger.info("Glass_1 output - {}, Glass_2 output - {}".format(common_data_1[3], common_data_2[3]))
common_look_time = int(time.time() * 1000)
logger_run_length.info(
";time-{};Glass1_hit_scan_output-{};Glass2_hit_scan_output-{};Glass1_run_length_output-{};Glass2_run_length_output-{};Frame1-{};Frame2-{}"
.format(common_look_time, common_data_1[4], common_data_2[4],
common_data_1[3], common_data_2[3], common_data_1[2],
common_data_2[2]))
if common_data_1[3] == common_data_2[3] and sum(
common_data_1[3]) != 0:
if not self.prev_state:
self.prev_state = True
obj_idx = np.argmax(common_data_1[3])
logger2.info("Look Detected;{}:{}".format(
self.objects[obj_idx], common_look_time))
play_sound()
else:
self.prev_state = False
self.last_frame_index_1 = common_data_1[2]
self.last_frame_index_2 = common_data_2[2]
def init(self):
has_gpu = True
self.lib = CDLL(self.dark_net_path, RTLD_GLOBAL)
self.lib.network_width.argtypes = [c_void_p]
self.lib.network_width.restype = c_int
self.lib.network_height.argtypes = [c_void_p]
self.lib.network_height.restype = c_int
self.predict = self.lib.network_predict
self.predict.argtypes = [c_void_p, POINTER(c_float)]
self.predict.restype = POINTER(c_float)
if has_gpu:
set_gpu = self.lib.cuda_set_device
set_gpu.argtypes = [c_int]
self.make_image = self.lib.make_image
self.make_image.argtypes = [c_int, c_int, c_int]
self.make_image.restype = IMAGE
self.get_network_boxes = self.lib.get_network_boxes
self.get_network_boxes.argtypes = [
c_void_p, c_int, c_int, c_float, c_float,
POINTER(c_int), c_int,
POINTER(c_int), c_int
]
self.get_network_boxes.restype = POINTER(DETECTION)
self.make_network_boxes = self.lib.make_network_boxes
self.make_network_boxes.argtypes = [c_void_p]
self.make_network_boxes.restype = POINTER(DETECTION)
self.free_detections = self.lib.free_detections
self.free_detections.argtypes = [POINTER(DETECTION), c_int]
self.free_ptrs = self.lib.free_ptrs
self.free_ptrs.argtypes = [POINTER(c_void_p), c_int]
self.network_predict = self.lib.network_predict
self.network_predict.argtypes = [c_void_p, POINTER(c_float)]
self.reset_rnn = self.lib.reset_rnn
self.reset_rnn.argtypes = [c_void_p]
self.load_net = self.lib.load_network
self.load_net.argtypes = [c_char_p, c_char_p, c_int]
self.load_net.restype = c_void_p
self.load_net_custom = self.lib.load_network_custom
self.load_net_custom.argtypes = [c_char_p, c_char_p, c_int, c_int]
self.load_net_custom.restype = c_void_p
self.do_nms_obj = self.lib.do_nms_obj
self.do_nms_obj.argtypes = [POINTER(DETECTION), c_int, c_int, c_float]
self.do_nms_sort = self.lib.do_nms_sort
self.do_nms_sort.argtypes = [POINTER(DETECTION), c_int, c_int, c_float]
self.free_image = self.lib.free_image
self.free_image.argtypes = [IMAGE]
self.letterbox_image = self.lib.letterbox_image
self.letterbox_image.argtypes = [IMAGE, c_int, c_int]
self.letterbox_image.restype = IMAGE
self.load_meta = self.lib.get_metadata
self.lib.get_metadata.argtypes = [c_char_p]
self.lib.get_metadata.restype = METADATA
self.load_image = self.lib.load_image_color
self.load_image.argtypes = [c_char_p, c_int, c_int]
self.load_image.restype = IMAGE
self.rgbgr_image = self.lib.rgbgr_image
self.rgbgr_image.argtypes = [IMAGE]
self.predict_image = self.lib.network_predict_image
self.predict_image.argtypes = [c_void_p, IMAGE]
self.predict_image.restype = POINTER(c_float)
assert 0 < self.thresh < 1, "Threshold should be a float between zero and one (non-inclusive)"
if not os.path.exists(self.config_path):
raise ValueError("Invalid config path `" +
os.path.abspath(self.config_path) + "`")
if not os.path.exists(self.weight_path):
raise ValueError("Invalid weight path `" +
os.path.abspath(self.weight_path) + "`")
if not os.path.exists(self.meta_path):
raise ValueError("Invalid data file path `" +
os.path.abspath(self.meta_path) + "`")
self.net_main = self.load_net_custom(self.config_path.encode("ascii"),
self.weight_path.encode("ascii"),
0, 1) # batch size = 1
self.meta_main = self.load_meta(self.meta_path.encode("ascii"))
# In Python 3, the metafile default access craps out on Windows (but not Linux)
# Read the names file and create a list to feed to detect
try:
with open(self.meta_path) as metaFH:
metaContents = metaFH.read()
import re
match = re.search("names *= *(.*)$", metaContents,
re.IGNORECASE | re.MULTILINE)
if match:
result = match.group(1)
else:
result = None
try:
if os.path.exists(result):
with open(result) as namesFH:
names_list = namesFH.read().strip().split("\n")
self.alt_names = [x.strip() for x in names_list]
except TypeError:
pass
except Exception:
pass
logger.info("Initialized Object Detection...")
time.sleep(1)
logger.info("Application started successfully")
world_receiver_1.join()
eye_0_receiver_1.join()
do_some_stuff_1.join()
world_receiver_2.join()
eye_0_receiver_2.join()
do_some_stuff_2.join()
stuff_together.join()
if __name__ == "__main__":
logger.info("Starting application...")
# input parameters to the application
port_glass_1 = 50020
port_glass_2 = 50021
confidence_threshold = 0.80
num_objects = 6
use_both_eyes = True
debug = True
# Proxy objects for common data for both glasses
BaseManager.register('CommonData_1', CommonData)
manager_1 = BaseManager()
manager_1.start()
common_data_proxy_1 = manager_1.CommonData_1()
def do_some_stuff(self, world_proxy, eye_0_proxy, eye_1_proxy,
common_data_proxy):
logger.info('Starting Do_Stuff...')
# print('do')
while True:
world = world_proxy.get_values(
) # world: [0]glass_id, [1]timestamp, [2]index, [3]frame
pupil_0 = eye_0_proxy.get_values(
) # pupil: [0]glass_id, [1]timestamp, [2]eye_id, [3]norm_pos, [4]confidence
pupil_1 = eye_1_proxy.get_values()
# print(world)
# if world[0] is None or pupil_0[0] is None or pupil_1[0] is None or self.last_frame_processed == world[2]:
# continue
if (world[0] is None) or (pupil_0[0] is None
and pupil_1[0] is None):
# print(world[2],pupil_0[3],pupil_1[3],'b')
continue
# print('do')
logger.info("Frame - {}, Timestamp - {}".format(
world[2], world[1]))
# logger.info("Eye_Id - {}, Norm_Pos - {}, Confidence - {}, Timestamp - {}".format(pupil_0[2], pupil_0[3], pupil_0[4], pupil_0[1]))
# logger.info("Eye_Id - {}, Norm_Pos - {}, Confidence - {}, Timestamp - {}".format(pupil_1[2], pupil_1[3], pupil_1[4], pupil_1[1]))
if pupil_0[0] is None:
pupil_0 = pupil_1
elif pupil_1[0] is None:
pupil_1 = pupil_0
# print(world[0],pupil_0[3],pupil_1[3],'a')
if pupil_0[4] > self.confidence_threshold and pupil_1[
4] > self.confidence_threshold:
pupil_loc = np.mean([pupil_0[3], pupil_1[3]], axis=0)
confidence = np.mean([pupil_0[4], pupil_1[4]])
elif pupil_0[4] > pupil_1[4]:
pupil_loc = pupil_0[3]
confidence = pupil_0[4]
else:
pupil_loc = pupil_1[3]
confidence = pupil_1[4]
try:
detections = self.object_detect.perform_detect(world[3])
detections = denormalize_detections(detections, confidence)
except Exception as e:
raise e
if self.debug:
self.display_image(detections, pupil_loc, world[3])
self.last_frame_processed = world[2]
# run_length_output = self.perform_run_length(detections, pupil_loc)
# common_data_proxy.set_values(self.glass_id, world[1], world[2], run_length_output[0], run_length_output[1])
crf_output = self.perform_crf(detections, pupil_loc)
common_data_proxy.set_values(self.glass_id, world[1], world[2],
crf_output[0], crf_output[1])
def perform_crf(self, detections, pupil_loc):
# output = [0] * self.num_objects
hit = [0] * self.num_objects
print("num_objects:", self.num_objects)
x = pupil_loc[0]
y = pupil_loc[1]
d = ((x - self.x_temp)**2 + (y - self.y_temp)**2)**0.5
print("x:", x, "y:", y, "d:", d)
self.current.update({"gaze_displacement": d})
self.x_temp = x
self.y_temp = y
for detection in detections:
index = self.object_detect.get_alt_names().index(detection[0]) + 1
# print(index, "--", detection[0])
bounds = detection[
2] # bounds: [0]bbox_x, [1]bbox_y, [2]bbox_w, [3]bbox_h
dist = ((x - bounds[0])**2 + (y - bounds[1])**2)**0.5
dial = ((bounds[2] / 2)**2 + (bounds[3] / 2)**2)**0.5
norm_dist = dist / dial
# self.current.update({detection[0] + "_distance": norm_dist})
self.current.update({REV_LABELS[index] + "_distance": norm_dist})
x1 = bounds[0] - bounds[2] / 2
x2 = bounds[0] + bounds[2] / 2
y1 = bounds[1] - bounds[3] / 2
y2 = bounds[1] + bounds[3] / 2
xin = (x1 <= x <= x2)
yin = (y1 <= y <= y2)
# print("bbox_x:", bounds[0], "bbox_y:", bounds[1], "bbox_w:", bounds[2], "bbox_h:", bounds[3])
if xin and yin:
hit[index - 1] = 1
# self.current.update({"is_" + detection[0]: True})
self.current.update({"is_" + REV_LABELS[index]: True})
break
feature = self.current
feature.update(self.previous)
feature.update(self.pprev)
features = [feature]
# training
tagger = pycrfsuite.Tagger()
tagger.open('exp_4')
pred = tagger.tag(features)[0]
logger.info("features - {}".format(features))
logger.info("pred - {}".format(pred))
print("features:", features)
print("pred:", pred)
# logger.info("CRF features - {}".format(self.nex))
# output[LABELS[pred]] = 1
# logger.info("hit scan output - {}".format(hit))
# logger.info("CRF output - {}".format(output))
# print("hit scan output. - {}".format(hit))
# print("CRF output - {}".format(output))
self.pprev.update({
"pprev_cards_distance":
self.previous["previous_cards_distance"],
"pprev_dice_distance":
self.previous["previous_dice_distance"],
"pprev_key_distance":
self.previous["previous_key_distance"],
"pprev_map_distance":
self.previous["previous_map_distance"],
"pprev_ball_distance":
self.previous["previous_ball_distance"],
"pprev_face_distance":
self.previous["previous_face_distance"],
"pprev_is_cards":
self.previous["previous_is_cards"],
"pprev_is_dice":
self.previous["previous_is_dice"],
"pprev_is_key":
self.previous["previous_is_key"],
"pprev_is_map":
self.previous["previous_is_map"],
"pprev_is_ball":
self.previous["previous_is_ball"],
"pprev_is_face":
self.previous["previous_is_face"],
"pprev_gaze_displacement":
self.previous["previous_gaze_displacement"]
})
self.previous.update({
"previous_cards_distance":
self.current["cards_distance"],
"previous_dice_distance":
self.current["dice_distance"],
"previous_key_distance":
self.current["key_distance"],
"previous_map_distance":
self.current["map_distance"],
"previous_ball_distance":
self.current["ball_distance"],
"previous_face_distance":
self.current["face_distance"],
"previous_is_cards":
self.current["is_cards"],
"previous_is_dice":
self.current["is_dice"],
"previous_is_key":
self.current["is_key"],
"previous_is_map":
self.current["is_map"],
"previous_is_ball":
self.current["is_ball"],
"previous_is_face":
self.current["is_face"],
"previous_gaze_displacement":
self.current["gaze_displacement"]
})
self.current.update({
"cards_distance": 10000,
"dice_distance": 10000,
"key_distance": 10000,
"map_distance": 10000,
"ball_distance": 10000,
"face_distance": 10000,
"is_cards": False,
"is_dice": False,
"is_key": False,
"is_map": False,
"is_ball": False,
"is_face": False,
"gaze_displacement": 0
})
# return output, hit
return pred, hit
def do_some_stuff_together(self, common_data_proxy_1, common_data_proxy_2):
logger.info("Starting Do_Stuff_Together...")
# gaze_detect = GazeSeq() #Initialize the object
while True:
common_data_1 = common_data_proxy_1.get_values()
common_data_2 = common_data_proxy_2.get_values()
if common_data_1[0] is None or common_data_2[0] is None or (
common_data_1[2] == self.last_frame_index_1
and common_data_2[2] == self.last_frame_index_2):
continue
# logger.info("Glass_1 Frame - {}, Glass_2 Frame - {}, Glass_1 Timestamp - {}, Glass_2 Timestamp - {}".format(
# common_data_1[2], common_data_2[2], common_data_1[1], common_data_2[1]))
# logger.info("Glass_1 output - {}, Glass_2 output - {}".format(common_data_1[3], common_data_2[3]))
common_look_time = int(time.time() * 1000)
logger_run_length.info(
";time-{};Glass1_hit_scan_output-{};Glass2_hit_scan_output-{};Glass1_run_length_output-{};Glass2_run_length_output-{};Frame1-{};Frame2-{}"
.format(common_look_time, common_data_1[4], common_data_2[4],
common_data_1[3], common_data_2[3], common_data_1[2],
common_data_2[2]))
if common_data_1[3] == common_data_2[3] and sum(
common_data_1[3]) != 0:
if not self.prev_state:
self.prev_state = True
obj_idx = np.argmax(common_data_1[3])
# gaze_status = seq_detect(obj_idx, self.triad_state)
# logger2.info("Look Detected;{}:{} Gaze Status {}".format(self.objects[obj_idx], common_look_time, gaze_status))
if self.triad_state == 'reset': # reset
if obj_idx == 5: # + face
print("Face Detected")
logger2.info("Face Detected;{}:{} ".format(
self.objects[obj_idx], common_look_time))
self.triad_state = "face"
play_sound()
else:
continue
elif self.triad_state == 'face': # face
if obj_idx == 5: # + face
continue
else: # + object
print("Face and Object Detected")
logger2.info(
"Face and Object Detected;{}:{} ".format(
self.objects[obj_idx], common_look_time))
self.triad_state = 'face_object'
play_sound()
elif self.state == 'face_object': # face-object
if obj_idx == 5: # + face
print("Gaze Triad Complete")
logger2.info("Gaze Triad Complete;{}:{} ".format(
self.objects[obj_idx], common_look_time))
self.triad_state = 'face'
play_sound()
# elif object_detected == self.obj_indx: # + object
# continue
else:
print("Triad incomplete")
logger2.info("Triad incomplete ".format(
self.objects[obj_idx], common_look_time))
self.triad_state = 'reset'
# play_sound()
else:
self.prev_state = False
self.last_frame_index_1 = common_data_1[2]
self.last_frame_index_2 = common_data_2[2]