def main():
default_model_dir = './models'
#default_model = 'mobilenet_ssd_v2_coco_quant_postprocess_edgetpu.tflite'
default_model = 'mobilenet_ssd_v2_face_quant_postprocess_edgetpu.tflite'
default_labels = 'coco_labels.txt'
parser = argparse.ArgumentParser(formatter_class=argparse.ArgumentDefaultsHelpFormatter)
parser.add_argument('-m', '--model', help='File path of .tflite file.', default=os.path.join(default_model_dir,default_model))
parser.add_argument('-l', '--labels', help='File path of labels file.',default=os.path.join(default_model_dir,default_labels))
parser.add_argument('-t', '--threshold', type=float, default=0.1, help='Score threshold for detected objects.')
parser.add_argument('--camera_idx', type=int, help='Index of which video source to use. ', default = 1)
args = parser.parse_args()
labels = load_labels(args.labels) if args.labels else {}
interpreter = make_interpreter(args.model)
interpreter.allocate_tensors()
#open camera
cap = cv2.VideoCapture(args.camera_idx)
while cap.isOpened():
ret, frame = cap.read()
if not ret:
break
cv2_im = frame
cv2_im_rgb = cv2.cvtColor(cv2_im, cv2.COLOR_BGR2RGB)
pil_im = Image.fromarray(cv2_im_rgb)
#common.set_input(interpreter, pil_im)
scale = detect.set_input(interpreter,pil_im.size,lambda size: pil_im.resize(size, Image.ANTIALIAS))
interpreter.invoke()
#print(scale)
objs = detect.get_output(interpreter, args.threshold, scale)
#print(objs)
#draw_objects(ImageDraw.Draw(pil_im), objs, labels)
cv2_im = append_objs_to_img(cv2_im, objs, labels)
cv2.imshow('frame', cv2_im)
if cv2.waitKey(1) & 0xFF == ord('q'):
break
cap.release()
cv2.destroyAllWindows()
while True:
#ret, frame = cap.read()
frame = cap.read()
# if not ret:
# break
(h, w) = frame.shape[:2]
vision_middle = (int(w / 2), int(h / 2))
cv2.circle(frame, vision_middle, target_circle_size * 2,
(255, 0, 255), 2)
#Detect the object and get the target middle
cv2_im = frame
cv2_im_rgb = cv2.cvtColor(cv2_im, cv2.COLOR_BGR2RGB)
pil_im = Image.fromarray(cv2_im_rgb)
#common.set_input(interpreter, pil_im)
scale = detect.set_input(
interpreter, pil_im.size,
lambda size: pil_im.resize(size, Image.ANTIALIAS))
interpreter.invoke()
#print(scale)
objs = detect.get_output(interpreter, args.threshold, scale)
#print(objs)
#draw_objects(ImageDraw.Draw(pil_im), objs, labels)
cv2_im = append_objs_to_img(cv2_im, objs, labels)
#the robot will only target the first object (human/face)
if objs:
x0, y0, x1, y1 = list(objs[0].bbox)
target_middle = (int(x0 + (x1 - x0) / 2),
int(y0 + (y1 - y0) / 2))
#print("target_middle: ",target_middle)
success_flag = True
cv2.circle(cv2_im, target_middle, int(target_circle_size / 2),
def cam(cap_id,e,target_middle_shared,delta_shared,success_flag_shared,vision_middle_shared,h_shared,w_shared,elapsed_time,beer_time):
args = utilities.parseConnectionArguments()
# Create connection to the device and get the router
with utilities.DeviceConnection.createTcpConnection(args) as router:
#Start EdgeTPU
default_model_dir = './models'
#default_model = 'mobilenet_ssd_v2_coco_quant_postprocess_edgetpu.tflite'
default_model = 'mobilenet_ssd_v2_face_quant_postprocess_edgetpu.tflite'
default_labels = 'coco_labels.txt'
default_threshold = 0.1
# parser = argparse.ArgumentParser(formatter_class=argparse.ArgumentDefaultsHelpFormatter)
# parser.add_argument('-m', '--model', help='File path of .tflite file.', default=os.path.join(default_model_dir,default_model))
# parser.add_argument('-l', '--labels', help='File path of labels file.',default=os.path.join(default_model_dir,default_labels))
# parser.add_argument('-t', '--threshold', type=float, default=default_threshold, help='Score threshold for detected objects.')
# #parser.add_argument('--camera_idx', type=int, help='Index of which video source to use. ', default = 1)
# args = parser.parse_args()
labels = load_labels(os.path.join(default_model_dir,default_labels))
interpreter = make_interpreter(os.path.join(default_model_dir,default_model))
interpreter.allocate_tensors()
cap = WebcamVideoStream(cap_id).start()
old_face_position=0
face_gone_timer=time.time()
success_flag=False
while True:
frame = cap.read()
(h, w) = frame.shape[:2]
vision_middle=(int(w/2),int(h/2)-150)
cv2.circle(frame,vision_middle,target_circle_size*2,(255,0,255),2)
#Detect the object and get the target middle
cv2_im = frame
cv2_im_rgb = cv2.cvtColor(cv2_im, cv2.COLOR_BGR2RGB)
pil_im = Image.fromarray(cv2_im_rgb)
#common.set_input(interpreter, pil_im)
scale = detect.set_input(interpreter,pil_im.size,lambda size: pil_im.resize(size, Image.ANTIALIAS))
interpreter.invoke()
#print(scale)
objs = detect.get_output(interpreter, default_threshold, scale)
#print(objs)
#draw_objects(ImageDraw.Draw(pil_im), objs, labels)
cv2_im = append_objs_to_img(cv2_im, objs, labels)
if testing == True:
#the robot will only target the first object (human/face)
faces_list=[]
if objs:
target_middle=0
for face_id in range(len(objs)):
x0, y0, x1, y1 = list(objs[face_id].bbox)
target_middle_temp=(int(x0+(x1-x0)/2),int(y0+(y1-y0)/2))
cv2.circle(cv2_im,target_middle_temp,int(target_circle_size/2),(0, 255, 0),2)
faces_list.append([face_id,target_middle_temp])
# print("old_face_position: ",old_face_position)
# print("faces_list[face_id][1]: ",faces_list[face_id][1])
if old_face_position==0: #never seen a face before, first assignment:
old_face_position=target_middle_temp
target_middle=target_middle_temp
#success_flag=True
elif ((old_face_position[0]-(x1-x0)<faces_list[face_id][1][0]<old_face_position[0]+(x1-x0)) and
(old_face_position[1]-(x1-x0)<faces_list[face_id][1][1]<old_face_position[1]+(x1-x0))):
target_middle=(int(x0+(x1-x0)/2),int(y0+(y1-y0)/2))
old_face_position=target_middle
success_flag=True
face_gone_timer=time.time()
# if time.time()-face_gone_timer>2 and success_flag==False: #if the face left the image, after 2 second set old_face_position=0
# old_face_position=0
#cv2_im = age_gender(cv2_im,face_boxes=objs)
else:
success_flag=False
if success_flag==False:
#human/face was not detected
#base.Stop()
target_middle=vision_middle
else:
if objs:
x0, y0, x1, y1 = list(objs[0].bbox)
target_middle=(int(x0+(x1-x0)/2),int(y0+(y1-y0)/2))
#print("target_middle: ",target_middle)
success_flag=True
cv2.circle(cv2_im,target_middle,int(target_circle_size/2),(0, 255, 0),2)
else:
success_flag=False
if success_flag==False:
#human/face was not detected
#base.Stop()
target_middle=vision_middle
#draw the delta
delta=[0,0]
delta[0]=target_middle[0]-vision_middle[0]
delta[1]=target_middle[1]-vision_middle[1]
cv2.line(cv2_im,vision_middle,(vision_middle[0]+delta[0],vision_middle[1]+delta[1]),(0,255,0),1)
#what needs to be given to othe process:
#target_middle,delta,success_flag
target_middle_shared[0]=target_middle[0]
target_middle_shared[1]=target_middle[1]
delta_shared[0]=delta[0]
delta_shared[1]=delta[1]
success_flag_shared.value=success_flag
vision_middle_shared[0]=vision_middle[0]
vision_middle_shared[1]=vision_middle[1]
h_shared.value=h
w_shared.value=w
if elapsed_time.value>1:
cv2_im=functions_.draw_loading_circle(
img=cv2_im, radius=target_circle_size, center=vision_middle,
elapsed_time=elapsed_time.value-1, end_time=beer_time-1)
if elapsed_time.value>beer_time:
#write text
font = cv2.FONT_HERSHEY_SIMPLEX
org = (vision_middle[0]-300,vision_middle[1]-100)
fontScale = 2
color = (255, 255, 255)
thickness = 3
cv2_im = cv2.putText(cv2_im, 'Have a beer, buddy!', org, font,
fontScale, color, thickness, cv2.LINE_AA)
cv2.imshow("robot camera", frame)
if cv2.waitKey(1) & 0xFF == ord('q') or e.is_set():
break
cap.stop()
cv2.destroyWindow(str(cap_id))
return
def cam(cap_id, e, target_middle_shared, delta_shared, success_flag_shared,
vision_middle_shared, h_shared, w_shared, elapsed_time, beer_time):
global args
if not args:
args = utilities.parseConnectionArguments()
# Create connection to the device and get the router
with utilities.DeviceConnection.createTcpConnection(args) as router:
#Start EdgeTPU
default_model_dir = './models'
#default_model = 'mobilenet_ssd_v2_coco_quant_postprocess_edgetpu.tflite'
default_model = 'mobilenet_ssd_v2_face_quant_postprocess_edgetpu.tflite'
default_labels = 'coco_labels.txt'
default_threshold = 0.1
labels = load_labels(os.path.join(default_model_dir, default_labels))
interpreter = make_interpreter(
os.path.join(default_model_dir, default_model))
interpreter.allocate_tensors()
cap = WebcamVideoStream(cap_id).start() #multithreading Video Capture
#Window Settings
window_name = "Robot_Camera"
size_wh = (1920 - 300, 1080)
location_xy = (0, 0)
cv2.namedWindow(window_name,
cv2.WINDOW_KEEPRATIO | cv2.WINDOW_GUI_NORMAL)
cv2.resizeWindow(window_name, *size_wh)
# cv2.namedWindow(window_name, cv2.WINDOW_AUTOSIZE)
cv2.moveWindow(window_name, *location_xy)
# cv2.imshow(window_name, image)
success_flag = False
t0 = time.time()
auto_focus_time = 7
device_manager = DeviceManagerClient(router)
vision_config = VisionConfigClient(router)
vision_device_id = vision_action.example_vision_get_device_id(
device_manager)
#Disable Auto-Focus
vision_action.autofocus_action(vision_config,
vision_device_id,
action_id=1)
while True:
frame = cap.read()
(h, w) = frame.shape[:2]
vision_middle = (int(w / 2), int(h / 2) - 150)
cv2.circle(frame, vision_middle, target_circle_size * 2,
(255, 0, 255), 2)
#Detect the object and get the target middle
cv2_im = frame
cv2_im_rgb = cv2.cvtColor(cv2_im, cv2.COLOR_BGR2RGB)
pil_im = Image.fromarray(cv2_im_rgb)
scale = detect.set_input(
interpreter, pil_im.size,
lambda size: pil_im.resize(size, Image.ANTIALIAS))
interpreter.invoke()
objs = detect.get_output(interpreter, default_threshold, scale)
cv2_im = append_objs_to_img(cv2_im, objs, labels)
if objs:
#find the biggest bounding box
area = 0
for i0 in range(len(objs)):
x0, y0, x1, y1 = list(objs[i0].bbox)
area_temp = (x1 - x0) * (y1 - y0)
if area_temp > area:
area = area_temp
biggest_area_index = i0
x0, y0, x1, y1 = list(objs[biggest_area_index].bbox)
target_middle = (int(x0 + (x1 - x0) / 2),
int(y0 + (y1 - y0) / 2))
success_flag = True
cv2.circle(cv2_im, target_middle, int(target_circle_size / 2),
(0, 255, 0), 2)
else:
success_flag = False
if success_flag == False:
#human/face was not detected
target_middle = vision_middle
if time.time() - t0 > auto_focus_time:
try: #sometimes when stream corrupted because of Schleifring, auotfocus leads to error
vision_action.autofocus_action(
vision_config,
vision_device_id,
action_id=6,
focus_point=target_middle_shared[:])
except:
None
t0 = time.time()
#draw the delta
delta = [0, 0]
delta[0] = target_middle[0] - vision_middle[0]
delta[1] = target_middle[1] - vision_middle[1]
cv2.line(
cv2_im, vision_middle,
(vision_middle[0] + delta[0], vision_middle[1] + delta[1]),
(0, 255, 0), 1)
#what needs to be given to the process:
target_middle_shared[0] = target_middle[0]
target_middle_shared[1] = target_middle[1]
delta_shared[0] = delta[0]
delta_shared[1] = delta[1]
success_flag_shared.value = success_flag
vision_middle_shared[0] = vision_middle[0]
vision_middle_shared[1] = vision_middle[1]
h_shared.value = h
w_shared.value = w
if elapsed_time.value > 1:
cv2_im = functions_.draw_loading_circle(
img=cv2_im,
radius=target_circle_size,
center=vision_middle,
elapsed_time=elapsed_time.value - 1,
end_time=beer_time - 1)
if elapsed_time.value > beer_time:
#write text
font = cv2.FONT_HERSHEY_SIMPLEX
org = (vision_middle[0] - 300, vision_middle[1] - 100)
fontScale = 2
color = (255, 255, 255)
thickness = 3
cv2_im = cv2.putText(cv2_im, 'Have a beer, buddy', org, font,
fontScale, color, thickness, cv2.LINE_AA)
#Call to high five
if elapsed_time.value == -1:
#write text
font = cv2.FONT_HERSHEY_SIMPLEX
org = (vision_middle[0] - 550, vision_middle[1])
fontScale = 4
color = (255, 255, 255)
thickness = 4
cv2_im = cv2.putText(cv2_im, ' High Five', org, font,
fontScale, color, thickness, cv2.LINE_AA)
if elapsed_time.value == -2:
#write text
font = cv2.FONT_HERSHEY_SIMPLEX
org = (vision_middle[0] - 550, vision_middle[1])
fontScale = 4
color = (255, 255, 255)
thickness = 4
cv2_im = cv2.putText(cv2_im, ' Good Night', org, font,
fontScale, color, thickness, cv2.LINE_AA)
cv2.imshow(window_name, frame)
if (cv2.waitKey(1) & 0xFF == ord('q')) or e.is_set():
break
cap.stop()
cv2.destroyWindow(str(cap_id))
return