class FrontEnd(object): """ Maintains the Tello display and moves it through the keyboard keys. Press escape key to quit. The controls are: - T: Takeoff - L: Land - Arrow keys: Forward, backward, left and right. - A and D: Counter clockwise and clockwise rotations - W and S: Up and down. """ def __init__(self): # Init pygame pygame.init() # Creat pygame window pygame.display.set_caption("Tello video stream") self.screen = pygame.display.set_mode([960, 720]) # Init Tello object that interacts with the Tello drone self.tello = Tello() # Drone velocities between -100~100 self.for_back_velocity = 0 self.left_right_velocity = 0 self.up_down_velocity = 0 self.yaw_velocity = 0 self.speed = 10 self.send_rc_control = False # create update timer pygame.time.set_timer(pygame.USEREVENT + 1, 50) def run(self): if not self.tello.connect(): print("Tello not connected") return if not self.tello.set_speed(self.speed): print("Not set speed to lowest possible") return # In case streaming is on. This happens when we quit this program without the escape key. if not self.tello.streamoff(): print("Could not stop video stream") return if not self.tello.streamon(): print("Could not start video stream") return frame_read = self.tello.get_frame_read() should_stop = False while not should_stop: for event in pygame.event.get(): if event.type == pygame.USEREVENT + 1: self.update() elif event.type == pygame.QUIT: should_stop = True elif event.type == pygame.KEYDOWN: if event.key == pygame.K_ESCAPE: should_stop = True else: self.keydown(event.key) elif event.type == pygame.KEYUP: self.keyup(event.key) if frame_read.stopped: frame_read.stop() break self.screen.fill([0, 0, 0]) frame = cv2.cvtColor(frame_read.frame, cv2.COLOR_BGR2RGB) frame = np.rot90(frame) frame = np.flipud(frame) frame = pygame.surfarray.make_surface(frame) self.screen.blit(frame, (0, 0)) pygame.display.update() time.sleep(1 / FPS) # Call it always before finishing. To deallocate resources. self.tello.end() def keydown(self, key): """ Update velocities based on key pressed Arguments: key: pygame key """ if key == pygame.K_UP: # set forward velocity self.for_back_velocity = S elif key == pygame.K_DOWN: # set backward velocity self.for_back_velocity = -S elif key == pygame.K_LEFT: # set left velocity self.left_right_velocity = -S elif key == pygame.K_RIGHT: # set right velocity self.left_right_velocity = S elif key == pygame.K_w: # set up velocity self.up_down_velocity = S elif key == pygame.K_s: # set down velocity self.up_down_velocity = -S elif key == pygame.K_a: # set yaw counter clockwise velocity self.yaw_velocity = -S elif key == pygame.K_d: # set yaw clockwise velocity self.yaw_velocity = S elif key == pygame.K_b: # get battery print(self.tello.get_battery()) elif key == pygame.K_c: # send command from terminal command = input( 'Enter command (speed?time?battery?tof?height?baro?temp?wifi?sn?sdk?altitude?)' ) print(self.tello.send_read_command(command)) elif key == pygame.K_1: print('flip right') self.tello.flip('r') elif key == pygame.K_2: print('flip left') self.tello.flip('l') elif key == pygame.K_3: print('flip forward') self.tello.flip('f') elif key == pygame.K_4: print('flip back') self.tello.flip('b') def keyup(self, key): """ Update velocities based on key released Arguments: key: pygame key """ if key == pygame.K_UP or key == pygame.K_DOWN: # set zero forward/backward velocity self.for_back_velocity = 0 elif key == pygame.K_LEFT or key == pygame.K_RIGHT: # set zero left/right velocity self.left_right_velocity = 0 elif key == pygame.K_w or key == pygame.K_s: # set zero up/down velocity self.up_down_velocity = 0 elif key == pygame.K_a or key == pygame.K_d: # set zero yaw velocity self.yaw_velocity = 0 elif key == pygame.K_t: # takeoff self.tello.takeoff() self.send_rc_control = True elif key == pygame.K_l: # land self.tello.land() self.send_rc_control = False def update(self): """ Update routine. Send velocities to Tello.""" if self.send_rc_control: self.tello.send_rc_control(self.left_right_velocity, self.for_back_velocity, self.up_down_velocity, self.yaw_velocity)
print("Rotating left") tello.rotate_clockwise(360) if _split[1] == "rotate_right" and takeoff == True: print('Received', data) print("Rotating left") tello.rotate_counter_clockwise(360) if _split[1] == "stop" and takeoff == True: print('Received', data) _move = (0, 0, 0, 0) print("Stopping", _move) tello.send_rc_control(int(_move[0]), int(_move[1]), int(_move[2]), int(_move[3])) if _split[1] == "move" and takeoff == True: print('Received', data) _move = (_split[2], _split[3], _split[4], _split[5]) print("Moving", _move) tello.send_rc_control(int(_move[0]), int(_move[1]), int(_move[2]), int(_move[3])) if _split[1] == "flip" and takeoff == True: print('Received', data) tello.flip(_split[2]) if _split[1] == "quit": break except Exception as e: print(e)
def main(): tello = Tello() tello.connect() vk_session = vk_api.VkApi(token='TOKEN') vk = vk_session.get_api() letsgo = VkKeyboard(one_time=True) letsgo.add_button('Начать', color=VkKeyboardColor.DEFAULT) keyboard_takeoff = VkKeyboard(one_time=True) keyboard_takeoff.add_button('Взлёт', color=VkKeyboardColor.POSITIVE) keyboard = VkKeyboard(one_time=True) keyboard.add_button('Вверх', color=VkKeyboardColor.PRIMARY) keyboard.add_button('Вниз', color=VkKeyboardColor.PRIMARY) keyboard.add_line() keyboard.add_button('Вперёд', color=VkKeyboardColor.PRIMARY) keyboard.add_line() keyboard.add_button('Влево', color=VkKeyboardColor.PRIMARY) keyboard.add_button('Флип', color=VkKeyboardColor.NEGATIVE) keyboard.add_button('Вправо', color=VkKeyboardColor.PRIMARY) keyboard.add_line() keyboard.add_button('Назад', color=VkKeyboardColor.PRIMARY) keyboard.add_line() keyboard.add_button('Проверка', color=VkKeyboardColor.DEFAULT) keyboard.add_line() keyboard.add_button('Посадка', color=VkKeyboardColor.NEGATIVE) flip_keyboard = VkKeyboard(one_time=False) flip_keyboard.add_button('Флип вперёд', color=VkKeyboardColor.DEFAULT) flip_keyboard.add_line() flip_keyboard.add_button('Флип влево', color=VkKeyboardColor.DEFAULT) flip_keyboard.add_button('Флип вправо', color=VkKeyboardColor.DEFAULT) flip_keyboard.add_line() flip_keyboard.add_button('Флип назад', color=VkKeyboardColor.DEFAULT) flip_keyboard.add_line() flip_keyboard.add_button('Доступность флипа', color=VkKeyboardColor.NEGATIVE) flip_keyboard.add_line() flip_keyboard.add_button('Вернуться в меню управления', color=VkKeyboardColor.NEGATIVE) mainmenu_keyboard = VkKeyboard(one_time=True) mainmenu_keyboard.add_button('Начать', color=VkKeyboardColor.DEFAULT) mainmenu_keyboard.add_line() mainmenu_keyboard.add_button('Конец работы', color=VkKeyboardColor.DEFAULT) longpoll = VkLongPoll(vk_session) for event in longpoll.listen(): if event.type == VkEventType.MESSAGE_NEW and event.to_me and event.text: if event.text.lower() == 'флип': vk.messages.send( # Отправляем сообщение user_id=event.user_id, keyboard=flip_keyboard.get_keyboard(), message='Переход в меню флипов') if event.text.lower() == 'начать': vk.messages.send( # Отправляем сообщение user_id=event.user_id, keyboard=keyboard_takeoff.get_keyboard(), message='Взлет?') if event.text.lower() == 'конец работы': vk.messages.send( # Отправляем сообщение user_id=event.user_id, message='Конец работы') if event.text.lower() == 'взлёт': vk.messages.send( # Отправляем сообщение user_id=event.user_id, keyboard=keyboard.get_keyboard(), message='Куда летим?') tello.takeoff() print('Взлёт') if event.text.lower() == 'посадка': vk.messages.send( # Отправляем сообщение user_id=event.user_id, keyboard=mainmenu_keyboard.get_keyboard(), message='Посадка') tello.land() print('Посадка') if event.text.lower() == 'вперёд': vk.messages.send( # Отправляем сообщение user_id=event.user_id, keyboard=keyboard.get_keyboard(), message='Полёт вперёд') tello.move_forward(100) print('Вперёд') if event.text.lower() == 'влево': vk.messages.send( # Отправляем сообщение user_id=event.user_id, keyboard=keyboard.get_keyboard(), message='Полёт влево') tello.move_left(100) print('Влево') if event.text.lower() == 'вправо': vk.messages.send( # Отправляем сообщение user_id=event.user_id, keyboard=keyboard.get_keyboard(), message='Полёт вправо') tello.move_right(100) print('Вправо') if event.text.lower() == 'назад': vk.messages.send( # Отправляем сообщение user_id=event.user_id, keyboard=keyboard.get_keyboard(), message='Полёт назад') tello.move_back(100) print('Назад') if event.text.lower() == 'проверка': x = tello.get_battery() vk.messages.send( # Отправляем сообщение user_id=event.user_id, keyboard=keyboard.get_keyboard(), message='Заряд баттареи ' + x) print('Проверка') if event.text.lower() == 'вверх': vk.messages.send( # Отправляем сообщение user_id=event.user_id, keyboard=keyboard.get_keyboard(), message='Полёт вверх') tello.move_up(20) print('Вверх') if event.text.lower() == 'вниз': vk.messages.send( # Отправляем сообщение user_id=event.user_id, keyboard=keyboard.get_keyboard(), message='Полёт вниз') tello.move_down(20) print('Вниз') if event.text.lower() == 'флип назад': vk.messages.send( # Отправляем сообщение user_id=event.user_id, keyboard=flip_keyboard.get_keyboard(), message='Флип назад') tello.flip('b') print('Флип назад') if event.text.lower() == 'флип вперёд': vk.messages.send( # Отправляем сообщение user_id=event.user_id, keyboard=flip_keyboard.get_keyboard(), message='Флип вперёд') tello.flip('f') print('Флип вперёд') if event.text.lower() == 'флип влево': vk.messages.send( # Отправляем сообщение user_id=event.user_id, keyboard=flip_keyboard.get_keyboard(), message='Флип влево') tello.flip('l') print('Флип влево') if event.text.lower() == 'флип вправо': vk.messages.send( # Отправляем сообщение user_id=event.user_id, keyboard=flip_keyboard.get_keyboard(), message='Флип вправо') tello.flip('r') print('Флип вправо') if event.text.lower() == 'доступность флипа': x = tello.get_battery() if x == 'ok': vk.messages.send(user_id=event.user_id, keyboard=flip_keyboard.get_keyboard(), message='Нажмите ещё раз') else: if int(x) > 50: vk.messages.send(user_id=event.user_id, keyboard=flip_keyboard.get_keyboard(), message='Заряд баттареи ' + x + 'Флип доступен') else: vk.messages.send(user_id=event.user_id, keyboard=flip_keyboard.get_keyboard(), message='Заряд баттареи ' + x + 'Флип недоступен') if event.text.lower() == 'вернуться в меню управления': vk.messages.send( # Отправляем сообщение user_id=event.user_id, keyboard=keyboard.get_keyboard(), message='Возврат в меню управления') print('Возврат в меню управления') tello.end()
class Drone: SEARCH_TIMEOUT = 10 * 1000 def __init__(self): pygame.init() pygame.display.set_caption("Tello video stream") self.screen = pygame.display.set_mode([960, 720]) self.tello = Tello() # Drone velocities between -100~100 self.for_back_velocity = 0 self.left_right_velocity = 0 self.up_down_velocity = 0 self.yaw_velocity = 0 self.speed = 10 self.command_queue = [] self.shutdown = False self.should_stop = False # create update timer pygame.time.set_timer(USEREVENT + 1, 50) self.tello.state = State.initializing def flip(self, direction='b'): for event in pygame.event.get(): if event.type == GameEvents.VIDEO_EVENT.value: log.info('Flipping') self.tello.state = State.flipping self.tello.flip(direction) log.info('Idle') self.tello.state = State.idle def fly(self): if not self.tello.connect(): print("Tello not connected") return if not self.tello.set_speed(self.speed): print("Not set speed to lowest possible") return # In case streaming is on. This happens when we quit this program without the escape key. if not self.tello.streamoff(): print("Could not stop video stream") return if not self.tello.streamon(): print("Could not start video stream") return frame_read = self.tello.get_frame_read() time.sleep(2) self.tello.takeoff() if self.tello.state is State.initializing: self.tello.state = State.yawing self.yaw_velocity = 100 while not self.should_stop: for event in pygame.event.get(): if event.type == USEREVENT + 1: if (self.tello.state is State.initializing or self.tello.state is State.yawing or self.tello.state is State.idle): self.update() if event.type == GameEvents.VIDEO_EVENT.value: if self.tello.state is State.yawing: self.yaw_velocity = 0 self.tello.state = State.idle elif self.tello.state is State.idle: self.flip() elif event.type == KEYDOWN: self.should_stop = True # if event.key == K_ESCAPE: # log.info('Landing') # self.tello.land() # break if frame_read.stopped: frame_read.stop() break self.screen.fill([0, 0, 0]) frame = cv2.cvtColor(frame_read.frame, cv2.COLOR_BGR2RGB) frame = np.rot90(frame) frame = np.flipud(frame) frame = pygame.surfarray.make_surface(frame) self.screen.blit(frame, (0, 0)) pygame.display.update() time.sleep(1 / FPS) self.tello.end() def update(self): """ Update routine. Send velocities to Tello.""" if self.tello.state is not State.flipping: self.tello.send_rc_control( self.left_right_velocity, self.for_back_velocity, self.up_down_velocity, self.yaw_velocity)
elif _cmd == "move_right": # left_right_velocity, forward_backward_velocity, up_down_velocity, yaw_velocity command = f"move,{-SPEED},0,0,0" elif _cmd == "rotate_left": # left_right_velocity, forward_backward_velocity, up_down_velocity, yaw_velocity command = f"rotate_left" elif _cmd == "rotate_right": # left_right_velocity, forward_backward_velocity, up_down_velocity, yaw_velocity command = f"rotate_right" elif _cmd == "flip_right": command = f"flip,r" if takeoff == True: tello.flip('r') elif _cmd == "flip_left": command = f"flip,l" if takeoff == True: tello.flip('l') elif _cmd == "flip_forward": command = f"flip,f" if takeoff == True: tello.flip('f') elif _cmd == "flip_backward": command = f"flip,b" if takeoff == True: tello.flip('b')
def thread1(): detection_buffer = 10 # buffer between when the drone can respond again to seeing a person detect_gate = False # logic gate before a drone will turn based on seeing someone detection_time = float( 'inf') #Getting around initializing error det_label = 100 print("Running Thread 1") footage = tello.get_frame_read() # print("I got a frame!") # cv2.imshow("DetectionResults", footage.frame) #no need to have it up here #raw_key = cv2.waitKey(1) is_async_mode = False wait_key_code = 0 # ----------------------------------------- 5. Loading model to the plugin ----------------------------------------- log.info("Loading model to the plugin") exec_net = plugin.load(network=net, num_requests=2) cur_request_id = 0 next_request_id = 1 render_time = 0 parsing_time = 0 # ----------------------------------------------- 6. Doing inference ----------------------------------------------- print( "To close the application, press 'CTRL+C' or any key with focus on the output window" ) print("Here's the type of footage.drone: " + str(type(footage.frame))) if is_async_mode: ret = True #footage.frame next_frame = footage.frame # In cap.read its 91600 size else: ret = True #footage.frame frame = footage.frame # print ("footage.frame type is: " + str(type(footage.drone))) print("Output Representation: " + str(footage.frame)) print("Here's the size of the array: " + str(footage.frame.size)) # if not ret: #commented due to truth value # print("RET VALUE IS FALSE!") # break if is_async_mode: request_id = next_request_id in_frame = cv2.resize(next_frame, (w, h)) else: request_id = cur_request_id in_frame = cv2.resize(frame, (w, h)) # resize input_frame to network size in_frame = in_frame.transpose( (2, 0, 1)) # Change data layout from HWC to CHW in_frame = in_frame.reshape((n, c, h, w)) #It looks like I copied it twice # in_frame = in_frame.transpose((2, 0, 1)) # Change data layout from HWC to CHW # in_frame = in_frame.reshape((n, c, h, w)) # Start inference start_time = time() exec_net.start_async(request_id=request_id, inputs={input_blob: in_frame}) det_time = time() - start_time # Collecting object detection results objects = list() if exec_net.requests[cur_request_id].wait(-1) == 0: output = exec_net.requests[cur_request_id].outputs start_time = time() for layer_name, out_blob in output.items(): layer_params = YoloV3Params( net.layers[layer_name].params, out_blob.shape[2]) log.info("Layer {} parameters: ".format(layer_name)) layer_params.log_params() objects += parse_yolo_region(out_blob, in_frame.shape[2:], frame.shape[:-1], layer_params, args.prob_threshold) parsing_time = time() - start_time # Filtering overlapping boxes with respect to the --iou_threshold CLI parameter for i in range(len(objects)): if objects[i]['confidence'] == 0: continue for j in range(i + 1, len(objects)): if intersection_over_union( objects[i], objects[j]) > args.iou_threshold: objects[j]['confidence'] = 0 # Drawing objects with respect to the --prob_threshold CLI parameter objects = [ obj for obj in objects if obj['confidence'] >= args.prob_threshold ] if len(objects) and args.raw_output_message: log.info("\nDetected boxes for batch {}:".format(1)) log.info( " Class ID | Confidence | XMIN | YMIN | XMAX | YMAX | COLOR " ) origin_im_size = frame.shape[:-1] for obj in objects: # Validation bbox of detected object if obj['xmax'] > origin_im_size[1] or obj[ 'ymax'] > origin_im_size[0] or obj[ 'xmin'] < 0 or obj['ymin'] < 0: continue color = (int(min(obj['class_id'] * 12.5, 255)), min(obj['class_id'] * 7, 255), min(obj['class_id'] * 5, 255)) det_label = labels_map[obj['class_id']] if labels_map and len(labels_map) >= obj['class_id'] else \ str(obj['class_id']) if det_label == 0: print("Configuring tello Drone") tello = Tello() if not tello.connect(): print("Tello not connected") return if not tello.streamon(): print("Could not start video stream") return # tello.takeoff() #Start the drone flight with takeoff (I'm putting a 3 minute limit before the drone comes down) if not tello.takeoff(): print("Takeoff failed") log.info("Takeoff should have occured.") if args.raw_output_message: log.info( "{:^9} | {:10f} | {:4} | {:4} | {:4} | {:4} | {} ". format(det_label, obj['confidence'], obj['xmin'], obj['ymin'], obj['xmax'], obj['ymax'], color)) cv2.rectangle(frame, (obj['xmin'], obj['ymin']), (obj['xmax'], obj['ymax']), color, 2) cv2.putText( frame, "#" + det_label + ' ' + str(round(obj['confidence'] * 100, 1)) + ' %', (obj['xmin'], obj['ymin'] - 7), cv2.FONT_HERSHEY_COMPLEX, 0.6, color, 1) print("YOLOV3 IS DETECTING OBJECT: " + str(coco_labels[int(det_label)])) if (int(det_label) == 0): detection_time = time() #if int(det_label) == 80: # detection_time = time() # print("seeing nada") if (time() - detection_time) > detection_buffer: detect_gate = True #detect_gate = False #person not detected.. classification id 80 if int(det_label) == 0 and detect_gate: detection_time = time() tello.rotate_clockwise( 90) #Current reaction to seeing a drone detect_gate = False str1 = str(coco_labels[int(det_label)]) str2 = 'person' if str1 == str2: print("sees person and rotate_clockwise") tello.flip('f') #tello.rotate_clockwise(90) # Draw performance stats over frame inf_time_message = "Inference time: N\A for async mode" if is_async_mode else \ "Inference time: {:.3f} ms".format(det_time * 1e3) render_time_message = "OpenCV rendering time: {:.3f} ms".format( render_time * 1e3) async_mode_message = "Async mode is on. Processing request {}".format(cur_request_id) if is_async_mode else \ "Async mode is off. Processing request {}".format(cur_request_id) parsing_message = "YOLO parsing time is {:.3f}".format( parsing_time * 1e3) cv2.putText(frame, inf_time_message, (15, 15), cv2.FONT_HERSHEY_COMPLEX, 0.5, (200, 10, 10), 1) cv2.putText(frame, render_time_message, (15, 45), cv2.FONT_HERSHEY_COMPLEX, 0.5, (10, 10, 200), 1) cv2.putText(frame, async_mode_message, (10, int(origin_im_size[0] - 20)), cv2.FONT_HERSHEY_COMPLEX, 0.5, (10, 10, 200), 1) cv2.putText(frame, parsing_message, (15, 30), cv2.FONT_HERSHEY_COMPLEX, 0.5, (10, 10, 200), 1) if is_async_mode: cur_request_id, next_request_id = next_request_id, cur_request_id frame = next_frame key = cv2.waitKey(wait_key_code) # Tab key # if key == 27: # break # ESC key if key == 9: exec_net.requests[cur_request_id].wait() is_async_mode = not is_async_mode log.info("Switched to {} mode".format( "async" if is_async_mode else "sync"))
def main(): args = build_argparser().parse_args() model_xml = args.model model_bin = os.path.splitext(model_xml)[0] + ".bin" # ------------- 1. Plugin initialization for specified device and load extensions library if specified ------------- plugin = IEPlugin(device=args.device, plugin_dirs=args.plugin_dir) if args.cpu_extension and 'CPU' in args.device: plugin.add_cpu_extension(args.cpu_extension) # -------------------- 2. Reading the IR generated by the Model Optimizer (.xml and .bin files) -------------------- log.info("Loading network files:\n\t{}\n\t{}".format(model_xml, model_bin)) net = IENetwork(model=model_xml, weights=model_bin) # ---------------------------------- 3. Load CPU extension for support specific layer ------------------------------ if plugin.device == "CPU": supported_layers = plugin.get_supported_layers(net) not_supported_layers = [ l for l in net.layers.keys() if l not in supported_layers ] if len(not_supported_layers) != 0: log.error( "Following layers are not supported by the plugin for specified device {}:\n {}" .format(plugin.device, ', '.join(not_supported_layers))) log.error( "Please try to specify cpu extensions library path in sample's command line parameters using -l " "or --cpu_extension command line argument") sys.exit(1) assert len(net.inputs.keys( )) == 1, "Sample supports only YOLO V3 based single input topologies" assert len( net.outputs ) == 3, "Sample supports only YOLO V3 based triple output topologies" # ---------------------------------------------- 4. Preparing inputs ----------------------------------------------- log.info("Preparing inputs") input_blob = next(iter(net.inputs)) # Defaulf batch_size is 1 net.batch_size = 1 # Read and pre-process input images n, c, h, w = net.inputs[input_blob].shape if args.labels: with open(args.labels, 'r') as f: labels_map = [x.strip() for x in f] else: labels_map = None input_stream = 0 if args.input == "cam" else args.input is_async_mode = True if (args.input != "drone"): cap = cv2.VideoCapture(input_stream) # Drone insertion number_input_frames = int(cap.get(cv2.CAP_PROP_FRAME_COUNT)) number_input_frames = 1 if number_input_frames != -1 and number_input_frames < 0 else number_input_frames wait_key_code = 1 # Number of frames in picture is 1 and this will be read in cycle. Sync mode is default value for this case if number_input_frames != 1: ret, frame = cap.read() #frames need to be taken from def in tello # print("Here's the type of cap.read(): " + str(type(cap.read()))) # print("Here's the raw read: " + str(cap.read())) # print ("Debugging Drone cam Array size is: " + str(ret.size)) else: is_async_mode = False wait_key_code = 0 # ----------------------------------------- 5. Loading model to the plugin ----------------------------------------- log.info("Loading model to the plugin") exec_net = plugin.load(network=net, num_requests=2) cur_request_id = 0 next_request_id = 1 render_time = 0 parsing_time = 0 # ----------------------------------------------- 6. Doing inference ----------------------------------------------- print( "To close the application, press 'CTRL+C' or any key with focus on the output window" ) while cap.isOpened(): # Here is the first asynchronous point: in the Async mode, we capture frame to populate the NEXT infer request # in the regular mode, we capture frame to the CURRENT infer request if is_async_mode: ret, next_frame = cap.read() else: ret, frame = cap.read() print("Here's the size of the frame array: " + str(frame.size)) if not ret: break if is_async_mode: request_id = next_request_id in_frame = cv2.resize(next_frame, (w, h)) else: request_id = cur_request_id in_frame = cv2.resize(frame, (w, h)) # resize input_frame to network size in_frame = in_frame.transpose( (2, 0, 1)) # Change data layout from HWC to CHW in_frame = in_frame.reshape((n, c, h, w)) # Start inference start_time = time() exec_net.start_async(request_id=request_id, inputs={input_blob: in_frame}) det_time = time() - start_time # Collecting object detection results objects = list() if exec_net.requests[cur_request_id].wait(-1) == 0: output = exec_net.requests[cur_request_id].outputs start_time = time() for layer_name, out_blob in output.items(): layer_params = YoloV3Params(net.layers[layer_name].params, out_blob.shape[2]) log.info("Layer {} parameters: ".format(layer_name)) layer_params.log_params() objects += parse_yolo_region(out_blob, in_frame.shape[2:], frame.shape[:-1], layer_params, args.prob_threshold) parsing_time = time() - start_time # Filtering overlapping boxes with respect to the --iou_threshold CLI parameter for i in range(len(objects)): if objects[i]['confidence'] == 0: continue for j in range(i + 1, len(objects)): if intersection_over_union( objects[i], objects[j]) > args.iou_threshold: objects[j]['confidence'] = 0 # Drawing objects with respect to the --prob_threshold CLI parameter objects = [ obj for obj in objects if obj['confidence'] >= args.prob_threshold ] if len(objects) and args.raw_output_message: log.info("\nDetected boxes for batch {}:".format(1)) log.info( " Class ID | Confidence | XMIN | YMIN | XMAX | YMAX | COLOR " ) origin_im_size = frame.shape[:-1] for obj in objects: # Validation bbox of detected object if obj['xmax'] > origin_im_size[1] or obj[ 'ymax'] > origin_im_size[0] or obj['xmin'] < 0 or obj[ 'ymin'] < 0: continue color = (int(min(obj['class_id'] * 12.5, 255)), min(obj['class_id'] * 7, 255), min(obj['class_id'] * 5, 255)) det_label = labels_map[obj['class_id']] if labels_map and len(labels_map) >= obj['class_id'] else \ str(obj['class_id']) if args.raw_output_message: log.info( "{:^9} | {:10f} | {:4} | {:4} | {:4} | {:4} | {} ". format(det_label, obj['confidence'], obj['xmin'], obj['ymin'], obj['xmax'], obj['ymax'], color)) cv2.rectangle(frame, (obj['xmin'], obj['ymin']), (obj['xmax'], obj['ymax']), color, 2) cv2.putText( frame, "#" + det_label + ' ' + str(round(obj['confidence'] * 100, 1)) + ' %', (obj['xmin'], obj['ymin'] - 7), cv2.FONT_HERSHEY_COMPLEX, 0.6, color, 1) # Draw performance stats over frame inf_time_message = "Inference time: N\A for async mode" if is_async_mode else \ "Inference time: {:.3f} ms".format(det_time * 1e3) render_time_message = "OpenCV rendering time: {:.3f} ms".format( render_time * 1e3) async_mode_message = "Async mode is on. Processing request {}".format(cur_request_id) if is_async_mode else \ "Async mode is off. Processing request {}".format(cur_request_id) parsing_message = "YOLO parsing time is {:.3f}".format( parsing_time * 1e3) cv2.putText(frame, inf_time_message, (15, 15), cv2.FONT_HERSHEY_COMPLEX, 0.5, (200, 10, 10), 1) cv2.putText(frame, render_time_message, (15, 45), cv2.FONT_HERSHEY_COMPLEX, 0.5, (10, 10, 200), 1) cv2.putText(frame, async_mode_message, (10, int(origin_im_size[0] - 20)), cv2.FONT_HERSHEY_COMPLEX, 0.5, (10, 10, 200), 1) cv2.putText(frame, parsing_message, (15, 30), cv2.FONT_HERSHEY_COMPLEX, 0.5, (10, 10, 200), 1) start_time = time() # cv2.imshow("DetectionResults", frame) render_time = time() - start_time if is_async_mode: cur_request_id, next_request_id = next_request_id, cur_request_id frame = next_frame key = cv2.waitKey(wait_key_code) # Tab key if key == 27: break # ESC key if key == 9: exec_net.requests[cur_request_id].wait() is_async_mode = not is_async_mode log.info("Switched to {} mode".format( "async" if is_async_mode else "sync")) cv2.destroyAllWindows() ############################################################################ else: detection_buffer = 10 # buffer between when the drone can respond again to seeing a person detect_gate = True # logic gate before a drone will turn based on seeing someone detection_time = float('inf') det_label = 100 flight_max = 300 print("Configuring tello Drone") tello = Tello() print("Drone battery level: " + str(tello.get_battery())) if not tello.connect(): print("Tello not connected") return log.info("Loading model to the plugin") exec_net = plugin.load(network=net, num_requests=2) if not tello.streamon(): print("Could not start video stream") return #Start the drone flight with takeoff (I'm putting a 3 minute limit before the drone comes down) if not tello.takeoff(): print("Takeoff failed") takeoff_time = time() # while(stripper_int(tello.get_height()) > 210) or (stripper_int(tello.get_height()) < 150): # if (stripper_int(tello.get_height()) > 210): # tello.move("down",3) # else: # tello.move("up",3) while (True): det_label = 100 #set to be initialized each time if (time() - takeoff_time) > flight_max: print( "Max flight time have been logged - triggering autolanding" ) tello.land() footage = tello.get_frame_read() log.info("New Frame Loaded") # cv2.imshow("DetectionResults", footage.frame) #no need to have it up here raw_key = cv2.waitKey(1) is_async_mode = False wait_key_code = 0 # ----------------------------------------- 5. Loading model to the plugin ----------------------------------------- # log.info("Loading model to the plugin") # exec_net = plugin.load(network=net, num_requests=2) cur_request_id = 0 next_request_id = 1 render_time = 0 parsing_time = 0 # ----------------------------------------------- 6. Doing inference ----------------------------------------------- # print("To close the application, press 'CTRL+C' or any key with focus on the output window") # print ("Here's the type of footage.drone: " + str(type(footage.frame))) if is_async_mode: ret = True #footage.frame next_frame = footage.frame # In cap.read its 91600 size else: ret = True #footage.frame frame = footage.frame # print ("footage.frame type is: " + str(type(footage.drone))) # print("Output Representation: " + str(footage.frame)) # print("Here's the size of the array: " + str(footage.frame.size)) if not ret: #commented due to truth value print("RET VALUE IS FALSE!") break if is_async_mode: request_id = next_request_id in_frame = cv2.resize(next_frame, (w, h)) else: request_id = cur_request_id in_frame = cv2.resize(frame, (w, h)) # resize input_frame to network size in_frame = in_frame.transpose( (2, 0, 1)) # Change data layout from HWC to CHW in_frame = in_frame.reshape((n, c, h, w)) #It looks like I copied it twice # in_frame = in_frame.transpose((2, 0, 1)) # Change data layout from HWC to CHW # in_frame = in_frame.reshape((n, c, h, w)) # Start inference start_time = time() exec_net.start_async(request_id=request_id, inputs={input_blob: in_frame}) det_time = time() - start_time # Collecting object detection results objects = list() if exec_net.requests[cur_request_id].wait(-1) == 0: output = exec_net.requests[cur_request_id].outputs start_time = time() for layer_name, out_blob in output.items(): layer_params = YoloV3Params(net.layers[layer_name].params, out_blob.shape[2]) log.info("Layer {} parameters: ".format(layer_name)) layer_params.log_params() objects += parse_yolo_region(out_blob, in_frame.shape[2:], frame.shape[:-1], layer_params, args.prob_threshold) parsing_time = time() - start_time # Filtering overlapping boxes with respect to the --iou_threshold CLI parameter for i in range(len(objects)): if objects[i]['confidence'] == 0: continue for j in range(i + 1, len(objects)): if intersection_over_union( objects[i], objects[j]) > args.iou_threshold: objects[j]['confidence'] = 0 # Drawing objects with respect to the --prob_threshold CLI parameter objects = [ obj for obj in objects if obj['confidence'] >= args.prob_threshold ] if len(objects) and args.raw_output_message: log.info("\nDetected boxes for batch {}:".format(1)) log.info( " Class ID | Confidence | XMIN | YMIN | XMAX | YMAX | COLOR " ) origin_im_size = frame.shape[:-1] for obj in objects: # Validation bbox of detected object if obj['xmax'] > origin_im_size[1] or obj[ 'ymax'] > origin_im_size[0] or obj['xmin'] < 0 or obj[ 'ymin'] < 0: continue color = (int(min(obj['class_id'] * 12.5, 255)), min(obj['class_id'] * 7, 255), min(obj['class_id'] * 5, 255)) det_label = labels_map[obj['class_id']] if labels_map and len(labels_map) >= obj['class_id'] else \ str(obj['class_id']) if args.raw_output_message: log.info( "{:^9} | {:10f} | {:4} | {:4} | {:4} | {:4} | {} ". format(det_label, obj['confidence'], obj['xmin'], obj['ymin'], obj['xmax'], obj['ymax'], color)) cv2.rectangle(frame, (obj['xmin'], obj['ymin']), (obj['xmax'], obj['ymax']), color, 2) cv2.putText( frame, "#" + det_label + ' ' + str(round(obj['confidence'] * 100, 1)) + ' %', (obj['xmin'], obj['ymin'] - 7), cv2.FONT_HERSHEY_COMPLEX, 0.6, color, 1) print("Drone is seeing: " + str(coco_labels[int(det_label)])) if (time() - detection_time) > detection_buffer: detect_gate = True if int(det_label) == 0 and detect_gate: detection_time = time() tello.flip('f') #Current reaction to seeing a drone detect_gate = False # Draw performance stats over frame inf_time_message = "Inference time: N\A for async mode" if is_async_mode else \ "Inference time: {:.3f} ms".format(det_time * 1e3) render_time_message = "OpenCV rendering time: {:.3f} ms".format( render_time * 1e3) async_mode_message = "Async mode is on. Processing request {}".format(cur_request_id) if is_async_mode else \ "Async mode is off. Processing request {}".format(cur_request_id) parsing_message = "YOLO parsing time is {:.3f}".format( parsing_time * 1e3) cv2.putText(frame, inf_time_message, (15, 15), cv2.FONT_HERSHEY_COMPLEX, 0.5, (200, 10, 10), 1) cv2.putText(frame, render_time_message, (15, 45), cv2.FONT_HERSHEY_COMPLEX, 0.5, (10, 10, 200), 1) cv2.putText(frame, async_mode_message, (10, int(origin_im_size[0] - 20)), cv2.FONT_HERSHEY_COMPLEX, 0.5, (10, 10, 200), 1) cv2.putText(frame, parsing_message, (15, 30), cv2.FONT_HERSHEY_COMPLEX, 0.5, (10, 10, 200), 1) start_time = time() # cv2.imshow("DetectionResults", frame) #Debug Testing render_time = time() - start_time if is_async_mode: cur_request_id, next_request_id = next_request_id, cur_request_id frame = next_frame key = cv2.waitKey(wait_key_code) # Tab key if key == 27: break # ESC key if key == 9: exec_net.requests[cur_request_id].wait() is_async_mode = not is_async_mode log.info("Switched to {} mode".format( "async" if is_async_mode else "sync")) if raw_key == 27: #esc is break key break # number_input_frames = int(cap.get(cv2.CAP_PROP_FRAME_COUNT)) # number_input_frames = 1 if number_input_frames != -1 and number_input_frames < 0 else number_input_frames # wait_key_code = 1 #causing issues in compile # Number of frames in picture is 1 and this will be read in cycle. Sync mode is default value for this case # if number_input_frames != 1: # ret, frame = cap.read() #frames need to be taken from def in tello # else: # is_async_mode = False # wait_key_code = 0 cv2.destroyAllWindows() tello.end()