def main(): args = get_args() # Setup video displays orig_video_disp = Display({'name': 'Original_Video'}) thresh_video_disp = Display({'name': 'Tresholded_Video'}) # Setup controls setup_trackbars(controls_window_name) # Get input video video = Video(args['video']) num_frames = video.get_num_frames() # Get the first frame to start with frame = video.next_frame() # To communicate with seek callback global seek_callback_action while True: if play_or_pause == 'Play': if not seek_callback_action: frame = video.next_frame() else: frame = video.get_frame(cur_seek_pos * num_frames / 100) seek_callback_action = False if video.end_reached(): # Wait indefinitely if end of video reached # Or until keypress and then exit cv2.waitKey(0) break # Refresh original video display orig_video_disp.refresh(frame) # Get threshold values h_min, h_max, s_min, s_max, v_min, v_max = get_thresholds( controls_window_name) # Convert image to HSV and apply threshold frame_hsv = cv2.cvtColor(frame, cv2.COLOR_BGR2HSV) frame_thresh = cv2.inRange( frame_hsv, (h_min, s_min, v_min), (h_max, s_max, v_max)) # Refresh thresholded video display thresh_video_disp.refresh(frame_thresh) # Add quitting event if orig_video_disp.can_quit() or thresh_video_disp.can_quit(): break # On quit, save the thresholds save_config = SaveConfig('new_thresholds', 'thresholds') save_config.save(h_min=h_min, h_max=h_max, s_min=s_min, s_max=s_max, v_min=v_min, v_max=v_max)
def __init__(self): """Initialise app.""" # Init pygame pygame.init() # Init display self.display = Display() # Display commands print(INSTRUCTIONS)
def _answer_interstitial(request_attrs, session_attrs, game_engine_events): questions = utils._('QUESTIONS') current_question = int(session_attrs.get('current_question', 1)) shuffled_index = session_attrs['ordered_questions'][current_question - 1] trivia_question = next((q for q in questions if q['index'] == shuffled_index), None) data = {'question_number': current_question} response_message = utils._('ASK_QUESTION_DISPLAY', data) response_message['display_text'] = trivia_question['question'] Display.render(handler_input, response_message) Game.listen_for_answer(handler_input)
def handle_roll_call_complete(handler_input, event): print(f'ROLLCALL_HELPER: handle roll call complete: {event}') request_attrs = handler_input.attributes_manager.request_attributes session_attrs = handler_input.attributes_manager.session_attributes # Move to the button game state to begin the game session_attrs['STATE'] = settings.STATES['button_game'] session_attrs['buttons'] = [ { 'button_id': event.gadget_id, 'count': i + 1, } for i, event in enumerate(event.input_events) ] # clear animations on all other buttons that haven't been added to the game request_attrs['directives'].append(directives.GadgetController.set_idle_animation({ 'animations': animations.BasicAnimations.solid(1, "black", 100) })) # display roll call complete animation on all buttons that were added to the game request_attrs['directives'].append(directives.GadgetController.set_idle_animation({ 'target_gadgets': [b['button_id'] for b in session_attrs['buttons']], 'animations': settings.ANIMATIONS['roll_call_complete'] })) print(f"RollCall: resuming play from question: {session_attrs.get('current_question')}") current_prompts = None if settings.ROLLCALL_STATES['named_players']: # tell the next player to press their button. message = utils._('ROLL_CALL_HELLO_PLAYER', { 'player_number': len(session_attrs['buttons']) }) current_prompts = message message = utils._('ROLL_CALL_COMPLETE') mixed_output_speech = '' if current_prompts: mixed_output_speech = " ".join([ settings.AUDIO['roll_call_complete'], settings.pick_random(message['output_speech']), ]) else: mixed_output_speech = " ".join([ settings.AUDIO['roll_call_complete'], settings.pick_random(message['output_speech']), ]) Display.render(handler_input, message) request_attrs['output_speech'].append(mixed_output_speech) request_attrs['reprompt'].append(message['reprompt']) request_attrs['open_microphone'] = True
def main(): args = get_args() # Check for calib_images folder if not os.path.exists('calib_images'): print 'Please create a directory "calib_images"' return # Setup video display video_disp = Display({'name': 'Video'}) # Setup controls setup_trackbars('Controls') # , thresholds) # Get input video video = Video(args['video']) num_frames = video.get_num_frames() # Get the first frame to start with frame = video.next_frame() global seek_callback_action while True: if play_or_pause == 'Play': if not seek_callback_action: frame = video.next_frame() else: frame = video.get_frame(cur_seek_pos * num_frames / 100) seek_callback_action = False if video.end_reached(): # Wait indefinitely if end of video reached # Or until keypress and then exit cv2.waitKey(0) break video_disp.refresh(frame) cur_frame_num = video.get_cur_frame_num() # Service the key events # if s is pressed, save image # if b is pressed, go back 1s # if n is pressed, go ahead 1s if video_disp.key_pressed('s'): video_file = os.path.basename(args['video']).lower() img_file_name = 'calib_images/{}_{}.png'.format( video_file.strip('.mp4'), cur_frame_num) if cv2.imwrite(img_file_name, frame): print 'Saved', img_file_name elif video_disp.key_pressed('n'): seek_callback( min((((cur_frame_num + 60) * 100) // num_frames), num_frames)) elif video_disp.key_pressed('b'): seek_callback(max((((cur_frame_num - 60) * 100) // num_frames), 0)) # Add quitting event if video_disp.can_quit(): break
def take_turn(actions): player = internal.current_players.get(internal.active_player) player_name = player.get("name") player_character = player.get("character") location = internal.current_game.board.locate_character( player_character) data = actions.get('turn_selection') turn = internal.current_turn_options.get(int(data)) if 'suggestion' in turn: internal.make_suggestion(player) elif 'accusation' in turn: internal.make_accusation(player) else: space_id = turn space_type = internal.current_game.move_player_to_space_by_id( space_id, player_character, location) ServerProtocol.message_all_players("\n" + player_name + " moved " + player_character + " to " + space_id + ".\n") ServerProtocol.message_all_players( Display.display_board(internal.current_game.board)) if space_type is 'Room': internal.make_suggestion(player, space_id) else: internal.finish_turn()
def end_game(handler_input, reset_game): print('GAME: end game') request_attrs = handler_input.attributes_manager.request_attributes session_attrs = handler_input.attributes_manager.session_attributes # Clean the player state on the way out session_attrs.pop('repeat', None) session_attrs.pop('incorrect_answer_buttons', None) session_attrs.pop('correct', None) session_attrs.pop('answering_button', None) session_attrs.pop('answering_player', None) response_message = utils._('GAME_FINISHED' if reset_game else 'GAME_CANCELLED') Display.render(handler_input, response_message) request_attrs['open_microphone'] = False handler_input.response_builder.set_should_end_session(True) if session_attrs['STATE'] == settings.STATES['button_game']: request_attrs['directives'].append(GadgetController.set_idle_animation({ 'target_gadgets': [b['button_id'] for b in session_attrs['buttons']], 'animations': settings.ANIMATIONS['exit'], })) if reset_game: final_scores = GameHelper.get_formatted_scores( handler_input, session_attrs.get('scores'), session_attrs['player_count'] ) multi_player = session_attrs['STATE'] == settings.STATES['button_game'] msg_key = 'GAME_FINISHED_INTRO' if multi_player else 'SINGLE_PLAYER_GAME_FINISHED_INTRO' game_finished_message = utils._(msg_key) if len(request_attrs['output_speech']) == 0: request_attrs['output_speech'].append("<break time='2s'/>") request_attrs['output_speech'].extend([ game_finished_message['output_speech'], final_scores, "<break time='1s'/>", response_message['output_speech'], ]) handler_input.attributes_manager.session_attributes = {} else: request_attrs['output_speech'].append(response_message['output_speech'])
def start_roll_call(handler_input, message_key): print('ROLLCALL_HELPER: start roll call') request_attrs = handler_input.attributes_manager.request_attributes session_attrs = handler_input.attributes_manager.session_attributes config = Helper.generate_input_handler_config(**{ 'player_count': session_attrs['player_count'], 'timeout': 35000, # allow 35 seconds for roll call to complete }) Helper.listen_for_roll_call(handler_input, config) message = utils._(message_key) Display.render(handler_input, message) request_attrs['output_speech'].append(message['output_speech']) request_attrs['output_speech'].append(settings.AUDIO['waiting_for_roll_call']) request_attrs['open_microphone'] = True session_attrs['buttons'] = []
def scheme_1(file_name): video = Video(file_name) display = Display() # Emulate a do-while loop # with "while True" and breaking # if condition fails after executing while True: frame = video.next_frame() if video.end_reached(): break # Do some operation gray = cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY) # Refresh display with new image display.refresh(gray) if display.can_quit(): break
def scheme_2(file_name): video = Video(file_name) display = Display() # First get the number of frames num_frames = video.get_num_frames() # Get each frame from video and display # If step is greater than one (simulating random # seeks,) the playback will be slow for i in range(num_frames): # Get the frame desired frame = video.get_frame(i) # Do some operation gray = cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY) # Refresh display with new image display.refresh(gray) # Quit midway if required if display.can_quit(): break
def __init__(self): self.config = Config() self.cookies = login() logger.info(self.cookies) self.base_path = os.path.join(os.path.dirname(os.path.dirname(os.path.realpath(__file__))), self.config.config['output']['path']) self.live_infos = Infos() self.display = Display() self.decoder = Decoder() self.uploader = Upload() self.threadRecorder = threadRecorder() logger.info('基路径:%s' % (self.base_path)) self.load_room_info() self.get_live_url() logger.info('初始化完成')
def evaluate(csv_file, eval_path="", s3_bucket="bsivisiondata", train_config=None, write_output_images=False): ''' eval_path: top directory, look for frames/ and annotations/ underneath here. If csv specified, then paths will be relative to this eval_path csv_file: a csv of paths_to_images,labels. Paths are relative to eval_path s3_bucket: name of an s3 bucket where image and label data is read from. If csv is specified, labels will be taken from the csv train_config: a .json config file from train_history that gives information on the training job that we're evaluating write_output_images: bool, whether or not to write output images to data/eval_images/training_job_name_steps loop_eval_period: int, the period in seconds to loop evaluation runs. If not set, only runs once ''' if train_config: config_file = train_config else: # config_file = get_most_recent_file_from_dir('train_history/') config_file = choose_file_with_stdin('train_history/') print("Config file: {}".format(config_file)) cm = ConfigManager(config_file, folder='train_history') cfg = cm.get_json() data = DataExtractor(eval_path, s3_bucket, csv_file=csv_file) # batch_size = cfg['training']['hyperparameters']['batch_size'] batch_size = 1 target_shape = cfg['training']['model']['target_shape'] data_gen = data.get_processed_data_generator(True, batch_size, target_shape, True) print("Testing on {} samples".format(data.get_data_length())) displayer = Display() model, path_and_prefix = get_model_and_load_weights(cfg) compile_model(model, cfg['training']) ckpt_prefix = os.path.basename(path_and_prefix) total_loss = 0 num_testing_samples = 0 for X, y, filepaths, originals in data_gen: pred = model.predict_on_batch(X) mse_out = mean_squared_error(y, pred) loss = mse_out.eval(session=K.get_session()) total_loss += loss[0] num_testing_samples += 1 print("Loss: {}".format(loss[0])) H = np.reshape(pred, (3,3)) # print('Prediction:\n{}'.format(H)) if write_output_images: frame = originals[0] filepath = filepaths[0] image_text = "Pred: {}\nTruth: {}\nLoss: {}".format(np.array_str(pred), np.array_str(y[0]), str(loss[0])) warped = displayer.get_warp_overlay(frame, H, dsize=(1000,425)) vstacked = displayer.get_vstacked(frame, warped, dsize=(1000,850)) displayer.put_rectangle(vstacked, (10, 10), (800, 80)) displayer.put_text(vstacked, "Pred: {}".format(np.array_str(pred, precision=2)), text_loc=(10,30)) displayer.put_text(vstacked, "Truth: {}".format(np.array_str(y[0], precision=2)), text_loc=(10,50)) displayer.put_text(vstacked, "loss: {}".format(loss[0]), text_loc=(10,70)) step_number = path_and_prefix.split('-')[-1] job_name_and_step_num = cfg['sagemaker_job_info']['job_name'] + '-step' + str(step_number) job_name_dir = os.path.join('data/eval_images', job_name_and_step_num) if not os.path.exists(job_name_dir): os.makedirs(job_name_dir) filename = "{:.2E}-{}".format(loss[0], os.path.basename(filepath)) displayer.save_image(os.path.join(job_name_dir, filename), vstacked) # displayer.display_vstacked(frame, warped, dsize=(1000,850)) # displayer.display_warp_overlay(frame, H, dsize=(600,600)) average_loss = total_loss / num_testing_samples print("Average loss: {}".format(average_loss)) return average_loss, path_and_prefix
def main_worker(id, video_file, camera_model, K, D, R, T, measurements, quit_event): # Setup video displays video_disp = Display({'name': 'Camera_{}'.format(id)}) # Get input video video = Video(video_file) # Setup the undistortion stuff if camera_model == 'P': # Harcoded image size as # this is a test script img_size = (1920, 1080) # First create scaled intrinsics because we will undistort # into region beyond original image region new_K = cv2.getOptimalNewCameraMatrix(K, D, img_size, 0.35)[0] # Then calculate new image size according to the scaling # Unfortunately the Python API doesn't directly provide the # the new image size. They forgot? new_img_size = (int(img_size[0] + (new_K[0, 2] - K[0, 2])), int( img_size[1] + (new_K[1, 2] - K[1, 2]))) # Standard routine of creating a new rectification # map for the given intrinsics and mapping each # pixel onto the new map with linear interpolation map1, map2 = cv2.initUndistortRectifyMap( K, D, None, new_K, new_img_size, cv2.CV_16SC2) elif camera_model == 'F': # Harcoded image size img_size = (1920, 1080) # First create scaled intrinsics because we will undistort # into region beyond original image region. The alpha # parameter in pinhole model is equivalent to balance parameter here. new_K = cv2.fisheye.estimateNewCameraMatrixForUndistortRectify( K, D, img_size, np.eye(3), balance=1) # Then calculate new image size according to the scaling # Well if they forgot this in pinhole Python API, # can't complain about Fisheye model. Note the reversed # indexing here too. new_img_size = (int(img_size[0] + (new_K[0, 2] - K[0, 2])), int( img_size[1] + (new_K[1, 2] - K[1, 2]))) # Standard routine of creating a new rectification # map for the given intrinsics and mapping each # pixel onto the new map with linear interpolation map1, map2 = cv2.fisheye.initUndistortRectifyMap( K, D, np.eye(3), new_K, new_img_size_1, cv2.CV_16SC2) # Set up foreground and background separation fgbg = cv2.createBackgroundSubtractorMOG2() # Averaging kernel that will be used in opening kernel = np.ones((6, 6), np.uint8) # Code commented out because not using # confidence currently, but could be # used again with changes later # # Will be used for histogram comparison # # (Confidence measure) # ball_image_file = 'ball_image.jpg' # ball_image = cv2.imread(ball_image_file) # 2D ball detection and 3D ball tracking setup ball_position_frame = None ball_wc = [0, 0, 0] while not video.end_reached() and not quit_event.value: # Get each frame frame = video.next_frame() # Undistort the current frame img_undistorted = cv2.remap( frame, map1, map2, cv2.INTER_LINEAR, cv2.BORDER_CONSTANT) # Convert to HSV and threshold range of ball img_hsv = cv2.cvtColor(img_undistorted, cv2.COLOR_BGR2HSV) mask = cv2.inRange(img_hsv, np.array( (15, 190, 200)), np.array((25, 255, 255))) # Foreground and background separation mask fgmask = fgbg.apply(img_undistorted) mask_color_bgs = cv2.bitwise_and(mask, mask, mask=fgmask) masked_and_opened = cv2.morphologyEx( mask_color_bgs, cv2.MORPH_OPEN, kernel) # Hough transform to detect ball (circle) circles = cv2.HoughCircles(masked_and_opened, cv2.HOUGH_GRADIENT, dp=3, minDist=2500, param1=300, param2=5, minRadius=3, maxRadius=30) if circles is not None: # Make indexing easier and # convert everything to int circles = circles[0, :] circles = np.round(circles).astype("int") # Take only the first # (and hopefully largest) # circle detected x, y, r = circles[0] ball_position_frame = [x - r, y - r, 2 * r, 2 * r] else: ball_position_frame = None # Determine the correct ball radius mask_ball_radius = cv2.bitwise_and(fgmask, fgmask, mask=cv2.inRange( img_hsv, np.array((10, 150, 180)), np.array((40, 255, 255)))) if ball_position_frame: x1, y1, w1, h1 = ball_position_frame ball_crop_temp = mask_ball_radius[( y1 + h1 // 2 - 50):(y1 + h1 // 2 + 50), (x1 + w1 // 2 - 50):(x1 + w1 // 2 + 50)] height, width = ball_crop_temp.shape if height and width: # Successfully cropped image ball_crop = ball_crop_temp cnts = cv2.findContours( ball_crop.copy(), cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE)[-2] center = None if len(cnts) > 0: # contour detected c = max(cnts, key=cv2.contourArea) ellipse = cv2.fitEllipse(c) width = min(ellipse[1]) ball_position_frame = [ ball_position_frame[0], ball_position_frame[1], 2 * width, 2 * width] # Code commented out because not using # confidence currently, but could be # used again with changes later # # Calculate confidence # confidence = histogram_comparison(ball_image, img_undistorted, ball_position_frame) # print confidence if ball_position_frame: x1, y1, w1, h1 = ball_position_frame pixels_per_mm = ( K[0, 0] + K[1, 1]) / 2 / DEFAULT_FOCAL_LENGTH z = PING_PONG_DIAMETER * \ DEFAULT_FOCAL_LENGTH / (w1 / pixels_per_mm) x = ((x1 - K[0, 2]) / pixels_per_mm) * z / DEFAULT_FOCAL_LENGTH y = ((y1 - K[1, 2]) / pixels_per_mm) * z / DEFAULT_FOCAL_LENGTH ball_cc = np.array([x, y, z]) / 1000 ball_wc = np.dot(R.T, ball_cc - T.ravel()) # Push measurements to be processed/visualized measurement = { 'id': id, 'frame_num': video.get_cur_frame_num(), 'ball_ic': ball_position_frame, 'ball_wc': ball_wc } measurements.put(measurement) # Update video display video_disp.refresh(img_undistorted) # Add quitting event if video_disp.can_quit(): break # Setting this will signal # the other parallel process # to exit too. quit_event.value = 1
class DuckieControlApp(object): """Application class. This application contains an event listener to catch keyboard keys press and a UI to display information. All UI logic is defined in utils.display. """ def __init__(self): """Initialise app.""" # Init pygame pygame.init() # Init display self.display = Display() # Display commands print(INSTRUCTIONS) def register_image(self, image): """Register an image for future display.""" self.display.image = image def register_stack(self, stack): """Register a state stack for future display.""" self.display.stack = stack def register_action(self, action): """Register an action for future display.""" self.display.action = action def step(self, auto_pilote_mode): """Run a step of app logic. This will check if an arrow key is pressed if in autopilote_mode and return the action to take, listen to the a key pressed and return toggle autopilote boolean if it is. """ # Get keys currently being pressed keys = pygame.key.get_pressed() action = [] # If in auto pilote mode, convert key to action if not auto_pilote_mode: # Default if no key is pressed action = [0, 0] if keys[pygame.K_UP]: action[0] = 0.6 action[1] = 0.6 if keys[pygame.K_DOWN]: action[0] = -0.6 action[1] = -0.6 if keys[pygame.K_LEFT]: action[0] = 0.24 action[1] = 0.86 if keys[pygame.K_RIGHT]: action[0] = 0.86 action[1] = 0.24 # Register action to display self.register_action(action) # Retrive pygame event list pygame.event.pump() autopilote_toggle = False save_cmd = False # If A have been pressed once, toggle autopilote for event in pygame.event.get(): if event.type == pygame.KEYDOWN and event.key == pygame.K_a: autopilote_toggle = True if event.type == pygame.KEYDOWN and event.key == pygame.K_RETURN: save_cmd = True return action, autopilote_toggle, save_cmd def render(self): """Render display.""" self.display.render()
from utils.display import Display from utils.file_manager import FileManager from utils.warp_tools import * from utils.rink_specs import HockeyRink import random import cv2 fm = FileManager('bsivisiondata') d = Display() annotations = fm.get_folder_list('PHI-PIT_6m-8m/annotations', extension_filter='json') # random.shuffle(annotations) for f in annotations: print f im_dict = fm.read_image_dict('PHI-PIT_6m-8m/annotations', f) if not 'warp' in im_dict: continue imname = f.split('.')[0] + '.png' im = fm.read_image_file('PHI-PIT_6m-8m/frames', imname) im = cv2.cvtColor(im, cv2.COLOR_RGB2BGR) H = np.array(im_dict['warp']['M']) hr = HockeyRink() scaled_H = scale_homography(H, 600, 300) H1280 = scale_homography(H, 1280, 720) ''' NEEDED TO BE RESIZING IMAGES BEFORE CALLING WARP!!! '''
def main(): args = get_args() # Setup video displays orig_video_disp = Display({'name': 'Original_Video'}) thresh_video_disp = Display({'name': 'Thresholded_Video'}) mean_shift_video_disp = Display({'name': 'Mean-Shift Tracking Video'}) # Setup controls setup_trackbars(controls_window_name) # Get input video video = Video(args['video']) num_frames = video.get_num_frames() # Get the first frame to start with frame = video.next_frame() global seek_callback_action # setup initial location of window top, length, left, width = 450, 36, 1000, 43 # simply hardcoded the values track_window = (left, top, width, length) # set up the ROI for tracking roi = frame[top:top + length, left:left + width] hsv_roi = cv2.cvtColor(roi, cv2.COLOR_BGR2HSV) h_min, h_max, s_min, s_max, v_min, v_max = get_thresholds( controls_window_name) mask = cv2.inRange(hsv_roi, np.array( (h_min, s_min, v_min)), np.array((h_max, s_max, v_max))) roi_hist = cv2.calcHist([hsv_roi], [0], mask, [180], [0, 180]) cv2.normalize(roi_hist, roi_hist, 0, 255, cv2.NORM_MINMAX) # Setup the termination criteria, either 10 iteration or move by at least 1 pt term_criteria = (cv2.TERM_CRITERIA_EPS | cv2.TERM_CRITERIA_COUNT, 20, 1) while True: if play_or_pause == 'Play': if not seek_callback_action: frame = video.next_frame() else: frame = video.get_frame(cur_seek_pos * num_frames / 100) seek_callback_action = False if video.end_reached(): # Wait indefinitely if end of video reached # Or until keypress and then exit cv2.waitKey(0) break # Refresh original video display orig_video_disp.refresh(frame) # Get threshold values h_min, h_max, s_min, s_max, v_min, v_max = get_thresholds( controls_window_name) # Convert image to HSV and apply threshold frame_hsv = cv2.cvtColor(frame, cv2.COLOR_BGR2HSV) frame_thresh = cv2.inRange( frame_hsv, (h_min, s_min, v_min), (h_max, s_max, v_max)) # threshold image in hsv domain frame_hsv_threshold = cv2.bitwise_and( frame_hsv, frame_hsv, mask=frame_thresh) # Refresh thresholded video display thresh_video_disp.refresh(frame_hsv_threshold) # Find the backprojection of the histogram dst = cv2.calcBackProject([frame_hsv_threshold], [ 0], roi_hist, [0, 180], 1) # apply meanshift to get the new location ret, track_window = cv2.meanShift(dst, track_window, term_criteria) # Draw it on image x, y, w, h = track_window frame_mean_shift = cv2.rectangle(frame, (x, y), (x + w, y + h), 255, 2) # Refresh mean shift tracking video display mean_shift_video_disp.refresh(frame_mean_shift)
def begin_game(): # initialize the game internal.current_game.initialize_game() ServerProtocol.message_all_players( Display.display_board(internal.current_game.board)) internal.begin_turn(1)
if __name__ == "__main__": current_game = Game() # add players to game current_game.add_player("Roxanne", "Mr. Green") current_game.add_player("Julie", "Mrs. Peacock") # initialize the game current_game.initialize_game() weapons = current_game.get_cards().get_weapons() characters = current_game.get_cards().get_characters() rooms = current_game.get_cards().get_rooms() for weapon in weapons: print(weapon.card_type + " : " + weapon.value) for character in characters: print(character.card_type + " : " + character.value) for room in rooms: print(room.card_type + " : " + room.value) players = current_game.get_players() for player in players: print(player.name + " : " + player.selected_character) Display.display_board(current_game.board) current_game.terminate_game()
def ask_question(handler_input, is_following): """ gather the built responses, add them to the overall response retrieve and ask the next/same question """ request_env = handler_input.request_envelope request_attrs = handler_input.attributes_manager.request_attributes session_attrs = handler_input.attributes_manager.session_attributes questions = utils._('QUESTIONS') print(f"GAME: ask_question (currentQuestion = {session_attrs.get('current_question')})") if not is_following: # clean repeat state session_attrs.pop('repeat', None) session_attrs.pop('incorrect_answer_buttons', None) session_attrs['input_handler_id'] = request_env.request.request_id if 'current_question' in session_attrs: current_question = int(session_attrs['current_question']) else: session_attrs['current_question'] = current_question = 1 if 'ordered_questions' not in session_attrs or ( current_question == 1 and 'repeat' not in session_attrs ): if settings.GAME_OPTIONS['shuffle_questions']: print('GamePlay: producing ordered questions for new game (using shuffling)!') # if this is the first question, then shuffle the questions ordered_questions = [q['index'] for q in questions] random.shuffle(ordered_questions) else: print('GamePlay: producing ordered questions for new game (shuffling disabled)!') ordered_questions = [q['index'] for q in questions] ordered_questions = ordered_questions[:settings.GAME_OPTIONS['questions_per_game']] session_attrs['ordered_questions'] = ordered_questions if ( current_question > len(session_attrs['ordered_questions']) or current_question > settings.GAME_OPTIONS['questions_per_game'] ): return Game.end_game(handler_input, True) else: shuffle_question = session_attrs['ordered_questions'][current_question - 1] next_question = next((q for q in questions if q['index'] == shuffle_question), None) print( f"Ask question: {current_question} of " f"{settings.GAME_OPTIONS['questions_per_game']}, " f"next question {next_question}" ) interstitial_delay = 6000 if is_following else 3000 questions_per_round = int(settings.GAME_OPTIONS['questions_per_round']) if ( current_question > 2 and 'repeat' not in session_attrs and (current_question - 1) % questions_per_round == 0 ): interstitial_delay += 12000 round_summary = GameHelper.generate_round_summary_narration( handler_input, session_attrs['current_question'], session_attrs.get('scores'), session_attrs['player_count'], ) request_attrs['output_speech'].append(round_summary) if 'correct' in session_attrs: if session_attrs['correct']: keys = { True: 'ANSWER_QUESTION_CORRECT_DISPLAY', False: 'SINGLE_PLAYER_ANSWER_QUESTION_CORRECT_DISPLAY', } key = keys[session_attrs['STATE'] == settings.STATES['button_game']] image = settings.pick_random(settings.IMAGES['correct_answer']) else: keys = { True: 'ANSWER_QUESTION_INCORRECT_DISPLAY', False: 'SINGLE_PLAYER_ANSWER_QUESTION_INCORRECT_DISPLAY', } key = keys[session_attrs['STATE'] == settings.STATES['button_game']] image = settings.pick_random(settings.IMAGES['incorrect_answer']) message = utils._(key, {'player_number': session_attrs['answering_player']}) message['image'] = image Display.render(handler_input, message) else: key = 'NEW_GAME' if current_question == 1 else 'RESUME' message = utils._(f'ASK_FIRST_QUESTION_{key}_DISPLAY') Display.render(handler_input, message) # use a shorter break for buttonless games break_time = 4 if session_attrs['STATE'] == settings.STATES['button_game'] else 1 answers = f"<break time='{break_time}s'/> Is it " if next_question['answers']: if len(next_question['answers']) > 1: answers += ', '.join(next_question['answers'][:-1]) answers += f", or, {next_question['answers'][-1]}" else: answers = next_question['answers'][0] answers += "?" request_attrs['output_speech'].append(next_question['question']) request_attrs['output_speech'].append(answers) if session_attrs['STATE'] == settings.STATES['button_game']: request_attrs['output_speech'].append(settings.AUDIO['waiting_for_buzz_in']) Game.animate_buttons_after_answer(handler_input) Game.send_answer_interstitial(handler_input, interstitial_delay) session_attrs.pop('answering_button', None) session_attrs.pop('answering_player', None) else: request_attrs['reprompt'].append(answers) session_attrs['waiting_for_answer'] = True session_attrs['answering_player'] = 1 request_attrs['open_microphone'] = True message = utils._('ASK_QUESTION_DISPLAY', {'question_number': current_question}) if session_attrs.get('correct') is True: message['image'] = settings.pick_random(settings.IMAGES['correct_answer']) elif session_attrs.get('correct') is False: message['image'] = settings.pick_random(settings.IMAGES['incorrect_answer']) Display.render(handler_input, message) session_attrs.pop('correct', None)
def main(): args = get_args() # Read in configuration # load_config = LoadConfig('config/thresholds.npz', 'thresholds') # thresholds = load_config.load() # Setup video displays video_disp = Display({'name': 'Video'}) # Setup controls setup_trackbars('Controls') # , thresholds) # Get input video video = Video(args['video']) num_frames = video.get_num_frames() # Get the first frame to start with frame = video.next_frame() global seek_callback_action # Deck for storing calib images calib_img_deck = deque(maxlen=MAX_NUM_IMAGES_FOR_CALIB) # Deck for storing charuco info charuco_corners_deck = deque(maxlen=MAX_NUM_IMAGES_FOR_CALIB) charuco_ids_deck = deque(maxlen=MAX_NUM_IMAGES_FOR_CALIB) skip_count = 0 test_camera_matrix = np.array([ [11096.77, 0, 540], [0, 11096.77, 960], [0, 0, 1] ]) while True: if play_or_pause == 'Play': if not seek_callback_action: frame = video.next_frame() else: frame = video.get_frame(cur_seek_pos * num_frames / 100) seek_callback_action = False if video.end_reached(): # Wait indefinitely if end of video reached # Or until keypress and then exit cv2.waitKey(0) break corners, ids, rejected_img_points = cv2.aruco.detectMarkers( frame, dictionary) if ids is not None: img_markers = cv2.aruco.drawDetectedMarkers(frame, corners, ids) num_charuco, charuco_corners, charuco_ids = cv2.aruco.interpolateCornersCharuco( corners, ids, frame, board, cameraMatrix=test_camera_matrix) if charuco_corners is not None: img_markers = cv2.aruco.drawDetectedCornersCharuco( img_markers, charuco_corners, charuco_ids) if ids.shape[0] == MAX_ARUCO_IDS \ and num_charuco == MAX_CHARUCO_IDS \ and skip_count % 15 == 0: calib_img_deck.append(frame) charuco_corners_deck.append(charuco_corners) charuco_ids_deck.append(charuco_ids) else: img_markers = frame cv2.putText(img_markers, '{}/{}'.format(len(calib_img_deck), MAX_NUM_IMAGES_FOR_CALIB), (200, 100), cv2.FONT_HERSHEY_COMPLEX, 2, (0, 0, 255), 5) video_disp.refresh(img_markers) if video_disp.key_pressed('s'): pass skip_count = skip_count + 1 # Add quitting event if video_disp.can_quit(): break # On quit, save the params # save_config = SaveConfig('new_erode_dilate', 'erode_dilate') # save_config.save(dilate_size=dilate_size, erode_size=erode_size) img_size = calib_img_deck[0].shape[:2] # print charuco_ids_deck # error, camera_matrix, dist_coeffs = cv2.aruco.calibrateCameraCharuco( # charuco_corners_deck, charuco_ids_deck, board, img_size, test_camera_matrix, None)[:3] objPoints, imgPoints = [board.chessboardCorners.reshape( 1, -1, 3)] * len(charuco_corners_deck), charuco_corners_deck calibration_flags = cv2.fisheye.CALIB_USE_INTRINSIC_GUESS + \ cv2.fisheye.CALIB_FIX_PRINCIPAL_POINT + cv2.fisheye.CALIB_FIX_SKEW error, camera_matrix, dist_coeffs = cv2.fisheye.calibrate( objPoints, imgPoints, img_size, test_camera_matrix, np.zeros(4), flags=calibration_flags)[:3] print error, camera_matrix save_config = SaveConfig('new_calib', 'calib') save_config.save(camera_matrix=camera_matrix, dist_coeffs=dist_coeffs)
# return sorted(os.listdir('train_history/'), key=lambda x: os.path.getmtime(os.path.join('train_history', x)), reverse=True)[0] if options.config_file: config_file = options.config_file else: # config_file = get_most_recent_file_from_dir('train_history/') config_file = choose_file_with_stdin('train_history/') print("Config file: {}".format(config_file)) cm = ConfigManager(config_file, folder='train_history') cfg = cm.get_json() model = get_model_and_load_weights(cfg) compile_model(model, cfg['training']) data = Data(options.test_path, cfg['training'], train_test='test') print("Testing on {} samples".format(data.test_len)) data_gen = data.test_generator() displayer = Display() for frame_batch in data_gen: print('Frame batch') pred = model.predict_on_batch(frame_batch) # print pred H = np.reshape(pred, (3,3)) print('Prediction:\n{}'.format(H)) frame = np.array(frame_batch[0]*255).astype('uint8') warped = displayer.get_warp_overlay(frame, H, dsize=(300,300)) displayer.display_vstacked(frame, warped, dsize=(600,600)) # displayer.display_warp_overlay(frame, H, dsize=(600,600)) print 'done'
from machine import I2C, Pin from utils.display import Display ON = 1 OFF = 0 LED = Pin(1, Pin.OUT) DISPLAY = Display(I2C(sda=Pin(0), scl=Pin(2)))
def main(): args = get_args() # Read in intrinsic calibration load_config_1 = LoadConfig( 'config/intrinsic_calib_{}_camera_1.npz'.format(args['model'].lower()), 'calib_camera_1') intrinsic_1 = load_config_1.load() K_1 = intrinsic_1['camera_matrix'] D_1 = intrinsic_1['dist_coeffs'] load_config_2 = LoadConfig( 'config/intrinsic_calib_{}_camera_2.npz'.format(args['model'].lower()), 'calib_camera_2') intrinsic_2 = load_config_2.load() K_2 = intrinsic_2['camera_matrix'] D_2 = intrinsic_2['dist_coeffs'] # Read in extrinsic calibration load_config_e_1 = LoadConfig( 'config/extrinsic_calib_{}_camera_1.npz'.format(args['model'].lower()), 'extrinsic_camera_1') extrinsic_1 = load_config_e_1.load() R_1 = cv2.Rodrigues(extrinsic_1['rvec'])[0] T_1 = extrinsic_1['tvec'] load_config_e_2 = LoadConfig( 'config/extrinsic_calib_{}_camera_2.npz'.format(args['model'].lower()), 'extrinsic_camera_2') extrinsic_2 = load_config_e_2.load() R_2 = cv2.Rodrigues(extrinsic_2['rvec'])[0] T_2 = extrinsic_2['tvec'] # Setup video displays video_disp_1 = Display({'name': 'Camera_1'}) video_disp_2 = Display({'name': 'Camera_2'}) # Get input video video_1 = Video(args['video_1']) video_2 = Video(args['video_2']) # Get the first frame; to see # if video framework works frame_1 = video_1.next_frame() frame_2 = video_2.next_frame() # Original code was to multiprocess, but # found MACOSX doesn't like forking processes # with GUIs. However, retaining Array from # multiproc for future. shared_var = Array('d', [0, 0, 0]) end_reached = Value('b', False) # visu = Process(target=visualize_table, args=(shared_var,)) # visu.start() visu = Thread(target=visualize_table, args=(end_reached, shared_var)) visu.daemon = True visu.start() # Setup the undistortion stuff if args['model'].upper() == 'P': # Harcoded image size as # this is a test script img_size = (1920, 1080) # First create scaled intrinsics because we will undistort # into region beyond original image region new_K_1 = cv2.getOptimalNewCameraMatrix(K_1, D_1, img_size, 0.35)[0] new_K_2 = cv2.getOptimalNewCameraMatrix(K_2, D_2, img_size, 0.35)[0] # Then calculate new image size according to the scaling # Unfortunately the Python API doesn't directly provide the # the new image size. They forgot? new_img_size_1 = (int(img_size[0] + (new_K_1[0, 2] - K_1[0, 2])), int(img_size[1] + (new_K_1[1, 2] - K_1[1, 2]))) new_img_size_2 = (int(img_size[0] + (new_K_2[0, 2] - K_2[0, 2])), int(img_size[1] + (new_K_2[1, 2] - K_2[1, 2]))) # Standard routine of creating a new rectification # map for the given intrinsics and mapping each # pixel onto the new map with linear interpolation map1_1, map2_1 = cv2.initUndistortRectifyMap(K_1, D_1, None, new_K_1, new_img_size_1, cv2.CV_16SC2) map1_2, map2_2 = cv2.initUndistortRectifyMap(K_2, D_2, None, new_K_2, new_img_size_2, cv2.CV_16SC2) elif args['model'].upper() == 'F': # Harcoded image size img_size = (1920, 1080) # First create scaled intrinsics because we will undistort # into region beyond original image region. The alpha # parameter in pinhole model is equivalent to balance parameter here. new_K_1 = cv2.fisheye.estimateNewCameraMatrixForUndistortRectify( K_1, D_1, img_size, np.eye(3), balance=1) new_K_2 = cv2.fisheye.estimateNewCameraMatrixForUndistortRectify( K_2, D_2, img_size, np.eye(3), balance=1) # Then calculate new image size according to the scaling # Well if they forgot this in pinhole Python API, # can't complain about Fisheye model. Note the reversed # indexing here too. new_img_size_1 = (int(img_size[0] + (new_K_1[0, 2] - K_1[0, 2])), int(img_size[1] + (new_K_1[1, 2] - K_1[1, 2]))) new_img_size_2 = (int(img_size[0] + (new_K_2[0, 2] - K_2[0, 2])), int(img_size[1] + (new_K_2[1, 2] - K_2[1, 2]))) # Standard routine of creating a new rectification # map for the given intrinsics and mapping each # pixel onto the new map with linear interpolation map1_1, map2_1 = cv2.fisheye.initUndistortRectifyMap( K_1, D_1, np.eye(3), new_K_1, new_img_size_1, cv2.CV_16SC2) map1_2, map2_2 = cv2.fisheye.initUndistortRectifyMap( K_2, D_2, np.eye(3), new_K_2, new_img_size_2, cv2.CV_16SC2) # STUFF corr_threshold = -1 radius_change_threshold = 5 ball_image_file = 'ball_image.jpg' # will be used for histogram comparison ball_image = cv2.imread(ball_image_file) fgbg1 = cv2.createBackgroundSubtractorMOG2() fgbg2 = cv2.createBackgroundSubtractorMOG2() kernel = np.ones((6, 6), np.uint8) ball_position_frame1 = None ball_position_frame2 = None prev_frame1 = None prev_frame2 = None ball_wc = [0, 0, 0] while not video_1.end_reached() and not video_2.end_reached(): frame_1 = video_1.next_frame() frame_2 = video_2.next_frame() img_undistorted_1 = cv2.remap(frame_1, map1_1, map2_1, cv2.INTER_LINEAR, cv2.BORDER_CONSTANT) img_undistorted_2 = cv2.remap(frame_2, map1_2, map2_2, cv2.INTER_LINEAR, cv2.BORDER_CONSTANT) # STUFF1 frame_1_hsv = cv2.cvtColor(img_undistorted_1, cv2.COLOR_BGR2HSV) mask1 = cv2.inRange(frame_1_hsv, np.array((15, 190, 200)), np.array((25, 255, 255))) fgmask1 = fgbg1.apply(img_undistorted_1) mask1_color_bgs = cv2.bitwise_and(mask1, mask1, mask=fgmask1) frame1_hsv_bgs = cv2.bitwise_and(frame_1_hsv, frame_1_hsv, mask=mask1_color_bgs) # opening b1 = cv2.morphologyEx(mask1_color_bgs, cv2.MORPH_OPEN, kernel) circles1 = cv2.HoughCircles(b1, cv2.HOUGH_GRADIENT, dp=3, minDist=2500, param1=300, param2=5, minRadius=3, maxRadius=30) if circles1 is not None: # convert the (x, y) coordinates and radius of the circles to integers circles1 = np.round(circles1[0, :]).astype("int") x, y, r = circles1[0] ball_position_frame1 = [x - r, y - r, 2 * r, 2 * r] # loop over the (x, y) coordinates and radius of the circles else: ball_position_frame1 = None mask_ball_radius1 = cv2.bitwise_and(fgmask1, fgmask1, mask=cv2.inRange( frame_1_hsv, np.array((10, 150, 180)), np.array((40, 255, 255)))) # determine the correct radius if ball_position_frame1 != None: x1, y1, w1, h1 = ball_position_frame1 ball_crop_temp1 = mask_ball_radius1[(y1 + h1 // 2 - 30):(y1 + h1 // 2 + 30), (x1 + w1 // 2 - 30):(x1 + w1 // 2 + 30)] height, width = ball_crop_temp1.shape if height != 0 and width != 0: # successfully cropped image ball_crop1 = ball_crop_temp1 cnts = cv2.findContours(ball_crop1.copy(), cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE)[-2] center = None if len(cnts) > 0: # contour detected c = max(cnts, key=cv2.contourArea) rect = cv2.minAreaRect(c) width, height = rect[1] ball_position_frame1 = [ ball_position_frame1[0], ball_position_frame1[1], min(width, height), min(width, height) ] prev_frame1 = img_undistorted_1 if ball_position_frame1: x1, y1, w1, h1 = ball_position_frame1 pixels_per_mm = (K_1[0, 0] + K_1[1, 1]) / 2 / FOCAL_LENGTH z = PING_PONG_DIAMETER * FOCAL_LENGTH / (w1 / pixels_per_mm) x = ((x1 - K_1[0, 2]) / pixels_per_mm) * z / FOCAL_LENGTH y = ((y1 - K_1[1, 2]) / pixels_per_mm) * z / FOCAL_LENGTH ball_cc1 = np.array([x, y, z]) / 1000 ball_wc1 = np.dot(R_1.T, ball_cc1 - T_1.ravel()) # STUFF2 frame_2_hsv = cv2.cvtColor(img_undistorted_2, cv2.COLOR_BGR2HSV) mask2 = cv2.inRange(frame_2_hsv, np.array((15, 190, 200)), np.array((25, 255, 255))) fgmask2 = fgbg2.apply(img_undistorted_2) mask2_color_bgs = cv2.bitwise_and(mask2, mask2, mask=fgmask2) frame2_hsv_bgs = cv2.bitwise_and(frame_2_hsv, frame_2_hsv, mask=mask2_color_bgs) # opening b2 = cv2.morphologyEx(mask2_color_bgs, cv2.MORPH_OPEN, kernel) circles2 = cv2.HoughCircles(b2, cv2.HOUGH_GRADIENT, dp=3, minDist=2500, param1=300, param2=5, minRadius=3, maxRadius=30) if circles2 is not None: # convert the (x, y) coordinates and radius of the circles to integers circles2 = np.round(circles2[0, :]).astype("int") x, y, r = circles2[0] ball_position_frame2 = [x - r, y - r, 2 * r, 2 * r] # loop over the (x, y) coordinates and radius of the circles else: ball_position_frame2 = None mask_ball_radius2 = cv2.bitwise_and(fgmask2, fgmask2, mask=cv2.inRange( frame_2_hsv, np.array((10, 150, 180)), np.array((40, 255, 255)))) # determine the correct radius if ball_position_frame2 != None: x2, y2, w2, h2 = ball_position_frame2 ball_crop_temp2 = mask_ball_radius2[(y2 + h2 // 2 - 30):(y2 + h2 // 2 + 30), (x2 + w2 // 2 - 30):(x2 + w2 // 2 + 30)] height, width = ball_crop_temp2.shape if height != 0 and width != 0: # successfully cropped image ball_crop2 = ball_crop_temp2 cnts = cv2.findContours(ball_crop2.copy(), cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE)[-2] center = None if len(cnts) > 0: # contour detected c = max(cnts, key=cv2.contourArea) rect = cv2.minAreaRect(c) width, height = rect[1] ball_position_frame2 = [ ball_position_frame2[0], ball_position_frame2[1], min(width, height), min(width, height) ] prev_frame2 = img_undistorted_2 if ball_position_frame2: x2, y2, w2, h2 = ball_position_frame2 pixels_per_mm = (K_2[0, 0] + K_2[1, 1]) / 2 / FOCAL_LENGTH z = PING_PONG_DIAMETER * FOCAL_LENGTH / (w2 / pixels_per_mm) x = ((x2 - K_2[0, 2]) / pixels_per_mm) * z / FOCAL_LENGTH y = ((y2 - K_2[1, 2]) / pixels_per_mm) * z / FOCAL_LENGTH ball_cc2 = np.array([x, y, z]) / 1000 ball_wc2 = np.dot(R_2.T, ball_cc2 - T_2.ravel()) # Additional rotation for absolute coordinates R = np.array([[-1, 0, 0], [0, -1, 0], [0, 0, 1]]) ball_wc2 = np.dot(R, ball_wc2) # Combine STUFF1 and STUFF2 # If positions from either available, # update as predicted from either. # But if both available, predict the average if ball_position_frame1 and ball_position_frame2: ball_wc = (ball_wc1 + ball_wc2) / 2 elif ball_position_frame1: ball_wc = ball_wc1 elif ball_position_frame2: ball_wc = ball_wc2 # print 'Ball IC', np.array([x2, y2]), 'Dia', w2 # print 'Ball CC', ball_cc2 # print 'Ball WC', ball_wc2 shared_var[0] = ball_wc[0] shared_var[1] = ball_wc[1] shared_var[2] = ball_wc[2] # Update GUI with new image video_disp_1.refresh(img_undistorted_1) video_disp_2.refresh(img_undistorted_2) # Add quitting event if video_disp_2.can_quit() or video_disp_1.can_quit(): break end_reached.value = True visu.join()
def main(): args = get_args() # Read in intrinsic calibration load_config = LoadConfig( 'config/intrinsic_calib_{}.npz'.format(args['model'].lower()), 'calib') calib = load_config.load() # Read in extrinsic calibration load_config_e = LoadConfig('config/extrinsic_calib_p_camera_1.npz', 'extrinsics') extrinsics = load_config_e.load() # Setup video display video_disp = Display({'name': 'Video'}) # Get input video video = Video(args['video']) num_frames = video.get_num_frames() # Get the first frame; to see # if video framework works frame = video.next_frame() shared_var = Array('d', [0, 0, 0]) visu = Process(target=visualize_table, args=(shared_var, )) visu.start() # Setup the undistortion stuff if args['model'].upper() == 'P': # Harcoded image size as # this is a test script img_size = (1920, 1080) # First create scaled intrinsics because we will undistort # into region beyond original image region new_calib_matrix, _ = cv2.getOptimalNewCameraMatrix( calib['camera_matrix'], calib['dist_coeffs'], img_size, 0.35) # Then calculate new image size according to the scaling # Unfortunately the Python API doesn't directly provide the # the new image size. They forgot? new_img_size = ( int(img_size[0] + (new_calib_matrix[0, 2] - calib['camera_matrix'][0, 2])), int(img_size[1] + (new_calib_matrix[1, 2] - calib['camera_matrix'][1, 2]))) # Standard routine of creating a new rectification # map for the given intrinsics and mapping each # pixel onto the new map with linear interpolation map1, map2 = cv2.initUndistortRectifyMap(calib['camera_matrix'], calib['dist_coeffs'], None, new_calib_matrix, new_img_size, cv2.CV_16SC2) elif args['model'].upper() == 'F': # Harcoded image size img_size = (1920, 1080) # First create scaled intrinsics because we will undistort # into region beyond original image region. The alpha # parameter in pinhole model is equivalent to balance parameter here. new_calib_matrix = cv2.fisheye.estimateNewCameraMatrixForUndistortRectify( calib['camera_matrix'], calib['dist_coeffs'], img_size, np.eye(3), balance=1) # Then calculate new image size according to the scaling # Well if they forgot this in pinhole Python API, # can't complain about Fisheye model. Note the reversed # indexing here too. new_img_size = ( int(img_size[0] + (new_calib_matrix[0, 2] - calib['camera_matrix'][0, 2])), int(img_size[1] + (new_calib_matrix[1, 2] - calib['camera_matrix'][1, 2]))) # Standard routine of creating a new rectification # map for the given intrinsics and mapping each # pixel onto the new map with linear interpolation map1, map2 = cv2.fisheye.initUndistortRectifyMap( calib['camera_matrix'], calib['dist_coeffs'], np.eye(3), new_calib_matrix, new_img_size, cv2.CV_16SC2) # STUFF corr_threshold = -1 radius_change_threshold = 5 ball_image_file = 'ball_image.jpg' # will be used for histogram comparison ball_image = cv2.imread(ball_image_file) fgbg2 = cv2.createBackgroundSubtractorMOG2() kernel = np.ones((6, 6), np.uint8) ball_position_frame2 = None prev_frame2 = None # Get the rotation matrix R = cv2.Rodrigues(extrinsics['rvec'])[0] while not video.end_reached(): frame = video.next_frame() img_undistorted = cv2.remap(frame, map1, map2, cv2.INTER_LINEAR, cv2.BORDER_CONSTANT) # STUFF img_hsv = cv2.cvtColor(img_undistorted, cv2.COLOR_BGR2HSV) mask2 = cv2.inRange(img_hsv, np.array((15, 190, 200)), np.array((25, 255, 255))) fgmask2 = fgbg2.apply(img_undistorted) mask2_color_bgs = cv2.bitwise_and(mask2, mask2, mask=fgmask2) frame2_hsv_bgs = cv2.bitwise_and(img_hsv, img_hsv, mask=mask2_color_bgs) frame_hsv = img_hsv frame_gray = cv2.cvtColor(img_undistorted, cv2.COLOR_BGR2GRAY) mask = cv2.inRange(frame_hsv, np.array((10, 150, 150)), np.array((40, 255, 255))) open_mask = cv2.morphologyEx(mask, cv2.MORPH_OPEN, kernel) frame_thresholded_opened_gray = cv2.bitwise_and(frame_gray, frame_gray, mask=open_mask) frame_thresholded_opened_gray_smoothed = cv2.GaussianBlur( frame_thresholded_opened_gray, (11, 11), 0) # opening a = cv2.inRange(frame_thresholded_opened_gray_smoothed, 10, 256) b = cv2.morphologyEx(mask2_color_bgs, cv2.MORPH_OPEN, kernel) circles = cv2.HoughCircles(b, cv2.HOUGH_GRADIENT, dp=3, minDist=2500, param1=300, param2=5, minRadius=3, maxRadius=30) if circles is not None: # convert the (x, y) coordinates and radius of the circles to integers circles = np.round(circles[0, :]).astype("int") x, y, r = circles[0] ball_position_frame2 = [x - r, y - r, 2 * r, 2 * r] # loop over the (x, y) coordinates and radius of the circles else: ball_position_frame2 = None frame2 = img_undistorted mask2_ball_radius = cv2.bitwise_and(fgmask2, fgmask2, mask=cv2.inRange( img_hsv, np.array((10, 150, 180)), np.array((40, 255, 255)))) if ball_position_frame2 != None: x2, y2, w2, h2 = ball_position_frame2 ball_crop_temp = mask2_ball_radius[(y2 + h2 / 2 - 30):(y2 + h2 / 2 + 30), (x2 + w2 / 2 - 30):(x2 + w2 / 2 + 30)] ball_crop_color = frame2[(y2 + h2 / 2 - 30):(y2 + h2 / 2 + 30), (x2 + w2 / 2 - 30):(x2 + w2 / 2 + 30)] height, width = ball_crop_temp.shape else: ball_crop_temp = [] height = 0 width = 0 if height != 0 and width != 0: ball_crop = ball_crop_temp cnts = cv2.findContours(ball_crop.copy(), cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE)[-2] center = None if len(cnts) > 0 and ball_position_frame2: c = max(cnts, key=cv2.contourArea) rect = cv2.minAreaRect(c) width, height = rect[1] box = cv2.boxPoints(rect) box = np.int0(box) cv2.drawContours(ball_crop_color, [box], 0, (0, 0, 255), 2) ball_position_frame2 = [ ball_position_frame2[0], ball_position_frame2[1], min(width, height), min(width, height) ] prev_frame2 = frame2 # print ball_position_frame2 if ball_position_frame2: x2, y2, w2, h2 = ball_position_frame2 # x = (x2 - 960) / PIXELS_PER_MM # y = (y2 - 540) / PIXELS_PER_MM pixels_per_mm = (new_calib_matrix[0, 0] + new_calib_matrix[1, 1]) / 2 / FOCAL_LENGTH z = PING_PONG_DIAMETER * FOCAL_LENGTH / (w2 / pixels_per_mm) x = ((x2 - new_calib_matrix[0, 2]) / pixels_per_mm) * z / FOCAL_LENGTH y = ((y2 - new_calib_matrix[1, 2]) / pixels_per_mm) * z / FOCAL_LENGTH ball_cc = np.array([x, y, z]) / 1000 #ball_wc = np.dot(R.T, extrinsics['tvec'].ravel() - ball_cc) ball_wc = np.dot(R.T, ball_cc - extrinsics['tvec'].ravel()) print 'Ball IC', np.array([x2, y2]), 'Dia', w2 print 'Ball CC', ball_cc print 'Ball WC', ball_wc shared_var[0] = ball_wc[0] shared_var[1] = ball_wc[1] shared_var[2] = ball_wc[2] # Update GUI with new image video_disp.refresh(frame2) #print "Pixels", ball_position_frame2 # Add quitting event if video_disp.can_quit(): break global main_process_end_reached main_process_end_reached = True visu.join()
def main(): args = get_args() # Read in configuration load_config = LoadConfig('new_calib_{}.npz'.format(args['model'].lower()), 'calib') calib = load_config.load() # Setup video displays video_disp = Display({'name': 'Video'}) # Setup controls setup_trackbars('Controls') # , thresholds) # Get input video video = Video(args['video']) num_frames = video.get_num_frames() # Get the first frame to start with frame = video.next_frame() global seek_callback_action while True: if play_or_pause == 'Play': if not seek_callback_action: frame = video.next_frame() else: frame = video.get_frame(cur_seek_pos * num_frames / 100) seek_callback_action = False if video.end_reached(): # Wait indefinitely if end of video reached # Or until keypress and then exit cv2.waitKey(0) break # Undistort according to pinhole model if args['model'].upper() == 'P': # Make sure distortion coeffecients # follow pinhole model if calib['dist_coeffs'].shape[1] != 5: print 'Input configuration probably not pinhole' return # Harcoded image size as # this is a test script img_size = (1920, 1080) # First create scaled intrinsics because we will undistort # into region beyond original image region new_calib_matrix, _ = cv2.getOptimalNewCameraMatrix( calib['camera_matrix'], calib['dist_coeffs'], img_size, 0.35) # Then calculate new image size according to the scaling # Unfortunately the Python API doesn't directly provide the # the new image size. They forgot? new_img_size = ( int(img_size[0] + (new_calib_matrix[0, 2] - calib['camera_matrix'][0, 2])), int(img_size[1] + (new_calib_matrix[1, 2] - calib['camera_matrix'][1, 2]))) # Standard routine of creating a new rectification # map for the given intrinsics and mapping each # pixel onto the new map with linear interpolation map1, map2 = cv2.initUndistortRectifyMap(calib['camera_matrix'], calib['dist_coeffs'], None, new_calib_matrix, new_img_size, cv2.CV_16SC2) img_undistorted = cv2.remap(frame, map1, map2, cv2.INTER_LINEAR, cv2.BORDER_CONSTANT) # Undistort according to fisheye model elif args['model'].upper() == 'F': # Make sure distortion coeffecients # follow fisheye model if calib['dist_coeffs'].shape[0] != 4: print 'Input configuration probably not fisheye' return # Harcoded image size as # this is a test script. # As already ranted before # someone messed with the image # size indexing and reversed it. img_size = (1920, 1080) # Also, the basic undistortion DOES NOT work # with the fisheye module # img_undistorted = cv2.fisheye.undistortImage( # frame, calib['camera_matrix'], calib['dist_coeffs']) # First create scaled intrinsics because we will undistort # into region beyond original image region. The alpha # parameter in pinhole model is equivalent to balance parameter here. new_calib_matrix = cv2.fisheye.estimateNewCameraMatrixForUndistortRectify( calib['camera_matrix'], calib['dist_coeffs'], img_size, np.eye(3), balance=1) # Then calculate new image size according to the scaling # Well if they forgot this in pinhole Python API, # can't complain about Fisheye model. Note the reversed # indexing here too. new_img_size = ( int(img_size[0] + (new_calib_matrix[0, 2] - calib['camera_matrix'][0, 2])), int(img_size[1] + (new_calib_matrix[1, 2] - calib['camera_matrix'][1, 2]))) # Standard routine of creating a new rectification # map for the given intrinsics and mapping each # pixel onto the new map with linear interpolation map1, map2 = cv2.fisheye.initUndistortRectifyMap( calib['camera_matrix'], calib['dist_coeffs'], np.eye(3), new_calib_matrix, new_img_size, cv2.CV_16SC2) img_undistorted = cv2.remap(frame, map1, map2, cv2.INTER_LINEAR, cv2.BORDER_CONSTANT) # Update GUI with new image video_disp.refresh(img_undistorted) # Service the s key to save image if video_disp.key_pressed('s'): cur_frame_num = video.get_cur_frame_num() orig_img_file_name = 'image_for_markers_orig.png' undistorted_img_file_name = 'image_for_markers_undistorted.png' if cv2.imwrite(orig_img_file_name, frame): print 'Saved original {} at frame {}'.format( orig_img_file_name, cur_frame_num) if cv2.imwrite(undistorted_img_file_name, img_undistorted): print 'Saved undistorted {} at frame {}'.format( undistorted_img_file_name, cur_frame_num) # Add quitting event if video_disp.can_quit(): break
def main(): args = get_args() # Read in configuration load_config = LoadConfig('config/thresholds.npz', 'thresholds') thresholds = load_config.load() # Setup video displays orig_video_disp = Display({'name': 'Original_Video'}) processed_video_disp = Display({'name': 'Processed_Video'}) # Setup controls setup_trackbars(controls_window_name, thresholds) # Get input video video = Video(args['video']) num_frames = video.get_num_frames() # Get the first frame to start with frame = video.next_frame() global seek_callback_action while True: if play_or_pause == 'Play': if not seek_callback_action: frame = video.next_frame() else: frame = video.get_frame(cur_seek_pos * num_frames / 100) seek_callback_action = False if video.end_reached(): # Wait indefinitely if end of video reached # Or until keypress and then exit cv2.waitKey(0) break # Refresh original video display orig_video_disp.refresh(frame) # Get threshold values h_min, h_max, s_min, s_max, v_min, v_max, erode_size, dilate_size = get_params( controls_window_name) # Convert image to HSV and apply threshold frame_hsv = cv2.cvtColor(frame, cv2.COLOR_BGR2HSV) frame_thresh = cv2.inRange(frame_hsv, (h_min, s_min, v_min), (h_max, s_max, v_max)) # Apply erosion # Create a kernel first and then apply kernel erode_kernel = cv2.getStructuringElement( cv2.MORPH_ELLIPSE, (erode_size + 1, erode_size + 1)) frame_erode = cv2.erode(frame_thresh, erode_kernel) # Apply dilate # Create a kernel first and then apply kernel dilate_kernel = cv2.getStructuringElement( cv2.MORPH_ELLIPSE, (dilate_size + 1, dilate_size + 1)) frame_dilate = cv2.dilate(frame_erode, dilate_kernel) # Refresh thresholded video display processed_video_disp.refresh(frame_dilate) # Add quitting event if orig_video_disp.can_quit() or processed_video_disp.can_quit(): break # On quit, save the params save_config = SaveConfig('new_erode_dilate', 'erode_dilate') save_config.save(dilate_size=dilate_size, erode_size=erode_size)
def main(): args = get_args() # Read in intrinsic calibration configuration if not os.path.exists('config/intrinsic_calib_{}.npz'.format( args['model'].lower())): print 'First perform intrinsic calibration and save it in config' return load_config = LoadConfig( 'config/intrinsic_calib_{}.npz'.format(args['model'].lower()), 'calib') calib = load_config.load() # TODO: Error handling if image not readable by OpenCV img = cv2.imread(args['image']) # Threshold image to isolate the markers # put in by the user at the "standard" positions img_markers = cv2.inRange( img, (MARKER_B_MIN_THRESH, MARKER_G_MIN_THRESH, MARKER_R_MIN_THRESH), (MARKER_B_MAX_THRESH, MARKER_G_MAX_THRESH, MARKER_R_MAX_THRESH)) # Detect the circular markers circles = cv2.HoughCircles(img_markers, cv2.HOUGH_GRADIENT, 1.5, 10, param1=150, param2=15, minRadius=5, maxRadius=35) # Draw circles for visualization if circles is not None: # Make 'circles' easy to index circles = circles[0, :] # Arrange circles X-coordinates - left to right # to match the TABLE markers scheme circles = circles[circles[:, 0].argsort()] # Check if we have exactly 4 markers if circles.shape[0] != 4: print 'Number of markers not 4' return for (xx, yy, rr) in circles: # Convert to ints x, y, r = int(xx), int(yy), int(rr) # Draw circle outlines cv2.circle(img, (x, y), r, (0, 0, 0), 2) # Draw dots at their centers cv2.rectangle(img, (x - 2, y - 2), (x + 2, y + 2), (0, 128, 255), -1) # Find scaled intrinsics according to pinhole model if args['model'].upper() == 'P': # Make sure distortion coeffecients # follow pinhole model if calib['dist_coeffs'].shape[1] != 5: print 'Input configuration probably not pinhole' return if not args['distorted']: # NOTE: Harcoded image size img_size = (1920, 1080) # First create scaled intrinsics because we will undistort # into region beyond original image region new_calib_matrix = cv2.getOptimalNewCameraMatrix( calib['camera_matrix'], calib['dist_coeffs'], img_size, 0.35)[0] # If we have undistorted image, we don't # need to use distortion model for projection new_dist_coeffs = np.zeros((5, 1)) else: new_calib_matrix = calib['camera_matrix'] new_dist_coeffs = calib['dist_coeffs'] # Find scaled intrinsics according to fisheye model elif args['model'].upper() == 'F': # If distorted image supplied, # no support currently. if args['distorted']: print 'Please supply undistored image for fisheye model' return # Make sure distortion coeffecients # follow fisheye model if calib['dist_coeffs'].shape[0] != 4: print 'Input configuration probably not fisheye' return # Harcoded image size as # this is a test script. # TODO: Change this later. img_size = (1920, 1080) # Create scaled intrinsics. The alpha parameter # in pinhole model is equivalent to balance parameter here. new_calib_matrix = cv2.fisheye.estimateNewCameraMatrixForUndistortRectify( calib['camera_matrix'], calib['dist_coeffs'], img_size, np.eye(3), balance=1) # If we have undistorted image, we don't # need to use distortion model for projection new_dist_coeffs = np.zeros((5, 1)) # Solve a point to point correspondence with # ideal points in the 3D world with projections # as we obtained from the user. # NOTE: Found it the hard way that solvePnP # unfortunately requires the image and object # points to be contiguous elements in the memory # This is probably to do with the method using # some old style pointers. obj_points = np.ascontiguousarray(TABLE_MARKERS.reshape((-1, 1, 3))) img_points = np.ascontiguousarray(circles[:, :2].reshape((-1, 1, 2))) # RANSAC achieves better performance than DLT + LM # as the markers in the TABLE might be noisy # rvecs, tvecs = cv2.solvePnP( # obj_points, img_points, calib['camera_matrix'], calib['dist_coeffs'])[1:] rvecs, tvecs = cv2.solvePnPRansac(obj_points, img_points, new_calib_matrix, new_dist_coeffs)[1:3] # Project the origin of the table back # on to imager according to the found # extrinsics rotation and translation projected_points = cv2.projectPoints(AXIS_AT_ORIGIN, rvecs, tvecs, new_calib_matrix, new_dist_coeffs)[0] # Draw the axes at the origin on to image img = cv2.line(img, tuple(projected_points[0].ravel().astype(int)), tuple(projected_points[3].ravel().astype(int)), (255, 0, 0), 5) img = cv2.line(img, tuple(projected_points[1].ravel().astype(int)), tuple(projected_points[3].ravel().astype(int)), (0, 255, 0), 5) img = cv2.line(img, tuple(projected_points[2].ravel().astype(int)), tuple(projected_points[3].ravel().astype(int)), (0, 0, 255), 5) # Display the image with detected markers # and the projected axis. img_display = Display({'name': 'Image'}) img_display.refresh(img) # Save extrinsics in the current folder save_config = SaveConfig('new_extrinsics', 'extrinsics') save_config.save(rvec=rvecs, tvec=tvecs) # Wait indefinitely cv2.waitKey(0)