def _parse_teams(self, frame: Frame) -> Tuple[TeamComp, TeamComp]: agents = [] for i, agent_im in enumerate(self.REGIONS["teams"].extract( frame.image)): blurlevel = cv2.Laplacian(agent_im, cv2.CV_64F).var() if blurlevel < 100: agents.append(None) logger.debug(f"Got agent {i}=None (blurlevel={blurlevel:.2f})") else: templates = self.AGENT_TEMPLATES if i > 4: templates = self.AGENT_TEMPLATES_FLIP # cv2.imshow('agent', self.AGENT_TEMPLATES_FLIP['Raze'][0]) match, r_agent = imageops.match_templates( agent_im, templates, method=cv2.TM_SQDIFF_NORMED, required_match=self.AGENT_TEMPLATE_REQUIRED_MATCH, use_masks=True, previous_match_context=(self.__class__.__name__, "_parse_teams", i), # verbose=True ) agent = r_agent if match > self.AGENT_TEMPLATE_REQUIRED_MATCH: agent = None logger.debug( f"Got agent {i}={agent} (best={r_agent}, match={match:.3f}, blurlevel={blurlevel:.1f})" ) agents.append(agent) return cast_teams((agents[:5], agents[5:]))
def process(self, frame: Frame) -> bool: y = frame.image_yuv[:, :, 0] champions_eliminated = self.REGIONS[ "champions_eliminated"].extract_one(y) t, thresh = cv2.threshold(champions_eliminated, 0, 255, cv2.THRESH_BINARY | cv2.THRESH_OTSU) # cv2.imshow('thresh', thresh) match, key = imageops.match_templates(thresh, self.TEMPLATES, cv2.TM_CCORR_NORMED, self.REQUIRED_MATCH) frame.apex.squad_summary_match = round(match, 4) if match > self.REQUIRED_MATCH: champions = key in ["champions_of_the_arena"] duos_empty_area = self.REGIONS["duos_empty_area"].extract_one( frame.image_yuv[:, :, 0]) duos_sum = np.sum(duos_empty_area > 100) duos = duos_sum < 100 logger.debug(f"Got duos_sum={duos_sum} => duos={duos}") shunt = 0 if duos: duos_shunt_area = self.REGIONS["duos_shunt_area"].extract_one( frame.image_yuv[:, :, 0]) duos_shunt_sum = np.sum(duos_shunt_area > 100) duos_shunt = duos_shunt_sum < 100 logger.debug( f"Got duos_shunt_sum={duos_shunt_sum} => duos_shunt={duos_shunt}" ) if duos_shunt: shunt = 270 frame.apex.squad_summary = SquadSummary( champions=champions, placed=self._process_yellowtext( self.REGIONS["placed"].extract_one(frame.image)), squad_kills=self._process_yellowtext( self.REGIONS["squad_kills"].extract_one(frame.image)), player_stats=self._process_player_stats(y, duos, shunt), elite=False, mode="duos" if duos else None, image=lazy_upload( "squad_summary", self.REGIONS.blank_out(frame.image), frame.timestamp, selection="last", ), ) self.REGIONS.draw(frame.debug_image) _draw_squad_summary(frame.debug_image, frame.apex.squad_summary) return True return False
def process(self, frame: Frame) -> bool: y = frame.image_yuv[:, :, 0] region = self.REGIONS["map_name"].extract_one(y) _, thresh = cv2.threshold(region, 200, 255, cv2.THRESH_BINARY) match, map_name = imageops.match_templates(thresh, self.TEMPLATES, method=cv2.TM_CCORR_NORMED, required_match=0.95) if match > self.REQUIRED_MATCH: frame.apex.map_loading = MapLoading(map_name) return True return False
def _parse_weapon( self, frame, row, killer_agent_x, killer_agent) -> Tuple[Optional[str], float, float, float, int]: weapon_region_left = killer_agent_x + 60 weapon_region_right = row.center[0] - 20 weapon_gray = self._get_region( frame.image_yuv, row.center[1] - 15, row.center[1] + 17, weapon_region_left, weapon_region_right, 0, debug_name="weapon", debug_image=frame.debug_image, ) if weapon_gray.shape[1] == 0: return None, 0, 0, 0, weapon_region_right weapon_adapt_thresh = np.clip( np.convolve(np.percentile(weapon_gray, 10, axis=0), [0.2, 0.6, 0.2], mode="same"), 160, 200, ) weapon_thresh = ((weapon_gray - weapon_adapt_thresh > 30) * 255).astype(np.uint8) kill_modifiers_thresh = weapon_thresh[:, -75:] _, wallbang_match, _, wallbang_loc = cv2.minMaxLoc( cv2.matchTemplate(kill_modifiers_thresh, self.WALLBANG_TEMPLATE, cv2.TM_CCORR_NORMED)) _, headshot_match, _, headshot_loc = cv2.minMaxLoc( cv2.matchTemplate(kill_modifiers_thresh, self.HEADSHOT_TEMPLATE, cv2.TM_CCORR_NORMED)) wallbang_match, headshot_match = float(wallbang_match), float( headshot_match) logger.debug( f"wallbang_match={wallbang_match:.2f}, headshot_match={headshot_match:.2f}" ) right = weapon_thresh.shape[1] - 1 if wallbang_match > self.KILL_MODIFIER_THRESHOLD: right = min(right, (weapon_thresh.shape[1] - 75) + wallbang_loc[0]) if headshot_match > self.KILL_MODIFIER_THRESHOLD: right = min(right, (weapon_thresh.shape[1] - 75) + headshot_loc[0]) if right != weapon_thresh.shape[1] - 1: logger.debug(f"Using right={right} (clipping kill modifier)") weapon_thresh = weapon_thresh[:, :right] # cv2.imwrite(f'C:/tmp/agents2/weap.png', weapon_thresh) # import matplotlib.pyplot as plt # f, figs = plt.subplots(4) # figs[0].imshow(weapon_gray) # figs[1].plot(weapon_adapt_thresh) # figs[2].imshow(weapon_gray - weapon_adapt_thresh) # figs[3].imshow(weapon_thresh) # plt.show() # cv2.imshow('weapon_thresh', weapon_thresh) weapon_image = cv2.dilate( cv2.copyMakeBorder( weapon_thresh, 5, 5, 5, 5, cv2.BORDER_CONSTANT, ), np.ones((2, 2)), ) contours, hierarchy = imageops.findContours(weapon_image, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE) contours_xywh = [(cnt, cv2.boundingRect(cnt)) for cnt in contours] best_weap_match, best_weap = 0, None for cnt, (x1, y1, w, h) in sorted(contours_xywh, key=lambda cnt_xywh: cnt_xywh[1][0], reverse=True): x2, y2 = x1 + w, y1 + h a = cv2.contourArea(cnt) fromright = weapon_image.shape[1] - x2 ignore = False if w > 145: logger.warning(f"Ignoring weapon contour with w={w}") ignore = True if fromright < 30: # contour is far right - could be small agent ability, so be less strict if a < 100 or h < 10: logger.debug( f"Ignoring right weapon contour {cv2.boundingRect(cnt)}, fromright={fromright}, a={a}" ) ignore = True else: logger.debug( f"Allowing potential ability contour {cv2.boundingRect(cnt)}, fromright={fromright}, a={a}" ) elif a < 200 or h < 16: # print('ignore', cv2.boundingRect(cnt), x2, a) logger.debug( f"Ignoring weapon contour {cv2.boundingRect(cnt)}, fromright={fromright}, a={a}" ) ignore = True if ignore: if frame.debug_image is not None and a > 1: cv2.drawContours( frame.debug_image, [cnt], -1, (0, 128, 255), 1, offset=( weapon_region_left - 5, row.center[1] - 20, ), ) continue # Draw contour to image, padding l=5, r=10, t=2, b=2 # The extra width padding prevents abilities matching small parts of large guns weapon_im = np.zeros((h + 4, w + 15), dtype=np.uint8) cv2.drawContours( weapon_im, [cnt], -1, 255, -1, offset=( -x1 + 5, -y1 + 2, ), ) if weapon_im.shape[1] > 150: weapon_im = weapon_im[:, :150] weapon_match, weapon = imageops.match_templates( weapon_im, { w: t for w, t in self.WEAPON_TEMPLATES.items() if "." not in w or w.lower().startswith(killer_agent.lower() + ".") }, cv2.TM_CCORR_NORMED, template_in_image=False, required_match=0.96, verbose=False, ) if best_weap_match < weapon_match: best_weap_match, best_weap = weapon_match, weapon valid = weapon_match > self.WEAPON_THRESHOLD if frame.debug_image is not None and a > 1: cv2.drawContours( frame.debug_image, [cnt], -1, (128, 255, 0) if valid else (0, 0, 255), 1, offset=( weapon_region_left - 5, row.center[1] - 20, ), ) if valid: if frame.debug_image is not None: x, y = 600, row.center[1] - 15 frame.debug_image[y:y + weapon_thresh.shape[0], x:x + weapon_thresh.shape[1]] = cv2.cvtColor( weapon_thresh, cv2.COLOR_GRAY2BGR) x -= weapon_im.shape[1] + 10 frame.debug_image[y:y + weapon_im.shape[0], x:x + weapon_im.shape[1]] = cv2.cvtColor( weapon_im, cv2.COLOR_GRAY2BGR) cv2.line( frame.debug_image, (x, y + weapon_im.shape[0] // 2), (450, self.WEAPON_NAMES.index(weapon) * 40 + 120), (0, 255, 0), 2, cv2.LINE_AA, ) return ( weapon, float(weapon_match), float(wallbang_match), float(headshot_match), int(weapon_region_left + x1), ) logger.warning( f"Unable to find weapon - best match was {best_weap!r} match={best_weap_match:.2f}" ) return None, 0, 0, 0, weapon_region_right
def process(self, frame: Frame) -> bool: result_y = self.REGIONS["result"].extract_one(frame.image_yuv[:, :, 0]) _, result_thresh = cv2.threshold(result_y, 220, 255, cv2.THRESH_BINARY) match, result = imageops.match_templates( result_thresh, self.RESULTS, cv2.TM_CCORR_NORMED, required_match=self.RESULT_TEMPLATE_REQUIRED_MATCH, previous_match_context=(self.__class__.__name__, "result"), ) if match > self.RESULT_TEMPLATE_REQUIRED_MATCH: logger.debug(f"Round result is {result} with match={match}") score_ims = self.REGIONS["scores"].extract(frame.image) score_gray = [ imageops.normalise(np.max(im, axis=2)) for im in score_ims ] scores = imageops.tesser_ocr_all( score_gray, expected_type=int, engine=din_next_regular_digits, invert=True, ) logger.debug(f"Round score is {scores}") frame.valorant.postgame = Postgame( victory=result == "victory", score=(scores[0], scores[1]), map=imageops.ocr_region(frame, self.REGIONS, "map"), game_mode=imageops.ocr_region(frame, self.REGIONS, "game_mode"), image=lazy_upload("postgame", self.REGIONS.blank_out(frame.image), frame.timestamp), ) draw_postgame(frame.debug_image, frame.valorant.postgame) sort_mode_gray = np.min( self.SCOREBOARD_REGIONS["scoreboard_sort_mode"].extract_one( frame.image), axis=2) sort_mode_filt = 255 - imageops.normalise(sort_mode_gray, bottom=75) # cv2.imshow('sort_mode_gray', sort_mode_gray) sort_mode = imageops.tesser_ocr(sort_mode_filt, engine=imageops.tesseract_lstm) sort_mode_match = max([ levenshtein.ratio( textops.strip_string(sort_mode).upper(), expected) for expected in self.SCOREBOARD_SORT_MODES ]) logger.debug( f"Got scoreboard sort mode: {sort_mode!r} match={sort_mode_match:.2f}" ) if sort_mode_match > 0.75: frame.valorant.scoreboard = self._parse_scoreboard(frame) draw_scoreboard(frame.debug_image, frame.valorant.scoreboard) return True return False
def _parse_scoreboard(self, frame: Frame) -> Scoreboard: agent_images = self.SCOREBOARD_REGIONS["agents"].extract(frame.image) name_images = self.SCOREBOARD_REGIONS["names"].extract(frame.image) stat_images = self.SCOREBOARD_REGIONS["stats"].extract(frame.image) stat_images_filt = [ self._filter_statrow_image(im) for im in stat_images ] stat_image_rows = [ stat_images_filt[r * 8:(r + 1) * 8] for r in range(10) ] # cv2.imshow( # 'stats', # np.vstack([ # np.hstack([self._filter_statrow_image(n)] + r) # for n, r in zip(name_images, stat_image_rows) # ]) # ) stats = [] for i, (agent_im, name_im, stat_row) in enumerate( zip(agent_images, name_images, stat_image_rows)): agent_match, agent = imageops.match_templates( agent_im, self.AGENT_TEMPLATES, method=cv2.TM_SQDIFF, required_match=self.AGENT_TEMPLATE_REQUIRED_MATCH, use_masks=True, previous_match_context=(self.__class__.__name__, "scoreboard", "agent", i), ) if agent_match > self.AGENT_TEMPLATE_REQUIRED_MATCH: agent = None row_bg = name_im[np.max(name_im, axis=2) < 200] row_color = np.median(row_bg, axis=0).astype(np.int) # cv2.imshow('name', self._filter_statrow_image(name_im)) # cv2.waitKey(0) stat = PlayerStats( agent, imageops.tesser_ocr( self._filter_statrow_image(name_im), engine=imageops.tesseract_lstm, ), row_color[0] > row_color[2], *imageops.tesser_ocr_all( stat_row, expected_type=int, engine=din_next_regular_digits, ), ) stats.append(stat) logger.debug( f"Got player stats: {stat} - agent match={agent_match:.2f}, row colour={tuple(row_color)}" ) return Scoreboard( stats, image=lazy_upload("scoreboard", self.SCOREBOARD_REGIONS.blank_out(frame.image), frame.timestamp), )
def process(self, frame: Frame): y = frame.image_yuv[:, :, 0] your_squad_image = self.REGIONS["your_squad"].extract_one(y) t, thresh = cv2.threshold(your_squad_image, 0, 255, cv2.THRESH_BINARY | cv2.THRESH_OTSU) match, key = imageops.match_templates(thresh, self.TEMPLATES, cv2.TM_CCORR_NORMED, self.REQUIRED_MATCH) frame.apex.your_squad_match = round(match, 4) if match < self.REQUIRED_MATCH: return False name1_trios = np.min(self.REGIONS["names"].extract_one(frame.image), axis=2) name1_duos = np.min(self.REGIONS["names_duos"].extract_one(frame.image), axis=2) # name1_thresh_value = max(np.max(name1_duos), np.max(name1_trios)) * 0.95 name1_thresh_value = 240 # logger.debug(f"Name thresh: {name1_thresh_value}") name1_trios_score = int(np.sum(name1_trios > name1_thresh_value)) name1_duos_score = int(np.sum(name1_duos > name1_thresh_value)) logger.debug(f"Trios name score: {name1_trios_score} vs duos name score: {name1_duos_score}") # self.duos = name1_duos_score and name1_duos_score > name1_trios_score self.duos = name1_trios_score < 100 logger.info(f"Using duos={self.duos}") if key == "your_squad": names_region_name = "names_duos" if self.duos else "names" names = imageops.tesser_ocr_all( self.REGIONS[names_region_name].extract(y), engine=imageops.tesseract_lstm, invert=True, ) frame.apex.your_squad = YourSquad( tuple(self._to_name(n) for n in names), mode="duos" if self.duos else None, images=lazy_upload( "your_squad", np.hstack(self.REGIONS[names_region_name].extract(frame.image)), frame.timestamp, ), ) self.REGIONS.draw(frame.debug_image) _draw_squad(frame.debug_image, frame.apex.your_squad) elif key == "your_selection": frame.apex.your_selection = YourSelection( name=self._to_name( imageops.tesser_ocr( self.REGIONS["names"].extract(y)[1], engine=imageops.tesseract_lstm, invert=True, ) ), image=lazy_upload( "your_selection", self.REGIONS["names"].extract(frame.image)[1], frame.timestamp, ), ) self.REGIONS.draw(frame.debug_image) _draw_squad(frame.debug_image, frame.apex.your_selection) elif key == "champion_squad": names_region_name = "names_duos" if self.duos else "names" names = imageops.tesser_ocr_all( self.REGIONS[names_region_name].extract(y), engine=imageops.tesseract_lstm, invert=True, ) frame.apex.champion_squad = ChampionSquad( tuple(self._to_name(n) for n in names), mode="duos" if self.duos else None, images=lazy_upload( "champion_squad", np.hstack(self.REGIONS[names_region_name].extract(frame.image)), frame.timestamp, ), ) self.REGIONS.draw(frame.debug_image) _draw_squad(frame.debug_image, frame.apex.champion_squad) return True
def process(self, frame: Frame) -> bool: agent_name_yuv = self.REGIONS["agent_name"].extract_one( frame.image_yuv) agent_name_thresh = cv2.inRange(agent_name_yuv, (200, 85, 120), (255, 115, 150)) # if hasattr(frame, 'source_image'): # cv2.imshow('agent_name_yuv', agent_name_yuv) # cv2.imshow('agent_name_thresh', agent_name_thresh) # cv2.imwrite( # os.path.join(os.path.dirname(__file__), 'data', 'agent_names', os.path.basename(frame.source_image)), # agent_name_thresh # ) match, best_match = imageops.match_templates( agent_name_thresh, self.AGENT_NAME_TEMPLATES, method=cv2.TM_CCORR_NORMED, required_match=0.95, # verbose=True, ) # self.REGIONS.draw(frame.debug_image) if match > self.AGENT_TEMPLATE_REQUIRED_MATCH: selected_agent_ims = self.REGIONS["selected_agents"].extract( frame.image) selected_agent_ims_gray = [ 255 - imageops.normalise(np.max(im, axis=2), bottom=50) for im in selected_agent_ims ] selected_agent_texts = imageops.tesser_ocr_all( selected_agent_ims_gray, engine=imageops.tesseract_lstm, ) logger.info(f"Got selected_agent_texts={selected_agent_texts}") picking = True for i, text in enumerate(selected_agent_texts): for word in textops.strip_string(text, string.ascii_letters + " .").split(" "): match = levenshtein.ratio(word, best_match) logger.debug( f"Player {i}: Got match {match:.2f} for {word!r} = {best_match!r}" ) if match > 0.7: logger.info( f"Found matching locked in agent {text!r} for selecting agent {best_match!r} - selection locked" ) picking = False game_mode = imageops.ocr_region(frame, self.REGIONS, "game_mode") ranks = [] for i, im in enumerate(self.REGIONS["player_ranks"].extract( frame.image)): match, matched_rank = imageops.match_templates( im, self.RANK_TEMPLATES, method=cv2.TM_SQDIFF, use_masks=True, required_match=15, previous_match_context=("player_ranks", i), ) ranks.append((matched_rank, round(match, 3))) player_name_ims = self.REGIONS["player_names"].extract(frame.image) player_name_gray = [ 255 - imageops.normalise(np.max(im, axis=2), bottom=50) for im in player_name_ims ] player_names = imageops.tesser_ocr_all( player_name_gray, engine=imageops.tesseract_lstm) frame.valorant.agent_select = AgentSelect( best_match, locked_in=not picking, map=imageops.ocr_region(frame, self.REGIONS, "map"), game_mode=game_mode, player_names=player_names, agents=selected_agent_texts, ranks=ranks, image=lazy_upload("agent_select", self.REGIONS.blank_out(frame.image), frame.timestamp, selection="last"), ) draw_agent_select(frame.debug_image, frame.valorant.agent_select) return True return False