def _parse_teams(self, frame: Frame) -> Tuple[TeamComp, TeamComp]:
agents = []
for i, agent_im in enumerate(self.REGIONS["teams"].extract(
frame.image)):
blurlevel = cv2.Laplacian(agent_im, cv2.CV_64F).var()
if blurlevel < 100:
agents.append(None)
logger.debug(f"Got agent {i}=None (blurlevel={blurlevel:.2f})")
else:
templates = self.AGENT_TEMPLATES
if i > 4:
templates = self.AGENT_TEMPLATES_FLIP
# cv2.imshow('agent', self.AGENT_TEMPLATES_FLIP['Raze'][0])
match, r_agent = imageops.match_templates(
agent_im,
templates,
method=cv2.TM_SQDIFF_NORMED,
required_match=self.AGENT_TEMPLATE_REQUIRED_MATCH,
use_masks=True,
previous_match_context=(self.__class__.__name__,
"_parse_teams", i),
# verbose=True
)
agent = r_agent
if match > self.AGENT_TEMPLATE_REQUIRED_MATCH:
agent = None
logger.debug(
f"Got agent {i}={agent} (best={r_agent}, match={match:.3f}, blurlevel={blurlevel:.1f})"
)
agents.append(agent)
return cast_teams((agents[:5], agents[5:]))
def process(self, frame: Frame) -> bool:
y = frame.image_yuv[:, :, 0]
champions_eliminated = self.REGIONS[
"champions_eliminated"].extract_one(y)
t, thresh = cv2.threshold(champions_eliminated, 0, 255,
cv2.THRESH_BINARY | cv2.THRESH_OTSU)
# cv2.imshow('thresh', thresh)
match, key = imageops.match_templates(thresh, self.TEMPLATES,
cv2.TM_CCORR_NORMED,
self.REQUIRED_MATCH)
frame.apex.squad_summary_match = round(match, 4)
if match > self.REQUIRED_MATCH:
champions = key in ["champions_of_the_arena"]
duos_empty_area = self.REGIONS["duos_empty_area"].extract_one(
frame.image_yuv[:, :, 0])
duos_sum = np.sum(duos_empty_area > 100)
duos = duos_sum < 100
logger.debug(f"Got duos_sum={duos_sum} => duos={duos}")
shunt = 0
if duos:
duos_shunt_area = self.REGIONS["duos_shunt_area"].extract_one(
frame.image_yuv[:, :, 0])
duos_shunt_sum = np.sum(duos_shunt_area > 100)
duos_shunt = duos_shunt_sum < 100
logger.debug(
f"Got duos_shunt_sum={duos_shunt_sum} => duos_shunt={duos_shunt}"
)
if duos_shunt:
shunt = 270
frame.apex.squad_summary = SquadSummary(
champions=champions,
placed=self._process_yellowtext(
self.REGIONS["placed"].extract_one(frame.image)),
squad_kills=self._process_yellowtext(
self.REGIONS["squad_kills"].extract_one(frame.image)),
player_stats=self._process_player_stats(y, duos, shunt),
elite=False,
mode="duos" if duos else None,
image=lazy_upload(
"squad_summary",
self.REGIONS.blank_out(frame.image),
frame.timestamp,
selection="last",
),
)
self.REGIONS.draw(frame.debug_image)
_draw_squad_summary(frame.debug_image, frame.apex.squad_summary)
return True
return False
def process(self, frame: Frame) -> bool:
y = frame.image_yuv[:, :, 0]
region = self.REGIONS["map_name"].extract_one(y)
_, thresh = cv2.threshold(region, 200, 255, cv2.THRESH_BINARY)
match, map_name = imageops.match_templates(thresh,
self.TEMPLATES,
method=cv2.TM_CCORR_NORMED,
required_match=0.95)
if match > self.REQUIRED_MATCH:
frame.apex.map_loading = MapLoading(map_name)
return True
return False
def _parse_weapon(
self, frame, row, killer_agent_x,
killer_agent) -> Tuple[Optional[str], float, float, float, int]:
weapon_region_left = killer_agent_x + 60
weapon_region_right = row.center[0] - 20
weapon_gray = self._get_region(
frame.image_yuv,
row.center[1] - 15,
row.center[1] + 17,
weapon_region_left,
weapon_region_right,
0,
debug_name="weapon",
debug_image=frame.debug_image,
)
if weapon_gray.shape[1] == 0:
return None, 0, 0, 0, weapon_region_right
weapon_adapt_thresh = np.clip(
np.convolve(np.percentile(weapon_gray, 10, axis=0),
[0.2, 0.6, 0.2],
mode="same"),
160,
200,
)
weapon_thresh = ((weapon_gray - weapon_adapt_thresh > 30) *
255).astype(np.uint8)
kill_modifiers_thresh = weapon_thresh[:, -75:]
_, wallbang_match, _, wallbang_loc = cv2.minMaxLoc(
cv2.matchTemplate(kill_modifiers_thresh, self.WALLBANG_TEMPLATE,
cv2.TM_CCORR_NORMED))
_, headshot_match, _, headshot_loc = cv2.minMaxLoc(
cv2.matchTemplate(kill_modifiers_thresh, self.HEADSHOT_TEMPLATE,
cv2.TM_CCORR_NORMED))
wallbang_match, headshot_match = float(wallbang_match), float(
headshot_match)
logger.debug(
f"wallbang_match={wallbang_match:.2f}, headshot_match={headshot_match:.2f}"
)
right = weapon_thresh.shape[1] - 1
if wallbang_match > self.KILL_MODIFIER_THRESHOLD:
right = min(right, (weapon_thresh.shape[1] - 75) + wallbang_loc[0])
if headshot_match > self.KILL_MODIFIER_THRESHOLD:
right = min(right, (weapon_thresh.shape[1] - 75) + headshot_loc[0])
if right != weapon_thresh.shape[1] - 1:
logger.debug(f"Using right={right} (clipping kill modifier)")
weapon_thresh = weapon_thresh[:, :right]
# cv2.imwrite(f'C:/tmp/agents2/weap.png', weapon_thresh)
# import matplotlib.pyplot as plt
# f, figs = plt.subplots(4)
# figs[0].imshow(weapon_gray)
# figs[1].plot(weapon_adapt_thresh)
# figs[2].imshow(weapon_gray - weapon_adapt_thresh)
# figs[3].imshow(weapon_thresh)
# plt.show()
# cv2.imshow('weapon_thresh', weapon_thresh)
weapon_image = cv2.dilate(
cv2.copyMakeBorder(
weapon_thresh,
5,
5,
5,
5,
cv2.BORDER_CONSTANT,
),
np.ones((2, 2)),
)
contours, hierarchy = imageops.findContours(weapon_image,
cv2.RETR_EXTERNAL,
cv2.CHAIN_APPROX_SIMPLE)
contours_xywh = [(cnt, cv2.boundingRect(cnt)) for cnt in contours]
best_weap_match, best_weap = 0, None
for cnt, (x1, y1, w, h) in sorted(contours_xywh,
key=lambda cnt_xywh: cnt_xywh[1][0],
reverse=True):
x2, y2 = x1 + w, y1 + h
a = cv2.contourArea(cnt)
fromright = weapon_image.shape[1] - x2
ignore = False
if w > 145:
logger.warning(f"Ignoring weapon contour with w={w}")
ignore = True
if fromright < 30:
# contour is far right - could be small agent ability, so be less strict
if a < 100 or h < 10:
logger.debug(
f"Ignoring right weapon contour {cv2.boundingRect(cnt)}, fromright={fromright}, a={a}"
)
ignore = True
else:
logger.debug(
f"Allowing potential ability contour {cv2.boundingRect(cnt)}, fromright={fromright}, a={a}"
)
elif a < 200 or h < 16:
# print('ignore', cv2.boundingRect(cnt), x2, a)
logger.debug(
f"Ignoring weapon contour {cv2.boundingRect(cnt)}, fromright={fromright}, a={a}"
)
ignore = True
if ignore:
if frame.debug_image is not None and a > 1:
cv2.drawContours(
frame.debug_image,
[cnt],
-1,
(0, 128, 255),
1,
offset=(
weapon_region_left - 5,
row.center[1] - 20,
),
)
continue
# Draw contour to image, padding l=5, r=10, t=2, b=2
# The extra width padding prevents abilities matching small parts of large guns
weapon_im = np.zeros((h + 4, w + 15), dtype=np.uint8)
cv2.drawContours(
weapon_im,
[cnt],
-1,
255,
-1,
offset=(
-x1 + 5,
-y1 + 2,
),
)
if weapon_im.shape[1] > 150:
weapon_im = weapon_im[:, :150]
weapon_match, weapon = imageops.match_templates(
weapon_im,
{
w: t
for w, t in self.WEAPON_TEMPLATES.items() if "." not in w
or w.lower().startswith(killer_agent.lower() + ".")
},
cv2.TM_CCORR_NORMED,
template_in_image=False,
required_match=0.96,
verbose=False,
)
if best_weap_match < weapon_match:
best_weap_match, best_weap = weapon_match, weapon
valid = weapon_match > self.WEAPON_THRESHOLD
if frame.debug_image is not None and a > 1:
cv2.drawContours(
frame.debug_image,
[cnt],
-1,
(128, 255, 0) if valid else (0, 0, 255),
1,
offset=(
weapon_region_left - 5,
row.center[1] - 20,
),
)
if valid:
if frame.debug_image is not None:
x, y = 600, row.center[1] - 15
frame.debug_image[y:y + weapon_thresh.shape[0], x:x +
weapon_thresh.shape[1]] = cv2.cvtColor(
weapon_thresh, cv2.COLOR_GRAY2BGR)
x -= weapon_im.shape[1] + 10
frame.debug_image[y:y + weapon_im.shape[0],
x:x + weapon_im.shape[1]] = cv2.cvtColor(
weapon_im, cv2.COLOR_GRAY2BGR)
cv2.line(
frame.debug_image,
(x, y + weapon_im.shape[0] // 2),
(450, self.WEAPON_NAMES.index(weapon) * 40 + 120),
(0, 255, 0),
2,
cv2.LINE_AA,
)
return (
weapon,
float(weapon_match),
float(wallbang_match),
float(headshot_match),
int(weapon_region_left + x1),
)
logger.warning(
f"Unable to find weapon - best match was {best_weap!r} match={best_weap_match:.2f}"
)
return None, 0, 0, 0, weapon_region_right
def process(self, frame: Frame) -> bool:
result_y = self.REGIONS["result"].extract_one(frame.image_yuv[:, :, 0])
_, result_thresh = cv2.threshold(result_y, 220, 255, cv2.THRESH_BINARY)
match, result = imageops.match_templates(
result_thresh,
self.RESULTS,
cv2.TM_CCORR_NORMED,
required_match=self.RESULT_TEMPLATE_REQUIRED_MATCH,
previous_match_context=(self.__class__.__name__, "result"),
)
if match > self.RESULT_TEMPLATE_REQUIRED_MATCH:
logger.debug(f"Round result is {result} with match={match}")
score_ims = self.REGIONS["scores"].extract(frame.image)
score_gray = [
imageops.normalise(np.max(im, axis=2)) for im in score_ims
]
scores = imageops.tesser_ocr_all(
score_gray,
expected_type=int,
engine=din_next_regular_digits,
invert=True,
)
logger.debug(f"Round score is {scores}")
frame.valorant.postgame = Postgame(
victory=result == "victory",
score=(scores[0], scores[1]),
map=imageops.ocr_region(frame, self.REGIONS, "map"),
game_mode=imageops.ocr_region(frame, self.REGIONS,
"game_mode"),
image=lazy_upload("postgame",
self.REGIONS.blank_out(frame.image),
frame.timestamp),
)
draw_postgame(frame.debug_image, frame.valorant.postgame)
sort_mode_gray = np.min(
self.SCOREBOARD_REGIONS["scoreboard_sort_mode"].extract_one(
frame.image),
axis=2)
sort_mode_filt = 255 - imageops.normalise(sort_mode_gray,
bottom=75)
# cv2.imshow('sort_mode_gray', sort_mode_gray)
sort_mode = imageops.tesser_ocr(sort_mode_filt,
engine=imageops.tesseract_lstm)
sort_mode_match = max([
levenshtein.ratio(
textops.strip_string(sort_mode).upper(), expected)
for expected in self.SCOREBOARD_SORT_MODES
])
logger.debug(
f"Got scoreboard sort mode: {sort_mode!r} match={sort_mode_match:.2f}"
)
if sort_mode_match > 0.75:
frame.valorant.scoreboard = self._parse_scoreboard(frame)
draw_scoreboard(frame.debug_image, frame.valorant.scoreboard)
return True
return False
def _parse_scoreboard(self, frame: Frame) -> Scoreboard:
agent_images = self.SCOREBOARD_REGIONS["agents"].extract(frame.image)
name_images = self.SCOREBOARD_REGIONS["names"].extract(frame.image)
stat_images = self.SCOREBOARD_REGIONS["stats"].extract(frame.image)
stat_images_filt = [
self._filter_statrow_image(im) for im in stat_images
]
stat_image_rows = [
stat_images_filt[r * 8:(r + 1) * 8] for r in range(10)
]
# cv2.imshow(
# 'stats',
# np.vstack([
# np.hstack([self._filter_statrow_image(n)] + r)
# for n, r in zip(name_images, stat_image_rows)
# ])
# )
stats = []
for i, (agent_im, name_im, stat_row) in enumerate(
zip(agent_images, name_images, stat_image_rows)):
agent_match, agent = imageops.match_templates(
agent_im,
self.AGENT_TEMPLATES,
method=cv2.TM_SQDIFF,
required_match=self.AGENT_TEMPLATE_REQUIRED_MATCH,
use_masks=True,
previous_match_context=(self.__class__.__name__, "scoreboard",
"agent", i),
)
if agent_match > self.AGENT_TEMPLATE_REQUIRED_MATCH:
agent = None
row_bg = name_im[np.max(name_im, axis=2) < 200]
row_color = np.median(row_bg, axis=0).astype(np.int)
# cv2.imshow('name', self._filter_statrow_image(name_im))
# cv2.waitKey(0)
stat = PlayerStats(
agent,
imageops.tesser_ocr(
self._filter_statrow_image(name_im),
engine=imageops.tesseract_lstm,
),
row_color[0] > row_color[2],
*imageops.tesser_ocr_all(
stat_row,
expected_type=int,
engine=din_next_regular_digits,
),
)
stats.append(stat)
logger.debug(
f"Got player stats: {stat} - agent match={agent_match:.2f}, row colour={tuple(row_color)}"
)
return Scoreboard(
stats,
image=lazy_upload("scoreboard",
self.SCOREBOARD_REGIONS.blank_out(frame.image),
frame.timestamp),
)
def process(self, frame: Frame):
y = frame.image_yuv[:, :, 0]
your_squad_image = self.REGIONS["your_squad"].extract_one(y)
t, thresh = cv2.threshold(your_squad_image, 0, 255, cv2.THRESH_BINARY | cv2.THRESH_OTSU)
match, key = imageops.match_templates(thresh, self.TEMPLATES, cv2.TM_CCORR_NORMED, self.REQUIRED_MATCH)
frame.apex.your_squad_match = round(match, 4)
if match < self.REQUIRED_MATCH:
return False
name1_trios = np.min(self.REGIONS["names"].extract_one(frame.image), axis=2)
name1_duos = np.min(self.REGIONS["names_duos"].extract_one(frame.image), axis=2)
# name1_thresh_value = max(np.max(name1_duos), np.max(name1_trios)) * 0.95
name1_thresh_value = 240
# logger.debug(f"Name thresh: {name1_thresh_value}")
name1_trios_score = int(np.sum(name1_trios > name1_thresh_value))
name1_duos_score = int(np.sum(name1_duos > name1_thresh_value))
logger.debug(f"Trios name score: {name1_trios_score} vs duos name score: {name1_duos_score}")
# self.duos = name1_duos_score and name1_duos_score > name1_trios_score
self.duos = name1_trios_score < 100
logger.info(f"Using duos={self.duos}")
if key == "your_squad":
names_region_name = "names_duos" if self.duos else "names"
names = imageops.tesser_ocr_all(
self.REGIONS[names_region_name].extract(y),
engine=imageops.tesseract_lstm,
invert=True,
)
frame.apex.your_squad = YourSquad(
tuple(self._to_name(n) for n in names),
mode="duos" if self.duos else None,
images=lazy_upload(
"your_squad",
np.hstack(self.REGIONS[names_region_name].extract(frame.image)),
frame.timestamp,
),
)
self.REGIONS.draw(frame.debug_image)
_draw_squad(frame.debug_image, frame.apex.your_squad)
elif key == "your_selection":
frame.apex.your_selection = YourSelection(
name=self._to_name(
imageops.tesser_ocr(
self.REGIONS["names"].extract(y)[1],
engine=imageops.tesseract_lstm,
invert=True,
)
),
image=lazy_upload(
"your_selection",
self.REGIONS["names"].extract(frame.image)[1],
frame.timestamp,
),
)
self.REGIONS.draw(frame.debug_image)
_draw_squad(frame.debug_image, frame.apex.your_selection)
elif key == "champion_squad":
names_region_name = "names_duos" if self.duos else "names"
names = imageops.tesser_ocr_all(
self.REGIONS[names_region_name].extract(y),
engine=imageops.tesseract_lstm,
invert=True,
)
frame.apex.champion_squad = ChampionSquad(
tuple(self._to_name(n) for n in names),
mode="duos" if self.duos else None,
images=lazy_upload(
"champion_squad",
np.hstack(self.REGIONS[names_region_name].extract(frame.image)),
frame.timestamp,
),
)
self.REGIONS.draw(frame.debug_image)
_draw_squad(frame.debug_image, frame.apex.champion_squad)
return True
def process(self, frame: Frame) -> bool:
agent_name_yuv = self.REGIONS["agent_name"].extract_one(
frame.image_yuv)
agent_name_thresh = cv2.inRange(agent_name_yuv, (200, 85, 120),
(255, 115, 150))
# if hasattr(frame, 'source_image'):
# cv2.imshow('agent_name_yuv', agent_name_yuv)
# cv2.imshow('agent_name_thresh', agent_name_thresh)
# cv2.imwrite(
# os.path.join(os.path.dirname(__file__), 'data', 'agent_names', os.path.basename(frame.source_image)),
# agent_name_thresh
# )
match, best_match = imageops.match_templates(
agent_name_thresh,
self.AGENT_NAME_TEMPLATES,
method=cv2.TM_CCORR_NORMED,
required_match=0.95,
# verbose=True,
)
# self.REGIONS.draw(frame.debug_image)
if match > self.AGENT_TEMPLATE_REQUIRED_MATCH:
selected_agent_ims = self.REGIONS["selected_agents"].extract(
frame.image)
selected_agent_ims_gray = [
255 - imageops.normalise(np.max(im, axis=2), bottom=50)
for im in selected_agent_ims
]
selected_agent_texts = imageops.tesser_ocr_all(
selected_agent_ims_gray,
engine=imageops.tesseract_lstm,
)
logger.info(f"Got selected_agent_texts={selected_agent_texts}")
picking = True
for i, text in enumerate(selected_agent_texts):
for word in textops.strip_string(text, string.ascii_letters +
" .").split(" "):
match = levenshtein.ratio(word, best_match)
logger.debug(
f"Player {i}: Got match {match:.2f} for {word!r} = {best_match!r}"
)
if match > 0.7:
logger.info(
f"Found matching locked in agent {text!r} for selecting agent {best_match!r} - selection locked"
)
picking = False
game_mode = imageops.ocr_region(frame, self.REGIONS, "game_mode")
ranks = []
for i, im in enumerate(self.REGIONS["player_ranks"].extract(
frame.image)):
match, matched_rank = imageops.match_templates(
im,
self.RANK_TEMPLATES,
method=cv2.TM_SQDIFF,
use_masks=True,
required_match=15,
previous_match_context=("player_ranks", i),
)
ranks.append((matched_rank, round(match, 3)))
player_name_ims = self.REGIONS["player_names"].extract(frame.image)
player_name_gray = [
255 - imageops.normalise(np.max(im, axis=2), bottom=50)
for im in player_name_ims
]
player_names = imageops.tesser_ocr_all(
player_name_gray, engine=imageops.tesseract_lstm)
frame.valorant.agent_select = AgentSelect(
best_match,
locked_in=not picking,
map=imageops.ocr_region(frame, self.REGIONS, "map"),
game_mode=game_mode,
player_names=player_names,
agents=selected_agent_texts,
ranks=ranks,
image=lazy_upload("agent_select",
self.REGIONS.blank_out(frame.image),
frame.timestamp,
selection="last"),
)
draw_agent_select(frame.debug_image, frame.valorant.agent_select)
return True
return False