def process(self, frame: Frame):
y = cv2.cvtColor(frame.image, cv2.COLOR_BGR2YUV)[:, :, 0]
weapon_images = self.REGIONS["weapon_names"].extract(y)
weapon_images = [255 - imageops.normalise(i) for i in weapon_images]
weapon_names = imageops.tesser_ocr_all(
weapon_images,
whitelist=string.ascii_uppercase,
engine=imageops.tesseract_lstm,
scale=2,
)
selected_weapons_regions = self.REGIONS[
"selected_weapon_tell"].extract(frame.image)
selected_weapons_colours = [
np.median(r, axis=(0, 1)) for r in selected_weapons_regions
]
def thresh_clip(im):
im = np.max(im, axis=2)
threshim = np.tile(im[:, 0], (im.shape[1], 1)).T
im = cv2.subtract(im, threshim)
tim = im > 20
return tim
frame.apex.weapons = Weapons(
weapon_names,
selected_weapons=(
(
int(selected_weapons_colours[0][0]),
int(selected_weapons_colours[0][1]),
int(selected_weapons_colours[0][2]),
),
(
int(selected_weapons_colours[1][0]),
int(selected_weapons_colours[1][1]),
int(selected_weapons_colours[1][2]),
),
),
clip=self._ocr_digits(
[im > 200 for im in self.REGIONS["clip"].extract(y)],
self.CLIP_WEIGHTS),
ammo=self._ocr_digits(
[
thresh_clip(im)
for im in self.REGIONS["ammo"].extract(frame.image)
],
self.AMMO_WEIGHTS,
),
)
# self.REGIONS.draw(frame.debug_image)
_draw_weapons(frame.debug_image, frame.apex.weapons)
return frame.apex.weapons.selected_weapons is not None
def process(self, frame: Frame) -> bool:
y = frame.image_yuv[:, :, 0]
tank_region = np.max(self.REGIONS["tank_region"].extract_one(frame.image), axis=2)
_, thresh = cv2.threshold(tank_region, 100, 255, cv2.THRESH_BINARY)
# cv2.imshow('thresh1', thresh)
tank_match_sm = cv2.matchTemplate(thresh, self.TANK_TEMPLATE, cv2.TM_CCORR_NORMED)
_, match_sm, _, mxloc_sm = cv2.minMaxLoc(tank_match_sm)
tank_match_lg = cv2.matchTemplate(thresh, self.TANK_LARGE_TEMPLATE, cv2.TM_CCORR_NORMED)
_, match_lg, _, mxloc_lg = cv2.minMaxLoc(tank_match_lg)
lock_match = cv2.matchTemplate(thresh, self.LOCK_TEMPLATE, cv2.TM_CCORR_NORMED)
_, match_lock, _, mxloc_lock = cv2.minMaxLoc(lock_match)
matched_i = arrayops.argmax([match_sm, match_lg, match_lock])
# print([match_sm, match_lg, match_lock])
match = [match_sm, match_lg, match_lock][matched_i]
matched = ["tank", "tank_lg", "lock"][matched_i]
best_match_pos = [mxloc_sm, mxloc_lg, mxloc_lock][matched_i]
match_x = best_match_pos[0]
# print(matched, match_x)
frame.overwatch.role_select_match = round(match, 2)
if match > self.REQUIRED_MATCH:
grouped = match_x < 150
logger.debug(
f"Found match for {matched!r} with match={match:0.3f} ({match_sm:.2f}, {match_lg:.2f}, {match_lock:.2f}), x={match_x} => grouped={grouped}"
)
suffix = "_group" if grouped else "_solo"
frame.overwatch.role_select = RoleSelect(
placement_text=imageops.tesser_ocr_all(
self.REGIONS["placements" + suffix].extract(y), whitelist=string.digits + "/-"
),
sr_text=big_noodle.ocr_all(self.REGIONS["srs" + suffix].extract(y), height=23, invert=True),
account_name=imageops.tesser_ocr(
self.REGIONS["account_name"].extract_one(y), engine=imageops.tesseract_lstm
),
grouped=grouped,
image=lazy_upload(
"role_select", self.REGIONS.blank_out(frame.image), frame.timestamp, selection="last"
),
)
if frame.debug_image is not None:
self.REGIONS.draw(frame.debug_image)
_draw_role_select(frame.debug_image, frame.overwatch.role_select)
return True
return False
def parse_score(self, frame: Frame) -> Tuple[Optional[int], Optional[int]]:
score_ims = self.REGIONS["scores"].extract(frame.image)
score_gray = [np.min(im, axis=2) for im in score_ims]
score_norm = [
imageops.normalise(im, bottom=80, top=100) for im in score_gray
]
# debugops.normalise(score_gray[0])
# cv2.imshow('score_ims', np.hstack(score_ims))
# cv2.imshow('score_gray', np.hstack(score_gray))
# cv2.imshow('score_ys_norm', np.hstack(score_norm))
score = imageops.tesser_ocr_all(score_norm,
expected_type=int,
invert=True,
engine=din_next_regular_digits)
logger.debug(f"Got score={score}")
return score[0], score[1]
def process(self, frame: Frame) -> bool:
result_y = self.REGIONS["result"].extract_one(frame.image_yuv[:, :, 0])
_, result_thresh = cv2.threshold(result_y, 220, 255, cv2.THRESH_BINARY)
match, result = imageops.match_templates(
result_thresh,
self.RESULTS,
cv2.TM_CCORR_NORMED,
required_match=self.RESULT_TEMPLATE_REQUIRED_MATCH,
previous_match_context=(self.__class__.__name__, "result"),
)
if match > self.RESULT_TEMPLATE_REQUIRED_MATCH:
logger.debug(f"Round result is {result} with match={match}")
score_ims = self.REGIONS["scores"].extract(frame.image)
score_gray = [
imageops.normalise(np.max(im, axis=2)) for im in score_ims
]
scores = imageops.tesser_ocr_all(
score_gray,
expected_type=int,
engine=din_next_regular_digits,
invert=True,
)
logger.debug(f"Round score is {scores}")
frame.valorant.postgame = Postgame(
victory=result == "victory",
score=(scores[0], scores[1]),
map=imageops.ocr_region(frame, self.REGIONS, "map"),
game_mode=imageops.ocr_region(frame, self.REGIONS,
"game_mode"),
image=lazy_upload("postgame",
self.REGIONS.blank_out(frame.image),
frame.timestamp),
)
draw_postgame(frame.debug_image, frame.valorant.postgame)
sort_mode_gray = np.min(
self.SCOREBOARD_REGIONS["scoreboard_sort_mode"].extract_one(
frame.image),
axis=2)
sort_mode_filt = 255 - imageops.normalise(sort_mode_gray,
bottom=75)
# cv2.imshow('sort_mode_gray', sort_mode_gray)
sort_mode = imageops.tesser_ocr(sort_mode_filt,
engine=imageops.tesseract_lstm)
sort_mode_match = max([
levenshtein.ratio(
textops.strip_string(sort_mode).upper(), expected)
for expected in self.SCOREBOARD_SORT_MODES
])
logger.debug(
f"Got scoreboard sort mode: {sort_mode!r} match={sort_mode_match:.2f}"
)
if sort_mode_match > 0.75:
frame.valorant.scoreboard = self._parse_scoreboard(frame)
draw_scoreboard(frame.debug_image, frame.valorant.scoreboard)
return True
return False
def _parse_scoreboard(self, frame: Frame) -> Scoreboard:
agent_images = self.SCOREBOARD_REGIONS["agents"].extract(frame.image)
name_images = self.SCOREBOARD_REGIONS["names"].extract(frame.image)
stat_images = self.SCOREBOARD_REGIONS["stats"].extract(frame.image)
stat_images_filt = [
self._filter_statrow_image(im) for im in stat_images
]
stat_image_rows = [
stat_images_filt[r * 8:(r + 1) * 8] for r in range(10)
]
# cv2.imshow(
# 'stats',
# np.vstack([
# np.hstack([self._filter_statrow_image(n)] + r)
# for n, r in zip(name_images, stat_image_rows)
# ])
# )
stats = []
for i, (agent_im, name_im, stat_row) in enumerate(
zip(agent_images, name_images, stat_image_rows)):
agent_match, agent = imageops.match_templates(
agent_im,
self.AGENT_TEMPLATES,
method=cv2.TM_SQDIFF,
required_match=self.AGENT_TEMPLATE_REQUIRED_MATCH,
use_masks=True,
previous_match_context=(self.__class__.__name__, "scoreboard",
"agent", i),
)
if agent_match > self.AGENT_TEMPLATE_REQUIRED_MATCH:
agent = None
row_bg = name_im[np.max(name_im, axis=2) < 200]
row_color = np.median(row_bg, axis=0).astype(np.int)
# cv2.imshow('name', self._filter_statrow_image(name_im))
# cv2.waitKey(0)
stat = PlayerStats(
agent,
imageops.tesser_ocr(
self._filter_statrow_image(name_im),
engine=imageops.tesseract_lstm,
),
row_color[0] > row_color[2],
*imageops.tesser_ocr_all(
stat_row,
expected_type=int,
engine=din_next_regular_digits,
),
)
stats.append(stat)
logger.debug(
f"Got player stats: {stat} - agent match={agent_match:.2f}, row colour={tuple(row_color)}"
)
return Scoreboard(
stats,
image=lazy_upload("scoreboard",
self.SCOREBOARD_REGIONS.blank_out(frame.image),
frame.timestamp),
)
def process(self, frame: Frame):
y = frame.image_yuv[:, :, 0]
your_squad_image = self.REGIONS["your_squad"].extract_one(y)
t, thresh = cv2.threshold(your_squad_image, 0, 255, cv2.THRESH_BINARY | cv2.THRESH_OTSU)
match, key = imageops.match_templates(thresh, self.TEMPLATES, cv2.TM_CCORR_NORMED, self.REQUIRED_MATCH)
frame.apex.your_squad_match = round(match, 4)
if match < self.REQUIRED_MATCH:
return False
name1_trios = np.min(self.REGIONS["names"].extract_one(frame.image), axis=2)
name1_duos = np.min(self.REGIONS["names_duos"].extract_one(frame.image), axis=2)
# name1_thresh_value = max(np.max(name1_duos), np.max(name1_trios)) * 0.95
name1_thresh_value = 240
# logger.debug(f"Name thresh: {name1_thresh_value}")
name1_trios_score = int(np.sum(name1_trios > name1_thresh_value))
name1_duos_score = int(np.sum(name1_duos > name1_thresh_value))
logger.debug(f"Trios name score: {name1_trios_score} vs duos name score: {name1_duos_score}")
# self.duos = name1_duos_score and name1_duos_score > name1_trios_score
self.duos = name1_trios_score < 100
logger.info(f"Using duos={self.duos}")
if key == "your_squad":
names_region_name = "names_duos" if self.duos else "names"
names = imageops.tesser_ocr_all(
self.REGIONS[names_region_name].extract(y),
engine=imageops.tesseract_lstm,
invert=True,
)
frame.apex.your_squad = YourSquad(
tuple(self._to_name(n) for n in names),
mode="duos" if self.duos else None,
images=lazy_upload(
"your_squad",
np.hstack(self.REGIONS[names_region_name].extract(frame.image)),
frame.timestamp,
),
)
self.REGIONS.draw(frame.debug_image)
_draw_squad(frame.debug_image, frame.apex.your_squad)
elif key == "your_selection":
frame.apex.your_selection = YourSelection(
name=self._to_name(
imageops.tesser_ocr(
self.REGIONS["names"].extract(y)[1],
engine=imageops.tesseract_lstm,
invert=True,
)
),
image=lazy_upload(
"your_selection",
self.REGIONS["names"].extract(frame.image)[1],
frame.timestamp,
),
)
self.REGIONS.draw(frame.debug_image)
_draw_squad(frame.debug_image, frame.apex.your_selection)
elif key == "champion_squad":
names_region_name = "names_duos" if self.duos else "names"
names = imageops.tesser_ocr_all(
self.REGIONS[names_region_name].extract(y),
engine=imageops.tesseract_lstm,
invert=True,
)
frame.apex.champion_squad = ChampionSquad(
tuple(self._to_name(n) for n in names),
mode="duos" if self.duos else None,
images=lazy_upload(
"champion_squad",
np.hstack(self.REGIONS[names_region_name].extract(frame.image)),
frame.timestamp,
),
)
self.REGIONS.draw(frame.debug_image)
_draw_squad(frame.debug_image, frame.apex.champion_squad)
return True
def process(self, frame: Frame) -> bool:
agent_name_yuv = self.REGIONS["agent_name"].extract_one(
frame.image_yuv)
agent_name_thresh = cv2.inRange(agent_name_yuv, (200, 85, 120),
(255, 115, 150))
# if hasattr(frame, 'source_image'):
# cv2.imshow('agent_name_yuv', agent_name_yuv)
# cv2.imshow('agent_name_thresh', agent_name_thresh)
# cv2.imwrite(
# os.path.join(os.path.dirname(__file__), 'data', 'agent_names', os.path.basename(frame.source_image)),
# agent_name_thresh
# )
match, best_match = imageops.match_templates(
agent_name_thresh,
self.AGENT_NAME_TEMPLATES,
method=cv2.TM_CCORR_NORMED,
required_match=0.95,
# verbose=True,
)
# self.REGIONS.draw(frame.debug_image)
if match > self.AGENT_TEMPLATE_REQUIRED_MATCH:
selected_agent_ims = self.REGIONS["selected_agents"].extract(
frame.image)
selected_agent_ims_gray = [
255 - imageops.normalise(np.max(im, axis=2), bottom=50)
for im in selected_agent_ims
]
selected_agent_texts = imageops.tesser_ocr_all(
selected_agent_ims_gray,
engine=imageops.tesseract_lstm,
)
logger.info(f"Got selected_agent_texts={selected_agent_texts}")
picking = True
for i, text in enumerate(selected_agent_texts):
for word in textops.strip_string(text, string.ascii_letters +
" .").split(" "):
match = levenshtein.ratio(word, best_match)
logger.debug(
f"Player {i}: Got match {match:.2f} for {word!r} = {best_match!r}"
)
if match > 0.7:
logger.info(
f"Found matching locked in agent {text!r} for selecting agent {best_match!r} - selection locked"
)
picking = False
game_mode = imageops.ocr_region(frame, self.REGIONS, "game_mode")
ranks = []
for i, im in enumerate(self.REGIONS["player_ranks"].extract(
frame.image)):
match, matched_rank = imageops.match_templates(
im,
self.RANK_TEMPLATES,
method=cv2.TM_SQDIFF,
use_masks=True,
required_match=15,
previous_match_context=("player_ranks", i),
)
ranks.append((matched_rank, round(match, 3)))
player_name_ims = self.REGIONS["player_names"].extract(frame.image)
player_name_gray = [
255 - imageops.normalise(np.max(im, axis=2), bottom=50)
for im in player_name_ims
]
player_names = imageops.tesser_ocr_all(
player_name_gray, engine=imageops.tesseract_lstm)
frame.valorant.agent_select = AgentSelect(
best_match,
locked_in=not picking,
map=imageops.ocr_region(frame, self.REGIONS, "map"),
game_mode=game_mode,
player_names=player_names,
agents=selected_agent_texts,
ranks=ranks,
image=lazy_upload("agent_select",
self.REGIONS.blank_out(frame.image),
frame.timestamp,
selection="last"),
)
draw_agent_select(frame.debug_image, frame.valorant.agent_select)
return True
return False
def _process_player_stats(self,
y: np.ndarray,
duos: bool = False,
shunt: int = 0) -> Tuple[PlayerStats, ...]:
name_images = self.REGIONS["names"].shunt(x=shunt).extract(y)
names = []
for im in name_images:
# self._mask_components_touching_edges(im)
im = 255 - cv2.bitwise_and(
im,
cv2.dilate(
cv2.threshold(im, 0, 255,
cv2.THRESH_BINARY | cv2.THRESH_OTSU)[1],
None,
),
)
im = cv2.resize(im, (0, 0), fx=2, fy=2)
im = cv2.GaussianBlur(im, (0, 0), 1)
name = imageops.tesser_ocr(
im,
engine=imageops.tesseract_lstm,
).replace(" ", "")
match = np.mean(imageops.tesseract_lstm.AllWordConfidences())
logger.info(f"Got name {name!r} ~ {match:1.2f}")
if match < 0.75:
name = imageops.tesser_ocr(
im,
engine=imageops.tesseract_only,
)
logger.info(f"Using {name!r} instead")
names.append(name)
stat_images = self.REGIONS["stats"].shunt(x=shunt).extract(y)
# for im in stat_images:
# self._mask_components_touching_edges(im)
stats = imageops.tesser_ocr_all(
stat_images,
engine=ocr.tesseract_ttlakes_digits_specials,
)
for i in range(len(stats)):
value = stats[i]
logger.debug(f"Got stat {i}: {value!r}")
if value:
value = value.lower().replace(" ", "")
for c1, c2 in "l1", "i1", "o0", (":", ""):
value = value.replace(c1, c2)
value = textops.strip_string(value, string.digits + "/")
if i < 3:
try:
stats[i] = tuple([int(v) for v in value.split("/")])
except ValueError as e:
logger.warning(f'Could not parse {value!r} as 3 ints" {e}')
stats[i] = None
elif 6 <= i <= 8:
# survival time
if stats[i] is not None:
try:
seconds = int(value)
except ValueError as e:
logger.warning(
f'Could not parse "{stats[i]}" as int: {e}')
seconds = None
else:
seconds = mmss_to_seconds(seconds)
logger.info(f"MM:SS {stats[i]} -> {seconds}")
stats[i] = seconds
else:
try:
stats[i] = int(value)
except ValueError as e:
logger.warning(f'Could not parse {value!r} as int" {e}')
stats[i] = None
# typing: ignore
# noinspection PyTypeChecker
count = 3 if not duos else 2
r = tuple([PlayerStats(names[i], *stats[i::3]) for i in range(count)])
for s in r:
if not s.kills:
pass
elif len(s.kills) == 3:
s.assists = s.kills[1]
s.knocks = s.kills[2]
s.kills = s.kills[0]
else:
s.kills = s.kills[0]
logger.info(f"Got {pprint.pformat(r)}")
return r