def parse_result_and_map(
self, frame: Frame) -> Tuple[Optional[str], Optional[str]]:
result_im = self.REGIONS["result"].extract_one(frame.image)
gray = np.max(result_im, axis=2)
# mask out white/gray text (this is map and match time info)
white_text = ((gray > 100) &
(np.ptp(result_im, axis=2) < 20)).astype(np.uint8) * 255
white_text = cv2.erode(white_text, None)
white_text = np.sum(white_text, axis=0) / 255
right = np.argmax(white_text > 2)
if right > 150:
right -= 10
logger.info(
f"Trimming width of result image {gray.shape[1]} -> {right} to cut white text"
)
gray = gray[:, :right]
else:
right = gray.shape[1]
result_text = imageops.tesser_ocr(gray,
whitelist="".join(
set("".join(self.RESULTS))),
invert=True)
result = textops.matches(result_text, self.RESULTS)
if np.min(result) > 2:
logger.warning(
f"Could not identify result from {result_text!r} (match={np.min(result)})"
)
return None, None
result = self.RESULTS[arrayops.argmin(result)]
logger.debug(f"Got result {result} from {result_text!r}")
# TODO: test this with "draw" result
map_image = self.REGIONS["map_name"].extract_one(frame.image)[:,
right:]
gray = np.min(map_image, axis=2)
map_text = textops.strip_string(
imageops.tesser_ocr(gray,
whitelist=string.ascii_uppercase + " :'",
invert=True,
scale=2),
string.ascii_uppercase + " ",
)
logger.debug(f"Parsed map as {map_text}")
return result, map_text
def _parse_killed_name(self, frame, row, killed_agent_x) -> Optional[str]:
killed_name_gray = self._get_region(
frame.image_yuv,
row.center[1] - 10,
row.center[1] + 10,
row.center[0] + 10,
killed_agent_x - 10,
0,
debug_name="killed_name",
debug_image=frame.debug_image,
)
if killed_name_gray.shape[1] == 0:
return None
killed_name_norm = 255 - imageops.normalise(killed_name_gray, min=170)
return textops.strip_string(
imageops.tesser_ocr(killed_name_norm,
engine=imageops.tesseract_lstm).upper(),
alphabet=string.ascii_uppercase + string.digits + "# ",
)
def process(self, frame: Frame) -> bool:
result_y = self.REGIONS["result"].extract_one(frame.image_yuv[:, :, 0])
_, result_thresh = cv2.threshold(result_y, 220, 255, cv2.THRESH_BINARY)
match, result = imageops.match_templates(
result_thresh,
self.RESULTS,
cv2.TM_CCORR_NORMED,
required_match=self.RESULT_TEMPLATE_REQUIRED_MATCH,
previous_match_context=(self.__class__.__name__, "result"),
)
if match > self.RESULT_TEMPLATE_REQUIRED_MATCH:
logger.debug(f"Round result is {result} with match={match}")
score_ims = self.REGIONS["scores"].extract(frame.image)
score_gray = [
imageops.normalise(np.max(im, axis=2)) for im in score_ims
]
scores = imageops.tesser_ocr_all(
score_gray,
expected_type=int,
engine=din_next_regular_digits,
invert=True,
)
logger.debug(f"Round score is {scores}")
frame.valorant.postgame = Postgame(
victory=result == "victory",
score=(scores[0], scores[1]),
map=imageops.ocr_region(frame, self.REGIONS, "map"),
game_mode=imageops.ocr_region(frame, self.REGIONS,
"game_mode"),
image=lazy_upload("postgame",
self.REGIONS.blank_out(frame.image),
frame.timestamp),
)
draw_postgame(frame.debug_image, frame.valorant.postgame)
sort_mode_gray = np.min(
self.SCOREBOARD_REGIONS["scoreboard_sort_mode"].extract_one(
frame.image),
axis=2)
sort_mode_filt = 255 - imageops.normalise(sort_mode_gray,
bottom=75)
# cv2.imshow('sort_mode_gray', sort_mode_gray)
sort_mode = imageops.tesser_ocr(sort_mode_filt,
engine=imageops.tesseract_lstm)
sort_mode_match = max([
levenshtein.ratio(
textops.strip_string(sort_mode).upper(), expected)
for expected in self.SCOREBOARD_SORT_MODES
])
logger.debug(
f"Got scoreboard sort mode: {sort_mode!r} match={sort_mode_match:.2f}"
)
if sort_mode_match > 0.75:
frame.valorant.scoreboard = self._parse_scoreboard(frame)
draw_scoreboard(frame.debug_image, frame.valorant.scoreboard)
return True
return False
class PostgameProcessor(Processor):
REGIONS = ExtractionRegionsCollection(
os.path.join(os.path.dirname(__file__), "data", "regions", "16_9.zip"))
SCOREBOARD_REGIONS = ExtractionRegionsCollection(
os.path.join(os.path.dirname(__file__), "data", "regions",
"scoreboard", "16_9.zip"))
RESULTS = {
"victory":
imageops.imread(
os.path.join(os.path.dirname(__file__), "data", "victory.png"), 0),
"defeat":
imageops.imread(
os.path.join(os.path.dirname(__file__), "data", "defeat.png"), 0),
}
RESULT_TEMPLATE_REQUIRED_MATCH = 0.8
# TAB_SELECTED_TEMPLATE = np.array([0] * 50 + [3] * 73 + [5] * 24 + [3] * 73 + [0] * 50)
# TABS = [
# ('summary', 240),
# ('scoreboard', 335)
# ]
SCOREBOARD_SORT_MODES = [
textops.strip_string("Individually Sorted").upper(),
textops.strip_string("Grouped By Team").upper(),
]
AGENT_TEMPLATES = {
name: load_agent_template(
os.path.join(os.path.dirname(__file__), "data", "agents",
name.lower() + ".png"))
for name in agents
}
AGENT_TEMPLATE_REQUIRED_MATCH = 50
def process(self, frame: Frame) -> bool:
result_y = self.REGIONS["result"].extract_one(frame.image_yuv[:, :, 0])
_, result_thresh = cv2.threshold(result_y, 220, 255, cv2.THRESH_BINARY)
match, result = imageops.match_templates(
result_thresh,
self.RESULTS,
cv2.TM_CCORR_NORMED,
required_match=self.RESULT_TEMPLATE_REQUIRED_MATCH,
previous_match_context=(self.__class__.__name__, "result"),
)
if match > self.RESULT_TEMPLATE_REQUIRED_MATCH:
logger.debug(f"Round result is {result} with match={match}")
score_ims = self.REGIONS["scores"].extract(frame.image)
score_gray = [
imageops.normalise(np.max(im, axis=2)) for im in score_ims
]
scores = imageops.tesser_ocr_all(
score_gray,
expected_type=int,
engine=din_next_regular_digits,
invert=True,
)
logger.debug(f"Round score is {scores}")
frame.valorant.postgame = Postgame(
victory=result == "victory",
score=(scores[0], scores[1]),
map=imageops.ocr_region(frame, self.REGIONS, "map"),
game_mode=imageops.ocr_region(frame, self.REGIONS,
"game_mode"),
image=lazy_upload("postgame",
self.REGIONS.blank_out(frame.image),
frame.timestamp),
)
draw_postgame(frame.debug_image, frame.valorant.postgame)
sort_mode_gray = np.min(
self.SCOREBOARD_REGIONS["scoreboard_sort_mode"].extract_one(
frame.image),
axis=2)
sort_mode_filt = 255 - imageops.normalise(sort_mode_gray,
bottom=75)
# cv2.imshow('sort_mode_gray', sort_mode_gray)
sort_mode = imageops.tesser_ocr(sort_mode_filt,
engine=imageops.tesseract_lstm)
sort_mode_match = max([
levenshtein.ratio(
textops.strip_string(sort_mode).upper(), expected)
for expected in self.SCOREBOARD_SORT_MODES
])
logger.debug(
f"Got scoreboard sort mode: {sort_mode!r} match={sort_mode_match:.2f}"
)
if sort_mode_match > 0.75:
frame.valorant.scoreboard = self._parse_scoreboard(frame)
draw_scoreboard(frame.debug_image, frame.valorant.scoreboard)
return True
return False
def _parse_scoreboard(self, frame: Frame) -> Scoreboard:
agent_images = self.SCOREBOARD_REGIONS["agents"].extract(frame.image)
name_images = self.SCOREBOARD_REGIONS["names"].extract(frame.image)
stat_images = self.SCOREBOARD_REGIONS["stats"].extract(frame.image)
stat_images_filt = [
self._filter_statrow_image(im) for im in stat_images
]
stat_image_rows = [
stat_images_filt[r * 8:(r + 1) * 8] for r in range(10)
]
# cv2.imshow(
# 'stats',
# np.vstack([
# np.hstack([self._filter_statrow_image(n)] + r)
# for n, r in zip(name_images, stat_image_rows)
# ])
# )
stats = []
for i, (agent_im, name_im, stat_row) in enumerate(
zip(agent_images, name_images, stat_image_rows)):
agent_match, agent = imageops.match_templates(
agent_im,
self.AGENT_TEMPLATES,
method=cv2.TM_SQDIFF,
required_match=self.AGENT_TEMPLATE_REQUIRED_MATCH,
use_masks=True,
previous_match_context=(self.__class__.__name__, "scoreboard",
"agent", i),
)
if agent_match > self.AGENT_TEMPLATE_REQUIRED_MATCH:
agent = None
row_bg = name_im[np.max(name_im, axis=2) < 200]
row_color = np.median(row_bg, axis=0).astype(np.int)
# cv2.imshow('name', self._filter_statrow_image(name_im))
# cv2.waitKey(0)
stat = PlayerStats(
agent,
imageops.tesser_ocr(
self._filter_statrow_image(name_im),
engine=imageops.tesseract_lstm,
),
row_color[0] > row_color[2],
*imageops.tesser_ocr_all(
stat_row,
expected_type=int,
engine=din_next_regular_digits,
),
)
stats.append(stat)
logger.debug(
f"Got player stats: {stat} - agent match={agent_match:.2f}, row colour={tuple(row_color)}"
)
return Scoreboard(
stats,
image=lazy_upload("scoreboard",
self.SCOREBOARD_REGIONS.blank_out(frame.image),
frame.timestamp),
)
def _filter_statrow_image(self, im):
im_gray = np.min(im, axis=2).astype(np.float)
bgcol = np.percentile(im_gray, 90)
im_norm = im_gray - bgcol
im_norm = im_norm / np.max(im_norm)
im = 255 - np.clip(im_norm * 255, 0, 255).astype(np.uint8)
return im
def process(self, frame: Frame) -> bool:
agent_name_yuv = self.REGIONS["agent_name"].extract_one(
frame.image_yuv)
agent_name_thresh = cv2.inRange(agent_name_yuv, (200, 85, 120),
(255, 115, 150))
# if hasattr(frame, 'source_image'):
# cv2.imshow('agent_name_yuv', agent_name_yuv)
# cv2.imshow('agent_name_thresh', agent_name_thresh)
# cv2.imwrite(
# os.path.join(os.path.dirname(__file__), 'data', 'agent_names', os.path.basename(frame.source_image)),
# agent_name_thresh
# )
match, best_match = imageops.match_templates(
agent_name_thresh,
self.AGENT_NAME_TEMPLATES,
method=cv2.TM_CCORR_NORMED,
required_match=0.95,
# verbose=True,
)
# self.REGIONS.draw(frame.debug_image)
if match > self.AGENT_TEMPLATE_REQUIRED_MATCH:
selected_agent_ims = self.REGIONS["selected_agents"].extract(
frame.image)
selected_agent_ims_gray = [
255 - imageops.normalise(np.max(im, axis=2), bottom=50)
for im in selected_agent_ims
]
selected_agent_texts = imageops.tesser_ocr_all(
selected_agent_ims_gray,
engine=imageops.tesseract_lstm,
)
logger.info(f"Got selected_agent_texts={selected_agent_texts}")
picking = True
for i, text in enumerate(selected_agent_texts):
for word in textops.strip_string(text, string.ascii_letters +
" .").split(" "):
match = levenshtein.ratio(word, best_match)
logger.debug(
f"Player {i}: Got match {match:.2f} for {word!r} = {best_match!r}"
)
if match > 0.7:
logger.info(
f"Found matching locked in agent {text!r} for selecting agent {best_match!r} - selection locked"
)
picking = False
game_mode = imageops.ocr_region(frame, self.REGIONS, "game_mode")
ranks = []
for i, im in enumerate(self.REGIONS["player_ranks"].extract(
frame.image)):
match, matched_rank = imageops.match_templates(
im,
self.RANK_TEMPLATES,
method=cv2.TM_SQDIFF,
use_masks=True,
required_match=15,
previous_match_context=("player_ranks", i),
)
ranks.append((matched_rank, round(match, 3)))
player_name_ims = self.REGIONS["player_names"].extract(frame.image)
player_name_gray = [
255 - imageops.normalise(np.max(im, axis=2), bottom=50)
for im in player_name_ims
]
player_names = imageops.tesser_ocr_all(
player_name_gray, engine=imageops.tesseract_lstm)
frame.valorant.agent_select = AgentSelect(
best_match,
locked_in=not picking,
map=imageops.ocr_region(frame, self.REGIONS, "map"),
game_mode=game_mode,
player_names=player_names,
agents=selected_agent_texts,
ranks=ranks,
image=lazy_upload("agent_select",
self.REGIONS.blank_out(frame.image),
frame.timestamp,
selection="last"),
)
draw_agent_select(frame.debug_image, frame.valorant.agent_select)
return True
return False
def _parse_score_report(self, y: np.ndarray) -> ScoreReport:
rp_report_image = self.REGIONS["rp_fields"].extract_one(y)
lines = []
for line in range(3):
line_im = rp_report_image[line * 40 + 5:(line + 1) * 40 - 7, 5:]
lines.append(
imageops.tesser_ocr(line_im,
engine=imageops.tesseract_lstm,
invert=True,
scale=2))
score_report = ScoreReport()
for line in lines:
valid = False
if ":" in line:
stat_name, stat_value = line.lower().replace(" ",
"").split(":", 1)
if stat_name == "entrycost":
score_report.entry_rank = stat_value.lower()
valid = True
elif stat_name == "kills":
try:
score_report.kills = int(stat_value.replace("o", "0"))
except ValueError:
logger.warning(
f'Could not parse Score Report > kills: {stat_value!r}" as int'
)
else:
valid = True
elif stat_name == "matchplacement":
stat_value = stat_value.replace("#", "")
try:
score_report.placement = int(
stat_value.replace("o", "0").split("/", 1)[0])
except ValueError:
logger.warning(
f'Could not parse Score Report > placement: {stat_value!r}" as placement'
)
else:
valid = True
if not valid:
logger.warning(f"Unknown line in score report: {line!r}")
score_adjustment_image = self.REGIONS["score_adjustment"].extract_one(
y)
score_adjustment_text = imageops.tesser_ocr(
score_adjustment_image,
engine=imageops.tesseract_lstm,
invert=True,
scale=1)
score_adjustment_text_strip = (textops.strip_string(
score_adjustment_text, alphabet=string.digits + "RP+-").replace(
"RP", "").replace("+", "").replace("-", ""))
try:
score_report.rp_adjustment = int(score_adjustment_text_strip)
except ValueError:
logger.warning(
f'Could not parse Score Report > score adjustment: {score_adjustment_text!r}" as valid adjustment'
)
current_rp_image = self.REGIONS["current_rp"].extract_one(y)
current_rp_text = imageops.tesser_ocr(current_rp_image,
engine=imageops.tesseract_lstm,
invert=True,
scale=1)
current_rp_text_strip = textops.strip_string(current_rp_text,
alphabet=string.digits +
"RP").replace("RP", "")
try:
score_report.current_rp = int(current_rp_text_strip)
except ValueError:
logger.warning(
f'Could not parse Score Report > current RP: {current_rp_text!r}" as valid RP'
)
return score_report
def _process_player_stats(self,
y: np.ndarray,
duos: bool = False,
shunt: int = 0) -> Tuple[PlayerStats, ...]:
name_images = self.REGIONS["names"].shunt(x=shunt).extract(y)
names = []
for im in name_images:
# self._mask_components_touching_edges(im)
im = 255 - cv2.bitwise_and(
im,
cv2.dilate(
cv2.threshold(im, 0, 255,
cv2.THRESH_BINARY | cv2.THRESH_OTSU)[1],
None,
),
)
im = cv2.resize(im, (0, 0), fx=2, fy=2)
im = cv2.GaussianBlur(im, (0, 0), 1)
name = imageops.tesser_ocr(
im,
engine=imageops.tesseract_lstm,
).replace(" ", "")
match = np.mean(imageops.tesseract_lstm.AllWordConfidences())
logger.info(f"Got name {name!r} ~ {match:1.2f}")
if match < 0.75:
name = imageops.tesser_ocr(
im,
engine=imageops.tesseract_only,
)
logger.info(f"Using {name!r} instead")
names.append(name)
stat_images = self.REGIONS["stats"].shunt(x=shunt).extract(y)
# for im in stat_images:
# self._mask_components_touching_edges(im)
stats = imageops.tesser_ocr_all(
stat_images,
engine=ocr.tesseract_ttlakes_digits_specials,
)
for i in range(len(stats)):
value = stats[i]
logger.debug(f"Got stat {i}: {value!r}")
if value:
value = value.lower().replace(" ", "")
for c1, c2 in "l1", "i1", "o0", (":", ""):
value = value.replace(c1, c2)
value = textops.strip_string(value, string.digits + "/")
if i < 3:
try:
stats[i] = tuple([int(v) for v in value.split("/")])
except ValueError as e:
logger.warning(f'Could not parse {value!r} as 3 ints" {e}')
stats[i] = None
elif 6 <= i <= 8:
# survival time
if stats[i] is not None:
try:
seconds = int(value)
except ValueError as e:
logger.warning(
f'Could not parse "{stats[i]}" as int: {e}')
seconds = None
else:
seconds = mmss_to_seconds(seconds)
logger.info(f"MM:SS {stats[i]} -> {seconds}")
stats[i] = seconds
else:
try:
stats[i] = int(value)
except ValueError as e:
logger.warning(f'Could not parse {value!r} as int" {e}')
stats[i] = None
# typing: ignore
# noinspection PyTypeChecker
count = 3 if not duos else 2
r = tuple([PlayerStats(names[i], *stats[i::3]) for i in range(count)])
for s in r:
if not s.kills:
pass
elif len(s.kills) == 3:
s.assists = s.kills[1]
s.knocks = s.kills[2]
s.kills = s.kills[0]
else:
s.kills = s.kills[0]
logger.info(f"Got {pprint.pformat(r)}")
return r