def append(self, image: "np.ndarray", timestamp: float) -> None:
intermittent_log(
logger,
f"{self}: Adding image @ {timestamp:.1f}",
frequency=2 if not self.dropped else self.dropped,
caller_extra_id=self.key,
)
if len(self.images) == self.images.maxlen:
self.dropped += 1
self.images.append((timestamp, image))
def capture(
self,
interpolation=cv2.INTER_LINEAR
) -> Tuple[np.ndarray, np.ndarray, bool]:
if self.destroyed:
raise ValueError(
f"Cannot capture from destroyed {self.__class__.__name__}")
if not self.ready:
since_last = time.time() - self.last_reset
if since_last > self.backoff:
if self._reset():
self.backoff = 0
else:
self.backoff = min(max(1, self.backoff) * 2, 10)
intermittent_log(
logger,
f"_reset failed - retrying in {self.backoff}s",
frequency=500,
level=logging.WARNING,
negative_level=logging.DEBUG,
caller_extra_id=(self.hwnd, self.backoff),
)
return zimg, zimg, False
else:
return zimg, zimg, False
if dpi_detection:
ctypes.windll.user32.SetThreadDpiAwarenessContext(-3)
if self.check_needs_reset():
self.ready = False
return zimg, zimg, False
elif self.foreground:
if not dll.capture():
logger.warning(f"Monitor capture failed - resetting")
self.ready = False
return zimg, zimg, False
else:
screen = self.img.copy()
screen = _fix_size(screen)
if not self.fullscreen:
screen = self._clip_windows(screen)
client_area = screen
if self.crop:
client_area = self.crop.apply(client_area)
crop_bars = self.maximised and not self.borderless
return client_area, resize_to_1080(
client_area, crop_bars, interpolation=interpolation), True
else:
return zimg, zimg, False
def _get_monitors() -> List[Rect]:
monitors = []
def _enum_monitor(hMonitor, hdcMonitor, rect, dwData):
monitors.append(Rect.from_RECT(rect[0]))
return True
ctypes.windll.user32.EnumDisplayMonitors(None, None,
MonitorEnumProc(_enum_monitor), 0)
intermittent_log(
logger,
f"Got monitors: {monitors}",
frequency=1500,
level=logging.INFO,
negative_level=logging.DEBUG,
caller_extra_id=(tuple([str(m) for m in monitors]), ),
)
return monitors
def _reset(self) -> bool:
if dpi_detection:
ctypes.windll.user32.SetThreadDpiAwarenessContext(-3)
intermittent_log(
logger,
f"Resetting {self}",
frequency=500,
level=logging.INFO,
negative_level=logging.DEBUG,
caller_extra_id=(
self.hwnd,
self.window_title,
self.executable,
self.foreground_window_pid,
self.process.pid if self.process else None,
),
)
self.ready = False
self.last_reset = time.time()
self.monitors = _get_monitors()
if not self.hwnd or not win32gui.IsWindow(self.hwnd):
if self.window_title or self.executable:
intermittent_log(
logger,
f"HWND invalid - looking for window matching window_title={self.window_title}, executable={self.executable}",
frequency=1500,
level=logging.INFO,
negative_level=logging.DEBUG,
caller_extra_id=(self.hwnd, self.window_title,
self.executable),
)
try:
self.hwnd, self.process = get_hwnd(self.window_title,
self.executable)
except ProcessNotFoundError as e:
logger.warning(f"Could not find window: {e}")
self.hwnd, self.process = None, None
return False
elif self.foreground_window_pid:
intermittent_log(
logger,
f"HWND invalid - looking for matching foreground window with PID={self.foreground_window_pid}",
frequency=1500,
level=logging.INFO,
negative_level=logging.DEBUG,
caller_extra_id=(self.foreground_window_pid, ),
)
foreground_hwnd = win32gui.GetForegroundWindow()
_, foreground_process_id = win32process.GetWindowThreadProcessId(
foreground_hwnd)
if foreground_process_id == self.foreground_window_pid:
logger.info(
f"Found foreground HWND matching PID: {foreground_hwnd}"
)
self.hwnd = foreground_hwnd
else:
intermittent_log(
logger,
f"Could not find foreground window matching PID",
frequency=1500,
level=logging.INFO,
negative_level=logging.DEBUG,
caller_extra_id=(self.foreground_window_pid, ),
)
self.hwnd = False
return False
if self.hwnd:
self.style = win32api.GetWindowLong(self.hwnd, GWL_STYLE)
logger.info(f"Got GWL_STYLE={self.style:08x}")
self.ex_style = win32api.GetWindowLong(self.hwnd, GWL_EXSTYLE)
logger.info(f"Got GWL_EXSTYLE={self.ex_style:08x}")
self.win_rect = Rect(*win32gui.GetWindowRect(self.hwnd))
logger.info(f"Got WindowRect={self.win_rect}")
self.client_rect = Rect(*win32gui.GetClientRect(self.hwnd))
logger.info(f"Got ClientRect={self.client_rect}")
try:
self.true_rect = Rect.from_RECT(
DwmGetWindowAttribute(self.hwnd))
logger.info(f"Got true_rect={self.true_rect}")
except Exception as e:
logger.warning(
f"Failed to get true_rect (DwmGetWindowAttribute): {e} - using GetWindowRect"
)
self.true_rect = self.win_rect
self.window_info = GetWindowInfo(self.hwnd)
logger.info(f"Got WindowInfo={self.window_info}")
self.foreground = False
if not self.style & WS_MINIMIZE:
center = self.win_rect.center
if center:
logger.info(f"Looking for monitor containing {center}")
self.monitor = _get_monitor_containing(
center, self.monitors)
logger.info(f"Got monitor={self.monitor}")
if self.monitor:
self.foreground = True
self.fullscreen = bool(self.ex_style & WS_EX_TOPMOST)
self.borderless = bool(not self.style & WS_BORDER)
if self.fullscreen or self.borderless:
self.maximised = True
else:
self.maximised = bool(self.style & WS_MAXIMIZE)
if self.foreground:
self.crop = self._make_crop()
logger.info(f"Got crop: {self.crop}")
else:
# no window to track - just use first monitor
self.monitor = self.monitors[0]
self.foreground = True
self.fullscreen = True
self.borderless = True
self.maximised = True
self.crop = None
self.source = self._make_source()
if self.foreground:
monitor_center = self.monitor.center
dll.deinit()
logger.info(
f"Trying to get monitor capture for center={monitor_center}")
if dll.init(int(monitor_center[0]), int(monitor_center[1])):
logger.warning("ScreenCapture.dll failed to init")
return False
bufaddr = dll.capture()
if not bufaddr:
logger.warning(
"ScreenCapture.dll capture failed to return image data")
return False
buffsize = dll.get_height() * dll.get_width() * 4
d11buf = PyMemoryView_FromMemory(bufaddr, buffsize, PyBUF_READ)
self.img = np.ndarray((dll.get_height(), dll.get_width(), 4),
np.uint8,
d11buf,
order="C")
logger.info("Done resetting DirectXCapture")
self.ready = True
return True
else:
self.ready = True
return True
def parse_kill(self, index: int, predictions: RowPredictionsDict,
decoded: RowDecodedDict, y: int,
extrapolated: bool) -> Optional[KillRow]:
# hero_pos, details = scipy.signal.find_peaks(1 - predictions['heroes_softmax'][index, :, -1], height=0.5, distance=3)
hero_pos, details = find_peaks(
1 - predictions["heroes_softmax"][index, :, -1],
height=0.5,
distance=3)
heroes = decoded["heroes"][index]
assists = decoded["assists"][index]
abilities = decoded["abilities"][index]
if not len(heroes):
intermittent_log(
logger,
f"Ignoring detected kill row {index} (y={y}, extrapolated={extrapolated}) with no heroes "
f"(best match {np.max(1 - predictions['heroes_softmax'][index, :, -1]):1.2f})",
frequency=30,
level=logging.WARNING,
caller_extra_id=round(y / 10),
)
return None
if len(heroes) > 2:
raise InvalidKillRow(
f"Got {len(heroes)} heroes in kill row {index} (y={y}, extrapolated={extrapolated}): {heroes}"
)
if len(hero_pos) != len(heroes) and not extrapolated:
raise InvalidKillRow(
f"Got {len(hero_pos)} hero peaks in kill row {index} (y={y}, extrapolated={extrapolated}), "
f"but had {len(heroes)} heroes: {hero_pos}, {heroes}")
if len(hero_pos) and len(heroes) <= 2:
if len(hero_pos) == 2:
splitpos = int(
np.mean([self.predictors.heropos2img(x)
for x in hero_pos]))
else:
splitpos = self.predictors.heropos2img(hero_pos[0])
splitpos_textspace = self.predictors.img2textpos(splitpos)
text_logs_left, text_logs_right = (
predictions["text"][index, :splitpos_textspace],
predictions["text"][index, splitpos_textspace:],
)
text_left = "".join(
self.predictors.decode_ctc(
[text_logs_left],
alphabet=self.predictors.outputs[0]["values"])[0])
text_right = "".join(
self.predictors.decode_ctc(
[text_logs_right],
alphabet=self.predictors.outputs[0]["values"])[0])
if len(hero_pos) == 1:
if len(text_left) >= 3:
# one hero, but text was to the left of it so there must be a (unknown) right hero
hero_left = heroes[0]
hero_right = "UNKNOWN"
else:
text_left = None
hero_left = None
hero_right = heroes[0]
else:
hero_left, hero_right = heroes
ability = None
if len(abilities) > 1:
logger.warning(
f"Got multiple abilities {abilities} for row {index} (hero={hero_left!r}"
)
elif len(abilities):
ability = abilities[0]
ability_hero = ability.split(".")[0]
if ability_hero != "ANY" and ability_hero != hero_left:
logger.warning(
f"Got mismatching ability {ability!r} but hero was {hero_left!r} for row {index}"
)
return KillRow(
left=Player(hero_left, text_left) if hero_left else None,
right=Player(hero_right, text_right),
y=int(y),
ability=ability,
assists=list(assists),
resurrect=bool(ability and "resurrect" in ability
and "mercy" == hero_left),
)
def segment(
gray_image: np.ndarray,
segmentation: Optional[str] = "connected_components",
threshold: Optional[str] = "otsu_above_mean",
min_area: float = 10,
height: int = None,
multiline=False,
debug: bool = False,
) -> List[np.ndarray]:
segments = []
# TODO: implement simpler/faster segmentation
if threshold is None:
thresh = gray_image
elif isinstance(threshold, np.ndarray):
thresh = threshold
elif isinstance(threshold, int) and 0 <= threshold <= 255:
_, thresh = cv2.threshold(gray_image, threshold, 255,
cv2.THRESH_BINARY)
elif isinstance(threshold, str) and threshold.startswith("otsu"):
if threshold == "otsu_above_mean":
thresh = imageops.otsu_thresh_lb_fraction(gray_image, 1)
elif threshold == "otsu":
_, thresh = cv2.threshold(gray_image, 0, 255,
cv2.THRESH_BINARY | cv2.THRESH_OTSU)
else:
raise ValueError(
f"Don\t know how to threshold image using { threshold }")
else:
raise ValueError(
f"Don\t know how to threshold image using { threshold }")
if segmentation == "connected_components":
labels, components = imageops.connected_components(thresh)
# TODO: estimate the size of the characters, and discard things that don't match
components = [c for c in components[1:] if c.area > min_area]
if height:
components = [
c for c in components if height * 0.9 < c.h < height * 1.1
]
if not len(components):
intermittent_log(logger,
"Could not find any characters",
frequency=30)
return []
if multiline:
lines: Dict[ConnectedComponent, int] = {}
comps = defaultdict(list)
for c in sorted(components, key=lambda c: c.y):
for other_c, other_line in lines.items():
if other_c.y < c.y + c.h and other_c.y + other_c.h > c.y:
lines[c] = lines[other_c]
break
else:
lines[c] = max(lines.values()) + 1 if lines else 0
comps[lines[c]].append(c)
components = sorted(components, key=lambda c: (lines[c], c.x))
else:
components = sorted(components, key=lambda c: c.x)
average_top = int(np.median([c.y for c in components]))
average_height = int(np.median([c.h for c in components]))
# top = Counter([y for (x, y, w, h, a) in stats[1:]]).most_common(1)[0][0]
# height = Counter([h for (x, y, w, h, a) in stats[1:]]).most_common(1)[0][0]
border_size = average_height * 0.05
height_tolerance = average_height * 0.2
# I is approx 40% as wide as it is high
# 1 can be 30%
min_width = average_height * 0.2
# M is approx 80% as wide as it is high
max_width = average_height * 0.9
top = round(max(0.0, average_top - border_size))
if not height:
height = round(
min(average_height + border_size, gray_image.shape[0]))
for component in components:
if abs(
min(component.h, gray_image.shape[0] - component.y) -
height) > height_tolerance:
logger.debug(
f"Found component with height={component.h} but expected height={height}"
)
continue
if not min_width < component.w < max_width:
logger.debug(
f"Found component with width={component.w} - expected range was [{min_width :1.1f}, {max_width :1.1f}]"
)
continue
if multiline:
ttop = int(component.y - border_size)
y1 = ttop
y2 = ttop + height
else:
y1 = top
y2 = top + height
x1 = round(max(0.0, component.x - border_size))
x2 = round(
min(component.x + component.w + border_size,
gray_image.shape[1]))
if y2 > gray_image.shape[0]:
logger.debug(
f"Found component with y={component.y}, using height={height} would take this outside the image"
)
continue
elif y2 - y1 < 2 or x2 - x1 < 2:
logger.debug(
f"Found component with height={y2 - y1}, width={x2 - x1} - ignoring"
)
continue
elif y1 < 0 or x1 < 0:
logger.debug(
f"Found component with left={x1}, top={y1} - ignoring")
continue
mask = (labels[y1:y2, x1:x2] == component.label).astype(
np.uint8) * 255
mask = cv2.dilate(mask, np.ones((3, 3)))
character = cv2.bitwise_and(gray_image[y1:y2, x1:x2], mask)
segments.append(character)
if debug:
print("-" * 25)
print(f"average_top: {average_top}")
print(f"average_height: {average_height}")
print(f"border_size: {border_size}")
print(f"height_tolerance: {height_tolerance}")
print(f"min_width: {min_width}")
print(f"max_width: {max_width}")
print(f"top: {top}")
print(f"height: {height}")
import matplotlib.pyplot as plt
f, (figs1, figs2) = plt.subplots(2, 2)
ax = figs1[0]
ax.imshow(gray_image, interpolation="none")
ax.set_title("segment image")
ax = figs1[1]
ax.imshow(thresh, interpolation="none")
ax.set_title("segment thresh")
ax = figs2[0]
ax.imshow(labels, interpolation="none")
ax.set_title("segment components")
ax = figs2[1]
ax.imshow(np.hstack(segments) if len(segments) else np.zeros(
(1, 1)),
interpolation="none")
ax.set_title("segments")
plt.show()
else:
raise ValueError(f"Don't know how to segment using {segmentation}")
return segments