def append(self, image: "np.ndarray", timestamp: float) -> None: intermittent_log( logger, f"{self}: Adding image @ {timestamp:.1f}", frequency=2 if not self.dropped else self.dropped, caller_extra_id=self.key, ) if len(self.images) == self.images.maxlen: self.dropped += 1 self.images.append((timestamp, image))
def capture( self, interpolation=cv2.INTER_LINEAR ) -> Tuple[np.ndarray, np.ndarray, bool]: if self.destroyed: raise ValueError( f"Cannot capture from destroyed {self.__class__.__name__}") if not self.ready: since_last = time.time() - self.last_reset if since_last > self.backoff: if self._reset(): self.backoff = 0 else: self.backoff = min(max(1, self.backoff) * 2, 10) intermittent_log( logger, f"_reset failed - retrying in {self.backoff}s", frequency=500, level=logging.WARNING, negative_level=logging.DEBUG, caller_extra_id=(self.hwnd, self.backoff), ) return zimg, zimg, False else: return zimg, zimg, False if dpi_detection: ctypes.windll.user32.SetThreadDpiAwarenessContext(-3) if self.check_needs_reset(): self.ready = False return zimg, zimg, False elif self.foreground: if not dll.capture(): logger.warning(f"Monitor capture failed - resetting") self.ready = False return zimg, zimg, False else: screen = self.img.copy() screen = _fix_size(screen) if not self.fullscreen: screen = self._clip_windows(screen) client_area = screen if self.crop: client_area = self.crop.apply(client_area) crop_bars = self.maximised and not self.borderless return client_area, resize_to_1080( client_area, crop_bars, interpolation=interpolation), True else: return zimg, zimg, False
def _get_monitors() -> List[Rect]: monitors = [] def _enum_monitor(hMonitor, hdcMonitor, rect, dwData): monitors.append(Rect.from_RECT(rect[0])) return True ctypes.windll.user32.EnumDisplayMonitors(None, None, MonitorEnumProc(_enum_monitor), 0) intermittent_log( logger, f"Got monitors: {monitors}", frequency=1500, level=logging.INFO, negative_level=logging.DEBUG, caller_extra_id=(tuple([str(m) for m in monitors]), ), ) return monitors
def _reset(self) -> bool: if dpi_detection: ctypes.windll.user32.SetThreadDpiAwarenessContext(-3) intermittent_log( logger, f"Resetting {self}", frequency=500, level=logging.INFO, negative_level=logging.DEBUG, caller_extra_id=( self.hwnd, self.window_title, self.executable, self.foreground_window_pid, self.process.pid if self.process else None, ), ) self.ready = False self.last_reset = time.time() self.monitors = _get_monitors() if not self.hwnd or not win32gui.IsWindow(self.hwnd): if self.window_title or self.executable: intermittent_log( logger, f"HWND invalid - looking for window matching window_title={self.window_title}, executable={self.executable}", frequency=1500, level=logging.INFO, negative_level=logging.DEBUG, caller_extra_id=(self.hwnd, self.window_title, self.executable), ) try: self.hwnd, self.process = get_hwnd(self.window_title, self.executable) except ProcessNotFoundError as e: logger.warning(f"Could not find window: {e}") self.hwnd, self.process = None, None return False elif self.foreground_window_pid: intermittent_log( logger, f"HWND invalid - looking for matching foreground window with PID={self.foreground_window_pid}", frequency=1500, level=logging.INFO, negative_level=logging.DEBUG, caller_extra_id=(self.foreground_window_pid, ), ) foreground_hwnd = win32gui.GetForegroundWindow() _, foreground_process_id = win32process.GetWindowThreadProcessId( foreground_hwnd) if foreground_process_id == self.foreground_window_pid: logger.info( f"Found foreground HWND matching PID: {foreground_hwnd}" ) self.hwnd = foreground_hwnd else: intermittent_log( logger, f"Could not find foreground window matching PID", frequency=1500, level=logging.INFO, negative_level=logging.DEBUG, caller_extra_id=(self.foreground_window_pid, ), ) self.hwnd = False return False if self.hwnd: self.style = win32api.GetWindowLong(self.hwnd, GWL_STYLE) logger.info(f"Got GWL_STYLE={self.style:08x}") self.ex_style = win32api.GetWindowLong(self.hwnd, GWL_EXSTYLE) logger.info(f"Got GWL_EXSTYLE={self.ex_style:08x}") self.win_rect = Rect(*win32gui.GetWindowRect(self.hwnd)) logger.info(f"Got WindowRect={self.win_rect}") self.client_rect = Rect(*win32gui.GetClientRect(self.hwnd)) logger.info(f"Got ClientRect={self.client_rect}") try: self.true_rect = Rect.from_RECT( DwmGetWindowAttribute(self.hwnd)) logger.info(f"Got true_rect={self.true_rect}") except Exception as e: logger.warning( f"Failed to get true_rect (DwmGetWindowAttribute): {e} - using GetWindowRect" ) self.true_rect = self.win_rect self.window_info = GetWindowInfo(self.hwnd) logger.info(f"Got WindowInfo={self.window_info}") self.foreground = False if not self.style & WS_MINIMIZE: center = self.win_rect.center if center: logger.info(f"Looking for monitor containing {center}") self.monitor = _get_monitor_containing( center, self.monitors) logger.info(f"Got monitor={self.monitor}") if self.monitor: self.foreground = True self.fullscreen = bool(self.ex_style & WS_EX_TOPMOST) self.borderless = bool(not self.style & WS_BORDER) if self.fullscreen or self.borderless: self.maximised = True else: self.maximised = bool(self.style & WS_MAXIMIZE) if self.foreground: self.crop = self._make_crop() logger.info(f"Got crop: {self.crop}") else: # no window to track - just use first monitor self.monitor = self.monitors[0] self.foreground = True self.fullscreen = True self.borderless = True self.maximised = True self.crop = None self.source = self._make_source() if self.foreground: monitor_center = self.monitor.center dll.deinit() logger.info( f"Trying to get monitor capture for center={monitor_center}") if dll.init(int(monitor_center[0]), int(monitor_center[1])): logger.warning("ScreenCapture.dll failed to init") return False bufaddr = dll.capture() if not bufaddr: logger.warning( "ScreenCapture.dll capture failed to return image data") return False buffsize = dll.get_height() * dll.get_width() * 4 d11buf = PyMemoryView_FromMemory(bufaddr, buffsize, PyBUF_READ) self.img = np.ndarray((dll.get_height(), dll.get_width(), 4), np.uint8, d11buf, order="C") logger.info("Done resetting DirectXCapture") self.ready = True return True else: self.ready = True return True
def parse_kill(self, index: int, predictions: RowPredictionsDict, decoded: RowDecodedDict, y: int, extrapolated: bool) -> Optional[KillRow]: # hero_pos, details = scipy.signal.find_peaks(1 - predictions['heroes_softmax'][index, :, -1], height=0.5, distance=3) hero_pos, details = find_peaks( 1 - predictions["heroes_softmax"][index, :, -1], height=0.5, distance=3) heroes = decoded["heroes"][index] assists = decoded["assists"][index] abilities = decoded["abilities"][index] if not len(heroes): intermittent_log( logger, f"Ignoring detected kill row {index} (y={y}, extrapolated={extrapolated}) with no heroes " f"(best match {np.max(1 - predictions['heroes_softmax'][index, :, -1]):1.2f})", frequency=30, level=logging.WARNING, caller_extra_id=round(y / 10), ) return None if len(heroes) > 2: raise InvalidKillRow( f"Got {len(heroes)} heroes in kill row {index} (y={y}, extrapolated={extrapolated}): {heroes}" ) if len(hero_pos) != len(heroes) and not extrapolated: raise InvalidKillRow( f"Got {len(hero_pos)} hero peaks in kill row {index} (y={y}, extrapolated={extrapolated}), " f"but had {len(heroes)} heroes: {hero_pos}, {heroes}") if len(hero_pos) and len(heroes) <= 2: if len(hero_pos) == 2: splitpos = int( np.mean([self.predictors.heropos2img(x) for x in hero_pos])) else: splitpos = self.predictors.heropos2img(hero_pos[0]) splitpos_textspace = self.predictors.img2textpos(splitpos) text_logs_left, text_logs_right = ( predictions["text"][index, :splitpos_textspace], predictions["text"][index, splitpos_textspace:], ) text_left = "".join( self.predictors.decode_ctc( [text_logs_left], alphabet=self.predictors.outputs[0]["values"])[0]) text_right = "".join( self.predictors.decode_ctc( [text_logs_right], alphabet=self.predictors.outputs[0]["values"])[0]) if len(hero_pos) == 1: if len(text_left) >= 3: # one hero, but text was to the left of it so there must be a (unknown) right hero hero_left = heroes[0] hero_right = "UNKNOWN" else: text_left = None hero_left = None hero_right = heroes[0] else: hero_left, hero_right = heroes ability = None if len(abilities) > 1: logger.warning( f"Got multiple abilities {abilities} for row {index} (hero={hero_left!r}" ) elif len(abilities): ability = abilities[0] ability_hero = ability.split(".")[0] if ability_hero != "ANY" and ability_hero != hero_left: logger.warning( f"Got mismatching ability {ability!r} but hero was {hero_left!r} for row {index}" ) return KillRow( left=Player(hero_left, text_left) if hero_left else None, right=Player(hero_right, text_right), y=int(y), ability=ability, assists=list(assists), resurrect=bool(ability and "resurrect" in ability and "mercy" == hero_left), )
def segment( gray_image: np.ndarray, segmentation: Optional[str] = "connected_components", threshold: Optional[str] = "otsu_above_mean", min_area: float = 10, height: int = None, multiline=False, debug: bool = False, ) -> List[np.ndarray]: segments = [] # TODO: implement simpler/faster segmentation if threshold is None: thresh = gray_image elif isinstance(threshold, np.ndarray): thresh = threshold elif isinstance(threshold, int) and 0 <= threshold <= 255: _, thresh = cv2.threshold(gray_image, threshold, 255, cv2.THRESH_BINARY) elif isinstance(threshold, str) and threshold.startswith("otsu"): if threshold == "otsu_above_mean": thresh = imageops.otsu_thresh_lb_fraction(gray_image, 1) elif threshold == "otsu": _, thresh = cv2.threshold(gray_image, 0, 255, cv2.THRESH_BINARY | cv2.THRESH_OTSU) else: raise ValueError( f"Don\t know how to threshold image using { threshold }") else: raise ValueError( f"Don\t know how to threshold image using { threshold }") if segmentation == "connected_components": labels, components = imageops.connected_components(thresh) # TODO: estimate the size of the characters, and discard things that don't match components = [c for c in components[1:] if c.area > min_area] if height: components = [ c for c in components if height * 0.9 < c.h < height * 1.1 ] if not len(components): intermittent_log(logger, "Could not find any characters", frequency=30) return [] if multiline: lines: Dict[ConnectedComponent, int] = {} comps = defaultdict(list) for c in sorted(components, key=lambda c: c.y): for other_c, other_line in lines.items(): if other_c.y < c.y + c.h and other_c.y + other_c.h > c.y: lines[c] = lines[other_c] break else: lines[c] = max(lines.values()) + 1 if lines else 0 comps[lines[c]].append(c) components = sorted(components, key=lambda c: (lines[c], c.x)) else: components = sorted(components, key=lambda c: c.x) average_top = int(np.median([c.y for c in components])) average_height = int(np.median([c.h for c in components])) # top = Counter([y for (x, y, w, h, a) in stats[1:]]).most_common(1)[0][0] # height = Counter([h for (x, y, w, h, a) in stats[1:]]).most_common(1)[0][0] border_size = average_height * 0.05 height_tolerance = average_height * 0.2 # I is approx 40% as wide as it is high # 1 can be 30% min_width = average_height * 0.2 # M is approx 80% as wide as it is high max_width = average_height * 0.9 top = round(max(0.0, average_top - border_size)) if not height: height = round( min(average_height + border_size, gray_image.shape[0])) for component in components: if abs( min(component.h, gray_image.shape[0] - component.y) - height) > height_tolerance: logger.debug( f"Found component with height={component.h} but expected height={height}" ) continue if not min_width < component.w < max_width: logger.debug( f"Found component with width={component.w} - expected range was [{min_width :1.1f}, {max_width :1.1f}]" ) continue if multiline: ttop = int(component.y - border_size) y1 = ttop y2 = ttop + height else: y1 = top y2 = top + height x1 = round(max(0.0, component.x - border_size)) x2 = round( min(component.x + component.w + border_size, gray_image.shape[1])) if y2 > gray_image.shape[0]: logger.debug( f"Found component with y={component.y}, using height={height} would take this outside the image" ) continue elif y2 - y1 < 2 or x2 - x1 < 2: logger.debug( f"Found component with height={y2 - y1}, width={x2 - x1} - ignoring" ) continue elif y1 < 0 or x1 < 0: logger.debug( f"Found component with left={x1}, top={y1} - ignoring") continue mask = (labels[y1:y2, x1:x2] == component.label).astype( np.uint8) * 255 mask = cv2.dilate(mask, np.ones((3, 3))) character = cv2.bitwise_and(gray_image[y1:y2, x1:x2], mask) segments.append(character) if debug: print("-" * 25) print(f"average_top: {average_top}") print(f"average_height: {average_height}") print(f"border_size: {border_size}") print(f"height_tolerance: {height_tolerance}") print(f"min_width: {min_width}") print(f"max_width: {max_width}") print(f"top: {top}") print(f"height: {height}") import matplotlib.pyplot as plt f, (figs1, figs2) = plt.subplots(2, 2) ax = figs1[0] ax.imshow(gray_image, interpolation="none") ax.set_title("segment image") ax = figs1[1] ax.imshow(thresh, interpolation="none") ax.set_title("segment thresh") ax = figs2[0] ax.imshow(labels, interpolation="none") ax.set_title("segment components") ax = figs2[1] ax.imshow(np.hstack(segments) if len(segments) else np.zeros( (1, 1)), interpolation="none") ax.set_title("segments") plt.show() else: raise ValueError(f"Don't know how to segment using {segmentation}") return segments