def __init__(self, settings, window_name, button_constructor, debug = False): """Make screenshot and find zones on screen""" super(MainWindow, self).__init__(settings, window_name, button_constructor, debug) self.x = None self.y = None self.cv = OpenCV()
def _find_image_result(self, img, screen_img, threshold): result = OpenCV().find_template(img, screen_img, threshold) if result is not None: return FindResult(*result, image=img, screen=screen_img, found=True) else: return FindResult(None, None, None, None, None, image=img, screen=screen_img, found=False) # nothing found
def find_multiple_images(self, image, threshold=0.99, cache=False, zone=None, screen=None): # _find_image(image, threshold, cache, zone) -> list(FindResult) threshold = float(threshold) cache = utils.to_bool(cache) screen_img = self._get_screen(cache, zone, screen) img = self.load_image(image) poses = OpenCV().find_multiple_templates(img, screen_img, threshold) result = [] for pos in poses: result.append(FindResult(*pos, image=img, screen=screen_img, found=True)) return result
def __init__(self, error_handler, output_dir): self.cv = OpenCV() self.cache_screenshot = None self.error_handler = error_handler self.output_dir = output_dir
class ImageProcessor(object): ###Image processing system - open files if needed, find them on screen, etc def __init__(self, error_handler, output_dir): self.cv = OpenCV() self.cache_screenshot = None self.error_handler = error_handler self.output_dir = output_dir def _screenshot(self, zone=None): # _screenshot([area]) -> Image # Get screenshot of specified area or whole game window if rect is None # Coordinates are calculating from left-top corner of window win_area = GUIProcess().get_window_area() if zone is not None: im = pyautogui.screenshot(region=(win_area[0] + int(zone[0]), win_area[1] + int(zone[1]), int(zone[2]), int(zone[3]))) else: im = pyautogui.screenshot(region=win_area) return im def get_screenshot(self): # for external use, without areas # self.win_area = GUIProcess().get_window_area() # return pyautogui.screenshot(region=self.win_area) return pyautogui.screenshot() def load_image(self, image): try: return Image.open(image) except IOError: pass # self.error_handler.report_error("Not opened image {}".format(image)) # raise CanNotOpenImageException(image) def _get_screen(self, cache=None, zone=None, screen=None): # _get_screen(cache, zone, screen) -> PIL image # cache - bool, use cached image or make new # zone - tuple(x, y, w, h) # screen - PIL image. Don't use with cache option # # Cache - user logic, screen - for internal use search_zone = zone scr = screen if scr is not None: img = scr return img elif cache: if self.cache_screenshot is None: raise CacheError img = self.cache_screenshot return img else: return self._screenshot(search_zone) # if area is not None: # return img.crop((search_zone[0], search_zone[1], search_zone[0] + search_zone[2], search_zone[1] + search_zone[3])) # else: # return img def take_cache_screenshot(self): return self.get_screenshot() def _find_image_result(self, img, screen_img, threshold): result = OpenCV().find_template(img, screen_img, threshold) if result is not None: return FindResult(*result, image=img, screen=screen_img, found=True) else: return FindResult(None, None, None, None, None, image=img, screen=screen_img, found=False) # nothing found def _find_image(self, image, threshold=0.95, cache=False, zone=None, screen=None): # _find_image(image, threshold, cache, zone) -> FindResult threshold = float(threshold) cache = utils.to_bool(cache) assert threshold > 0 and threshold <= 1, "Threshold must be in (0, 1)" screen_img = self._get_screen(cache, zone, screen) img = self.load_image(image) return self._find_image_result(img, screen_img, threshold) def find_image(self, image, threshold=0.95, cache=False, zone=None, screen=None): result = ImageProcessor(self.error_handler, self.output_dir)._find_image( image, threshold, cache, zone, screen).found if result: LOGGER.info( 'Template was found on screen with threshold:{}'.format( threshold)) return True else: LOGGER.info( 'Template was not found on screen with threshold {}'.format( threshold)) return False def _is_image_on_screen(self, image, threshold=0.95, cache=False, zone=None, screen=None): return self._find_image(image, threshold, cache, zone, screen).found def _image_should_be_on_screen(self, image, threshold=0.95, cache=False, zone=None, screen=None): result = self._find_image(image, threshold, cache, zone, screen) if result.found: return True self.error_handler.report_warning("First try was unsuccesful") # try again utils.sleep(0.020) result = self._find_image(image, threshold, cache, zone) if result.found: return True image_info = ("image", result.image) screen_info = ("screen", result.screen) msg = "Image was not found at screen with threshold {}".format( threshold) self.error_handler.report_error(msg, image_info, screen_info) raise RuntimeError(msg) def _image_should_not_be_on_screen(self, image, threshold=0.95, cache=False, zone=None, screen=None): result = self._find_image(image, threshold, cache, zone, screen) if not result.found: return True self.error_handler.report_warning("First try was unsuccesful") # try again utils.sleep(0.020) result = self._find_image(image, threshold, cache, zone, screen) if not result.found: return True image_info = ("image", result.image) screen_info = ("screen", result.screen) msg = "Image was found at screen with threshold {}".format(threshold) self.error_handler.report_error(msg, image_info, screen_info) raise RuntimeError(msg) def _wait_for_image(self, image, threshold=0.95, timeout=15, zone=None): timeout = float(timeout) start_time = datetime.datetime.now() img = self.load_image(image) while True: screen_img = self._get_screen(False, zone) result = self._find_image_result(img, screen_img, threshold) if result.found: return True utils.sleep(0) if (datetime.datetime.now() - start_time).seconds > int(timeout): break image_info = ("image", result.image) screen_info = ("screen", result.screen) msg = "Waiting for image was unsucessfull with threshold {} and timeout {} sec".format( threshold, int(timeout)) self.error_handler.report_warning(msg, image_info, screen_info) return False def _wait_for_image_to_hide(self, image, threshold=0.95, timeout=15, zone=None): timeout = float(timeout) start_time = datetime.datetime.now() img = self.load_image(image) while True: screen_img = self._screenshot(zone) result = self._find_image_result(img, screen_img, threshold) if not result.found: return True utils.sleep(0) if (datetime.datetime.now() - start_time).seconds > int(timeout): break image_info = ("image", result.image) screen_info = ("screen", result.screen) msg = "Waiting for image hide was unsucessfull for threshold {} and timeout {}".format( threshold, int(timeout)) self.error_handler.report_warning(msg, image_info, screen_info) return False def _wait_for_image_to_stop(self, image, threshold=0.95, timeout=15, move_threshold=0.99, step=0.1): timeout = float(timeout) threshold = float(threshold) move_threshold = float(move_threshold) step = float(step) assert threshold > 0 and threshold <= 1, "Threshold must be in (0, 1]" assert move_threshold > 0 and move_threshold <= 1, "Move threshold must be in (0, 1]" img = self.load_image(image) start_time = datetime.datetime.now() new_screen = self._screenshot() new_pos = self._find_image_result(img, new_screen, threshold) while True: old_scren = new_screen old_pos = new_pos utils.sleep(step) new_screen = self._screenshot() new_pos = self._find_image_result(img, new_screen, threshold) if old_pos.found and new_pos.found: # template is on screen, not blinking and whatever else ds = math.hypot(new_pos.x - old_pos.x, new_pos.y - old_pos.y) diag = 1280 # hypot for 1024x768 if 1 - ds / diag > threshold: return True if (datetime.datetime.now() - start_time).seconds > int(timeout): break image_info = ("image", new_pos.image) screen_info = ("screen", new_pos.screen) msg = "Waiting for image stop was unsucessfull for threshold {} and timeout {}".format( threshold, int(timeout)) self.error_handler.report_warning(msg, image_info, screen_info) return False def find_multiple_images(self, image, threshold=0.95, cache=False, zone=None, screen=None): # _find_image(image, threshold, cache, zone) -> list(FindResult) threshold = float(threshold) cache = utils.to_bool(cache) screen_img = self._get_screen(cache, zone, screen) img = self.load_image(image) poses = OpenCV().find_multiple_templates(img, screen_img, threshold) result = [] for pos in poses: result.append( FindResult(*pos, image=img, screen=screen_img, found=True)) return result def _get_images_count(self, image, threshold=0.95, cache=False, zone=None, screen=None): return len( self.find_multiple_images(image, threshold, cache, zone, screen)) def find_one_of(self, images, cache=False, zone=None, screen=None): ###Find one of images. The one with max threshold wins and will be returned or None if no one found assert len(images) > 0, "At least one image must be set" screen_img = self._get_screen(False, zone, screen) results = [] for image_info in images: result = self._find_image_result(image_info[0], screen_img, float(image_info[1])) if result.found: results.append(result) if not results: return None return sorted(results, key=lambda res: res.threshold, reverse=True)[0] def find_all_of(self, images, cache=False, zone=None, screen=None): pass def wait_for_one_of(self, images, timeout=15, zone=None): # wait_for_one_of_images(images, timeout, step, zone) -> bool # images - list of (image, threshold, bool) # image - PIL image # threshold - threshold for image # bool - True for wait for show, False for wait for hide assert len( images ) > 0, "You are trying to wait for empty list of images. Really?" timeout = float(timeout) start_time = datetime.datetime.now() while True: screen_img = self._screenshot(zone) for image_info in images: # todo: optimize result = self._find_image_result(image_info[0], screen_img, float(image_info[1])) if result.found == utils.to_bool(image_info[2]): return result utils.sleep(0) if (datetime.datetime.now() - start_time).seconds > timeout: break images_info = [] for index, image in enumerate(images): images_info.append( ("image_{}_threshold_{}".format(index, image[1]), image[0])) images_info.append(("screen", screen_img)) self.error_handler.report_warning( "Waiting for one of the images was unsuccessful", *images_info) return result def wait_for_all_of(self, images, timeout=15, zone=None): pass def resize_image(self, resize_percent, origFile): origresize = Image.open(origFile) width, height = origresize.size width_resize = width * int(resize_percent) / width + width height_resize = height * int(resize_percent) / height + height origresize = origresize.resize( (int(round(width_resize)), int(round(height_resize))), Image.ANTIALIAS) return origresize def get_image_to_recognize(self, zone, cache, resize_percent, contrast, invert, brightness, change_mode, win_area): im = self.check_to_resize(zone, resize_percent) if change_mode: im = self.cv.prepare_image_to_recognize(im) if invert: im = self.cv.prepare_image_to_recognize(im) im = ImageOps.invert(im) if contrast != 0 and brightness != 0: im = ScreenshotOperations().change_brightness(im, brightness) im = ScreenshotOperations().change_contrast(im, contrast) else: if contrast != 0: im = ScreenshotOperations().change_contrast(im, contrast) elif brightness > 0: im = ScreenshotOperations().change_brightness(im, brightness) return im def get_image_to_recognize_with_background(self, zone, cache, resize_percent, contrast, background, brightness, invert): ### Merges the screenshot image and the background and makes a new image with a plain background. After converts image to black and grey colors and inverts # colors. Backgorund images of each game are stored in l_screens\background folder. Use the same image name and 'background' parameter name # in test: background = game_name im = self.check_to_resize(zone, resize_percent) directory = os.path.abspath(os.path.dirname(__file__)) background_dir = os.path.abspath( os.path.join(directory, r'..\..\launcher\l_screens\background')) output_dir = os.path.abspath(r'..\output') for file in os.listdir(background_dir): if file.startswith(background): image1 = Image.open(background_dir + '\\' + file) image = ImageChops.difference(image1, im) image = image.convert('L') result = ImageOps.invert(image) if contrast and brightness != 0: result = ScreenshotOperations().change_contrast( result, contrast) result = ScreenshotOperations().change_brightness( result, brightness) elif brightness != 0: result = ScreenshotOperations().change_brightness( result, brightness) elif contrast != 0: result = ScreenshotOperations().change_contrast( result, contrast) if invert: result = ImageOps.invert(result) return result def check_to_resize(self, zone, resize_percent): orig = self._screenshot(zone) origFile = self._make_up_filename() if resize_percent != 0: resized = self.resize_image(resize_percent, origFile) else: resized = orig return resized _screenshot_counter = 0 def _make_up_filename(self): try: output = BuiltIn().get_variable_value('${OUTPUT_DIR}') except RobotNotRunningError: LOGGER.info('Could not get output dir, using default - output') output = os.path.join(os.getcwd(), 'output') output = os.path.abspath(output) if not os.path.exists(output): os.mkdir(output) self._screenshot_counter += 1 return os.path.join( output, "guiproc-screenshot-%d.png" % (self._screenshot_counter)) # animations def _wait_for_animation_stops(self, zone, timeout=DEFAULT_TIMEOUT, threshold=DEFAULT_THRESHOLD, step=0.05): timeout = float(timeout) threshold = float(threshold) step = float(step) new_screen = self._screenshot(zone) start_time = datetime.datetime.now() while True: utils.sleep(step) old_screen = new_screen new_screen = self._get_screen(False, zone) result = self._find_image_result(new_screen, old_screen, threshold) if result.found: return True if (datetime.datetime.now() - start_time).seconds > timeout: break self.error_handler.report_warning( "Timeout exceeded while waiting for animation stops") return False def _wait_for_animation_starts(self, zone, timeout=DEFAULT_TIMEOUT, threshold=DEFAULT_THRESHOLD, step=0.05): timeout = float(timeout) threshold = float(threshold) step = float(step) new_screen = self._screenshot(zone) start_time = datetime.datetime.now() while True: utils.sleep(step) old_screen = new_screen new_screen = self._get_screen(False, zone) result = self._find_image_result(new_screen, old_screen, threshold) if not result.found: return True if (datetime.datetime.now() - start_time).seconds > timeout: break self.error_handler.report_warning( "Timeout exceeded while waiting for animation starts") return False def _is_zone_animating(self, zone, threshold=DEFAULT_THRESHOLD, step=0.5): threshold = float(threshold) step = float(step) old_screen = self._screenshot(zone) utils.sleep(step) new_screen = self._screenshot(zone) return not self._find_image_result(new_screen, old_screen, threshold).found def save_zone_content_to_output(self, zone): ###FOR DEBUG: saves the content (screenshot) of the provided zone. Pass zone. Image is saved in the output folder in launcher screen_img = self._screenshot(zone) ErrorHandler().save_pictures([(screen_img, "zone")]) def _is_animating(self, zone, threshold, step): threshold = float(threshold) step = float(step) old_screen = self._get_screen(False, zone) utils.sleep(step) new_screen = self._get_screen(False, zone) return not self._find_image_result(new_screen, old_screen, threshold).found
def init(self, settings, references, area=None): """Initializes ImageLibrary for current suite run. Pass as args: settings_file - path to .yaml configs as list reference_folders - path to screens folders as list area - window area taken with keyword Get Window Area Examples: | Init = | ${Settings} | ${References} | ${windowArea} | update_config=${update_config} """ #old, works from robot execution context #self.debug = utils.to_bool(BuiltIn().get_variable_value("${DEBUG_MODE}", False)) #to work in unittests avoiding robot execution context # for i in sys.argv: # if re.search('DEBUG_MODE:\w+', i): # self.debug = utils.to_bool(re.search('DEBUG_MODE:\w+', i).group(0)[11:]) self.settings = {} self.button_registry = GlobalButtonRegistry('hack') assert settings is not None, "YAML config file must not be empty and must contain 'main' window section with at least one of buttons|templates|zones etc" if hasattr(settings, '__iter__'): for setting in settings: with open(setting, 'r') as f: config = yaml.safe_load(f) if "global_buttons_defs" in config: self.button_registry.update_info( config["global_buttons_defs"]) del config["global_buttons_defs"] self.settings.update(config) else: with open(settings) as f: config = yaml.safe_load(f) if "global_buttons_defs" in config: self.button_registry.update_info(config["global_buttons_defs"]) del config["global_buttons_defs"] self.settings.update(config) self.reference_folders = references _check_config(self.settings, self.reference_folders) if area is None: screen_width = tk.Tk().winfo_screenwidth() screen_height = tk.Tk().winfo_screenheight() self.area = (0, 0, screen_width, screen_height) else: self.area = area if "main" not in self.settings: raise errors.ConfigError('config must contain "main" section') self.error_handler = ErrorHandler(self.screenshot_folder, self.area) try: self.image_processor = ImageProcessor(self.area, OpenCV(), references, self.error_handler) except TypeError: LOGGER.info( "Something went wrong while the ImageLibrary library init process: it doesn't get the required params.\n" "Program may not be loaded.") self.button_registry.report_merge_errors() self.button_constructor = ButtonConstructor() # init all windows self.windows = {} for name, config in sorted(self.settings.items()): if name == 'main': self.mw = MainWindow(config, "main", self.button_constructor, self.debug) elif isinstance(config, dict): self.windows[name] = Window(config, name, self.button_constructor, self.debug) # window has multiple screens elif isinstance(config, list): self.windows[name] = [] for index, screen in enumerate(config): if not isinstance(screen, dict): raise errors.ConfigError( "screen {} of window {} not properly configured: dict expected" .format(index + 1, name)) self.windows[name].append( Window(screen, name, self.button_constructor, self.debug))