def __init__(self, settings, window_name, button_constructor, debug = False):
"""Make screenshot and find zones on screen"""
super(MainWindow, self).__init__(settings, window_name, button_constructor, debug)
self.x = None
self.y = None
self.cv = OpenCV()
def _find_image_result(self, img, screen_img, threshold):
result = OpenCV().find_template(img, screen_img, threshold)
if result is not None:
return FindResult(*result,
image=img,
screen=screen_img,
found=True)
else:
return FindResult(None,
None,
None,
None,
None,
image=img,
screen=screen_img,
found=False) # nothing found
def find_multiple_images(self, image, threshold=0.99, cache=False, zone=None, screen=None):
# _find_image(image, threshold, cache, zone) -> list(FindResult)
threshold = float(threshold)
cache = utils.to_bool(cache)
screen_img = self._get_screen(cache, zone, screen)
img = self.load_image(image)
poses = OpenCV().find_multiple_templates(img, screen_img, threshold)
result = []
for pos in poses:
result.append(FindResult(*pos, image=img, screen=screen_img, found=True))
return result
def __init__(self, error_handler, output_dir):
self.cv = OpenCV()
self.cache_screenshot = None
self.error_handler = error_handler
self.output_dir = output_dir
class ImageProcessor(object):
###Image processing system - open files if needed, find them on screen, etc
def __init__(self, error_handler, output_dir):
self.cv = OpenCV()
self.cache_screenshot = None
self.error_handler = error_handler
self.output_dir = output_dir
def _screenshot(self, zone=None):
# _screenshot([area]) -> Image
# Get screenshot of specified area or whole game window if rect is None
# Coordinates are calculating from left-top corner of window
win_area = GUIProcess().get_window_area()
if zone is not None:
im = pyautogui.screenshot(region=(win_area[0] + int(zone[0]),
win_area[1] + int(zone[1]),
int(zone[2]), int(zone[3])))
else:
im = pyautogui.screenshot(region=win_area)
return im
def get_screenshot(self):
# for external use, without areas
# self.win_area = GUIProcess().get_window_area()
# return pyautogui.screenshot(region=self.win_area)
return pyautogui.screenshot()
def load_image(self, image):
try:
return Image.open(image)
except IOError:
pass
# self.error_handler.report_error("Not opened image {}".format(image))
# raise CanNotOpenImageException(image)
def _get_screen(self, cache=None, zone=None, screen=None):
# _get_screen(cache, zone, screen) -> PIL image
# cache - bool, use cached image or make new
# zone - tuple(x, y, w, h)
# screen - PIL image. Don't use with cache option
#
# Cache - user logic, screen - for internal use
search_zone = zone
scr = screen
if scr is not None:
img = scr
return img
elif cache:
if self.cache_screenshot is None:
raise CacheError
img = self.cache_screenshot
return img
else:
return self._screenshot(search_zone)
# if area is not None:
# return img.crop((search_zone[0], search_zone[1], search_zone[0] + search_zone[2], search_zone[1] + search_zone[3]))
# else:
# return img
def take_cache_screenshot(self):
return self.get_screenshot()
def _find_image_result(self, img, screen_img, threshold):
result = OpenCV().find_template(img, screen_img, threshold)
if result is not None:
return FindResult(*result,
image=img,
screen=screen_img,
found=True)
else:
return FindResult(None,
None,
None,
None,
None,
image=img,
screen=screen_img,
found=False) # nothing found
def _find_image(self,
image,
threshold=0.95,
cache=False,
zone=None,
screen=None):
# _find_image(image, threshold, cache, zone) -> FindResult
threshold = float(threshold)
cache = utils.to_bool(cache)
assert threshold > 0 and threshold <= 1, "Threshold must be in (0, 1)"
screen_img = self._get_screen(cache, zone, screen)
img = self.load_image(image)
return self._find_image_result(img, screen_img, threshold)
def find_image(self,
image,
threshold=0.95,
cache=False,
zone=None,
screen=None):
result = ImageProcessor(self.error_handler,
self.output_dir)._find_image(
image, threshold, cache, zone,
screen).found
if result:
LOGGER.info(
'Template was found on screen with threshold:{}'.format(
threshold))
return True
else:
LOGGER.info(
'Template was not found on screen with threshold {}'.format(
threshold))
return False
def _is_image_on_screen(self,
image,
threshold=0.95,
cache=False,
zone=None,
screen=None):
return self._find_image(image, threshold, cache, zone, screen).found
def _image_should_be_on_screen(self,
image,
threshold=0.95,
cache=False,
zone=None,
screen=None):
result = self._find_image(image, threshold, cache, zone, screen)
if result.found:
return True
self.error_handler.report_warning("First try was unsuccesful")
# try again
utils.sleep(0.020)
result = self._find_image(image, threshold, cache, zone)
if result.found:
return True
image_info = ("image", result.image)
screen_info = ("screen", result.screen)
msg = "Image was not found at screen with threshold {}".format(
threshold)
self.error_handler.report_error(msg, image_info, screen_info)
raise RuntimeError(msg)
def _image_should_not_be_on_screen(self,
image,
threshold=0.95,
cache=False,
zone=None,
screen=None):
result = self._find_image(image, threshold, cache, zone, screen)
if not result.found:
return True
self.error_handler.report_warning("First try was unsuccesful")
# try again
utils.sleep(0.020)
result = self._find_image(image, threshold, cache, zone, screen)
if not result.found:
return True
image_info = ("image", result.image)
screen_info = ("screen", result.screen)
msg = "Image was found at screen with threshold {}".format(threshold)
self.error_handler.report_error(msg, image_info, screen_info)
raise RuntimeError(msg)
def _wait_for_image(self, image, threshold=0.95, timeout=15, zone=None):
timeout = float(timeout)
start_time = datetime.datetime.now()
img = self.load_image(image)
while True:
screen_img = self._get_screen(False, zone)
result = self._find_image_result(img, screen_img, threshold)
if result.found:
return True
utils.sleep(0)
if (datetime.datetime.now() - start_time).seconds > int(timeout):
break
image_info = ("image", result.image)
screen_info = ("screen", result.screen)
msg = "Waiting for image was unsucessfull with threshold {} and timeout {} sec".format(
threshold, int(timeout))
self.error_handler.report_warning(msg, image_info, screen_info)
return False
def _wait_for_image_to_hide(self,
image,
threshold=0.95,
timeout=15,
zone=None):
timeout = float(timeout)
start_time = datetime.datetime.now()
img = self.load_image(image)
while True:
screen_img = self._screenshot(zone)
result = self._find_image_result(img, screen_img, threshold)
if not result.found:
return True
utils.sleep(0)
if (datetime.datetime.now() - start_time).seconds > int(timeout):
break
image_info = ("image", result.image)
screen_info = ("screen", result.screen)
msg = "Waiting for image hide was unsucessfull for threshold {} and timeout {}".format(
threshold, int(timeout))
self.error_handler.report_warning(msg, image_info, screen_info)
return False
def _wait_for_image_to_stop(self,
image,
threshold=0.95,
timeout=15,
move_threshold=0.99,
step=0.1):
timeout = float(timeout)
threshold = float(threshold)
move_threshold = float(move_threshold)
step = float(step)
assert threshold > 0 and threshold <= 1, "Threshold must be in (0, 1]"
assert move_threshold > 0 and move_threshold <= 1, "Move threshold must be in (0, 1]"
img = self.load_image(image)
start_time = datetime.datetime.now()
new_screen = self._screenshot()
new_pos = self._find_image_result(img, new_screen, threshold)
while True:
old_scren = new_screen
old_pos = new_pos
utils.sleep(step)
new_screen = self._screenshot()
new_pos = self._find_image_result(img, new_screen, threshold)
if old_pos.found and new_pos.found: # template is on screen, not blinking and whatever else
ds = math.hypot(new_pos.x - old_pos.x, new_pos.y - old_pos.y)
diag = 1280 # hypot for 1024x768
if 1 - ds / diag > threshold:
return True
if (datetime.datetime.now() - start_time).seconds > int(timeout):
break
image_info = ("image", new_pos.image)
screen_info = ("screen", new_pos.screen)
msg = "Waiting for image stop was unsucessfull for threshold {} and timeout {}".format(
threshold, int(timeout))
self.error_handler.report_warning(msg, image_info, screen_info)
return False
def find_multiple_images(self,
image,
threshold=0.95,
cache=False,
zone=None,
screen=None):
# _find_image(image, threshold, cache, zone) -> list(FindResult)
threshold = float(threshold)
cache = utils.to_bool(cache)
screen_img = self._get_screen(cache, zone, screen)
img = self.load_image(image)
poses = OpenCV().find_multiple_templates(img, screen_img, threshold)
result = []
for pos in poses:
result.append(
FindResult(*pos, image=img, screen=screen_img, found=True))
return result
def _get_images_count(self,
image,
threshold=0.95,
cache=False,
zone=None,
screen=None):
return len(
self.find_multiple_images(image, threshold, cache, zone, screen))
def find_one_of(self, images, cache=False, zone=None, screen=None):
###Find one of images. The one with max threshold wins and will be returned or None if no one found
assert len(images) > 0, "At least one image must be set"
screen_img = self._get_screen(False, zone, screen)
results = []
for image_info in images:
result = self._find_image_result(image_info[0], screen_img,
float(image_info[1]))
if result.found:
results.append(result)
if not results:
return None
return sorted(results, key=lambda res: res.threshold, reverse=True)[0]
def find_all_of(self, images, cache=False, zone=None, screen=None):
pass
def wait_for_one_of(self, images, timeout=15, zone=None):
# wait_for_one_of_images(images, timeout, step, zone) -> bool
# images - list of (image, threshold, bool)
# image - PIL image
# threshold - threshold for image
# bool - True for wait for show, False for wait for hide
assert len(
images
) > 0, "You are trying to wait for empty list of images. Really?"
timeout = float(timeout)
start_time = datetime.datetime.now()
while True:
screen_img = self._screenshot(zone)
for image_info in images:
# todo: optimize
result = self._find_image_result(image_info[0], screen_img,
float(image_info[1]))
if result.found == utils.to_bool(image_info[2]):
return result
utils.sleep(0)
if (datetime.datetime.now() - start_time).seconds > timeout:
break
images_info = []
for index, image in enumerate(images):
images_info.append(
("image_{}_threshold_{}".format(index, image[1]), image[0]))
images_info.append(("screen", screen_img))
self.error_handler.report_warning(
"Waiting for one of the images was unsuccessful", *images_info)
return result
def wait_for_all_of(self, images, timeout=15, zone=None):
pass
def resize_image(self, resize_percent, origFile):
origresize = Image.open(origFile)
width, height = origresize.size
width_resize = width * int(resize_percent) / width + width
height_resize = height * int(resize_percent) / height + height
origresize = origresize.resize(
(int(round(width_resize)), int(round(height_resize))),
Image.ANTIALIAS)
return origresize
def get_image_to_recognize(self, zone, cache, resize_percent, contrast,
invert, brightness, change_mode, win_area):
im = self.check_to_resize(zone, resize_percent)
if change_mode:
im = self.cv.prepare_image_to_recognize(im)
if invert:
im = self.cv.prepare_image_to_recognize(im)
im = ImageOps.invert(im)
if contrast != 0 and brightness != 0:
im = ScreenshotOperations().change_brightness(im, brightness)
im = ScreenshotOperations().change_contrast(im, contrast)
else:
if contrast != 0:
im = ScreenshotOperations().change_contrast(im, contrast)
elif brightness > 0:
im = ScreenshotOperations().change_brightness(im, brightness)
return im
def get_image_to_recognize_with_background(self, zone, cache,
resize_percent, contrast,
background, brightness, invert):
### Merges the screenshot image and the background and makes a new image with a plain background. After converts image to black and grey colors and inverts
# colors. Backgorund images of each game are stored in l_screens\background folder. Use the same image name and 'background' parameter name
# in test: background = game_name
im = self.check_to_resize(zone, resize_percent)
directory = os.path.abspath(os.path.dirname(__file__))
background_dir = os.path.abspath(
os.path.join(directory, r'..\..\launcher\l_screens\background'))
output_dir = os.path.abspath(r'..\output')
for file in os.listdir(background_dir):
if file.startswith(background):
image1 = Image.open(background_dir + '\\' + file)
image = ImageChops.difference(image1, im)
image = image.convert('L')
result = ImageOps.invert(image)
if contrast and brightness != 0:
result = ScreenshotOperations().change_contrast(
result, contrast)
result = ScreenshotOperations().change_brightness(
result, brightness)
elif brightness != 0:
result = ScreenshotOperations().change_brightness(
result, brightness)
elif contrast != 0:
result = ScreenshotOperations().change_contrast(
result, contrast)
if invert:
result = ImageOps.invert(result)
return result
def check_to_resize(self, zone, resize_percent):
orig = self._screenshot(zone)
origFile = self._make_up_filename()
if resize_percent != 0:
resized = self.resize_image(resize_percent, origFile)
else:
resized = orig
return resized
_screenshot_counter = 0
def _make_up_filename(self):
try:
output = BuiltIn().get_variable_value('${OUTPUT_DIR}')
except RobotNotRunningError:
LOGGER.info('Could not get output dir, using default - output')
output = os.path.join(os.getcwd(), 'output')
output = os.path.abspath(output)
if not os.path.exists(output):
os.mkdir(output)
self._screenshot_counter += 1
return os.path.join(
output, "guiproc-screenshot-%d.png" % (self._screenshot_counter))
# animations
def _wait_for_animation_stops(self,
zone,
timeout=DEFAULT_TIMEOUT,
threshold=DEFAULT_THRESHOLD,
step=0.05):
timeout = float(timeout)
threshold = float(threshold)
step = float(step)
new_screen = self._screenshot(zone)
start_time = datetime.datetime.now()
while True:
utils.sleep(step)
old_screen = new_screen
new_screen = self._get_screen(False, zone)
result = self._find_image_result(new_screen, old_screen, threshold)
if result.found:
return True
if (datetime.datetime.now() - start_time).seconds > timeout:
break
self.error_handler.report_warning(
"Timeout exceeded while waiting for animation stops")
return False
def _wait_for_animation_starts(self,
zone,
timeout=DEFAULT_TIMEOUT,
threshold=DEFAULT_THRESHOLD,
step=0.05):
timeout = float(timeout)
threshold = float(threshold)
step = float(step)
new_screen = self._screenshot(zone)
start_time = datetime.datetime.now()
while True:
utils.sleep(step)
old_screen = new_screen
new_screen = self._get_screen(False, zone)
result = self._find_image_result(new_screen, old_screen, threshold)
if not result.found:
return True
if (datetime.datetime.now() - start_time).seconds > timeout:
break
self.error_handler.report_warning(
"Timeout exceeded while waiting for animation starts")
return False
def _is_zone_animating(self, zone, threshold=DEFAULT_THRESHOLD, step=0.5):
threshold = float(threshold)
step = float(step)
old_screen = self._screenshot(zone)
utils.sleep(step)
new_screen = self._screenshot(zone)
return not self._find_image_result(new_screen, old_screen,
threshold).found
def save_zone_content_to_output(self, zone):
###FOR DEBUG: saves the content (screenshot) of the provided zone. Pass zone. Image is saved in the output folder in launcher
screen_img = self._screenshot(zone)
ErrorHandler().save_pictures([(screen_img, "zone")])
def _is_animating(self, zone, threshold, step):
threshold = float(threshold)
step = float(step)
old_screen = self._get_screen(False, zone)
utils.sleep(step)
new_screen = self._get_screen(False, zone)
return not self._find_image_result(new_screen, old_screen,
threshold).found
def init(self, settings, references, area=None):
"""Initializes ImageLibrary for current suite run.
Pass as args:
settings_file - path to .yaml configs as list
reference_folders - path to screens folders as list
area - window area taken with keyword Get Window Area
Examples:
| Init = | ${Settings} | ${References} | ${windowArea} | update_config=${update_config}
"""
#old, works from robot execution context
#self.debug = utils.to_bool(BuiltIn().get_variable_value("${DEBUG_MODE}", False))
#to work in unittests avoiding robot execution context
# for i in sys.argv:
# if re.search('DEBUG_MODE:\w+', i):
# self.debug = utils.to_bool(re.search('DEBUG_MODE:\w+', i).group(0)[11:])
self.settings = {}
self.button_registry = GlobalButtonRegistry('hack')
assert settings is not None, "YAML config file must not be empty and must contain 'main' window section with at least one of buttons|templates|zones etc"
if hasattr(settings, '__iter__'):
for setting in settings:
with open(setting, 'r') as f:
config = yaml.safe_load(f)
if "global_buttons_defs" in config:
self.button_registry.update_info(
config["global_buttons_defs"])
del config["global_buttons_defs"]
self.settings.update(config)
else:
with open(settings) as f:
config = yaml.safe_load(f)
if "global_buttons_defs" in config:
self.button_registry.update_info(config["global_buttons_defs"])
del config["global_buttons_defs"]
self.settings.update(config)
self.reference_folders = references
_check_config(self.settings, self.reference_folders)
if area is None:
screen_width = tk.Tk().winfo_screenwidth()
screen_height = tk.Tk().winfo_screenheight()
self.area = (0, 0, screen_width, screen_height)
else:
self.area = area
if "main" not in self.settings:
raise errors.ConfigError('config must contain "main" section')
self.error_handler = ErrorHandler(self.screenshot_folder, self.area)
try:
self.image_processor = ImageProcessor(self.area, OpenCV(),
references,
self.error_handler)
except TypeError:
LOGGER.info(
"Something went wrong while the ImageLibrary library init process: it doesn't get the required params.\n"
"Program may not be loaded.")
self.button_registry.report_merge_errors()
self.button_constructor = ButtonConstructor()
# init all windows
self.windows = {}
for name, config in sorted(self.settings.items()):
if name == 'main':
self.mw = MainWindow(config, "main", self.button_constructor,
self.debug)
elif isinstance(config, dict):
self.windows[name] = Window(config, name,
self.button_constructor,
self.debug)
# window has multiple screens
elif isinstance(config, list):
self.windows[name] = []
for index, screen in enumerate(config):
if not isinstance(screen, dict):
raise errors.ConfigError(
"screen {} of window {} not properly configured: dict expected"
.format(index + 1, name))
self.windows[name].append(
Window(screen, name, self.button_constructor,
self.debug))