def _find_sift_in_predict_area(self, image, screen): if not self.record_pos: return None # calc predict area in screen image_wh, screen_resolution = aircv.get_resolution( image), aircv.get_resolution(screen) xmin, ymin, xmax, ymax = Predictor.get_predict_area( self.record_pos, image_wh, self.resolution, screen_resolution) # crop predict image from screen predict_area = aircv.crop_image(screen, (xmin, ymin, xmax, ymax)) if not predict_area.any(): return None # find sift in that image ret_in_area = aircv.find_sift(predict_area, image, threshold=self.threshold, rgb=self.rgb) # calc cv ret if found if not ret_in_area: return None ret = deepcopy(ret_in_area) if "rectangle" in ret: for idx, item in enumerate(ret["rectangle"]): ret["rectangle"][idx] = (item[0] + xmin, item[1] + ymin) ret["result"] = (ret_in_area["result"][0] + xmin, ret_in_area["result"][1] + ymin) return ret
def _find_keypoint_result_in_predict_area(self, func, image, screen): G.LOGGING.debug("teststtsstt") if not self.record_pos: return None # calc predict area in screen image_wh, screen_resolution = aircv.get_resolution( image), aircv.get_resolution(screen) xmin, ymin, xmax, ymax = Predictor.get_predict_area( self.record_pos, image_wh, self.resolution, screen_resolution) G.LOGGING.debug("111111111111111111111try match with %s" % xmin) # crop predict image from screen predict_area = aircv.crop_image(screen, (xmin, ymin, xmax, ymax)) if not predict_area.any(): return None # keypoint matching in predicted area: ret_in_area = func(image, predict_area, threshold=self.threshold, rgb=self.rgb) # calc cv ret if found if not ret_in_area: return None ret = deepcopy(ret_in_area) if "rectangle" in ret: for idx, item in enumerate(ret["rectangle"]): ret["rectangle"][idx] = (item[0] + xmin, item[1] + ymin) ret["result"] = (ret_in_area["result"][0] + xmin, ret_in_area["result"][1] + ymin) return ret
def _translate_code(self, step): if step["tag"] != "function": return None step_data = step["data"] args = [] code = { "name": step_data["name"], "args": args, } for key, value in step_data["call_args"].items(): args.append({ "key": key, "value": value, }) for k, arg in enumerate(args): value = arg["value"] if isinstance(value, dict) and value.get("__class__") == "Template": if self.export_dir: # all relative path image_path = value['filename'] if not os.path.isfile(os.path.join(self.script_root, image_path)) and value['_filepath']: # copy image used by using statement shutil.copyfile(value['_filepath'], os.path.join(self.script_root, value['filename'])) else: image_path = os.path.abspath(value['_filepath'] or value['filename']) arg["image"] = image_path if not value['_filepath'] and not os.path.exists(value['filename']): crop_img = imread(os.path.join(self.script_root, value['filename'])) else: crop_img = imread(value['_filepath'] or value['filename']) arg["resolution"] = get_resolution(crop_img) return code
def _translate_code(self, step): if step["tag"] != "function": return None step_data = step["data"] args = [] code = { "name": step_data["name"], "args": args, } for key, value in step_data["call_args"].items(): args.append({ "key": key, "value": value, }) for k, arg in enumerate(args): value = arg["value"] if isinstance(value, dict) and value.get("__class__") == "Template": image_path = str(value['filename']) if not self.export_dir: if os.path.isfile( os.path.join(self.script_root, image_path)): image_path = os.path.join(self.script_root, image_path) else: image_path = value['_filepath'] else: if not os.path.isfile( os.path.join(self.script_root, image_path)): shutil.copy(value['_filepath'], self.script_root) arg["image"] = image_path crop_img = imread( os.path.join(self.script_root, str(value['filename']))) arg["resolution"] = get_resolution(crop_img) return code
def try_log_screen(screen=None, quality=None, max_size=None): """ Save screenshot to file Args: screen: screenshot to be saved quality: The image quality, default is ST.SNAPSHOT_QUALITY max_size: the maximum size of the picture, e.g 1200 Returns: None """ if not ST.LOG_DIR: return if not quality: quality = ST.SNAPSHOT_QUALITY if not max_size: max_size = ST.IMAGE_MAXSIZE if screen is None: screen = G.DEVICE.snapshot(quality=quality) filename = "%(time)d.jpg" % {'time': time.time() * 1000} filepath = os.path.join(ST.LOG_DIR, filename) aircv.imwrite(filepath, screen, quality, max_size=max_size) return {"screen": filename, "resolution": aircv.get_resolution(screen)}
def _get_left_up_offset(self): window_pos = self.get_window_position() window_size = self.get_window_size() mouse = Controller() screen = self.screenshot() screen_size = get_resolution(screen) offset=window_size["width"] - \ screen_size[0], window_size["height"] - screen_size[1] pos = (int(offset[0] / 2 + window_pos['x']), int(offset[1] + window_pos['y'] - offset[0] / 2)) return pos
def loop_find(query, driver=None, timeout=10, threshold=None, interval=0.5, intervalfunc=None): """ Search for image template in the screen until timeout Args: query: image template to be found in screenshot timeout: time interval how long to look for the image template threshold: default is None interval: sleep interval before next attempt to find the image template intervalfunc: function that is executed after unsuccessful attempt to find the image template Raises: TargetNotFoundError: when image template is not found in screenshot Returns: TargetNotFoundError if image template not found, otherwise returns the position where the image template has been found in screenshot """ threshold = 0.6 start_time = time.time() while True: screen = driver.screenshot() query.resolution = get_resolution(screen) if screen is None: print("Screen is None, may be locked") else: if threshold: query.threshold = threshold match_pos = query.match_in(screen) if match_pos: try_log_screen(screen) return match_pos if intervalfunc is not None: intervalfunc() # 超时则raise,未超时则进行下次循环: if (time.time() - start_time) > timeout: try_log_screen(screen) raise TargetNotFoundError('Picture %s not found in screen' % query) else: time.sleep(interval)
def _find_sift_in_predict_area(self, image, screen): if not self.record_pos: return None # calc predict area in screen screen_resolution = aircv.get_resolution(screen) xmin, ymin, xmax, ymax = Predictor.get_predict_area(self.record_pos, screen_resolution) # crop predict image from screen predict_area = aircv.crop_image(screen, (xmin, ymin, xmax, ymax)) # aircv.show(predict_area) # find sift in that image ret_in_area = aircv.find_sift(predict_area, image, threshold=self.threshold, rgb=self.rgb) # calc cv ret if found if not ret_in_area: return None ret = deepcopy(ret_in_area) ret["result"] = (ret_in_area["result"][0] + xmin, ret_in_area["result"][1] + ymin) return ret
def _resize_image(self, image, screen, resize_method): """模板匹配中,将输入的截图适配成 等待模板匹配的截图.""" screen_resolution = aircv.get_resolution(screen) # 如果分辨率一致,则不需要进行im_search的适配: if tuple(self.resolution) == tuple(screen_resolution) or resize_method is None: return image # 分辨率不一致则进行适配,默认使用cocos_min_strategy: h, w = image.shape[:2] w_re, h_re = resize_method(w, h, self.resolution, screen_resolution) # 确保w_re和h_re > 0, 至少有1个像素: w_re, h_re = max(1, w_re), max(1, h_re) # 调试代码: 输出调试信息. G.LOGGING.debug("resize: (%s, %s)->(%s, %s), resolution: %s=>%s" % ( w, h, w_re, h_re, self.resolution, screen_resolution)) # 进行图片缩放: image = cv2.resize(image, (w_re, h_re)) return image
def try_log_screen(screen=None): """ Save screenshot to file Args: screen: screenshot to be saved Returns: None """ if not ST.LOG_DIR: return if screen is None: screen = G.DEVICE.snapshot(quality=ST.SNAPSHOT_QUALITY) filename = "%(time)d.jpg" % {'time': time.time() * 1000} filepath = os.path.join(ST.LOG_DIR, filename) aircv.imwrite(filepath, screen, ST.SNAPSHOT_QUALITY) return {"screen": filename, "resolution": aircv.get_resolution(screen)}
def _translate_code(self, step): if step["tag"] != "function": return None step_data = step["data"] args = [] code = { "name": step_data["name"], "args": args, } for key, value in step_data["call_args"].items(): args.append({ "key": key, "value": value, }) for k, arg in enumerate(args): value = arg["value"] if isinstance(value, dict) and value.get("__class__") == "Template": if self.export_dir: # all relative path image_path = value['filename'] if not os.path.isfile( os.path.join(self.script_root, image_path)): shutil.copy(value['_filepath'], self.script_root ) # copy image used by using statement else: if self.online_path: _filepath = os.path.abspath(value['_filepath']) jk_filepath = _filepath.split(self.proj_name)[1] image_path = self.online_path + jk_filepath.replace( '\\', '/') else: image_path = os.path.abspath(value['_filepath'] or value['filename']) arg["image"] = image_path print(' arg["image"]: ', arg["image"]) crop_img = imread(value['_filepath'] or value['filename']) arg["resolution"] = get_resolution(crop_img) return code
def loop_find3(total_pic, query, posi, tag=None, timeout=ST.FIND_TIMEOUT, threshold=None, interval=0.5, intervalfunc=None): # print("ST.FIND_TIMEOUT",ST.FIND_TIMEOUT) G.LOGGING.info("Try finding:\n%s", query.record_pos) G.LOGGING.info(query._imread().shape) # screen_resolution = aircv.get_resolution(screen) part_size_width = query._imread().shape[0] part_size_height = query._imread().shape[1] part_resolution_width = query.resolution[0] part_resolution_height = query.resolution[1] # pairt_origin_xmin_relative = posi[0]; # pairt_origin_xmax_relative = posi[2] ; pairt_origin_ymin_relative = posi[0] / part_resolution_height pairt_origin_ymax_relative = posi[1] / part_resolution_height # print("香菇:",(pairt_origin_ymin_relative,pairt_origin_ymax_relative)) posi_x = query.record_pos[0] posi_y = query.record_pos[1] # print("posi_x",posi_x) start_time = time.time() screen = aircv.imread(total_pic) G.LOGGING.debug("query path %s" % query.filepath) # predict_area = aircv.crop_image(screen, (xmin, ymin, xmax, ymax)) screen_resolution = aircv.get_resolution(screen) # print("总图片的分辨率",screen_resolution); total_resolution_width = screen_resolution[0] total_resolution_height = screen_resolution[1] width_scale = round(total_resolution_width / part_resolution_width, 2) height_scale = round(total_resolution_height / part_resolution_height, 2) # print("width_scale",width_scale) # print("width_scale",height_scale) # print("总图的分辨率",(total_resolution_width,total_resolution_height)) # print("缩放之后的相对坐标",(pairt_origin_xmin_relative * width_scale,pairt_origin_ymin_relative * height_scale,pairt_origin_xmax_relative * width_scale,pairt_origin_ymax_relative * height_scale)) # pairt_end_xmin_absolute = round( (pairt_origin_xmin_relative * width_scale) * total_resolution_width ); pairt_end_xmin_absolute = 0 # pairt_end_xmax_absolute = round( (pairt_origin_xmax_relative * width_scale) * total_resolution_width ); pairt_end_xmax_absolute = total_resolution_width pairt_end_ymin_absolute = round( (pairt_origin_ymin_relative * height_scale) * total_resolution_height) - 30 pairt_end_ymax_absolute = round( (pairt_origin_ymax_relative * height_scale) * total_resolution_height) + 30 # print("enddddd",(pairt_end_xmin_absolute,pairt_end_ymin_absolute,pairt_end_xmax_absolute,pairt_end_ymax_absolute)) # part_resolution_width_scaled = part_size_width * width_scale; # part_resolution_height_scaled = part_size_height * height_scale; # part_end_xmax = ((posi_x + 0.5) if (posi_x + 0.5)<=1 else 1)*total_resolution_width # part_end_ymax = ((posi_y + 0.5) if (posi_y + 0.5)<=1 else 1)*total_resolution_height # part_end_xmin = part_end_xmax - part_resolution_width_scaled # part_end_ymin = part_end_ymax - part_resolution_height_scaled predict_area = aircv.crop_image( screen, (pairt_end_xmin_absolute, pairt_end_ymin_absolute, pairt_end_xmax_absolute, pairt_end_ymax_absolute)) screen[0:pairt_end_ymin_absolute, 0:total_resolution_width] = 0 screen[pairt_end_ymax_absolute:total_resolution_height, 0:total_resolution_width] = 0 # print("黑素区域",(pairt_end_ymin_absolute,pairt_end_ymax_absolute)) # screen[200:250,0:50] = 0; aircv.imwrite("part_black.png", screen, 99) # predict_area = aircv.crop_image(screen, (0, 0, 1126, 2436)); # print("hahahahah") # print((part_end_xmin, part_end_ymin, part_end_xmax, part_end_ymax)) # print(predict_area.shape) aircv.imwrite("predict_area3.png", predict_area, 99) while True: if predict_area is None: G.LOGGING.warning("预测区域有问题!") else: if threshold: query.threshold = threshold match_pos = query.match_in(screen) if match_pos: try_log_screen(screen) return match_pos if intervalfunc is not None: intervalfunc() # 超时则raise,未超时则进行下次循环: if (time.time() - start_time) > timeout: try_log_screen(screen) # raise TargetNotFoundError('Picture %s not found in screen' % query) G.LOGGING.warning("图片查找超时!") break else: time.sleep(interval)
def loop_find2(total_pic, query, timeout=ST.FIND_TIMEOUT, threshold=None, interval=0.5, intervalfunc=None): G.LOGGING.info("Try finding:\n%s", query.record_pos) G.LOGGING.info(query._imread().shape) part_size_width = query._imread().shape[0] part_size_height = query._imread().shape[1] part_resolution_width = query.resolution[0] part_resolution_height = query.resolution[1] # print("part_resolution_width:" + str(part_resolution_width)) # print("part_resolution_height:" + str(part_resolution_height)) posi_x = query.record_pos[0] posi_y = query.record_pos[1] print("posi_x", posi_x) start_time = time.time() screen = aircv.imread(total_pic) G.LOGGING.debug("query path %s" % query.filepath) # predict_area = aircv.crop_image(screen, (xmin, ymin, xmax, ymax)) screen_resolution = aircv.get_resolution(screen) total_resolution_width = query.resolution[0] total_resolution_height = query.resolution[1] width_scale = total_resolution_width / part_resolution_width height_scale = total_resolution_height / part_resolution_height print("test_scale:" + str(1 / 2)) print("width_scale:" + str(width_scale)) print("height_scase:" + str(height_scale)) part_resolution_width_scaled = part_size_width * width_scale part_resolution_height_scaled = part_size_height * height_scale print("part_resolution_width_scaled:" + str(part_resolution_width_scaled)) print("part_resolution_height_scaled:" + str(part_resolution_height_scaled)) G.LOGGING.debug("图片的分辨率") G.LOGGING.debug(screen_resolution) part_end_xmax = ((posi_x + 0.5) if (posi_x + 0.5) <= 1 else 1) * total_resolution_width part_end_ymax = ((posi_y + 0.5) if (posi_y + 0.5) <= 1 else 1) * total_resolution_height part_end_xmin = part_end_xmax - part_resolution_width_scaled part_end_ymin = part_end_ymax - part_resolution_height_scaled predict_area = aircv.crop_image( screen, (part_end_xmin, part_end_ymin, part_end_xmax, part_end_ymax)) print("hahahahah") print((part_end_xmin, part_end_ymin, part_end_xmax, part_end_ymax)) print(predict_area.shape) aircv.imwrite("predict_area.png", screen, 99) while True: if predict_area is None: G.LOGGING.warning("预测区域有问题!") else: if threshold: query.threshold = threshold match_pos = query.match_in(screen) if match_pos: try_log_screen(screen) return match_pos if intervalfunc is not None: intervalfunc() # 超时则raise,未超时则进行下次循环: if (time.time() - start_time) > timeout: try_log_screen(predict_area) # raise TargetNotFoundError('Picture %s not found in screen' % query) G.LOGGING.warning("图片查找超时!") break else: time.sleep(interval)