def _find_sift_in_predict_area(self, image, screen):
if not self.record_pos:
return None
# calc predict area in screen
image_wh, screen_resolution = aircv.get_resolution(
image), aircv.get_resolution(screen)
xmin, ymin, xmax, ymax = Predictor.get_predict_area(
self.record_pos, image_wh, self.resolution, screen_resolution)
# crop predict image from screen
predict_area = aircv.crop_image(screen, (xmin, ymin, xmax, ymax))
if not predict_area.any():
return None
# find sift in that image
ret_in_area = aircv.find_sift(predict_area,
image,
threshold=self.threshold,
rgb=self.rgb)
# calc cv ret if found
if not ret_in_area:
return None
ret = deepcopy(ret_in_area)
if "rectangle" in ret:
for idx, item in enumerate(ret["rectangle"]):
ret["rectangle"][idx] = (item[0] + xmin, item[1] + ymin)
ret["result"] = (ret_in_area["result"][0] + xmin,
ret_in_area["result"][1] + ymin)
return ret
def _find_keypoint_result_in_predict_area(self, func, image, screen):
G.LOGGING.debug("teststtsstt")
if not self.record_pos:
return None
# calc predict area in screen
image_wh, screen_resolution = aircv.get_resolution(
image), aircv.get_resolution(screen)
xmin, ymin, xmax, ymax = Predictor.get_predict_area(
self.record_pos, image_wh, self.resolution, screen_resolution)
G.LOGGING.debug("111111111111111111111try match with %s" % xmin)
# crop predict image from screen
predict_area = aircv.crop_image(screen, (xmin, ymin, xmax, ymax))
if not predict_area.any():
return None
# keypoint matching in predicted area:
ret_in_area = func(image,
predict_area,
threshold=self.threshold,
rgb=self.rgb)
# calc cv ret if found
if not ret_in_area:
return None
ret = deepcopy(ret_in_area)
if "rectangle" in ret:
for idx, item in enumerate(ret["rectangle"]):
ret["rectangle"][idx] = (item[0] + xmin, item[1] + ymin)
ret["result"] = (ret_in_area["result"][0] + xmin,
ret_in_area["result"][1] + ymin)
return ret
def _translate_code(self, step):
if step["tag"] != "function":
return None
step_data = step["data"]
args = []
code = {
"name": step_data["name"],
"args": args,
}
for key, value in step_data["call_args"].items():
args.append({
"key": key,
"value": value,
})
for k, arg in enumerate(args):
value = arg["value"]
if isinstance(value, dict) and value.get("__class__") == "Template":
if self.export_dir: # all relative path
image_path = value['filename']
if not os.path.isfile(os.path.join(self.script_root, image_path)) and value['_filepath']:
# copy image used by using statement
shutil.copyfile(value['_filepath'], os.path.join(self.script_root, value['filename']))
else:
image_path = os.path.abspath(value['_filepath'] or value['filename'])
arg["image"] = image_path
if not value['_filepath'] and not os.path.exists(value['filename']):
crop_img = imread(os.path.join(self.script_root, value['filename']))
else:
crop_img = imread(value['_filepath'] or value['filename'])
arg["resolution"] = get_resolution(crop_img)
return code
def _translate_code(self, step):
if step["tag"] != "function":
return None
step_data = step["data"]
args = []
code = {
"name": step_data["name"],
"args": args,
}
for key, value in step_data["call_args"].items():
args.append({
"key": key,
"value": value,
})
for k, arg in enumerate(args):
value = arg["value"]
if isinstance(value,
dict) and value.get("__class__") == "Template":
image_path = str(value['filename'])
if not self.export_dir:
if os.path.isfile(
os.path.join(self.script_root, image_path)):
image_path = os.path.join(self.script_root, image_path)
else:
image_path = value['_filepath']
else:
if not os.path.isfile(
os.path.join(self.script_root, image_path)):
shutil.copy(value['_filepath'], self.script_root)
arg["image"] = image_path
crop_img = imread(
os.path.join(self.script_root, str(value['filename'])))
arg["resolution"] = get_resolution(crop_img)
return code
def try_log_screen(screen=None, quality=None, max_size=None):
"""
Save screenshot to file
Args:
screen: screenshot to be saved
quality: The image quality, default is ST.SNAPSHOT_QUALITY
max_size: the maximum size of the picture, e.g 1200
Returns:
None
"""
if not ST.LOG_DIR:
return
if not quality:
quality = ST.SNAPSHOT_QUALITY
if not max_size:
max_size = ST.IMAGE_MAXSIZE
if screen is None:
screen = G.DEVICE.snapshot(quality=quality)
filename = "%(time)d.jpg" % {'time': time.time() * 1000}
filepath = os.path.join(ST.LOG_DIR, filename)
aircv.imwrite(filepath, screen, quality, max_size=max_size)
return {"screen": filename, "resolution": aircv.get_resolution(screen)}
def _get_left_up_offset(self):
window_pos = self.get_window_position()
window_size = self.get_window_size()
mouse = Controller()
screen = self.screenshot()
screen_size = get_resolution(screen)
offset=window_size["width"] - \
screen_size[0], window_size["height"] - screen_size[1]
pos = (int(offset[0] / 2 + window_pos['x']),
int(offset[1] + window_pos['y'] - offset[0] / 2))
return pos
def loop_find(query,
driver=None,
timeout=10,
threshold=None,
interval=0.5,
intervalfunc=None):
"""
Search for image template in the screen until timeout
Args:
query: image template to be found in screenshot
timeout: time interval how long to look for the image template
threshold: default is None
interval: sleep interval before next attempt to find the image template
intervalfunc: function that is executed after unsuccessful attempt to find the image template
Raises:
TargetNotFoundError: when image template is not found in screenshot
Returns:
TargetNotFoundError if image template not found, otherwise returns the position where the image template has
been found in screenshot
"""
threshold = 0.6
start_time = time.time()
while True:
screen = driver.screenshot()
query.resolution = get_resolution(screen)
if screen is None:
print("Screen is None, may be locked")
else:
if threshold:
query.threshold = threshold
match_pos = query.match_in(screen)
if match_pos:
try_log_screen(screen)
return match_pos
if intervalfunc is not None:
intervalfunc()
# 超时则raise，未超时则进行下次循环:
if (time.time() - start_time) > timeout:
try_log_screen(screen)
raise TargetNotFoundError('Picture %s not found in screen' % query)
else:
time.sleep(interval)
def _find_sift_in_predict_area(self, image, screen):
if not self.record_pos:
return None
# calc predict area in screen
screen_resolution = aircv.get_resolution(screen)
xmin, ymin, xmax, ymax = Predictor.get_predict_area(self.record_pos, screen_resolution)
# crop predict image from screen
predict_area = aircv.crop_image(screen, (xmin, ymin, xmax, ymax))
# aircv.show(predict_area)
# find sift in that image
ret_in_area = aircv.find_sift(predict_area, image, threshold=self.threshold, rgb=self.rgb)
# calc cv ret if found
if not ret_in_area:
return None
ret = deepcopy(ret_in_area)
ret["result"] = (ret_in_area["result"][0] + xmin, ret_in_area["result"][1] + ymin)
return ret
def _resize_image(self, image, screen, resize_method):
"""模板匹配中，将输入的截图适配成 等待模板匹配的截图."""
screen_resolution = aircv.get_resolution(screen)
# 如果分辨率一致，则不需要进行im_search的适配:
if tuple(self.resolution) == tuple(screen_resolution) or resize_method is None:
return image
# 分辨率不一致则进行适配，默认使用cocos_min_strategy:
h, w = image.shape[:2]
w_re, h_re = resize_method(w, h, self.resolution, screen_resolution)
# 确保w_re和h_re > 0, 至少有1个像素:
w_re, h_re = max(1, w_re), max(1, h_re)
# 调试代码: 输出调试信息.
G.LOGGING.debug("resize: (%s, %s)->(%s, %s), resolution: %s=>%s" % (
w, h, w_re, h_re, self.resolution, screen_resolution))
# 进行图片缩放:
image = cv2.resize(image, (w_re, h_re))
return image
def try_log_screen(screen=None):
"""
Save screenshot to file
Args:
screen: screenshot to be saved
Returns:
None
"""
if not ST.LOG_DIR:
return
if screen is None:
screen = G.DEVICE.snapshot(quality=ST.SNAPSHOT_QUALITY)
filename = "%(time)d.jpg" % {'time': time.time() * 1000}
filepath = os.path.join(ST.LOG_DIR, filename)
aircv.imwrite(filepath, screen, ST.SNAPSHOT_QUALITY)
return {"screen": filename, "resolution": aircv.get_resolution(screen)}
def _translate_code(self, step):
if step["tag"] != "function":
return None
step_data = step["data"]
args = []
code = {
"name": step_data["name"],
"args": args,
}
for key, value in step_data["call_args"].items():
args.append({
"key": key,
"value": value,
})
for k, arg in enumerate(args):
value = arg["value"]
if isinstance(value,
dict) and value.get("__class__") == "Template":
if self.export_dir: # all relative path
image_path = value['filename']
if not os.path.isfile(
os.path.join(self.script_root, image_path)):
shutil.copy(value['_filepath'], self.script_root
) # copy image used by using statement
else:
if self.online_path:
_filepath = os.path.abspath(value['_filepath'])
jk_filepath = _filepath.split(self.proj_name)[1]
image_path = self.online_path + jk_filepath.replace(
'\\', '/')
else:
image_path = os.path.abspath(value['_filepath']
or value['filename'])
arg["image"] = image_path
print(' arg["image"]: ', arg["image"])
crop_img = imread(value['_filepath'] or value['filename'])
arg["resolution"] = get_resolution(crop_img)
return code
def loop_find3(total_pic,
query,
posi,
tag=None,
timeout=ST.FIND_TIMEOUT,
threshold=None,
interval=0.5,
intervalfunc=None):
# print("ST.FIND_TIMEOUT",ST.FIND_TIMEOUT)
G.LOGGING.info("Try finding:\n%s", query.record_pos)
G.LOGGING.info(query._imread().shape)
# screen_resolution = aircv.get_resolution(screen)
part_size_width = query._imread().shape[0]
part_size_height = query._imread().shape[1]
part_resolution_width = query.resolution[0]
part_resolution_height = query.resolution[1]
# pairt_origin_xmin_relative = posi[0];
# pairt_origin_xmax_relative = posi[2] ;
pairt_origin_ymin_relative = posi[0] / part_resolution_height
pairt_origin_ymax_relative = posi[1] / part_resolution_height
# print("香菇：",(pairt_origin_ymin_relative,pairt_origin_ymax_relative))
posi_x = query.record_pos[0]
posi_y = query.record_pos[1]
# print("posi_x",posi_x)
start_time = time.time()
screen = aircv.imread(total_pic)
G.LOGGING.debug("query path %s" % query.filepath)
# predict_area = aircv.crop_image(screen, (xmin, ymin, xmax, ymax))
screen_resolution = aircv.get_resolution(screen)
# print("总图片的分辨率",screen_resolution);
total_resolution_width = screen_resolution[0]
total_resolution_height = screen_resolution[1]
width_scale = round(total_resolution_width / part_resolution_width, 2)
height_scale = round(total_resolution_height / part_resolution_height, 2)
# print("width_scale",width_scale)
# print("width_scale",height_scale)
# print("总图的分辨率",(total_resolution_width,total_resolution_height))
# print("缩放之后的相对坐标",(pairt_origin_xmin_relative * width_scale,pairt_origin_ymin_relative * height_scale,pairt_origin_xmax_relative * width_scale,pairt_origin_ymax_relative * height_scale))
# pairt_end_xmin_absolute = round( (pairt_origin_xmin_relative * width_scale) * total_resolution_width );
pairt_end_xmin_absolute = 0
# pairt_end_xmax_absolute = round( (pairt_origin_xmax_relative * width_scale) * total_resolution_width );
pairt_end_xmax_absolute = total_resolution_width
pairt_end_ymin_absolute = round(
(pairt_origin_ymin_relative * height_scale) *
total_resolution_height) - 30
pairt_end_ymax_absolute = round(
(pairt_origin_ymax_relative * height_scale) *
total_resolution_height) + 30
# print("enddddd",(pairt_end_xmin_absolute,pairt_end_ymin_absolute,pairt_end_xmax_absolute,pairt_end_ymax_absolute))
# part_resolution_width_scaled = part_size_width * width_scale;
# part_resolution_height_scaled = part_size_height * height_scale;
# part_end_xmax = ((posi_x + 0.5) if (posi_x + 0.5)<=1 else 1)*total_resolution_width
# part_end_ymax = ((posi_y + 0.5) if (posi_y + 0.5)<=1 else 1)*total_resolution_height
# part_end_xmin = part_end_xmax - part_resolution_width_scaled
# part_end_ymin = part_end_ymax - part_resolution_height_scaled
predict_area = aircv.crop_image(
screen, (pairt_end_xmin_absolute, pairt_end_ymin_absolute,
pairt_end_xmax_absolute, pairt_end_ymax_absolute))
screen[0:pairt_end_ymin_absolute, 0:total_resolution_width] = 0
screen[pairt_end_ymax_absolute:total_resolution_height,
0:total_resolution_width] = 0
# print("黑素区域",(pairt_end_ymin_absolute,pairt_end_ymax_absolute))
# screen[200:250,0:50] = 0;
aircv.imwrite("part_black.png", screen, 99)
# predict_area = aircv.crop_image(screen, (0, 0, 1126, 2436));
# print("hahahahah")
# print((part_end_xmin, part_end_ymin, part_end_xmax, part_end_ymax))
# print(predict_area.shape)
aircv.imwrite("predict_area3.png", predict_area, 99)
while True:
if predict_area is None:
G.LOGGING.warning("预测区域有问题！")
else:
if threshold:
query.threshold = threshold
match_pos = query.match_in(screen)
if match_pos:
try_log_screen(screen)
return match_pos
if intervalfunc is not None:
intervalfunc()
# 超时则raise，未超时则进行下次循环:
if (time.time() - start_time) > timeout:
try_log_screen(screen)
# raise TargetNotFoundError('Picture %s not found in screen' % query)
G.LOGGING.warning("图片查找超时！")
break
else:
time.sleep(interval)
def loop_find2(total_pic,
query,
timeout=ST.FIND_TIMEOUT,
threshold=None,
interval=0.5,
intervalfunc=None):
G.LOGGING.info("Try finding:\n%s", query.record_pos)
G.LOGGING.info(query._imread().shape)
part_size_width = query._imread().shape[0]
part_size_height = query._imread().shape[1]
part_resolution_width = query.resolution[0]
part_resolution_height = query.resolution[1]
# print("part_resolution_width:" + str(part_resolution_width))
# print("part_resolution_height:" + str(part_resolution_height))
posi_x = query.record_pos[0]
posi_y = query.record_pos[1]
print("posi_x", posi_x)
start_time = time.time()
screen = aircv.imread(total_pic)
G.LOGGING.debug("query path %s" % query.filepath)
# predict_area = aircv.crop_image(screen, (xmin, ymin, xmax, ymax))
screen_resolution = aircv.get_resolution(screen)
total_resolution_width = query.resolution[0]
total_resolution_height = query.resolution[1]
width_scale = total_resolution_width / part_resolution_width
height_scale = total_resolution_height / part_resolution_height
print("test_scale:" + str(1 / 2))
print("width_scale:" + str(width_scale))
print("height_scase:" + str(height_scale))
part_resolution_width_scaled = part_size_width * width_scale
part_resolution_height_scaled = part_size_height * height_scale
print("part_resolution_width_scaled:" + str(part_resolution_width_scaled))
print("part_resolution_height_scaled:" +
str(part_resolution_height_scaled))
G.LOGGING.debug("图片的分辨率")
G.LOGGING.debug(screen_resolution)
part_end_xmax = ((posi_x + 0.5) if
(posi_x + 0.5) <= 1 else 1) * total_resolution_width
part_end_ymax = ((posi_y + 0.5) if
(posi_y + 0.5) <= 1 else 1) * total_resolution_height
part_end_xmin = part_end_xmax - part_resolution_width_scaled
part_end_ymin = part_end_ymax - part_resolution_height_scaled
predict_area = aircv.crop_image(
screen, (part_end_xmin, part_end_ymin, part_end_xmax, part_end_ymax))
print("hahahahah")
print((part_end_xmin, part_end_ymin, part_end_xmax, part_end_ymax))
print(predict_area.shape)
aircv.imwrite("predict_area.png", screen, 99)
while True:
if predict_area is None:
G.LOGGING.warning("预测区域有问题！")
else:
if threshold:
query.threshold = threshold
match_pos = query.match_in(screen)
if match_pos:
try_log_screen(screen)
return match_pos
if intervalfunc is not None:
intervalfunc()
# 超时则raise，未超时则进行下次循环:
if (time.time() - start_time) > timeout:
try_log_screen(predict_area)
# raise TargetNotFoundError('Picture %s not found in screen' % query)
G.LOGGING.warning("图片查找超时！")
break
else:
time.sleep(interval)
