def load_info(only_app, only_scrs, filename):
(appname, caseid,
scrname) = re.search('([a-z0-9]+)_?([0-9]+)?_([a-z0-9]+)',
filename).groups()
if only_app is not None and appname != only_app:
return None
if only_scrs is not None and scrname not in only_scrs:
return None
filebase = os.path.splitext(filename)[0]
tree = analyze.load_tree(filename)
hidden.find_hidden_ocr(tree)
hidden.mark_children_hidden_ocr(tree)
pngfile = filebase + '.png'
path = get_path(filebase)
return {
'tree': tree,
'img': pngfile,
'regs': collect_tags(tree),
'app': appname,
'file': filename,
'path': path,
'scr': scrname
}
def dump_tree(self, filename):
tree = analyze.load_tree(filename)
app = os.path.basename(filename).split('_')[0]
if self.hide:
hidden.find_hidden_ocr(tree)
if not self.nofill:
hidden.mark_children_hidden_ocr(tree)
for nodeid in tree:
node = tree[nodeid]
if node['visible'] == 'hidden':
self.hidden_cnt += 1
self.hidden_app[app] = self.hidden_app.get(app, 0) + 1
self.total_cnt += 1
self.total_app[app] = self.total_app.get(app, 0) + 1
if (node['regs']
or node['tags']) and node['visible'] == 'hidden':
logger.error(
"ERROR!: INVISIBLE %s %d %s", filename, nodeid,
node['regs'][0] if node['regs'] else node['tags'])
if self.print_tree or self.show:
util.print_tree(tree, show_hidden=self.show)
def process_input(filename):
filebase = os.path.splitext(filename)[0]
basename = os.path.basename(filename)
pagename = basename.split(".")[0]
if basename.count("_") == 1:
(appname, scrname) = pagename.split("_")
else:
(appname, casename, scrname) = pagename.split("_")
if scrname == 'cat1':
scrname = "cat"
elif scrname == 'cat':
pass
elif scrname.startswith('cat'):
scrname = "cat2"
# if scrname == "cat1":
# continue
# if scrname == "searchret":
# scrname = "list"
ptfile = filebase + '.pt'
if os.path.exists(ptfile) and not nopt:
ptf = open(ptfile, 'rb')
unpickler = pickle.Unpickler(ptf)
try:
pt = unpickler.load()
return (scrname, pt)
except:
pass
ptf = open(ptfile, 'wb')
pickler = pickle.Pickler(ptf)
#with open(filename, 'r') as f:
# xmldata = f.read()
tree = analyze.load_tree(filename)
hidden.find_hidden_ocr(tree)
hidden.mark_children_hidden_ocr(tree)
if '.xml' in filename:
actfile = filebase + '.txt'
actname = open(actfile).read()
# tree = analyze.analyze([filename], show_progress=False)[0]
elif '.hier' in filename:
urlfile = filebase + '.url'
actname = util.url_to_actname(open(urlfile).read())
# loaded = webdriver.load(filebase)
# tree = analyze.analyze_items(loaded['items'])
imgfile = filebase + '.png'
# treeinfo = preprocess(xmldata)
# # data = ""
#
# actinfo = ''
# if os.path.exists(actfile):
# with open(actfile) as txtf:
# actinfo = preprocess_txt(txtf.read())
# else:
# if only_good_pt:
# logger.info("skipping %s: no act", filename)
# continue
# else:
# if missing_warn:
# logger.warn("WARN: %s missing activity info" % filename)
#
# imginfo = ''
# if os.path.exists(imgfile):
# if not use_fake_ocr:
# imginfo = preprocess_img(imgfile)
# else:
# imginfo = preprocess_img_fake(xmldata)
# else:
# if only_good_pt:
# logger.info("skipping %s: no img", filename)
# continue
# else:
# if missing_warn:
# logger.warn("WARN: %s missing img info" % filename)
# pt = {'app': appname, 'tree': treeinfo, 'act': actinfo, 'img': imginfo,
#'file': filename}
# if config.use_postprocess:
pt = prepare_point(actname, imgfile, tree)
# else:
# pt = prepare_point(xmldata, open(actfile).read(), imgfile)
pt['file'] = filename
pt['app'] = appname
pt['scr'] = appname
# treeinfo = ' '.join([treeinfo, actinfo, imginfo])
pickler.dump(pt)
return (scrname, pt)
def load(self, prev=False):
if self.file_idx == len(self.filenames):
Gtk.main_quit()
return
if prev:
self.file_idx -= 2
filename = self.filenames[self.file_idx]
(self.app, self.scr) = util.get_aux_info(filename)
if self.app not in self.memory:
self.memory[self.app] = {}
self.set_title(filename)
self.file_idx += 1
print("Loading %s" % filename)
self.pngfile = os.path.splitext(filename)[0] + '.png'
self.descname = os.path.splitext(filename)[0] + '.%s.txt' % self.kind
starttime = time.time()
self.tree = analyze.load_tree(filename)
hidden.find_hidden_ocr(self.tree)
hidden.mark_children_hidden_ocr(self.tree)
util.print_tree(self.tree, show_hidden=self.show_hidden)
if self.ml:
self.get_ml_rets()
else:
self.load_desc()
endtime = time.time()
print("Load time: %.3fs" % (endtime - starttime))
self.focus_id = -1
self.colors = {}
self.ptx = self.pty = 0
self.img = cairo.ImageSurface.create_from_png(self.pngfile)
print('Image:', self.img.get_width(), self.img.get_height())
root_item_id = min(self.tree)
root_node = self.tree[root_item_id]
print('Root node:', root_node['width'], root_node['height'])
self.scale = 1.0 * self.img.get_width() / config.width
#self.scale = analyze.find_closest(self.scale, analyze.SCALE_RATIOS)
print('Scale:', '%.3f' % self.scale, '->', '%.3f' % self.scale)
self.resize(self.img.get_width(), self.img.get_height())
self.mark_depth(self.tree)
for item_id in self.tree:
color_r = random.random() / 2
color_g = random.random() / 2
color_b = random.random() / 2
self.colors[item_id] = (color_r, color_g, color_b)
imgocr = Image.open(self.pngfile)
self.imgwidth = imgocr.width
self.imgheight = imgocr.height
#imgocr2 = imgocr.convert("RGB").resize(
# (imgocr.width * OCR_RATIO, imgocr.height * OCR_RATIO))
self.tesapi.SetImage(imgocr)
self.tesapi.SetSourceResolution(config.ocr_resolution)
self.dump_memory()
def load_point(no_appname, nopt, filename):
points = []
labels = []
apps = []
scrs = []
filebase = os.path.splitext(filename)[0]
(appname, caseid,
scrname) = re.search('([a-z0-9]+)_?([0-9]+)?_([a-z0-9]+)',
filename).groups()
if appname is not None and appname == no_appname:
return (points, labels, apps, scrs)
if scrname in tags.tag['ignored_screens']:
return (points, labels, apps, scrs)
if not nopt:
cached = load_cached_point(filebase)
if cached is not None:
if discard_middle_layer:
tree = analyze.load_tree(filename)
discard_middle(cached[0], tree)
return cached
tree = analyze.load_tree(filename)
hidden.find_hidden_ocr(tree)
hidden.mark_children_hidden_ocr(tree)
if not has_label(tree):
logger.info("%s: no label", filename)
return (points, labels, apps, scrs)
logger.debug("analyzing %s", filename)
pngfile = filebase + '.png'
imgdata = skimage.io.imread(pngfile, as_grey=True)
imgdata = skimage.transform.resize(imgdata, (config.height, config.width),
mode='constant')
treeinfo = analyze.collect_treeinfo(tree)
path = get_path(filebase)
collector = FeatureCollector(tree, pngfile)
for itemid in sorted(tree):
if cannot_be_region(tree[itemid]):
continue
try:
point = collector.prepare_point(itemid, appname, scrname, caseid,
imgdata, treeinfo, path)
except ValueError:
logger.exception("ERROR at %s" % filename)
return (points, labels, apps, scrs)
if point['empty']:
continue
pt_tag = 'NONE'
if len(tree[itemid]['regs']) > 0:
firsttag = tree[itemid]['regs'][0]
pt_tag = firsttag
points.append(point)
labels.append(pt_tag)
apps.append(appname)
scrs.append(scrname)
cache_points(filebase, points, labels, apps, scrs)
if discard_middle_layer:
tree = analyze.load_tree(filename)
discard_middle(points, tree)
return (points, labels, apps, scrs)
def load_point(no_appname, filename):
points = []
labels = []
apps = []
scrs = []
filebase = os.path.splitext(filename)[0]
featurefile = filebase + '.pts'
(appname, caseid, scrname) = re.search('([a-z0-9]+)_?([0-9]+)?_([a-z0-9]+)',
filename).groups()
if appname is not None and appname == no_appname:
return (points, labels, apps, scrs)
if scrname in tags.tag['ignored_screens']:
return (points, labels, apps, scrs)
if os.path.exists(featurefile) and not nopt:
featuref = open(featurefile, 'rb')
unpickler = pickle.Unpickler(featuref)
count = unpickler.load()
for i in range(count):
try:
point = unpickler.load()
pt_tag = unpickler.load()
appname = unpickler.load()
scrname = unpickler.load()
points.append(point)
labels.append(pt_tag)
apps.append(appname)
scrs.append(scrname)
except:
break
return (points, labels, apps, scrs)
if not nopt and print_empty_pt:
logger.info("analyzing %s", filename)
tree = analyze.load_tree(filename)
hidden.find_hidden_ocr(tree)
hidden.mark_children_hidden_ocr(tree)
#if '.xml' in filename:
# tree = analyze.analyze([filename], show_progress=False)[0]
# # if scrname.startswith('cat'):
# # scrname = "cat"
#else:
# loaded = webdriver.load(filebase)
# descs = util.load_desc(filebase)
# tree = analyze.analyze_items(loaded['items'], descs=descs)
if not has_label(tree):
logger.info("%s: no label", filename)
return (points, labels, apps, scrs)
pngfile = filebase + '.png'
imgdata = skimage.io.imread(pngfile, as_grey=True)
imgdata = skimage.transform.resize(imgdata, (config.height, config.width),
mode='constant')
featuref = open(featurefile, 'wb')
pickler = pickle.Pickler(featuref)
pickler.dump(len(sorted(tree)))
treeinfo = analyze.collect_treeinfo(tree)
tesapi = get_tesapi()
set_tes_image(tesapi, imgdata)
for itemid in sorted(tree):
try:
point = prepare_point(tree, itemid, appname, scrname, caseid, imgdata,
treeinfo, tesapi)
except ValueError:
logger.exception("ERROR at %s" % filename)
return (points, labels, apps, scrs)
pt_tag = 'NONE'
if len(tree[itemid]['tags']) > 0:
firsttag = tree[itemid]['tags'][0]
#if tags.valid(scrname, firsttag):
pt_tag = firsttag
# firsttag = CONV.get(firsttag, firsttag)
# if firsttag is not None:
# if (not scrname in ONLY_CARE_LABEL or
# not ONLY_CARE_LABEL[scrname] or
# firsttag in ONLY_CARE_LABEL[scrname]):
# pt_tag = firsttag
#
# if scrname in IGNORE_LABELS and pt_tag in IGNORE_LABELS[scrname]:
# pt_tag = 'NONE'
# cnt[pt_tag] = cnt.get(pt_tag, 0) + 1
# skimage.io.imsave("%s_%d.png" % (pt_tag, cnt[pt_tag]), point['img_thr'])
for i in range(REP_COUNT.get(scrname, 1) if use_rep else 1):
points.append(point)
labels.append(pt_tag)
apps.append(appname)
scrs.append(scrname)
pickler.dump(point)
pickler.dump(pt_tag)
pickler.dump(appname)
pickler.dump(scrname)
if print_points:
print_point(point)
return (points, labels, apps, scrs)
def grab_adb(self, dev, no_img):
screen = None
if (self.last_state is not None and self.last_state_screen is not None
and (not self.last_state.get('noimg', False) or no_img)):
screen = sense.grab_full(dev, no_img=True)
if not screen['xml']:
logger.warning("fail to grab screen info")
return [None, None]
if screen['xml'] == self.last_state.get('xml'):
# not 'WebView' in screen['xml']):
logger.info("reuse last screen")
return [
copy.deepcopy(self.last_state),
sense.restore_screen(self.last_state_screen)
]
if screen is None or not no_img:
screen = sense.grab_full(dev, no_img=no_img)
if not screen['xml']:
logger.warning("fail to grab screen info")
return [None, None]
actname = screen['act']
logger.debug("screen grabbed")
curr_state = state.State()
curr_state.set('xml', screen['xml'])
curr_state.set('act', screen['act'])
curr_state.set('noimg', no_img)
items = analyze.parse_xml(screen['xml'])
curr_url = ''
if 'WebView' in screen['xml'] and config.GRAB_WEBVIEW:
curr_url = self.grab_webview(items, dev)
curr_state.set('url', curr_url)
if 1 not in items:
# not loaded yet, we should wait and retry
logger.warning("got empty screen")
return [None, None]
tree = analyze.analyze_items(items, history=self.history)
if len(tree) == 0:
logger.warning("got empty tree")
return [None, None]
if 'WebView' in screen['xml'] and config.GRAB_WEBVIEW:
self.annotate_webview(tree)
if 'scr' in screen:
hidden.add_ocrinfo(tree, screen['scr'])
if config.observe_remove_hidden_ocr:
hidden.find_hidden_ocr(tree)
hidden.mark_children_hidden_ocr(tree)
if config.GRAB_WEBVIEW and curr_url != '' and curr_url != 'about:blank':
actname += util.url_to_actname(curr_url)
if screen['act'] is None:
logger.warning("fail to observe activity")
curr_app = 'unknown'
else:
logger.info("at %s", screen['act'])
curr_app = util.get_app(screen['act'])
curr_state.set('app', curr_app)
return (screen, actname, tree, items, curr_state)
def analyze(files,
print_rets=False,
show_progress=False,
print_items=False,
print_error=False,
show_ocr=False,
show_stat=False,
use_ocr=False):
ret = []
if show_progress:
progress = progressbar.ProgressBar()
items = progress(files)
else:
items = files
analyzer = Analyzer()
for filename in items:
filebase = os.path.splitext(filename)[0]
logger.debug("analyzing %s" % filename)
(items, descs, regs) = load_case(filename)
if print_items:
util.printitems(items)
start_time = time.time()
newtree = analyzer.analyze_items(items, descs, regs, print_items,
print_error, [])
ret.append(newtree)
if print_rets:
util.print_tree(newtree)
logger.info("Time used: %.3fs", time.time() - start_time)
if use_ocr:
hidden.add_ocrinfo(newtree, filebase + '.png')
hidden.find_hidden_ocr(newtree)
util.print_tree(newtree)
if print_rets:
dlg = dialog.detect_dialog(newtree)
if dlg[0]:
logger.info("I think this is dialog")
for btnid in dlg[1]:
logger.info("btn: %s", util.describe_node(newtree[btnid]))
logger.info(
"is: %s",
dialog.detect_dialog_button(newtree, btnid, dlg[1]))
logger.info("decide to click: %s",
dialog.decide_dialog_action(newtree))
if print_error:
for itemid in descs:
found = False
for nodeid in newtree:
if itemid in newtree[nodeid]['raw']:
found = True
break
if not found:
logger.error("REMOVED: %s %d %s %s",
os.path.basename(filename), itemid,
descs[itemid], util.describe(items[itemid]))
if show_stat:
analyzer.show_stat()
return ret
