def remove_nest_click(tree, itemid, origid):
orignode = tree[origid]
for otherid in tree:
if tree[otherid]['parent'] == itemid:
othernode = tree[otherid]
if (othernode['click'] and othernode['x'] >= orignode['x']
and othernode['x'] + othernode['width'] <=
orignode['x'] + orignode['width']
and othernode['y'] >= orignode['y']
and othernode['y'] + othernode['height'] <=
orignode['y'] + orignode['height']):
print("because of %s" % util.describe_node(orignode))
print("removing click on %s" % util.describe_node(othernode))
othernode['click'] = False
remove_nest_click(tree, otherid, origid)
def prepare_point(tree, itemid, app, scr, caseid, imgdata, treeinfo, tesapi):
point = {}
point['app'] = app
point['scr'] = scr
point['case'] = caseid
point['id'] = itemid
point['str'] = util.describe_node(tree[itemid], None)
node = tree[itemid]
#cls = node['class']
add_ctx_attr(point, 'node_text', collect_text(node, tree), text_re, 10)
add_ctx(point, 'node_ctx', node, ['desc'], text_re)
add_ctx(point, 'node_ctx', node, ['id'], id_re)
add_ctx(point, 'node_class', node, ['class'], id_re)
if 'Recycler' in node['class'] or 'ListView' in node['class']:
point['node_class'] += " ListContainer"
point['node_x'] = node['x']
point['node_y'] = node['y']
point['node_w'] = node['width']
point['node_h'] = node['height']
point['node_word_count'] = len(text_re.findall(node['text']))
point['node_digits'] = len(digit_re.findall(node['text']))
prepare_neighbour(tree, itemid, point)
parent = node['parent']
point['parent_ctx'] = ''
parent_click = parent_scroll = parent_manychild = False
parent_depth = 0
while parent != 0 and parent != -1 and parent_depth < PARENT_DEPTH_LIMIT:
add_ctx(point, 'parent_ctx', tree[parent], ['class', 'id'], id_re)
parent_click |= tree[parent]['click']
parent_scroll |= tree[parent]['scroll']
parent_manychild |= len(tree[parent]['children']) > 1
parent = tree[parent]['parent']
parent_depth += 1
point['parent_prop'] = [parent_click, parent_scroll, parent_manychild]
has_dupid = False
is_itemlike = False
is_listlike = False
for _id in node['raw']:
if _id in treeinfo['dupid']:
has_dupid = True
break
for _id in node['raw']:
if _id in treeinfo['itemlike']:
is_itemlike = True
break
for _id in node['raw']:
if _id in treeinfo['listlike']:
is_listlike = True
break
point['node_prop'] = [node['click'], node['scroll'], len(node['children']) > 1,
has_dupid, is_itemlike, is_listlike]
prepare_img(point, node, imgdata, tesapi)
return point
def handle_dialog(self, curr_state, dev, observer):
if self.disallow_dialog:
return False
if curr_state.get('tree', None) is None:
return False
tree = curr_state.get('tree')
ret = dialog.detect_dialog(tree)
if not ret[0]:
return False
# sometimes pop up slowly
dev.wait_idle()
gui_state = observer.grab_state(dev, no_img=True)
curr_state.merge(gui_state)
tree = curr_state.get('tree')
ret = dialog.detect_dialog(tree)
if not ret[0]:
return False
logger.info("dialog detected")
buttonsid = ret[1]
if len(buttonsid) == 1:
logger.info("single button dialog")
# what else can you do?
loc = locator.itemid_locator(buttonsid[0])
op = operation.Operation("click", loc)
ret = op.do(dev, observer, curr_state, environ.empty, self)
gui_state = observer.grab_state(dev)
curr_state.merge(gui_state)
return ret
if len(buttonsid) == 0:
logger.info("no buttons")
return False
types = {}
for buttonid in buttonsid:
logger.info("%s", util.describe_node(tree[buttonid], short=True))
button_type = dialog.detect_dialog_button(tree, buttonid, buttonsid)
if button_type is None:
return False
types[button_type] = buttonid
action_to_do = dialog.decide_dialog_action(tree)
logger.info("types: %s, decide to click %s", types, action_to_do)
if action_to_do in types:
loc = locator.itemid_locator(types[action_to_do])
op = operation.Operation("click", loc)
ret = op.do(dev, observer, curr_state, environ.empty, self)
gui_state = observer.grab_state(dev)
curr_state.merge(gui_state)
return ret
else:
logger.info("can't find the decided action")
return False
def mark_item(scr_name, item_id, item_desc, items):
if not item_valid(scr_name, item_desc):
print("warn: illegal item %s" % item_desc)
rets[item_id] = item_desc
print("Marked as <%s> %s" %
(item_desc,
util.describe_node(find_node(item_id, tree),
short=True).strip()))
history.append([item_id, item_desc])
save_results()
def record_action(self, type_, args={}):
self.flush_cache()
ret = "ACTION: %s" % type_
for item in sorted(args):
if item == 'target':
ret += ' target: %s' % util.describe_node(args[item])
else:
ret += ' %s: %r' % (item, args[item])
print(ret)
if self.save:
args['action'] = type_
saction = json.dumps(args)
self.action_file.write(saction)
self.action_file.write('\n')
self.action_file.flush()
def prepare_self(self):
node = self.tree[self.nodeid]
# AUX info
self.point['id'] = self.nodeid
self.point['str'] = util.describe_node(node, None)
self.add_ctx('node_text', node, ['text'], text_re, 10)
self.add_ctx('node_ctx', node, ['desc'], text_re)
self.add_ctx('node_ctx', node, ['id'], id_re)
self.add_ctx('node_class', node, ['class'], id_re)
if 'Recycler' in node['class'] or 'ListView' in node['class']:
self.point['node_class'] += " ListContainer"
self.point['node_x'] = node['x']
self.point['node_y'] = node['y']
self.point['node_w'] = node['width']
self.point['node_h'] = node['height']
def get_listinfo(tree, nodeid):
node = tree[nodeid]
list_items = []
for childid in node['children']:
child = tree[childid]
list_item = {}
list_item['text'] = collect_text(tree, child)
list_item['click'] = child['click']
list_items.append(list_item)
print("Items of list %s %dx%d @%d-%d" %
(util.describe_node(node), node['origw'], node['origh'],
node['origx'], node['origy']))
for item in list_items:
print(item)
return list_items
def explore():
print("learning")
clas = classify.learn("../guis/", None)
element_clas = elements.learn("../guis/", None)
print("exploring")
dev = device.Device()
while True:
hier = sense.grab_full(dev)
if not hier['xml']:
logger.error("fail to grab hierarchy")
return
print(hier['act'])
guess_scr = clas.classify(hier['xml'], hier['act'], hier['scr'])
print("classify: %s" % (guess_scr))
items = analyze.parse_xml(hier['xml'])
tree = analyze.analyze_items(items)
imgdata = sense.load_image(hier['scr'])
guess_descs = {}
treeinfo = analyze.collect_treeinfo(tree)
for itemid in tree:
guess_element = element_clas.classify(guess_scr, tree, itemid,
imgdata, treeinfo)
if guess_element != 'NONE':
guess_descs[itemid] = guess_element
util.print_tree(tree, guess_descs)
input('enter')
if False:
itemid = random.choice(sorted(tree))
if 'click' in tree[itemid] and tree[itemid]['click']:
print("Clicking %s" % util.describe_node(tree[itemid]))
widget.click(dev, tree[itemid])
def handle_console_cmd(cons, cmd, envs):
threading.current_thread().name = 'MainThread'
dev = envs['dev']
ob = envs['ob']
st = envs['state']
tlib = envs['tlib']
env = envs['env']
if 'app' in envs:
app = envs['app']
else:
app = 'cui'
def save_stat():
tlib.save_stat(os.path.join("../stat/", "%s.txt" % app))
if cmd == 'q':
save_stat()
dev.finish()
sys.exit(0)
elif cmd == 'sense':
scr = ob.grab_state(dev, no_verify=True, no_print=True)
if scr.get('guess_descs') is None:
util.print_tree(scr.get('tree'))
else:
util.print_tree(scr.get('tree'), scr.get('guess_descs'),
scr.get('guess_score'))
st.merge(scr)
return
elif cmd == 'tags':
config.show_guess_tags = True
scr = ob.grab_state(dev, no_verify=True, no_print=True)
util.print_tree(scr.get('tree'), scr.get('guess_descs'), scr.get('guess_score'))
st.merge(scr)
config.show_guess_tags = False
return
elif cmd == 'tree':
scr = ob.grab_state(dev, no_img=True)
util.print_tree(scr.get('tree'))
st.merge(scr)
return
elif cmd.startswith('app '):
app = cmd.split(' ')[1]
load_app(app, ob, tlib, envs)
return
elif cmd == 'load':
tlib = testlib.collect_pieces("../tlib/")
envs['tlib'] = tlib
load_app(app, ob, tlib, envs)
return
elif cmd == 'reload':
value.init_params("../etc/", app)
try:
if app:
os.remove("../model/screen_%s" % app)
os.remove("../model/element_%s" % app)
else:
os.remove("../model/screen")
os.remove("../model/element")
except:
pass
ob.load("../model/", "../guis/", "../guis-extra/", config.extra_screens,
config.extra_element_scrs)
return
elif cmd == 'tlib':
tlib = testlib.collect_pieces("../tlib/")
envs['tlib'] = tlib
return
elif cmd == 'test':
tlib = testlib.collect_pieces("../tlib/")
tlib.set_app(app)
tlib.add_test(microtest.init_test(appdb.get_app(app)))
envs['tlib'] = tlib
config.show_guess_tags = True
scr = ob.grab_state(dev)
config.show_guess_tags = False
st.merge(scr)
begin_state = st.to_essential(tlib.essential_props())
tests = tlib.usable_tests(st)
tests.sort(key=lambda test: test.prio)
for i in range(len(tests)):
testinfo = tlib.get_testinfo(tests[i])
print("%2d. %s: %s [%d %d]" % (i, tests[i].feature_name, tests[i].name,
testinfo.succs, testinfo.fails))
if len(tests) == 0:
print("no test available!")
return
if len(tests) == 1:
tid = 0
else:
tid = input("which one? ")
try:
tid = int(tid)
except:
return
test = tests[tid]
ret = test.attempt(dev, ob, st, tlib, environ.empty)
util.print_tree(st.get('tree'), st.get('guess_descs'))
end_state = st.to_essential(tlib.essential_props())
if ret:
print("test SUCC")
tlib.clear_stat(test)
tlib.mark_succ(test, begin_state, end_state)
else:
print("test FAIL")
tlib.mark_fail(test, begin_state)
save_stat()
return
elif cmd == 'save':
save_stat()
return
elif cmd == 'stat':
tlib.print_stat()
return
elif cmd == 'items':
scr = ob.grab_state(dev, no_img=True)
util.printitems(scr.get('items'))
return
elif cmd.startswith('item'):
itemid = int(cmd.split(' ')[1])
scr = ob.grab_state(dev, no_img=True)
items = scr.get('items')
if itemid in items:
print(util.describe(items[itemid]))
else:
print("no such item: %d" % itemid)
return
elif cmd == 'debug':
logging.basicConfig(level=logging.DEBUG)
return
elif cmd == 'idle':
dev.wait_idle()
return
elif cmd == 'dialog':
scr = ob.grab_state(dev)
ret = tlib.handle_dialog(scr, dev, ob)
if ret:
print("dialog handled")
else:
print("dialog not handled")
return
elif cmd.startswith('find '):
tag = cmd.split(' ', 1)[1]
con = concept.parse(tag)
if con is None:
print("invalid locator")
return
scr = ob.grab_state(dev)
widgets = con.locate(scr, ob, environ.Environment())
if widgets == []:
print("can't find")
else:
for widget in widgets:
print("FOUND: %s" % widget)
print(" content:", widget.content())
return
elif cmd == 'perf':
perfmon.print_stat()
return
elif cmd == 'clear':
init = tlib.find_test('meta', 'start app')
st.reset()
init.attempt(dev, ob, st, tlib, None)
ob.update_state(dev, st)
tlib.mark_succ(init, state.init_state, st)
return
elif cmd.startswith('set'):
parts = cmd.split(' ', 2)
if len(parts) == 2:
key = parts[1]
val = "1"
else:
(key, val) = parts[1:]
st.set(key, val)
return
elif cmd.startswith('del'):
parts = cmd.split(' ', 1)
key = parts[1]
st.remove(key)
return
elif cmd == 'dump':
filename = util.choose_filename("../cap/", "page%d")
logger.info("dumping to page %s", filename)
dev.dump(filename)
#sense.dump_page(dev, "../cap/")
return
elif cmd == 'list':
scr = ob.grab_state(dev, no_img=True)
tree = scr.get('tree')
treeinfo = analyze.collect_treeinfo(tree)
for root in sorted(treeinfo['listlike']):
print("ROOT:", util.describe_node(tree[root]))
for item in sorted(treeinfo['itemlike']):
if listinfo.get_lca(tree, [root, item]) == root:
print(" NODE:", util.describe_node(tree[item]))
for field in sorted(treeinfo['dupid']):
if listinfo.get_lca(tree, [item, field]) == item:
print(" FIELD:", util.describe_node(tree[field]))
return
elif cmd == 'webon':
config.GRAB_WEBVIEW = True
return
elif cmd == 'weboff':
config.GRAB_WEBVIEW = False
return
elif cmd.startswith('ob '):
prop = cmd.split(' ', 1)[1]
wd = watchdog.Watchdog(100000)
smgr = statemgr.StateMgr(tlib, None, dev, ob, wd)
ret = smgr.observe_prop(prop, st)
if ret:
print("prop %s = %s" % (prop, st.get(prop, '')))
else:
print("observe error")
return
elif cmd.startswith('clean '):
parts = cmd.split(' ', 2)
if len(parts) == 2:
prop = parts[1]
val = "1"
else:
(prop, val) = parts[1:]
wd = watchdog.Watchdog(100000)
smgr = statemgr.StateMgr(tlib, None, dev, ob, wd)
ret = smgr.cleanup_prop(prop, val, st)
if ret:
print("prop %s cleaned" % prop)
else:
print("clean error")
return
elif cmd.startswith('oc '):
prop = cmd.split(' ', 1)[1]
wd = watchdog.Watchdog(100000)
smgr = statemgr.StateMgr(tlib, None, dev, ob, wd)
ret = smgr.observe_and_clean(prop, st)
if ret:
print("prop %s is clean now" % prop)
else:
print("observe/clean error")
return
elif cmd == 'dbg':
print(tlib)
return
elif cmd.startswith('skip '):
name = cmd.split(' ', 1)[1]
add_skip(app, name)
print("skipping %s" % name)
return
elif cmd == 'skip':
skips = load_skip(app)
for skip in skips:
print("now skipping %s" % skip)
return
elif cmd.startswith('noskip '):
name = cmd.split(' ', 1)[1]
del_skip(app, name)
print("not skipping %s" % name)
return
elif cmd == 'src':
if st.get('xml') is not None:
print(st.get('xml'))
if st.get('src') is not None:
print(st.get('src'))
return
elif cmd == 'install':
apputils.install(dev, app)
return
elif cmd == 'uninstall':
apputils.uninstall(dev, app)
return
op = operation.parse_line(cmd)
if op is None:
print("unknown op")
return
ret = op.do(dev, ob, st, env, tlib)
if not ret:
print("op failed")
def add_label(self, node, desc):
print('%s -> %s' % (util.describe_node(node, short=True), desc))
node['label'] = desc
def prepare_img(point, node, imgdata, tesapi):
# your widget should be inside the screenshot
#assert(node['y'] + node['height'] <= imgdata.shape[0])
#assert(node['x'] + node['width'] <= imgdata.shape[1])
(min_x, min_y) = (node['x'], node['y'])
if min_x < 0:
min_x = 0
if min_y < 0:
min_y = 0
(max_x, max_y) = (node['x'] + node['width'], node['y'] + node['height'])
if max_x > imgdata.shape[1]:
if not node['webview']:
logger.debug("%s widget x2 %d > %d" % (util.describe_node(node, short=True),
max_x, imgdata.shape[1]))
max_x = imgdata.shape[1]
if max_y > imgdata.shape[0]:
if not node['webview']:
logger.debug("widget y2 %s %d > %d" % (util.describe_node(node, short=True),
max_y, imgdata.shape[0]))
max_y = imgdata.shape[0]
real_width = max(max_x - min_x, 0)
real_height = max(max_y - min_y, 0)
#min_dim = min(real_width, real_height)
if real_width * real_height == 0:
myimg_thr = numpy.zeros([32, 32], float)
logger.debug("empty image!")
else:
try:
myimg = imgdata[min_y: max_y, min_x: max_x]
#myimg = imgdata[min_y: min_y + min_dim, min_x: min_x + min_dim]
myimg = skimage.transform.resize(myimg, (32, 32), mode='constant')
except:
logger.error("ERROR at %s %dx%d-%dx%d" % (util.describe_node(node),
min_x, min_y, max_x, max_y))
raise
# myimg = imgdata[node['origy']: node['origy'] + node['origh'],
# node['origx']: node['origx'] + node['origw']]
# point['img'] = myimg
if use_threshold:
if myimg.max() - myimg.min() < 1e-6:
thres = 0.5
else:
thres = skimage.filters.threshold_otsu(myimg)
myimg_thr = myimg >= thres
myimg_thr = skimage.img_as_float(myimg_thr)
else:
myimg_thr = myimg
if myimg_thr.mean() < 0.2:
myimg_thr = 1.0 - myimg_thr
point['img_thr'] = myimg_thr
# point['img_flat'] = myimg_thr.flatten()
img_feature = skimage.feature.hog(myimg_thr, orientations=8, pixels_per_cell=(8, 8),
cells_per_block=(1, 1), block_norm='L1')
# print(len(img_feature))
point['img_hog'] = img_feature
if config.elements_use_ocr:
ocr_text = node['ocr']
#and real_width * real_height > 0:
#tesapi.SetRectangle(node['x'], node['y'], node['width'], node['height'])
#try:
# ocr_text = tesapi.GetUTF8Text()
#except:
# logger.warning("tessearact fail to recognize")
# ocr_text = ''
#ocr_text = ocr_text.strip().replace('\n', ' ')
else:
ocr_text = 'dummy'
point['node_ocr'] = ocr_text
#if point['node_text'].strip() == '':
# point['node_text'] = ocr_text
logger.debug("%s VS %s" % (ocr_text, node['text']))
def __str__(self):
return util.describe_node(self.node, short=True)
def delnode(self, tree, nodeid):
logger.debug("removing %s: %s", nodeid,
util.describe_node(tree[nodeid]))
del tree[nodeid]
def analyze(files,
print_rets=False,
show_progress=False,
print_items=False,
print_error=False,
show_ocr=False,
show_stat=False,
use_ocr=False):
ret = []
if show_progress:
progress = progressbar.ProgressBar()
items = progress(files)
else:
items = files
analyzer = Analyzer()
for filename in items:
filebase = os.path.splitext(filename)[0]
logger.debug("analyzing %s" % filename)
(items, descs, regs) = load_case(filename)
if print_items:
util.printitems(items)
start_time = time.time()
newtree = analyzer.analyze_items(items, descs, regs, print_items,
print_error, [])
ret.append(newtree)
if print_rets:
util.print_tree(newtree)
logger.info("Time used: %.3fs", time.time() - start_time)
if use_ocr:
hidden.add_ocrinfo(newtree, filebase + '.png')
hidden.find_hidden_ocr(newtree)
util.print_tree(newtree)
if print_rets:
dlg = dialog.detect_dialog(newtree)
if dlg[0]:
logger.info("I think this is dialog")
for btnid in dlg[1]:
logger.info("btn: %s", util.describe_node(newtree[btnid]))
logger.info(
"is: %s",
dialog.detect_dialog_button(newtree, btnid, dlg[1]))
logger.info("decide to click: %s",
dialog.decide_dialog_action(newtree))
if print_error:
for itemid in descs:
found = False
for nodeid in newtree:
if itemid in newtree[nodeid]['raw']:
found = True
break
if not found:
logger.error("REMOVED: %s %d %s %s",
os.path.basename(filename), itemid,
descs[itemid], util.describe(items[itemid]))
if show_stat:
analyzer.show_stat()
return ret
def observe(files):
viewproc = None
for filename in files:
filebase = os.path.splitext(filename)[0]
descname = filebase + '.desc.txt'
imgname = filebase + '.png'
if viewproc is not None:
try:
viewproc.kill()
except:
pass
viewproc = subprocess.Popen([config.picviewer_path, imgname])
scr_name = filename.split('/')[-1].split('.')[0].split('_')[-1]
if scr_name.startswith('cat'):
scr_name = 'cat'
if '.xml' in filename:
with open(filename) as f:
src = f.read()
root = ET.fromstring(src)
output_stack = []
items = {}
attrs = {}
parse(root, 0, 0, None, config.width, config.real_height,
output_stack, 0, items, attrs)
def get_depth(line):
line = line[3:]
return len(line) - len(line.lstrip())
max_item_id = max(items)
ext_items = {}
for i in range(max_item_id + 1):
if i in items:
my_depth = get_depth(items[i])
my_lines = [items[i]]
for j in range(i + 1, max_item_id + 1):
if j in items:
his_depth = get_depth(items[j])
if his_depth > my_depth:
my_lines.append(items[j])
else:
break
ext_items[i] = my_lines
for output in output_stack:
print(output)
print("==== IMPORTANT ====")
for item_id in sorted(attrs):
if attrs[item_id]['important']:
print(attrs[item_id]['output'])
rets = {}
if os.path.exists(descname):
with open(descname) as inf:
for line in inf.read().split('\n'):
if not line:
continue
(item_id, desc) = line.split(' ')
rets[int(item_id)] = desc
if '.xml' in filename:
tree = analyze.analyze([filename])[0]
elif '.hier' in filename:
loaded = webdriver.load(filebase)
items = loaded['items']
tree = analyze.analyze_items(items)
print("=== ANALYZED ===")
util.print_tree(tree)
def removed_from_tree(itemid, tree):
for nodeid in tree:
if itemid in tree[nodeid]['raw']:
return False
return True
print("=== CURRENT ===")
for itemid in sorted(rets):
for nodeid in tree:
if itemid in tree[nodeid]['raw']:
print("%3s %15s %s" % (itemid, rets[itemid],
util.describe_node(tree[nodeid])))
#print("%3s %15s %s" % (itemid, rets[itemid], items[itemid]))
if removed_from_tree(itemid, tree):
print("MISSING FROM TREE!", itemid)
print_tags(scr_name)
def save_results():
with open(descname, 'w') as outf:
for item_id in sorted(rets):
outf.write("%s %s\n" % (item_id, rets[item_id]))
def find_node(itemid, tree):
for nodeid in tree:
if itemid in tree[nodeid]['raw']:
return tree[nodeid]
return None
history = []
def mark_item(scr_name, item_id, item_desc, items):
if not item_valid(scr_name, item_desc):
print("warn: illegal item %s" % item_desc)
rets[item_id] = item_desc
print("Marked as <%s> %s" %
(item_desc,
util.describe_node(find_node(item_id, tree),
short=True).strip()))
history.append([item_id, item_desc])
save_results()
def unmark_item(item_id):
del rets[item_id]
print("deleted item %d" % item_id)
save_results()
cur_item = -1
while True:
line = input("%s> " % filename)
if line == '':
break
if line == '?':
for item_id in sorted(rets):
print("%20s %-100s" % (rets[item_id], items[item_id]))
continue
if line == ';':
util.print_tree(tree)
#for line in output_stack:
# print(line)
continue
if line == '!':
print_tags(scr_name)
continue
if line == 'q':
sys.exit(0)
mode = 'l'
if ' ' in line:
parts = line.split(' ')
try:
cur_item = int(parts[0])
item_desc = parts[1]
item_idx = -1
except:
mode = parts[0]
if mode == 'r':
rep = int(parts[1])
offset = int(parts[2])
if len(parts) > 3:
num = int(parts[3])
else:
num = 1
elif mode == 'd':
del_idx = int(parts[1])
else:
try:
item_idx = int(line)
except:
item_desc = line
item_idx = -1
if mode == 'r':
cur_len = len(history)
for j in range(num):
for i in range(rep):
old_entry = history[cur_len - rep + i]
mark_item(scr_name, old_entry[0] + offset + j * offset,
old_entry[1], items)
elif mode == 'd':
unmark_item(del_idx)
else:
if item_idx == -1:
if cur_item != -1:
mark_item(scr_name, cur_item, item_desc, items)
else:
print("You must choose an item first")
else:
if item_idx in items:
cur_item = item_idx
print("item %d is:" % item_idx)
for line in ext_items[item_idx]:
print(line)
else:
print("item does not exist")
if viewproc is not None:
viewproc.kill()
