def remove_nest_click(tree, itemid, origid): orignode = tree[origid] for otherid in tree: if tree[otherid]['parent'] == itemid: othernode = tree[otherid] if (othernode['click'] and othernode['x'] >= orignode['x'] and othernode['x'] + othernode['width'] <= orignode['x'] + orignode['width'] and othernode['y'] >= orignode['y'] and othernode['y'] + othernode['height'] <= orignode['y'] + orignode['height']): print("because of %s" % util.describe_node(orignode)) print("removing click on %s" % util.describe_node(othernode)) othernode['click'] = False remove_nest_click(tree, otherid, origid)
def prepare_point(tree, itemid, app, scr, caseid, imgdata, treeinfo, tesapi): point = {} point['app'] = app point['scr'] = scr point['case'] = caseid point['id'] = itemid point['str'] = util.describe_node(tree[itemid], None) node = tree[itemid] #cls = node['class'] add_ctx_attr(point, 'node_text', collect_text(node, tree), text_re, 10) add_ctx(point, 'node_ctx', node, ['desc'], text_re) add_ctx(point, 'node_ctx', node, ['id'], id_re) add_ctx(point, 'node_class', node, ['class'], id_re) if 'Recycler' in node['class'] or 'ListView' in node['class']: point['node_class'] += " ListContainer" point['node_x'] = node['x'] point['node_y'] = node['y'] point['node_w'] = node['width'] point['node_h'] = node['height'] point['node_word_count'] = len(text_re.findall(node['text'])) point['node_digits'] = len(digit_re.findall(node['text'])) prepare_neighbour(tree, itemid, point) parent = node['parent'] point['parent_ctx'] = '' parent_click = parent_scroll = parent_manychild = False parent_depth = 0 while parent != 0 and parent != -1 and parent_depth < PARENT_DEPTH_LIMIT: add_ctx(point, 'parent_ctx', tree[parent], ['class', 'id'], id_re) parent_click |= tree[parent]['click'] parent_scroll |= tree[parent]['scroll'] parent_manychild |= len(tree[parent]['children']) > 1 parent = tree[parent]['parent'] parent_depth += 1 point['parent_prop'] = [parent_click, parent_scroll, parent_manychild] has_dupid = False is_itemlike = False is_listlike = False for _id in node['raw']: if _id in treeinfo['dupid']: has_dupid = True break for _id in node['raw']: if _id in treeinfo['itemlike']: is_itemlike = True break for _id in node['raw']: if _id in treeinfo['listlike']: is_listlike = True break point['node_prop'] = [node['click'], node['scroll'], len(node['children']) > 1, has_dupid, is_itemlike, is_listlike] prepare_img(point, node, imgdata, tesapi) return point
def handle_dialog(self, curr_state, dev, observer): if self.disallow_dialog: return False if curr_state.get('tree', None) is None: return False tree = curr_state.get('tree') ret = dialog.detect_dialog(tree) if not ret[0]: return False # sometimes pop up slowly dev.wait_idle() gui_state = observer.grab_state(dev, no_img=True) curr_state.merge(gui_state) tree = curr_state.get('tree') ret = dialog.detect_dialog(tree) if not ret[0]: return False logger.info("dialog detected") buttonsid = ret[1] if len(buttonsid) == 1: logger.info("single button dialog") # what else can you do? loc = locator.itemid_locator(buttonsid[0]) op = operation.Operation("click", loc) ret = op.do(dev, observer, curr_state, environ.empty, self) gui_state = observer.grab_state(dev) curr_state.merge(gui_state) return ret if len(buttonsid) == 0: logger.info("no buttons") return False types = {} for buttonid in buttonsid: logger.info("%s", util.describe_node(tree[buttonid], short=True)) button_type = dialog.detect_dialog_button(tree, buttonid, buttonsid) if button_type is None: return False types[button_type] = buttonid action_to_do = dialog.decide_dialog_action(tree) logger.info("types: %s, decide to click %s", types, action_to_do) if action_to_do in types: loc = locator.itemid_locator(types[action_to_do]) op = operation.Operation("click", loc) ret = op.do(dev, observer, curr_state, environ.empty, self) gui_state = observer.grab_state(dev) curr_state.merge(gui_state) return ret else: logger.info("can't find the decided action") return False
def mark_item(scr_name, item_id, item_desc, items): if not item_valid(scr_name, item_desc): print("warn: illegal item %s" % item_desc) rets[item_id] = item_desc print("Marked as <%s> %s" % (item_desc, util.describe_node(find_node(item_id, tree), short=True).strip())) history.append([item_id, item_desc]) save_results()
def record_action(self, type_, args={}): self.flush_cache() ret = "ACTION: %s" % type_ for item in sorted(args): if item == 'target': ret += ' target: %s' % util.describe_node(args[item]) else: ret += ' %s: %r' % (item, args[item]) print(ret) if self.save: args['action'] = type_ saction = json.dumps(args) self.action_file.write(saction) self.action_file.write('\n') self.action_file.flush()
def prepare_self(self): node = self.tree[self.nodeid] # AUX info self.point['id'] = self.nodeid self.point['str'] = util.describe_node(node, None) self.add_ctx('node_text', node, ['text'], text_re, 10) self.add_ctx('node_ctx', node, ['desc'], text_re) self.add_ctx('node_ctx', node, ['id'], id_re) self.add_ctx('node_class', node, ['class'], id_re) if 'Recycler' in node['class'] or 'ListView' in node['class']: self.point['node_class'] += " ListContainer" self.point['node_x'] = node['x'] self.point['node_y'] = node['y'] self.point['node_w'] = node['width'] self.point['node_h'] = node['height']
def get_listinfo(tree, nodeid): node = tree[nodeid] list_items = [] for childid in node['children']: child = tree[childid] list_item = {} list_item['text'] = collect_text(tree, child) list_item['click'] = child['click'] list_items.append(list_item) print("Items of list %s %dx%d @%d-%d" % (util.describe_node(node), node['origw'], node['origh'], node['origx'], node['origy'])) for item in list_items: print(item) return list_items
def explore(): print("learning") clas = classify.learn("../guis/", None) element_clas = elements.learn("../guis/", None) print("exploring") dev = device.Device() while True: hier = sense.grab_full(dev) if not hier['xml']: logger.error("fail to grab hierarchy") return print(hier['act']) guess_scr = clas.classify(hier['xml'], hier['act'], hier['scr']) print("classify: %s" % (guess_scr)) items = analyze.parse_xml(hier['xml']) tree = analyze.analyze_items(items) imgdata = sense.load_image(hier['scr']) guess_descs = {} treeinfo = analyze.collect_treeinfo(tree) for itemid in tree: guess_element = element_clas.classify(guess_scr, tree, itemid, imgdata, treeinfo) if guess_element != 'NONE': guess_descs[itemid] = guess_element util.print_tree(tree, guess_descs) input('enter') if False: itemid = random.choice(sorted(tree)) if 'click' in tree[itemid] and tree[itemid]['click']: print("Clicking %s" % util.describe_node(tree[itemid])) widget.click(dev, tree[itemid])
def handle_console_cmd(cons, cmd, envs): threading.current_thread().name = 'MainThread' dev = envs['dev'] ob = envs['ob'] st = envs['state'] tlib = envs['tlib'] env = envs['env'] if 'app' in envs: app = envs['app'] else: app = 'cui' def save_stat(): tlib.save_stat(os.path.join("../stat/", "%s.txt" % app)) if cmd == 'q': save_stat() dev.finish() sys.exit(0) elif cmd == 'sense': scr = ob.grab_state(dev, no_verify=True, no_print=True) if scr.get('guess_descs') is None: util.print_tree(scr.get('tree')) else: util.print_tree(scr.get('tree'), scr.get('guess_descs'), scr.get('guess_score')) st.merge(scr) return elif cmd == 'tags': config.show_guess_tags = True scr = ob.grab_state(dev, no_verify=True, no_print=True) util.print_tree(scr.get('tree'), scr.get('guess_descs'), scr.get('guess_score')) st.merge(scr) config.show_guess_tags = False return elif cmd == 'tree': scr = ob.grab_state(dev, no_img=True) util.print_tree(scr.get('tree')) st.merge(scr) return elif cmd.startswith('app '): app = cmd.split(' ')[1] load_app(app, ob, tlib, envs) return elif cmd == 'load': tlib = testlib.collect_pieces("../tlib/") envs['tlib'] = tlib load_app(app, ob, tlib, envs) return elif cmd == 'reload': value.init_params("../etc/", app) try: if app: os.remove("../model/screen_%s" % app) os.remove("../model/element_%s" % app) else: os.remove("../model/screen") os.remove("../model/element") except: pass ob.load("../model/", "../guis/", "../guis-extra/", config.extra_screens, config.extra_element_scrs) return elif cmd == 'tlib': tlib = testlib.collect_pieces("../tlib/") envs['tlib'] = tlib return elif cmd == 'test': tlib = testlib.collect_pieces("../tlib/") tlib.set_app(app) tlib.add_test(microtest.init_test(appdb.get_app(app))) envs['tlib'] = tlib config.show_guess_tags = True scr = ob.grab_state(dev) config.show_guess_tags = False st.merge(scr) begin_state = st.to_essential(tlib.essential_props()) tests = tlib.usable_tests(st) tests.sort(key=lambda test: test.prio) for i in range(len(tests)): testinfo = tlib.get_testinfo(tests[i]) print("%2d. %s: %s [%d %d]" % (i, tests[i].feature_name, tests[i].name, testinfo.succs, testinfo.fails)) if len(tests) == 0: print("no test available!") return if len(tests) == 1: tid = 0 else: tid = input("which one? ") try: tid = int(tid) except: return test = tests[tid] ret = test.attempt(dev, ob, st, tlib, environ.empty) util.print_tree(st.get('tree'), st.get('guess_descs')) end_state = st.to_essential(tlib.essential_props()) if ret: print("test SUCC") tlib.clear_stat(test) tlib.mark_succ(test, begin_state, end_state) else: print("test FAIL") tlib.mark_fail(test, begin_state) save_stat() return elif cmd == 'save': save_stat() return elif cmd == 'stat': tlib.print_stat() return elif cmd == 'items': scr = ob.grab_state(dev, no_img=True) util.printitems(scr.get('items')) return elif cmd.startswith('item'): itemid = int(cmd.split(' ')[1]) scr = ob.grab_state(dev, no_img=True) items = scr.get('items') if itemid in items: print(util.describe(items[itemid])) else: print("no such item: %d" % itemid) return elif cmd == 'debug': logging.basicConfig(level=logging.DEBUG) return elif cmd == 'idle': dev.wait_idle() return elif cmd == 'dialog': scr = ob.grab_state(dev) ret = tlib.handle_dialog(scr, dev, ob) if ret: print("dialog handled") else: print("dialog not handled") return elif cmd.startswith('find '): tag = cmd.split(' ', 1)[1] con = concept.parse(tag) if con is None: print("invalid locator") return scr = ob.grab_state(dev) widgets = con.locate(scr, ob, environ.Environment()) if widgets == []: print("can't find") else: for widget in widgets: print("FOUND: %s" % widget) print(" content:", widget.content()) return elif cmd == 'perf': perfmon.print_stat() return elif cmd == 'clear': init = tlib.find_test('meta', 'start app') st.reset() init.attempt(dev, ob, st, tlib, None) ob.update_state(dev, st) tlib.mark_succ(init, state.init_state, st) return elif cmd.startswith('set'): parts = cmd.split(' ', 2) if len(parts) == 2: key = parts[1] val = "1" else: (key, val) = parts[1:] st.set(key, val) return elif cmd.startswith('del'): parts = cmd.split(' ', 1) key = parts[1] st.remove(key) return elif cmd == 'dump': filename = util.choose_filename("../cap/", "page%d") logger.info("dumping to page %s", filename) dev.dump(filename) #sense.dump_page(dev, "../cap/") return elif cmd == 'list': scr = ob.grab_state(dev, no_img=True) tree = scr.get('tree') treeinfo = analyze.collect_treeinfo(tree) for root in sorted(treeinfo['listlike']): print("ROOT:", util.describe_node(tree[root])) for item in sorted(treeinfo['itemlike']): if listinfo.get_lca(tree, [root, item]) == root: print(" NODE:", util.describe_node(tree[item])) for field in sorted(treeinfo['dupid']): if listinfo.get_lca(tree, [item, field]) == item: print(" FIELD:", util.describe_node(tree[field])) return elif cmd == 'webon': config.GRAB_WEBVIEW = True return elif cmd == 'weboff': config.GRAB_WEBVIEW = False return elif cmd.startswith('ob '): prop = cmd.split(' ', 1)[1] wd = watchdog.Watchdog(100000) smgr = statemgr.StateMgr(tlib, None, dev, ob, wd) ret = smgr.observe_prop(prop, st) if ret: print("prop %s = %s" % (prop, st.get(prop, ''))) else: print("observe error") return elif cmd.startswith('clean '): parts = cmd.split(' ', 2) if len(parts) == 2: prop = parts[1] val = "1" else: (prop, val) = parts[1:] wd = watchdog.Watchdog(100000) smgr = statemgr.StateMgr(tlib, None, dev, ob, wd) ret = smgr.cleanup_prop(prop, val, st) if ret: print("prop %s cleaned" % prop) else: print("clean error") return elif cmd.startswith('oc '): prop = cmd.split(' ', 1)[1] wd = watchdog.Watchdog(100000) smgr = statemgr.StateMgr(tlib, None, dev, ob, wd) ret = smgr.observe_and_clean(prop, st) if ret: print("prop %s is clean now" % prop) else: print("observe/clean error") return elif cmd == 'dbg': print(tlib) return elif cmd.startswith('skip '): name = cmd.split(' ', 1)[1] add_skip(app, name) print("skipping %s" % name) return elif cmd == 'skip': skips = load_skip(app) for skip in skips: print("now skipping %s" % skip) return elif cmd.startswith('noskip '): name = cmd.split(' ', 1)[1] del_skip(app, name) print("not skipping %s" % name) return elif cmd == 'src': if st.get('xml') is not None: print(st.get('xml')) if st.get('src') is not None: print(st.get('src')) return elif cmd == 'install': apputils.install(dev, app) return elif cmd == 'uninstall': apputils.uninstall(dev, app) return op = operation.parse_line(cmd) if op is None: print("unknown op") return ret = op.do(dev, ob, st, env, tlib) if not ret: print("op failed")
def add_label(self, node, desc): print('%s -> %s' % (util.describe_node(node, short=True), desc)) node['label'] = desc
def prepare_img(point, node, imgdata, tesapi): # your widget should be inside the screenshot #assert(node['y'] + node['height'] <= imgdata.shape[0]) #assert(node['x'] + node['width'] <= imgdata.shape[1]) (min_x, min_y) = (node['x'], node['y']) if min_x < 0: min_x = 0 if min_y < 0: min_y = 0 (max_x, max_y) = (node['x'] + node['width'], node['y'] + node['height']) if max_x > imgdata.shape[1]: if not node['webview']: logger.debug("%s widget x2 %d > %d" % (util.describe_node(node, short=True), max_x, imgdata.shape[1])) max_x = imgdata.shape[1] if max_y > imgdata.shape[0]: if not node['webview']: logger.debug("widget y2 %s %d > %d" % (util.describe_node(node, short=True), max_y, imgdata.shape[0])) max_y = imgdata.shape[0] real_width = max(max_x - min_x, 0) real_height = max(max_y - min_y, 0) #min_dim = min(real_width, real_height) if real_width * real_height == 0: myimg_thr = numpy.zeros([32, 32], float) logger.debug("empty image!") else: try: myimg = imgdata[min_y: max_y, min_x: max_x] #myimg = imgdata[min_y: min_y + min_dim, min_x: min_x + min_dim] myimg = skimage.transform.resize(myimg, (32, 32), mode='constant') except: logger.error("ERROR at %s %dx%d-%dx%d" % (util.describe_node(node), min_x, min_y, max_x, max_y)) raise # myimg = imgdata[node['origy']: node['origy'] + node['origh'], # node['origx']: node['origx'] + node['origw']] # point['img'] = myimg if use_threshold: if myimg.max() - myimg.min() < 1e-6: thres = 0.5 else: thres = skimage.filters.threshold_otsu(myimg) myimg_thr = myimg >= thres myimg_thr = skimage.img_as_float(myimg_thr) else: myimg_thr = myimg if myimg_thr.mean() < 0.2: myimg_thr = 1.0 - myimg_thr point['img_thr'] = myimg_thr # point['img_flat'] = myimg_thr.flatten() img_feature = skimage.feature.hog(myimg_thr, orientations=8, pixels_per_cell=(8, 8), cells_per_block=(1, 1), block_norm='L1') # print(len(img_feature)) point['img_hog'] = img_feature if config.elements_use_ocr: ocr_text = node['ocr'] #and real_width * real_height > 0: #tesapi.SetRectangle(node['x'], node['y'], node['width'], node['height']) #try: # ocr_text = tesapi.GetUTF8Text() #except: # logger.warning("tessearact fail to recognize") # ocr_text = '' #ocr_text = ocr_text.strip().replace('\n', ' ') else: ocr_text = 'dummy' point['node_ocr'] = ocr_text #if point['node_text'].strip() == '': # point['node_text'] = ocr_text logger.debug("%s VS %s" % (ocr_text, node['text']))
def __str__(self): return util.describe_node(self.node, short=True)
def delnode(self, tree, nodeid): logger.debug("removing %s: %s", nodeid, util.describe_node(tree[nodeid])) del tree[nodeid]
def analyze(files, print_rets=False, show_progress=False, print_items=False, print_error=False, show_ocr=False, show_stat=False, use_ocr=False): ret = [] if show_progress: progress = progressbar.ProgressBar() items = progress(files) else: items = files analyzer = Analyzer() for filename in items: filebase = os.path.splitext(filename)[0] logger.debug("analyzing %s" % filename) (items, descs, regs) = load_case(filename) if print_items: util.printitems(items) start_time = time.time() newtree = analyzer.analyze_items(items, descs, regs, print_items, print_error, []) ret.append(newtree) if print_rets: util.print_tree(newtree) logger.info("Time used: %.3fs", time.time() - start_time) if use_ocr: hidden.add_ocrinfo(newtree, filebase + '.png') hidden.find_hidden_ocr(newtree) util.print_tree(newtree) if print_rets: dlg = dialog.detect_dialog(newtree) if dlg[0]: logger.info("I think this is dialog") for btnid in dlg[1]: logger.info("btn: %s", util.describe_node(newtree[btnid])) logger.info( "is: %s", dialog.detect_dialog_button(newtree, btnid, dlg[1])) logger.info("decide to click: %s", dialog.decide_dialog_action(newtree)) if print_error: for itemid in descs: found = False for nodeid in newtree: if itemid in newtree[nodeid]['raw']: found = True break if not found: logger.error("REMOVED: %s %d %s %s", os.path.basename(filename), itemid, descs[itemid], util.describe(items[itemid])) if show_stat: analyzer.show_stat() return ret
def observe(files): viewproc = None for filename in files: filebase = os.path.splitext(filename)[0] descname = filebase + '.desc.txt' imgname = filebase + '.png' if viewproc is not None: try: viewproc.kill() except: pass viewproc = subprocess.Popen([config.picviewer_path, imgname]) scr_name = filename.split('/')[-1].split('.')[0].split('_')[-1] if scr_name.startswith('cat'): scr_name = 'cat' if '.xml' in filename: with open(filename) as f: src = f.read() root = ET.fromstring(src) output_stack = [] items = {} attrs = {} parse(root, 0, 0, None, config.width, config.real_height, output_stack, 0, items, attrs) def get_depth(line): line = line[3:] return len(line) - len(line.lstrip()) max_item_id = max(items) ext_items = {} for i in range(max_item_id + 1): if i in items: my_depth = get_depth(items[i]) my_lines = [items[i]] for j in range(i + 1, max_item_id + 1): if j in items: his_depth = get_depth(items[j]) if his_depth > my_depth: my_lines.append(items[j]) else: break ext_items[i] = my_lines for output in output_stack: print(output) print("==== IMPORTANT ====") for item_id in sorted(attrs): if attrs[item_id]['important']: print(attrs[item_id]['output']) rets = {} if os.path.exists(descname): with open(descname) as inf: for line in inf.read().split('\n'): if not line: continue (item_id, desc) = line.split(' ') rets[int(item_id)] = desc if '.xml' in filename: tree = analyze.analyze([filename])[0] elif '.hier' in filename: loaded = webdriver.load(filebase) items = loaded['items'] tree = analyze.analyze_items(items) print("=== ANALYZED ===") util.print_tree(tree) def removed_from_tree(itemid, tree): for nodeid in tree: if itemid in tree[nodeid]['raw']: return False return True print("=== CURRENT ===") for itemid in sorted(rets): for nodeid in tree: if itemid in tree[nodeid]['raw']: print("%3s %15s %s" % (itemid, rets[itemid], util.describe_node(tree[nodeid]))) #print("%3s %15s %s" % (itemid, rets[itemid], items[itemid])) if removed_from_tree(itemid, tree): print("MISSING FROM TREE!", itemid) print_tags(scr_name) def save_results(): with open(descname, 'w') as outf: for item_id in sorted(rets): outf.write("%s %s\n" % (item_id, rets[item_id])) def find_node(itemid, tree): for nodeid in tree: if itemid in tree[nodeid]['raw']: return tree[nodeid] return None history = [] def mark_item(scr_name, item_id, item_desc, items): if not item_valid(scr_name, item_desc): print("warn: illegal item %s" % item_desc) rets[item_id] = item_desc print("Marked as <%s> %s" % (item_desc, util.describe_node(find_node(item_id, tree), short=True).strip())) history.append([item_id, item_desc]) save_results() def unmark_item(item_id): del rets[item_id] print("deleted item %d" % item_id) save_results() cur_item = -1 while True: line = input("%s> " % filename) if line == '': break if line == '?': for item_id in sorted(rets): print("%20s %-100s" % (rets[item_id], items[item_id])) continue if line == ';': util.print_tree(tree) #for line in output_stack: # print(line) continue if line == '!': print_tags(scr_name) continue if line == 'q': sys.exit(0) mode = 'l' if ' ' in line: parts = line.split(' ') try: cur_item = int(parts[0]) item_desc = parts[1] item_idx = -1 except: mode = parts[0] if mode == 'r': rep = int(parts[1]) offset = int(parts[2]) if len(parts) > 3: num = int(parts[3]) else: num = 1 elif mode == 'd': del_idx = int(parts[1]) else: try: item_idx = int(line) except: item_desc = line item_idx = -1 if mode == 'r': cur_len = len(history) for j in range(num): for i in range(rep): old_entry = history[cur_len - rep + i] mark_item(scr_name, old_entry[0] + offset + j * offset, old_entry[1], items) elif mode == 'd': unmark_item(del_idx) else: if item_idx == -1: if cur_item != -1: mark_item(scr_name, cur_item, item_desc, items) else: print("You must choose an item first") else: if item_idx in items: cur_item = item_idx print("item %d is:" % item_idx) for line in ext_items[item_idx]: print(line) else: print("item does not exist") if viewproc is not None: viewproc.kill()