def run(args):
if args.input_file:
with open(args.input_file, "r") as input_file:
source = input_file.read()
else:
import sys
source = sys.stdin.read()
if args.assemble:
with open(args.output, "w") as binout:
write_binary(parse_instructions(source), binout)
else:
tree = parse(preprocess(source))
if args.print_ast:
print_tree(tree)
if not tree:
return
instructions = link(*translate(tree))
if args.print_assembly:
print_instructions(instructions)
with open(args.output, "w") as binout:
write_binary(instructions, binout)
def invoke(self):
for i in range(self.repetitions):
self.m_metrics.reset()
mcts.player_one_tree = {}
mcts.player_two_tree = {}
self.search("", iterations=self.iterations)
self.m_metrics.repeated_rewards.append(self.m_metrics.rewards)
self.m_metrics.repeated_exploitability_values.append(
self.m_metrics.exploitability_values)
util.print_tree(mcts.player_one_tree)
self.m_metrics.show_cumulative_reward()
self.m_metrics.show_cumulative_reward_slope()
self.m_metrics.show_exploitability()
persistance.save_deterministic_strategy(
mcts.player_one_tree,
"Smooth_Deterministic_" + str(self.iterations) + "_random.json")
persistance.save_stochastic_strategy(
mcts.player_one_tree,
"Smooth_Stochastic_" + str(self.iterations) + "_random.json")
persistance.save_deterministic_strategy(
mcts.player_one_tree,
"Smooth_Deterministic_" + str(self.iterations) + "_self-play.json")
persistance.save_stochastic_strategy(
mcts.player_one_tree,
"Smooth_Stochastic_" + str(self.iterations) + "_self-play.json")
def build(filename, path, if_print_ast=True):
build_path = os.path.join(path, 'build')
with open(os.path.join(path, filename + ".barter"), "r") as listing:
listing_text = listing.read()
ast = parse_program(listing_text)
if if_print_ast:
print_tree(ast)
res = create_source(ast)
if not os.path.exists(build_path):
os.makedirs(build_path)
with open(f'{build_path}/{filename}.cpp', 'w') as f:
f.write(res)
def classify_elements(self, curr_state, no_print):
guess_descs = {}
guess_items = {} # type: Dict[str, List[int]]
guess_score = {}
tree = curr_state.get('tree')
guess_scr = curr_state.get('screen')
imgdata = curr_state.get('screen_img')
treeinfo = analyze.collect_treeinfo(tree)
self.element_clas.set_page(imgdata)
points = {}
for itemid in tree:
(guess_element, score,
point) = self.element_clas.classify(guess_scr,
tree,
itemid,
imgdata,
treeinfo,
with_point=True)
points[itemid] = point
#elements.print_point(point)
if guess_element != 'NONE':
if tags.single(guess_element,
guess_scr) and guess_element in guess_items:
old_item = guess_items[guess_element][0]
if guess_score[old_item] < score:
guess_items[guess_element] = [itemid]
guess_descs[itemid] = guess_element
guess_score[itemid] = score
del guess_descs[old_item]
del guess_score[old_item]
else:
guess_descs[itemid] = guess_element
guess_score[itemid] = score
guess_items[guess_element] = (
guess_items.get(guess_element, []) + [itemid])
if not no_print:
util.print_tree(tree,
guess_descs,
scores=guess_score,
tagged_only=False,
use_log=True)
curr_state.set('guess_descs', guess_descs)
curr_state.set('guess_score', guess_score)
curr_state.set('guess_items', guess_items)
def dump_tree(self, filename):
tree = analyze.load_tree(filename)
app = os.path.basename(filename).split('_')[0]
if self.hide:
hidden.find_hidden_ocr(tree)
if not self.nofill:
hidden.mark_children_hidden_ocr(tree)
for nodeid in tree:
node = tree[nodeid]
if node['visible'] == 'hidden':
self.hidden_cnt += 1
self.hidden_app[app] = self.hidden_app.get(app, 0) + 1
self.total_cnt += 1
self.total_app[app] = self.total_app.get(app, 0) + 1
if (node['regs']
or node['tags']) and node['visible'] == 'hidden':
logger.error(
"ERROR!: INVISIBLE %s %d %s", filename, nodeid,
node['regs'][0] if node['regs'] else node['tags'])
if self.print_tree or self.show:
util.print_tree(tree, show_hidden=self.show)
def explore():
print("learning")
clas = classify.learn("../guis/", None)
element_clas = elements.learn("../guis/", None)
print("exploring")
dev = device.Device()
while True:
hier = sense.grab_full(dev)
if not hier['xml']:
logger.error("fail to grab hierarchy")
return
print(hier['act'])
guess_scr = clas.classify(hier['xml'], hier['act'], hier['scr'])
print("classify: %s" % (guess_scr))
items = analyze.parse_xml(hier['xml'])
tree = analyze.analyze_items(items)
imgdata = sense.load_image(hier['scr'])
guess_descs = {}
treeinfo = analyze.collect_treeinfo(tree)
for itemid in tree:
guess_element = element_clas.classify(guess_scr, tree, itemid,
imgdata, treeinfo)
if guess_element != 'NONE':
guess_descs[itemid] = guess_element
util.print_tree(tree, guess_descs)
input('enter')
if False:
itemid = random.choice(sorted(tree))
if 'click' in tree[itemid] and tree[itemid]['click']:
print("Clicking %s" % util.describe_node(tree[itemid]))
widget.click(dev, tree[itemid])
def run(args):
if args.input_file:
with open(args.input_file, 'r') as input_file:
source = input_file.read()
else:
import sys
source = sys.stdin.read()
if args.assemble:
with open(args.output, 'w') as binout:
write_binary(parse_instructions(source), binout)
else:
tree = parse(preprocess(source))
if args.print_ast:
print_tree(tree)
if not tree:
return
instructions = link(*translate(tree))
if args.print_assembly:
print_instructions(instructions)
with open(args.output, 'w') as binout:
write_binary(instructions, binout)
def observe(files):
viewproc = None
for filename in files:
filebase = os.path.splitext(filename)[0]
descname = filebase + '.desc.txt'
imgname = filebase + '.png'
if viewproc is not None:
try:
viewproc.kill()
except:
pass
viewproc = subprocess.Popen([config.picviewer_path, imgname])
scr_name = filename.split('/')[-1].split('.')[0].split('_')[-1]
if scr_name.startswith('cat'):
scr_name = 'cat'
if '.xml' in filename:
with open(filename) as f:
src = f.read()
root = ET.fromstring(src)
output_stack = []
items = {}
attrs = {}
parse(root, 0, 0, None, config.width, config.real_height,
output_stack, 0, items, attrs)
def get_depth(line):
line = line[3:]
return len(line) - len(line.lstrip())
max_item_id = max(items)
ext_items = {}
for i in range(max_item_id + 1):
if i in items:
my_depth = get_depth(items[i])
my_lines = [items[i]]
for j in range(i + 1, max_item_id + 1):
if j in items:
his_depth = get_depth(items[j])
if his_depth > my_depth:
my_lines.append(items[j])
else:
break
ext_items[i] = my_lines
for output in output_stack:
print(output)
print("==== IMPORTANT ====")
for item_id in sorted(attrs):
if attrs[item_id]['important']:
print(attrs[item_id]['output'])
rets = {}
if os.path.exists(descname):
with open(descname) as inf:
for line in inf.read().split('\n'):
if not line:
continue
(item_id, desc) = line.split(' ')
rets[int(item_id)] = desc
if '.xml' in filename:
tree = analyze.analyze([filename])[0]
elif '.hier' in filename:
loaded = webdriver.load(filebase)
items = loaded['items']
tree = analyze.analyze_items(items)
print("=== ANALYZED ===")
util.print_tree(tree)
def removed_from_tree(itemid, tree):
for nodeid in tree:
if itemid in tree[nodeid]['raw']:
return False
return True
print("=== CURRENT ===")
for itemid in sorted(rets):
for nodeid in tree:
if itemid in tree[nodeid]['raw']:
print("%3s %15s %s" % (itemid, rets[itemid],
util.describe_node(tree[nodeid])))
#print("%3s %15s %s" % (itemid, rets[itemid], items[itemid]))
if removed_from_tree(itemid, tree):
print("MISSING FROM TREE!", itemid)
print_tags(scr_name)
def save_results():
with open(descname, 'w') as outf:
for item_id in sorted(rets):
outf.write("%s %s\n" % (item_id, rets[item_id]))
def find_node(itemid, tree):
for nodeid in tree:
if itemid in tree[nodeid]['raw']:
return tree[nodeid]
return None
history = []
def mark_item(scr_name, item_id, item_desc, items):
if not item_valid(scr_name, item_desc):
print("warn: illegal item %s" % item_desc)
rets[item_id] = item_desc
print("Marked as <%s> %s" %
(item_desc,
util.describe_node(find_node(item_id, tree),
short=True).strip()))
history.append([item_id, item_desc])
save_results()
def unmark_item(item_id):
del rets[item_id]
print("deleted item %d" % item_id)
save_results()
cur_item = -1
while True:
line = input("%s> " % filename)
if line == '':
break
if line == '?':
for item_id in sorted(rets):
print("%20s %-100s" % (rets[item_id], items[item_id]))
continue
if line == ';':
util.print_tree(tree)
#for line in output_stack:
# print(line)
continue
if line == '!':
print_tags(scr_name)
continue
if line == 'q':
sys.exit(0)
mode = 'l'
if ' ' in line:
parts = line.split(' ')
try:
cur_item = int(parts[0])
item_desc = parts[1]
item_idx = -1
except:
mode = parts[0]
if mode == 'r':
rep = int(parts[1])
offset = int(parts[2])
if len(parts) > 3:
num = int(parts[3])
else:
num = 1
elif mode == 'd':
del_idx = int(parts[1])
else:
try:
item_idx = int(line)
except:
item_desc = line
item_idx = -1
if mode == 'r':
cur_len = len(history)
for j in range(num):
for i in range(rep):
old_entry = history[cur_len - rep + i]
mark_item(scr_name, old_entry[0] + offset + j * offset,
old_entry[1], items)
elif mode == 'd':
unmark_item(del_idx)
else:
if item_idx == -1:
if cur_item != -1:
mark_item(scr_name, cur_item, item_desc, items)
else:
print("You must choose an item first")
else:
if item_idx in items:
cur_item = item_idx
print("item %d is:" % item_idx)
for line in ext_items[item_idx]:
print(line)
else:
print("item does not exist")
if viewproc is not None:
viewproc.kill()
def built_tree(_str, verbose=False):
"""
params:
_str = input string
verbose: print stuff?
returns:
suffix tree: list where each node is a list of
[
children: list of nodes,
ancestor: node in tree,
range of incoming edge: tuple, (start, end),
s_ptr (list) or starting index of suffix if node is a terminal node
]
if children is [] -> node is a terminal node,
if ancestor is None -> node is the root node
"""
def get_head(node):
if node is None or node[2] is None:
# node[2] None means that there is no ancestors
return 0
else:
# Repeats with getting the ancestor of the current node + the length between the end - start of incoming edges
return get_head(node[1]) + (node[2][1] - node[2][0])
def fastscan(node, __str):
if __str == "":
# Returns the current node, and the start of the incoming edges
return node, node[2][0]
for child in node[0]:
c_str = _str[child[2][0]:child[2][1]]
# Characters in string from the start to the end index of the incoming edges in the built string
if __str[0] == c_str[0]:
#Matches first character
if len(c_str) <= len(__str):
# if the length is less than the length of the total string, it can be expanded
return fastscan(child, __str[len(c_str):])
else:
# Returns list of children nodes, and the whole current string
return slowscan(child[1], __str)
def slowscan(node, __str):
if verbose:
print("q: %s" % __str)
scan_node = node
position = scan_node[2][0]
if __str != "":
# If the string is not empty, not all children will probably match
for child in node[0]:
full_match = False
for k, i in enumerate(range(*child[2])):
# Go through each character in child
if k >= len(__str) or _str[i] != __str[k]:
# Either the string is longer than the one we are looking at, meaning it can't be full match
# Or if the prefix does not match with the string at the current location
full_match = False
break
else:
if full_match:
continue
else:
full_match = True
scan_node = child
if full_match:
if (len(__str) == k + 1):
# See if string is one longer than the range of edges coming to the curent child's string
return scan_node, scan_node[2][0]
else:
# Cut off the string at K + 1 and do it again on the child node
return slowscan(child, __str[k + 1:])
# partial match
if i != child[2][0]:
position = i
break
if verbose:
print("scan_node: %s, position: %i" %
(''.join([_str[j] for j in range(*scan_node[2])]), position))
return scan_node, position
def add_terminal(node, _range, i):
# Add node with no children, the current node as the ancestor, the range and the starting index of suffix
node[0].append([[], node, _range, i])
def add_inner(node_add, position):
ancestor = node_add[1]
# New node is inserted between the ancestor and the node to add
new_node = [[node_add], ancestor, (node_add[2][0], position), None]
# Adding all children to the ancestor as long as the child node's range is not the same as node_add
ancestor[0] = [i for i in ancestor[0] if i[2] != node_add[2]]
# Adding new node the ancestor
ancestor[0].append(new_node)
# So new range of node_add is its position to its original end point of incoming edges
node_add[2] = (position, node_add[2][1])
# Making the ancestor the new inner node
node_add[1] = new_node
return new_node
# build init tree
T = [[[[], None, (0, len(_str)), 0]], None, (0, 0), None]
T[0][0][1] = T
T[3] = T
head_old = T
head_new = None
tail = _str
# iterate
for i in range(1, len(_str)):
if verbose:
print("===============::============")
# Range of starting node
empty_head = head_old[2] == (0, 0)
# Ancestor of current node
u = None if empty_head else head_old[1]
# v is the string range, taken from the slice of the range of the ancestor node
v = "" if empty_head else _str[head_old[2][0]:head_old[2][1]]
w = None
# Check to make sure range is not (0,0) for example
if u is not None and (u[2][1] - u[2][0]) > 0:
# Use fastcan with the string stored at u and the slice of string v
w, position = fastscan(u[3], v)
else:
# Use fastscan with the head node and all but the beginning letter of the string
w, position = fastscan(T, v[1:])
if position != w[2][0]:
# If the start position from the scan does not equal the start position of the scanned node
head_new = add_inner(w, position)
# Add a node and split the range up, based on the position, continuing with the new inner node
w = head_new
else:
# If it does match, then can do a slowscan
head_tmp, position = slowscan(w, tail[1:] if empty_head else tail)
if position != head_tmp[2][0]:
# Same as above
head_new = add_inner(head_tmp, position)
else:
head_new = head_tmp
# Get the range of the head to the root, or basically the length, as each ancestor has the number of end - start of edges added
# But also add the how far into the string we are with + i
head_new_len = get_head(head_new) + i
head_old[3] = w
# Create new terminal node right under the current node, like adding the $ in the string
add_terminal(head_new, (head_new_len, len(_str)), i)
if verbose:
print_tree(T, _str)
# Take the original string and cut off the characters up to now, since we have gone through them
tail = _str[head_new_len:]
head_old = head_new
return T
def handle_console_cmd(cons, cmd, envs):
threading.current_thread().name = 'MainThread'
dev = envs['dev']
ob = envs['ob']
st = envs['state']
tlib = envs['tlib']
env = envs['env']
if 'app' in envs:
app = envs['app']
else:
app = 'cui'
def save_stat():
tlib.save_stat(os.path.join("../stat/", "%s.txt" % app))
if cmd == 'q':
save_stat()
dev.finish()
sys.exit(0)
elif cmd == 'sense':
scr = ob.grab_state(dev, no_verify=True, no_print=True)
if scr.get('guess_descs') is None:
util.print_tree(scr.get('tree'))
else:
util.print_tree(scr.get('tree'), scr.get('guess_descs'),
scr.get('guess_score'))
st.merge(scr)
return
elif cmd == 'tags':
config.show_guess_tags = True
scr = ob.grab_state(dev, no_verify=True, no_print=True)
util.print_tree(scr.get('tree'), scr.get('guess_descs'), scr.get('guess_score'))
st.merge(scr)
config.show_guess_tags = False
return
elif cmd == 'tree':
scr = ob.grab_state(dev, no_img=True)
util.print_tree(scr.get('tree'))
st.merge(scr)
return
elif cmd.startswith('app '):
app = cmd.split(' ')[1]
load_app(app, ob, tlib, envs)
return
elif cmd == 'load':
tlib = testlib.collect_pieces("../tlib/")
envs['tlib'] = tlib
load_app(app, ob, tlib, envs)
return
elif cmd == 'reload':
value.init_params("../etc/", app)
try:
if app:
os.remove("../model/screen_%s" % app)
os.remove("../model/element_%s" % app)
else:
os.remove("../model/screen")
os.remove("../model/element")
except:
pass
ob.load("../model/", "../guis/", "../guis-extra/", config.extra_screens,
config.extra_element_scrs)
return
elif cmd == 'tlib':
tlib = testlib.collect_pieces("../tlib/")
envs['tlib'] = tlib
return
elif cmd == 'test':
tlib = testlib.collect_pieces("../tlib/")
tlib.set_app(app)
tlib.add_test(microtest.init_test(appdb.get_app(app)))
envs['tlib'] = tlib
config.show_guess_tags = True
scr = ob.grab_state(dev)
config.show_guess_tags = False
st.merge(scr)
begin_state = st.to_essential(tlib.essential_props())
tests = tlib.usable_tests(st)
tests.sort(key=lambda test: test.prio)
for i in range(len(tests)):
testinfo = tlib.get_testinfo(tests[i])
print("%2d. %s: %s [%d %d]" % (i, tests[i].feature_name, tests[i].name,
testinfo.succs, testinfo.fails))
if len(tests) == 0:
print("no test available!")
return
if len(tests) == 1:
tid = 0
else:
tid = input("which one? ")
try:
tid = int(tid)
except:
return
test = tests[tid]
ret = test.attempt(dev, ob, st, tlib, environ.empty)
util.print_tree(st.get('tree'), st.get('guess_descs'))
end_state = st.to_essential(tlib.essential_props())
if ret:
print("test SUCC")
tlib.clear_stat(test)
tlib.mark_succ(test, begin_state, end_state)
else:
print("test FAIL")
tlib.mark_fail(test, begin_state)
save_stat()
return
elif cmd == 'save':
save_stat()
return
elif cmd == 'stat':
tlib.print_stat()
return
elif cmd == 'items':
scr = ob.grab_state(dev, no_img=True)
util.printitems(scr.get('items'))
return
elif cmd.startswith('item'):
itemid = int(cmd.split(' ')[1])
scr = ob.grab_state(dev, no_img=True)
items = scr.get('items')
if itemid in items:
print(util.describe(items[itemid]))
else:
print("no such item: %d" % itemid)
return
elif cmd == 'debug':
logging.basicConfig(level=logging.DEBUG)
return
elif cmd == 'idle':
dev.wait_idle()
return
elif cmd == 'dialog':
scr = ob.grab_state(dev)
ret = tlib.handle_dialog(scr, dev, ob)
if ret:
print("dialog handled")
else:
print("dialog not handled")
return
elif cmd.startswith('find '):
tag = cmd.split(' ', 1)[1]
con = concept.parse(tag)
if con is None:
print("invalid locator")
return
scr = ob.grab_state(dev)
widgets = con.locate(scr, ob, environ.Environment())
if widgets == []:
print("can't find")
else:
for widget in widgets:
print("FOUND: %s" % widget)
print(" content:", widget.content())
return
elif cmd == 'perf':
perfmon.print_stat()
return
elif cmd == 'clear':
init = tlib.find_test('meta', 'start app')
st.reset()
init.attempt(dev, ob, st, tlib, None)
ob.update_state(dev, st)
tlib.mark_succ(init, state.init_state, st)
return
elif cmd.startswith('set'):
parts = cmd.split(' ', 2)
if len(parts) == 2:
key = parts[1]
val = "1"
else:
(key, val) = parts[1:]
st.set(key, val)
return
elif cmd.startswith('del'):
parts = cmd.split(' ', 1)
key = parts[1]
st.remove(key)
return
elif cmd == 'dump':
filename = util.choose_filename("../cap/", "page%d")
logger.info("dumping to page %s", filename)
dev.dump(filename)
#sense.dump_page(dev, "../cap/")
return
elif cmd == 'list':
scr = ob.grab_state(dev, no_img=True)
tree = scr.get('tree')
treeinfo = analyze.collect_treeinfo(tree)
for root in sorted(treeinfo['listlike']):
print("ROOT:", util.describe_node(tree[root]))
for item in sorted(treeinfo['itemlike']):
if listinfo.get_lca(tree, [root, item]) == root:
print(" NODE:", util.describe_node(tree[item]))
for field in sorted(treeinfo['dupid']):
if listinfo.get_lca(tree, [item, field]) == item:
print(" FIELD:", util.describe_node(tree[field]))
return
elif cmd == 'webon':
config.GRAB_WEBVIEW = True
return
elif cmd == 'weboff':
config.GRAB_WEBVIEW = False
return
elif cmd.startswith('ob '):
prop = cmd.split(' ', 1)[1]
wd = watchdog.Watchdog(100000)
smgr = statemgr.StateMgr(tlib, None, dev, ob, wd)
ret = smgr.observe_prop(prop, st)
if ret:
print("prop %s = %s" % (prop, st.get(prop, '')))
else:
print("observe error")
return
elif cmd.startswith('clean '):
parts = cmd.split(' ', 2)
if len(parts) == 2:
prop = parts[1]
val = "1"
else:
(prop, val) = parts[1:]
wd = watchdog.Watchdog(100000)
smgr = statemgr.StateMgr(tlib, None, dev, ob, wd)
ret = smgr.cleanup_prop(prop, val, st)
if ret:
print("prop %s cleaned" % prop)
else:
print("clean error")
return
elif cmd.startswith('oc '):
prop = cmd.split(' ', 1)[1]
wd = watchdog.Watchdog(100000)
smgr = statemgr.StateMgr(tlib, None, dev, ob, wd)
ret = smgr.observe_and_clean(prop, st)
if ret:
print("prop %s is clean now" % prop)
else:
print("observe/clean error")
return
elif cmd == 'dbg':
print(tlib)
return
elif cmd.startswith('skip '):
name = cmd.split(' ', 1)[1]
add_skip(app, name)
print("skipping %s" % name)
return
elif cmd == 'skip':
skips = load_skip(app)
for skip in skips:
print("now skipping %s" % skip)
return
elif cmd.startswith('noskip '):
name = cmd.split(' ', 1)[1]
del_skip(app, name)
print("not skipping %s" % name)
return
elif cmd == 'src':
if st.get('xml') is not None:
print(st.get('xml'))
if st.get('src') is not None:
print(st.get('src'))
return
elif cmd == 'install':
apputils.install(dev, app)
return
elif cmd == 'uninstall':
apputils.uninstall(dev, app)
return
op = operation.parse_line(cmd)
if op is None:
print("unknown op")
return
ret = op.do(dev, ob, st, env, tlib)
if not ret:
print("op failed")
verbose = len(sys.argv) > 2 and sys.argv[2] == '--verbose'
with open(filename, "r") as datafile:
data = datafile.read().replace('\n', ' ')
# Build tree from data file
if verbose:
print(data)
start = time.time()
tree = built_tree(data, False)
if False and verbose:
end = time.time()
sys.stdout.write("\n")
print_tree(tree, data)
sys.stdout.write("\n")
print("Construction Time: %f4" % (end - start))
print("Size per character (On average): %i" % (sys.getsizeof(data)/len(data)))
print("Size of suffix tree (on average): %i" % (sys.getsizeof(tree)/len(tree)))
start = time.time()
result = find_tandem_repeats(tree, data)
if verbose:
out_str = ""
if verbose:
end = time.time()
print("Search Time: %f4" % (end - start))
sys.stdout.write("\n")
def built_tree(_str, verbose=False):
"""
params:
_str = input string
verbose: print stuff?
returns:
suffix tree: list where each node is a list of
[
children: list of nodes,
ancestor: node in tree,
range of incoming edge: tuple, (start, end),
s_ptr (list) or starting index of suffix if node is a terminal node
]
if children is [] -> node is a terminal node,
if ancestor is None -> node is the root node
"""
def get_head(node):
if node is None or node[2] is None:
return 0
else:
return get_head(node[1]) + (node[2][1] - node[2][0])
def fastscan(node, __str):
if __str == "":
return node, node[2][0]
for child in node[0]:
c_str = _str[child[2][0]:child[2][1]]
if __str[0] == c_str[0]:
if len(c_str) <= len(__str):
return fastscan(child, __str[len(c_str):])
else:
return slowscan(child[1], __str)
def slowscan(node, __str):
if verbose:
print("q: %s" % __str)
scan_node = node
position = scan_node[2][0]
if __str != "":
for child in node[0]:
full_match = False
for k, i in enumerate(range(*child[2])):
if k >= len(__str) or _str[i] != __str[k]:
full_match = False
break
else:
if full_match:
continue
else:
full_match = True
scan_node = child
if full_match:
if (len(__str) == k + 1):
return scan_node, scan_node[2][0]
else:
return slowscan(child, __str[k + 1:])
# partial match
if i != child[2][0]:
position = i
break
if verbose:
print("scan_node: %s, position: %i" %
(''.join([_str[j] for j in range(*scan_node[2])]), position))
return scan_node, position
def add_terminal(node, _range, i):
node[0].append([[], node, _range, i])
def add_inner(node_add, position):
ancestor = node_add[1]
new_node = [[node_add], ancestor, (node_add[2][0], position), None]
ancestor[0] = [i for i in ancestor[0] if i[2] != node_add[2]]
ancestor[0].append(new_node)
node_add[2] = (position, node_add[2][1])
node_add[1] = new_node
return new_node
# build init tree
T = [[[[], None, (0, len(_str)), 0]], None, (0, 0), None]
T[0][0][1] = T
T[3] = T
head_old = T
head_new = None
tail = _str
# iterate
for i in range(1, len(_str)):
if verbose:
print("===============::============")
empty_head = head_old[2] == (0, 0)
u = None if empty_head else head_old[1]
v = "" if empty_head else _str[head_old[2][0]:head_old[2][1]]
w = None
if u is not None and (u[2][1] - u[2][0]) > 0:
w, position = fastscan(u[3], v)
else:
w, position = fastscan(T, v[1:])
if position != w[2][0]:
head_new = add_inner(w, position)
w = head_new
else:
head_tmp, position = slowscan(w, tail[1:] if empty_head else tail)
if position != head_tmp[2][0]:
head_new = add_inner(head_tmp, position)
else:
head_new = head_tmp
head_new_len = get_head(head_new) + i
head_old[3] = w
add_terminal(head_new, (head_new_len, len(_str)), i)
if verbose:
print_tree(T, _str)
tail = _str[head_new_len:]
head_old = head_new
return T
def load(self, prev=False):
if self.file_idx == len(self.filenames):
Gtk.main_quit()
return
if prev:
self.file_idx -= 2
filename = self.filenames[self.file_idx]
(self.app, self.scr) = util.get_aux_info(filename)
if self.app not in self.memory:
self.memory[self.app] = {}
self.set_title(filename)
self.file_idx += 1
print("Loading %s" % filename)
self.pngfile = os.path.splitext(filename)[0] + '.png'
self.descname = os.path.splitext(filename)[0] + '.%s.txt' % self.kind
starttime = time.time()
self.tree = analyze.load_tree(filename)
hidden.find_hidden_ocr(self.tree)
hidden.mark_children_hidden_ocr(self.tree)
util.print_tree(self.tree, show_hidden=self.show_hidden)
if self.ml:
self.get_ml_rets()
else:
self.load_desc()
endtime = time.time()
print("Load time: %.3fs" % (endtime - starttime))
self.focus_id = -1
self.colors = {}
self.ptx = self.pty = 0
self.img = cairo.ImageSurface.create_from_png(self.pngfile)
print('Image:', self.img.get_width(), self.img.get_height())
root_item_id = min(self.tree)
root_node = self.tree[root_item_id]
print('Root node:', root_node['width'], root_node['height'])
self.scale = 1.0 * self.img.get_width() / config.width
#self.scale = analyze.find_closest(self.scale, analyze.SCALE_RATIOS)
print('Scale:', '%.3f' % self.scale, '->', '%.3f' % self.scale)
self.resize(self.img.get_width(), self.img.get_height())
self.mark_depth(self.tree)
for item_id in self.tree:
color_r = random.random() / 2
color_g = random.random() / 2
color_b = random.random() / 2
self.colors[item_id] = (color_r, color_g, color_b)
imgocr = Image.open(self.pngfile)
self.imgwidth = imgocr.width
self.imgheight = imgocr.height
#imgocr2 = imgocr.convert("RGB").resize(
# (imgocr.width * OCR_RATIO, imgocr.height * OCR_RATIO))
self.tesapi.SetImage(imgocr)
self.tesapi.SetSourceResolution(config.ocr_resolution)
self.dump_memory()
def guess(filename, lasttag=None):
basename = filename.replace('.xml', '')
(items, descs) = analyze.load_case(filename)
# if descs:
# return
appname = basename.split('/')[-1].split('.')[0].split('_')[0]
scrname = basename.split('/')[-1].split('.')[0].split('_')[-1]
imgfile = filename.replace('.xml', '.png')
if not appname in tag_db:
tag_db[appname] = {}
if appname in tag_db:
for tag in tag_db[appname]:
xid = tag_db[appname][tag]
for itemid in items:
if items[itemid]['id'] == xid:
descs[itemid] = tag
print("%s Current:" % basename)
util.printdesc(items, descs)
# analyze.printitems(items)
tree = analyze.analyze([filename])[0]
util.print_tree(tree, descs)
viewproc = subprocess.Popen([config.picviewer_path, basename + ".png"])
prompt = ''
while True:
print(tags.tag['universal'])
if scrname in tags.tag:
print(tags.tag[scrname])
line = input(prompt + "> ")
parts = line.split(' ')
cmd = parts[0]
if cmd == '':
util.save_desc(basename, descs)
break
elif cmd[0] == 'q':
viewproc.kill()
sys.exit(0)
elif cmd[0] == 'l':
listid = int(parts[1])
listtype = parts[2]
guess_list(listid, listtype, items, descs)
util.printdesc(items, descs)
elif cmd[0] == 'a':
auto_guess(filename, items, descs)
util.print_tree(tree, descs)
elif cmd[0] == 'y':
util.save_desc(basename, descs)
break
elif cmd[0] >= '0' and cmd[0] <= '9':
if len(parts) > 1:
lasttag = parts[1]
util.mark_item(parts, items, descs, scrname, tag_db[appname])
util.print_tree(tree, descs)
else:
if lasttag is None:
print(util.describe(items[int(cmd)]))
else:
util.mark_item([cmd, lasttag], items, descs, scrname,
tag_db[appname])
util.print_tree(tree, descs)
elif cmd[0] == 'd':
if cmd == 'dr':
recursive = True
else:
recursive = False
util.remove_mark(parts[1:], items, descs, recursive)
util.print_tree(tree, descs)
elif cmd[0] == ';':
util.print_tree(tree, descs)
elif cmd[0] == '!':
util.printitems(items)
elif cmd[0] == 'c':
element_clas = elements.getmodel("../model/", "../guis/", None)
imgdata = skimage.io.imread(imgfile, as_grey=True)
imgdata = skimage.transform.resize(imgdata,
(config.height, config.width))
treeinfo = analyze.collect_treeinfo(tree)
for itemid in tree:
(guess_element,
score) = element_clas.classify(scrname, tree, itemid, imgdata,
treeinfo)
if guess_element != 'NONE':
descs[itemid] = guess_element
util.print_tree(tree, descs)
elif cmd[0] == '?':
print(util.describe(items[int(parts[1])]))
else:
print("Unknown command")
viewproc.kill()
def dump_tree():
dump.capture_layout("temp", ".", None)
tree = analyze.analyze(["temp.xml"])[0]
util.print_tree(tree)
return tree
def grab_state_once(self, dev, no_verify, no_print, no_img,
known_scr) -> state.State:
if not no_img:
logger.info("observing%s", " -img" if no_img else "")
if dev.use_web_grab:
grab_ret = self.grab_web(dev, no_img)
else:
grab_ret = self.grab_adb(dev, no_img)
if len(grab_ret) == 2:
reuse_state = grab_ret[0]
if (reuse_state is None
or reuse_state.get('guess_descs') is not None
or not config.show_guess_tags
or reuse_state.get('screen') == 'NONE'):
return reuse_state
else:
if reuse_state is None:
return None
tree = reuse_state.get('tree')
items = reuse_state.get('items')
curr_state = reuse_state
actname = reuse_state.get('act')
screen = copy.deepcopy(grab_ret[1])
else:
(screen, actname, tree, items, curr_state) = grab_ret
self.record_tree(tree, items, actname)
curr_state.set('items', items)
curr_state.set('tree', tree)
curr_app = curr_state.get('app')
if 'scr' in screen:
imgdata = sense.load_image(screen['scr'])
curr_state.set('screen_img', imgdata)
curr_state.set('screenshot', screen['scr'])
guess_scr = None
for (screen_name, obs) in value.get_screenobs().items():
for ob in obs:
if ob.check(curr_state):
if guess_scr is not None:
if guess_scr != screen_name:
if (guess_scr.startswith('sys_')
and screen_name.startswith('app_')):
guess_scr = screen_name
else:
logger.warning(
"multiple sys screen match: %s, %s",
guess_scr, screen_name)
else:
guess_scr = screen_name
if guess_scr is not None:
if guess_scr not in self.clas_stat:
self.clas_stat[guess_scr] = [0, 0]
scrstat = self.clas_stat[guess_scr]
logger.info("screen overrided to %s %d+ %d-", screen_name,
scrstat[0], scrstat[1])
curr_state.set('screen', guess_scr)
if (not no_img and self.err_cap_path is not None
and not is_sys_screen(guess_scr)
and scrstat[1] < config.ERR_CAP_LIMIT):
clas_scr = self.clas.classify(actname, screen['scr'], tree)
if guess_scr != clas_scr:
# yes, it's incorrect
logger.info("screen misclassified as %s", clas_scr)
if config.dump_err_page:
sense.dump_page(dev, self.err_cap_path,
self.appname, guess_scr,
self.guispath)
scrstat[1] += 1
else:
scrstat[0] += 1
break
if (self.app is not None and curr_app != self.app
and 'browser' not in curr_app
and 'com.google.android' not in curr_app):
logger.info("switched to %s from %s", curr_app, self.app)
if LAUNCHER_APP == curr_app:
logger.info("at launcher")
curr_state.set('exited', True)
curr_state.set('screen', 'init')
if curr_state.get('screen', '') == '':
curr_state.set('screen', 'other')
while self.tlib.handle_sys_screen(curr_state, dev, self):
pass
self.last_state = copy.deepcopy(curr_state)
self.last_state_screen = sense.duplicate_screen(screen)
return curr_state
if guess_scr is not None:
for count in range(config.HANDLE_SYS_LIMIT):
if not no_print and is_sys_screen(guess_scr):
util.print_tree(tree, tagged_only=False, use_log=True)
if not self.tlib.handle_sys_screen(curr_state, dev, self):
break
guess_scr = curr_state.get('screen')
tree = curr_state.get('tree')
if self.clas is None:
self.last_state = copy.deepcopy(curr_state)
return curr_state
if no_img:
# no img: no screen/element classify
self.last_state = copy.deepcopy(curr_state)
return curr_state
if guess_scr is None:
if known_scr is not None:
guess_scr = known_scr
else:
guess_scr = self.clas.classify(actname, screen['scr'], tree)
curr_state.set('screen', guess_scr)
if guess_scr != 'NONE':
if not self.verify_screen_tag(curr_state,
guess_scr) and not no_verify:
# failed the tag verify!
logger.info("the screen was considered %s, but it is not",
guess_scr)
guess_scr = 'NONE'
curr_state.set('screen', guess_scr)
if known_scr is None:
logger.info("I think this is %s", guess_scr)
else:
logger.info("Using old knowledge %s", known_scr)
if guess_scr != 'NONE' and config.show_guess_tags:
# I know! (I think)
self.classify_elements(curr_state, no_print)
else:
curr_state.set('guess_descs', None)
if not no_print:
util.print_tree(tree, tagged_only=False, use_log=True)
self.last_state = copy.deepcopy(curr_state)
self.last_state_screen = sense.duplicate_screen(screen)
return curr_state
def detect(self, filename):
img = cv2.imread(filename, cv2.IMREAD_COLOR)
if img is None:
return '{"error_message": "Can\'t open file\'' + filename + '\'"}'
self.step.log(img)
prefer_height = 1000.0
raw_height = img.shape[0]
raw_width = img.shape[1]
size_factor = prefer_height / raw_height
img = cv2.resize(img, (int(raw_width * size_factor), int(prefer_height)))
self.step.log(img)
after_img = img.copy()
original_img = img.copy()
img = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)
self.step.log(img)
img = cv2.GaussianBlur(img, (11, 11), 0)
self.step.log(img)
img = cv2.adaptiveThreshold(img, 255, cv2.ADAPTIVE_THRESH_MEAN_C, cv2.THRESH_BINARY, 51, 5)
self.step.log(img)
# src, result_intensity, method, type, block, area, weight_sum
# cv2.ADAPTIVE_THRESH_GAUSSIAN_C
# cv2.THRESH_OTSU cv2.THRESH_BINARY
img = cv2.bitwise_not(img)
# if self.gui:
# self.gui.show_image(0, img, 900)
height, width = img.shape
before_img = None
final_img = np.zeros((height, width, 3), np.uint8)
img = cv2.Canny(img, 128, 200) # min max
self.step.log(img)
# after_img = cv2.cvtColor(img, cv2.COLOR_GRAY2RGB)
# if self.gui:
# self.gui.show_image(1, img, 900)
# cv2.cv.CV_RETR_EXTERNAL cv2.cv.CV_RETR_LIST
contours, hierarchy = cv2.findContours(img, cv2.cv.CV_RETR_CCOMP, cv2.CHAIN_APPROX_SIMPLE)
elements = []
root_element = Element(0, [Point(0, 0), Point(0, height), Point(width, height), Point(width, 0)], "Root")
root_element.description = Description.Root
root_area = root_element.polygon.area
root_width = root_element.width
# print root_width
size_threshold = (0.05 / 100 * root_area)
### SHOW_WITHOUT_PREPROCESSING
img = cv2.cvtColor(img, cv2.COLOR_GRAY2BGR)
if self.step.active:
last_number = 0
for number, cnt in enumerate(contours):
last_number = number
if hierarchy[0, number, 3] == -1:
approx = cv2.approxPolyDP(cnt, 0.01 * cv2.arcLength(cnt, True), True)
vertices = util.get_vertices(approx)
self.step.draw_vertices(img, vertices, (0, 0, 255), str(number))
self.step.draw_vertices(original_img, vertices, (0, 0, 255), str(number))
self.step.log(img)
self.step.log(original_img)
###
last_number = 0
for number, cnt in enumerate(contours):
last_number = number
if hierarchy[0, number, 3] == -1:
approx = cv2.approxPolyDP(cnt, 0.01 * cv2.arcLength(cnt, True), True)
vertices = util.get_vertices(approx)
vertex_count = len(vertices)
# if self.gui:
# self.gui.raw_draw(before_img, vertices, util.rand_color(), str(number))
before_img = after_img.copy()
self.step.draw_vertices(before_img, util.get_bounding_vertices(vertices), (0, 255, 255), "")
self.step.log_vertices(before_img, vertices, (0, 0, 255), str(number))
vertices = util.reduce_vertex_by_length(vertices, 0.01 * root_width)
vertices = util.reduce_vertex_by_angle(vertices, 160)
vertices = util.reduce_vertex_by_average_length(vertices, 0.2)
vertices = util.reduce_vertex_by_angle(vertices, 145)
vertices = util.reduce_vertex_by_average_length(vertices, 0.25)
vertices = util.reduce_vertex_by_angle(vertices, 130)
vertices = util.reduce_vertex_by_average_length(vertices, 0.1)
before_img = after_img.copy()
self.step.draw_vertices(before_img, util.get_bounding_vertices(vertices), (0, 255, 255), "")
self.step.log_vertices(before_img, vertices, (0, 0, 255), str(number))
# vertices = util.reduce_vertex_by_average_length(vertices, 0.3)
vertex_count = len(vertices)
# if self.gui:
# self.gui.raw_draw(after_img, vertices, util.rand_color(), str(number))
self.step.log_vertices(after_img, vertices, (255, 0, 0), str(number))
self.step.draw_vertices(final_img, vertices, (0, 255, 0), str(number))
if (vertex_count == 3):
e = TriangleElement(number, vertices, "Tri#" + str(number))
if e.polygon.area > size_threshold:
elements.append(e)
continue
elif (vertex_count == 4):
e = QuadrilateralElement(number, vertices, "Quad#" + str(number))
if e.polygon.area > size_threshold:
elements.append(e)
continue
else:
pass
self.step.log(final_img)
# if self.gui:
# self.gui.show_image(2, before_img, 900)
# if self.gui:
# self.gui.show_image(3, after_img, 900)
elements = util.remove_resembling_element(elements, 0.5)
elements.append(root_element)
# START HANDLING ELEMENTS AS TREE
util.construct_tree_by_within(elements) # use root_element from now on
self.append_text_elements(filename, root_element, last_number)
self.destroy_all_children_of_triangle(root_element)
self.detect_video_player(root_element)
self.detect_image_placeholder(root_element)
self.detect_panel(root_element)
self.interpret_leaf_rectangle(root_element)
# if self.gui:
# self.gui.draw_tree(newimg, root_element)
util.print_tree(root_element)
# self.step.tree_log(root_element)
# if self.gui:
# self.gui.show_image(4, newimg, 900)
util.assign_depth(root_element)
json_result = "{"
json_result += '"error_message": null,'
json_result += '"json_elements": {'
json_result += '"width":' + str(width) + ','
json_result += '"height":' + str(height) + ','
json_result += '"elements":['
json_result += self.traverse_as_json(root_element)
json_result += "]"
json_result += "}"
json_result += "}"
return json_result
'url': self.driver.current_url,
'scr': screenshot,
}
return screen
if __name__ == '__main__':
logging.basicConfig(level=logging.DEBUG)
import analyze
import sys
device = WebDevice()
try:
device.connect(sys.argv[1])
device.goto("http://m.hm.com/us/product/69060?article=69060-B")
#https://www.amazon.com")
start_time = time.time()
items = device.capture()
end_time = time.time()
print('used:', end_time - start_time)
device.dump("/tmp/testdump")
finally:
device.close()
util.printitems(items)
tree = analyze.analyze_items(items)
util.print_tree(tree)
loaded = load("/tmp/testdump")
def analyze_items(self, items, descs, regs, print_rets, print_error,
history):
self.fix_size_to_scr(items)
self.mark_important(items)
parents = [0]
newtree = {}
for itemid in preorder(items, 0):
item = items[itemid]
if item['important']:
nodeinfo = {}
par = -1
while len(parents) > 0 and (
item['depth'] <= items[parents[-1]]['depth']
or items[item['parent']]['depth'] <
items[parents[-1]]['depth']):
parents.pop()
if len(parents) > 0:
par = parents[-1]
if print_rets:
print('%3d' % itemid, self.get_prefix(items, itemid),
util.describe(item), item['parent'], par)
# orig = item
nodeinfo['parent'] = par
nodeinfo['class'] = item['class']
nodeinfo['text'] = item['text']
nodeinfo['desc'] = item['desc']
nodeinfo['id'] = item['id']
nodeinfo['raw'] = [itemid]
nodeinfo['width'] = item['width']
nodeinfo['height'] = item['height']
nodeinfo['x'] = item['x']
nodeinfo['y'] = item['y']
nodeinfo['origw'] = nodeinfo['width']
nodeinfo['origh'] = nodeinfo['height']
nodeinfo['origx'] = nodeinfo['x']
nodeinfo['origy'] = nodeinfo['y']
nodeinfo['click'] = item['click']
nodeinfo['scroll'] = item['scroll']
nodeinfo['password'] = item['password']
nodeinfo['focused'] = item['focused']
nodeinfo['checkable'] = item['checkable']
nodeinfo['childid'] = 0 # placemarker
nodeinfo['webview'] = item['webview']
nodeinfo['origitem'] = item
if itemid in descs:
nodeinfo['tags'] = [descs[itemid]]
else:
nodeinfo['tags'] = []
if itemid in regs:
nodeinfo['regs'] = [regs[itemid]]
else:
nodeinfo['regs'] = []
newtree[itemid] = nodeinfo
parents.append(itemid)
self.incstat("before", len(newtree))
if print_rets:
util.print_tree(newtree)
while self.process(newtree, items, history):
if print_rets:
util.print_tree(newtree)
if print_error:
for itemid in descs:
found = False
for nodeid in newtree:
if itemid in newtree[nodeid]['raw']:
found = True
break
if not found:
logger.error("REMOVED: %d %s %s", itemid,
descs[itemid],
util.describe(items[itemid]))
pass
while self.postprocess(newtree, items, history):
pass
self.incstat("after", len(newtree))
collect_children(newtree)
return newtree
def analyze(files,
print_rets=False,
show_progress=False,
print_items=False,
print_error=False,
show_ocr=False,
show_stat=False,
use_ocr=False):
ret = []
if show_progress:
progress = progressbar.ProgressBar()
items = progress(files)
else:
items = files
analyzer = Analyzer()
for filename in items:
filebase = os.path.splitext(filename)[0]
logger.debug("analyzing %s" % filename)
(items, descs, regs) = load_case(filename)
if print_items:
util.printitems(items)
start_time = time.time()
newtree = analyzer.analyze_items(items, descs, regs, print_items,
print_error, [])
ret.append(newtree)
if print_rets:
util.print_tree(newtree)
logger.info("Time used: %.3fs", time.time() - start_time)
if use_ocr:
hidden.add_ocrinfo(newtree, filebase + '.png')
hidden.find_hidden_ocr(newtree)
util.print_tree(newtree)
if print_rets:
dlg = dialog.detect_dialog(newtree)
if dlg[0]:
logger.info("I think this is dialog")
for btnid in dlg[1]:
logger.info("btn: %s", util.describe_node(newtree[btnid]))
logger.info(
"is: %s",
dialog.detect_dialog_button(newtree, btnid, dlg[1]))
logger.info("decide to click: %s",
dialog.decide_dialog_action(newtree))
if print_error:
for itemid in descs:
found = False
for nodeid in newtree:
if itemid in newtree[nodeid]['raw']:
found = True
break
if not found:
logger.error("REMOVED: %s %d %s %s",
os.path.basename(filename), itemid,
descs[itemid], util.describe(items[itemid]))
if show_stat:
analyzer.show_stat()
return ret
