예제 #1
0
def load_points(files,
                show_progress=False,
                no_appname=None,
                extra=[],
                parallel=True,
                nopt=False):
    points = []
    labels = []
    apps = []
    scrs = []
    rets = util.parallel_work(functools.partial(load_point, no_appname, nopt),
                              files,
                              parallel,
                              show_progress=show_progress)
    for (xpoints, xlabels, xapps, xscrs) in rets:
        for i in range(len(xpoints)):
            xpoints[i]['extra'] = False
            xscrs[i] = 'region'
            if xlabels[i] != 'NONE' and not tags.valid(xscrs[i], xlabels[i]):
                xlabels[i] = 'NONE'

            points.append(xpoints[i])
            labels.append(xlabels[i])
            apps.append(xapps[i])
            scrs.append(xscrs[i])

    rets = util.parallel_work(functools.partial(load_point, no_appname, nopt),
                              extra,
                              parallel,
                              show_progress=show_progress)
    for (xpoints, xlabels, xapps, xscrs) in rets:
        for i in range(len(xpoints)):
            xpoints[i]['extra'] = True
            xscrs[i] = 'region'
            if xlabels[i] != 'NONE' and not tags.valid(xscrs[i], xlabels[i]):
                xlabels[i] = 'NONE'

            if xlabels[i] != 'NONE':
                points.append(xpoints[i])
                labels.append(xlabels[i])
                apps.append(xapps[i])
                scrs.append(xscrs[i])

    return (points, labels, apps, scrs)
예제 #2
0
def load_points(files, show_progress=True, no_appname=None, extra=[]):
    #    files = sklearn.utils.shuffle(files, random_state=0)
    #    rets = analyze.analyze(files, show_progress=show_progress)
    #    for tree in rets:
    #        analyze.print_tree(tree)

    points = []
    labels = []
    apps = []
    scrs = []
    #cnt = {}
    pool = multiprocessing.Pool(processes=config.threads)
    rets = pool.map(functools.partial(load_point, no_appname), files)
    #for i in progress(range(len(files))):
    for (xpoints, xlabels, xapps, xscrs) in rets:
        for i in range(len(xpoints)):
            xpoints[i]['extra'] = False
            if try_all:
                xscrs[i] = 'ALL'
            if xlabels[i] != 'NONE' and not tags.valid(xscrs[i], xlabels[i]):
                xlabels[i] = 'NONE'
        points += xpoints
        labels += xlabels
        apps += xapps
        scrs += xscrs
    rets = pool.map(functools.partial(load_point, no_appname), extra)
    #for i in progress(range(len(files))):
    for (xpoints, xlabels, xapps, xscrs) in rets:
        for i in range(len(xpoints)):
            xpoints[i]['extra'] = True
            if xlabels[i] != 'NONE' and not tags.valid(xscrs[i], xlabels[i]):
                xlabels[i] = 'NONE'

            if xlabels[i] != 'NONE':
                points.append(xpoints[i])
                labels.append(xlabels[i])
                apps.append(xapps[i])
                scrs.append(xscrs[i])
    pool.close()

    return (points, labels, apps, scrs)
예제 #3
0
def evaluate_app(parallel, guispath, extrapath, my_points):
    good_cnt = total_cnt = inside_cnt = 0
    app = my_points[0]['app']
    logger.info("app %s, learning...", app)
    start_time = time.time()
    clas = learn(guispath, app, extrapath, parallel=parallel)
    logger.info("%s: Learned in %.3fs, testing...", app,
                time.time() - start_time)
    correct = []
    missing = []
    extra = []
    correct_scr = {}
    total_scr = {}
    for point in my_points:
        cached_pts = load_cached_point(os.path.splitext(point['file'])[0])
        rets = clas.classify(point['tree'], point['img'], point['path'],
                             cached_pts[0] if cached_pts else None)
        regs = list(
            filter(lambda tag: tags.valid('region', tag), point['regs']))
        print(os.path.basename(point['file']), 'Tags:', regs, 'Rets:', rets)
        ret_region = set()
        for ret in rets:
            ret_region.add(ret[1])

        if set(regs) == ret_region:
            good_cnt += 1
            correct_scr[point['scr']] = correct_scr.get(point['scr'], 0) + 1

        found = 0
        for tag in regs:
            if tag in ret_region:
                found += 1
                correct.append(tag)
            else:
                missing.append(tag)

        for tag in ret_region:
            if tag not in regs:
                extra.append(tag)

        if found == len(regs):
            inside_cnt += 1

        total_cnt += 1
        total_scr[point['scr']] = total_scr.get(point['scr'], 0) + 1

    avg_scr_ratio = 0.0
    for scr in total_scr:
        scr_ratio = 1.0 * correct_scr.get(scr, 0) / total_scr[scr]
        avg_scr_ratio += scr_ratio

    avg_scr_ratio /= len(total_scr)

    return {
        'good': good_cnt,
        'total': total_cnt,
        'inside': inside_cnt,
        'correct': correct,
        'missing': missing,
        'extra': extra,
        'app': app,
        'scr_avg': avg_scr_ratio
    }
예제 #4
0
def run_clas(files, eval_app, extrafiles):
    print("Analyzing files")
    (points, labels, apps, scrs) = load_points(files, extra=extrafiles)

    print("Point count: %d" % len(points))

    if len(points) == 0:
        print("No point matches specification!")
        return

    for i in range(len(labels)):
        if not tags.valid(scrs[i], labels[i]):
            labels[i] = 'NONE'

    sorted_apps = sorted(apps)

    global_tag_good = global_none_good = global_tag_bad = global_none_bad = 0
    global_good_cnt = {}
    global_bad_cnt = {}
    global_badpred_cnt = {}
    conf_cnt = {}
    app_stat = {}

    for scr in sorted(set(scrs)):
        scr_points = []
        scr_apps = []
        scr_labels = []
        for i in range(len(points)):
            if scr == 'ALL' or scrs[i] == scr:
                scr_points.append(points[i])
                scr_apps.append(apps[i])
                scr_labels.append(labels[i])

        if len(set(scr_apps)) == 1:
            # single-app screen, can't test
            continue
        if eval_app is None:
            split = PredefinedSplit(test_fold=list(map(
                lambda x: sorted_apps.index(x), scr_apps))).split()
        else:
            train_idx = []
            test_idx = []
            for i in range(len(scr_apps)):
                if scr_apps[i] != eval_app:
                    train_idx.append(i)
                else:
                    test_idx.append(i)
            if test_idx == []:
                continue
            split = [(train_idx, test_idx)]
            #        split = PredefinedSplit(test_fold=list(map(lambda x: 1 if x == 'etsy'
            #        else 0, scr_apps)))
        all_tag_good = all_none_good = all_tag_bad = all_none_bad = 0
        good_cnt = {}
        bad_cnt = {}
        badpred_cnt = {}

        pool = multiprocessing.Pool(processes=config.threads)
        rets = pool.map(functools.partial(evaluate, scr_points, scr_labels, scr_apps),
                        split)

        for (tag_good, tag_bad, none_good, none_bad,
             xgood, xbad, xbadpred, xconf, test_app) in rets:
            if test_app is None:
                continue

            all_tag_good += tag_good
            all_tag_bad += tag_bad
            all_none_good += none_good
            all_none_bad += none_bad

            merge_cnt(good_cnt, xgood)
            merge_cnt(bad_cnt, xbad)
            merge_cnt(badpred_cnt, xbadpred)
            merge_cnt(conf_cnt, xconf)

            if test_app not in app_stat:
                app_stat[test_app] = {}
            merge_cnt(app_stat[test_app], {'tag_good': tag_good, 'tag_bad': tag_bad,
                                           'none_good': none_good, 'none_bad': none_bad})
        pool.close()
        #for train_idx, test_idx in split.split():
        # per-screen stat info
        print("SCREEN %20s   good: %4d,%4d   bad: %4d,%4d  %.3f %.3f" % (
            scr, all_tag_good, all_none_good, all_tag_bad, all_none_bad,
            1.0 * all_tag_good / (all_tag_good + all_tag_bad),
            1.0 * (all_tag_good + all_none_good) / (
                all_tag_bad + all_none_bad + all_tag_good + all_none_good)))
        if print_per_screen:
            print("IN ALL:  good: %d,%d   bad: %d,%d" % (
                all_tag_good, all_none_good, all_tag_bad, all_none_bad))
            print("PER TAG:")
            for item in sorted(set(list(good_cnt) + list(bad_cnt))):
                print("\t%s: %d+ %d- %d*" % (item, good_cnt.get(item, 0),
                                             bad_cnt.get(item, 0),
                                             badpred_cnt.get(item, 0)))
        global_tag_good += all_tag_good
        global_tag_bad += all_tag_bad
        global_none_good += all_none_good
        global_none_bad += all_none_bad
        merge_cnt(global_good_cnt, good_cnt)
        merge_cnt(global_bad_cnt, bad_cnt)
        merge_cnt(global_badpred_cnt, badpred_cnt)

    if len(scrs) > 1:
        print("GLOBAL:  good: %d,%d   bad: %d,%d  G/G: %.3f T/T: %.3f" % (
            global_tag_good, global_none_good, global_tag_bad, global_none_bad,
            1.0 * global_tag_good / (global_tag_bad + global_tag_good),
            1.0 * (global_tag_good + global_none_good) / (
                global_tag_bad + global_none_bad + global_tag_good + global_none_good)))
        print("PER TAG:")
        for item in sorted(set(list(global_good_cnt) + list(global_bad_cnt))):
            print("\t%s: %d+ %d- %d*" % (
                item, global_good_cnt.get(item, 0), global_bad_cnt.get(item, 0),
                global_badpred_cnt.get(item, 0)))

    global_tag_good = 0.0
    global_tag_good_cnt = 0
    global_all_good = 0.0
    global_all_good_cnt = 0
    global_none_good = 0.0
    global_none_good_cnt = 0
    for app in app_stat:
        entry = app_stat[app]
        if entry['tag_good'] + entry['tag_bad'] > 0:
            tag_good_per = 100.0 * entry['tag_good'] / (entry['tag_good'] +
                                                        entry['tag_bad'])
            global_tag_good += tag_good_per
            global_tag_good_cnt += 1
        else:
            tag_good_per = -1.0
        if (entry['tag_good'] + entry['tag_bad'] +
            entry['none_good'] + entry['none_bad'] > 0):
            all_good_per = 100.0 * (entry['tag_good'] + entry['none_good']) / (
                entry['tag_good'] + entry['tag_bad'] +
                entry['none_good'] + entry['none_bad'])
            global_all_good += all_good_per
            global_all_good_cnt += 1
            none_good_per = 100.0 * entry['none_good'] / (entry['none_good'] +
                                                          entry['none_bad'])
            global_none_good += none_good_per
            global_none_good_cnt += 1
        else:
            all_good_per = -1.0
        if print_perapp or print_detail:
            print("%s: good: %d, %d  bad: %d, %d  %.3f, %.3f" % (
                app, entry['tag_good'], entry['none_good'], entry['tag_bad'],
                entry['none_bad'], tag_good_per, all_good_per))
    global_tag_good /= global_tag_good_cnt
    global_all_good /= global_all_good_cnt
    global_none_good /= global_none_good_cnt
    print("FINAL app avg: +%.3f, *%.3f, -%.3f" % (
        global_tag_good, global_all_good, global_none_good))
    print("conf_order: %r" % conf_cnt)
예제 #5
0
    tree = analyze.load_tree(filename)
    (appname, scrname) = util.get_aux_info(filename)

    if not tags.validapp(appname):
        continue
    if not tags.validscr(scrname) or scrname in tags.tag['ignored_screens']:
        #print('skip', scrname)
        continue

    scr_cnt += 1

    for nodeid in tree:
        node = tree[nodeid]
        tagged = False
        for tag in node['tags']:
            if tags.valid(scrname, tag):
                tagged = True
                break

        if not node['click'] and is_click:
            continue
        if node['children'] != [] and is_leaf:
            continue
        if node['text'] == '' and is_text:
            continue
        if is_visible and (node['height'] < 5 or node['width'] < 5):
            continue
        leaf_cnt += 1
        if tagged:
            tag_cnt += 1
예제 #6
0
    wids = {}
    tags.load("../etc-%s" % catname)
    for filename in glob.glob("../guis-%s/*.xml" % catname):
        basename = os.path.basename(filename).split('.')[0]
        appname = basename.split('_')[0]
        scrname = basename.split('_')[-1]
        if appname not in scrs:
            scrs[appname] = {}
            wids[appname] = {}
        scrs[appname][scrname] = 1

        descfile = filename.replace('.xml', '.desc.txt')
        if os.path.exists(descfile):
            for line in open(descfile):
                tagname = line.strip().split(' ', 1)[1]
                if tags.valid(scrname, tagname):
                    wids[appname][tagname] = 1

    tot = 0
    totw = 0
    appc = 0
    for app in scrs:
        if app not in tags.apps:
            continue
        print(app, len(scrs[app]), len(wids[app]))
        tot += len(scrs[app])
        totw += len(wids[app])
        appc += 1

    print("CAT", catname, tot, 1.0 * tot / appc)
    print("CAT", catname, totw, 1.0 * totw / appc)
예제 #7
0
            if am_bridge:
                bridge_totline += 1
            if am_app:
                app_totline += 1

            if cus_flow:
                cus_totline += 1
            elif '@%s:%s' % (appname, test_name) in tests:
                totline += 1
                if "screen is" in line or "screen is not" in line:
                    cur_scr = line.split(' ')[-1]
                    used_scr.add(cur_scr)

                if "@" in line and line[0] != '@':
                    for wid in wid_re.findall(line):
                        wid = wid[1:]
                        if tags.valid(cur_scr, wid):
                            used_wid.add(cur_scr + ':' + wid)

    print("Cat: ", catname)
    if tests - found:
        print(tests - found)
    print('Lib:   ', testcnt, totline, "%.2f" % (1.0 * totline / testcnt))
    print('Custom:', cus_testcnt, cus_totline,
          "%.2f" % (1.0 * cus_totline / cus_testcnt))
    print('App:', app_cnt, app_totline)
    print('Bridge', bridge_cnt, bridge_totline)
    print('A+B:', app_cnt + bridge_cnt, app_totline + bridge_totline)
    print('Used screens:', len(used_scr), list(sorted(used_scr)))
    print('Used widgets:', len(used_wid), list(sorted(used_wid)))