def learn(self, datadir, appname, extrapath, parallel=True): files = util.collect_files(datadir) extrafiles = util.collect_files(extrapath) (points, labels, _, _) = load_points(files, no_appname=appname, extra=extrafiles, parallel=parallel, nopt=self.nopt) (points, labels) = self.filter_pts(points, labels) self.classes = list(set(labels)) for lbl in self.classes: self.pipelines[lbl] = prepare_pipeline() pts_with_lbl = [] my_labels = [] for i in range(len(labels)): if labels[i] == lbl: pts_with_lbl.append(points[i]) my_labels.append(1) elif not use_oneclass: pts_with_lbl.append(points[i]) my_labels.append(-1) self.pipelines[lbl].fit(pts_with_lbl, my_labels)
def build_srpm(r, b): if len(b.spec) <= len('.spec'): # should not really get here util.append_to(b.logfile, "error: No .spec given but build src.rpm wanted") return "FAIL" status.push("building %s" % b.spec) # messagebus.notify(topic="build_srpm.start", spec=b.spec, flags=r.flags, batch=b, request=r) b.src_rpm = "" builder_opts = "-nu -nm --nodeps --http --define \'_pld_builder 1\'" if ("test-build" in r.flags): tag_test = "" else: tag_test = " -Tp %s -tt" % (config.tag_prefixes[0], ) cmd = ( "cd rpm/packages; nice -n %s ./builder %s -bs %s -r %s %s %s %s %s 2>&1" % (config.nice, builder_opts, b.bconds_string(), b.branch, tag_test, b.kernel_string(), b.defines_string(), b.spec)) util.append_to(b.logfile, "request from: %s" % r.requester) util.append_to(b.logfile, "started at: %s" % time.asctime()) util.append_to(b.logfile, "building SRPM using: %s\n" % cmd) res = chroot.run(cmd, logfile=b.logfile) util.append_to(b.logfile, "exit status %d" % res) files = util.collect_files(b.logfile) if len(files) > 0: if len(files) > 1: util.append_to(b.logfile, "error: More than one file produced: %s" % files) res = "FAIL_TOOMANYFILES" last = files[len(files) - 1] b.src_rpm_file = last b.src_rpm = os.path.basename(last) r.chroot_files.extend(files) else: util.append_to(b.logfile, "error: No files produced.") res = "FAIL" if res == 0 and not "test-build" in r.flags: for pref in config.tag_prefixes: util.append_to(b.logfile, "Tagging with prefix: %s" % pref) res = chroot.run("cd rpm/packages; ./builder -bs %s -r %s -Tp %s -Tv %s %s" % \ (b.bconds_string(), b.branch, pref, b.defines_string(), b.spec), logfile = b.logfile) if res == 0: transfer_file(r, b) packagename = b.spec[:-5] packagedir = "rpm/packages/%s" % packagename chroot.run("rpm/packages/builder -m %s" % \ (b.spec,), logfile = b.logfile) chroot.run("rm -rf %s" % packagedir, logfile=b.logfile) status.pop() if res: res = "FAIL" # messagebus.notify(topic="build_srpm.finish", spec=b.spec) return res
def build_srpm(r, b): if len(b.spec) <= len('.spec'): # should not really get here util.append_to(b.logfile, "error: No .spec given but build src.rpm wanted") return "FAIL" status.push("building %s" % b.spec) b.src_rpm = "" builder_opts = "-nu -nm --nodeps --http" if ("test-build" in r.flags): tag_test="" else: tag_test=" -Tp %s -tt" % (config.tag_prefixes[0],) cmd = ("cd rpm/packages; nice -n %s ./builder %s -bs %s -r %s %s %s %s %s 2>&1" % (config.nice, builder_opts, b.bconds_string(), b.branch, tag_test, b.kernel_string(), b.defines_string(), b.spec)) util.append_to(b.logfile, "request from: %s" % r.requester) util.append_to(b.logfile, "started at: %s" % time.asctime()) util.append_to(b.logfile, "building SRPM using: %s\n" % cmd) res = chroot.run(cmd, logfile = b.logfile) util.append_to(b.logfile, "exit status %d" % res) files = util.collect_files(b.logfile) if len(files) > 0: if len(files) > 1: util.append_to(b.logfile, "error: More than one file produced: %s" % files) res = "FAIL_TOOMANYFILES" last = files[len(files) - 1] b.src_rpm_file = last b.src_rpm = os.path.basename(last) r.chroot_files.extend(files) else: util.append_to(b.logfile, "error: No files produced.") res = "FAIL" if res == 0 and not "test-build" in r.flags: for pref in config.tag_prefixes: util.append_to(b.logfile, "Tagging with prefix: %s" % pref) res = chroot.run("cd rpm/packages; ./builder -r %s -Tp %s -Tv %s" % \ (b.branch, pref, b.spec), logfile = b.logfile) if res == 0: transfer_file(r, b) packagename = b.spec[:-5] packagedir = "rpm/packages/%s" % packagename chroot.run("rpm/packages/builder -m %s" % \ (b.spec,), logfile = b.logfile) chroot.run("rm -rf %s" % packagedir, logfile = b.logfile) status.pop() if res: res = "FAIL" return res
def build_rpm(r, b): packagename = b.get_package_name() if not packagename: # should not really get here b.log_line("error: No .spec not given of malformed: '%s'" % b.spec) res = "FAIL_INTERNAL" return res status.push("building %s (%s)" % (b.spec, packagename)) b.log_line("request from: %s" % r.requester) if check_skip_build(r, b): b.log_line("build skipped due to src builder request") res = "SKIP_REQUESTED" return res b.log_line("started at: %s" % time.asctime()) fetch_src(r, b) b.log_line("installing srpm: %s" % b.src_rpm) res = chroot.run(""" set -ex; install -d %(topdir)s/{BUILD,RPMS}; rpm -Uhv --nodeps %(rpmdefs)s %(src_rpm)s; rm -f %(src_rpm)s; """ % { 'topdir': b._topdir, 'rpmdefs': b.rpmbuild_opts(), 'src_rpm': b.src_rpm }, logfile=b.logfile) b.files = [] tmpdir = b.tmpdir() if res: b.log_line("error: installing src rpm failed") res = "FAIL_SRPM_INSTALL" else: prepare_env() chroot.run("set -x; install -m 700 -d %s" % tmpdir, logfile=b.logfile) b.default_target(config.arch) # check for build arch before filling BR cmd = "set -ex; TMPDIR=%(tmpdir)s exec nice -n %(nice)s " \ "rpmbuild -bp --short-circuit --nodeps %(rpmdefs)s --define 'prep exit 0' %(topdir)s/%(spec)s" % { 'tmpdir': tmpdir, 'nice' : config.nice, 'topdir' : b._topdir, 'rpmdefs' : b.rpmbuild_opts(), 'spec': b.spec, } res = chroot.run(cmd, logfile=b.logfile) if res: res = "UNSUPP" b.log_line("error: build arch check (%s) failed" % cmd) if not res: if ("no-install-br" not in r.flags) and not install.uninstall_self_conflict(b): res = "FAIL_DEPS_UNINSTALL" if ("no-install-br" not in r.flags) and not install.install_br(r, b): res = "FAIL_DEPS_INSTALL" if not res: max_jobs = max( min(int(os.sysconf('SC_NPROCESSORS_ONLN') + 1), config.max_jobs), 1) if r.max_jobs > 0: max_jobs = max(min(config.max_jobs, r.max_jobs), 1) cmd = "set -ex; : build-id: %(r_id)s; TMPDIR=%(tmpdir)s exec nice -n %(nice)s " \ "rpmbuild -bb --define '_smp_mflags -j%(max_jobs)d' %(rpmdefs)s %(topdir)s/%(spec)s" % { 'r_id' : r.id, 'tmpdir': tmpdir, 'nice' : config.nice, 'rpmdefs' : b.rpmbuild_opts(), 'topdir' : b._topdir, 'max_jobs' : max_jobs, 'spec': b.spec, } b.log_line("building RPM using: %s" % cmd) begin_time = time.time() res = chroot.run(cmd, logfile=b.logfile) end_time = time.time() b.log_line("ended at: %s, done in %s" % (time.asctime(), datetime.timedelta(0, end_time - begin_time))) if res: res = "FAIL" files = util.collect_files(b.logfile, basedir=b._topdir) if len(files) > 0: r.chroot_files.extend(files) else: b.log_line("error: No files produced.") last_section = util.find_last_section(b.logfile) if last_section == None: res = "FAIL" else: res = "FAIL_%s" % last_section.upper() b.files = files # cleanup tmp and build files chroot.run(""" set -ex; chmod -R u+rwX %(topdir)s/BUILD; rm -rf %(topdir)s/{tmp,BUILD} """ % { 'topdir': b._topdir, }, logfile=b.logfile) def ll(l): util.append_to(b.logfile, l) if b.files != []: rpm_cache_dir = config.rpm_cache_dir if "test-build" not in r.flags: # NOTE: copying to cache dir doesn't mean that build failed, so ignore result b.log_line("copy rpm files to cache_dir: %s" % rpm_cache_dir) chroot.run( "cp -f %s %s && poldek --mo=nodiff --mkidxz -s %s/" % \ (string.join(b.files), rpm_cache_dir, rpm_cache_dir), logfile = b.logfile, user = "******" ) else: ll("test-build: not copying to " + rpm_cache_dir) ll("Begin-PLD-Builder-Info") if "upgrade" in r.flags: b.upgraded = install.upgrade_from_batch(r, b) else: ll("not upgrading") ll("End-PLD-Builder-Info") for f in b.files: local = r.tmp_dir + os.path.basename(f) chroot.cp(f, outfile=local, rm=True) ftp.add(local) # cleanup all remains from this build chroot.run(""" set -ex; rm -rf %(topdir)s; """ % { 'topdir': b._topdir, }, logfile=b.logfile) def uploadinfo(b): c = "file:SRPMS:%s\n" % b.src_rpm for f in b.files: c = c + "file:ARCH:%s\n" % os.path.basename(f) c = c + "END\n" return c if config.gen_upinfo and b.files != [] and 'test-build' not in r.flags: fname = r.tmp_dir + b.src_rpm + ".uploadinfo" f = open(fname, "w") f.write(uploadinfo(b)) f.close() ftp.add(fname, "uploadinfo") status.pop() return res
def main(): parser = argparse.ArgumentParser(description="Classifier") parser.add_argument('--guispath', help='data path', default="../guis/") parser.add_argument('--tagspath', help='tags path', default="../etc/tags.txt") parser.add_argument('--extrapath', help='extra data path', default="../guis-extra/") parser.add_argument('--app', help='only for app') parser.add_argument('--scrs', help='only for screens') parser.add_argument('--debug', help='debug', default=False, action='store_const', const=True) parser.add_argument('--nopt', help='no pt cache', default=False, action='store_const', const=True) parser.add_argument('--printpt', help='print points', default=False, action='store_const', const=True) parser.add_argument('--onlyfor', help='only print for') parser.add_argument('--analyze', help='show pt for one file', default=None) parser.add_argument('--thres', help='threshold') args = parser.parse_args() if args.debug: logging.basicConfig(level=logging.DEBUG) else: logging.basicConfig(level=logging.INFO) tags.load(args.tagspath) if args.analyze: (pts, lbls, _, _) = load_points([args.analyze], nopt=True) for i in range(len(pts)): print_point(pts[i], lbls[i]) return if args.thres is not None: config.REGION_SCORE_BOUND = float(args.thres) start_time = time.time() points = {} point_count = 0 files = util.collect_files(args.guispath) # load points first so we have cache later logger.info("Caching points...") (pts, lbls, _, _) = load_points(files, nopt=args.nopt, show_progress=True) if args.printpt: if args.onlyfor: onlyfor = args.onlyfor.split(',') else: onlyfor = None for i in range(len(pts)): if onlyfor is None or lbls[i] in onlyfor: if args.app is None or pts[i]['app'] == args.app: print_point(pts[i], lbls[i]) return logger.info("cached %d points.", len(pts)) logger.info("Load points...") if args.scrs is None: scrs = None else: scrs = args.scrs.split(',') rets = util.parallel_work(functools.partial(load_info, args.app, scrs), files, True, show_progress=True) for ret in rets: if ret is not None: points[ret['app']] = points.get(ret['app'], []) + [ret] point_count += 1 logger.info("Point count: %d, used %.3fs", point_count, time.time() - start_time) apps = list(set(points)) logger.info("Testing, app count %d", len(apps)) if args.app is None and config.parallel: pool = multiprocessing.Pool(processes=config.threads) rets = pool.map( functools.partial(evaluate_app, False, args.guispath, args.extrapath), points.values()) pool.close() else: rets = map( functools.partial(evaluate_app, True, args.guispath, args.extrapath), points.values()) good_cnt = total_cnt = inside_cnt = 0 correct = [] missing = [] extra = [] good_ratio = 0.0 for ret in rets: good_cnt += ret['good'] total_cnt += ret['total'] inside_cnt += ret['inside'] correct += ret['correct'] missing += ret['missing'] extra += ret['extra'] logger.info("app %s: %d/%d avg(scr): %.2f", ret['app'], ret['good'], ret['total'], ret['scr_avg']) good_ratio += ret['scr_avg'] good_ratio /= len(apps) logger.info("Good/Total: %d/%d %.2f", good_cnt, total_cnt, 1.0 * good_cnt / total_cnt) logger.info("Inside/Total: %d/%d %.2f", inside_cnt, total_cnt, 1.0 * inside_cnt / total_cnt) logger.info("Correct: %s", print_hash(count_items(correct))) logger.info("Missing: %s", print_hash(count_items(missing))) logger.info("Extra: %s", print_hash(count_items(extra))) logger.info("Good ratio(app avg): %.3f", good_ratio)
import util import tags catname = sys.argv[1] tags.load("../etc-%s" % catname) is_leaf = 'leaf' in sys.argv is_click = 'click' in sys.argv is_text = 'text' in sys.argv is_visible = 'visible' in sys.argv leaf_cnt = 0 tag_cnt = 0 scr_cnt = 0 for filename in util.collect_files("../guis-%s" % catname): tree = analyze.load_tree(filename) (appname, scrname) = util.get_aux_info(filename) if not tags.validapp(appname): continue if not tags.validscr(scrname) or scrname in tags.tag['ignored_screens']: #print('skip', scrname) continue scr_cnt += 1 for nodeid in tree: node = tree[nodeid] tagged = False for tag in node['tags']:
def build_rpm(r, b): packagename = b.get_package_name() if not packagename: # should not really get here b.log_line("error: No .spec not given of malformed: '%s'" % b.spec) res = "FAIL_INTERNAL" return res status.push("building %s (%s)" % (b.spec, packagename)) b.log_line("request from: %s" % r.requester) if check_skip_build(r, b): b.log_line("build skipped due to src builder request") res = "SKIP_REQUESTED" return res b.log_line("started at: %s" % time.asctime()) fetch_src(r, b) b.log_line("installing srpm: %s" % b.src_rpm) res = chroot.run(""" set -ex; install -d %(topdir)s/{BUILD,RPMS}; rpm -Uhv --nodeps %(rpmdefs)s %(src_rpm)s; rm -f %(src_rpm)s; """ % { 'topdir' : b._topdir, 'rpmdefs' : b.rpmbuild_opts(), 'src_rpm' : b.src_rpm }, logfile = b.logfile) b.files = [] tmpdir = b.tmpdir() if res: b.log_line("error: installing src rpm failed") res = "FAIL_SRPM_INSTALL" else: prepare_env() chroot.run("set -x; install -m 700 -d %s" % tmpdir, logfile=b.logfile) b.default_target(config.arch) # check for build arch before filling BR cmd = "set -ex; TMPDIR=%(tmpdir)s exec nice -n %(nice)s " \ "rpmbuild -bp --short-circuit --nodeps %(rpmdefs)s --define 'prep exit 0' %(topdir)s/%(spec)s" % { 'tmpdir': tmpdir, 'nice' : config.nice, 'topdir' : b._topdir, 'rpmdefs' : b.rpmbuild_opts(), 'spec': b.spec, } res = chroot.run(cmd, logfile = b.logfile) if res: res = "UNSUPP" b.log_line("error: build arch check (%s) failed" % cmd) if not res: if ("no-install-br" not in r.flags) and not install.uninstall_self_conflict(b): res = "FAIL_DEPS_UNINSTALL" if ("no-install-br" not in r.flags) and not install.install_br(r, b): res = "FAIL_DEPS_INSTALL" if not res: max_jobs = max(min(int(os.sysconf('SC_NPROCESSORS_ONLN') + 1), config.max_jobs), 1) if r.max_jobs > 0: max_jobs = max(min(config.max_jobs, r.max_jobs), 1) cmd = "set -ex; : build-id: %(r_id)s; TMPDIR=%(tmpdir)s exec nice -n %(nice)s " \ "rpmbuild -bb --define '_smp_mflags -j%(max_jobs)d' %(rpmdefs)s %(topdir)s/%(spec)s" % { 'r_id' : r.id, 'tmpdir': tmpdir, 'nice' : config.nice, 'rpmdefs' : b.rpmbuild_opts(), 'topdir' : b._topdir, 'max_jobs' : max_jobs, 'spec': b.spec, } b.log_line("building RPM using: %s" % cmd) begin_time = time.time() res = chroot.run(cmd, logfile = b.logfile) end_time = time.time() b.log_line("ended at: %s, done in %s" % (time.asctime(), datetime.timedelta(0, end_time - begin_time))) if res: res = "FAIL" files = util.collect_files(b.logfile, basedir = b._topdir) if len(files) > 0: r.chroot_files.extend(files) else: b.log_line("error: No files produced.") last_section = util.find_last_section(b.logfile) if last_section == None: res = "FAIL" else: res = "FAIL_%s" % last_section.upper() b.files = files # cleanup tmp and build files chroot.run(""" set -ex; chmod -R u+rwX %(topdir)s/BUILD; rm -rf %(topdir)s/{tmp,BUILD} """ % { 'topdir' : b._topdir, }, logfile = b.logfile) def ll(l): util.append_to(b.logfile, l) if b.files != []: rpm_cache_dir = config.rpm_cache_dir if "test-build" not in r.flags: # NOTE: copying to cache dir doesn't mean that build failed, so ignore result b.log_line("copy rpm files to cache_dir: %s" % rpm_cache_dir) chroot.run( "cp -f %s %s && poldek --mo=nodiff --mkidxz -s %s/" % \ (string.join(b.files), rpm_cache_dir, rpm_cache_dir), logfile = b.logfile, user = "******" ) else: ll("test-build: not copying to " + rpm_cache_dir) ll("Begin-PLD-Builder-Info") if "upgrade" in r.flags: b.upgraded = install.upgrade_from_batch(r, b) else: ll("not upgrading") ll("End-PLD-Builder-Info") for f in b.files: local = r.tmp_dir + os.path.basename(f) chroot.cp(f, outfile = local, rm = True) ftp.add(local) # cleanup all remains from this build chroot.run(""" set -ex; rm -rf %(topdir)s; """ % { 'topdir' : b._topdir, }, logfile = b.logfile) def uploadinfo(b): c="file:SRPMS:%s\n" % b.src_rpm for f in b.files: c=c + "file:ARCH:%s\n" % os.path.basename(f) c=c + "END\n" return c if config.gen_upinfo and b.files != [] and 'test-build' not in r.flags: fname = r.tmp_dir + b.src_rpm + ".uploadinfo" f = open(fname, "w") f.write(uploadinfo(b)) f.close() ftp.add(fname, "uploadinfo") status.pop() return res