def learn(self, datadir, appname, extrapath, parallel=True):
files = util.collect_files(datadir)
extrafiles = util.collect_files(extrapath)
(points, labels, _, _) = load_points(files,
no_appname=appname,
extra=extrafiles,
parallel=parallel,
nopt=self.nopt)
(points, labels) = self.filter_pts(points, labels)
self.classes = list(set(labels))
for lbl in self.classes:
self.pipelines[lbl] = prepare_pipeline()
pts_with_lbl = []
my_labels = []
for i in range(len(labels)):
if labels[i] == lbl:
pts_with_lbl.append(points[i])
my_labels.append(1)
elif not use_oneclass:
pts_with_lbl.append(points[i])
my_labels.append(-1)
self.pipelines[lbl].fit(pts_with_lbl, my_labels)
def build_srpm(r, b):
if len(b.spec) <= len('.spec'):
# should not really get here
util.append_to(b.logfile,
"error: No .spec given but build src.rpm wanted")
return "FAIL"
status.push("building %s" % b.spec)
# messagebus.notify(topic="build_srpm.start", spec=b.spec, flags=r.flags, batch=b, request=r)
b.src_rpm = ""
builder_opts = "-nu -nm --nodeps --http --define \'_pld_builder 1\'"
if ("test-build" in r.flags):
tag_test = ""
else:
tag_test = " -Tp %s -tt" % (config.tag_prefixes[0], )
cmd = (
"cd rpm/packages; nice -n %s ./builder %s -bs %s -r %s %s %s %s %s 2>&1"
% (config.nice, builder_opts, b.bconds_string(), b.branch, tag_test,
b.kernel_string(), b.defines_string(), b.spec))
util.append_to(b.logfile, "request from: %s" % r.requester)
util.append_to(b.logfile, "started at: %s" % time.asctime())
util.append_to(b.logfile, "building SRPM using: %s\n" % cmd)
res = chroot.run(cmd, logfile=b.logfile)
util.append_to(b.logfile, "exit status %d" % res)
files = util.collect_files(b.logfile)
if len(files) > 0:
if len(files) > 1:
util.append_to(b.logfile,
"error: More than one file produced: %s" % files)
res = "FAIL_TOOMANYFILES"
last = files[len(files) - 1]
b.src_rpm_file = last
b.src_rpm = os.path.basename(last)
r.chroot_files.extend(files)
else:
util.append_to(b.logfile, "error: No files produced.")
res = "FAIL"
if res == 0 and not "test-build" in r.flags:
for pref in config.tag_prefixes:
util.append_to(b.logfile, "Tagging with prefix: %s" % pref)
res = chroot.run("cd rpm/packages; ./builder -bs %s -r %s -Tp %s -Tv %s %s" % \
(b.bconds_string(), b.branch, pref, b.defines_string(), b.spec), logfile = b.logfile)
if res == 0:
transfer_file(r, b)
packagename = b.spec[:-5]
packagedir = "rpm/packages/%s" % packagename
chroot.run("rpm/packages/builder -m %s" % \
(b.spec,), logfile = b.logfile)
chroot.run("rm -rf %s" % packagedir, logfile=b.logfile)
status.pop()
if res:
res = "FAIL"
# messagebus.notify(topic="build_srpm.finish", spec=b.spec)
return res
def build_srpm(r, b):
if len(b.spec) <= len('.spec'):
# should not really get here
util.append_to(b.logfile, "error: No .spec given but build src.rpm wanted")
return "FAIL"
status.push("building %s" % b.spec)
b.src_rpm = ""
builder_opts = "-nu -nm --nodeps --http"
if ("test-build" in r.flags):
tag_test=""
else:
tag_test=" -Tp %s -tt" % (config.tag_prefixes[0],)
cmd = ("cd rpm/packages; nice -n %s ./builder %s -bs %s -r %s %s %s %s %s 2>&1" %
(config.nice, builder_opts, b.bconds_string(), b.branch,
tag_test, b.kernel_string(), b.defines_string(), b.spec))
util.append_to(b.logfile, "request from: %s" % r.requester)
util.append_to(b.logfile, "started at: %s" % time.asctime())
util.append_to(b.logfile, "building SRPM using: %s\n" % cmd)
res = chroot.run(cmd, logfile = b.logfile)
util.append_to(b.logfile, "exit status %d" % res)
files = util.collect_files(b.logfile)
if len(files) > 0:
if len(files) > 1:
util.append_to(b.logfile, "error: More than one file produced: %s" % files)
res = "FAIL_TOOMANYFILES"
last = files[len(files) - 1]
b.src_rpm_file = last
b.src_rpm = os.path.basename(last)
r.chroot_files.extend(files)
else:
util.append_to(b.logfile, "error: No files produced.")
res = "FAIL"
if res == 0 and not "test-build" in r.flags:
for pref in config.tag_prefixes:
util.append_to(b.logfile, "Tagging with prefix: %s" % pref)
res = chroot.run("cd rpm/packages; ./builder -r %s -Tp %s -Tv %s" % \
(b.branch, pref, b.spec), logfile = b.logfile)
if res == 0:
transfer_file(r, b)
packagename = b.spec[:-5]
packagedir = "rpm/packages/%s" % packagename
chroot.run("rpm/packages/builder -m %s" % \
(b.spec,), logfile = b.logfile)
chroot.run("rm -rf %s" % packagedir, logfile = b.logfile)
status.pop()
if res:
res = "FAIL"
return res
def build_rpm(r, b):
packagename = b.get_package_name()
if not packagename:
# should not really get here
b.log_line("error: No .spec not given of malformed: '%s'" % b.spec)
res = "FAIL_INTERNAL"
return res
status.push("building %s (%s)" % (b.spec, packagename))
b.log_line("request from: %s" % r.requester)
if check_skip_build(r, b):
b.log_line("build skipped due to src builder request")
res = "SKIP_REQUESTED"
return res
b.log_line("started at: %s" % time.asctime())
fetch_src(r, b)
b.log_line("installing srpm: %s" % b.src_rpm)
res = chroot.run("""
set -ex;
install -d %(topdir)s/{BUILD,RPMS};
rpm -Uhv --nodeps %(rpmdefs)s %(src_rpm)s;
rm -f %(src_rpm)s;
""" % {
'topdir': b._topdir,
'rpmdefs': b.rpmbuild_opts(),
'src_rpm': b.src_rpm
},
logfile=b.logfile)
b.files = []
tmpdir = b.tmpdir()
if res:
b.log_line("error: installing src rpm failed")
res = "FAIL_SRPM_INSTALL"
else:
prepare_env()
chroot.run("set -x; install -m 700 -d %s" % tmpdir, logfile=b.logfile)
b.default_target(config.arch)
# check for build arch before filling BR
cmd = "set -ex; TMPDIR=%(tmpdir)s exec nice -n %(nice)s " \
"rpmbuild -bp --short-circuit --nodeps %(rpmdefs)s --define 'prep exit 0' %(topdir)s/%(spec)s" % {
'tmpdir': tmpdir,
'nice' : config.nice,
'topdir' : b._topdir,
'rpmdefs' : b.rpmbuild_opts(),
'spec': b.spec,
}
res = chroot.run(cmd, logfile=b.logfile)
if res:
res = "UNSUPP"
b.log_line("error: build arch check (%s) failed" % cmd)
if not res:
if ("no-install-br"
not in r.flags) and not install.uninstall_self_conflict(b):
res = "FAIL_DEPS_UNINSTALL"
if ("no-install-br"
not in r.flags) and not install.install_br(r, b):
res = "FAIL_DEPS_INSTALL"
if not res:
max_jobs = max(
min(int(os.sysconf('SC_NPROCESSORS_ONLN') + 1),
config.max_jobs), 1)
if r.max_jobs > 0:
max_jobs = max(min(config.max_jobs, r.max_jobs), 1)
cmd = "set -ex; : build-id: %(r_id)s; TMPDIR=%(tmpdir)s exec nice -n %(nice)s " \
"rpmbuild -bb --define '_smp_mflags -j%(max_jobs)d' %(rpmdefs)s %(topdir)s/%(spec)s" % {
'r_id' : r.id,
'tmpdir': tmpdir,
'nice' : config.nice,
'rpmdefs' : b.rpmbuild_opts(),
'topdir' : b._topdir,
'max_jobs' : max_jobs,
'spec': b.spec,
}
b.log_line("building RPM using: %s" % cmd)
begin_time = time.time()
res = chroot.run(cmd, logfile=b.logfile)
end_time = time.time()
b.log_line("ended at: %s, done in %s" %
(time.asctime(),
datetime.timedelta(0, end_time - begin_time)))
if res:
res = "FAIL"
files = util.collect_files(b.logfile, basedir=b._topdir)
if len(files) > 0:
r.chroot_files.extend(files)
else:
b.log_line("error: No files produced.")
last_section = util.find_last_section(b.logfile)
if last_section == None:
res = "FAIL"
else:
res = "FAIL_%s" % last_section.upper()
b.files = files
# cleanup tmp and build files
chroot.run("""
set -ex;
chmod -R u+rwX %(topdir)s/BUILD;
rm -rf %(topdir)s/{tmp,BUILD}
""" % {
'topdir': b._topdir,
},
logfile=b.logfile)
def ll(l):
util.append_to(b.logfile, l)
if b.files != []:
rpm_cache_dir = config.rpm_cache_dir
if "test-build" not in r.flags:
# NOTE: copying to cache dir doesn't mean that build failed, so ignore result
b.log_line("copy rpm files to cache_dir: %s" % rpm_cache_dir)
chroot.run(
"cp -f %s %s && poldek --mo=nodiff --mkidxz -s %s/" % \
(string.join(b.files), rpm_cache_dir, rpm_cache_dir),
logfile = b.logfile, user = "******"
)
else:
ll("test-build: not copying to " + rpm_cache_dir)
ll("Begin-PLD-Builder-Info")
if "upgrade" in r.flags:
b.upgraded = install.upgrade_from_batch(r, b)
else:
ll("not upgrading")
ll("End-PLD-Builder-Info")
for f in b.files:
local = r.tmp_dir + os.path.basename(f)
chroot.cp(f, outfile=local, rm=True)
ftp.add(local)
# cleanup all remains from this build
chroot.run("""
set -ex;
rm -rf %(topdir)s;
""" % {
'topdir': b._topdir,
},
logfile=b.logfile)
def uploadinfo(b):
c = "file:SRPMS:%s\n" % b.src_rpm
for f in b.files:
c = c + "file:ARCH:%s\n" % os.path.basename(f)
c = c + "END\n"
return c
if config.gen_upinfo and b.files != [] and 'test-build' not in r.flags:
fname = r.tmp_dir + b.src_rpm + ".uploadinfo"
f = open(fname, "w")
f.write(uploadinfo(b))
f.close()
ftp.add(fname, "uploadinfo")
status.pop()
return res
def main():
parser = argparse.ArgumentParser(description="Classifier")
parser.add_argument('--guispath', help='data path', default="../guis/")
parser.add_argument('--tagspath',
help='tags path',
default="../etc/tags.txt")
parser.add_argument('--extrapath',
help='extra data path',
default="../guis-extra/")
parser.add_argument('--app', help='only for app')
parser.add_argument('--scrs', help='only for screens')
parser.add_argument('--debug',
help='debug',
default=False,
action='store_const',
const=True)
parser.add_argument('--nopt',
help='no pt cache',
default=False,
action='store_const',
const=True)
parser.add_argument('--printpt',
help='print points',
default=False,
action='store_const',
const=True)
parser.add_argument('--onlyfor', help='only print for')
parser.add_argument('--analyze', help='show pt for one file', default=None)
parser.add_argument('--thres', help='threshold')
args = parser.parse_args()
if args.debug:
logging.basicConfig(level=logging.DEBUG)
else:
logging.basicConfig(level=logging.INFO)
tags.load(args.tagspath)
if args.analyze:
(pts, lbls, _, _) = load_points([args.analyze], nopt=True)
for i in range(len(pts)):
print_point(pts[i], lbls[i])
return
if args.thres is not None:
config.REGION_SCORE_BOUND = float(args.thres)
start_time = time.time()
points = {}
point_count = 0
files = util.collect_files(args.guispath)
# load points first so we have cache later
logger.info("Caching points...")
(pts, lbls, _, _) = load_points(files, nopt=args.nopt, show_progress=True)
if args.printpt:
if args.onlyfor:
onlyfor = args.onlyfor.split(',')
else:
onlyfor = None
for i in range(len(pts)):
if onlyfor is None or lbls[i] in onlyfor:
if args.app is None or pts[i]['app'] == args.app:
print_point(pts[i], lbls[i])
return
logger.info("cached %d points.", len(pts))
logger.info("Load points...")
if args.scrs is None:
scrs = None
else:
scrs = args.scrs.split(',')
rets = util.parallel_work(functools.partial(load_info, args.app, scrs),
files,
True,
show_progress=True)
for ret in rets:
if ret is not None:
points[ret['app']] = points.get(ret['app'], []) + [ret]
point_count += 1
logger.info("Point count: %d, used %.3fs", point_count,
time.time() - start_time)
apps = list(set(points))
logger.info("Testing, app count %d", len(apps))
if args.app is None and config.parallel:
pool = multiprocessing.Pool(processes=config.threads)
rets = pool.map(
functools.partial(evaluate_app, False, args.guispath,
args.extrapath), points.values())
pool.close()
else:
rets = map(
functools.partial(evaluate_app, True, args.guispath,
args.extrapath), points.values())
good_cnt = total_cnt = inside_cnt = 0
correct = []
missing = []
extra = []
good_ratio = 0.0
for ret in rets:
good_cnt += ret['good']
total_cnt += ret['total']
inside_cnt += ret['inside']
correct += ret['correct']
missing += ret['missing']
extra += ret['extra']
logger.info("app %s: %d/%d avg(scr): %.2f", ret['app'], ret['good'],
ret['total'], ret['scr_avg'])
good_ratio += ret['scr_avg']
good_ratio /= len(apps)
logger.info("Good/Total: %d/%d %.2f", good_cnt, total_cnt,
1.0 * good_cnt / total_cnt)
logger.info("Inside/Total: %d/%d %.2f", inside_cnt, total_cnt,
1.0 * inside_cnt / total_cnt)
logger.info("Correct: %s", print_hash(count_items(correct)))
logger.info("Missing: %s", print_hash(count_items(missing)))
logger.info("Extra: %s", print_hash(count_items(extra)))
logger.info("Good ratio(app avg): %.3f", good_ratio)
import util
import tags
catname = sys.argv[1]
tags.load("../etc-%s" % catname)
is_leaf = 'leaf' in sys.argv
is_click = 'click' in sys.argv
is_text = 'text' in sys.argv
is_visible = 'visible' in sys.argv
leaf_cnt = 0
tag_cnt = 0
scr_cnt = 0
for filename in util.collect_files("../guis-%s" % catname):
tree = analyze.load_tree(filename)
(appname, scrname) = util.get_aux_info(filename)
if not tags.validapp(appname):
continue
if not tags.validscr(scrname) or scrname in tags.tag['ignored_screens']:
#print('skip', scrname)
continue
scr_cnt += 1
for nodeid in tree:
node = tree[nodeid]
tagged = False
for tag in node['tags']:
def build_rpm(r, b):
packagename = b.get_package_name()
if not packagename:
# should not really get here
b.log_line("error: No .spec not given of malformed: '%s'" % b.spec)
res = "FAIL_INTERNAL"
return res
status.push("building %s (%s)" % (b.spec, packagename))
b.log_line("request from: %s" % r.requester)
if check_skip_build(r, b):
b.log_line("build skipped due to src builder request")
res = "SKIP_REQUESTED"
return res
b.log_line("started at: %s" % time.asctime())
fetch_src(r, b)
b.log_line("installing srpm: %s" % b.src_rpm)
res = chroot.run("""
set -ex;
install -d %(topdir)s/{BUILD,RPMS};
rpm -Uhv --nodeps %(rpmdefs)s %(src_rpm)s;
rm -f %(src_rpm)s;
""" % {
'topdir' : b._topdir,
'rpmdefs' : b.rpmbuild_opts(),
'src_rpm' : b.src_rpm
}, logfile = b.logfile)
b.files = []
tmpdir = b.tmpdir()
if res:
b.log_line("error: installing src rpm failed")
res = "FAIL_SRPM_INSTALL"
else:
prepare_env()
chroot.run("set -x; install -m 700 -d %s" % tmpdir, logfile=b.logfile)
b.default_target(config.arch)
# check for build arch before filling BR
cmd = "set -ex; TMPDIR=%(tmpdir)s exec nice -n %(nice)s " \
"rpmbuild -bp --short-circuit --nodeps %(rpmdefs)s --define 'prep exit 0' %(topdir)s/%(spec)s" % {
'tmpdir': tmpdir,
'nice' : config.nice,
'topdir' : b._topdir,
'rpmdefs' : b.rpmbuild_opts(),
'spec': b.spec,
}
res = chroot.run(cmd, logfile = b.logfile)
if res:
res = "UNSUPP"
b.log_line("error: build arch check (%s) failed" % cmd)
if not res:
if ("no-install-br" not in r.flags) and not install.uninstall_self_conflict(b):
res = "FAIL_DEPS_UNINSTALL"
if ("no-install-br" not in r.flags) and not install.install_br(r, b):
res = "FAIL_DEPS_INSTALL"
if not res:
max_jobs = max(min(int(os.sysconf('SC_NPROCESSORS_ONLN') + 1), config.max_jobs), 1)
if r.max_jobs > 0:
max_jobs = max(min(config.max_jobs, r.max_jobs), 1)
cmd = "set -ex; : build-id: %(r_id)s; TMPDIR=%(tmpdir)s exec nice -n %(nice)s " \
"rpmbuild -bb --define '_smp_mflags -j%(max_jobs)d' %(rpmdefs)s %(topdir)s/%(spec)s" % {
'r_id' : r.id,
'tmpdir': tmpdir,
'nice' : config.nice,
'rpmdefs' : b.rpmbuild_opts(),
'topdir' : b._topdir,
'max_jobs' : max_jobs,
'spec': b.spec,
}
b.log_line("building RPM using: %s" % cmd)
begin_time = time.time()
res = chroot.run(cmd, logfile = b.logfile)
end_time = time.time()
b.log_line("ended at: %s, done in %s" % (time.asctime(), datetime.timedelta(0, end_time - begin_time)))
if res:
res = "FAIL"
files = util.collect_files(b.logfile, basedir = b._topdir)
if len(files) > 0:
r.chroot_files.extend(files)
else:
b.log_line("error: No files produced.")
last_section = util.find_last_section(b.logfile)
if last_section == None:
res = "FAIL"
else:
res = "FAIL_%s" % last_section.upper()
b.files = files
# cleanup tmp and build files
chroot.run("""
set -ex;
chmod -R u+rwX %(topdir)s/BUILD;
rm -rf %(topdir)s/{tmp,BUILD}
""" % {
'topdir' : b._topdir,
}, logfile = b.logfile)
def ll(l):
util.append_to(b.logfile, l)
if b.files != []:
rpm_cache_dir = config.rpm_cache_dir
if "test-build" not in r.flags:
# NOTE: copying to cache dir doesn't mean that build failed, so ignore result
b.log_line("copy rpm files to cache_dir: %s" % rpm_cache_dir)
chroot.run(
"cp -f %s %s && poldek --mo=nodiff --mkidxz -s %s/" % \
(string.join(b.files), rpm_cache_dir, rpm_cache_dir),
logfile = b.logfile, user = "******"
)
else:
ll("test-build: not copying to " + rpm_cache_dir)
ll("Begin-PLD-Builder-Info")
if "upgrade" in r.flags:
b.upgraded = install.upgrade_from_batch(r, b)
else:
ll("not upgrading")
ll("End-PLD-Builder-Info")
for f in b.files:
local = r.tmp_dir + os.path.basename(f)
chroot.cp(f, outfile = local, rm = True)
ftp.add(local)
# cleanup all remains from this build
chroot.run("""
set -ex;
rm -rf %(topdir)s;
""" % {
'topdir' : b._topdir,
}, logfile = b.logfile)
def uploadinfo(b):
c="file:SRPMS:%s\n" % b.src_rpm
for f in b.files:
c=c + "file:ARCH:%s\n" % os.path.basename(f)
c=c + "END\n"
return c
if config.gen_upinfo and b.files != [] and 'test-build' not in r.flags:
fname = r.tmp_dir + b.src_rpm + ".uploadinfo"
f = open(fname, "w")
f.write(uploadinfo(b))
f.close()
ftp.add(fname, "uploadinfo")
status.pop()
return res
