def upgrade_from_batch(r, b):
f = chroot.popen("rpm --test -F %s 2>&1" % string.join(b.files), user = "******")
killset = {}
rx = re.compile(r' \(installed\) (?P<name>[^\s]+)-[^-]+-[^-]+$')
for l in f.xreadlines():
m = rx.search(l)
if m: killset[m.group('name')] = 1
f.close()
if len(killset) != 0:
err = close_killset(killset)
if err != "":
util.append_to(b.logfile, err)
log.notice("cannot upgrade rpms")
return False
k = string.join(killset.keys())
if True:
b.log_line("upgrade requires removal of %s" % k)
res = chroot.run("rpm -e %s" % k, logfile = b.logfile, user = "******")
if res != 0:
b.log_line("package removal failed")
return False
else:
b.log_line("packages removed sucessfuly")
else:
b.log_line("upgrade would need removal of %s" % k)
return False
b.log_line("upgrading packages")
logbuf = StringIO.StringIO()
res = chroot.run("rpm -Fvh %s" % string.join(b.files), user = "******", logfile = b.logfile)
if res != 0:
b.log_line("package upgrade failed")
logbuf.close()
return False
logbuf.close()
return True
def send_rpmqa():
tmp = path.build_dir + '/' + util.uuid() + '/'
os.mkdir(tmp)
log = tmp + config.rpmqa_filename
open(log, 'a').write("Query done at: %s\n" % datetime.datetime.now().isoformat(' '))
chroot.run("rpm -qa|sort", logfile=log)
os.chmod(log,0644)
ftp.init(rpmqa=True)
ftp.add(log)
ftp.flush()
os.unlink(log)
os.rmdir(tmp)
def send_rpmqa():
tmp = path.build_dir + '/' + util.uuid() + '/'
os.mkdir(tmp)
log = tmp + config.rpmqa_filename
open(log, 'a').write("Query done at: %s\n" %
datetime.datetime.now().isoformat(' '))
chroot.run("rpm -qa|sort", logfile=log)
os.chmod(log, 0644)
ftp.init(rpmqa=True)
ftp.add(log)
ftp.flush()
os.unlink(log)
os.rmdir(tmp)
def run_command(batch):
# we want to keep "skip" in queue.html
command = batch.command
# rewrite special "skip:BUILD_ID into touch
if command[:5] == "skip:":
c = ""
for id in command[5:].split(','):
if os.path.isdir(path.srpms_dir + '/' + id):
c = c + "echo skip:%s;\n" % (id)
c = c + "touch %s/%s/skipme;\n" % (path.srpms_dir, id)
else:
c = c + "echo %s is not valid build-id;\n" % (id)
command = c
if "no-chroot" in batch.command_flags:
# TODO: the append here by shell hack should be solved in python
c = "(%s) >> %s 2>&1" % (command, batch.logfile)
f = os.popen(c)
for l in f.xreadlines():
pass
r = f.close()
if r == None:
return 0
else:
return r
else:
user = "******"
if "as-builder" in batch.command_flags:
user = "******"
return chroot.run(command, logfile = batch.logfile, user = user)
def run_command(batch):
# we want to keep "skip" in queue.html
command = batch.command
# rewrite special "skip:BUILD_ID into touch
if command[:5] == "skip:":
c = ""
for id in command[5:].split(','):
if os.path.isdir(path.srpms_dir + '/' + id):
c = c + "echo skip:%s;\n" % (id)
c = c + "touch %s/%s/skipme;\n" % (path.srpms_dir, id)
else:
c = c + "echo %s is not valid build-id;\n" % (id)
command = c
if "no-chroot" in batch.command_flags:
# TODO: the append here by shell hack should be solved in python
c = "(%s) >> %s 2>&1" % (command, batch.logfile)
f = os.popen(c)
for l in f.xreadlines():
pass
r = f.close()
if r == None:
return 0
else:
return r
else:
user = "******"
if "as-builder" in batch.command_flags:
user = None
return chroot.run(command, logfile=batch.logfile, user=user)
def handle_bin():
send_rpmqa()
f=chroot.popen("""ls -l --time-style +%s /spools/ready""", 'root')
rmpkgs=[]
curtime=time.time()
for i in f:
if i[-4:-1]!='rpm':
continue
tmp=i.split()
mtime=int(tmp[5])
pkgname=tmp[6]
if curtime - mtime > config.max_keep_time:
rmpkgs.append(pkgname)
i=0
while rmpkgs[i:i+1000]:
chroot.run("cd /spools/ready; rm -f %s" % ' '.join(rmpkgs[i:i+1000]), 'root')
i=i+1000
f.close()
chroot.run("poldek --mo=nodiff --mkidxz -s /spools/ready")
def handle_bin():
send_rpmqa()
f = chroot.popen("""ls -l --time-style +%s /spools/ready""", 'root')
rmpkgs = []
curtime = time.time()
for i in f:
if i[-4:-1] != 'rpm':
continue
tmp = i.split()
mtime = int(tmp[5])
pkgname = tmp[6]
if curtime - mtime > config.max_keep_time:
rmpkgs.append(pkgname)
i = 0
while rmpkgs[i:i + 1000]:
chroot.run("cd /spools/ready; rm -f %s" % ' '.join(rmpkgs[i:i + 1000]),
'root')
i = i + 1000
f.close()
chroot.run("poldek --mo=nodiff --mkidxz -s /spools/ready")
def build_srpm(r, b):
if len(b.spec) <= len('.spec'):
# should not really get here
util.append_to(b.logfile,
"error: No .spec given but build src.rpm wanted")
return "FAIL"
status.push("building %s" % b.spec)
# messagebus.notify(topic="build_srpm.start", spec=b.spec, flags=r.flags, batch=b, request=r)
b.src_rpm = ""
builder_opts = "-nu -nm --nodeps --http --define \'_pld_builder 1\'"
if ("test-build" in r.flags):
tag_test = ""
else:
tag_test = " -Tp %s -tt" % (config.tag_prefixes[0], )
cmd = (
"cd rpm/packages; nice -n %s ./builder %s -bs %s -r %s %s %s %s %s 2>&1"
% (config.nice, builder_opts, b.bconds_string(), b.branch, tag_test,
b.kernel_string(), b.defines_string(), b.spec))
util.append_to(b.logfile, "request from: %s" % r.requester)
util.append_to(b.logfile, "started at: %s" % time.asctime())
util.append_to(b.logfile, "building SRPM using: %s\n" % cmd)
res = chroot.run(cmd, logfile=b.logfile)
util.append_to(b.logfile, "exit status %d" % res)
files = util.collect_files(b.logfile)
if len(files) > 0:
if len(files) > 1:
util.append_to(b.logfile,
"error: More than one file produced: %s" % files)
res = "FAIL_TOOMANYFILES"
last = files[len(files) - 1]
b.src_rpm_file = last
b.src_rpm = os.path.basename(last)
r.chroot_files.extend(files)
else:
util.append_to(b.logfile, "error: No files produced.")
res = "FAIL"
if res == 0 and not "test-build" in r.flags:
for pref in config.tag_prefixes:
util.append_to(b.logfile, "Tagging with prefix: %s" % pref)
res = chroot.run("cd rpm/packages; ./builder -bs %s -r %s -Tp %s -Tv %s %s" % \
(b.bconds_string(), b.branch, pref, b.defines_string(), b.spec), logfile = b.logfile)
if res == 0:
transfer_file(r, b)
packagename = b.spec[:-5]
packagedir = "rpm/packages/%s" % packagename
chroot.run("rpm/packages/builder -m %s" % \
(b.spec,), logfile = b.logfile)
chroot.run("rm -rf %s" % packagedir, logfile=b.logfile)
status.pop()
if res:
res = "FAIL"
# messagebus.notify(topic="build_srpm.finish", spec=b.spec)
return res
def prepare_env(logfile = None):
chroot.run("""
test ! -f /proc/uptime && mount /proc 2>/dev/null
test ! -c /dev/full && rm -f /dev/full && mknod -m 666 /dev/full c 1 7
test ! -c /dev/null && rm -f /dev/null && mknod -m 666 /dev/null c 1 3
test ! -c /dev/random && rm -f /dev/random && mknod -m 644 /dev/random c 1 8
test ! -c /dev/urandom && rm -f /dev/urandom && mknod -m 644 /dev/urandom c 1 9
test ! -c /dev/zero && rm -f /dev/zero && mknod -m 666 /dev/zero c 1 5
# need entry for "/" in mtab, for diskspace() to work in rpm
[ -z $(awk '$2 == "/" {print $1; exit}' /etc/mtab) ] && mount -f -t rootfs rootfs /
# make neccessary files readable for builder user
# TODO: see if they really aren't readable for builder
for db in Packages Name Basenames Providename Pubkeys; do
db=/var/lib/rpm/$db
test -f $db && chmod a+r $db
done
# try to limit network access for builder account
/bin/setfacl -m u:builder:--- /etc/resolv.conf
""", 'root', logfile = logfile)
def prepare_env(logfile = None):
chroot.run("""
test ! -f /proc/uptime && mount /proc 2>/dev/null
test ! -c /dev/full && rm -f /dev/full && mknod -m 666 /dev/full c 1 7
test ! -c /dev/null && rm -f /dev/null && mknod -m 666 /dev/null c 1 3
test ! -c /dev/random && rm -f /dev/random && mknod -m 644 /dev/random c 1 8
test ! -c /dev/urandom && rm -f /dev/urandom && mknod -m 644 /dev/urandom c 1 9
test ! -c /dev/zero && rm -f /dev/zero && mknod -m 666 /dev/zero c 1 5
# need entry for "/" in mtab, for diskspace() to work in rpm
[ -z $(awk '$2 == "/" {print $1; exit}' /etc/mtab) ] && mount -f -t rootfs rootfs /
# make neccessary files readable for builder user
# TODO: see if they really aren't readable for builder
for db in Packages Name Basenames Providename Pubkeys; do
db=/var/lib/rpm/$db
test -f $db && chmod a+r $db
done
# try to limit network access for builder account
/bin/setfacl -m u:builder:--- /etc/resolv.conf
""", 'root', logfile = logfile)
def upgrade_from_batch(r, b):
f = chroot.popen("rpm --test -F %s 2>&1" % string.join(b.files),
user="******")
killset = {}
rx = re.compile(r' \(installed\) (?P<name>[^\s]+)-[^-]+-[^-]+$')
for l in f.xreadlines():
m = rx.search(l)
if m: killset[m.group('name')] = 1
f.close()
if len(killset) != 0:
err = close_killset(killset)
if err != "":
util.append_to(b.logfile, err)
log.notice("cannot upgrade rpms")
return False
k = string.join(killset.keys())
if True:
b.log_line("upgrade requires removal of %s" % k)
res = chroot.run("rpm -e %s" % k, logfile=b.logfile, user="******")
if res != 0:
b.log_line("package removal failed")
return False
else:
b.log_line("packages removed sucessfuly")
else:
b.log_line("upgrade would need removal of %s" % k)
return False
b.log_line("upgrading packages")
logbuf = StringIO.StringIO()
res = chroot.run("rpm -Fvh %s" % string.join(b.files),
user="******",
logfile=b.logfile)
if res != 0:
b.log_line("package upgrade failed")
logbuf.close()
return False
logbuf.close()
return True
def uninstall(conflicting, b):
b.log_line("uninstalling conflicting packages")
err = close_killset(conflicting)
if err != "":
util.append_to(b.logfile, err)
b.log_line("error: conflicting packages uninstallation failed")
return False
else:
for k in conflicting.keys():
b.log_line("removing %s" % k)
res = chroot.run("poldek --noask --erase %s" % k, logfile = b.logfile, user = "******")
if res != 0:
b.log_line("package %s removal failed" % k)
return True
def uninstall(conflicting, b):
b.log_line("uninstalling conflicting packages")
err = close_killset(conflicting)
if err != "":
util.append_to(b.logfile, err)
b.log_line("error: conflicting packages uninstallation failed")
return False
else:
for k in conflicting.keys():
b.log_line("removing %s" % k)
res = chroot.run("poldek --noask --erase %s" % k,
logfile=b.logfile,
user="******")
if res != 0:
b.log_line("package %s removal failed" % k)
return True
def build_srpm(r, b):
if len(b.spec) <= len('.spec'):
# should not really get here
util.append_to(b.logfile, "error: No .spec given but build src.rpm wanted")
return "FAIL"
status.push("building %s" % b.spec)
b.src_rpm = ""
builder_opts = "-nu -nm --nodeps --http"
if ("test-build" in r.flags):
tag_test=""
else:
tag_test=" -Tp %s -tt" % (config.tag_prefixes[0],)
cmd = ("cd rpm/packages; nice -n %s ./builder %s -bs %s -r %s %s %s %s %s 2>&1" %
(config.nice, builder_opts, b.bconds_string(), b.branch,
tag_test, b.kernel_string(), b.defines_string(), b.spec))
util.append_to(b.logfile, "request from: %s" % r.requester)
util.append_to(b.logfile, "started at: %s" % time.asctime())
util.append_to(b.logfile, "building SRPM using: %s\n" % cmd)
res = chroot.run(cmd, logfile = b.logfile)
util.append_to(b.logfile, "exit status %d" % res)
files = util.collect_files(b.logfile)
if len(files) > 0:
if len(files) > 1:
util.append_to(b.logfile, "error: More than one file produced: %s" % files)
res = "FAIL_TOOMANYFILES"
last = files[len(files) - 1]
b.src_rpm_file = last
b.src_rpm = os.path.basename(last)
r.chroot_files.extend(files)
else:
util.append_to(b.logfile, "error: No files produced.")
res = "FAIL"
if res == 0 and not "test-build" in r.flags:
for pref in config.tag_prefixes:
util.append_to(b.logfile, "Tagging with prefix: %s" % pref)
res = chroot.run("cd rpm/packages; ./builder -r %s -Tp %s -Tv %s" % \
(b.branch, pref, b.spec), logfile = b.logfile)
if res == 0:
transfer_file(r, b)
packagename = b.spec[:-5]
packagedir = "rpm/packages/%s" % packagename
chroot.run("rpm/packages/builder -m %s" % \
(b.spec,), logfile = b.logfile)
chroot.run("rm -rf %s" % packagedir, logfile = b.logfile)
status.pop()
if res:
res = "FAIL"
return res
def build_all(r, build_fnc):
status.email = r.requester_email
notify.begin(r)
tmp = path.build_dir + '/' + util.uuid() + "/"
r.tmp_dir = tmp
os.mkdir(tmp)
atexit.register(util.clean_tmp, tmp)
log.notice("started processing %s" % r.id)
r.chroot_files = []
r.some_ok = 0
for batch in r.batches:
can_build = 1
failed_dep = ""
for dep in batch.depends_on:
if dep.build_failed:
can_build = 0
failed_dep = dep.spec
if batch.is_command() and can_build:
batch.logfile = tmp + "command"
if config.builder in batch.builders:
log.notice("running %s" % batch.command)
stopwatch.start()
batch.build_failed = run_command(batch)
if batch.build_failed:
log.notice("running %s FAILED" % batch.command)
notify.add_batch(batch, "FAIL")
else:
r.some_ok = 1
log.notice("running %s OK" % batch.command)
notify.add_batch(batch, "OK")
batch.build_time = stopwatch.stop()
report.add_pld_builder_info(batch)
buildlogs.add(batch.logfile,
failed=batch.build_failed,
id=r.id)
else:
log.notice("not running command, not for me.")
batch.build_failed = 0
batch.log_line("queued command %s for other builders" %
batch.command)
r.some_ok = 1
buildlogs.add(batch.logfile,
failed=batch.build_failed,
id=r.id)
elif can_build:
log.notice("building %s" % batch.spec)
stopwatch.start()
batch.logfile = tmp + batch.spec + ".log"
batch.gb_id = r.id
batch.requester = r.requester
batch.requester_email = r.requester_email
batch.build_failed = build_fnc(r, batch)
if batch.build_failed:
log.notice("building %s FAILED (%s)" %
(batch.spec, batch.build_failed))
notify.add_batch(batch, batch.build_failed)
else:
r.some_ok = 1
log.notice("building %s OK" % (batch.spec))
notify.add_batch(batch, "OK")
batch.build_time = stopwatch.stop()
report.add_pld_builder_info(batch)
buildlogs.add(batch.logfile, failed=batch.build_failed, id=r.id)
else:
batch.build_failed = 1
batch.skip_reason = "SKIPED [%s failed]" % failed_dep
batch.logfile = None
batch.build_time = ""
log.notice("building %s %s" % (batch.spec, batch.skip_reason))
notify.add_batch(batch, "SKIP")
buildlogs.flush()
chroot.run("rm -f %s" % string.join(r.chroot_files))
def install_br(r, b):
def get_missing_br(r, b):
# ignore internal rpm dependencies, see lib/rpmns.c for list
ignore_br = re.compile(
r'^\s*(rpmlib|cpuinfo|getconf|uname|soname|user|group|mounted|diskspace|digest|gnupg|macro|envvar|running|sanitycheck|vcheck|signature|verify|exists|executable|readable|writable)\(.*'
)
tmpdir = b.tmpdir()
cmd = "set -e; TMPDIR=%(tmpdir)s rpmbuild --nobuild %(rpmdefs)s %(topdir)s/%(spec)s 2>&1" % {
'tmpdir': tmpdir,
'topdir': b._topdir,
'rpmdefs': b.rpmbuild_opts(),
'spec': b.spec,
}
f = chroot.popen(cmd)
rx = re.compile(r"^\s*(?P<name>[^\s]+) .*is needed by")
needed = {}
b.log_line("checking BR")
for l in f.xreadlines():
b.log_line("rpm: %s" % l.rstrip())
m = rx.search(l)
if m and not ignore_br.match(l):
needed[m.group('name')] = 1
f.close()
return needed
needed = get_missing_br(r, b)
if len(needed) == 0:
b.log_line("no BR needed")
return True
nbr = ""
for bre in needed.keys():
nbr = nbr + " " + re.escape(bre)
br = string.strip(nbr)
b.log_line("updating poldek cache...")
chroot.run("poldek --up --upa", user="******", logfile=b.logfile)
# check conflicts in BRed packages
b.log_line("checking conflicting packages in BRed packages")
f = chroot.popen(
"poldek --test --test --noask --caplookup -Q -v --upgrade %s" % br,
user="******")
# phonon-devel-4.3.1-1.i686 conflicts with qt4-phonon-devel-4.5.0-6.i686
# jdbc-stdext >= 2.0 is required by installed java-struts-1.3.10-1.noarch
# jmx is needed by (installed) java-commons-modeler-2.0-1.noarch
rx = re.compile(
r".*(conflicts with|is required by|is needed by)( installed| \(installed\)|) (?P<name>[^\s]+)-[^-]+-[^-]+($| .*)"
)
conflicting = {}
for l in f.xreadlines():
b.log_line("poldek: %s" % l.rstrip())
m = rx.search(l)
if m: conflicting[m.group('name')] = 1
f.close()
if len(conflicting) == 0:
b.log_line("no conflicts found")
else:
if not uninstall(conflicting, b):
return False
# recheck BuildRequires since above uninstallation could remove some required deps
needed = get_missing_br(r, b)
if len(needed) == 0:
b.log_line("no BR needed")
return True
nbr = ""
for bre in needed.keys():
nbr = nbr + " " + re.escape(bre)
br = string.strip(nbr)
b.log_line("installing BR: %s" % br)
res = chroot.run("poldek --noask --caplookup -Q -v --upgrade %s" % br,
user="******",
logfile=b.logfile)
if res != 0:
b.log_line("error: BR installation failed")
return False
return True
def build_rpm(r, b):
packagename = b.get_package_name()
if not packagename:
# should not really get here
b.log_line("error: No .spec not given of malformed: '%s'" % b.spec)
res = "FAIL_INTERNAL"
return res
status.push("building %s (%s)" % (b.spec, packagename))
b.log_line("request from: %s" % r.requester)
if check_skip_build(r, b):
b.log_line("build skipped due to src builder request")
res = "SKIP_REQUESTED"
return res
b.log_line("started at: %s" % time.asctime())
fetch_src(r, b)
b.log_line("installing srpm: %s" % b.src_rpm)
res = chroot.run("""
set -ex;
install -d %(topdir)s/{BUILD,RPMS};
rpm -Uhv --nodeps %(rpmdefs)s %(src_rpm)s;
rm -f %(src_rpm)s;
""" % {
'topdir' : b._topdir,
'rpmdefs' : b.rpmbuild_opts(),
'src_rpm' : b.src_rpm
}, logfile = b.logfile)
b.files = []
tmpdir = b.tmpdir()
if res:
b.log_line("error: installing src rpm failed")
res = "FAIL_SRPM_INSTALL"
else:
prepare_env()
chroot.run("set -x; install -m 700 -d %s" % tmpdir, logfile=b.logfile)
b.default_target(config.arch)
# check for build arch before filling BR
cmd = "set -ex; TMPDIR=%(tmpdir)s exec nice -n %(nice)s " \
"rpmbuild -bp --short-circuit --nodeps %(rpmdefs)s --define 'prep exit 0' %(topdir)s/%(spec)s" % {
'tmpdir': tmpdir,
'nice' : config.nice,
'topdir' : b._topdir,
'rpmdefs' : b.rpmbuild_opts(),
'spec': b.spec,
}
res = chroot.run(cmd, logfile = b.logfile)
if res:
res = "UNSUPP"
b.log_line("error: build arch check (%s) failed" % cmd)
if not res:
if ("no-install-br" not in r.flags) and not install.uninstall_self_conflict(b):
res = "FAIL_DEPS_UNINSTALL"
if ("no-install-br" not in r.flags) and not install.install_br(r, b):
res = "FAIL_DEPS_INSTALL"
if not res:
max_jobs = max(min(int(os.sysconf('SC_NPROCESSORS_ONLN') + 1), config.max_jobs), 1)
if r.max_jobs > 0:
max_jobs = max(min(config.max_jobs, r.max_jobs), 1)
cmd = "set -ex; : build-id: %(r_id)s; TMPDIR=%(tmpdir)s exec nice -n %(nice)s " \
"rpmbuild -bb --define '_smp_mflags -j%(max_jobs)d' %(rpmdefs)s %(topdir)s/%(spec)s" % {
'r_id' : r.id,
'tmpdir': tmpdir,
'nice' : config.nice,
'rpmdefs' : b.rpmbuild_opts(),
'topdir' : b._topdir,
'max_jobs' : max_jobs,
'spec': b.spec,
}
b.log_line("building RPM using: %s" % cmd)
begin_time = time.time()
res = chroot.run(cmd, logfile = b.logfile)
end_time = time.time()
b.log_line("ended at: %s, done in %s" % (time.asctime(), datetime.timedelta(0, end_time - begin_time)))
if res:
res = "FAIL"
files = util.collect_files(b.logfile, basedir = b._topdir)
if len(files) > 0:
r.chroot_files.extend(files)
else:
b.log_line("error: No files produced.")
last_section = util.find_last_section(b.logfile)
if last_section == None:
res = "FAIL"
else:
res = "FAIL_%s" % last_section.upper()
b.files = files
# cleanup tmp and build files
chroot.run("""
set -ex;
chmod -R u+rwX %(topdir)s/BUILD;
rm -rf %(topdir)s/{tmp,BUILD}
""" % {
'topdir' : b._topdir,
}, logfile = b.logfile)
def ll(l):
util.append_to(b.logfile, l)
if b.files != []:
rpm_cache_dir = config.rpm_cache_dir
if "test-build" not in r.flags:
# NOTE: copying to cache dir doesn't mean that build failed, so ignore result
b.log_line("copy rpm files to cache_dir: %s" % rpm_cache_dir)
chroot.run(
"cp -f %s %s && poldek --mo=nodiff --mkidxz -s %s/" % \
(string.join(b.files), rpm_cache_dir, rpm_cache_dir),
logfile = b.logfile, user = "******"
)
else:
ll("test-build: not copying to " + rpm_cache_dir)
ll("Begin-PLD-Builder-Info")
if "upgrade" in r.flags:
b.upgraded = install.upgrade_from_batch(r, b)
else:
ll("not upgrading")
ll("End-PLD-Builder-Info")
for f in b.files:
local = r.tmp_dir + os.path.basename(f)
chroot.cp(f, outfile = local, rm = True)
ftp.add(local)
# cleanup all remains from this build
chroot.run("""
set -ex;
rm -rf %(topdir)s;
""" % {
'topdir' : b._topdir,
}, logfile = b.logfile)
def uploadinfo(b):
c="file:SRPMS:%s\n" % b.src_rpm
for f in b.files:
c=c + "file:ARCH:%s\n" % os.path.basename(f)
c=c + "END\n"
return c
if config.gen_upinfo and b.files != [] and 'test-build' not in r.flags:
fname = r.tmp_dir + b.src_rpm + ".uploadinfo"
f = open(fname, "w")
f.write(uploadinfo(b))
f.close()
ftp.add(fname, "uploadinfo")
status.pop()
return res
def build_all(r, build_fnc):
status.email = r.requester_email
notify.begin(r)
tmp = path.build_dir + '/' + util.uuid() + "/"
r.tmp_dir = tmp
os.mkdir(tmp)
atexit.register(util.clean_tmp, tmp)
log.notice("started processing %s" % r.id)
r.chroot_files = []
r.some_ok = 0
for batch in r.batches:
can_build = 1
failed_dep = ""
for dep in batch.depends_on:
if dep.build_failed:
can_build = 0
failed_dep = dep.spec
if batch.is_command() and can_build:
batch.logfile = tmp + "command"
if config.builder in batch.builders:
log.notice("running %s" % batch.command)
stopwatch.start()
batch.build_failed = run_command(batch)
if batch.build_failed:
log.notice("running %s FAILED" % batch.command)
notify.add_batch(batch, "FAIL")
else:
r.some_ok = 1
log.notice("running %s OK" % batch.command)
notify.add_batch(batch, "OK")
batch.build_time = stopwatch.stop()
report.add_pld_builder_info(batch)
buildlogs.add(batch.logfile, failed = batch.build_failed, id=r.id)
else:
log.notice("not running command, not for me.")
batch.build_failed = 0
batch.log_line("queued command %s for other builders" % batch.command)
r.some_ok = 1
buildlogs.add(batch.logfile, failed = batch.build_failed, id=r.id)
elif can_build:
log.notice("building %s" % batch.spec)
stopwatch.start()
batch.logfile = tmp + batch.spec + ".log"
batch.gb_id=r.id
batch.requester=r.requester
batch.requester_email=r.requester_email
batch.build_failed = build_fnc(r, batch)
if batch.build_failed:
log.notice("building %s FAILED (%s)" % (batch.spec, batch.build_failed))
notify.add_batch(batch, batch.build_failed)
else:
r.some_ok = 1
log.notice("building %s OK" % (batch.spec))
notify.add_batch(batch, "OK")
batch.build_time = stopwatch.stop()
report.add_pld_builder_info(batch)
buildlogs.add(batch.logfile, failed = batch.build_failed, id=r.id)
else:
batch.build_failed = 1
batch.skip_reason = "SKIPED [%s failed]" % failed_dep
batch.logfile = None
batch.build_time = ""
log.notice("building %s %s" % (batch.spec, batch.skip_reason))
notify.add_batch(batch, "SKIP")
buildlogs.flush()
chroot.run("rm -f %s" % string.join(r.chroot_files))
def install_br(r, b):
def get_missing_br(r, b):
# ignore internal rpm dependencies, see lib/rpmns.c for list
ignore_br = re.compile(r'^\s*(rpmlib|cpuinfo|getconf|uname|soname|user|group|mounted|diskspace|digest|gnupg|macro|envvar|running|sanitycheck|vcheck|signature|verify|exists|executable|readable|writable)\(.*')
tmpdir = b.tmpdir()
cmd = "set -e; TMPDIR=%(tmpdir)s rpmbuild --nobuild %(rpmdefs)s %(topdir)s/%(spec)s 2>&1" % {
'tmpdir': tmpdir,
'topdir' : b._topdir,
'rpmdefs' : b.rpmbuild_opts(),
'spec': b.spec,
}
f = chroot.popen(cmd)
rx = re.compile(r"^\s*(?P<name>[^\s]+) .*is needed by")
needed = {}
b.log_line("checking BR")
for l in f.xreadlines():
b.log_line("rpm: %s" % l.rstrip())
m = rx.search(l)
if m and not ignore_br.match(l):
needed[m.group('name')] = 1
f.close()
return needed
needed = get_missing_br(r, b);
if len(needed) == 0:
b.log_line("no BR needed")
return True
nbr = ""
for bre in needed.keys():
nbr = nbr + " " + re.escape(bre)
br = string.strip(nbr)
b.log_line("updating poldek cache...")
chroot.run("poldek --up --upa", user = "******", logfile = b.logfile)
# check conflicts in BRed packages
b.log_line("checking conflicting packages in BRed packages")
f = chroot.popen("poldek --test --test --noask --caplookup -Q -v --upgrade %s" % br, user = "******")
# phonon-devel-4.3.1-1.i686 conflicts with qt4-phonon-devel-4.5.0-6.i686
# jdbc-stdext >= 2.0 is required by installed java-struts-1.3.10-1.noarch
# jmx is needed by (installed) java-commons-modeler-2.0-1.noarch
rx = re.compile(r".*(conflicts with|is required by|is needed by)( installed| \(installed\)|) (?P<name>[^\s]+)-[^-]+-[^-]+($| .*)")
conflicting = {}
for l in f.xreadlines():
b.log_line("poldek: %s" % l.rstrip())
m = rx.search(l)
if m: conflicting[m.group('name')] = 1
f.close()
if len(conflicting) == 0:
b.log_line("no conflicts found")
else:
if not uninstall(conflicting, b):
return False
# recheck BuildRequires since above uninstallation could remove some required deps
needed = get_missing_br(r, b);
if len(needed) == 0:
b.log_line("no BR needed")
return True
nbr = ""
for bre in needed.keys():
nbr = nbr + " " + re.escape(bre)
br = string.strip(nbr)
b.log_line("installing BR: %s" % br)
res = chroot.run("poldek --noask --caplookup -Q -v --upgrade %s" % br,
user = "******",
logfile = b.logfile)
if res != 0:
b.log_line("error: BR installation failed")
return False
return True
def build_rpm(r, b):
packagename = b.get_package_name()
if not packagename:
# should not really get here
b.log_line("error: No .spec not given of malformed: '%s'" % b.spec)
res = "FAIL_INTERNAL"
return res
status.push("building %s (%s)" % (b.spec, packagename))
b.log_line("request from: %s" % r.requester)
if check_skip_build(r, b):
b.log_line("build skipped due to src builder request")
res = "SKIP_REQUESTED"
return res
b.log_line("started at: %s" % time.asctime())
fetch_src(r, b)
b.log_line("installing srpm: %s" % b.src_rpm)
res = chroot.run("""
set -ex;
install -d %(topdir)s/{BUILD,RPMS};
rpm -Uhv --nodeps %(rpmdefs)s %(src_rpm)s;
rm -f %(src_rpm)s;
""" % {
'topdir': b._topdir,
'rpmdefs': b.rpmbuild_opts(),
'src_rpm': b.src_rpm
},
logfile=b.logfile)
b.files = []
tmpdir = b.tmpdir()
if res:
b.log_line("error: installing src rpm failed")
res = "FAIL_SRPM_INSTALL"
else:
prepare_env()
chroot.run("set -x; install -m 700 -d %s" % tmpdir, logfile=b.logfile)
b.default_target(config.arch)
# check for build arch before filling BR
cmd = "set -ex; TMPDIR=%(tmpdir)s exec nice -n %(nice)s " \
"rpmbuild -bp --short-circuit --nodeps %(rpmdefs)s --define 'prep exit 0' %(topdir)s/%(spec)s" % {
'tmpdir': tmpdir,
'nice' : config.nice,
'topdir' : b._topdir,
'rpmdefs' : b.rpmbuild_opts(),
'spec': b.spec,
}
res = chroot.run(cmd, logfile=b.logfile)
if res:
res = "UNSUPP"
b.log_line("error: build arch check (%s) failed" % cmd)
if not res:
if ("no-install-br"
not in r.flags) and not install.uninstall_self_conflict(b):
res = "FAIL_DEPS_UNINSTALL"
if ("no-install-br"
not in r.flags) and not install.install_br(r, b):
res = "FAIL_DEPS_INSTALL"
if not res:
max_jobs = max(
min(int(os.sysconf('SC_NPROCESSORS_ONLN') + 1),
config.max_jobs), 1)
if r.max_jobs > 0:
max_jobs = max(min(config.max_jobs, r.max_jobs), 1)
cmd = "set -ex; : build-id: %(r_id)s; TMPDIR=%(tmpdir)s exec nice -n %(nice)s " \
"rpmbuild -bb --define '_smp_mflags -j%(max_jobs)d' %(rpmdefs)s %(topdir)s/%(spec)s" % {
'r_id' : r.id,
'tmpdir': tmpdir,
'nice' : config.nice,
'rpmdefs' : b.rpmbuild_opts(),
'topdir' : b._topdir,
'max_jobs' : max_jobs,
'spec': b.spec,
}
b.log_line("building RPM using: %s" % cmd)
begin_time = time.time()
res = chroot.run(cmd, logfile=b.logfile)
end_time = time.time()
b.log_line("ended at: %s, done in %s" %
(time.asctime(),
datetime.timedelta(0, end_time - begin_time)))
if res:
res = "FAIL"
files = util.collect_files(b.logfile, basedir=b._topdir)
if len(files) > 0:
r.chroot_files.extend(files)
else:
b.log_line("error: No files produced.")
last_section = util.find_last_section(b.logfile)
if last_section == None:
res = "FAIL"
else:
res = "FAIL_%s" % last_section.upper()
b.files = files
# cleanup tmp and build files
chroot.run("""
set -ex;
chmod -R u+rwX %(topdir)s/BUILD;
rm -rf %(topdir)s/{tmp,BUILD}
""" % {
'topdir': b._topdir,
},
logfile=b.logfile)
def ll(l):
util.append_to(b.logfile, l)
if b.files != []:
rpm_cache_dir = config.rpm_cache_dir
if "test-build" not in r.flags:
# NOTE: copying to cache dir doesn't mean that build failed, so ignore result
b.log_line("copy rpm files to cache_dir: %s" % rpm_cache_dir)
chroot.run(
"cp -f %s %s && poldek --mo=nodiff --mkidxz -s %s/" % \
(string.join(b.files), rpm_cache_dir, rpm_cache_dir),
logfile = b.logfile, user = "******"
)
else:
ll("test-build: not copying to " + rpm_cache_dir)
ll("Begin-PLD-Builder-Info")
if "upgrade" in r.flags:
b.upgraded = install.upgrade_from_batch(r, b)
else:
ll("not upgrading")
ll("End-PLD-Builder-Info")
for f in b.files:
local = r.tmp_dir + os.path.basename(f)
chroot.cp(f, outfile=local, rm=True)
ftp.add(local)
# cleanup all remains from this build
chroot.run("""
set -ex;
rm -rf %(topdir)s;
""" % {
'topdir': b._topdir,
},
logfile=b.logfile)
def uploadinfo(b):
c = "file:SRPMS:%s\n" % b.src_rpm
for f in b.files:
c = c + "file:ARCH:%s\n" % os.path.basename(f)
c = c + "END\n"
return c
if config.gen_upinfo and b.files != [] and 'test-build' not in r.flags:
fname = r.tmp_dir + b.src_rpm + ".uploadinfo"
f = open(fname, "w")
f.write(uploadinfo(b))
f.close()
ftp.add(fname, "uploadinfo")
status.pop()
return res