def main():
init_conf("src")
if lock("building-srpm", non_block = 1) == None:
return
while True:
status.push("srpm: processing queue")
q = B_Queue(path.queue_file)
if not q.lock(1):
status.pop()
return
q.read()
if q.requests == []:
q.unlock()
status.pop()
return
r = pick_request(q)
q.write()
q.unlock()
status.pop()
status.push("srpm: handling request from %s" % r.requester)
handle_request(r)
status.pop()
def store_binary_request(r):
new_b = []
for b in r.batches:
if not b.build_failed: new_b.append(b)
if new_b == []:
return
r.batches = new_b
# store new queue and max_req_no for binary builders
num = int(string.strip(open(path.max_req_no_file, "r").read())) + 1
r.no = num
q = B_Queue(path.req_queue_file)
q.lock(0)
q.read()
q.add(r)
q.write()
q.dump(path.queue_stats_file)
q.dump_html(path.queue_html_stats_file)
q.write_signed(path.req_queue_signed_file)
q.unlock()
(fdno,
tmpfname) = tempfile.mkstemp(dir=os.path.dirname(path.max_req_no_file))
cnt_f = os.fdopen(fdno, "w")
cnt_f.seek(0)
cnt_f.write("%d\n" % num)
cnt_f.flush()
os.fsync(cnt_f.fileno())
cnt_f.close()
os.chmod(tmpfname, 0644)
os.rename(tmpfname, path.max_req_no_file)
def main():
init_conf("src")
if lock("building-srpm", non_block=1) == None:
return
while True:
status.push("srpm: processing queue")
q = B_Queue(path.queue_file)
if not q.lock(1):
status.pop()
return
q.read()
if q.requests == []:
q.unlock()
status.pop()
return
r = pick_request(q)
q.write()
q.unlock()
status.pop()
status.push("srpm: handling request from %s" % r.requester)
handle_request(r)
status.pop()
def store_binary_request(r):
new_b = []
for b in r.batches:
if not b.build_failed: new_b.append(b)
if new_b == []:
return
r.batches = new_b
# store new queue and max_req_no for binary builders
num = int(string.strip(open(path.max_req_no_file, "r").read())) + 1
r.no = num
q = B_Queue(path.req_queue_file)
q.lock(0)
q.read()
q.add(r)
q.write()
q.dump(path.queue_stats_file)
q.dump_html(path.queue_html_stats_file)
q.write_signed(path.req_queue_signed_file)
q.unlock()
(fdno, tmpfname) = tempfile.mkstemp(dir=os.path.dirname(path.max_req_no_file))
cnt_f = os.fdopen(fdno, "w")
cnt_f.seek(0)
cnt_f.write("%d\n" % num)
cnt_f.flush()
os.fsync(cnt_f.fileno())
cnt_f.close()
os.chmod(tmpfname, 0644)
os.rename(tmpfname, path.max_req_no_file)
def main_for(builder):
msg = ""
init_conf(builder)
q = B_Queue(path.queue_file + "-" + config.builder)
q.lock(0)
q.read()
if q.requests == []:
q.unlock()
return
req = pick_request(q)
q.unlock()
# high priority tasks have priority < 0, normal tasks >= 0
if req.priority >= 0:
# allow only one build in given builder at once
if not lock.lock("building-rpm-for-%s" % config.builder, non_block=1):
return
# don't kill server
check_load()
# not more then job_slots builds at once
locked = 0
for slot in range(config.job_slots):
if lock.lock("building-rpm-slot-%d" % slot, non_block=1):
locked = 1
break
if not locked:
return
# record fact that we got lock for this builder, load balancer
# will use it for fair-queuing
l = lock.lock("got-lock")
f = open(path.got_lock_file, "a")
f.write(config.builder + "\n")
f.close()
l.close()
else:
msg = "HIGH PRIORITY: "
msg += "handling request %s (%d) for %s from %s, priority %s" \
% (req.id, req.no, config.builder, req.requester, req.priority)
log.notice(msg)
status.push(msg)
handle_request(req)
status.pop()
def otherreqs(r):
if r.no == req.no:
return False
else:
return True
q = B_Queue(path.queue_file + "-" + config.builder)
q.lock(0)
q.read()
previouslen = len(q.requests)
q.requests = filter(otherreqs, q.requests)
if len(q.requests) < previouslen:
q.write()
q.unlock()
def handle_notification(r, user):
if not user.can_do("notify", r.builder):
log.alert("user %s is not allowed to notify:%s" %
(user.login, r.builder))
q = B_Queue(path.req_queue_file)
q.lock(0)
q.read()
not_fin = filter(lambda (r): not r.is_done(), q.requests)
r.apply_to(q)
for r in not_fin:
if r.is_done():
util.clean_tmp(path.srpms_dir + '/' + r.id)
now = time.time()
def leave_it(r):
# for ,,done'' set timeout to 4d
if r.is_done() and r.time + 4 * 24 * 60 * 60 < now:
return False
# and for not ,,done'' set it to 20d
if r.time + 20 * 24 * 60 * 60 < now:
util.clean_tmp(path.srpms_dir + '/' + r.id)
return False
return True
q.requests = filter(leave_it, q.requests)
q.write()
q.dump(path.queue_stats_file)
q.dump_html(path.queue_html_stats_file)
q.write_signed(path.req_queue_signed_file)
q.unlock()
def handle_group(r, user):
lockf = None
def fail_mail(msg):
if len(r.batches) >= 1:
spec = r.batches[0].spec
else:
spec = "None.spec"
log.error("%s: %s" % (spec, msg))
m = Message()
m.set_headers(to=r.requester_email, cc=config.builder_list)
m.set_headers(subject="building %s failed" % spec)
m.write_line(msg)
m.send()
lockf = lock("request")
if check_double_id(r.id):
lockf.close()
return
try:
if (user.change_requester and r.requester):
user = acl.user_by_login(r.requester)
except KeyError:
r.requester += '/' + user.get_login()
else:
r.requester = user.get_login()
r.requester_email = user.mail_to()
for batch in r.batches:
if not user.can_do("src", config.builder, batch.branch):
fail_mail("user %s is not allowed to src:%s:%s" \
% (user.get_login(), config.builder, batch.branch))
lockf.close()
return
if 'test-build' in r.flags and 'upgrade' in r.flags:
fail_mail("it's forbidden to upgrade from a test build")
lockf.close()
return
if "upgrade" in r.flags and not user.can_do("upgrade", config.builder,
batch.branch):
fail_mail("user %s is not allowed to upgrade:%s:%s" \
% (user.get_login(), config.builder, batch.branch))
lockf.close()
return
# src builder handles only special commands
if batch.is_command() and (batch.command in ["git pull"]
or batch.command[:5] == "skip:"
or config.builder in batch.builders):
batch.expand_builders(config.binary_builders +
[config.src_builder])
else:
batch.expand_builders(config.binary_builders)
if not batch.is_command() and config.builder in batch.builders:
batch.builders.remove(config.builder)
for bld in batch.builders:
batch.builders_status[bld] = '?'
batch.builders_status_time[bld] = time.time()
if bld not in config.binary_builders and bld != config.builder:
fail_mail("I (src rpm builder '%s') do not handle binary builder '%s', only '%s'" % \
(config.builder, bld, string.join(config.binary_builders)))
lockf.close()
return
if batch.is_command():
if "no-chroot" in batch.command_flags:
if not user.can_do("command-no-chroot", bld):
fail_mail("user %s is not allowed to command-no-chroot:%s" \
% (user.get_login(), bld))
lockf.close()
return
if not user.can_do("command", bld):
fail_mail("user %s is not allowed to command:%s" \
% (user.get_login(), bld))
lockf.close()
return
elif not user.can_do("binary", bld, batch.branch):
pkg = batch.spec
if pkg.endswith(".spec"):
pkg = pkg[:-5]
if not user.can_do("binary-" + pkg, bld, batch.branch):
fail_mail("user %s is not allowed to binary-%s:%s:%s" \
% (user.get_login(), pkg, bld, batch.branch))
lockf.close()
return
if not "test-build" in r.flags and not user.can_do(
"ready", bld, batch.branch):
fail_mail("user %s is not allowed to send ready builds (ready:%s:%s)" \
% (user.get_login(), bld, batch.branch))
lockf.close()
return
pkg = batch.spec
if pkg.endswith(".spec"):
pkg = pkg[:-5]
if not "test-build" in r.flags and blacklist.package(pkg):
fail_mail(
"package '%s' is blacklisted, only test-builds allowed" %
pkg)
lockf.close()
return
r.priority = user.check_priority(r.priority, config.builder)
r.time = time.time()
log.notice("queued %s from %s" % (r.id, user.get_login()))
q = B_Queue(path.queue_file)
q.lock(0)
q.read()
q.add(r)
q.write()
q.unlock()
lockf.close()
def handle_reqs(builder, reqs):
qpath = path.queue_file + "-" + builder
if not os.access(qpath, os.F_OK):
util.append_to(qpath, "<queue/>\n")
q = B_Queue(qpath)
q.lock(0)
q.read()
for r in reqs:
if r.kind != 'group':
raise Exception, 'handle_reqs: fatal: huh? %s' % r.kind
need_it = 0
for b in r.batches:
if builder in b.builders:
need_it = 1
if need_it:
log.notice("queued %s (%d) for %s" % (r.id, r.no, builder))
q.add(r)
q.write()
q.unlock()
def handle_group(r, user):
lockf = None
def fail_mail(msg):
if len(r.batches) >= 1:
spec = r.batches[0].spec
else:
spec = "None.spec"
log.error("%s: %s" % (spec, msg))
m = Message()
m.set_headers(to = r.requester_email, cc = config.builder_list)
m.set_headers(subject = "building %s failed" % spec)
m.write_line(msg)
m.send()
lockf = lock("request")
if check_double_id(r.id):
lockf.close()
return
try:
if (user.change_requester and r.requester):
user = acl.user_by_login(r.requester)
except KeyError:
r.requester += '/' + user.get_login()
else:
r.requester = user.get_login()
r.requester_email = user.mail_to()
for batch in r.batches:
if not user.can_do("src", config.builder, batch.branch):
fail_mail("user %s is not allowed to src:%s:%s" \
% (user.get_login(), config.builder, batch.branch))
lockf.close()
return
if 'test-build' in r.flags and 'upgrade' in r.flags:
fail_mail("it's forbidden to upgrade from a test build")
lockf.close()
return
if "upgrade" in r.flags and not user.can_do("upgrade", config.builder, batch.branch):
fail_mail("user %s is not allowed to upgrade:%s:%s" \
% (user.get_login(), config.builder, batch.branch))
lockf.close()
return
# src builder handles only special commands
if batch.is_command() and (batch.command in ["git pull"] or batch.command[:5] == "skip:" or config.builder in batch.builders):
batch.expand_builders(config.binary_builders + [config.src_builder])
else:
batch.expand_builders(config.binary_builders)
if not batch.is_command() and config.builder in batch.builders:
batch.builders.remove(config.builder)
for bld in batch.builders:
batch.builders_status[bld] = '?'
batch.builders_status_time[bld] = time.time()
if bld not in config.binary_builders and bld != config.builder:
fail_mail("I (src rpm builder '%s') do not handle binary builder '%s', only '%s'" % \
(config.builder, bld, string.join(config.binary_builders)))
lockf.close()
return
if batch.is_command():
if "no-chroot" in batch.command_flags:
if not user.can_do("command-no-chroot", bld):
fail_mail("user %s is not allowed to command-no-chroot:%s" \
% (user.get_login(), bld))
lockf.close()
return
if not user.can_do("command", bld):
fail_mail("user %s is not allowed to command:%s" \
% (user.get_login(), bld))
lockf.close()
return
elif not user.can_do("binary", bld, batch.branch):
pkg = batch.spec
if pkg.endswith(".spec"):
pkg = pkg[:-5]
if not user.can_do("binary-" + pkg, bld, batch.branch):
fail_mail("user %s is not allowed to binary-%s:%s:%s" \
% (user.get_login(), pkg, bld, batch.branch))
lockf.close()
return
if not "test-build" in r.flags and not user.can_do("ready", bld, batch.branch):
fail_mail("user %s is not allowed to send ready builds (ready:%s:%s)" \
% (user.get_login(), bld, batch.branch))
lockf.close()
return
r.priority = user.check_priority(r.priority,config.builder)
r.time = time.time()
log.notice("queued %s from %s" % (r.id, user.get_login()))
q = B_Queue(path.queue_file)
q.lock(0)
q.read()
q.add(r)
q.write()
q.unlock()
lockf.close()
def handle_notification(r, user):
if not user.can_do("notify", r.builder):
log.alert("user %s is not allowed to notify:%s" % (user.login, r.builder))
q = B_Queue(path.req_queue_file)
q.lock(0)
q.read()
not_fin = filter(lambda (r): not r.is_done(), q.requests)
r.apply_to(q)
for r in not_fin:
if r.is_done():
util.clean_tmp(path.srpms_dir + '/' + r.id)
now = time.time()
def leave_it(r):
# for ,,done'' set timeout to 4d
if r.is_done() and r.time + 4 * 24 * 60 * 60 < now:
return False
# and for not ,,done'' set it to 20d
if r.time + 20 * 24 * 60 * 60 < now:
util.clean_tmp(path.srpms_dir + '/' + r.id)
return False
return True
q.requests = filter(leave_it, q.requests)
q.write()
q.dump(path.queue_stats_file)
q.dump_html(path.queue_html_stats_file)
q.write_signed(path.req_queue_signed_file)
q.unlock()
def handle_reqs(builder, reqs):
qpath = path.queue_file + "-" + builder
if not os.access(qpath, os.F_OK):
util.append_to(qpath, "<queue/>\n")
q = B_Queue(qpath)
q.lock(0)
q.read()
for r in reqs:
if r.kind != 'group':
raise Exception, 'handle_reqs: fatal: huh? %s' % r.kind
need_it = 0
for b in r.batches:
if builder in b.builders:
need_it = 1
if need_it:
log.notice("queued %s (%d) for %s" % (r.id, r.no, builder))
q.add(r)
q.write()
q.unlock()
def main_for(builder):
msg = ""
init_conf(builder)
q = B_Queue(path.queue_file + "-" + config.builder)
q.lock(0)
q.read()
if q.requests == []:
q.unlock()
return
req = pick_request(q)
q.unlock()
# high priority tasks have priority < 0, normal tasks >= 0
if req.priority >= 0:
# allow only one build in given builder at once
if not lock.lock("building-rpm-for-%s" % config.builder, non_block = 1):
return
# don't kill server
check_load()
# not more then job_slots builds at once
locked = 0
for slot in range(config.job_slots):
if lock.lock("building-rpm-slot-%d" % slot, non_block = 1):
locked = 1
break
if not locked:
return
# record fact that we got lock for this builder, load balancer
# will use it for fair-queuing
l = lock.lock("got-lock")
f = open(path.got_lock_file, "a")
f.write(config.builder + "\n")
f.close()
l.close()
else:
msg = "HIGH PRIORITY: "
msg += "handling request %s (%d) for %s from %s, priority %s" \
% (req.id, req.no, config.builder, req.requester, req.priority)
log.notice(msg)
status.push(msg)
handle_request(req)
status.pop()
def otherreqs(r):
if r.no==req.no:
return False
else:
return True
q = B_Queue(path.queue_file + "-" + config.builder)
q.lock(0)
q.read()
previouslen=len(q.requests)
q.requests=filter(otherreqs, q.requests)
if len(q.requests)<previouslen:
q.write()
q.unlock()