def scheduler(arch):
query = "SELECT count(*) " + \
"FROM schedule AS p JOIN sources AS s ON p.package_id=s.id " + \
"WHERE s.architecture='{arch}'"
total = int(query_db(query.format(arch=arch))[0][0])
log.info('==============================================================')
log.info('Currently scheduled packages in all suites on ' + arch + ': ' + str(total))
if total > MAXIMA[arch]:
log.info(str(total) + ' packages already scheduled' +
', only scheduling new versions.')
empty_pkgs = {}
for suite in SUITES:
empty_pkgs[suite] = []
untested, msg_untested = empty_pkgs, ''
new, msg_new = schedule_new_versions(arch, total)
old_ftbfs, msg_old_ftbfs = empty_pkgs, ''
old_depwait, msg_old_depwait = empty_pkgs, ''
old, msg_old = empty_pkgs, ''
four04, msg_404 = empty_pkgs, ''
else:
log.info(str(total) + ' packages already scheduled' +
', scheduling some more...')
untested, msg_untested = schedule_untested_packages(arch, total)
new, msg_new = schedule_new_versions(arch, total+len(untested))
old_ftbfs, msg_old_ftbfs = schedule_old_ftbfs_versions(arch, total+len(untested)+len(new))
old_depwait, msg_old_depwait = schedule_old_depwait_versions(arch, total+len(untested)+len(new)+len(old_ftbfs))
four04, msg_404 = schedule_404_versions(arch, total+len(untested)+len(new)+len(old_ftbfs)+len(old_depwait))
old, msg_old = schedule_old_versions(arch, total+len(untested)+len(new)+len(old_ftbfs)+len(old_depwait)+len(four04))
now_queued_here = {}
# make sure to schedule packages in unstable first
# (but keep the view ordering everywhere else)
priotized_suite_order = ['unstable']
for suite in SUITES:
if suite not in priotized_suite_order:
priotized_suite_order.append(suite)
for suite in priotized_suite_order:
query = "SELECT count(*) " \
"FROM schedule AS p JOIN sources AS s ON p.package_id=s.id " \
"WHERE s.suite='{suite}' AND s.architecture='{arch}'"
query = query.format(suite=suite, arch=arch)
now_queued_here[suite] = int(query_db(query)[0][0]) + \
len(untested[suite]+new[suite]+old[suite])
# schedule packages differently in the queue...
to_be_scheduled = queue_packages({}, untested[suite], datetime.now()+timedelta(minutes=-720))
assert(isinstance(to_be_scheduled, dict))
to_be_scheduled = queue_packages(to_be_scheduled, new[suite], datetime.now()+timedelta(minutes=-1440))
to_be_scheduled = queue_packages(to_be_scheduled, old_ftbfs[suite], datetime.now()+timedelta(minutes=360))
to_be_scheduled = queue_packages(to_be_scheduled, old_depwait[suite], datetime.now()+timedelta(minutes=-360))
to_be_scheduled = queue_packages(to_be_scheduled, old[suite], datetime.now()+timedelta(minutes=720))
to_be_scheduled = queue_packages(to_be_scheduled, four04[suite], datetime.now())
schedule_packages(to_be_scheduled)
# update the scheduled page
generate_schedule(arch) # from reproducible_html_indexes
# build the message text for this arch
message = ' - ' + arch + ': '
if msg_untested:
message += msg_untested + ', '
if msg_new:
message += msg_new + ', '
if msg_404:
message += msg_404 + ', '
if msg_old_ftbfs:
message += msg_old_ftbfs + ', '
if msg_old_depwait:
message += msg_old_depwait + ', '
if msg_old:
message += msg_old + ', '
total = [now_queued_here[x] for x in SUITES]
message += 'for ' + str(sum(total))
message += ' or ' + '+'.join([str(now_queued_here[x]) for x in SUITES])
message += ' in total.'
log.info('Scheduling for architecture ' + arch + ' done.')
log.info('--------------------------------------------------------------')
# only notifiy if there were packages scheduled in any suite
for x in SUITES:
if len(untested[x])+len(new[x])+len(old[x])+len(old_ftbfs[x])+len(old_depwait[x]) > 0:
return message + '\n'
return ''
elif notify or artifacts:
do_notify = 1
else:
do_notify = 0
for id in ids:
to_schedule.append((id, date, artifacts_value, str(do_notify), requester,
None))
save_schedule.append((id, requester, epoch))
log.debug('Packages about to be scheduled: ' + str(to_schedule))
query1 = '''REPLACE INTO schedule
(package_id, date_scheduled, save_artifacts, notify, scheduler, message)
VALUES (?, ?, ?, ?, ?, ?)'''
query2 = '''INSERT INTO manual_scheduler
(package_id, requester, date_request) VALUES (?, ?, ?)'''
if not dry_run:
cursor = conn_db.cursor()
cursor.executemany(query1, to_schedule)
cursor.executemany(query2, save_schedule)
conn_db.commit()
else:
log.info('Ran with --dry-run, scheduled nothing')
log.info(bcolors.GOOD + message + bcolors.ENDC)
if not (local and requester == "jenkins maintenance job") and len(ids) != 0:
if not dry_run:
irc_msg(message)
generate_schedule(arch) # update the html page
def rest(scheduling_args, requester, local, suite, arch):
"Actually schedule a package for a single suite on a single arch."
# Shorter names
reason = scheduling_args.message
issue = scheduling_args.issue
status = scheduling_args.status
built_after = scheduling_args.after
built_before = scheduling_args.before
packages = scheduling_args.packages
artifacts = scheduling_args.keep_artifacts
notify = scheduling_args.notify
notify_on_start = scheduling_args.noisy
dry_run = scheduling_args.dry_run
log.info("Scheduling packages in %s/%s", arch, suite)
ids = []
pkgs = []
query1 = """SELECT id FROM sources WHERE name='{pkg}' AND suite='{suite}'
AND architecture='{arch}'"""
query2 = """SELECT p.date_build_started
FROM sources AS s JOIN schedule as p ON p.package_id=s.id
WHERE p.package_id='{id}'"""
for pkg in set(packages):
# test whether the package actually exists
result = query_db(query1.format(pkg=pkg, suite=suite, arch=arch))
# tests whether the package is already building
try:
result2 = query_db(query2.format(id=result[0][0]))
except IndexError:
log.error('%sThe package %s is not available in %s/%s%s',
bcolors.FAIL, pkg, suite, arch, bcolors.ENDC)
continue
try:
if not result2[0][0]:
ids.append(result[0][0])
pkgs.append(pkg)
else:
log.warning(bcolors.WARN + 'The package ' + pkg + ' is ' +
'already building, not scheduling it.' +
bcolors.ENDC)
except IndexError:
# it's not in the schedule
ids.append(result[0][0])
pkgs.append(pkg)
def compose_irc_message():
"One-shot closure to limit scope of the following local variables."
blablabla = '✂…' if len(' '.join(pkgs)) > 257 else ''
packages_txt = str(len(ids)) + ' packages ' if len(pkgs) > 1 else ''
trailing = ' - artifacts will be preserved' if artifacts else ''
trailing += ' - with irc notification' if notify else ''
trailing += ' - notify on start too' if notify_on_start else ''
message = requester + ' scheduled ' + packages_txt + \
'in ' + suite + '/' + arch
if reason:
message += ', reason: \'' + reason + '\''
message += ': ' + ' '.join(pkgs)[0:256] + blablabla + trailing
return message
info_msg = compose_irc_message()
del compose_irc_message
# these packages are manually scheduled, so should have high priority,
# so schedule them in the past, so they are picked earlier :)
# the current date is subtracted twice, so it sorts before early scheduling
# schedule on the full hour so we can recognize them easily
epoch = int(time.time())
now = datetime.now()
days = int(now.strftime('%j')) * 2
hours = int(now.strftime('%H')) * 2
minutes = int(now.strftime('%M'))
time_delta = timedelta(days=days, hours=hours, minutes=minutes)
date = (now - time_delta).strftime('%Y-%m-%d %H:%M')
log.debug('date_scheduled = ' + date + ' time_delta = ' + str(time_delta))
# a single person can't schedule more than 500 packages in the same day; this
# is actually easy to bypass, but let's give some trust to the Debian people
query = """SELECT count(*) FROM manual_scheduler
WHERE requester = '{}' AND date_request > '{}'"""
try:
amount = int(
query_db(query.format(requester, int(time.time() - 86400)))[0][0])
except IndexError:
amount = 0
log.debug(requester + ' already scheduled ' + str(amount) +
' packages today')
if amount + len(ids) > 500 and not local:
log.error(
bcolors.FAIL + 'You have exceeded the maximum number of manual ' +
'reschedulings allowed for a day. Please ask in ' +
'#debian-reproducible if you need to schedule more packages.' +
bcolors.ENDC)
sys.exit(1)
# do the actual scheduling
add_to_schedule = []
update_schedule = []
save_schedule = []
artifacts_value = 1 if artifacts else 0
if notify_on_start:
do_notify = 2
elif notify or artifacts:
do_notify = 1
else:
do_notify = 0
schedule_table = db_table('schedule')
if ids:
existing_pkg_ids = dict(
query_db(
sql.select([
schedule_table.c.package_id,
schedule_table.c.id,
]).where(schedule_table.c.package_id.in_(ids))))
for id in ids:
if id in existing_pkg_ids:
update_schedule.append({
'update_id': existing_pkg_ids[id],
'package_id': id,
'date_scheduled': date,
'save_artifacts': artifacts_value,
'notify': str(do_notify),
'scheduler': requester,
})
else:
add_to_schedule.append({
'package_id': id,
'date_scheduled': date,
'save_artifacts': artifacts_value,
'notify': str(do_notify),
'scheduler': requester,
})
save_schedule.append({
'package_id': id,
'requester': requester,
'date_request': epoch,
})
log.debug('Packages about to be scheduled: ' + str(add_to_schedule) +
str(update_schedule))
update_schedule_query = schedule_table.update().\
where(schedule_table.c.id == sql.bindparam('update_id'))
insert_schedule_query = schedule_table.insert()
insert_manual_query = db_table('manual_scheduler').insert()
if not dry_run:
transaction = conn_db.begin()
if add_to_schedule:
conn_db.execute(insert_schedule_query, add_to_schedule)
if update_schedule:
conn_db.execute(update_schedule_query, update_schedule)
if save_schedule:
conn_db.execute(insert_manual_query, save_schedule)
transaction.commit()
else:
log.info('Ran with --dry-run, scheduled nothing')
log.info(bcolors.GOOD + info_msg + bcolors.ENDC)
if not (local
and requester == "jenkins maintenance job") and len(ids) != 0:
if not dry_run:
irc_msg(info_msg)
from reproducible_html_live_status import generate_schedule
generate_schedule(arch) # update the HTML page
def scheduler(arch):
query = 'SELECT count(*) ' + \
'FROM schedule AS p JOIN sources AS s ON p.package_id=s.id ' + \
'WHERE s.architecture="{arch}"'
total = int(query_db(query.format(arch=arch))[0][0])
log.info('==============================================================')
log.info('Currently scheduled packages in all suites on ' + arch + ': ' + str(total))
if total > MAXIMA[arch]:
log.info(str(total) + ' packages already scheduled' +
', only scheduling new versions.')
empty_pkgs = {}
for suite in SUITES:
empty_pkgs[suite] = []
untested, msg_untested = empty_pkgs, ''
new, msg_new = schedule_new_versions(arch, total)
old_ftbfs_and_depwait, msg_old_ftbfs_and_depwait = empty_pkgs, ''
old, msg_old = empty_pkgs, ''
else:
log.info(str(total) + ' packages already scheduled' +
', scheduling some more...')
untested, msg_untested = schedule_untested_packages(arch, total)
new, msg_new = schedule_new_versions(arch, total+len(untested))
old_ftbfs_and_depwait, msg_old_ftbfs_and_depwait = schedule_old_ftbfs_and_depwait_versions(arch, total+len(untested)+len(new))
old, msg_old = schedule_old_versions(arch, total+len(untested)+len(new)+len(old_ftbfs_and_depwait))
now_queued_here = {}
# make sure to schedule packages in unstable first
# (but keep the view ordering everywhere else)
priotized_suite_order = ['unstable']
for suite in SUITES:
if suite not in priotized_suite_order:
priotized_suite_order.append(suite)
for suite in priotized_suite_order:
if arch == 'armhf' and suite != 'unstable':
now_queued_here[suite] = 0
continue
query = 'SELECT count(*) ' \
'FROM schedule AS p JOIN sources AS s ON p.package_id=s.id ' \
'WHERE s.suite="{suite}" AND s.architecture="{arch}"'
query = query.format(suite=suite, arch=arch)
now_queued_here[suite] = int(query_db(query)[0][0]) + \
len(untested[suite]+new[suite]+old[suite])
# schedule packages differently in the queue...
to_be_scheduled = queue_packages({}, untested[suite], datetime.now())
assert(isinstance(to_be_scheduled, dict))
to_be_scheduled = queue_packages(to_be_scheduled, new[suite], datetime.now()+timedelta(minutes=-720))
to_be_scheduled = queue_packages(to_be_scheduled, old_ftbfs_and_depwait[suite], datetime.now()+timedelta(minutes=360))
to_be_scheduled = queue_packages(to_be_scheduled, old[suite], datetime.now()+timedelta(minutes=720))
schedule_packages(to_be_scheduled)
# update the scheduled page
generate_schedule(arch) # from reproducible_html_indexes
# build the kgb message text
if arch != 'armhf':
message = 'Scheduled in ' + '+'.join(SUITES) + ' (' + arch + '): '
else:
message = 'Scheduled in unstable (' + arch + '): '
if msg_untested:
message += msg_untested
message += ' and ' if msg_new and not msg_old_ftbfs_and_depwait and not msg_old else ''
message += ', ' if ( msg_new and msg_old_ftbfs_and_depwait ) or ( msg_new and msg_old ) else ''
if msg_new:
message += msg_new
message += ' and ' if msg_old_ftbfs_and_depwait and not msg_old else ''
message += ', ' if msg_old_ftbfs_and_depwait and msg_old else ''
if msg_old_ftbfs_and_depwait:
message += msg_old_ftbfs_and_depwait
message += ' and ' if msg_old_ftbfs_and_depwait else ''
if msg_old:
message += msg_old
total = [now_queued_here[x] for x in SUITES]
message += ', for ' + str(sum(total))
if arch != 'armhf':
message += ' or ' + '+'.join([str(now_queued_here[x]) for x in SUITES])
message += ' packages in total.'
# only notifiy irc if there were packages scheduled in any suite
for x in SUITES:
if len(untested[x])+len(new[x])+len(old[x])+len(old_ftbfs_and_depwait[x]) > 0:
log.info(message)
irc_msg(message)
break
log.info('Scheduling for architecture ' + arch + ' done.')
log.info('--------------------------------------------------------------')
def scheduler(arch):
query = (
"SELECT count(*) "
+ "FROM schedule AS p JOIN sources AS s ON p.package_id=s.id "
+ 'WHERE s.architecture="{arch}"'
)
total = int(query_db(query.format(arch=arch))[0][0])
log.info("==============================================================")
log.info("Currently scheduled packages in all suites on " + arch + ": " + str(total))
if total > MAXIMA[arch]:
log.info(str(total) + " packages already scheduled" + ", only scheduling new versions.")
empty_pkgs = {}
for suite in SUITES:
empty_pkgs[suite] = []
untested, msg_untested = empty_pkgs, ""
new, msg_new = schedule_new_versions(arch, total)
old_ftbfs_and_depwait, msg_old_ftbfs_and_depwait = empty_pkgs, ""
old, msg_old = empty_pkgs, ""
four04, msg_404 = empty_pkgs, ""
else:
log.info(str(total) + " packages already scheduled" + ", scheduling some more...")
untested, msg_untested = schedule_untested_packages(arch, total)
new, msg_new = schedule_new_versions(arch, total + len(untested))
old_ftbfs_and_depwait, msg_old_ftbfs_and_depwait = schedule_old_ftbfs_and_depwait_versions(
arch, total + len(untested) + len(new)
)
old, msg_old = schedule_old_versions(arch, total + len(untested) + len(new) + len(old_ftbfs_and_depwait))
four04, msg_404 = schedule_404_versions(
arch, total + len(untested) + len(new) + len(old_ftbfs_and_depwait) + len(old)
)
now_queued_here = {}
# make sure to schedule packages in unstable first
# (but keep the view ordering everywhere else)
priotized_suite_order = ["unstable"]
for suite in SUITES:
if suite not in priotized_suite_order:
priotized_suite_order.append(suite)
for suite in priotized_suite_order:
query = (
"SELECT count(*) "
"FROM schedule AS p JOIN sources AS s ON p.package_id=s.id "
'WHERE s.suite="{suite}" AND s.architecture="{arch}"'
)
query = query.format(suite=suite, arch=arch)
now_queued_here[suite] = int(query_db(query)[0][0]) + len(untested[suite] + new[suite] + old[suite])
# schedule packages differently in the queue...
to_be_scheduled = queue_packages({}, untested[suite], datetime.now())
assert isinstance(to_be_scheduled, dict)
to_be_scheduled = queue_packages(to_be_scheduled, new[suite], datetime.now() + timedelta(minutes=-720))
to_be_scheduled = queue_packages(
to_be_scheduled, old_ftbfs_and_depwait[suite], datetime.now() + timedelta(minutes=360)
)
to_be_scheduled = queue_packages(to_be_scheduled, old[suite], datetime.now() + timedelta(minutes=720))
to_be_scheduled = queue_packages(to_be_scheduled, four04[suite], datetime.now())
schedule_packages(to_be_scheduled)
# update the scheduled page
generate_schedule(arch) # from reproducible_html_indexes
# build the message text for this arch
message = " - " + arch + ": "
if msg_untested:
message += msg_untested + ", "
if msg_new:
message += msg_new + ", "
if msg_404:
message += msg_404 + ", "
if msg_old_ftbfs_and_depwait:
message += msg_old_ftbfs_and_depwait + ", "
if msg_old:
message += msg_old + ", "
total = [now_queued_here[x] for x in SUITES]
message += " for " + str(sum(total))
message += " or " + "+".join([str(now_queued_here[x]) for x in SUITES])
message += " packages in total."
log.info("Scheduling for architecture " + arch + " done.")
log.info("--------------------------------------------------------------")
# only notifiy irc if there were packages scheduled in any suite
for x in SUITES:
if len(untested[x]) + len(new[x]) + len(old[x]) + len(old_ftbfs_and_depwait[x]) > 0:
return message
return ""
elif notify or artifacts:
do_notify = 1
else:
do_notify = 0
for id in ids:
to_schedule.append(
(id, date, artifacts_value, str(do_notify), requester, None))
save_schedule.append((id, requester, epoch))
log.debug('Packages about to be scheduled: ' + str(to_schedule))
query1 = '''REPLACE INTO schedule
(package_id, date_scheduled, save_artifacts, notify, scheduler, message)
VALUES (?, ?, ?, ?, ?, ?)'''
query2 = '''INSERT INTO manual_scheduler
(package_id, requester, date_request) VALUES (?, ?, ?)'''
if not dry_run:
cursor = conn_db.cursor()
cursor.executemany(query1, to_schedule)
cursor.executemany(query2, save_schedule)
conn_db.commit()
else:
log.info('Ran with --dry-run, scheduled nothing')
log.info(bcolors.GOOD + message + bcolors.ENDC)
if not (local and requester == "jenkins maintenance job") and len(ids) != 0:
if not dry_run:
irc_msg(message)
generate_schedule(arch) # update the html page