def schedule_packages(packages):
pkgs = [{
'package_id': x,
'date_scheduled': packages[x]
} for x in packages.keys()]
log.debug('IDs about to be scheduled: %s', packages.keys())
if pkgs:
conn_db.execute(db_table('schedule').insert(), pkgs)
def process_pkg(package, deactivate):
if deactivate:
_good('Deactivating notification for package ' + str(package))
flag = 0
else:
_good('Activating notification for package ' + str(package))
flag = 1
sources_table = db_table('sources')
update_query = sources_table.update().\
where(sources_table.c.name == package).\
values(notify_maintainer=flag)
rows = conn_db.execute(update_query).rowcount
if rows == 0:
log.error(bcolors.FAIL + str(package) + ' does not exists')
sys.exit(1)
if DEBUG:
log.debug('Double check the change:')
query = 'SELECT * FROM sources WHERE name="{}"'.format(package)
log.debug(query_db(query))
def store_notes():
log.debug('Removing all notes')
notes_table = db_table('notes')
conn_db.execute(notes_table.delete())
to_insert = []
for entry in [x for y in sorted(notes) for x in notes[y]]:
pkg_id = entry['id']
pkg_version = entry['version']
pkg_issues = json.dumps(entry['issues'])
pkg_bugs = json.dumps(entry['bugs'])
pkg_comments = entry['comments']
to_insert.append({
'package_id': pkg_id,
'version': pkg_version,
'issues': pkg_issues,
'bugs': pkg_bugs,
'comments': pkg_comments
})
if (len(to_insert)):
conn_db.execute(notes_table.insert(), to_insert)
log.info('Saved ' + str(len(to_insert)) + ' notes in the database')
def store_issues():
issues_table = db_table('issues')
# Get existing issues
results = conn_db.execute(sql.select([issues_table.c.name]))
existing_issues = set([row[0] for row in results])
to_insert = []
to_update = []
for name in issues:
url = issues[name]['url'] if 'url' in issues[name] else ''
desc = issues[name]['description']
if name in existing_issues:
to_update.append({
'issuename': name,
'url': url,
'description': desc
})
# remove this package from the set, to know who to delete later
existing_issues.remove(name)
else:
to_insert.append({'name': name, 'url': url, 'description': desc})
if to_update:
update_query = issues_table.update().\
where(issues_table.c.name == sql.bindparam('issuename'))
conn_db.execute(update_query, to_update)
log.debug('Issues updated in the database')
if to_insert:
conn_db.execute(issues_table.insert(), to_insert)
log.debug('Issues added to the database')
# if there are any existing issues left, delete them.
if existing_issues:
to_delete = [{'issuename': name} for name in existing_issues]
delete_query = issues_table.delete().\
where(issues_table.c.name == sql.bindparam('issuename'))
conn_db.execute(delete_query, to_delete)
log.info("Removed the following issues: " + str(existing_issues))
- force the suite/arch to the defaults
+ notes: if true the query also takes the value "status"
Technically speaking, a page can be empty (we all love nonsense) but every
section must have at least a `query` defining what to file in.
"""
timespan_date_map = {}
timespan_date_map[24] = (datetime.now() -
timedelta(hours=24)).strftime('%Y-%m-%d %H:%M')
timespan_date_map[48] = (datetime.now() -
timedelta(hours=48)).strftime('%Y-%m-%d %H:%M')
# sqlalchemy table definitions needed for queries
results = db_table('results')
sources = db_table('sources')
notes = db_table('notes')
# filtered_issues is defined in reproducible_common.py and
# can be used to excludes some FTBFS issues
filter_issues_list = []
for issue in filtered_issues:
filter_issues_list.append(notes.c.issues.contains(issue))
if not filtered_issues:
filter_issues_list = [None]
count_results = select([func.count(results.c.id)
]).select_from(results.join(sources)).where(
and_(sources.c.suite == bindparam('suite'),
sources.c.architecture == bindparam('arch')))
def gen_html_issue(issue, suite):
"""
Given a issue as input (as a dict:
{"issue_identifier": {"description": "blablabla", "url": "blabla"}}
) it returns the html body
"""
# links to the issue in other suites
suite_links = ''
for i in SUITES:
if suite_links != '':
suite_links += ' / '
if i != suite:
suite_links += '<a href="' + REPRODUCIBLE_URL + ISSUES_URI + '/' + i + '/' + issue + '_issue.html">' + i + '</a>'
else:
suite_links += '<em>' + i + '</em>'
# check for url:
if 'url' in issues[issue]:
url = issue_html_url.substitute(url=issues[issue]['url'])
else:
url = ''
# add affected packages:
affected = ''
results = db_table('results')
sources = db_table('sources')
sql = select(
[sources.c.name]
).select_from(
results.join(sources)
).where(
and_(
sources.c.suite == bindparam('suite'),
sources.c.architecture == bindparam('arch'),
results.c.status == bindparam('status'),
)
).order_by(
sources.c.name
)
try:
arch = 'amd64'
for status in Status:
status = status.value
pkgs = query_db(sql.where(sources.c.name.in_(issues_count[issue]))\
.params({'suite': suite, 'arch': arch, 'status': status.name}))
pkgs = [p[0] for p in pkgs]
if not pkgs:
continue
affected += tab*4 + '<p>\n'
affected += tab*5 + '<img src="/static/{}"'.format(status.icon)
affected += ' alt="' + status.name + ' icon" />\n'
affected += tab*5 + str(len(pkgs)) + ' ' + status.spokenstatus
affected += ' packages in ' + suite + '/' + arch +':\n'
affected += tab*5 + '<code>\n'
pkgs_popcon = issues_popcon_annotate(pkgs)
try:
for pkg, popc_num, is_popular in sorted(pkgs_popcon, key=lambda x: x[0] in bugs):
affected += tab*6 + Package(pkg).html_link(suite, arch, bugs, popc_num, is_popular)
except ValueError:
pass
affected += tab*5 + '</code>\n'
affected += tab*4 + '</p>\n'
except KeyError: # The note is not listed in any package, that is
affected = '<i>None</i>'
# check for description:
try:
desc = issues[issue]['description']
except KeyError:
log.warning('You should really include a description in the ' +
issue + ' issue')
desc = 'N/A'
desc = url2html.sub(r'<a href="\1">\1</a>', desc)
desc = desc.replace('\n', '<br />')
return issue_html.substitute(issue=issue, urls=url, description=desc,
affected_pkgs=affected,
suite=suite, suite_links=suite_links,
notesgit_description=NOTESGIT_DESCRIPTION)
def rest(scheduling_args, requester, local, suite, arch):
"Actually schedule a package for a single suite on a single arch."
# Shorter names
reason = scheduling_args.message
issue = scheduling_args.issue
status = scheduling_args.status
built_after = scheduling_args.after
built_before = scheduling_args.before
packages = scheduling_args.packages
artifacts = scheduling_args.keep_artifacts
notify = scheduling_args.notify
notify_on_start = scheduling_args.notify_on_start
dry_run = scheduling_args.dry_run
log.info("Scheduling packages in %s/%s", arch, suite)
ids = []
pkgs = []
query1 = """SELECT id FROM sources WHERE name='{pkg}' AND suite='{suite}'
AND architecture='{arch}'"""
query2 = """SELECT p.date_build_started
FROM sources AS s JOIN schedule as p ON p.package_id=s.id
WHERE p.package_id='{id}'"""
for pkg in set(packages):
# test whether the package actually exists
result = query_db(query1.format(pkg=pkg, suite=suite, arch=arch))
# tests whether the package is already building
try:
result2 = query_db(query2.format(id=result[0][0]))
except IndexError:
log.error('%sThe package %s is not available in %s/%s%s',
bcolors.FAIL, pkg, suite, arch, bcolors.ENDC)
continue
try:
if not result2[0][0]:
ids.append(result[0][0])
pkgs.append(pkg)
else:
log.warning(bcolors.WARN + 'The package ' + pkg + ' is ' +
'already building, not scheduling it.' +
bcolors.ENDC)
except IndexError:
# it's not in the schedule
ids.append(result[0][0])
pkgs.append(pkg)
def compose_irc_message():
"One-shot closure to limit scope of the following local variables."
blablabla = '✂…' if len(' '.join(pkgs)) > 257 else ''
packages_txt = str(len(ids)) + ' packages ' if len(pkgs) > 1 else ''
trailing = ' - artifacts will be preserved' if artifacts else ''
trailing += ' - with irc notification' if notify else ''
trailing += ' - notify on start too' if notify_on_start else ''
message = requester + ' scheduled ' + packages_txt + \
'in ' + suite + '/' + arch
if reason:
message += ', reason: \'' + reason + '\''
message += ': ' + ' '.join(pkgs)[0:256] + blablabla + trailing
return message
info_msg = compose_irc_message()
del compose_irc_message
# these packages are manually scheduled, so should have high priority,
# so schedule them in the past, so they are picked earlier :)
# the current date is subtracted twice, so it sorts before early scheduling
# schedule on the full hour so we can recognize them easily
epoch = int(time.time())
now = datetime.now()
days = int(now.strftime('%j')) * 2
hours = int(now.strftime('%H')) * 2
minutes = int(now.strftime('%M'))
time_delta = timedelta(days=days, hours=hours, minutes=minutes)
date = (now - time_delta).strftime('%Y-%m-%d %H:%M')
log.debug('date_scheduled = ' + date + ' time_delta = ' + str(time_delta))
# a single person can't schedule more than 500 packages in the same day; this
# is actually easy to bypass, but let's give some trust to the Debian people
query = """SELECT count(*) FROM manual_scheduler
WHERE requester = '{}' AND date_request > '{}'"""
try:
amount = int(
query_db(query.format(requester, int(time.time() - 86400)))[0][0])
except IndexError:
amount = 0
log.debug(requester + ' already scheduled ' + str(amount) +
' packages today')
if amount + len(ids) > 500 and not local:
log.error(
bcolors.FAIL + 'You have exceeded the maximum number of manual ' +
'reschedulings allowed for a day. Please ask in ' +
'#debian-reproducible if you need to schedule more packages.' +
bcolors.ENDC)
sys.exit(1)
# do the actual scheduling
add_to_schedule = []
update_schedule = []
save_schedule = []
artifacts_value = 1 if artifacts else 0
if notify_on_start:
do_notify = 2
elif notify or artifacts:
do_notify = 1
else:
do_notify = 0
schedule_table = db_table('schedule')
if ids:
existing_pkg_ids = dict(
query_db(
sql.select([
schedule_table.c.package_id,
schedule_table.c.id,
]).where(schedule_table.c.package_id.in_(ids))))
for id in ids:
if id in existing_pkg_ids:
update_schedule.append({
'update_id': existing_pkg_ids[id],
'package_id': id,
'date_scheduled': date,
'save_artifacts': artifacts_value,
'notify': str(do_notify),
'scheduler': requester,
})
else:
add_to_schedule.append({
'package_id': id,
'date_scheduled': date,
'save_artifacts': artifacts_value,
'notify': str(do_notify),
'scheduler': requester,
})
save_schedule.append({
'package_id': id,
'requester': requester,
'date_request': epoch,
})
log.debug('Packages about to be scheduled: ' + str(add_to_schedule) +
str(update_schedule))
update_schedule_query = schedule_table.update().\
where(schedule_table.c.id == sql.bindparam('update_id'))
insert_schedule_query = schedule_table.insert()
insert_manual_query = db_table('manual_scheduler').insert()
if not dry_run:
transaction = conn_db.begin()
if add_to_schedule:
conn_db.execute(insert_schedule_query, add_to_schedule)
if update_schedule:
conn_db.execute(update_schedule_query, update_schedule)
if save_schedule:
conn_db.execute(insert_manual_query, save_schedule)
transaction.commit()
else:
log.info('Ran with --dry-run, scheduled nothing')
log.info(bcolors.GOOD + info_msg + bcolors.ENDC)
if not (local
and requester == "jenkins maintenance job") and len(ids) != 0:
if not dry_run:
# Always announce on -changes
irc_msg(info_msg, 'debian-reproducible-changes')
# Announce some messages on main channel
if notify_on_start or artifacts:
irc_msg(info_msg)
from rblib.confparse import log
from rblib.models import Package, Status
from rblib.utils import convert_into_hms_string
from rblib.html import tab, create_main_navigation, write_html_page
from reproducible_html_indexes import build_leading_text_section
from rblib.const import (
DISTRO_BASE,
DISTRO_URL,
DISTRO_URI,
ARCHS,
SUITES,
defaultsuite,
)
# sqlalchemy table definitions needed for queries
results = db_table('results')
sources = db_table('sources')
schedule = db_table('schedule')
stats_build = db_table('stats_build')
def convert_into_status_html(statusname):
if statusname == 'None':
return ''
status = Status.get(statusname)
return '{n} <img src="/static/{icon}" alt="{n}" title="{n}" />'.format(
n=status.value.name, icon=status.value.icon)
def generate_schedule(arch):
""" the schedule pages are very different than others index pages """
def update_sources_db(suite, arch, sources):
# extract relevant info (package name and version) from the sources file
new_pkgs = set()
newest_version = {}
for src in deb822.Sources.iter_paragraphs(sources.split('\n')):
pkg = (src['Package'], src['Version'], suite, arch)
if 'Extra-Source-Only' in src and src['Extra-Source-Only'] == 'yes':
log.debug('Ignoring {} due to Extra-Source-Only'.format(pkg))
continue
# only keep the most recent version of a src for each package/suite/arch
key = src['Package'] + suite + arch
if key in newest_version:
oldversion = newest_version[key]
oldpackage = (src['Package'], oldversion, suite, arch)
new_pkgs.remove(oldpackage)
newest_version[key] = src['Version']
new_pkgs.add(pkg)
# get the current packages in the database
query = "SELECT name, version, suite, architecture FROM sources " + \
"WHERE suite='{}' AND architecture='{}'".format(suite, arch)
cur_pkgs = set([(p.name, p.version, p.suite, p.architecture)
for p in query_db(query)])
pkgs_to_add = []
updated_pkgs = []
different_pkgs = [x for x in new_pkgs if x not in cur_pkgs]
log.debug('Packages different in the archive and in the db: %s',
different_pkgs)
for pkg in different_pkgs:
# pkg: (name, version, suite, arch)
query = "SELECT id, version, notify_maintainer FROM sources " + \
"WHERE name='{}' AND suite='{}' AND architecture='{}'"
query = query.format(pkg[0], pkg[2], pkg[3])
try:
result = query_db(query)[0]
except IndexError: # new package
pkgs_to_add.append({
'name': pkg[0],
'version': pkg[1],
'suite': pkg[2],
'architecture': pkg[3],
})
continue
pkg_id = result[0]
old_version = result[1]
notify_maint = int(result[2])
if apt_pkg.version_compare(pkg[1], old_version) > 0:
log.debug('New version: ' + str(pkg) + ' (we had ' + old_version +
')')
updated_pkgs.append({
'update_id': pkg_id,
'name': pkg[0],
'version': pkg[1],
'suite': pkg[2],
'architecture': pkg[3],
'notify_maintainer': notify_maint,
})
# Now actually update the database:
sources_table = db_table('sources')
# updated packages
log.info('Pushing ' + str(len(updated_pkgs)) +
' updated packages to the database...')
if updated_pkgs:
transaction = conn_db.begin()
update_query = sources_table.update().\
where(sources_table.c.id == sql.bindparam('update_id'))
conn_db.execute(update_query, updated_pkgs)
transaction.commit()
# new packages
if pkgs_to_add:
log.info('Now inserting %i new sources in the database: %s',
len(pkgs_to_add), pkgs_to_add)
transaction = conn_db.begin()
conn_db.execute(sources_table.insert(), pkgs_to_add)
transaction.commit()
# RM'ed packages
cur_pkgs_name = [x[0] for x in cur_pkgs]
new_pkgs_name = [x[0] for x in new_pkgs]
rmed_pkgs = [x for x in cur_pkgs_name if x not in new_pkgs_name]
log.info('Now deleting %i removed packages: %s', len(rmed_pkgs), rmed_pkgs)
rmed_pkgs_id = []
pkgs_to_rm = []
query = "SELECT id FROM sources WHERE name='{}' AND suite='{}' " + \
"AND architecture='{}'"
for pkg in rmed_pkgs:
result = query_db(query.format(pkg, suite, arch))
rmed_pkgs_id.append({'deleteid': result[0][0]})
pkgs_to_rm.append({'name': pkg, 'suite': suite, 'architecture': arch})
log.debug('removed packages ID: %s',
[str(x['deleteid']) for x in rmed_pkgs_id])
log.debug('removed packages: %s', pkgs_to_rm)
if rmed_pkgs_id:
transaction = conn_db.begin()
results_table = db_table('results')
schedule_table = db_table('schedule')
notes_table = db_table('notes')
removed_packages_table = db_table('removed_packages')
delete_results_query = results_table.delete().\
where(results_table.c.package_id == sql.bindparam('deleteid'))
delete_schedule_query = schedule_table.delete().\
where(schedule_table.c.package_id == sql.bindparam('deleteid'))
delete_notes_query = notes_table.delete().\
where(notes_table.c.package_id == sql.bindparam('deleteid'))
delete_sources_query = sources_table.delete().\
where(sources_table.c.id == sql.bindparam('deleteid'))
conn_db.execute(delete_results_query, rmed_pkgs_id)
conn_db.execute(delete_schedule_query, rmed_pkgs_id)
conn_db.execute(delete_notes_query, rmed_pkgs_id)
conn_db.execute(delete_sources_query, rmed_pkgs_id)
conn_db.execute(removed_packages_table.insert(), pkgs_to_rm)
transaction.commit()
# finally check whether the db has the correct number of packages
query = "SELECT count(*) FROM sources WHERE suite='{}' " + \
"AND architecture='{}'"
pkgs_end = query_db(query.format(suite, arch))
count_new_pkgs = len(set([x[0] for x in new_pkgs]))
if int(pkgs_end[0][0]) != count_new_pkgs:
print_critical_message('AH! The number of source in the Sources file' +
' is different than the one in the DB!')
log.critical('source in the debian archive for the %s suite: %s',
suite, str(count_new_pkgs))
log.critical('source in the reproducible db for the %s suite: %s',
suite, str(pkgs_end[0][0]))
sys.exit(1)
if pkgs_to_add:
log.info('Building pages for the new packages')
gen_packages_html([Package(x['name']) for x in pkgs_to_add],
no_clean=True)