def update_stats_breakages(diffoscope_timeouts, diffoscope_crashes):
# we only do stats up until yesterday
YESTERDAY = (datetime.now() - timedelta(days=1)).strftime('%Y-%m-%d')
result = query_db("""
SELECT datum, diffoscope_timeouts, diffoscope_crashes
FROM stats_breakages
WHERE datum = '{date}'
""".format(date=YESTERDAY))
# if there is not a result for this day, add one
if not result:
insert = "INSERT INTO stats_breakages VALUES ('{date}', " + \
"'{diffoscope_timeouts}', '{diffoscope_crashes}')"
query_db(
insert.format(date=YESTERDAY,
diffoscope_timeouts=diffoscope_timeouts,
diffoscope_crashes=diffoscope_crashes))
log.info(
"Updating db table stats_breakages on %s with %s timeouts and %s crashes.",
YESTERDAY, diffoscope_timeouts, diffoscope_crashes)
else:
log.debug(
"Not updating db table stats_breakages as it already has data for %s.",
YESTERDAY)
def unrep_with_dbd_issues():
log.info('running unrep_with_dbd_issues check...')
without_dbd = []
bad_dbd = []
sources_without_dbd = set()
query = '''SELECT s.name, r.version, s.suite, s.architecture
FROM sources AS s JOIN results AS r ON r.package_id=s.id
WHERE r.status='unreproducible'
ORDER BY s.name ASC, s.suite DESC, s.architecture ASC'''
results = query_db(query)
for pkg, version, suite, arch in results:
eversion = strip_epoch(version)
dbd = DBD_PATH + '/' + suite + '/' + arch + '/' + pkg + '_' + \
eversion + '.diffoscope.html'
if not os.access(dbd, os.R_OK):
without_dbd.append((pkg, version, suite, arch))
sources_without_dbd.add(pkg)
log.warning(suite + '/' + arch + '/' + pkg + ' (' + version +
') is '
'unreproducible without diffoscope file.')
else:
log.debug(dbd + ' found.')
data = open(dbd, 'br').read(3)
if b'<' not in data:
bad_dbd.append((pkg, version, suite, arch))
log.warning(suite + '/' + arch + '/' + pkg + ' (' + version +
') has '
'diffoscope output, but it does not seem to '
'be an HTML page.')
sources_without_dbd.add(pkg)
return without_dbd, bad_dbd, sources_without_dbd
def update_stats(suite, arch, stats, pkgset_name):
result = query_db("""
SELECT datum, meta_pkg, suite
FROM stats_meta_pkg_state
WHERE datum = '{date}' AND suite = '{suite}'
AND architecture = '{arch}' AND meta_pkg = '{name}'
""".format(date=YESTERDAY, suite=suite, arch=arch, name=pkgset_name))
# if there is not a result for this day, add one
if not result:
insert = "INSERT INTO stats_meta_pkg_state VALUES ('{date}', " + \
"'{suite}', '{arch}', '{pkgset_name}', '{count_good}', " + \
"'{count_bad}', '{count_ugly}', '{count_rest}')"
query_db(
insert.format(date=YESTERDAY,
suite=suite,
arch=arch,
pkgset_name=pkgset_name,
count_good=stats['count_good'],
count_bad=stats['count_bad'],
count_ugly=stats['count_ugly'],
count_rest=stats['count_rest']))
log.info("Updating db entry for meta pkgset %s in %s/%s on %s.",
pkgset_name, suite, arch, YESTERDAY)
else:
log.debug(
"Not updating db entry for meta pkgset %s in %s/%s on %s as one exists already.",
pkgset_name, suite, arch, YESTERDAY)
def pbuilder_dep_fail():
log.info('running pbuilder_dep_fail check...')
bad_pkgs = []
# we only care about these failures in the !unstable !experimental suites
# as they happen all the time in there, as packages are buggy
# and specific versions also come and go
query = '''SELECT s.name, r.version, s.suite, s.architecture
FROM sources AS s JOIN results AS r ON r.package_id=s.id
WHERE r.status = 'FTBFS'
AND s.suite NOT IN ('unstable', 'experimental')
ORDER BY s.name ASC, s.suite DESC, s.architecture ASC'''
results = query_db(query)
for pkg, version, suite, arch in results:
eversion = strip_epoch(version)
rbuild = RBUILD_PATH + '/' + suite + '/' + arch + '/' + pkg + '_' + \
eversion + '.rbuild.log'
if os.access(rbuild, os.R_OK):
log.debug('\tlooking at ' + rbuild)
with open(rbuild, "br") as fd:
for line in fd:
if re.search(b'E: pbuilder-satisfydepends failed.', line):
bad_pkgs.append((pkg, version, suite, arch))
log.warning(suite + '/' + arch + '/' + pkg + ' (' +
version +
') failed to satisfy its dependencies.')
return bad_pkgs
def schedule_packages(packages):
pkgs = [{
'package_id': x,
'date_scheduled': packages[x]
} for x in packages.keys()]
log.debug('IDs about to be scheduled: %s', packages.keys())
if pkgs:
conn_db.execute(db_table('schedule').insert(), pkgs)
def load_issues():
"""
format:
{ 'issue_name': {'description': 'blabla', 'url': 'blabla'} }
"""
with open(ISSUES) as fd:
issues = yaml.load(fd)
log.debug("issues loaded. There are " + str(len(issues)) +
" issues listed")
return issues
def purge_old_pages():
for suite in SUITES:
for arch in ARCHS:
log.info('Removing old pages from ' + suite + '/' + arch + '.')
try:
presents = sorted(
os.listdir(RB_PKG_PATH + '/' + suite + '/' + arch))
except OSError as e:
if e.errno != errno.ENOENT: # that's 'No such file or
raise # directory' error (errno 17)
presents = []
log.debug('page presents: ' + str(presents))
# get the existing packages
query = "SELECT name, suite, architecture FROM sources " + \
"WHERE suite='{}' AND architecture='{}'".format(suite, arch)
cur_pkgs = set([(p.name, p.suite, p.architecture)
for p in query_db(query)])
for page in presents:
# When diffoscope results exist for a package, we create a page
# that displays the diffoscope results by default in the main iframe
# in this subdirectory. Ignore this directory.
if page == 'diffoscope-results':
continue
pkg = page.rsplit('.', 1)[0]
if (pkg, suite, arch) not in cur_pkgs:
log.info('There is no package named ' + pkg + ' from ' +
suite + '/' + arch + ' in the database. ' +
'Removing old page.')
os.remove(RB_PKG_PATH + '/' + suite + '/' + arch + '/' +
page)
# Additionally clean up the diffoscope results default pages
log.info('Removing old pages from ' + suite + '/' + arch +
'/diffoscope-results/.')
try:
presents = sorted(
os.listdir(RB_PKG_PATH + '/' + suite + '/' + arch +
'/diffoscope-results'))
except OSError as e:
if e.errno != errno.ENOENT: # that's 'No such file or
raise # directory' error (errno 17)
presents = []
log.debug('diffoscope page presents: ' + str(presents))
for page in presents:
pkg = page.rsplit('.', 1)[0]
if (pkg, suite, arch) not in cur_pkgs:
log.info('There is no package named ' + pkg + ' from ' +
suite + '/' + arch + '/diffoscope-results in ' +
'the database. Removing old page.')
os.remove(RB_PKG_PATH + '/' + suite + '/' + arch + '/' +
'diffoscope-results/' + page)
def update_sources(suite):
# download the sources file for this suite
mirror = 'http://deb.debian.org/debian'
remotefile = mirror + '/dists/' + suite + '/main/source/Sources.xz'
log.info('Downloading sources file for %s: %s', suite, remotefile)
sources = lzma.decompress(urlopen(remotefile).read()).decode('utf8')
log.debug('\tdownloaded')
for arch in ARCHS:
log.info('Updating sources db for %s/%s...', suite, arch)
update_sources_db(suite, arch, sources)
log.info('DB update done for %s/%s done at %s.', suite, arch, datetime.now().strftime("%Y-%m-%d %H:%M:%S"))
def process_pkg(package, deactivate):
if deactivate:
_good('Deactivating notification for package ' + str(package))
flag = 0
else:
_good('Activating notification for package ' + str(package))
flag = 1
sources_table = db_table('sources')
update_query = sources_table.update().\
where(sources_table.c.name == package).\
values(notify_maintainer=flag)
rows = conn_db.execute(update_query).rowcount
if rows == 0:
log.error(bcolors.FAIL + str(package) + ' does not exists')
sys.exit(1)
if DEBUG:
log.debug('Double check the change:')
query = 'SELECT * FROM sources WHERE name="{}"'.format(package)
log.debug(query_db(query))
def build_page_section(page, section, suite, arch):
try:
if pages[page].get('global') and pages[page]['global']:
suite = defaultsuite
arch = defaultarch
if pages[page].get('notes') and pages[page]['notes']:
db_status = section['status'].value.name
query = queries[section['query']].params({
'status': db_status,
'suite': suite,
'arch': arch
})
section['icon_status'] = section['status'].value.icon
else:
query = queries[section['query']].params({
'suite': suite,
'arch': arch
})
rows = query_db(query)
except:
print_critical_message('A query failed: %s' % query)
raise
html = ''
footnote = True if rows else False
if not rows: # there are no package in this set, do not output anything
log.debug('empty query: %s' %
query.compile(compile_kwargs={"literal_binds": True}))
return (html, footnote)
html += build_leading_text_section(section, rows, suite, arch)
html += '<p>\n' + tab + '<code>\n'
for row in rows:
pkg = row[0]
html += tab * 2 + Package(pkg).html_link(suite, arch)
else:
html += tab + '</code>\n'
html += '</p>'
if section.get('bottom'):
html += section['bottom']
html = (tab * 2).join(html.splitlines(True))
return (html, footnote)
def store_notes():
log.debug('Removing all notes')
notes_table = db_table('notes')
conn_db.execute(notes_table.delete())
to_insert = []
for entry in [x for y in sorted(notes) for x in notes[y]]:
pkg_id = entry['id']
pkg_version = entry['version']
pkg_issues = json.dumps(entry['issues'])
pkg_bugs = json.dumps(entry['bugs'])
pkg_comments = entry['comments']
to_insert.append({
'package_id': pkg_id,
'version': pkg_version,
'issues': pkg_issues,
'bugs': pkg_bugs,
'comments': pkg_comments
})
if (len(to_insert)):
conn_db.execute(notes_table.insert(), to_insert)
log.info('Saved ' + str(len(to_insert)) + ' notes in the database')
def purge_old_notes(notes):
removed_pages = []
to_rebuild = []
presents = sorted(os.listdir(NOTES_PATH))
for page in presents:
pkg = page.rsplit('_', 1)[0]
log.debug('Checking if ' + page + ' (from ' + pkg + ') is still needed')
if pkg not in notes:
log.info('There are no notes for ' + pkg + '. Removing old page.')
os.remove(NOTES_PATH + '/' + page)
removed_pages.append(pkg)
for pkg in removed_pages:
for suite in SUITES:
try:
query = "SELECT s.name " + \
"FROM results AS r JOIN sources AS s ON r.package_id=s.id " + \
"WHERE s.name='{pkg}' AND r.status != '' AND s.suite='{suite}'"
query = query.format(pkg=pkg, suite=suite)
to_rebuild.append(query_db(query)[0][0])
except IndexError: # the package is not tested. this can happen if
pass # a package got removed from the archive
if to_rebuild:
gen_packages_html([Package(x) for x in to_rebuild])
def store_issues():
issues_table = db_table('issues')
# Get existing issues
results = conn_db.execute(sql.select([issues_table.c.name]))
existing_issues = set([row[0] for row in results])
to_insert = []
to_update = []
for name in issues:
url = issues[name]['url'] if 'url' in issues[name] else ''
desc = issues[name]['description']
if name in existing_issues:
to_update.append({
'issuename': name,
'url': url,
'description': desc
})
# remove this package from the set, to know who to delete later
existing_issues.remove(name)
else:
to_insert.append({'name': name, 'url': url, 'description': desc})
if to_update:
update_query = issues_table.update().\
where(issues_table.c.name == sql.bindparam('issuename'))
conn_db.execute(update_query, to_update)
log.debug('Issues updated in the database')
if to_insert:
conn_db.execute(issues_table.insert(), to_insert)
log.debug('Issues added to the database')
# if there are any existing issues left, delete them.
if existing_issues:
to_delete = [{'issuename': name} for name in existing_issues]
delete_query = issues_table.delete().\
where(issues_table.c.name == sql.bindparam('issuename'))
conn_db.execute(delete_query, to_delete)
log.info("Removed the following issues: " + str(existing_issues))
def load_notes():
"""
format:
{ 'package_name': {'version': '0.0', 'comments'<etc>}, 'package_name':{} }
"""
with open(NOTES) as fd:
possible_notes = yaml.load(fd)
log.debug("notes loaded. There are " + str(len(possible_notes)) +
" package listed")
notes = copy.copy(possible_notes)
for package in possible_notes: # check if every package listed on the notes
try: # actually have been tested
query = "SELECT s.name " + \
"FROM results AS r JOIN sources AS s ON r.package_id=s.id " + \
"WHERE s.name='{pkg}' AND r.status != ''"
query = query.format(pkg=package)
query_db(query)[0] # just discard this result, we only care of its success
except IndexError:
log.warning("This query produces no results: " + query)
log.warning("This means there is no tested package with the name " + package + ".")
del notes[package]
log.debug("notes checked. There are " + str(len(notes)) +
" package listed")
return notes
def iterate_over_notes(notes):
num_notes = str(len(notes))
i = 0
for package in sorted(notes):
log.debug('iterating over notes... ' + str(i) + '/' + num_notes)
note = notes[package]
note['package'] = package
log.debug('\t' + str(note))
html = gen_html_note(package, note)
title = 'Notes for ' + package + ' - reproducible builds result'
destfile = NOTES_PATH + '/' + package + '_note.html'
write_html_page(title=title, body=html, destfile=destfile)
desturl = REPRODUCIBLE_URL + NOTES_URI + '/' + package + '_note.html'
log.debug("Note created: " + desturl)
i = i + 1
log.info('Created ' + str(i) + ' note pages.')
def iterate_over_issues(issues):
num_issues = str(len(issues))
for suite in SUITES:
i = 0
for issue in sorted(issues):
log.debug('iterating over issues in ' + suite +'... ' + str(i) + '/' + num_issues)
log.debug('\t' + str(issue))
html = gen_html_issue(issue, suite)
title = 'Notes about issue ' + issue + ' in ' + suite
destfile = ISSUES_PATH + '/' + suite + '/' + issue + '_issue.html'
left_nav_html = create_main_navigation(displayed_page='issues')
write_html_page(title=title, body=html, destfile=destfile,
style_note=True, left_nav_html=left_nav_html)
desturl = REPRODUCIBLE_URL + ISSUES_URI + '/' + suite + '/' + issue + '_issue.html'
log.debug("Issue created: " + desturl)
i = i + 1
log.info('Created ' + str(i) + ' issue pages for ' + suite)
def build_page(page, suite=None, arch=None):
gpage = False
if pages[page].get('global') and pages[page]['global']:
gpage = True
suite = defaultsuite
arch = defaultarch
if not gpage and suite and not arch:
print_critical_message('The architecture was not specified while ' +
'building a suite-specific page.')
sys.exit(1)
if gpage:
log.debug('Building the ' + page + ' global index page...')
title = pages[page]['title']
else:
log.debug('Building the ' + page + ' index page for ' + suite + '/' +
arch + '...')
title = pages[page]['title'].format(suite=suite, arch=arch)
page_sections = pages[page]['body']
html = ''
footnote = False
if pages[page].get('header'):
if pages[page].get('notes_hint') and pages[page][
'notes_hint'] and suite == defaultsuite:
hint = ' <em>These</em> are the packages with failures that <em>still need to be investigated</em>.'
else:
hint = ''
if pages[page].get('header_query'):
html += pages[page]['header'].format(tot=query_db(
pages[page]['header_query'].format(suite=suite,
arch=arch))[0][0],
suite=suite,
arch=arch,
hint=hint)
else:
html += pages[page].get('header')
for section in page_sections:
if gpage:
if section.get('nosuite') and section['nosuite']: # only defaults
html += build_page_section(page, section, suite, arch)[0]
else:
for suite in SUITES:
for arch in ARCHS:
log.debug('global page §' + section['status'].name +
' in ' + page + ' for ' + suite + '/' + arch)
html += build_page_section(page, section, suite,
arch)[0]
footnote = True
else:
html1, footnote1 = build_page_section(page, section, suite, arch)
html += html1
footnote = True if footnote1 else footnote
suite_arch_nav_template = None
if gpage:
destfile = DISTRO_BASE + '/index_' + page + '.html'
desturl = DISTRO_URL + '/index_' + page + '.html'
suite = defaultsuite # used for the links in create_main_navigation
else:
destfile = DISTRO_BASE + '/' + suite + '/' + arch + '/index_' + \
page + '.html'
desturl = DISTRO_URL + '/' + suite + '/' + arch + '/index_' + \
page + '.html'
suite_arch_nav_template = DISTRO_URI + '/{{suite}}/{{arch}}/index_' + \
page + '.html'
left_nav_html = create_main_navigation(
suite=suite,
arch=arch,
displayed_page=page,
suite_arch_nav_template=suite_arch_nav_template,
)
write_html_page(title=title,
body=html,
destfile=destfile,
style_note=True,
left_nav_html=left_nav_html)
log.info('"' + title + '" now available at ' + desturl)
def update_sources_db(suite, arch, sources):
# extract relevant info (package name and version) from the sources file
new_pkgs = set()
newest_version = {}
for src in deb822.Sources.iter_paragraphs(sources.split('\n')):
pkg = (src['Package'], src['Version'], suite, arch)
if 'Extra-Source-Only' in src and src['Extra-Source-Only'] == 'yes':
log.debug('Ignoring {} due to Extra-Source-Only'.format(pkg))
continue
# only keep the most recent version of a src for each package/suite/arch
key = src['Package'] + suite + arch
if key in newest_version:
oldversion = newest_version[key]
oldpackage = (src['Package'], oldversion, suite, arch)
new_pkgs.remove(oldpackage)
newest_version[key] = src['Version']
new_pkgs.add(pkg)
# get the current packages in the database
query = "SELECT name, version, suite, architecture FROM sources " + \
"WHERE suite='{}' AND architecture='{}'".format(suite, arch)
cur_pkgs = set([(p.name, p.version, p.suite, p.architecture)
for p in query_db(query)])
pkgs_to_add = []
updated_pkgs = []
different_pkgs = [x for x in new_pkgs if x not in cur_pkgs]
log.debug('Packages different in the archive and in the db: %s',
different_pkgs)
for pkg in different_pkgs:
# pkg: (name, version, suite, arch)
query = "SELECT id, version, notify_maintainer FROM sources " + \
"WHERE name='{}' AND suite='{}' AND architecture='{}'"
query = query.format(pkg[0], pkg[2], pkg[3])
try:
result = query_db(query)[0]
except IndexError: # new package
pkgs_to_add.append({
'name': pkg[0],
'version': pkg[1],
'suite': pkg[2],
'architecture': pkg[3],
})
continue
pkg_id = result[0]
old_version = result[1]
notify_maint = int(result[2])
if apt_pkg.version_compare(pkg[1], old_version) > 0:
log.debug('New version: ' + str(pkg) + ' (we had ' + old_version +
')')
updated_pkgs.append({
'update_id': pkg_id,
'name': pkg[0],
'version': pkg[1],
'suite': pkg[2],
'architecture': pkg[3],
'notify_maintainer': notify_maint,
})
# Now actually update the database:
sources_table = db_table('sources')
# updated packages
log.info('Pushing ' + str(len(updated_pkgs)) +
' updated packages to the database...')
if updated_pkgs:
transaction = conn_db.begin()
update_query = sources_table.update().\
where(sources_table.c.id == sql.bindparam('update_id'))
conn_db.execute(update_query, updated_pkgs)
transaction.commit()
# new packages
if pkgs_to_add:
log.info('Now inserting %i new sources in the database: %s',
len(pkgs_to_add), pkgs_to_add)
transaction = conn_db.begin()
conn_db.execute(sources_table.insert(), pkgs_to_add)
transaction.commit()
# RM'ed packages
cur_pkgs_name = [x[0] for x in cur_pkgs]
new_pkgs_name = [x[0] for x in new_pkgs]
rmed_pkgs = [x for x in cur_pkgs_name if x not in new_pkgs_name]
log.info('Now deleting %i removed packages: %s', len(rmed_pkgs), rmed_pkgs)
rmed_pkgs_id = []
pkgs_to_rm = []
query = "SELECT id FROM sources WHERE name='{}' AND suite='{}' " + \
"AND architecture='{}'"
for pkg in rmed_pkgs:
result = query_db(query.format(pkg, suite, arch))
rmed_pkgs_id.append({'deleteid': result[0][0]})
pkgs_to_rm.append({'name': pkg, 'suite': suite, 'architecture': arch})
log.debug('removed packages ID: %s',
[str(x['deleteid']) for x in rmed_pkgs_id])
log.debug('removed packages: %s', pkgs_to_rm)
if rmed_pkgs_id:
transaction = conn_db.begin()
results_table = db_table('results')
schedule_table = db_table('schedule')
notes_table = db_table('notes')
removed_packages_table = db_table('removed_packages')
delete_results_query = results_table.delete().\
where(results_table.c.package_id == sql.bindparam('deleteid'))
delete_schedule_query = schedule_table.delete().\
where(schedule_table.c.package_id == sql.bindparam('deleteid'))
delete_notes_query = notes_table.delete().\
where(notes_table.c.package_id == sql.bindparam('deleteid'))
delete_sources_query = sources_table.delete().\
where(sources_table.c.id == sql.bindparam('deleteid'))
conn_db.execute(delete_results_query, rmed_pkgs_id)
conn_db.execute(delete_schedule_query, rmed_pkgs_id)
conn_db.execute(delete_notes_query, rmed_pkgs_id)
conn_db.execute(delete_sources_query, rmed_pkgs_id)
conn_db.execute(removed_packages_table.insert(), pkgs_to_rm)
transaction.commit()
# finally check whether the db has the correct number of packages
query = "SELECT count(*) FROM sources WHERE suite='{}' " + \
"AND architecture='{}'"
pkgs_end = query_db(query.format(suite, arch))
count_new_pkgs = len(set([x[0] for x in new_pkgs]))
if int(pkgs_end[0][0]) != count_new_pkgs:
print_critical_message('AH! The number of source in the Sources file' +
' is different than the one in the DB!')
log.critical('source in the debian archive for the %s suite: %s',
suite, str(count_new_pkgs))
log.critical('source in the reproducible db for the %s suite: %s',
suite, str(pkgs_end[0][0]))
sys.exit(1)
if pkgs_to_add:
log.info('Building pages for the new packages')
gen_packages_html([Package(x['name']) for x in pkgs_to_add],
no_clean=True)
else:
log.error('Failed to get the ' + suite + 'sources')
continue
query = "SELECT s.name " + \
"FROM results AS r JOIN sources AS s ON r.package_id=s.id " + \
"WHERE r.status='unreproducible' AND s.suite='{suite}'"
try:
pkgs = [x[0] for x in query_db(query.format(suite=suite))]
except IndexError:
log.error('Looks like there are no unreproducible packages...')
p = Popen(('dd-list --stdin --sources ' + sources.name).split(),
stdout=PIPE, stdin=PIPE, stderr=PIPE)
out, err = p.communicate(input=('\n'.join(pkgs)).encode())
if err:
log.error('dd-list printed some errors:\n' + err.decode())
log.debug('dd-list output:\n' + out.decode())
html = '<p>The following maintainers and uploaders are listed '
html += 'for packages in ' + suite + ' which have built '
html += 'unreproducibly. Please note that the while the link '
html += 'always points to the amd64 version, it\'s possible that'
html += 'the unreproducibility is only present in another architecture(s).</p>\n<p><pre>'
out = out.decode().splitlines()
get_mail = re.compile('<(.*)>')
for line in out:
if line[0:3] == ' ':
line = line.strip().split(None, 1)
html += ' '
# the final strip() is to avoid a newline
html += Package(line[0]).html_link(suite, arch).strip()
try:
def rest(scheduling_args, requester, local, suite, arch):
"Actually schedule a package for a single suite on a single arch."
# Shorter names
reason = scheduling_args.message
issue = scheduling_args.issue
status = scheduling_args.status
built_after = scheduling_args.after
built_before = scheduling_args.before
packages = scheduling_args.packages
artifacts = scheduling_args.keep_artifacts
notify = scheduling_args.notify
notify_on_start = scheduling_args.notify_on_start
dry_run = scheduling_args.dry_run
log.info("Scheduling packages in %s/%s", arch, suite)
ids = []
pkgs = []
query1 = """SELECT id FROM sources WHERE name='{pkg}' AND suite='{suite}'
AND architecture='{arch}'"""
query2 = """SELECT p.date_build_started
FROM sources AS s JOIN schedule as p ON p.package_id=s.id
WHERE p.package_id='{id}'"""
for pkg in set(packages):
# test whether the package actually exists
result = query_db(query1.format(pkg=pkg, suite=suite, arch=arch))
# tests whether the package is already building
try:
result2 = query_db(query2.format(id=result[0][0]))
except IndexError:
log.error('%sThe package %s is not available in %s/%s%s',
bcolors.FAIL, pkg, suite, arch, bcolors.ENDC)
continue
try:
if not result2[0][0]:
ids.append(result[0][0])
pkgs.append(pkg)
else:
log.warning(bcolors.WARN + 'The package ' + pkg + ' is ' +
'already building, not scheduling it.' +
bcolors.ENDC)
except IndexError:
# it's not in the schedule
ids.append(result[0][0])
pkgs.append(pkg)
def compose_irc_message():
"One-shot closure to limit scope of the following local variables."
blablabla = '✂…' if len(' '.join(pkgs)) > 257 else ''
packages_txt = str(len(ids)) + ' packages ' if len(pkgs) > 1 else ''
trailing = ' - artifacts will be preserved' if artifacts else ''
trailing += ' - with irc notification' if notify else ''
trailing += ' - notify on start too' if notify_on_start else ''
message = requester + ' scheduled ' + packages_txt + \
'in ' + suite + '/' + arch
if reason:
message += ', reason: \'' + reason + '\''
message += ': ' + ' '.join(pkgs)[0:256] + blablabla + trailing
return message
info_msg = compose_irc_message()
del compose_irc_message
# these packages are manually scheduled, so should have high priority,
# so schedule them in the past, so they are picked earlier :)
# the current date is subtracted twice, so it sorts before early scheduling
# schedule on the full hour so we can recognize them easily
epoch = int(time.time())
now = datetime.now()
days = int(now.strftime('%j')) * 2
hours = int(now.strftime('%H')) * 2
minutes = int(now.strftime('%M'))
time_delta = timedelta(days=days, hours=hours, minutes=minutes)
date = (now - time_delta).strftime('%Y-%m-%d %H:%M')
log.debug('date_scheduled = ' + date + ' time_delta = ' + str(time_delta))
# a single person can't schedule more than 500 packages in the same day; this
# is actually easy to bypass, but let's give some trust to the Debian people
query = """SELECT count(*) FROM manual_scheduler
WHERE requester = '{}' AND date_request > '{}'"""
try:
amount = int(
query_db(query.format(requester, int(time.time() - 86400)))[0][0])
except IndexError:
amount = 0
log.debug(requester + ' already scheduled ' + str(amount) +
' packages today')
if amount + len(ids) > 500 and not local:
log.error(
bcolors.FAIL + 'You have exceeded the maximum number of manual ' +
'reschedulings allowed for a day. Please ask in ' +
'#debian-reproducible if you need to schedule more packages.' +
bcolors.ENDC)
sys.exit(1)
# do the actual scheduling
add_to_schedule = []
update_schedule = []
save_schedule = []
artifacts_value = 1 if artifacts else 0
if notify_on_start:
do_notify = 2
elif notify or artifacts:
do_notify = 1
else:
do_notify = 0
schedule_table = db_table('schedule')
if ids:
existing_pkg_ids = dict(
query_db(
sql.select([
schedule_table.c.package_id,
schedule_table.c.id,
]).where(schedule_table.c.package_id.in_(ids))))
for id in ids:
if id in existing_pkg_ids:
update_schedule.append({
'update_id': existing_pkg_ids[id],
'package_id': id,
'date_scheduled': date,
'save_artifacts': artifacts_value,
'notify': str(do_notify),
'scheduler': requester,
})
else:
add_to_schedule.append({
'package_id': id,
'date_scheduled': date,
'save_artifacts': artifacts_value,
'notify': str(do_notify),
'scheduler': requester,
})
save_schedule.append({
'package_id': id,
'requester': requester,
'date_request': epoch,
})
log.debug('Packages about to be scheduled: ' + str(add_to_schedule) +
str(update_schedule))
update_schedule_query = schedule_table.update().\
where(schedule_table.c.id == sql.bindparam('update_id'))
insert_schedule_query = schedule_table.insert()
insert_manual_query = db_table('manual_scheduler').insert()
if not dry_run:
transaction = conn_db.begin()
if add_to_schedule:
conn_db.execute(insert_schedule_query, add_to_schedule)
if update_schedule:
conn_db.execute(update_schedule_query, update_schedule)
if save_schedule:
conn_db.execute(insert_manual_query, save_schedule)
transaction.commit()
else:
log.info('Ran with --dry-run, scheduled nothing')
log.info(bcolors.GOOD + info_msg + bcolors.ENDC)
if not (local
and requester == "jenkins maintenance job") and len(ids) != 0:
if not dry_run:
# Always announce on -changes
irc_msg(info_msg, 'debian-reproducible-changes')
# Announce some messages on main channel
if notify_on_start or artifacts:
irc_msg(info_msg)
result = sorted(query_db(query))
log.info('\tprocessing ' + str(len(result)))
keys = ['package', 'version', 'suite', 'architecture', 'status', 'build_date']
crossarchkeys = ['package', 'version', 'suite', 'status']
archdetailkeys = ['architecture', 'version', 'status', 'build_date']
# crossarch is a dictionary of all packages used to build a summary of the
# package's test results across all archs (for suite=unstable only)
crossarch = {}
crossarchversions = {}
for row in result:
pkg = dict(zip(keys, row))
log.debug(pkg)
output.append(pkg)
# tracker.d.o should only care about results in testing
if pkg['suite'] == 'buster':
package = pkg['package']
if package in crossarch:
# compare statuses to get cross-arch package status
status1 = crossarch[package]['status']
status2 = pkg['status']
newstatus = ''
# compare the versions (only keep most up to date!)
version1 = crossarch[package]['version']
version2 = pkg['version']
def parse_args():
parser = argparse.ArgumentParser(
description='Reschedule packages to re-test their reproducibility',
epilog='The build results will be announced on the #debian-reproducible'
' IRC channel if -n is provided. Specifying two or more filters'
' (namely two or more -r/-i/-t/-b) means "all packages with that'
' issue AND that status AND that date". Blacklisted package '
"can't be selected by a filter, but needs to be explitely listed"
' in the package list.')
parser.add_argument('--dry-run', action='store_true')
parser.add_argument('--null',
action='store_true',
help='The arguments are '
'considered null-separated and coming from stdin.')
parser.add_argument('-k',
'--keep-artifacts',
action='store_true',
help='Save artifacts (for further offline study).')
parser.add_argument('-n',
'--notify',
action='store_true',
help='Notify the channel when the build finishes.')
parser.add_argument(
'--notify-on-start',
action='store_true',
help='Also '
'notify when the build starts, linking to the build url.')
parser.add_argument(
'-m',
'--message',
default='',
help='A text to be sent to the IRC channel when notifying' +
' about the scheduling.')
parser.add_argument('-r',
'--status',
required=False,
help='Schedule all package with this status.')
parser.add_argument('-i',
'--issue',
required=False,
help='Schedule all packages with this issue.')
parser.add_argument('-t',
'--after',
required=False,
help='Schedule all packages built after this date.')
parser.add_argument('-b',
'--before',
required=False,
help='Schedule all packages built before this date.')
parser.add_argument('-a',
'--architecture',
required=False,
default='amd64',
help='Specify the architectures to schedule in ' +
'(space or comma separated).' + "Default: 'amd64'.")
parser.add_argument(
'-s',
'--suite',
required=False,
default='unstable',
help=
"Specify the suites to schedule in (space or comma separated). Default: 'unstable'."
)
parser.add_argument('packages',
metavar='package',
nargs='*',
help='Space seperated list of packages to reschedule.')
# only consider "unknown_args", i.e. the arguments that the common.py script
# doesn't know about and therefor ignored.
scheduling_args = parser.parse_known_args(unknown_args)[0]
if scheduling_args.null:
scheduling_args = parser.parse_known_args(
sys.stdin.read().split('\0'))[0]
scheduling_args.packages = [x for x in scheduling_args.packages if x]
if scheduling_args.notify_on_start:
scheduling_args.notify = True
# this variable is expected to come from the remote host
try:
requester = os.environ['LC_USER']
except KeyError:
log.critical(
bcolors.FAIL + 'You should use the provided script to '
'schedule packages. Ask in #debian-reproducible if you have '
'trouble with that.' + bcolors.ENDC)
sys.exit(1)
# this variable is set by reproducible scripts and so it only available in calls made on the local host (=main node)
try:
local = True if os.environ['LOCAL_CALL'] == 'true' else False
except KeyError:
local = False
# Shorter names
suites = [
x.strip()
for x in re.compile(r'[, \t]').split(scheduling_args.suite or "")
]
suites = [x for x in suites if x]
archs = [
x.strip() for x in re.compile(r'[, \t]').split(
scheduling_args.architecture or "")
]
archs = [x for x in archs if x]
reason = scheduling_args.message
issue = scheduling_args.issue
status = scheduling_args.status
built_after = scheduling_args.after
built_before = scheduling_args.before
packages = scheduling_args.packages
artifacts = scheduling_args.keep_artifacts
notify = scheduling_args.notify
notify_on_start = scheduling_args.notify_on_start
dry_run = scheduling_args.dry_run
log.debug('Requester: ' + requester)
log.debug('Dry run: ' + str(dry_run))
log.debug('Local call: ' + str(local))
log.debug('Reason: ' + reason)
log.debug('Artifacts: ' + str(artifacts))
log.debug('Notify: ' + str(notify))
log.debug('Debug url: ' + str(notify_on_start))
log.debug('Issue: ' + issue if issue else str(None))
log.debug('Status: ' + status if status else str(None))
log.debug('Date: after ' + built_after if built_after else str(None) +
' before ' + built_before if built_before else str(None))
log.debug('Suites: ' + repr(suites))
log.debug('Architectures: ' + repr(archs))
log.debug('Packages: ' + ' '.join(packages))
if not suites[0]:
log.critical('You need to specify the suite name')
sys.exit(1)
if set(suites) - set(SUITES): # Some command-line suites don't exist.
log.critical('Some of the specified suites %r are not being tested.',
suites)
log.critical('Please choose among ' + ', '.join(SUITES) + '.')
sys.exit(1)
if set(archs) - set(ARCHS): # Some command-line archs don't exist.
log.critical('Some of the specified archs %r are not being tested.',
archs)
log.critical('Please choose among' + ', '.join(ARCHS) + '.')
sys.exit(1)
if issue or status or built_after or built_before:
# Note: this .extend() operation modifies scheduling_args.packages, which
# is used by rest()
for suite in suites:
for arch in archs:
packages.extend(
packages_matching_criteria(
arch,
suite,
(issue, status, built_after, built_before),
))
del arch
del suite
if len(packages) > 50 and notify:
log.critical(bcolors.RED + bcolors.BOLD)
subprocess.run(('figlet', 'No.'))
log.critical(bcolors.FAIL + 'Do not reschedule more than 50 packages ',
'with notification.\nIf you think you need to do this, ',
'please discuss this with the IRC channel first.',
bcolors.ENDC)
sys.exit(1)
if artifacts:
log.info('The artifacts of the build(s) will be saved to the location '
'mentioned at the end of the build log(s).')
if notify_on_start:
log.info('The channel will be notified when the build starts')
return scheduling_args, requester, local, suites, archs
def load_notes():
"""
format:
{ 'package_name': [
{'suite': 'unstable', 'version': '0.0', 'comments': None,
'bugs': [1234, 5678], 'issues': ['blalba','auauau']},
{'suite': 'stretch', 'version': None, 'comments': 'strstr',
'bugs': [], 'issues': []}],
'package_name':<etc> }
"""
with open(NOTES) as fd:
original = yaml.load(fd)
log.info("notes loaded. There are " + str(len(original)) +
" packages listed")
notes = {}
for pkg in sorted(original):
assert isinstance(pkg, str)
try:
assert 'version' in original[pkg]
except AssertionError:
print_critical_message(pkg + ' does not include a version')
irc_msg('The note for ' + pkg + ' does not include a version.')
query = """SELECT s.id, s.version, s.suite
FROM results AS r JOIN sources AS s ON r.package_id=s.id
WHERE s.name='{pkg}' AND r.status != ''"""
# AND s.architecture='amd64'"""
query = query.format(pkg=pkg)
result = query_db(query)
if not result:
log.info('Warning: This query produces no results: ' + query +
'\nThis means there is no tested ' +
'package with the name ' + pkg)
try:
irc_msg(
"There is problem with the note for {} (it may "
"have been removed from the archive). Please check {} and {}"
.format(pkg, os.environ['BUILD_URL'],
"https://tracker.debian.org/pkg/" + pkg))
except KeyError:
log.error(
'There is a problem with the note for %s - please '
'check.', pkg)
else:
notes[pkg] = []
for suite in result:
pkg_details = {}
# https://image-store.slidesharecdn.com/c2c44a06-5e28-4296-8d87-419529750f6b-original.jpeg
try:
if apt_pkg.version_compare(str(original[pkg]['version']),
str(suite[1])) > 0:
continue
except KeyError:
pass
pkg_details['suite'] = suite[2]
try:
pkg_details['version'] = original[pkg]['version']
except KeyError:
pkg_details['version'] = ''
pkg_details['comments'] = original[pkg]['comments'] if \
'comments' in original[pkg] else None
pkg_details['bugs'] = original[pkg]['bugs'] if \
'bugs' in original[pkg] else []
pkg_details['issues'] = original[pkg]['issues'] if \
'issues' in original[pkg] else []
pkg_details['id'] = int(suite[0])
log.debug('adding %s => %s', pkg, pkg_details)
notes[pkg].append(pkg_details)
log.info("notes checked. There are " + str(len(notes)) +
" packages listed")
return notes
def gen_packages_html(packages, no_clean=False):
"""
generate the /rb-pkg/package.HTML pages.
packages should be a list of Package objects.
"""
total = len(packages)
log.info('Generating the pages of ' + str(total) + ' package(s)')
for package in sorted(packages, key=lambda x: x.name):
assert isinstance(package, Package)
gen_history_page(package)
for arch in ARCHS:
gen_history_page(package, arch)
pkg = package.name
notes_uri = ''
notes_file = NOTES_PATH + '/' + pkg + '_note.html'
if os.access(notes_file, os.R_OK):
notes_uri = NOTES_URI + '/' + pkg + '_note.html'
for suite in SUITES:
for arch in ARCHS:
status = package.builds[suite][arch].status
version = package.builds[suite][arch].version
build_date = package.builds[suite][arch].build_date
if status is None: # the package is not in the checked suite
continue
log.debug('Generating the page of %s/%s/%s @ %s built at %s',
pkg, suite, arch, version, build_date)
suitearch_section_html, default_view, reproducible = \
gen_suitearch_section(package, suite, arch)
history_uri = '{}/{}.html'.format(HISTORY_URI, pkg)
history_archs = []
for a in ARCHS:
history_archs.append({
'history_arch':
a,
'history_arch_uri':
'{}/{}/{}.html'.format(HISTORY_URI, a, pkg)
})
project_links = renderer.render(project_links_template)
desturl = '{}{}/{}/{}/{}.html'.format(
REPRODUCIBLE_URL,
RB_PKG_URI,
suite,
arch,
pkg,
)
navigation_html = renderer.render(
package_navigation_template, {
'package': pkg,
'suite': suite,
'arch': arch,
'version': version,
'history_uri': history_uri,
'history_archs': history_archs,
'notes_uri': notes_uri,
'notify_maintainer': package.notify_maint,
'suitearch_section_html': suitearch_section_html,
'project_links_html': project_links,
'reproducible': reproducible,
'dashboard_url': DISTRO_URL,
'desturl': desturl,
})
body_html = renderer.render(package_page_template, {
'default_view': default_view,
})
destfile = os.path.join(RB_PKG_PATH, suite, arch,
pkg + '.html')
title = pkg + ' - reproducible builds result'
write_html_page(title=title,
body=body_html,
destfile=destfile,
no_header=True,
noendpage=True,
left_nav_html=navigation_html)
log.debug("Package page generated at " + desturl)
# Optionally generate a page in which the main iframe shows the
# diffoscope results by default. Needed for navigation between
# diffoscope pages for different suites/archs
eversion = strip_epoch(version)
dbd_links = get_and_clean_dbd_links(pkg, eversion, suite, arch,
status)
# only generate the diffoscope page if diffoscope results exist
if 'dbd_uri' in dbd_links:
body_html = renderer.render(
package_page_template, {
'default_view': dbd_links['dbd_uri'],
})
destfile = dbd_links['dbd_page_file']
desturl = REPRODUCIBLE_URL + "/" + dbd_links['dbd_page_uri']
title = "{} ({}) diffoscope results in {}/{}".format(
pkg, version, suite, arch)
write_html_page(title=title,
body=body_html,
destfile=destfile,
no_header=True,
noendpage=True,
left_nav_html=navigation_html)
log.debug("Package diffoscope page generated at " +
desturl)
if not no_clean:
purge_old_pages() # housekeep is always good
