def gen_history_page(package, arch=None):
keys = [
'build date', 'version', 'suite', 'architecture', 'result',
'build duration', 'node1', 'node2', 'job'
]
context = {}
try:
package.history[
0] # we don't care about the actual value, it jst need to exist
except IndexError:
context['arch'] = arch
else:
context['keys'] = [{'key': key} for key in keys]
rows = []
for r in package.history:
# make a copy, since we modify in place
record = dict(r)
# FIXME - hacky, should be rethought one day
# skip records for suites that are unknown to us (i.e. other distro)
if record['suite'] not in SUITES:
continue
# skip records for other archs if we care about arch
if arch and record['architecture'] != arch:
continue
# remove trailing .debian.net from hostnames
record['node1'] = shorten_if_debiannet(record['node1'])
record['node2'] = shorten_if_debiannet(record['node2'])
# add icon to result
status = Status.get(record['result'])
result_html = '{spokenstatus}<img src="/static/{icon}" alt="{spokenstatus}" title="{spokenstatus}"/>'
record['result'] = result_html.format(
icon=status.value.icon, spokenstatus=status.value.spokenstatus)
# human formatting of build duration
record['build duration'] = convert_into_hms_string(
int(record['build duration']))
row_items = [{'item': record[key]} for key in keys]
rows.append({'row_items': row_items})
context['rows'] = rows
html = renderer.render(package_history_template, context)
if arch:
destfile = os.path.join(HISTORY_PATH, arch, package.name + '.html')
else:
destfile = os.path.join(HISTORY_PATH, package.name + '.html')
title = 'build history of {}'.format(package.name)
if arch:
title += ' on {}'.format(arch)
write_html_page(title=title, body=html, destfile=destfile, noendpage=True)
def iterate_over_notes(notes):
num_notes = str(len(notes))
i = 0
for package in sorted(notes):
log.debug('iterating over notes... ' + str(i) + '/' + num_notes)
note = notes[package]
note['package'] = package
log.debug('\t' + str(note))
html = gen_html_note(package, note)
title = 'Notes for ' + package + ' - reproducible builds result'
destfile = NOTES_PATH + '/' + package + '_note.html'
write_html_page(title=title, body=html, destfile=destfile)
desturl = REPRODUCIBLE_URL + NOTES_URI + '/' + package + '_note.html'
log.debug("Note created: " + desturl)
i = i + 1
log.info('Created ' + str(i) + ' note pages.')
def create_index_page(suite, arch):
title = 'Package sets in %s/%s' % (suite, arch)
body = create_pkgset_navigation(suite, arch)
destfile = os.path.join(DISTRO_BASE, suite, arch, "index_pkg_sets.html")
suite_arch_nav_template = DISTRO_URI + \
'/{{suite}}/{{arch}}/index_pkg_sets.html'
left_nav_html = create_main_navigation(
suite=suite,
arch=arch,
displayed_page='pkg_set',
suite_arch_nav_template=suite_arch_nav_template,
ignore_experimental=True,
)
log.info("Creating pkgset index page for %s/%s.", suite, arch)
write_html_page(title=title,
body=body,
destfile=destfile,
left_nav_html=left_nav_html)
def iterate_over_issues(issues):
num_issues = str(len(issues))
for suite in SUITES:
i = 0
for issue in sorted(issues):
log.debug('iterating over issues in ' + suite +'... ' + str(i) + '/' + num_issues)
log.debug('\t' + str(issue))
html = gen_html_issue(issue, suite)
title = 'Notes about issue ' + issue + ' in ' + suite
destfile = ISSUES_PATH + '/' + suite + '/' + issue + '_issue.html'
left_nav_html = create_main_navigation(displayed_page='issues')
write_html_page(title=title, body=html, destfile=destfile,
style_note=True, left_nav_html=left_nav_html)
desturl = REPRODUCIBLE_URL + ISSUES_URI + '/' + suite + '/' + issue + '_issue.html'
log.debug("Issue created: " + desturl)
i = i + 1
log.info('Created ' + str(i) + ' issue pages for ' + suite)
def index_issues(issues, scorefuncs):
firstscorefunc = next(iter(scorefuncs.values()))
templ = "\n<table class=\"body\">\n" + tab + "<tr>\n" + tab*2 + "<th>\n" \
+ tab*3 + "Identified issues\n" + tab*2 + "</th>\n" + tab*2 + "<th>\n" \
+ "".join(
tab*3 + k + "\n" + tab*2 + "</th>\n" + tab*2 + "<th>\n"
for k in scorefuncs.keys()) \
+ tab*3 + "Affected packages<br/>\n" \
+ tab*3 + "(the 1/4 most-popular ones (within the issue) are underlined)\n" \
+ tab*2 + "</th>\n" + tab + "</tr>\n"
html = (tab*2).join(templ.splitlines(True))
for issue in sorted(issues, key=lambda issue: sort_issues(firstscorefunc, issue)):
html += tab*3 + '<tr>\n'
html += tab*4 + '<td><a href="' + ISSUES_URI + '/' + defaultsuite + \
'/'+ issue + '_issue.html">' + issue.replace("_", " ") + '</a></td>\n'
issues_list = issues_count.get(issue, [])
for scorefunc in scorefuncs.values():
html += tab*4 + '<td><b>' + str(scorefunc(issues_list)) + '</b></td>\n'
html += tab*4 + '<td>\n'
issues_with_popcon = issues_popcon_annotate(issues_list)
issue_strings = [
'<span %stitle="popcon score: %s">%s</span>' % (
'class="package-popular" ' if p[2] else '', p[1], p[0]
) for p in issues_with_popcon]
html += tab*5 + ', '.join(issue_strings) + '\n'
html += tab*4 + '</td>\n'
html += tab*3 + '</tr>\n'
html += tab*2 + '</table>\n'
html += tab*2 + '<p>For a total of <b>' + \
str(len([x for x in notes if notes[x].get('issues')])) + \
'</b> packages categorized in <b>' + str(len(issues)) + \
'</b> issues.</p>'
html += tab*2 + '<p>' + NOTESGIT_DESCRIPTION + '</p>'
title = 'Known issues related to reproducible builds'
destfile = DISTRO_BASE + '/index_issues.html'
desturl = DISTRO_URL + '/index_issues.html'
left_nav_html = create_main_navigation(displayed_page='issues')
write_html_page(title=title, body=html, destfile=destfile,
left_nav_html=left_nav_html)
log.info('Issues index now available at ' + desturl)
def generate_oldies(arch):
log.info('Building the oldies page for ' + arch + '...')
title = 'Oldest results on ' + arch
html = ''
for suite in SUITES:
query = select([
sources.c.suite, sources.c.architecture, sources.c.name,
results.c.status, results.c.build_date
]).select_from(results.join(sources)).where(
and_(sources.c.suite == bindparam('suite'),
sources.c.architecture == bindparam('arch'),
results.c.status != 'blacklisted')).order_by(
results.c.build_date).limit(15)
text = Template('Oldest results on $suite/$arch:')
rows = query_db(query.params({'arch': arch, 'suite': suite}))
html += build_leading_text_section({'text': text}, rows, suite, arch)
html += '<p><table class="scheduled">\n' + tab
html += '<tr><th class="center">#</th><th class="center">suite</th><th class="center">arch</th>'
html += '<th class="center">source package</th><th class="center">status</th><th class="center">build date</th></tr>\n'
for row in rows:
# 0: suite, 1: arch, 2: pkg name 3: status 4: build date
pkg = row[2]
html += tab + '<tr><td> </td><td>' + row[0] + '</td>'
html += '<td>' + row[1] + '</td><td><code>'
html += Package(pkg).html_link(row[0], row[1])
html += '</code></td><td>' + convert_into_status_html(str(
row[3])) + '</td><td>' + row[4] + '</td></tr>\n'
html += '</table></p>\n'
destfile = DISTRO_BASE + '/index_' + arch + '_oldies.html'
desturl = DISTRO_URL + '/index_' + arch + '_oldies.html'
left_nav_html = create_main_navigation(arch=arch)
write_html_page(title=title,
body=html,
destfile=destfile,
style_note=True,
refresh_every=60,
left_nav_html=left_nav_html)
log.info("Page generated at " + desturl)
def build_page(page, suite=None, arch=None):
gpage = False
if pages[page].get('global') and pages[page]['global']:
gpage = True
suite = defaultsuite
arch = defaultarch
if not gpage and suite and not arch:
print_critical_message('The architecture was not specified while ' +
'building a suite-specific page.')
sys.exit(1)
if gpage:
log.debug('Building the ' + page + ' global index page...')
title = pages[page]['title']
else:
log.debug('Building the ' + page + ' index page for ' + suite + '/' +
arch + '...')
title = pages[page]['title'].format(suite=suite, arch=arch)
page_sections = pages[page]['body']
html = ''
footnote = False
if pages[page].get('header'):
if pages[page].get('notes_hint') and pages[page][
'notes_hint'] and suite == defaultsuite:
hint = ' <em>These</em> are the packages with failures that <em>still need to be investigated</em>.'
else:
hint = ''
if pages[page].get('header_query'):
html += pages[page]['header'].format(tot=query_db(
pages[page]['header_query'].format(suite=suite,
arch=arch))[0][0],
suite=suite,
arch=arch,
hint=hint)
else:
html += pages[page].get('header')
for section in page_sections:
if gpage:
if section.get('nosuite') and section['nosuite']: # only defaults
html += build_page_section(page, section, suite, arch)[0]
else:
for suite in SUITES:
for arch in ARCHS:
log.debug('global page §' + section['status'].name +
' in ' + page + ' for ' + suite + '/' + arch)
html += build_page_section(page, section, suite,
arch)[0]
footnote = True
else:
html1, footnote1 = build_page_section(page, section, suite, arch)
html += html1
footnote = True if footnote1 else footnote
suite_arch_nav_template = None
if gpage:
destfile = DISTRO_BASE + '/index_' + page + '.html'
desturl = DISTRO_URL + '/index_' + page + '.html'
suite = defaultsuite # used for the links in create_main_navigation
else:
destfile = DISTRO_BASE + '/' + suite + '/' + arch + '/index_' + \
page + '.html'
desturl = DISTRO_URL + '/' + suite + '/' + arch + '/index_' + \
page + '.html'
suite_arch_nav_template = DISTRO_URI + '/{{suite}}/{{arch}}/index_' + \
page + '.html'
left_nav_html = create_main_navigation(
suite=suite,
arch=arch,
displayed_page=page,
suite_arch_nav_template=suite_arch_nav_template,
)
write_html_page(title=title,
body=html,
destfile=destfile,
style_note=True,
left_nav_html=left_nav_html)
log.info('"' + title + '" now available at ' + desturl)
def create_pkgset_page_and_graphs(suite, arch, stats, pkgset_name):
html_body = ""
html_body += create_pkgset_navigation(suite, arch, pkgset_name)
pkgset_context = ({
'pkgset_name':
pkgset_name,
'suite':
suite,
'arch':
arch,
'pkg_symbol_legend_html':
renderer.render(pkg_legend_template, {}),
})
png_file, png_href = stats_png_file_href(suite, arch, pkgset_name)
thumb_file, thumb_href = stats_thumb_file_href(suite, arch, pkgset_name)
yesterday_timestamp = (datetime.now() - timedelta(days=1)).timestamp()
if (not os.access(png_file, os.R_OK)
or os.stat(png_file).st_mtime < yesterday_timestamp):
create_pkgset_graph(png_file, suite, arch, pkgset_name)
check_call(
['convert', png_file, '-adaptive-resize', '160x80', thumb_file])
pkgset_context['png'] = png_href
other_archs = [a for a in ARCHS if a != arch]
pkgset_context['other_archs']= \
gen_other_arch_context(other_archs, suite, pkgset_name)
pkgset_context['status_details'] = []
status_cutename_descriptions = [
('FTBR', 'bad', 'failed to build reproducibly'),
('FTBFS', 'ugly', 'failed to build from source'),
('rest', 'rest',
'are either in depwait state, blacklisted, not for us, or cannot be downloaded'
),
('reproducible', 'good', 'successfully build reproducibly'),
]
for (status, cutename, description) in status_cutename_descriptions:
icon_html = ''
if status == 'rest':
for x in ['depwait', 'blacklisted', 'NFU', 'E404']:
s = Status.get(x)
icon_html += gen_status_link_icon(s.value.name, None,
s.value.icon, suite, arch)
else:
s = Status.get(status)
icon_html += gen_status_link_icon(s.value.name, None, s.value.icon,
suite, arch)
details_context = {
'icon_html':
icon_html,
'description':
description,
'package_list_html':
''.join(
[Package(x).html_link(suite, arch) for x in stats[cutename]]),
'status_count':
stats["count_" + cutename],
'status_percent':
stats["percent_" + cutename],
}
if (status in ('reproducible', 'FTBR')
or stats["count_" + cutename] != 0):
pkgset_context['status_details'].append(details_context)
html_body += renderer.render(pkgset_details_template, pkgset_context)
title = '%s package set for %s/%s' % \
(pkgset_name, suite, arch)
page = "pkg_set_" + pkgset_name + ".html"
destfile = os.path.join(DISTRO_BASE, suite, arch, page)
suite_arch_nav_template = DISTRO_URI + '/{{suite}}/{{arch}}/' + page
left_nav_html = create_main_navigation(
suite=suite,
arch=arch,
displayed_page='pkg_set',
suite_arch_nav_template=suite_arch_nav_template,
ignore_experimental=True,
)
log.info("Creating meta pkgset page for %s in %s/%s.", pkgset_name, suite,
arch)
write_html_page(title=title,
body=html_body,
destfile=destfile,
left_nav_html=left_nav_html)
lack_rbuild())
html += _gen_packages_html(
'have been built but don\'t have a .buildinfo file:', lack_buildinfo())
return html
if __name__ == '__main__':
html = '<p>This page lists unexpected things a human should look at and '
html += 'fix, like packages with an incoherent status or files that '
html += 'should not be there. Some of these breakages are caused by '
html += 'bugs in <a href="https://salsa.debian.org/reproducible-builds/diffoscope">diffoscope</a> '
html += 'while others are probably due to bugs in the scripts run by jenkins. '
html += '<em>Please help making this page empty!</em></p>\n'
breakages = gen_html()
if breakages:
html += breakages
else:
html += '<p><b>COOL!!!</b> Everything is GOOD and not a single issue was '
html += 'detected. <i>Enjoy!</i></p>'
title = 'Breakage on the Debian pages of tests.reproducible-builds.org'
destfile = DISTRO_BASE + '/index_breakages.html'
desturl = DISTRO_URL + '/index_breakages.html'
left_nav_html = create_main_navigation(displayed_page='breakages')
write_html_page(title,
html,
destfile,
style_note=True,
left_nav_html=left_nav_html)
log.info('Breakages page created at ' + desturl)
def generate_schedule(arch):
""" the schedule pages are very different than others index pages """
log.info('Building the schedule index page for ' + arch + '...')
title = 'Packages currently scheduled on ' + arch + ' for testing for build reproducibility'
# 'AND h.name=s.name AND h.suite=s.suite AND h.architecture=s.architecture'
# in this query and the query below is needed due to not using package_id
# in the stats_build table, which should be fixed...
averagesql = select([
func.coalesce(func.avg(cast(stats_build.c.build_duration, Integer)), 0)
]).where(
and_(
stats_build.c.status.in_(('reproducible', 'FTBR')),
stats_build.c.name == sources.c.name,
stats_build.c.suite == sources.c.suite,
stats_build.c.architecture == sources.c.architecture,
)).as_scalar()
query = select([
schedule.c.date_scheduled, sources.c.suite, sources.c.architecture,
sources.c.name, results.c.status, results.c.build_duration, averagesql
]).select_from(sources.join(schedule).join(results, isouter=True)).where(
and_(
schedule.c.date_build_started == None,
sources.c.architecture == bindparam('arch'),
)).order_by(schedule.c.date_scheduled)
text = Template(
'$tot packages are currently scheduled for testing on $arch:')
html = ''
rows = query_db(query.params({'arch': arch}))
html += build_leading_text_section({'text': text}, rows, defaultsuite,
arch)
html += generate_live_status_table(arch)
html += '<p><table class="scheduled">\n' + tab
html += '<tr><th class="center">#</th><th class="center">scheduled at</th><th class="center">suite</th>'
html += '<th class="center">arch</th><th class="center">source package</th><th class="center">previous build status</th><th class="center">previous build duration</th><th class="center">average build duration</th></tr>\n'
for row in rows:
# 0: date_scheduled, 1: suite, 2: arch, 3: pkg name 4: previous status 5: previous build duration 6. avg build duration
pkg = row[3]
duration = convert_into_hms_string(row[5])
avg_duration = convert_into_hms_string(row[6])
html += tab + '<tr><td> </td><td>' + row[0] + '</td>'
html += '<td>' + row[1] + '</td><td>' + row[2] + '</td><td><code>'
html += Package(pkg).html_link(row[1], row[2])
html += '</code></td><td>' + convert_into_status_html(
str(row[4])
) + '</td><td>' + duration + '</td><td>' + avg_duration + '</td></tr>\n'
html += '</table></p>\n'
destfile = DISTRO_BASE + '/index_' + arch + '_scheduled.html'
desturl = DISTRO_URL + '/index_' + arch + '_scheduled.html'
suite_arch_nav_template = DISTRO_URI + '/index_{{arch}}_scheduled.html'
left_nav_html = create_main_navigation(
arch=arch,
no_suite=True,
displayed_page='scheduled',
suite_arch_nav_template=suite_arch_nav_template)
write_html_page(title=title,
body=html,
destfile=destfile,
style_note=True,
refresh_every=60,
left_nav_html=left_nav_html)
log.info("Page generated at " + desturl)
def gen_packages_html(packages, no_clean=False):
"""
generate the /rb-pkg/package.HTML pages.
packages should be a list of Package objects.
"""
total = len(packages)
log.info('Generating the pages of ' + str(total) + ' package(s)')
for package in sorted(packages, key=lambda x: x.name):
assert isinstance(package, Package)
gen_history_page(package)
for arch in ARCHS:
gen_history_page(package, arch)
pkg = package.name
notes_uri = ''
notes_file = NOTES_PATH + '/' + pkg + '_note.html'
if os.access(notes_file, os.R_OK):
notes_uri = NOTES_URI + '/' + pkg + '_note.html'
for suite in SUITES:
for arch in ARCHS:
status = package.builds[suite][arch].status
version = package.builds[suite][arch].version
build_date = package.builds[suite][arch].build_date
if status is None: # the package is not in the checked suite
continue
log.debug('Generating the page of %s/%s/%s @ %s built at %s',
pkg, suite, arch, version, build_date)
suitearch_section_html, default_view, reproducible = \
gen_suitearch_section(package, suite, arch)
history_uri = '{}/{}.html'.format(HISTORY_URI, pkg)
history_archs = []
for a in ARCHS:
history_archs.append({
'history_arch':
a,
'history_arch_uri':
'{}/{}/{}.html'.format(HISTORY_URI, a, pkg)
})
project_links = renderer.render(project_links_template)
desturl = '{}{}/{}/{}/{}.html'.format(
REPRODUCIBLE_URL,
RB_PKG_URI,
suite,
arch,
pkg,
)
navigation_html = renderer.render(
package_navigation_template, {
'package': pkg,
'suite': suite,
'arch': arch,
'version': version,
'history_uri': history_uri,
'history_archs': history_archs,
'notes_uri': notes_uri,
'notify_maintainer': package.notify_maint,
'suitearch_section_html': suitearch_section_html,
'project_links_html': project_links,
'reproducible': reproducible,
'dashboard_url': DISTRO_URL,
'desturl': desturl,
})
body_html = renderer.render(package_page_template, {
'default_view': default_view,
})
destfile = os.path.join(RB_PKG_PATH, suite, arch,
pkg + '.html')
title = pkg + ' - reproducible builds result'
write_html_page(title=title,
body=body_html,
destfile=destfile,
no_header=True,
noendpage=True,
left_nav_html=navigation_html)
log.debug("Package page generated at " + desturl)
# Optionally generate a page in which the main iframe shows the
# diffoscope results by default. Needed for navigation between
# diffoscope pages for different suites/archs
eversion = strip_epoch(version)
dbd_links = get_and_clean_dbd_links(pkg, eversion, suite, arch,
status)
# only generate the diffoscope page if diffoscope results exist
if 'dbd_uri' in dbd_links:
body_html = renderer.render(
package_page_template, {
'default_view': dbd_links['dbd_uri'],
})
destfile = dbd_links['dbd_page_file']
desturl = REPRODUCIBLE_URL + "/" + dbd_links['dbd_page_uri']
title = "{} ({}) diffoscope results in {}/{}".format(
pkg, version, suite, arch)
write_html_page(title=title,
body=body_html,
destfile=destfile,
no_header=True,
noendpage=True,
left_nav_html=navigation_html)
log.debug("Package diffoscope page generated at " +
desturl)
if not no_clean:
purge_old_pages() # housekeep is always good
