def gen_history_page(package, arch=None): keys = [ 'build date', 'version', 'suite', 'architecture', 'result', 'build duration', 'node1', 'node2', 'job' ] context = {} try: package.history[ 0] # we don't care about the actual value, it jst need to exist except IndexError: context['arch'] = arch else: context['keys'] = [{'key': key} for key in keys] rows = [] for r in package.history: # make a copy, since we modify in place record = dict(r) # FIXME - hacky, should be rethought one day # skip records for suites that are unknown to us (i.e. other distro) if record['suite'] not in SUITES: continue # skip records for other archs if we care about arch if arch and record['architecture'] != arch: continue # remove trailing .debian.net from hostnames record['node1'] = shorten_if_debiannet(record['node1']) record['node2'] = shorten_if_debiannet(record['node2']) # add icon to result status = Status.get(record['result']) result_html = '{spokenstatus}<img src="/static/{icon}" alt="{spokenstatus}" title="{spokenstatus}"/>' record['result'] = result_html.format( icon=status.value.icon, spokenstatus=status.value.spokenstatus) # human formatting of build duration record['build duration'] = convert_into_hms_string( int(record['build duration'])) row_items = [{'item': record[key]} for key in keys] rows.append({'row_items': row_items}) context['rows'] = rows html = renderer.render(package_history_template, context) if arch: destfile = os.path.join(HISTORY_PATH, arch, package.name + '.html') else: destfile = os.path.join(HISTORY_PATH, package.name + '.html') title = 'build history of {}'.format(package.name) if arch: title += ' on {}'.format(arch) write_html_page(title=title, body=html, destfile=destfile, noendpage=True)
def iterate_over_notes(notes): num_notes = str(len(notes)) i = 0 for package in sorted(notes): log.debug('iterating over notes... ' + str(i) + '/' + num_notes) note = notes[package] note['package'] = package log.debug('\t' + str(note)) html = gen_html_note(package, note) title = 'Notes for ' + package + ' - reproducible builds result' destfile = NOTES_PATH + '/' + package + '_note.html' write_html_page(title=title, body=html, destfile=destfile) desturl = REPRODUCIBLE_URL + NOTES_URI + '/' + package + '_note.html' log.debug("Note created: " + desturl) i = i + 1 log.info('Created ' + str(i) + ' note pages.')
def create_index_page(suite, arch): title = 'Package sets in %s/%s' % (suite, arch) body = create_pkgset_navigation(suite, arch) destfile = os.path.join(DISTRO_BASE, suite, arch, "index_pkg_sets.html") suite_arch_nav_template = DISTRO_URI + \ '/{{suite}}/{{arch}}/index_pkg_sets.html' left_nav_html = create_main_navigation( suite=suite, arch=arch, displayed_page='pkg_set', suite_arch_nav_template=suite_arch_nav_template, ignore_experimental=True, ) log.info("Creating pkgset index page for %s/%s.", suite, arch) write_html_page(title=title, body=body, destfile=destfile, left_nav_html=left_nav_html)
def iterate_over_issues(issues): num_issues = str(len(issues)) for suite in SUITES: i = 0 for issue in sorted(issues): log.debug('iterating over issues in ' + suite +'... ' + str(i) + '/' + num_issues) log.debug('\t' + str(issue)) html = gen_html_issue(issue, suite) title = 'Notes about issue ' + issue + ' in ' + suite destfile = ISSUES_PATH + '/' + suite + '/' + issue + '_issue.html' left_nav_html = create_main_navigation(displayed_page='issues') write_html_page(title=title, body=html, destfile=destfile, style_note=True, left_nav_html=left_nav_html) desturl = REPRODUCIBLE_URL + ISSUES_URI + '/' + suite + '/' + issue + '_issue.html' log.debug("Issue created: " + desturl) i = i + 1 log.info('Created ' + str(i) + ' issue pages for ' + suite)
def index_issues(issues, scorefuncs): firstscorefunc = next(iter(scorefuncs.values())) templ = "\n<table class=\"body\">\n" + tab + "<tr>\n" + tab*2 + "<th>\n" \ + tab*3 + "Identified issues\n" + tab*2 + "</th>\n" + tab*2 + "<th>\n" \ + "".join( tab*3 + k + "\n" + tab*2 + "</th>\n" + tab*2 + "<th>\n" for k in scorefuncs.keys()) \ + tab*3 + "Affected packages<br/>\n" \ + tab*3 + "(the 1/4 most-popular ones (within the issue) are underlined)\n" \ + tab*2 + "</th>\n" + tab + "</tr>\n" html = (tab*2).join(templ.splitlines(True)) for issue in sorted(issues, key=lambda issue: sort_issues(firstscorefunc, issue)): html += tab*3 + '<tr>\n' html += tab*4 + '<td><a href="' + ISSUES_URI + '/' + defaultsuite + \ '/'+ issue + '_issue.html">' + issue.replace("_", " ") + '</a></td>\n' issues_list = issues_count.get(issue, []) for scorefunc in scorefuncs.values(): html += tab*4 + '<td><b>' + str(scorefunc(issues_list)) + '</b></td>\n' html += tab*4 + '<td>\n' issues_with_popcon = issues_popcon_annotate(issues_list) issue_strings = [ '<span %stitle="popcon score: %s">%s</span>' % ( 'class="package-popular" ' if p[2] else '', p[1], p[0] ) for p in issues_with_popcon] html += tab*5 + ', '.join(issue_strings) + '\n' html += tab*4 + '</td>\n' html += tab*3 + '</tr>\n' html += tab*2 + '</table>\n' html += tab*2 + '<p>For a total of <b>' + \ str(len([x for x in notes if notes[x].get('issues')])) + \ '</b> packages categorized in <b>' + str(len(issues)) + \ '</b> issues.</p>' html += tab*2 + '<p>' + NOTESGIT_DESCRIPTION + '</p>' title = 'Known issues related to reproducible builds' destfile = DISTRO_BASE + '/index_issues.html' desturl = DISTRO_URL + '/index_issues.html' left_nav_html = create_main_navigation(displayed_page='issues') write_html_page(title=title, body=html, destfile=destfile, left_nav_html=left_nav_html) log.info('Issues index now available at ' + desturl)
def generate_oldies(arch): log.info('Building the oldies page for ' + arch + '...') title = 'Oldest results on ' + arch html = '' for suite in SUITES: query = select([ sources.c.suite, sources.c.architecture, sources.c.name, results.c.status, results.c.build_date ]).select_from(results.join(sources)).where( and_(sources.c.suite == bindparam('suite'), sources.c.architecture == bindparam('arch'), results.c.status != 'blacklisted')).order_by( results.c.build_date).limit(15) text = Template('Oldest results on $suite/$arch:') rows = query_db(query.params({'arch': arch, 'suite': suite})) html += build_leading_text_section({'text': text}, rows, suite, arch) html += '<p><table class="scheduled">\n' + tab html += '<tr><th class="center">#</th><th class="center">suite</th><th class="center">arch</th>' html += '<th class="center">source package</th><th class="center">status</th><th class="center">build date</th></tr>\n' for row in rows: # 0: suite, 1: arch, 2: pkg name 3: status 4: build date pkg = row[2] html += tab + '<tr><td> </td><td>' + row[0] + '</td>' html += '<td>' + row[1] + '</td><td><code>' html += Package(pkg).html_link(row[0], row[1]) html += '</code></td><td>' + convert_into_status_html(str( row[3])) + '</td><td>' + row[4] + '</td></tr>\n' html += '</table></p>\n' destfile = DISTRO_BASE + '/index_' + arch + '_oldies.html' desturl = DISTRO_URL + '/index_' + arch + '_oldies.html' left_nav_html = create_main_navigation(arch=arch) write_html_page(title=title, body=html, destfile=destfile, style_note=True, refresh_every=60, left_nav_html=left_nav_html) log.info("Page generated at " + desturl)
def build_page(page, suite=None, arch=None): gpage = False if pages[page].get('global') and pages[page]['global']: gpage = True suite = defaultsuite arch = defaultarch if not gpage and suite and not arch: print_critical_message('The architecture was not specified while ' + 'building a suite-specific page.') sys.exit(1) if gpage: log.debug('Building the ' + page + ' global index page...') title = pages[page]['title'] else: log.debug('Building the ' + page + ' index page for ' + suite + '/' + arch + '...') title = pages[page]['title'].format(suite=suite, arch=arch) page_sections = pages[page]['body'] html = '' footnote = False if pages[page].get('header'): if pages[page].get('notes_hint') and pages[page][ 'notes_hint'] and suite == defaultsuite: hint = ' <em>These</em> are the packages with failures that <em>still need to be investigated</em>.' else: hint = '' if pages[page].get('header_query'): html += pages[page]['header'].format(tot=query_db( pages[page]['header_query'].format(suite=suite, arch=arch))[0][0], suite=suite, arch=arch, hint=hint) else: html += pages[page].get('header') for section in page_sections: if gpage: if section.get('nosuite') and section['nosuite']: # only defaults html += build_page_section(page, section, suite, arch)[0] else: for suite in SUITES: for arch in ARCHS: log.debug('global page §' + section['status'].name + ' in ' + page + ' for ' + suite + '/' + arch) html += build_page_section(page, section, suite, arch)[0] footnote = True else: html1, footnote1 = build_page_section(page, section, suite, arch) html += html1 footnote = True if footnote1 else footnote suite_arch_nav_template = None if gpage: destfile = DISTRO_BASE + '/index_' + page + '.html' desturl = DISTRO_URL + '/index_' + page + '.html' suite = defaultsuite # used for the links in create_main_navigation else: destfile = DISTRO_BASE + '/' + suite + '/' + arch + '/index_' + \ page + '.html' desturl = DISTRO_URL + '/' + suite + '/' + arch + '/index_' + \ page + '.html' suite_arch_nav_template = DISTRO_URI + '/{{suite}}/{{arch}}/index_' + \ page + '.html' left_nav_html = create_main_navigation( suite=suite, arch=arch, displayed_page=page, suite_arch_nav_template=suite_arch_nav_template, ) write_html_page(title=title, body=html, destfile=destfile, style_note=True, left_nav_html=left_nav_html) log.info('"' + title + '" now available at ' + desturl)
def create_pkgset_page_and_graphs(suite, arch, stats, pkgset_name): html_body = "" html_body += create_pkgset_navigation(suite, arch, pkgset_name) pkgset_context = ({ 'pkgset_name': pkgset_name, 'suite': suite, 'arch': arch, 'pkg_symbol_legend_html': renderer.render(pkg_legend_template, {}), }) png_file, png_href = stats_png_file_href(suite, arch, pkgset_name) thumb_file, thumb_href = stats_thumb_file_href(suite, arch, pkgset_name) yesterday_timestamp = (datetime.now() - timedelta(days=1)).timestamp() if (not os.access(png_file, os.R_OK) or os.stat(png_file).st_mtime < yesterday_timestamp): create_pkgset_graph(png_file, suite, arch, pkgset_name) check_call( ['convert', png_file, '-adaptive-resize', '160x80', thumb_file]) pkgset_context['png'] = png_href other_archs = [a for a in ARCHS if a != arch] pkgset_context['other_archs']= \ gen_other_arch_context(other_archs, suite, pkgset_name) pkgset_context['status_details'] = [] status_cutename_descriptions = [ ('FTBR', 'bad', 'failed to build reproducibly'), ('FTBFS', 'ugly', 'failed to build from source'), ('rest', 'rest', 'are either in depwait state, blacklisted, not for us, or cannot be downloaded' ), ('reproducible', 'good', 'successfully build reproducibly'), ] for (status, cutename, description) in status_cutename_descriptions: icon_html = '' if status == 'rest': for x in ['depwait', 'blacklisted', 'NFU', 'E404']: s = Status.get(x) icon_html += gen_status_link_icon(s.value.name, None, s.value.icon, suite, arch) else: s = Status.get(status) icon_html += gen_status_link_icon(s.value.name, None, s.value.icon, suite, arch) details_context = { 'icon_html': icon_html, 'description': description, 'package_list_html': ''.join( [Package(x).html_link(suite, arch) for x in stats[cutename]]), 'status_count': stats["count_" + cutename], 'status_percent': stats["percent_" + cutename], } if (status in ('reproducible', 'FTBR') or stats["count_" + cutename] != 0): pkgset_context['status_details'].append(details_context) html_body += renderer.render(pkgset_details_template, pkgset_context) title = '%s package set for %s/%s' % \ (pkgset_name, suite, arch) page = "pkg_set_" + pkgset_name + ".html" destfile = os.path.join(DISTRO_BASE, suite, arch, page) suite_arch_nav_template = DISTRO_URI + '/{{suite}}/{{arch}}/' + page left_nav_html = create_main_navigation( suite=suite, arch=arch, displayed_page='pkg_set', suite_arch_nav_template=suite_arch_nav_template, ignore_experimental=True, ) log.info("Creating meta pkgset page for %s in %s/%s.", pkgset_name, suite, arch) write_html_page(title=title, body=html_body, destfile=destfile, left_nav_html=left_nav_html)
lack_rbuild()) html += _gen_packages_html( 'have been built but don\'t have a .buildinfo file:', lack_buildinfo()) return html if __name__ == '__main__': html = '<p>This page lists unexpected things a human should look at and ' html += 'fix, like packages with an incoherent status or files that ' html += 'should not be there. Some of these breakages are caused by ' html += 'bugs in <a href="https://salsa.debian.org/reproducible-builds/diffoscope">diffoscope</a> ' html += 'while others are probably due to bugs in the scripts run by jenkins. ' html += '<em>Please help making this page empty!</em></p>\n' breakages = gen_html() if breakages: html += breakages else: html += '<p><b>COOL!!!</b> Everything is GOOD and not a single issue was ' html += 'detected. <i>Enjoy!</i></p>' title = 'Breakage on the Debian pages of tests.reproducible-builds.org' destfile = DISTRO_BASE + '/index_breakages.html' desturl = DISTRO_URL + '/index_breakages.html' left_nav_html = create_main_navigation(displayed_page='breakages') write_html_page(title, html, destfile, style_note=True, left_nav_html=left_nav_html) log.info('Breakages page created at ' + desturl)
def generate_schedule(arch): """ the schedule pages are very different than others index pages """ log.info('Building the schedule index page for ' + arch + '...') title = 'Packages currently scheduled on ' + arch + ' for testing for build reproducibility' # 'AND h.name=s.name AND h.suite=s.suite AND h.architecture=s.architecture' # in this query and the query below is needed due to not using package_id # in the stats_build table, which should be fixed... averagesql = select([ func.coalesce(func.avg(cast(stats_build.c.build_duration, Integer)), 0) ]).where( and_( stats_build.c.status.in_(('reproducible', 'FTBR')), stats_build.c.name == sources.c.name, stats_build.c.suite == sources.c.suite, stats_build.c.architecture == sources.c.architecture, )).as_scalar() query = select([ schedule.c.date_scheduled, sources.c.suite, sources.c.architecture, sources.c.name, results.c.status, results.c.build_duration, averagesql ]).select_from(sources.join(schedule).join(results, isouter=True)).where( and_( schedule.c.date_build_started == None, sources.c.architecture == bindparam('arch'), )).order_by(schedule.c.date_scheduled) text = Template( '$tot packages are currently scheduled for testing on $arch:') html = '' rows = query_db(query.params({'arch': arch})) html += build_leading_text_section({'text': text}, rows, defaultsuite, arch) html += generate_live_status_table(arch) html += '<p><table class="scheduled">\n' + tab html += '<tr><th class="center">#</th><th class="center">scheduled at</th><th class="center">suite</th>' html += '<th class="center">arch</th><th class="center">source package</th><th class="center">previous build status</th><th class="center">previous build duration</th><th class="center">average build duration</th></tr>\n' for row in rows: # 0: date_scheduled, 1: suite, 2: arch, 3: pkg name 4: previous status 5: previous build duration 6. avg build duration pkg = row[3] duration = convert_into_hms_string(row[5]) avg_duration = convert_into_hms_string(row[6]) html += tab + '<tr><td> </td><td>' + row[0] + '</td>' html += '<td>' + row[1] + '</td><td>' + row[2] + '</td><td><code>' html += Package(pkg).html_link(row[1], row[2]) html += '</code></td><td>' + convert_into_status_html( str(row[4]) ) + '</td><td>' + duration + '</td><td>' + avg_duration + '</td></tr>\n' html += '</table></p>\n' destfile = DISTRO_BASE + '/index_' + arch + '_scheduled.html' desturl = DISTRO_URL + '/index_' + arch + '_scheduled.html' suite_arch_nav_template = DISTRO_URI + '/index_{{arch}}_scheduled.html' left_nav_html = create_main_navigation( arch=arch, no_suite=True, displayed_page='scheduled', suite_arch_nav_template=suite_arch_nav_template) write_html_page(title=title, body=html, destfile=destfile, style_note=True, refresh_every=60, left_nav_html=left_nav_html) log.info("Page generated at " + desturl)
def gen_packages_html(packages, no_clean=False): """ generate the /rb-pkg/package.HTML pages. packages should be a list of Package objects. """ total = len(packages) log.info('Generating the pages of ' + str(total) + ' package(s)') for package in sorted(packages, key=lambda x: x.name): assert isinstance(package, Package) gen_history_page(package) for arch in ARCHS: gen_history_page(package, arch) pkg = package.name notes_uri = '' notes_file = NOTES_PATH + '/' + pkg + '_note.html' if os.access(notes_file, os.R_OK): notes_uri = NOTES_URI + '/' + pkg + '_note.html' for suite in SUITES: for arch in ARCHS: status = package.builds[suite][arch].status version = package.builds[suite][arch].version build_date = package.builds[suite][arch].build_date if status is None: # the package is not in the checked suite continue log.debug('Generating the page of %s/%s/%s @ %s built at %s', pkg, suite, arch, version, build_date) suitearch_section_html, default_view, reproducible = \ gen_suitearch_section(package, suite, arch) history_uri = '{}/{}.html'.format(HISTORY_URI, pkg) history_archs = [] for a in ARCHS: history_archs.append({ 'history_arch': a, 'history_arch_uri': '{}/{}/{}.html'.format(HISTORY_URI, a, pkg) }) project_links = renderer.render(project_links_template) desturl = '{}{}/{}/{}/{}.html'.format( REPRODUCIBLE_URL, RB_PKG_URI, suite, arch, pkg, ) navigation_html = renderer.render( package_navigation_template, { 'package': pkg, 'suite': suite, 'arch': arch, 'version': version, 'history_uri': history_uri, 'history_archs': history_archs, 'notes_uri': notes_uri, 'notify_maintainer': package.notify_maint, 'suitearch_section_html': suitearch_section_html, 'project_links_html': project_links, 'reproducible': reproducible, 'dashboard_url': DISTRO_URL, 'desturl': desturl, }) body_html = renderer.render(package_page_template, { 'default_view': default_view, }) destfile = os.path.join(RB_PKG_PATH, suite, arch, pkg + '.html') title = pkg + ' - reproducible builds result' write_html_page(title=title, body=body_html, destfile=destfile, no_header=True, noendpage=True, left_nav_html=navigation_html) log.debug("Package page generated at " + desturl) # Optionally generate a page in which the main iframe shows the # diffoscope results by default. Needed for navigation between # diffoscope pages for different suites/archs eversion = strip_epoch(version) dbd_links = get_and_clean_dbd_links(pkg, eversion, suite, arch, status) # only generate the diffoscope page if diffoscope results exist if 'dbd_uri' in dbd_links: body_html = renderer.render( package_page_template, { 'default_view': dbd_links['dbd_uri'], }) destfile = dbd_links['dbd_page_file'] desturl = REPRODUCIBLE_URL + "/" + dbd_links['dbd_page_uri'] title = "{} ({}) diffoscope results in {}/{}".format( pkg, version, suite, arch) write_html_page(title=title, body=body_html, destfile=destfile, no_header=True, noendpage=True, left_nav_html=navigation_html) log.debug("Package diffoscope page generated at " + desturl) if not no_clean: purge_old_pages() # housekeep is always good