def create_index_page(suite, arch): title = 'Package sets in %s/%s' % (suite, arch) body = create_pkgset_navigation(suite, arch) destfile = os.path.join(DISTRO_BASE, suite, arch, "index_pkg_sets.html") suite_arch_nav_template = DISTRO_URI + \ '/{{suite}}/{{arch}}/index_pkg_sets.html' left_nav_html = create_main_navigation( suite=suite, arch=arch, displayed_page='pkg_set', suite_arch_nav_template=suite_arch_nav_template, ignore_experimental=True, ) log.info("Creating pkgset index page for %s/%s.", suite, arch) write_html_page(title=title, body=body, destfile=destfile, left_nav_html=left_nav_html)
def iterate_over_issues(issues): num_issues = str(len(issues)) for suite in SUITES: i = 0 for issue in sorted(issues): log.debug('iterating over issues in ' + suite +'... ' + str(i) + '/' + num_issues) log.debug('\t' + str(issue)) html = gen_html_issue(issue, suite) title = 'Notes about issue ' + issue + ' in ' + suite destfile = ISSUES_PATH + '/' + suite + '/' + issue + '_issue.html' left_nav_html = create_main_navigation(displayed_page='issues') write_html_page(title=title, body=html, destfile=destfile, style_note=True, left_nav_html=left_nav_html) desturl = REPRODUCIBLE_URL + ISSUES_URI + '/' + suite + '/' + issue + '_issue.html' log.debug("Issue created: " + desturl) i = i + 1 log.info('Created ' + str(i) + ' issue pages for ' + suite)
def index_issues(issues, scorefuncs): firstscorefunc = next(iter(scorefuncs.values())) templ = "\n<table class=\"body\">\n" + tab + "<tr>\n" + tab*2 + "<th>\n" \ + tab*3 + "Identified issues\n" + tab*2 + "</th>\n" + tab*2 + "<th>\n" \ + "".join( tab*3 + k + "\n" + tab*2 + "</th>\n" + tab*2 + "<th>\n" for k in scorefuncs.keys()) \ + tab*3 + "Affected packages<br/>\n" \ + tab*3 + "(the 1/4 most-popular ones (within the issue) are underlined)\n" \ + tab*2 + "</th>\n" + tab + "</tr>\n" html = (tab*2).join(templ.splitlines(True)) for issue in sorted(issues, key=lambda issue: sort_issues(firstscorefunc, issue)): html += tab*3 + '<tr>\n' html += tab*4 + '<td><a href="' + ISSUES_URI + '/' + defaultsuite + \ '/'+ issue + '_issue.html">' + issue.replace("_", " ") + '</a></td>\n' issues_list = issues_count.get(issue, []) for scorefunc in scorefuncs.values(): html += tab*4 + '<td><b>' + str(scorefunc(issues_list)) + '</b></td>\n' html += tab*4 + '<td>\n' issues_with_popcon = issues_popcon_annotate(issues_list) issue_strings = [ '<span %stitle="popcon score: %s">%s</span>' % ( 'class="package-popular" ' if p[2] else '', p[1], p[0] ) for p in issues_with_popcon] html += tab*5 + ', '.join(issue_strings) + '\n' html += tab*4 + '</td>\n' html += tab*3 + '</tr>\n' html += tab*2 + '</table>\n' html += tab*2 + '<p>For a total of <b>' + \ str(len([x for x in notes if notes[x].get('issues')])) + \ '</b> packages categorized in <b>' + str(len(issues)) + \ '</b> issues.</p>' html += tab*2 + '<p>' + NOTESGIT_DESCRIPTION + '</p>' title = 'Known issues related to reproducible builds' destfile = DISTRO_BASE + '/index_issues.html' desturl = DISTRO_URL + '/index_issues.html' left_nav_html = create_main_navigation(displayed_page='issues') write_html_page(title=title, body=html, destfile=destfile, left_nav_html=left_nav_html) log.info('Issues index now available at ' + desturl)
def generate_oldies(arch): log.info('Building the oldies page for ' + arch + '...') title = 'Oldest results on ' + arch html = '' for suite in SUITES: query = select([ sources.c.suite, sources.c.architecture, sources.c.name, results.c.status, results.c.build_date ]).select_from(results.join(sources)).where( and_(sources.c.suite == bindparam('suite'), sources.c.architecture == bindparam('arch'), results.c.status != 'blacklisted')).order_by( results.c.build_date).limit(15) text = Template('Oldest results on $suite/$arch:') rows = query_db(query.params({'arch': arch, 'suite': suite})) html += build_leading_text_section({'text': text}, rows, suite, arch) html += '<p><table class="scheduled">\n' + tab html += '<tr><th class="center">#</th><th class="center">suite</th><th class="center">arch</th>' html += '<th class="center">source package</th><th class="center">status</th><th class="center">build date</th></tr>\n' for row in rows: # 0: suite, 1: arch, 2: pkg name 3: status 4: build date pkg = row[2] html += tab + '<tr><td> </td><td>' + row[0] + '</td>' html += '<td>' + row[1] + '</td><td><code>' html += Package(pkg).html_link(row[0], row[1]) html += '</code></td><td>' + convert_into_status_html(str( row[3])) + '</td><td>' + row[4] + '</td></tr>\n' html += '</table></p>\n' destfile = DISTRO_BASE + '/index_' + arch + '_oldies.html' desturl = DISTRO_URL + '/index_' + arch + '_oldies.html' left_nav_html = create_main_navigation(arch=arch) write_html_page(title=title, body=html, destfile=destfile, style_note=True, refresh_every=60, left_nav_html=left_nav_html) log.info("Page generated at " + desturl)
def build_page(page, suite=None, arch=None): gpage = False if pages[page].get('global') and pages[page]['global']: gpage = True suite = defaultsuite arch = defaultarch if not gpage and suite and not arch: print_critical_message('The architecture was not specified while ' + 'building a suite-specific page.') sys.exit(1) if gpage: log.debug('Building the ' + page + ' global index page...') title = pages[page]['title'] else: log.debug('Building the ' + page + ' index page for ' + suite + '/' + arch + '...') title = pages[page]['title'].format(suite=suite, arch=arch) page_sections = pages[page]['body'] html = '' footnote = False if pages[page].get('header'): if pages[page].get('notes_hint') and pages[page][ 'notes_hint'] and suite == defaultsuite: hint = ' <em>These</em> are the packages with failures that <em>still need to be investigated</em>.' else: hint = '' if pages[page].get('header_query'): html += pages[page]['header'].format(tot=query_db( pages[page]['header_query'].format(suite=suite, arch=arch))[0][0], suite=suite, arch=arch, hint=hint) else: html += pages[page].get('header') for section in page_sections: if gpage: if section.get('nosuite') and section['nosuite']: # only defaults html += build_page_section(page, section, suite, arch)[0] else: for suite in SUITES: for arch in ARCHS: log.debug('global page §' + section['status'].name + ' in ' + page + ' for ' + suite + '/' + arch) html += build_page_section(page, section, suite, arch)[0] footnote = True else: html1, footnote1 = build_page_section(page, section, suite, arch) html += html1 footnote = True if footnote1 else footnote suite_arch_nav_template = None if gpage: destfile = DISTRO_BASE + '/index_' + page + '.html' desturl = DISTRO_URL + '/index_' + page + '.html' suite = defaultsuite # used for the links in create_main_navigation else: destfile = DISTRO_BASE + '/' + suite + '/' + arch + '/index_' + \ page + '.html' desturl = DISTRO_URL + '/' + suite + '/' + arch + '/index_' + \ page + '.html' suite_arch_nav_template = DISTRO_URI + '/{{suite}}/{{arch}}/index_' + \ page + '.html' left_nav_html = create_main_navigation( suite=suite, arch=arch, displayed_page=page, suite_arch_nav_template=suite_arch_nav_template, ) write_html_page(title=title, body=html, destfile=destfile, style_note=True, left_nav_html=left_nav_html) log.info('"' + title + '" now available at ' + desturl)
def create_pkgset_page_and_graphs(suite, arch, stats, pkgset_name): html_body = "" html_body += create_pkgset_navigation(suite, arch, pkgset_name) pkgset_context = ({ 'pkgset_name': pkgset_name, 'suite': suite, 'arch': arch, 'pkg_symbol_legend_html': renderer.render(pkg_legend_template, {}), }) png_file, png_href = stats_png_file_href(suite, arch, pkgset_name) thumb_file, thumb_href = stats_thumb_file_href(suite, arch, pkgset_name) yesterday_timestamp = (datetime.now() - timedelta(days=1)).timestamp() if (not os.access(png_file, os.R_OK) or os.stat(png_file).st_mtime < yesterday_timestamp): create_pkgset_graph(png_file, suite, arch, pkgset_name) check_call( ['convert', png_file, '-adaptive-resize', '160x80', thumb_file]) pkgset_context['png'] = png_href other_archs = [a for a in ARCHS if a != arch] pkgset_context['other_archs']= \ gen_other_arch_context(other_archs, suite, pkgset_name) pkgset_context['status_details'] = [] status_cutename_descriptions = [ ('FTBR', 'bad', 'failed to build reproducibly'), ('FTBFS', 'ugly', 'failed to build from source'), ('rest', 'rest', 'are either in depwait state, blacklisted, not for us, or cannot be downloaded' ), ('reproducible', 'good', 'successfully build reproducibly'), ] for (status, cutename, description) in status_cutename_descriptions: icon_html = '' if status == 'rest': for x in ['depwait', 'blacklisted', 'NFU', 'E404']: s = Status.get(x) icon_html += gen_status_link_icon(s.value.name, None, s.value.icon, suite, arch) else: s = Status.get(status) icon_html += gen_status_link_icon(s.value.name, None, s.value.icon, suite, arch) details_context = { 'icon_html': icon_html, 'description': description, 'package_list_html': ''.join( [Package(x).html_link(suite, arch) for x in stats[cutename]]), 'status_count': stats["count_" + cutename], 'status_percent': stats["percent_" + cutename], } if (status in ('reproducible', 'FTBR') or stats["count_" + cutename] != 0): pkgset_context['status_details'].append(details_context) html_body += renderer.render(pkgset_details_template, pkgset_context) title = '%s package set for %s/%s' % \ (pkgset_name, suite, arch) page = "pkg_set_" + pkgset_name + ".html" destfile = os.path.join(DISTRO_BASE, suite, arch, page) suite_arch_nav_template = DISTRO_URI + '/{{suite}}/{{arch}}/' + page left_nav_html = create_main_navigation( suite=suite, arch=arch, displayed_page='pkg_set', suite_arch_nav_template=suite_arch_nav_template, ignore_experimental=True, ) log.info("Creating meta pkgset page for %s in %s/%s.", pkgset_name, suite, arch) write_html_page(title=title, body=html_body, destfile=destfile, left_nav_html=left_nav_html)
lack_rbuild()) html += _gen_packages_html( 'have been built but don\'t have a .buildinfo file:', lack_buildinfo()) return html if __name__ == '__main__': html = '<p>This page lists unexpected things a human should look at and ' html += 'fix, like packages with an incoherent status or files that ' html += 'should not be there. Some of these breakages are caused by ' html += 'bugs in <a href="https://salsa.debian.org/reproducible-builds/diffoscope">diffoscope</a> ' html += 'while others are probably due to bugs in the scripts run by jenkins. ' html += '<em>Please help making this page empty!</em></p>\n' breakages = gen_html() if breakages: html += breakages else: html += '<p><b>COOL!!!</b> Everything is GOOD and not a single issue was ' html += 'detected. <i>Enjoy!</i></p>' title = 'Breakage on the Debian pages of tests.reproducible-builds.org' destfile = DISTRO_BASE + '/index_breakages.html' desturl = DISTRO_URL + '/index_breakages.html' left_nav_html = create_main_navigation(displayed_page='breakages') write_html_page(title, html, destfile, style_note=True, left_nav_html=left_nav_html) log.info('Breakages page created at ' + desturl)
def generate_schedule(arch): """ the schedule pages are very different than others index pages """ log.info('Building the schedule index page for ' + arch + '...') title = 'Packages currently scheduled on ' + arch + ' for testing for build reproducibility' # 'AND h.name=s.name AND h.suite=s.suite AND h.architecture=s.architecture' # in this query and the query below is needed due to not using package_id # in the stats_build table, which should be fixed... averagesql = select([ func.coalesce(func.avg(cast(stats_build.c.build_duration, Integer)), 0) ]).where( and_( stats_build.c.status.in_(('reproducible', 'FTBR')), stats_build.c.name == sources.c.name, stats_build.c.suite == sources.c.suite, stats_build.c.architecture == sources.c.architecture, )).as_scalar() query = select([ schedule.c.date_scheduled, sources.c.suite, sources.c.architecture, sources.c.name, results.c.status, results.c.build_duration, averagesql ]).select_from(sources.join(schedule).join(results, isouter=True)).where( and_( schedule.c.date_build_started == None, sources.c.architecture == bindparam('arch'), )).order_by(schedule.c.date_scheduled) text = Template( '$tot packages are currently scheduled for testing on $arch:') html = '' rows = query_db(query.params({'arch': arch})) html += build_leading_text_section({'text': text}, rows, defaultsuite, arch) html += generate_live_status_table(arch) html += '<p><table class="scheduled">\n' + tab html += '<tr><th class="center">#</th><th class="center">scheduled at</th><th class="center">suite</th>' html += '<th class="center">arch</th><th class="center">source package</th><th class="center">previous build status</th><th class="center">previous build duration</th><th class="center">average build duration</th></tr>\n' for row in rows: # 0: date_scheduled, 1: suite, 2: arch, 3: pkg name 4: previous status 5: previous build duration 6. avg build duration pkg = row[3] duration = convert_into_hms_string(row[5]) avg_duration = convert_into_hms_string(row[6]) html += tab + '<tr><td> </td><td>' + row[0] + '</td>' html += '<td>' + row[1] + '</td><td>' + row[2] + '</td><td><code>' html += Package(pkg).html_link(row[1], row[2]) html += '</code></td><td>' + convert_into_status_html( str(row[4]) ) + '</td><td>' + duration + '</td><td>' + avg_duration + '</td></tr>\n' html += '</table></p>\n' destfile = DISTRO_BASE + '/index_' + arch + '_scheduled.html' desturl = DISTRO_URL + '/index_' + arch + '_scheduled.html' suite_arch_nav_template = DISTRO_URI + '/index_{{arch}}_scheduled.html' left_nav_html = create_main_navigation( arch=arch, no_suite=True, displayed_page='scheduled', suite_arch_nav_template=suite_arch_nav_template) write_html_page(title=title, body=html, destfile=destfile, style_note=True, refresh_every=60, left_nav_html=left_nav_html) log.info("Page generated at " + desturl)
html += 'always points to the amd64 version, it\'s possible that' html += 'the unreproducibility is only present in another architecture(s).</p>\n<p><pre>' out = out.decode().splitlines() get_mail = re.compile('<(.*)>') for line in out: if line[0:3] == ' ': line = line.strip().split(None, 1) html += ' ' # the final strip() is to avoid a newline html += Package(line[0]).html_link(suite, arch).strip() try: html += ' ' + line[1] # eventual uploaders sign except IndexError: pass elif line.strip(): # be sure this is not just an empty line email = get_mail.findall(line.strip())[0] html += HTML.escape(line.strip()) html += '<a name="{maint}" href="#{maint}">¶</a>'.format( maint=email) html += '\n' html += '</pre></p>' title = 'Maintainers of unreproducible packages in ' + suite destfile = DISTRO_BASE + '/' + suite + '/index_dd-list.html' suite_arch_nav_template = DISTRO_URI + '/{{suite}}/index_dd-list.html' left_nav_html = create_main_navigation(suite=suite, arch=arch, displayed_page='dd_list', no_arch=True, suite_arch_nav_template=suite_arch_nav_template) write_html_page(title, html, destfile, style_note=True, left_nav_html=left_nav_html) log.info('%s/%s/index_dd-list.html published', DISTRO_URL, suite)