Esempio n. 1
0
def create_index_page(suite, arch):
    title = 'Package sets in %s/%s' % (suite, arch)
    body = create_pkgset_navigation(suite, arch)
    destfile = os.path.join(DISTRO_BASE, suite, arch, "index_pkg_sets.html")
    suite_arch_nav_template = DISTRO_URI + \
                              '/{{suite}}/{{arch}}/index_pkg_sets.html'
    left_nav_html = create_main_navigation(
        suite=suite,
        arch=arch,
        displayed_page='pkg_set',
        suite_arch_nav_template=suite_arch_nav_template,
        ignore_experimental=True,
    )
    log.info("Creating pkgset index page for %s/%s.", suite, arch)
    write_html_page(title=title,
                    body=body,
                    destfile=destfile,
                    left_nav_html=left_nav_html)
Esempio n. 2
0
def iterate_over_issues(issues):
    num_issues = str(len(issues))
    for suite in SUITES:
        i = 0
        for issue in sorted(issues):
            log.debug('iterating over issues in ' + suite +'... ' + str(i) + '/' + num_issues)
            log.debug('\t' + str(issue))
            html = gen_html_issue(issue, suite)

            title = 'Notes about issue ' + issue + ' in ' + suite
            destfile = ISSUES_PATH + '/' + suite + '/' + issue + '_issue.html'
            left_nav_html = create_main_navigation(displayed_page='issues')
            write_html_page(title=title, body=html, destfile=destfile,
                            style_note=True, left_nav_html=left_nav_html)

            desturl = REPRODUCIBLE_URL + ISSUES_URI + '/' + suite + '/' + issue + '_issue.html'
            log.debug("Issue created: " + desturl)
            i = i + 1
        log.info('Created ' + str(i) + ' issue pages for ' + suite)
Esempio n. 3
0
def index_issues(issues, scorefuncs):
    firstscorefunc = next(iter(scorefuncs.values()))
    templ = "\n<table class=\"body\">\n" + tab + "<tr>\n" + tab*2 + "<th>\n" \
          + tab*3 + "Identified issues\n" + tab*2 + "</th>\n" + tab*2 + "<th>\n" \
          + "".join(
            tab*3 + k + "\n" + tab*2 + "</th>\n" + tab*2 + "<th>\n"
            for k in scorefuncs.keys()) \
          + tab*3 + "Affected packages<br/>\n" \
          + tab*3 + "(the 1/4 most-popular ones (within the issue) are underlined)\n" \
          + tab*2 + "</th>\n" + tab + "</tr>\n"
    html = (tab*2).join(templ.splitlines(True))
    for issue in sorted(issues, key=lambda issue: sort_issues(firstscorefunc, issue)):
        html += tab*3 + '<tr>\n'
        html += tab*4 + '<td><a href="' + ISSUES_URI + '/' + defaultsuite + \
                '/'+ issue + '_issue.html">' + issue.replace("_", " ") + '</a></td>\n'
        issues_list = issues_count.get(issue, [])
        for scorefunc in scorefuncs.values():
            html += tab*4 + '<td><b>' + str(scorefunc(issues_list)) + '</b></td>\n'
        html += tab*4 + '<td>\n'
        issues_with_popcon = issues_popcon_annotate(issues_list)
        issue_strings = [
            '<span %stitle="popcon score: %s">%s</span>' % (
                'class="package-popular" ' if p[2] else '', p[1], p[0]
            ) for p in issues_with_popcon]
        html += tab*5 + ', '.join(issue_strings) + '\n'
        html += tab*4 + '</td>\n'
        html += tab*3 + '</tr>\n'
    html += tab*2 + '</table>\n'
    html += tab*2 + '<p>For a total of <b>' + \
            str(len([x for x in notes if notes[x].get('issues')])) + \
            '</b> packages categorized in <b>' + str(len(issues)) + \
            '</b> issues.</p>'
    html += tab*2 + '<p>' + NOTESGIT_DESCRIPTION + '</p>'
    title = 'Known issues related to reproducible builds'
    destfile = DISTRO_BASE + '/index_issues.html'
    desturl = DISTRO_URL + '/index_issues.html'
    left_nav_html = create_main_navigation(displayed_page='issues')
    write_html_page(title=title, body=html, destfile=destfile,
                    left_nav_html=left_nav_html)
    log.info('Issues index now available at ' + desturl)
Esempio n. 4
0
def generate_oldies(arch):
    log.info('Building the oldies page for ' + arch + '...')
    title = 'Oldest results on ' + arch
    html = ''
    for suite in SUITES:
        query = select([
            sources.c.suite, sources.c.architecture, sources.c.name,
            results.c.status, results.c.build_date
        ]).select_from(results.join(sources)).where(
            and_(sources.c.suite == bindparam('suite'),
                 sources.c.architecture == bindparam('arch'),
                 results.c.status != 'blacklisted')).order_by(
                     results.c.build_date).limit(15)
        text = Template('Oldest results on $suite/$arch:')
        rows = query_db(query.params({'arch': arch, 'suite': suite}))
        html += build_leading_text_section({'text': text}, rows, suite, arch)
        html += '<p><table class="scheduled">\n' + tab
        html += '<tr><th class="center">#</th><th class="center">suite</th><th class="center">arch</th>'
        html += '<th class="center">source package</th><th class="center">status</th><th class="center">build date</th></tr>\n'
        for row in rows:
            # 0: suite, 1: arch, 2: pkg name 3: status 4: build date
            pkg = row[2]
            html += tab + '<tr><td>&nbsp;</td><td>' + row[0] + '</td>'
            html += '<td>' + row[1] + '</td><td><code>'
            html += Package(pkg).html_link(row[0], row[1])
            html += '</code></td><td>' + convert_into_status_html(str(
                row[3])) + '</td><td>' + row[4] + '</td></tr>\n'
        html += '</table></p>\n'
    destfile = DISTRO_BASE + '/index_' + arch + '_oldies.html'
    desturl = DISTRO_URL + '/index_' + arch + '_oldies.html'
    left_nav_html = create_main_navigation(arch=arch)
    write_html_page(title=title,
                    body=html,
                    destfile=destfile,
                    style_note=True,
                    refresh_every=60,
                    left_nav_html=left_nav_html)
    log.info("Page generated at " + desturl)
Esempio n. 5
0
def build_page(page, suite=None, arch=None):
    gpage = False
    if pages[page].get('global') and pages[page]['global']:
        gpage = True
        suite = defaultsuite
        arch = defaultarch
    if not gpage and suite and not arch:
        print_critical_message('The architecture was not specified while ' +
                               'building a suite-specific page.')
        sys.exit(1)
    if gpage:
        log.debug('Building the ' + page + ' global index page...')
        title = pages[page]['title']
    else:
        log.debug('Building the ' + page + ' index page for ' + suite + '/' +
                  arch + '...')
        title = pages[page]['title'].format(suite=suite, arch=arch)
    page_sections = pages[page]['body']
    html = ''
    footnote = False
    if pages[page].get('header'):
        if pages[page].get('notes_hint') and pages[page][
                'notes_hint'] and suite == defaultsuite:
            hint = ' <em>These</em> are the packages with failures that <em>still need to be investigated</em>.'
        else:
            hint = ''
        if pages[page].get('header_query'):
            html += pages[page]['header'].format(tot=query_db(
                pages[page]['header_query'].format(suite=suite,
                                                   arch=arch))[0][0],
                                                 suite=suite,
                                                 arch=arch,
                                                 hint=hint)
        else:
            html += pages[page].get('header')
    for section in page_sections:
        if gpage:
            if section.get('nosuite') and section['nosuite']:  # only defaults
                html += build_page_section(page, section, suite, arch)[0]
            else:
                for suite in SUITES:
                    for arch in ARCHS:
                        log.debug('global page §' + section['status'].name +
                                  ' in ' + page + ' for ' + suite + '/' + arch)
                        html += build_page_section(page, section, suite,
                                                   arch)[0]
            footnote = True
        else:
            html1, footnote1 = build_page_section(page, section, suite, arch)
            html += html1
            footnote = True if footnote1 else footnote
    suite_arch_nav_template = None
    if gpage:
        destfile = DISTRO_BASE + '/index_' + page + '.html'
        desturl = DISTRO_URL + '/index_' + page + '.html'
        suite = defaultsuite  # used for the links in create_main_navigation
    else:
        destfile = DISTRO_BASE + '/' + suite + '/' + arch + '/index_' + \
                   page + '.html'
        desturl = DISTRO_URL + '/' + suite + '/' + arch + '/index_' + \
                  page + '.html'
        suite_arch_nav_template = DISTRO_URI + '/{{suite}}/{{arch}}/index_' + \
                                  page + '.html'
    left_nav_html = create_main_navigation(
        suite=suite,
        arch=arch,
        displayed_page=page,
        suite_arch_nav_template=suite_arch_nav_template,
    )
    write_html_page(title=title,
                    body=html,
                    destfile=destfile,
                    style_note=True,
                    left_nav_html=left_nav_html)
    log.info('"' + title + '" now available at ' + desturl)
Esempio n. 6
0
def create_pkgset_page_and_graphs(suite, arch, stats, pkgset_name):
    html_body = ""
    html_body += create_pkgset_navigation(suite, arch, pkgset_name)
    pkgset_context = ({
        'pkgset_name':
        pkgset_name,
        'suite':
        suite,
        'arch':
        arch,
        'pkg_symbol_legend_html':
        renderer.render(pkg_legend_template, {}),
    })

    png_file, png_href = stats_png_file_href(suite, arch, pkgset_name)
    thumb_file, thumb_href = stats_thumb_file_href(suite, arch, pkgset_name)
    yesterday_timestamp = (datetime.now() - timedelta(days=1)).timestamp()

    if (not os.access(png_file, os.R_OK)
            or os.stat(png_file).st_mtime < yesterday_timestamp):
        create_pkgset_graph(png_file, suite, arch, pkgset_name)
        check_call(
            ['convert', png_file, '-adaptive-resize', '160x80', thumb_file])

    pkgset_context['png'] = png_href
    other_archs = [a for a in ARCHS if a != arch]
    pkgset_context['other_archs']= \
        gen_other_arch_context(other_archs, suite, pkgset_name)

    pkgset_context['status_details'] = []

    status_cutename_descriptions = [
        ('FTBR', 'bad', 'failed to build reproducibly'),
        ('FTBFS', 'ugly', 'failed to build from source'),
        ('rest', 'rest',
         'are either in depwait state, blacklisted, not for us, or cannot be downloaded'
         ),
        ('reproducible', 'good', 'successfully build reproducibly'),
    ]

    for (status, cutename, description) in status_cutename_descriptions:
        icon_html = ''
        if status == 'rest':
            for x in ['depwait', 'blacklisted', 'NFU', 'E404']:
                s = Status.get(x)
                icon_html += gen_status_link_icon(s.value.name, None,
                                                  s.value.icon, suite, arch)
        else:
            s = Status.get(status)
            icon_html += gen_status_link_icon(s.value.name, None, s.value.icon,
                                              suite, arch)

        details_context = {
            'icon_html':
            icon_html,
            'description':
            description,
            'package_list_html':
            ''.join(
                [Package(x).html_link(suite, arch) for x in stats[cutename]]),
            'status_count':
            stats["count_" + cutename],
            'status_percent':
            stats["percent_" + cutename],
        }

        if (status in ('reproducible', 'FTBR')
                or stats["count_" + cutename] != 0):
            pkgset_context['status_details'].append(details_context)

    html_body += renderer.render(pkgset_details_template, pkgset_context)
    title = '%s package set for %s/%s' % \
            (pkgset_name, suite, arch)
    page = "pkg_set_" + pkgset_name + ".html"
    destfile = os.path.join(DISTRO_BASE, suite, arch, page)
    suite_arch_nav_template = DISTRO_URI + '/{{suite}}/{{arch}}/' + page
    left_nav_html = create_main_navigation(
        suite=suite,
        arch=arch,
        displayed_page='pkg_set',
        suite_arch_nav_template=suite_arch_nav_template,
        ignore_experimental=True,
    )
    log.info("Creating meta pkgset page for %s in %s/%s.", pkgset_name, suite,
             arch)
    write_html_page(title=title,
                    body=html_body,
                    destfile=destfile,
                    left_nav_html=left_nav_html)
                               lack_rbuild())
    html += _gen_packages_html(
        'have been built but don\'t have a .buildinfo file:', lack_buildinfo())
    return html


if __name__ == '__main__':
    html = '<p>This page lists unexpected things a human should look at and '
    html += 'fix, like packages with an incoherent status or files that '
    html += 'should not be there. Some of these breakages are caused by '
    html += 'bugs in <a href="https://salsa.debian.org/reproducible-builds/diffoscope">diffoscope</a> '
    html += 'while others are probably due to bugs in the scripts run by jenkins. '
    html += '<em>Please help making this page empty!</em></p>\n'
    breakages = gen_html()
    if breakages:
        html += breakages
    else:
        html += '<p><b>COOL!!!</b> Everything is GOOD and not a single issue was '
        html += 'detected. <i>Enjoy!</i></p>'
    title = 'Breakage on the Debian pages of tests.reproducible-builds.org'
    destfile = DISTRO_BASE + '/index_breakages.html'
    desturl = DISTRO_URL + '/index_breakages.html'

    left_nav_html = create_main_navigation(displayed_page='breakages')
    write_html_page(title,
                    html,
                    destfile,
                    style_note=True,
                    left_nav_html=left_nav_html)
    log.info('Breakages page created at ' + desturl)
Esempio n. 8
0
def generate_schedule(arch):
    """ the schedule pages are very different than others index pages """
    log.info('Building the schedule index page for ' + arch + '...')
    title = 'Packages currently scheduled on ' + arch + ' for testing for build reproducibility'

    # 'AND h.name=s.name AND h.suite=s.suite AND h.architecture=s.architecture'
    # in this query and the query below is needed due to not using package_id
    # in the stats_build table, which should be fixed...
    averagesql = select([
        func.coalesce(func.avg(cast(stats_build.c.build_duration, Integer)), 0)
    ]).where(
        and_(
            stats_build.c.status.in_(('reproducible', 'FTBR')),
            stats_build.c.name == sources.c.name,
            stats_build.c.suite == sources.c.suite,
            stats_build.c.architecture == sources.c.architecture,
        )).as_scalar()

    query = select([
        schedule.c.date_scheduled, sources.c.suite, sources.c.architecture,
        sources.c.name, results.c.status, results.c.build_duration, averagesql
    ]).select_from(sources.join(schedule).join(results, isouter=True)).where(
        and_(
            schedule.c.date_build_started == None,
            sources.c.architecture == bindparam('arch'),
        )).order_by(schedule.c.date_scheduled)

    text = Template(
        '$tot packages are currently scheduled for testing on $arch:')
    html = ''
    rows = query_db(query.params({'arch': arch}))
    html += build_leading_text_section({'text': text}, rows, defaultsuite,
                                       arch)
    html += generate_live_status_table(arch)
    html += '<p><table class="scheduled">\n' + tab
    html += '<tr><th class="center">#</th><th class="center">scheduled at</th><th class="center">suite</th>'
    html += '<th class="center">arch</th><th class="center">source package</th><th class="center">previous build status</th><th class="center">previous build duration</th><th class="center">average build duration</th></tr>\n'
    for row in rows:
        # 0: date_scheduled, 1: suite, 2: arch, 3: pkg name 4: previous status 5: previous build duration 6. avg build duration
        pkg = row[3]
        duration = convert_into_hms_string(row[5])
        avg_duration = convert_into_hms_string(row[6])
        html += tab + '<tr><td>&nbsp;</td><td>' + row[0] + '</td>'
        html += '<td>' + row[1] + '</td><td>' + row[2] + '</td><td><code>'
        html += Package(pkg).html_link(row[1], row[2])
        html += '</code></td><td>' + convert_into_status_html(
            str(row[4])
        ) + '</td><td>' + duration + '</td><td>' + avg_duration + '</td></tr>\n'
    html += '</table></p>\n'
    destfile = DISTRO_BASE + '/index_' + arch + '_scheduled.html'
    desturl = DISTRO_URL + '/index_' + arch + '_scheduled.html'
    suite_arch_nav_template = DISTRO_URI + '/index_{{arch}}_scheduled.html'
    left_nav_html = create_main_navigation(
        arch=arch,
        no_suite=True,
        displayed_page='scheduled',
        suite_arch_nav_template=suite_arch_nav_template)
    write_html_page(title=title,
                    body=html,
                    destfile=destfile,
                    style_note=True,
                    refresh_every=60,
                    left_nav_html=left_nav_html)
    log.info("Page generated at " + desturl)
        html += 'always points to the amd64 version, it\'s possible that'
        html += 'the unreproducibility is only present in another architecture(s).</p>\n<p><pre>'
        out = out.decode().splitlines()
        get_mail = re.compile('<(.*)>')
        for line in out:
            if line[0:3] == '   ':
                line = line.strip().split(None, 1)
                html += '    '
                # the final strip() is to avoid a newline
                html += Package(line[0]).html_link(suite, arch).strip()
                try:
                    html += ' ' + line[1]  # eventual uploaders sign
                except IndexError:
                    pass
            elif line.strip():  # be sure this is not just an empty line
                email = get_mail.findall(line.strip())[0]
                html += HTML.escape(line.strip())
                html += '<a name="{maint}" href="#{maint}">&para;</a>'.format(
                    maint=email)
            html += '\n'
        html += '</pre></p>'
        title = 'Maintainers of unreproducible packages in ' + suite
        destfile = DISTRO_BASE + '/' + suite + '/index_dd-list.html'
        suite_arch_nav_template = DISTRO_URI + '/{{suite}}/index_dd-list.html'
        left_nav_html = create_main_navigation(suite=suite, arch=arch,
            displayed_page='dd_list', no_arch=True,
            suite_arch_nav_template=suite_arch_nav_template)
        write_html_page(title, html, destfile, style_note=True,
                        left_nav_html=left_nav_html)
        log.info('%s/%s/index_dd-list.html published', DISTRO_URL, suite)