예제 #1
0
def make_sections(allruns, common=None):
    # print allruns.keys()
    if common is None:
        common = {}

    # print('Selecting %d with %s' % (len(allruns), common))

    if len(allruns) == 1:
        key = list(allruns.keys())[0]
        value = allruns[key]
        return dict(type='sample', common=common, key=key, value=value)

    fields_size = [(field, len(list(allruns.groups_by_field_value(field))))
                   for field in allruns.field_names_in_all_keys()]

    # Now choose the one with the least choices
    fields_size.sort(key=lambda x: x[1])

    if not fields_size:
        # [frozendict({'i': 1, 'n': 3}), frozendict({'i': 2, 'n': 3}), frozendict({}), frozendict({'i': 0, 'n': 3})]
        msg = 'Not all records of the same type have the same fields'
        msg += pformat(list(allruns.keys()))
        raise ValueError(msg)

    field = fields_size[0][0]
    division = {}
    for value, samples in allruns.groups_by_field_value(field):
        samples = samples.remove_field(field)
        c = dict(common)
        c[field] = value
        try:
            division[value] = make_sections(samples, common=c)
        except Exception:
            msg = 'Error occurred inside grouping by field %r = %r' % (field, value)
            msg += '\nCommon: %r' % common
            msg += '\nSamples: %s' % list(samples.keys())
            logger.error(msg)
            raise

    return dict(type='division', field=field,
                division=division, common=common)
예제 #2
0
def check_fails(f, *args, **kwargs):
    try:
        f(*args, **kwargs)
    except BaseException as e:
        logger.error('Known failure for %s ' % f)
        logger.warn('Fails with error %s' % e)
        #comptest_fails = kwargs.get('comptest_fails', f.__name__)
        d = 'out/comptests-failures'
        if not os.path.exists(d):
            os.makedirs(d)
        job_id = JobCompute.current_job_id
        if job_id is None:
            job_id = 'nojob-%s' % f.__name__
        out = os.path.join(d, job_id + '.txt')
        #         for i in range(1000):
        #             outi = out % i
        #             if not os.path.exists(outi):
        with open(out, 'w') as f:
            f.write(traceback.format_exc(e))
    else:
        msg = 'Function was supposed to fail.'
        raise_desc(Exception, msg, f=f, args=args, kwargs=kwargs)
예제 #3
0
def index_reports(reports, index, update=None):  # @UnusedVariable
    """
        Writes an index for the report to the file given. 
        The special key "report" gives the report type.
        
        report[dict(report=...,param1=..., param2=...) ] => filename
    """
    # print('Updating because of new report %s' % update)

    dirname = os.path.dirname(index)
    if not os.path.exists(dirname):
        os.makedirs(dirname)

    # logger.info('Writing on %s' % friendly_path(index))

    f = open(index, 'w')

    f.write("""
        <html>
        <head>
        <style type="text/css">
        span.when { float: right; }
        li { clear: both; }
        a.self { color: black; text-decoration: none; }
        </style>
        </head>
        <body>
    """)

    mtime = lambda x: os.path.getmtime(x)
    existing = list([x for x in list(reports.items()) if os.path.exists(x[1])])

    # create order statistics
    alltimes = np.array([mtime(b) for _, b in existing])

    def order(filename):
        """ returns between 0 and 1 the order statistics """
        assert os.path.exists(filename)
        histime = mtime(filename)
        compare = (alltimes < histime)
        return np.mean(compare * 1.0)

    def style_order(order):
        if order > 0.95:
            return "color: green;"
        if order > 0.9:
            return "color: orange;"
        if order < 0.5:
            return "color: gray;"
        return ""

    @contract(k=dict, filename=str)
    def write_li(k, filename, element='li'):
        desc = ",  ".join('%s = %s' % (a, b) for a, b in list(k.items()))
        href = os.path.relpath(os.path.realpath(filename),
                               os.path.dirname(os.path.realpath(index)))
        if os.path.exists(filename):
            when = duration_compact(time.time() - mtime(filename))
            span_when = '<span class="when">%s ago</span>' % when
            style = style_order(order(filename))
            a = '<a href="%s">%s</a>' % (href, desc)
        else:
            # print('File %s does not exist yet' % filename)
            style = ""
            span_when = '<span class="when">missing</span>'
            a = '<a href="%s">%s</a>' % (href, desc)
        f.write('<%s style="%s">%s %s</%s>' % (element, style, a, span_when,
                                               element))

    # write the first 10
    existing.sort(key=lambda x: (-mtime(x[1])))
    nlast = min(len(existing), 10)
    last = existing[:nlast]
    f.write('<h2 id="last">Last %d report</h2>\n' % (nlast))

    f.write('<ul>')
    for i in range(nlast):
        write_li(*last[i])
    f.write('</ul>')

    if False:
        for report_type, r in reports.groups_by_field_value('report'):
            f.write('<h2 id="%s">%s</h2>\n' % (report_type, report_type))
            f.write('<ul>')
            r = reports.select(report=report_type)
            items = list(r.items())
            items.sort(key=lambda x: str(x[0]))  # XXX use natsort   
            for k, filename in items:
                write_li(k, filename)

            f.write('</ul>')

    f.write('<h2>All report</h2>\n')

    try:
        sections = make_sections(reports)
    except:
        logger.error(str(list(reports.keys())))
        raise

    if sections['type'] == 'sample':
        # only one...
        sections = dict(type='division', field='raw',
                        division=dict(raw1=sections), common=dict())

    def write_sections(sections, parents):
        assert 'type' in sections
        assert sections['type'] == 'division', sections
        field = sections['field']
        division = sections['division']

        f.write('<ul>')
        sorted_values = natsorted(list(division.keys()))
        for value in sorted_values:
            parents.append(value)
            html_id = "-".join(map(str, parents))
            bottom = division[value]
            if bottom['type'] == 'sample':
                d = {field: value}
                if not bottom['key']:
                    write_li(k=d, filename=bottom['value'], element='li')
                else:
                    f.write('<li> <p id="%s"><a class="self" href="#%s">%s = %s</a></p>\n'
                            % (html_id, html_id, field, value))
                    f.write('<ul>')
                    write_li(k=bottom['key'], filename=bottom['value'], element='li')
                    f.write('</ul>')
                    f.write('</li>')
            else:
                f.write('<li> <p id="%s"><a class="self" href="#%s">%s = %s</a></p>\n'
                        % (html_id, html_id, field, value))

                write_sections(bottom, parents)
                f.write('</li>')
        f.write('</ul>')

    write_sections(sections, parents=[])

    f.write('''
    
    </body>
    </html>
    
    ''')
    f.close()