Ejemplo n.º 1
0
 def add_result(self, run):
     self.counter += 1
     self.dic[run.category] += 1
     self.dic[(run.category, result.get_result_classification(run.status))] += 1
     self.score += result.score_for_task(run.identifier, run.properties, run.category, run.status)
     #if run.properties:
     self.max_score += result.score_for_task(run.identifier, run.properties, result.CATEGORY_CORRECT, None)
Ejemplo n.º 2
0
 def add_result(self, run):
     self.counter += 1
     self.dic[run.category] += 1
     self.dic[(run.category, result.get_result_classification(run.status))] += 1
     self.score += result.score_for_task(run.identifier, run.properties, run.category, run.status)
     #if run.properties:
     self.max_score += result.score_for_task(run.identifier, run.properties, result.CATEGORY_CORRECT, None)
Ejemplo n.º 3
0
    def create_from_xml(sourcefileTag, get_value_from_logfile, listOfColumns, correct_only):
        '''
        This function collects the values from one run.
        Only columns that should be part of the table are collected.
        '''

        def read_logfile_lines(logfilename):
            if not logfilename:
                return []
            try:
                with open(logfilename, 'rt') as logfile:
                    return logfile.readlines()
            except IOError as e:
                logging.warning("Could not read value from logfile: %s", e)
                return []

        status = Util.get_column_value(sourcefileTag, 'status', '')
        category = Util.get_column_value(sourcefileTag, 'category', result.CATEGORY_MISSING)
        score = result.score_for_task(sourcefileTag.get('name'),
                                      sourcefileTag.get('properties', '').split(),
                                      category)
        logfileLines = None

        values = []

        for column in listOfColumns: # for all columns that should be shown
            value = None # default value
            if column.title.lower() == 'score':
                value = str(score)
            elif column.title.lower() == 'status':
                value = status

            elif not correct_only or category == result.CATEGORY_CORRECT:
                if not column.pattern: # collect values from XML
                    value = Util.get_column_value(sourcefileTag, column.title)

                else: # collect values from logfile
                    if logfileLines is None: # cache content
                        logfileLines = read_logfile_lines(sourcefileTag.get('logfile'))

                    value = get_value_from_logfile(logfileLines, column.pattern)

            if column.number_of_digits is not None:
                value = Util.format_number(value, column.number_of_digits)

            values.append(value)

        return RunResult(get_task_id(sourcefileTag), status, category, score, sourcefileTag.get('logfile'), listOfColumns, values)
Ejemplo n.º 4
0
def get_stats(rows):
    stats = parallel.map(get_stats_of_run_set, rows_to_columns(rows)) # column-wise
    rowsForStats = list(map(Util.flatten, zip(*stats))) # row-wise

    # Calculate maximal score and number of true/false files for the given properties
    count_true = count_false = max_score = 0
    for row in rows:
        if not row.properties:
            # properties missing for at least one task, result would be wrong
            count_true = count_false = 0
            logging.info('Missing property for %s.', row.filename)
            break
        correct_result = result.satisfies_file_property(row.filename, row.properties)
        if correct_result is True:
            count_true += 1
        elif correct_result is False:
            count_false += 1
        max_score += result.score_for_task(row.filename, row.properties, result.CATEGORY_CORRECT)
    task_counts = 'in total {0} true tasks, {1} false tasks'.format(count_true, count_false)

    if max_score:
        score_row = tempita.bunch(id='score',
                                  title='score ({0} tasks, max score: {1})'.format(len(rows), max_score),
                                  description=task_counts,
                                  content=rowsForStats[7])

    def indent(n):
        return ' '*(n*4)

    return [tempita.bunch(id=None, title='total tasks', description=task_counts, content=rowsForStats[0]),
            tempita.bunch(id=None, title=indent(1)+'correct results', description='(property holds + result is true) OR (property does not hold + result is false)', content=rowsForStats[1]),
            tempita.bunch(id=None, title=indent(2)+'correct true', description='property holds + result is true', content=rowsForStats[2]),
            tempita.bunch(id=None, title=indent(2)+'correct false', description='property does not hold + result is false', content=rowsForStats[3]),
            tempita.bunch(id=None, title=indent(1)+'incorrect results', description='(property holds + result is false) OR (property does not hold + result is true)', content=rowsForStats[4]),
            tempita.bunch(id=None, title=indent(2)+'incorrect true', description='property does not hold + result is true', content=rowsForStats[5]),
            tempita.bunch(id=None, title=indent(2)+'incorrect false', description='property holds + result is false', content=rowsForStats[6]),
            ] + ([score_row] if max_score else [])