def add_result(self, run): self.counter += 1 self.dic[run.category] += 1 self.dic[(run.category, result.get_result_classification(run.status))] += 1 self.score += result.score_for_task(run.identifier, run.properties, run.category, run.status) #if run.properties: self.max_score += result.score_for_task(run.identifier, run.properties, result.CATEGORY_CORRECT, None)
def create_from_xml(sourcefileTag, get_value_from_logfile, listOfColumns, correct_only): ''' This function collects the values from one run. Only columns that should be part of the table are collected. ''' def read_logfile_lines(logfilename): if not logfilename: return [] try: with open(logfilename, 'rt') as logfile: return logfile.readlines() except IOError as e: logging.warning("Could not read value from logfile: %s", e) return [] status = Util.get_column_value(sourcefileTag, 'status', '') category = Util.get_column_value(sourcefileTag, 'category', result.CATEGORY_MISSING) score = result.score_for_task(sourcefileTag.get('name'), sourcefileTag.get('properties', '').split(), category) logfileLines = None values = [] for column in listOfColumns: # for all columns that should be shown value = None # default value if column.title.lower() == 'score': value = str(score) elif column.title.lower() == 'status': value = status elif not correct_only or category == result.CATEGORY_CORRECT: if not column.pattern: # collect values from XML value = Util.get_column_value(sourcefileTag, column.title) else: # collect values from logfile if logfileLines is None: # cache content logfileLines = read_logfile_lines(sourcefileTag.get('logfile')) value = get_value_from_logfile(logfileLines, column.pattern) if column.number_of_digits is not None: value = Util.format_number(value, column.number_of_digits) values.append(value) return RunResult(get_task_id(sourcefileTag), status, category, score, sourcefileTag.get('logfile'), listOfColumns, values)
def get_stats(rows): stats = parallel.map(get_stats_of_run_set, rows_to_columns(rows)) # column-wise rowsForStats = list(map(Util.flatten, zip(*stats))) # row-wise # Calculate maximal score and number of true/false files for the given properties count_true = count_false = max_score = 0 for row in rows: if not row.properties: # properties missing for at least one task, result would be wrong count_true = count_false = 0 logging.info('Missing property for %s.', row.filename) break correct_result = result.satisfies_file_property(row.filename, row.properties) if correct_result is True: count_true += 1 elif correct_result is False: count_false += 1 max_score += result.score_for_task(row.filename, row.properties, result.CATEGORY_CORRECT) task_counts = 'in total {0} true tasks, {1} false tasks'.format(count_true, count_false) if max_score: score_row = tempita.bunch(id='score', title='score ({0} tasks, max score: {1})'.format(len(rows), max_score), description=task_counts, content=rowsForStats[7]) def indent(n): return ' '*(n*4) return [tempita.bunch(id=None, title='total tasks', description=task_counts, content=rowsForStats[0]), tempita.bunch(id=None, title=indent(1)+'correct results', description='(property holds + result is true) OR (property does not hold + result is false)', content=rowsForStats[1]), tempita.bunch(id=None, title=indent(2)+'correct true', description='property holds + result is true', content=rowsForStats[2]), tempita.bunch(id=None, title=indent(2)+'correct false', description='property does not hold + result is false', content=rowsForStats[3]), tempita.bunch(id=None, title=indent(1)+'incorrect results', description='(property holds + result is false) OR (property does not hold + result is true)', content=rowsForStats[4]), tempita.bunch(id=None, title=indent(2)+'incorrect true', description='property does not hold + result is true', content=rowsForStats[5]), tempita.bunch(id=None, title=indent(2)+'incorrect false', description='property holds + result is false', content=rowsForStats[6]), ] + ([score_row] if max_score else [])