def add_result(self, run): self.counter += 1 self.dic[run.category] += 1 self.dic[(run.category, result.get_result_classification(run.status))] += 1 for prop in run.properties: self.score += prop.compute_score(run.category, run.status) self.max_score += prop.max_score(run.expected_results.get(prop.filename))
def add_result(self, run): self.counter += 1 self.dic[run.category] += 1 self.dic[(run.category, result.get_result_classification(run.status))] += 1 self.score += result.score_for_task(run.identifier, run.properties, run.category, run.status) #if run.properties: self.max_score += result.score_for_task(run.identifier, run.properties, result.CATEGORY_CORRECT, None)
def merge(result_xml, witness_sets, overwrite_status): for run in result_xml.findall("run"): try: status_from_verification = run.find('column[@title="status"]').get( "value") category_from_verification = run.find( 'column[@title="category"]').get("value") except AttributeError: status_from_verification = "not found" category_from_verification = "not found" ( statusWit, categoryWit, status_from_verification, category_from_verification, ) = get_validation_result( run, witness_sets, status_from_verification, category_from_verification, ) # Overwrite status with status from witness if ((overwrite_status or result.RESULT_CLASS_FALSE == result.get_result_classification(status_from_verification)) and "correct" == category_from_verification and statusWit is not None and categoryWit is not None): try: run.find('column[@title="status"]').set("value", statusWit) run.find('column[@title="category"]').set("value", categoryWit) except AttributeError: pass # Clean-up an entry that can be inferred by table-generator automatically, avoids path confusion del run.attrib["logfile"]
def determine_result(self, returncode, returnsignal, output, isTimeout): for line in output: line = line.strip() if result.get_result_classification( line) != result.RESULT_CLASS_OTHER: return line return result.RESULT_UNKNOWN
def get_stats_of_number_column(values, categoryList, columnTitle): assert len(values) == len(categoryList) try: valueList = [Util.to_decimal(v) for v in values] except InvalidOperation as e: if columnTitle != "host": # we ignore values of column host, used in cloud-mode logging.warning("%s. Statistics may be wrong.", e) return (StatValue(0), StatValue(0), StatValue(0), StatValue(0), StatValue(0), StatValue(0), StatValue(0)) valuesPerCategory = collections.defaultdict(list) for value, catStat in zip(valueList, categoryList): category, status = catStat if status is None: continue valuesPerCategory[category, result.get_result_classification(status)].append(value) return (StatValue.from_list(valueList), StatValue.from_list(valuesPerCategory[result.CATEGORY_CORRECT, result.RESULT_CLASS_TRUE] + valuesPerCategory[result.CATEGORY_CORRECT, result.RESULT_CLASS_FALSE]), StatValue.from_list(valuesPerCategory[result.CATEGORY_CORRECT, result.RESULT_CLASS_TRUE]), StatValue.from_list(valuesPerCategory[result.CATEGORY_CORRECT, result.RESULT_CLASS_FALSE]), StatValue.from_list(valuesPerCategory[result.CATEGORY_WRONG, result.RESULT_CLASS_TRUE] + valuesPerCategory[result.CATEGORY_WRONG, result.RESULT_CLASS_FALSE]), StatValue.from_list(valuesPerCategory[result.CATEGORY_WRONG, result.RESULT_CLASS_TRUE]), StatValue.from_list(valuesPerCategory[result.CATEGORY_WRONG, result.RESULT_CLASS_FALSE]), )
def _get_stats_of_number_column(values, categoryList, correct_only): valueList = [util.to_decimal(v) for v in values] assert len(valueList) == len(categoryList) valuesPerCategory = collections.defaultdict(list) for value, (category, status) in zip(valueList, categoryList): if status is None: continue valuesPerCategory[category, result.get_result_classification(status)].append( value ) stats = ColumnStatistics() stats.total = StatValue.from_list(valueList) def create_stat_value_for(*keys): all_values_for_keys = list( itertools.chain.from_iterable(valuesPerCategory[key] for key in keys) ) return StatValue.from_list(all_values_for_keys) stats.correct = create_stat_value_for( (result.CATEGORY_CORRECT, result.RESULT_CLASS_TRUE), (result.CATEGORY_CORRECT, result.RESULT_CLASS_FALSE), ) stats.correct_true = create_stat_value_for( (result.CATEGORY_CORRECT, result.RESULT_CLASS_TRUE) ) stats.correct_false = create_stat_value_for( (result.CATEGORY_CORRECT, result.RESULT_CLASS_FALSE) ) stats.correct_unconfirmed = create_stat_value_for( (result.CATEGORY_CORRECT_UNCONFIRMED, result.RESULT_CLASS_TRUE), (result.CATEGORY_CORRECT_UNCONFIRMED, result.RESULT_CLASS_FALSE), ) stats.correct_unconfirmed_true = create_stat_value_for( (result.CATEGORY_CORRECT_UNCONFIRMED, result.RESULT_CLASS_TRUE) ) stats.correct_unconfirmed_false = create_stat_value_for( (result.CATEGORY_CORRECT_UNCONFIRMED, result.RESULT_CLASS_FALSE) ) if not correct_only: stats.wrong = create_stat_value_for( (result.CATEGORY_WRONG, result.RESULT_CLASS_TRUE), (result.CATEGORY_WRONG, result.RESULT_CLASS_FALSE), ) stats.wrong_true = create_stat_value_for( (result.CATEGORY_WRONG, result.RESULT_CLASS_TRUE) ) stats.wrong_false = create_stat_value_for( (result.CATEGORY_WRONG, result.RESULT_CLASS_FALSE) ) return stats
def _get_stats_of_status_column(run_results, col): stats = ColumnStatistics() stats.score = StatValue(sum(run_result.score or 0 for run_result in run_results)) stats.total = StatValue( sum(1 for run_result in run_results if run_result.values[col]) ) counts = collections.Counter( (run_result.category, result.get_result_classification(run_result.values[col])) for run_result in run_results ) def create_stat_value_for(*keys): return StatValue(sum(counts[key] for key in keys)) stats.correct = create_stat_value_for( (result.CATEGORY_CORRECT, result.RESULT_CLASS_TRUE), (result.CATEGORY_CORRECT, result.RESULT_CLASS_FALSE), ) stats.correct_true = create_stat_value_for( (result.CATEGORY_CORRECT, result.RESULT_CLASS_TRUE) ) stats.correct_false = create_stat_value_for( (result.CATEGORY_CORRECT, result.RESULT_CLASS_FALSE) ) stats.correct_unconfirmed = create_stat_value_for( (result.CATEGORY_CORRECT_UNCONFIRMED, result.RESULT_CLASS_TRUE), (result.CATEGORY_CORRECT_UNCONFIRMED, result.RESULT_CLASS_FALSE), ) stats.correct_unconfirmed_true = create_stat_value_for( (result.CATEGORY_CORRECT_UNCONFIRMED, result.RESULT_CLASS_TRUE) ) stats.correct_unconfirmed_false = create_stat_value_for( (result.CATEGORY_CORRECT_UNCONFIRMED, result.RESULT_CLASS_FALSE) ) stats.wrong = create_stat_value_for( (result.CATEGORY_WRONG, result.RESULT_CLASS_TRUE), (result.CATEGORY_WRONG, result.RESULT_CLASS_FALSE), ) stats.wrong_true = create_stat_value_for( (result.CATEGORY_WRONG, result.RESULT_CLASS_TRUE) ) stats.wrong_false = create_stat_value_for( (result.CATEGORY_WRONG, result.RESULT_CLASS_FALSE) ) return stats
def main(argv=None): if argv is None: argv = sys.argv if len(argv) < 3: sys.exit( 'Usage: ' + argv[0] + ' <results-xml> [<witness-xml>]* [--no-overwrite-status-true].\n') resultFile = argv[1] witnessFiles = [] isOverwrite = True for i in range(2, len(argv)): if len(argv) > i and not argv[i].startswith('--'): witnessFiles.append(argv[i]) if argv[i] == '--no-overwrite-status-true': isOverwrite = False if not os.path.exists(resultFile) or not os.path.isfile(resultFile): sys.exit('File {0} does not exist.'.format(repr(resultFile))) resultXML = TableGenerator.parse_results_file(resultFile) witnessSets = [] for witnessFile in witnessFiles: if not os.path.exists(witnessFile) or not os.path.isfile(witnessFile): sys.exit('File {0} does not exist.'.format(repr(witnessFile))) witnessXML = TableGenerator.parse_results_file(witnessFile) witnessSets.append(getWitnesses(witnessXML)) for result in resultXML.findall('run'): run = result.get('name') statusWit, categoryWit = (None, None) i = 0 for witnessSet in witnessSets: i = i + 1 witness = witnessSet.get(run, None) # copy data from witness if witness is not None: statusWitNew, categoryWitNew = getWitnessResult( witness, result) if (categoryWit is None or not categoryWit.startswith(Result.CATEGORY_CORRECT) or categoryWitNew == Result.CATEGORY_CORRECT or statusWitNew.startswith('witness invalid')): statusWit, categoryWit = (statusWitNew, categoryWitNew) # Overwrite status with status from witness if ((isOverwrite or Result.RESULT_CLASS_FALSE == Result.get_result_classification( result.findall('column[@title="status"]')[0].get('value'))) and 'correct' == result.findall('column[@title="category"]')[0].get('value') and statusWit is not None and categoryWit is not None): #print(run, statusWit, categoryWit) result.findall('column[@title="status"]')[0].set( 'value', statusWit) result.findall('column[@title="category"]')[0].set( 'value', categoryWit) # Clean-up an entry that can be inferred by table-generator automatically, avoids path confusion del result.attrib['logfile'] filename = resultFile + '.merged.xml.bz2' print(' ' + filename) open_func = bz2.BZ2File if hasattr(bz2.BZ2File, 'writable') else util.BZ2FileHack with io.TextIOWrapper(open_func(filename, 'wb'), encoding='utf-8') as xml_file: xml_file.write( xml_to_string(resultXML).replace(' \n', '').replace(' \n', ''))
def main(argv=None): if argv is None: argv = sys.argv if len(argv) < 3: sys.exit( "Usage: " + argv[0] + " <results-xml> [<witness-xml>]* [--no-overwrite-status-true].\n" ) resultFile = argv[1] witnessFiles = [] isOverwrite = True for i in range(2, len(argv)): if len(argv) > i and not argv[i].startswith("--"): witnessFiles.append(argv[i]) if argv[i] == "--no-overwrite-status-true": isOverwrite = False if not os.path.exists(resultFile) or not os.path.isfile(resultFile): sys.exit("File {0} does not exist.".format(repr(resultFile))) resultXML = tablegenerator.parse_results_file(resultFile) witnessSets = [] for witnessFile in witnessFiles: if not os.path.exists(witnessFile) or not os.path.isfile(witnessFile): sys.exit("File {0} does not exist.".format(repr(witnessFile))) witnessXML = tablegenerator.parse_results_file(witnessFile) witnessSets.append(getWitnesses(witnessXML)) for result_tag in resultXML.findall("run"): run = result_tag.get("name") try: status_from_verification = result_tag.find('column[@title="status"]').get( "value" ) category_from_verification = result_tag.find( 'column[@title="category"]' ).get("value") except AttributeError: status_from_verification = "not found" category_from_verification = "not found" print(run, status_from_verification, category_from_verification) statusWit, categoryWit = (None, None) for witnessSet in witnessSets: witness = witnessSet.get(run, None) # copy data from witness if witness is not None and len(witness) > 0: if result_tag.get("properties") == "coverage-error-call": status_from_validation = witness.find( 'column[@title="status"]' ).get("value") if status_from_validation == "true": statusWit, categoryWit = (status_from_verification, "correct") category_from_verification = "correct" scoreColumn = ElementTree.Element( "column", {"title": "score", "value": "1"} ) result_tag.append(scoreColumn) elif result_tag.get("properties") == "coverage-branches": try: coverage_value = ( witness.find('column[@title="branches_covered"]') .get("value") .replace("%", "") ) except AttributeError: coverage_value = "0.00" statusWit, categoryWit = (status_from_verification, "correct") category_from_verification = "correct" try: coverage_float = float(coverage_value) except ValueError: continue scoreColumn = ElementTree.Element( "column", {"title": "score", "value": str(coverage_float / 100)}, ) result_tag.append(scoreColumn) else: # For verification statusWitNew, categoryWitNew = getWitnessResult(witness, result_tag) print(statusWitNew, categoryWitNew) if ( categoryWit is None or not categoryWit.startswith(result.CATEGORY_CORRECT) or categoryWitNew == result.CATEGORY_CORRECT ): statusWit, categoryWit = (statusWitNew, categoryWitNew) # Overwrite status with status from witness if ( ( isOverwrite or result.RESULT_CLASS_FALSE == result.get_result_classification(status_from_verification) ) and "correct" == category_from_verification and statusWit is not None and categoryWit is not None ): try: result_tag.find('column[@title="status"]').set("value", statusWit) result_tag.find('column[@title="category"]').set("value", categoryWit) except AttributeError: pass # Clean-up an entry that can be inferred by table-generator automatically, avoids path confusion del result_tag.attrib["logfile"] filename = resultFile + ".merged.xml.bz2" print(" " + filename) with io.TextIOWrapper(bz2.BZ2File(filename, "wb"), encoding="utf-8") as xml_file: xml_file.write( xml_to_string(resultXML).replace(" \n", "").replace(" \n", "") )
def determine_result(self, run): for line in run.output: if result.get_result_classification(line) != result.RESULT_CLASS_OTHER: return line return result.RESULT_UNKNOWN
def main(argv=None): if argv is None: argv = sys.argv if len(argv) < 3: sys.exit( 'Usage: ' + argv[0] + ' <results-xml> [<witness-xml>]* [--no-overwrite-status-true].\n') resultFile = argv[1] witnessFiles = [] isOverwrite = True for i in range(2, len(argv)): if len(argv) > i and not argv[i].startswith('--'): witnessFiles.append(argv[i]) if argv[i] == '--no-overwrite-status-true': isOverwrite = False if not os.path.exists(resultFile) or not os.path.isfile(resultFile): sys.exit('File {0} does not exist.'.format(repr(resultFile))) resultXML = TableGenerator.parse_results_file(resultFile) witnessSets = [] for witnessFile in witnessFiles: if not os.path.exists(witnessFile) or not os.path.isfile(witnessFile): sys.exit('File {0} does not exist.'.format(repr(witnessFile))) witnessXML = TableGenerator.parse_results_file(witnessFile) witnessSets.append(getWitnesses(witnessXML)) for result in resultXML.findall('run'): run = result.get('name') try: status_from_verification = result.findall( 'column[@title="status"]')[0].get('value') category_from_verification = result.findall( 'column[@title="category"]')[0].get('value') except: status_from_verification = "not found" category_from_verification = "not found" statusWit, categoryWit = (None, None) for witnessSet in witnessSets: witness = witnessSet.get(run, None) # copy data from witness if witness is not None: if result.get('properties') == 'coverage-error-call': status_from_validation = witness.findall( 'column[@title="status"]')[0].get('value') if status_from_validation == "true": statusWit, categoryWit = (status_from_verification, 'correct') category_from_verification = 'correct' scoreColumn = ET.Element('column', { 'title': 'score', 'value': '1' }) result.append(scoreColumn) elif result.get('properties') == 'coverage-branches': try: coverage_value = witness.findall( 'column[@title="branches_covered"]')[0].get( 'value').replace("%", "") except IndexError: coverage_value = '0.00' statusWit, categoryWit = (status_from_verification, 'correct') category_from_verification = 'correct' scoreColumn = ET.Element( 'column', { 'title': 'score', 'value': str(float(coverage_value) / 100) }) result.append(scoreColumn) else: # For verification statusWitNew, categoryWitNew = getWitnessResult( witness, result) if (categoryWit is None or not categoryWit.startswith(Result.CATEGORY_CORRECT) or categoryWitNew == Result.CATEGORY_CORRECT or statusWitNew.startswith('witness invalid')): statusWit, categoryWit = (statusWitNew, categoryWitNew) # Overwrite status with status from witness if ((isOverwrite or Result.RESULT_CLASS_FALSE == Result.get_result_classification(status_from_verification)) and 'correct' == category_from_verification and statusWit is not None and categoryWit is not None): #print(run, statusWit, categoryWit) try: result.findall('column[@title="status"]')[0].set( 'value', statusWit) result.findall('column[@title="category"]')[0].set( 'value', categoryWit) except: pass # Clean-up an entry that can be inferred by table-generator automatically, avoids path confusion del result.attrib['logfile'] filename = resultFile + '.merged.xml.bz2' print(' ' + filename) open_func = bz2.BZ2File if hasattr(bz2.BZ2File, 'writable') else util.BZ2FileHack with io.TextIOWrapper(open_func(filename, 'wb'), encoding='utf-8') as xml_file: xml_file.write( xml_to_string(resultXML).replace(' \n', '').replace(' \n', ''))
def main(argv=None): if argv is None: argv = sys.argv if len(argv) < 3: sys.exit('Usage: ' + argv[0] + ' <results-xml> [<witness-xml>]* [--no-overwrite-status-true].\n') resultFile = argv[1] witnessFiles = [] isOverwrite = True for i in range(2, len(argv)): if len(argv) > i and not argv[i].startswith('--'): witnessFiles.append(argv[i]) if argv[i] == '--no-overwrite-status-true': isOverwrite = False if not os.path.exists(resultFile) or not os.path.isfile(resultFile): sys.exit('File {0} does not exist.'.format(repr(resultFile))) resultXML = TableGenerator.parse_results_file(resultFile) witnessSets = [] for witnessFile in witnessFiles: if not os.path.exists(witnessFile) or not os.path.isfile(witnessFile): sys.exit('File {0} does not exist.'.format(repr(witnessFile))) witnessXML = TableGenerator.parse_results_file(witnessFile) witnessSets.append(getWitnesses(witnessXML)) for result in resultXML.findall('run'): run = result.get('name') statusWit, categoryWit = (None, None) i = 0 for witnessSet in witnessSets: i = i + 1 witness = witnessSet.get(run, None) # copy data from witness if witness is not None: statusWitNew, categoryWitNew = getWitnessResult(witness, result) if ( categoryWit is None or not categoryWit.startswith(Result.CATEGORY_CORRECT) or categoryWitNew == Result.CATEGORY_CORRECT or statusWitNew.startswith('witness invalid') ): statusWit, categoryWit = (statusWitNew, categoryWitNew) # Overwrite status with status from witness if ( ( isOverwrite or Result.RESULT_CLASS_FALSE == Result.get_result_classification( result.findall('column[@title="status"]')[0].get('value')) ) and 'correct' == result.findall('column[@title="category"]')[0].get('value') and statusWit is not None and categoryWit is not None ): #print(run, statusWit, categoryWit) result.findall('column[@title="status"]')[0].set('value', statusWit) result.findall('column[@title="category"]')[0].set('value', categoryWit) # Clean-up an entry that can be inferred by table-generator automatically, avoids path confusion del result.attrib['logfile'] filename = resultFile + '.merged.xml.bz2' print (' ' + filename) open_func = bz2.BZ2File if hasattr(bz2.BZ2File, 'writable') else util.BZ2FileHack with io.TextIOWrapper(open_func(filename, 'wb'), encoding='utf-8') as xml_file: xml_file.write(xml_to_string(resultXML).replace(' \n','').replace(' \n',''))
def get_stats_of_run_set(runResults): """ This function returns the numbers of the statistics. @param runResults: All the results of the execution of one run set (as list of RunResult objects) """ # convert: # [['TRUE', 0,1], ['FALSE', 0,2]] --> [['TRUE', 'FALSE'], [0,1, 0,2]] # in python2 this is a list, in python3 this is the iterator of the list # this works, because we iterate over the list some lines below listsOfValues = zip(*[runResult.values for runResult in runResults]) columns = runResults[0].columns statusList = [(runResult.category, runResult.status) for runResult in runResults] # collect some statistics totalRow = [] correctRow = [] correctTrueRow = [] correctFalseRow = [] incorrectRow = [] wrongTrueRow = [] wrongFalseRow = [] scoreRow = [] for column, values in zip(columns, listsOfValues): if column.title == 'status': total = StatValue(len([runResult.status for runResult in runResults if runResult.status])) counts = collections.Counter((category, result.get_result_classification(status)) for category, status in statusList) countCorrectTrue = counts[result.CATEGORY_CORRECT, result.RESULT_CLASS_TRUE] countCorrectFalse = counts[result.CATEGORY_CORRECT, result.RESULT_CLASS_FALSE] countWrongTrue = counts[result.CATEGORY_WRONG, result.RESULT_CLASS_TRUE] countWrongFalse = counts[result.CATEGORY_WRONG, result.RESULT_CLASS_FALSE] correct = StatValue(countCorrectTrue + countCorrectFalse) correctTrue = StatValue(countCorrectTrue) correctFalse = StatValue(countCorrectFalse) incorrect = StatValue(countWrongTrue + countWrongFalse) wrongTrue = StatValue(countWrongTrue) wrongFalse = StatValue(countWrongFalse) score = StatValue(sum(run_result.score for run_result in runResults)) else: total, correct, correctTrue, correctFalse, incorrect, wrongTrue, wrongFalse = get_stats_of_number_column(values, statusList, column.title) score = '' if (total.sum, correct.sum, correctTrue.sum, correctFalse.sum, incorrect.sum, wrongTrue.sum, wrongFalse.sum) == (0,0,0,0,0,0,0): (total, correct, correctTrue, correctFalse, incorrect, wrongTrue, wrongFalse) = (None, None, None, None, None, None, None) totalRow.append(total) correctRow.append(correct) correctTrueRow.append(correctTrue) correctFalseRow.append(correctFalse) incorrectRow.append(incorrect) wrongTrueRow.append(wrongTrue) wrongFalseRow.append(wrongFalse) scoreRow.append(score) def replace_irrelevant(row): if not row: return count = row[0] if not count or not count.sum: for i in range(1, len(row)): row[i] = None replace_irrelevant(totalRow) replace_irrelevant(correctRow) replace_irrelevant(correctTrueRow) replace_irrelevant(correctFalseRow) replace_irrelevant(incorrectRow) replace_irrelevant(wrongTrueRow) replace_irrelevant(wrongFalseRow) replace_irrelevant(scoreRow) return (totalRow, correctRow, correctTrueRow, correctFalseRow, incorrectRow, wrongTrueRow, wrongFalseRow, scoreRow)