def add_result(self, run):
self.counter += 1
self.dic[run.category] += 1
self.dic[(run.category, result.get_result_classification(run.status))] += 1
for prop in run.properties:
self.score += prop.compute_score(run.category, run.status)
self.max_score += prop.max_score(run.expected_results.get(prop.filename))
def add_result(self, run):
self.counter += 1
self.dic[run.category] += 1
self.dic[(run.category, result.get_result_classification(run.status))] += 1
self.score += result.score_for_task(run.identifier, run.properties, run.category, run.status)
#if run.properties:
self.max_score += result.score_for_task(run.identifier, run.properties, result.CATEGORY_CORRECT, None)
def merge(result_xml, witness_sets, overwrite_status):
for run in result_xml.findall("run"):
try:
status_from_verification = run.find('column[@title="status"]').get(
"value")
category_from_verification = run.find(
'column[@title="category"]').get("value")
except AttributeError:
status_from_verification = "not found"
category_from_verification = "not found"
(
statusWit,
categoryWit,
status_from_verification,
category_from_verification,
) = get_validation_result(
run,
witness_sets,
status_from_verification,
category_from_verification,
)
# Overwrite status with status from witness
if ((overwrite_status or result.RESULT_CLASS_FALSE
== result.get_result_classification(status_from_verification))
and "correct" == category_from_verification
and statusWit is not None and categoryWit is not None):
try:
run.find('column[@title="status"]').set("value", statusWit)
run.find('column[@title="category"]').set("value", categoryWit)
except AttributeError:
pass
# Clean-up an entry that can be inferred by table-generator automatically, avoids path confusion
del run.attrib["logfile"]
def determine_result(self, returncode, returnsignal, output, isTimeout):
for line in output:
line = line.strip()
if result.get_result_classification(
line) != result.RESULT_CLASS_OTHER:
return line
return result.RESULT_UNKNOWN
def add_result(self, run):
self.counter += 1
self.dic[run.category] += 1
self.dic[(run.category, result.get_result_classification(run.status))] += 1
self.score += result.score_for_task(run.identifier, run.properties, run.category, run.status)
#if run.properties:
self.max_score += result.score_for_task(run.identifier, run.properties, result.CATEGORY_CORRECT, None)
def get_stats_of_number_column(values, categoryList, columnTitle):
assert len(values) == len(categoryList)
try:
valueList = [Util.to_decimal(v) for v in values]
except InvalidOperation as e:
if columnTitle != "host": # we ignore values of column host, used in cloud-mode
logging.warning("%s. Statistics may be wrong.", e)
return (StatValue(0), StatValue(0), StatValue(0), StatValue(0), StatValue(0), StatValue(0), StatValue(0))
valuesPerCategory = collections.defaultdict(list)
for value, catStat in zip(valueList, categoryList):
category, status = catStat
if status is None:
continue
valuesPerCategory[category, result.get_result_classification(status)].append(value)
return (StatValue.from_list(valueList),
StatValue.from_list(valuesPerCategory[result.CATEGORY_CORRECT, result.RESULT_CLASS_TRUE]
+ valuesPerCategory[result.CATEGORY_CORRECT, result.RESULT_CLASS_FALSE]),
StatValue.from_list(valuesPerCategory[result.CATEGORY_CORRECT, result.RESULT_CLASS_TRUE]),
StatValue.from_list(valuesPerCategory[result.CATEGORY_CORRECT, result.RESULT_CLASS_FALSE]),
StatValue.from_list(valuesPerCategory[result.CATEGORY_WRONG, result.RESULT_CLASS_TRUE]
+ valuesPerCategory[result.CATEGORY_WRONG, result.RESULT_CLASS_FALSE]),
StatValue.from_list(valuesPerCategory[result.CATEGORY_WRONG, result.RESULT_CLASS_TRUE]),
StatValue.from_list(valuesPerCategory[result.CATEGORY_WRONG, result.RESULT_CLASS_FALSE]),
)
def _get_stats_of_number_column(values, categoryList, correct_only):
valueList = [util.to_decimal(v) for v in values]
assert len(valueList) == len(categoryList)
valuesPerCategory = collections.defaultdict(list)
for value, (category, status) in zip(valueList, categoryList):
if status is None:
continue
valuesPerCategory[category, result.get_result_classification(status)].append(
value
)
stats = ColumnStatistics()
stats.total = StatValue.from_list(valueList)
def create_stat_value_for(*keys):
all_values_for_keys = list(
itertools.chain.from_iterable(valuesPerCategory[key] for key in keys)
)
return StatValue.from_list(all_values_for_keys)
stats.correct = create_stat_value_for(
(result.CATEGORY_CORRECT, result.RESULT_CLASS_TRUE),
(result.CATEGORY_CORRECT, result.RESULT_CLASS_FALSE),
)
stats.correct_true = create_stat_value_for(
(result.CATEGORY_CORRECT, result.RESULT_CLASS_TRUE)
)
stats.correct_false = create_stat_value_for(
(result.CATEGORY_CORRECT, result.RESULT_CLASS_FALSE)
)
stats.correct_unconfirmed = create_stat_value_for(
(result.CATEGORY_CORRECT_UNCONFIRMED, result.RESULT_CLASS_TRUE),
(result.CATEGORY_CORRECT_UNCONFIRMED, result.RESULT_CLASS_FALSE),
)
stats.correct_unconfirmed_true = create_stat_value_for(
(result.CATEGORY_CORRECT_UNCONFIRMED, result.RESULT_CLASS_TRUE)
)
stats.correct_unconfirmed_false = create_stat_value_for(
(result.CATEGORY_CORRECT_UNCONFIRMED, result.RESULT_CLASS_FALSE)
)
if not correct_only:
stats.wrong = create_stat_value_for(
(result.CATEGORY_WRONG, result.RESULT_CLASS_TRUE),
(result.CATEGORY_WRONG, result.RESULT_CLASS_FALSE),
)
stats.wrong_true = create_stat_value_for(
(result.CATEGORY_WRONG, result.RESULT_CLASS_TRUE)
)
stats.wrong_false = create_stat_value_for(
(result.CATEGORY_WRONG, result.RESULT_CLASS_FALSE)
)
return stats
def _get_stats_of_status_column(run_results, col):
stats = ColumnStatistics()
stats.score = StatValue(sum(run_result.score or 0 for run_result in run_results))
stats.total = StatValue(
sum(1 for run_result in run_results if run_result.values[col])
)
counts = collections.Counter(
(run_result.category, result.get_result_classification(run_result.values[col]))
for run_result in run_results
)
def create_stat_value_for(*keys):
return StatValue(sum(counts[key] for key in keys))
stats.correct = create_stat_value_for(
(result.CATEGORY_CORRECT, result.RESULT_CLASS_TRUE),
(result.CATEGORY_CORRECT, result.RESULT_CLASS_FALSE),
)
stats.correct_true = create_stat_value_for(
(result.CATEGORY_CORRECT, result.RESULT_CLASS_TRUE)
)
stats.correct_false = create_stat_value_for(
(result.CATEGORY_CORRECT, result.RESULT_CLASS_FALSE)
)
stats.correct_unconfirmed = create_stat_value_for(
(result.CATEGORY_CORRECT_UNCONFIRMED, result.RESULT_CLASS_TRUE),
(result.CATEGORY_CORRECT_UNCONFIRMED, result.RESULT_CLASS_FALSE),
)
stats.correct_unconfirmed_true = create_stat_value_for(
(result.CATEGORY_CORRECT_UNCONFIRMED, result.RESULT_CLASS_TRUE)
)
stats.correct_unconfirmed_false = create_stat_value_for(
(result.CATEGORY_CORRECT_UNCONFIRMED, result.RESULT_CLASS_FALSE)
)
stats.wrong = create_stat_value_for(
(result.CATEGORY_WRONG, result.RESULT_CLASS_TRUE),
(result.CATEGORY_WRONG, result.RESULT_CLASS_FALSE),
)
stats.wrong_true = create_stat_value_for(
(result.CATEGORY_WRONG, result.RESULT_CLASS_TRUE)
)
stats.wrong_false = create_stat_value_for(
(result.CATEGORY_WRONG, result.RESULT_CLASS_FALSE)
)
return stats
def main(argv=None):
if argv is None:
argv = sys.argv
if len(argv) < 3:
sys.exit(
'Usage: ' + argv[0] +
' <results-xml> [<witness-xml>]* [--no-overwrite-status-true].\n')
resultFile = argv[1]
witnessFiles = []
isOverwrite = True
for i in range(2, len(argv)):
if len(argv) > i and not argv[i].startswith('--'):
witnessFiles.append(argv[i])
if argv[i] == '--no-overwrite-status-true':
isOverwrite = False
if not os.path.exists(resultFile) or not os.path.isfile(resultFile):
sys.exit('File {0} does not exist.'.format(repr(resultFile)))
resultXML = TableGenerator.parse_results_file(resultFile)
witnessSets = []
for witnessFile in witnessFiles:
if not os.path.exists(witnessFile) or not os.path.isfile(witnessFile):
sys.exit('File {0} does not exist.'.format(repr(witnessFile)))
witnessXML = TableGenerator.parse_results_file(witnessFile)
witnessSets.append(getWitnesses(witnessXML))
for result in resultXML.findall('run'):
run = result.get('name')
statusWit, categoryWit = (None, None)
i = 0
for witnessSet in witnessSets:
i = i + 1
witness = witnessSet.get(run, None)
# copy data from witness
if witness is not None:
statusWitNew, categoryWitNew = getWitnessResult(
witness, result)
if (categoryWit is None
or not categoryWit.startswith(Result.CATEGORY_CORRECT)
or categoryWitNew == Result.CATEGORY_CORRECT
or statusWitNew.startswith('witness invalid')):
statusWit, categoryWit = (statusWitNew, categoryWitNew)
# Overwrite status with status from witness
if ((isOverwrite
or Result.RESULT_CLASS_FALSE == Result.get_result_classification(
result.findall('column[@title="status"]')[0].get('value')))
and 'correct'
== result.findall('column[@title="category"]')[0].get('value')
and statusWit is not None and categoryWit is not None):
#print(run, statusWit, categoryWit)
result.findall('column[@title="status"]')[0].set(
'value', statusWit)
result.findall('column[@title="category"]')[0].set(
'value', categoryWit)
# Clean-up an entry that can be inferred by table-generator automatically, avoids path confusion
del result.attrib['logfile']
filename = resultFile + '.merged.xml.bz2'
print(' ' + filename)
open_func = bz2.BZ2File if hasattr(bz2.BZ2File,
'writable') else util.BZ2FileHack
with io.TextIOWrapper(open_func(filename, 'wb'),
encoding='utf-8') as xml_file:
xml_file.write(
xml_to_string(resultXML).replace(' \n', '').replace(' \n', ''))
def main(argv=None):
if argv is None:
argv = sys.argv
if len(argv) < 3:
sys.exit(
"Usage: "
+ argv[0]
+ " <results-xml> [<witness-xml>]* [--no-overwrite-status-true].\n"
)
resultFile = argv[1]
witnessFiles = []
isOverwrite = True
for i in range(2, len(argv)):
if len(argv) > i and not argv[i].startswith("--"):
witnessFiles.append(argv[i])
if argv[i] == "--no-overwrite-status-true":
isOverwrite = False
if not os.path.exists(resultFile) or not os.path.isfile(resultFile):
sys.exit("File {0} does not exist.".format(repr(resultFile)))
resultXML = tablegenerator.parse_results_file(resultFile)
witnessSets = []
for witnessFile in witnessFiles:
if not os.path.exists(witnessFile) or not os.path.isfile(witnessFile):
sys.exit("File {0} does not exist.".format(repr(witnessFile)))
witnessXML = tablegenerator.parse_results_file(witnessFile)
witnessSets.append(getWitnesses(witnessXML))
for result_tag in resultXML.findall("run"):
run = result_tag.get("name")
try:
status_from_verification = result_tag.find('column[@title="status"]').get(
"value"
)
category_from_verification = result_tag.find(
'column[@title="category"]'
).get("value")
except AttributeError:
status_from_verification = "not found"
category_from_verification = "not found"
print(run, status_from_verification, category_from_verification)
statusWit, categoryWit = (None, None)
for witnessSet in witnessSets:
witness = witnessSet.get(run, None)
# copy data from witness
if witness is not None and len(witness) > 0:
if result_tag.get("properties") == "coverage-error-call":
status_from_validation = witness.find(
'column[@title="status"]'
).get("value")
if status_from_validation == "true":
statusWit, categoryWit = (status_from_verification, "correct")
category_from_verification = "correct"
scoreColumn = ElementTree.Element(
"column", {"title": "score", "value": "1"}
)
result_tag.append(scoreColumn)
elif result_tag.get("properties") == "coverage-branches":
try:
coverage_value = (
witness.find('column[@title="branches_covered"]')
.get("value")
.replace("%", "")
)
except AttributeError:
coverage_value = "0.00"
statusWit, categoryWit = (status_from_verification, "correct")
category_from_verification = "correct"
try:
coverage_float = float(coverage_value)
except ValueError:
continue
scoreColumn = ElementTree.Element(
"column",
{"title": "score", "value": str(coverage_float / 100)},
)
result_tag.append(scoreColumn)
else:
# For verification
statusWitNew, categoryWitNew = getWitnessResult(witness, result_tag)
print(statusWitNew, categoryWitNew)
if (
categoryWit is None
or not categoryWit.startswith(result.CATEGORY_CORRECT)
or categoryWitNew == result.CATEGORY_CORRECT
):
statusWit, categoryWit = (statusWitNew, categoryWitNew)
# Overwrite status with status from witness
if (
(
isOverwrite
or result.RESULT_CLASS_FALSE
== result.get_result_classification(status_from_verification)
)
and "correct" == category_from_verification
and statusWit is not None
and categoryWit is not None
):
try:
result_tag.find('column[@title="status"]').set("value", statusWit)
result_tag.find('column[@title="category"]').set("value", categoryWit)
except AttributeError:
pass
# Clean-up an entry that can be inferred by table-generator automatically, avoids path confusion
del result_tag.attrib["logfile"]
filename = resultFile + ".merged.xml.bz2"
print(" " + filename)
with io.TextIOWrapper(bz2.BZ2File(filename, "wb"), encoding="utf-8") as xml_file:
xml_file.write(
xml_to_string(resultXML).replace(" \n", "").replace(" \n", "")
)
def determine_result(self, run):
for line in run.output:
if result.get_result_classification(line) != result.RESULT_CLASS_OTHER:
return line
return result.RESULT_UNKNOWN
def main(argv=None):
if argv is None:
argv = sys.argv
if len(argv) < 3:
sys.exit(
'Usage: ' + argv[0] +
' <results-xml> [<witness-xml>]* [--no-overwrite-status-true].\n')
resultFile = argv[1]
witnessFiles = []
isOverwrite = True
for i in range(2, len(argv)):
if len(argv) > i and not argv[i].startswith('--'):
witnessFiles.append(argv[i])
if argv[i] == '--no-overwrite-status-true':
isOverwrite = False
if not os.path.exists(resultFile) or not os.path.isfile(resultFile):
sys.exit('File {0} does not exist.'.format(repr(resultFile)))
resultXML = TableGenerator.parse_results_file(resultFile)
witnessSets = []
for witnessFile in witnessFiles:
if not os.path.exists(witnessFile) or not os.path.isfile(witnessFile):
sys.exit('File {0} does not exist.'.format(repr(witnessFile)))
witnessXML = TableGenerator.parse_results_file(witnessFile)
witnessSets.append(getWitnesses(witnessXML))
for result in resultXML.findall('run'):
run = result.get('name')
try:
status_from_verification = result.findall(
'column[@title="status"]')[0].get('value')
category_from_verification = result.findall(
'column[@title="category"]')[0].get('value')
except:
status_from_verification = "not found"
category_from_verification = "not found"
statusWit, categoryWit = (None, None)
for witnessSet in witnessSets:
witness = witnessSet.get(run, None)
# copy data from witness
if witness is not None:
if result.get('properties') == 'coverage-error-call':
status_from_validation = witness.findall(
'column[@title="status"]')[0].get('value')
if status_from_validation == "true":
statusWit, categoryWit = (status_from_verification,
'correct')
category_from_verification = 'correct'
scoreColumn = ET.Element('column', {
'title': 'score',
'value': '1'
})
result.append(scoreColumn)
elif result.get('properties') == 'coverage-branches':
try:
coverage_value = witness.findall(
'column[@title="branches_covered"]')[0].get(
'value').replace("%", "")
except IndexError:
coverage_value = '0.00'
statusWit, categoryWit = (status_from_verification,
'correct')
category_from_verification = 'correct'
scoreColumn = ET.Element(
'column', {
'title': 'score',
'value': str(float(coverage_value) / 100)
})
result.append(scoreColumn)
else:
# For verification
statusWitNew, categoryWitNew = getWitnessResult(
witness, result)
if (categoryWit is None or
not categoryWit.startswith(Result.CATEGORY_CORRECT)
or categoryWitNew == Result.CATEGORY_CORRECT
or statusWitNew.startswith('witness invalid')):
statusWit, categoryWit = (statusWitNew, categoryWitNew)
# Overwrite status with status from witness
if ((isOverwrite or Result.RESULT_CLASS_FALSE
== Result.get_result_classification(status_from_verification))
and 'correct' == category_from_verification
and statusWit is not None and categoryWit is not None):
#print(run, statusWit, categoryWit)
try:
result.findall('column[@title="status"]')[0].set(
'value', statusWit)
result.findall('column[@title="category"]')[0].set(
'value', categoryWit)
except:
pass
# Clean-up an entry that can be inferred by table-generator automatically, avoids path confusion
del result.attrib['logfile']
filename = resultFile + '.merged.xml.bz2'
print(' ' + filename)
open_func = bz2.BZ2File if hasattr(bz2.BZ2File,
'writable') else util.BZ2FileHack
with io.TextIOWrapper(open_func(filename, 'wb'),
encoding='utf-8') as xml_file:
xml_file.write(
xml_to_string(resultXML).replace(' \n', '').replace(' \n', ''))
def main(argv=None):
if argv is None:
argv = sys.argv
if len(argv) < 3:
sys.exit('Usage: ' + argv[0] + ' <results-xml> [<witness-xml>]* [--no-overwrite-status-true].\n')
resultFile = argv[1]
witnessFiles = []
isOverwrite = True
for i in range(2, len(argv)):
if len(argv) > i and not argv[i].startswith('--'):
witnessFiles.append(argv[i])
if argv[i] == '--no-overwrite-status-true':
isOverwrite = False
if not os.path.exists(resultFile) or not os.path.isfile(resultFile):
sys.exit('File {0} does not exist.'.format(repr(resultFile)))
resultXML = TableGenerator.parse_results_file(resultFile)
witnessSets = []
for witnessFile in witnessFiles:
if not os.path.exists(witnessFile) or not os.path.isfile(witnessFile):
sys.exit('File {0} does not exist.'.format(repr(witnessFile)))
witnessXML = TableGenerator.parse_results_file(witnessFile)
witnessSets.append(getWitnesses(witnessXML))
for result in resultXML.findall('run'):
run = result.get('name')
statusWit, categoryWit = (None, None)
i = 0
for witnessSet in witnessSets:
i = i + 1
witness = witnessSet.get(run, None)
# copy data from witness
if witness is not None:
statusWitNew, categoryWitNew = getWitnessResult(witness, result)
if (
categoryWit is None or
not categoryWit.startswith(Result.CATEGORY_CORRECT) or
categoryWitNew == Result.CATEGORY_CORRECT or
statusWitNew.startswith('witness invalid')
):
statusWit, categoryWit = (statusWitNew, categoryWitNew)
# Overwrite status with status from witness
if (
( isOverwrite
or Result.RESULT_CLASS_FALSE == Result.get_result_classification(
result.findall('column[@title="status"]')[0].get('value'))
)
and 'correct' == result.findall('column[@title="category"]')[0].get('value')
and statusWit is not None
and categoryWit is not None
):
#print(run, statusWit, categoryWit)
result.findall('column[@title="status"]')[0].set('value', statusWit)
result.findall('column[@title="category"]')[0].set('value', categoryWit)
# Clean-up an entry that can be inferred by table-generator automatically, avoids path confusion
del result.attrib['logfile']
filename = resultFile + '.merged.xml.bz2'
print (' ' + filename)
open_func = bz2.BZ2File if hasattr(bz2.BZ2File, 'writable') else util.BZ2FileHack
with io.TextIOWrapper(open_func(filename, 'wb'), encoding='utf-8') as xml_file:
xml_file.write(xml_to_string(resultXML).replace(' \n','').replace(' \n',''))
def get_stats_of_run_set(runResults):
"""
This function returns the numbers of the statistics.
@param runResults: All the results of the execution of one run set (as list of RunResult objects)
"""
# convert:
# [['TRUE', 0,1], ['FALSE', 0,2]] --> [['TRUE', 'FALSE'], [0,1, 0,2]]
# in python2 this is a list, in python3 this is the iterator of the list
# this works, because we iterate over the list some lines below
listsOfValues = zip(*[runResult.values for runResult in runResults])
columns = runResults[0].columns
statusList = [(runResult.category, runResult.status) for runResult in runResults]
# collect some statistics
totalRow = []
correctRow = []
correctTrueRow = []
correctFalseRow = []
incorrectRow = []
wrongTrueRow = []
wrongFalseRow = []
scoreRow = []
for column, values in zip(columns, listsOfValues):
if column.title == 'status':
total = StatValue(len([runResult.status for runResult in runResults if runResult.status]))
counts = collections.Counter((category, result.get_result_classification(status))
for category, status in statusList)
countCorrectTrue = counts[result.CATEGORY_CORRECT, result.RESULT_CLASS_TRUE]
countCorrectFalse = counts[result.CATEGORY_CORRECT, result.RESULT_CLASS_FALSE]
countWrongTrue = counts[result.CATEGORY_WRONG, result.RESULT_CLASS_TRUE]
countWrongFalse = counts[result.CATEGORY_WRONG, result.RESULT_CLASS_FALSE]
correct = StatValue(countCorrectTrue + countCorrectFalse)
correctTrue = StatValue(countCorrectTrue)
correctFalse = StatValue(countCorrectFalse)
incorrect = StatValue(countWrongTrue + countWrongFalse)
wrongTrue = StatValue(countWrongTrue)
wrongFalse = StatValue(countWrongFalse)
score = StatValue(sum(run_result.score for run_result in runResults))
else:
total, correct, correctTrue, correctFalse, incorrect, wrongTrue, wrongFalse = get_stats_of_number_column(values, statusList, column.title)
score = ''
if (total.sum, correct.sum, correctTrue.sum, correctFalse.sum, incorrect.sum, wrongTrue.sum, wrongFalse.sum) == (0,0,0,0,0,0,0):
(total, correct, correctTrue, correctFalse, incorrect, wrongTrue, wrongFalse) = (None, None, None, None, None, None, None)
totalRow.append(total)
correctRow.append(correct)
correctTrueRow.append(correctTrue)
correctFalseRow.append(correctFalse)
incorrectRow.append(incorrect)
wrongTrueRow.append(wrongTrue)
wrongFalseRow.append(wrongFalse)
scoreRow.append(score)
def replace_irrelevant(row):
if not row:
return
count = row[0]
if not count or not count.sum:
for i in range(1, len(row)):
row[i] = None
replace_irrelevant(totalRow)
replace_irrelevant(correctRow)
replace_irrelevant(correctTrueRow)
replace_irrelevant(correctFalseRow)
replace_irrelevant(incorrectRow)
replace_irrelevant(wrongTrueRow)
replace_irrelevant(wrongFalseRow)
replace_irrelevant(scoreRow)
return (totalRow, correctRow, correctTrueRow, correctFalseRow, incorrectRow, wrongTrueRow, wrongFalseRow, scoreRow)