def end_test(self, name, attrs):
self.log_devcmd("# Test Case end: %s (#%d) - %s" %
(helpers.utf8(attrs['longname']),
self.testcase_counter, helpers.utf8(attrs['status'])),
no_timestamp=True)
self.log("%-12s name: '%s'" % ('end_test', helpers.utf8(name)))
self.log('--------')
def start_test(self, name, attrs):
self.testcase_counter += 1
self.log_devcmd(
"# Test Case start: %s (#%d)" %
(helpers.utf8(attrs['longname']), self.testcase_counter),
no_timestamp=True)
self.log("%-12s name: '%s'" % ('start_test', helpers.utf8(name)))
self.log('--------')
def test_suite_author_mapping(self, build_name):
query = {"build_name": build_name}
cursor = self.test_suites_collection().find(query)
test_suite_author_map = {}
for tc in cursor:
test_suite_author_map[helpers.utf8(
tc["product_suite"])] = helpers.utf8(tc["author"])
return test_suite_author_map
def print_failed_tests(args):
db = TestCatalog()
ts_author_dict = db.test_suite_author_mapping(args.build)
query = {
"build_name": args.build,
"tags": {
"$all": [args.release]
},
"status": "FAIL",
}
tc_archive_collection = db.test_cases_archive_collection().find(
query).sort([("product_suite", pymongo.ASCENDING),
("name", pymongo.ASCENDING)])
total_tc = tc_archive_collection.count()
i = 0
for tc in tc_archive_collection:
i += 1
string = ("TC-%03d: %12s %-55s %s" %
(i, ts_author_dict[tc["product_suite"]], tc["product_suite"],
tc["name"]))
if args.show_tags:
string = ("%s %s" % (string, helpers.utf8(tc["tags"])))
print string
print "\nTotal test cases failed: %s" % total_tc
def print_testcases(testcases):
i = 0
for tc in sorted(testcases):
i += 1
print("\t%03d. %s" % (i, helpers.utf8(tc)))
if i > 0:
print("")
def testsuites(self, **kwargs):
"""
Returns a list containing product_suite, total_tests, and author.
"""
testsuites = self.catalog().test_suites_collection()
collection_suites = testsuites.find({"build_name": self._build_name}, {
"product_suite": 1,
"author": 1,
"total_tests": 1,
"_id": 0
})
suites_lookup = {}
for s in collection_suites:
name = helpers.utf8(s['product_suite'])
suites_lookup[name] = s
testcases = self.testcases(**kwargs)
suite_names = {}
for tc in testcases:
name = helpers.utf8(tc['product_suite'])
if name not in suite_names:
suite_names[name] = 1 # initialization
else:
suite_names[name] += 1
suites = []
for k, v in suite_names.items():
if k in suites_lookup:
author = suites_lookup[k]['author']
else:
author = '???'
suites.append({
"product_suite": k,
"total_tests": v,
"author": author,
})
return suites
def manual_by_tag(self, tag=None, collection="test_cases"):
authors = self.suite_authors()
if tag == None:
tags = helpers.list_flatten([self.release_lowercase(), "manual"])
else:
tags = helpers.list_flatten(
[self.release_lowercase(), "manual", tag])
query = {"tags": {"$all": tags}, "build_name": self._build_name}
testcases = self.catalog().db()[collection]
cases = testcases.find(query)
tests = []
for x in cases:
tests.append("%s %s %s %s" %
(authors[x['product_suite']], x['product_suite'],
x['name'], [helpers.utf8(tag) for tag in x['tags']]))
return tests
testcases = db.test_cases
testsuites = db.test_suites
suites = testsuites.find()
tc = testcases.find({
"build_name": os.environ['BUILD_NAME'],
"tags": {
"$all": ["ironhorse"]
}
})
authors = {}
ironhorse_suites = {}
for x in suites:
product_suite = helpers.utf8(x['product_suite'])
authors[product_suite] = helpers.utf8(x['author'])
for x in tc:
product_suite = helpers.utf8(x['product_suite'])
ironhorse_suites[product_suite] = authors[product_suite]
authors_list = []
for suite, author in ironhorse_suites.items():
authors_list.append("%-12s %s" % (author, suite))
print "\n".join(sorted(authors_list))
print ""
print "Total suites: %s" % len(authors_list)
def end_keyword(self, name, attrs):
self.log("%-12s name: '%s'" % ('end_keyword', helpers.utf8(name)))
self.log('--------')
def start_keyword(self, name, attrs):
# self.log_devcmd("# Keyword: %s" % name, no_timestamp=True)
self.log("%-12s name: '%s'" % ('start_keyword', helpers.utf8(name)))
self.log('--------')
def end_suite(self, name, attrs):
self.log("%-12s name: '%s'" % ('end_suite', helpers.utf8(name)))
def start_suite(self, name, attrs):
self.log_devcmd(
"# Test Suite: %s\n# Source: %s" %
(helpers.utf8(attrs['longname']), helpers.utf8(attrs['source'])),
no_timestamp=True)
self.log("%-12s name: '%s'" % ('start_suite', helpers.utf8(name)))
def display_stats(args):
build = args.build
ih = BuildStats(args.release, build)
total = {}
cat = TestCatalog()
if args.show_all:
print_stat("Total of all test suites:", ih.total_testsuites())
print_stat("Total of all test cases:", ih.total_testcases())
suites_in_release = ih.testsuites(release=ih.release_lowercase())
product_suites = {}
for _suites in suites_in_release:
product_suites[_suites['product_suite']] = _suites
print ""
print "%s Release Metrics" % (ih.release())
print "==================================================================="
total_testsuites_in_release = ih.total_testsuites(
release=ih.release_lowercase())
print_stat("Total test suites:", total_testsuites_in_release)
total_tc = ih.total_testcases(release=ih.release_lowercase())
total_tc_untested = ih.total_testcases_by_tag(["manual-untested"])[0]
total_tc_manual = ih.total_testcases_by_tag(["manual"])[0]
# total_tc_pct = percentage(total_tc - total_tc_untested, total_tc)
print_stat2("Total test cases:", total_tc)
total['tests'] = total_tc
# !!! FIXME: unspecified-topology will need to be removed.
for functionality in cat.test_types() + [
"virtual", "physical", "unspecified-topology", "generic-topology",
"missing-topology", "manual", "manual-untested"
]:
total_tc_func = ih.total_testcases_by_tag(functionality)[0]
# total_tc_func_untested = ih.total_testcases_by_tag(
# [functionality, "manual-untested"])[0]
# total_tc_func_manual = ih.total_testcases_by_tag(
# [functionality, "manual"])[0]
# test_pct = percentage(total_tc_func - total_tc_func_untested,
# total_tc_func)
total[functionality] = total_tc_func
total_virtual = ih.total_testcases_by_tag('virtual')[0]
print_stat3("Total virtual test cases:", total_virtual,
percentage(total_virtual, total_tc))
total_physical = ih.total_testcases_by_tag('physical')[0]
print_stat3("Total physical test cases:", total_physical,
percentage(total_physical, total_tc))
total_unspecified_topology = ih.total_testcases_by_tag(
'unspecified-topology')[0]
print_stat3("Total unspecified-topology test cases:",
total_unspecified_topology,
percentage(total_unspecified_topology, total_tc))
total_generic_topology = ih.total_testcases_by_tag('generic-topology')[0]
print_stat3("Total generic-topology test cases:", total_generic_topology,
percentage(total_generic_topology, total_tc))
total_missing_topology = ih.total_testcases_by_tag('missing-topology')[0]
print_stat3("Total missing-topology test cases:", total_missing_topology,
percentage(total_missing_topology, total_tc))
total_manual = ih.total_testcases_by_tag('manual')[0]
print_stat3("Total manual test cases:", total_manual,
percentage(total_manual, total_tc))
total_untested = ih.total_testcases_by_tag('manual-untested')[0]
print_stat3("Total manual-untested test cases:", total_untested,
percentage(total_untested, total_tc))
total_tc_executable = ih.total_executable_testcases()
print_stat3("Total test cases executable:", total_tc_executable,
percentage(total_tc_executable, total_tc))
total_tc_automated = total_tc - total_tc_manual - total_tc_untested
# total_tc_automated_executable = total_tc_executable - total_tc_manual
print_stat3("Total test cases automated:", total_tc_automated,
percentage(total_tc_automated, total_tc))
print ""
print "Regression Run Results (Build: %s)" % (build)
i = 0
for a_build in cat.aggregated_build(build):
i += 1
print "\tBuild %02d: %s" % (i, a_build)
print "==================================================================="
total_testsuites_in_release_executed = ih.total_testsuites_executed(
build_name=build)[0]
print_stat("Total test suites executed:",
total_testsuites_in_release_executed)
if args.show_suites:
suites_executed = ih.testsuites(
release=ih.release_lowercase(),
build=build,
collection_testcases="test_cases_archive")
print_suites(suites_executed)
print ""
print_stat(
"Total test suites not executed:",
total_testsuites_in_release - total_testsuites_in_release_executed)
if args.show_suites:
print_suites_not_executed(suites_in_release, suites_executed)
print ""
print "Test Summary exec% pass% auto%"
print "-------------------------------------------------------------------------- ----------------------- -------------------- -------------- ----- ----- -----"
total_executed = ih.total_testcases_executed(build_name=build)
total_untested = ih.total_testcases_by_tag(["manual-untested"])[0]
total_manual = ih.total_testcases_by_tag(["manual"])[0]
test_pct = percentage(total_executed[0], total['tests'])
pass_pct = percentage(total_executed[1], total_executed[0])
total_tc_automated = total['tests'] - total_untested - total_manual
automation_pct = percentage(total_tc_automated, total['tests'])
print_stat2(
"Total test cases (total, executed, passed, failed):",
(total['tests'], ) + total_executed,
total_untested,
total_manual,
test_pct,
pass_pct,
automation_pct,
)
if args.show_untested and total_untested != 0:
print "\t----- manual-untested(%d)" % total_untested
print_testcases(ih.manual_untested_by_tag())
if args.show_manual and total_manual != 0:
print "\t----- manual(%d)" % total_manual
print_testcases(ih.manual_by_tag())
for functionality in cat.test_types() + [
'virtual', 'physical', 'unspecified-topology', 'generic-topology',
'missing-topology', "manual", "manual-untested"
]:
total_executed = ih.total_testcases_by_tag_executed(functionality)
total_untested = ih.total_testcases_by_tag(
[functionality, "manual-untested"])[0]
total_manual = ih.total_testcases_by_tag([functionality, "manual"])[0]
test_pct = percentage(total_executed[0], total[functionality])
pass_pct = percentage(total_executed[1], total_executed[0])
if functionality in ['manual', 'manual-untested']:
automation_pct = 0.0
else:
total_tc_automated = total[
functionality] - total_untested - total_manual
automation_pct = percentage(total_tc_automated,
total[functionality])
print_stat2(
"Total %s tests (total, executed, passed, failed):" %
functionality,
(total[functionality], ) + total_executed,
total_untested,
total_manual,
test_pct,
pass_pct,
automation_pct,
)
if args.show_untested and total_untested != 0:
print "\t----- manual-untested(%d)" % total_untested
print_testcases(ih.manual_untested_by_tag(functionality))
if args.show_manual and total_manual != 0:
print "\t----- manual(%d)" % total_manual
print_testcases(ih.manual_by_tag(functionality))
if not args.no_show_functional_areas:
print ""
print "Functional Areas exec% pass% auto%"
print "-------------------------------------------------------------------------- ----------------------- -------------------- -------------- ----- ----- -----"
functionality = "feature"
for feature in cat.features(release=ih.release_lowercase()):
total_tc = ih.total_testcases_by_tag([functionality, feature])[0]
total_executed = ih.total_testcases_by_tag_executed(
[functionality, feature])
total_untested = ih.total_testcases_by_tag(
[functionality, feature, "manual-untested"])[0]
total_manual = ih.total_testcases_by_tag(
[functionality, feature, "manual"])[0]
test_pct = percentage(total_executed[0], total_tc)
pass_pct = percentage(total_executed[1], total_executed[0])
total_tc_automated = total_tc - total_untested - total_manual
automation_pct = percentage(total_tc_automated, total_tc)
print_stat2(
"Total %s+%s tests (total, executed, passed, failed):" %
(functionality, feature),
(total_tc, ) + total_executed,
total_untested,
total_manual,
test_pct,
pass_pct,
automation_pct,
)
if args.show_untested and total_untested != 0:
print "\t----- manual-untested(%d)" % total_untested
print_testcases(
ih.manual_untested_by_tag([functionality, feature]))
if args.show_manual and total_manual != 0:
print "\t----- manual(%d)" % total_manual
print_testcases(ih.manual_by_tag([functionality, feature]))
print ""
print "Manual Verification Details"
print "==================================================================="
# test_case_cursor = cat.find_test_cases_archive_matching_build(build_name=build)
test_case_cursor = ih.testcases_archive(release=ih.release_lowercase(),
build=build)
# print "Total test cases for build '%s': %s" % (build, test_case_cursor.count())
i = 0
pass_rate = {}
details_str = ""
# print "XXXX release: %s, build: %s, tc total: %s" % (ih.release_lowercase(),
# build,
# test_case_cursor.count())
for test_case in test_case_cursor:
# if (('jira' in test_case and test_case['jira']) or
# ('notes' in test_case and test_case['notes'])):
if 'build_name_verified' in test_case:
i += 1
if test_case['product_suite'] in product_suites:
author = helpers.utf8(
product_suites[test_case['product_suite']]['author'])
else:
author = 'unknown'
if author in pass_rate:
pass_rate[author][test_case['status']] += 1
else:
pass_rate[author] = {'PASS': 0, 'FAIL': 0}
pass_rate[author][test_case['status']] += 1
if 'build_name_verified' in test_case and test_case[
'build_name_verified']:
build_name_verified = test_case['build_name_verified']
else:
build_name_verified = None
if 'jira' in test_case and test_case['jira']:
jira = test_case['jira']
else:
jira = None
if 'notes' in test_case and test_case['notes']:
notes = test_case['notes']
else:
notes = None
if jira:
jira += " - https://bigswitch.atlassian.net/browse/%s" % jira
details_str += "%03d. %s %s %s\n" % (
i, author, test_case['product_suite'], test_case['name'])
details_str += "\tverified in : %s\n" % build_name_verified
details_str += "\tstatus : %s\n" % test_case['status']
details_str += "\tjira : %s\n" % jira
details_str += "\tnotes : %s\n" % notes
details_str += ""
if i == 0:
print "None"
else:
total_passes = total_fails = 0
for author, status in pass_rate.items():
print "Author: %14s Passes: %3d Fails: %3d Total: %3d" % (
author, status['PASS'], status['FAIL'],
status['PASS'] + status['FAIL'])
total_passes += status['PASS']
total_fails += status['FAIL']
print " %14s Passes: %3d Fails: %3d Total: %3d" % (
'Combined', total_passes, total_fails, total_passes + total_fails)
print ""
print details_str
}
}, {
"tags": {
"$nin": test_types
}
}]
}, {
"product_suite": 1,
"name": 1,
"tags": 1,
"_id": 0
}).sort("product")
tc_dict = {}
for x in tc:
product_suite = helpers.utf8(x['product_suite'])
if product_suite not in tc_dict:
tc_dict[product_suite] = []
tc_dict[product_suite].append("%s, %s" %
(helpers.utf8(x['name']), x['tags']))
count = 0
for key, value in tc_dict.items():
ts = testsuites.find_one({"product_suite": key})
print "Name: %s, Source file: %s" % (ts["author"], ts["source"])
for x in ["\t%s" % x for x in value]:
print x
count += 1
print ""
