def end_test(self, name, attrs): self.log_devcmd("# Test Case end: %s (#%d) - %s" % (helpers.utf8(attrs['longname']), self.testcase_counter, helpers.utf8(attrs['status'])), no_timestamp=True) self.log("%-12s name: '%s'" % ('end_test', helpers.utf8(name))) self.log('--------')
def start_test(self, name, attrs): self.testcase_counter += 1 self.log_devcmd( "# Test Case start: %s (#%d)" % (helpers.utf8(attrs['longname']), self.testcase_counter), no_timestamp=True) self.log("%-12s name: '%s'" % ('start_test', helpers.utf8(name))) self.log('--------')
def test_suite_author_mapping(self, build_name): query = {"build_name": build_name} cursor = self.test_suites_collection().find(query) test_suite_author_map = {} for tc in cursor: test_suite_author_map[helpers.utf8( tc["product_suite"])] = helpers.utf8(tc["author"]) return test_suite_author_map
def print_failed_tests(args): db = TestCatalog() ts_author_dict = db.test_suite_author_mapping(args.build) query = { "build_name": args.build, "tags": { "$all": [args.release] }, "status": "FAIL", } tc_archive_collection = db.test_cases_archive_collection().find( query).sort([("product_suite", pymongo.ASCENDING), ("name", pymongo.ASCENDING)]) total_tc = tc_archive_collection.count() i = 0 for tc in tc_archive_collection: i += 1 string = ("TC-%03d: %12s %-55s %s" % (i, ts_author_dict[tc["product_suite"]], tc["product_suite"], tc["name"])) if args.show_tags: string = ("%s %s" % (string, helpers.utf8(tc["tags"]))) print string print "\nTotal test cases failed: %s" % total_tc
def print_testcases(testcases): i = 0 for tc in sorted(testcases): i += 1 print("\t%03d. %s" % (i, helpers.utf8(tc))) if i > 0: print("")
def testsuites(self, **kwargs): """ Returns a list containing product_suite, total_tests, and author. """ testsuites = self.catalog().test_suites_collection() collection_suites = testsuites.find({"build_name": self._build_name}, { "product_suite": 1, "author": 1, "total_tests": 1, "_id": 0 }) suites_lookup = {} for s in collection_suites: name = helpers.utf8(s['product_suite']) suites_lookup[name] = s testcases = self.testcases(**kwargs) suite_names = {} for tc in testcases: name = helpers.utf8(tc['product_suite']) if name not in suite_names: suite_names[name] = 1 # initialization else: suite_names[name] += 1 suites = [] for k, v in suite_names.items(): if k in suites_lookup: author = suites_lookup[k]['author'] else: author = '???' suites.append({ "product_suite": k, "total_tests": v, "author": author, }) return suites
def manual_by_tag(self, tag=None, collection="test_cases"): authors = self.suite_authors() if tag == None: tags = helpers.list_flatten([self.release_lowercase(), "manual"]) else: tags = helpers.list_flatten( [self.release_lowercase(), "manual", tag]) query = {"tags": {"$all": tags}, "build_name": self._build_name} testcases = self.catalog().db()[collection] cases = testcases.find(query) tests = [] for x in cases: tests.append("%s %s %s %s" % (authors[x['product_suite']], x['product_suite'], x['name'], [helpers.utf8(tag) for tag in x['tags']])) return tests
testcases = db.test_cases testsuites = db.test_suites suites = testsuites.find() tc = testcases.find({ "build_name": os.environ['BUILD_NAME'], "tags": { "$all": ["ironhorse"] } }) authors = {} ironhorse_suites = {} for x in suites: product_suite = helpers.utf8(x['product_suite']) authors[product_suite] = helpers.utf8(x['author']) for x in tc: product_suite = helpers.utf8(x['product_suite']) ironhorse_suites[product_suite] = authors[product_suite] authors_list = [] for suite, author in ironhorse_suites.items(): authors_list.append("%-12s %s" % (author, suite)) print "\n".join(sorted(authors_list)) print "" print "Total suites: %s" % len(authors_list)
def end_keyword(self, name, attrs): self.log("%-12s name: '%s'" % ('end_keyword', helpers.utf8(name))) self.log('--------')
def start_keyword(self, name, attrs): # self.log_devcmd("# Keyword: %s" % name, no_timestamp=True) self.log("%-12s name: '%s'" % ('start_keyword', helpers.utf8(name))) self.log('--------')
def end_suite(self, name, attrs): self.log("%-12s name: '%s'" % ('end_suite', helpers.utf8(name)))
def start_suite(self, name, attrs): self.log_devcmd( "# Test Suite: %s\n# Source: %s" % (helpers.utf8(attrs['longname']), helpers.utf8(attrs['source'])), no_timestamp=True) self.log("%-12s name: '%s'" % ('start_suite', helpers.utf8(name)))
def display_stats(args): build = args.build ih = BuildStats(args.release, build) total = {} cat = TestCatalog() if args.show_all: print_stat("Total of all test suites:", ih.total_testsuites()) print_stat("Total of all test cases:", ih.total_testcases()) suites_in_release = ih.testsuites(release=ih.release_lowercase()) product_suites = {} for _suites in suites_in_release: product_suites[_suites['product_suite']] = _suites print "" print "%s Release Metrics" % (ih.release()) print "===================================================================" total_testsuites_in_release = ih.total_testsuites( release=ih.release_lowercase()) print_stat("Total test suites:", total_testsuites_in_release) total_tc = ih.total_testcases(release=ih.release_lowercase()) total_tc_untested = ih.total_testcases_by_tag(["manual-untested"])[0] total_tc_manual = ih.total_testcases_by_tag(["manual"])[0] # total_tc_pct = percentage(total_tc - total_tc_untested, total_tc) print_stat2("Total test cases:", total_tc) total['tests'] = total_tc # !!! FIXME: unspecified-topology will need to be removed. for functionality in cat.test_types() + [ "virtual", "physical", "unspecified-topology", "generic-topology", "missing-topology", "manual", "manual-untested" ]: total_tc_func = ih.total_testcases_by_tag(functionality)[0] # total_tc_func_untested = ih.total_testcases_by_tag( # [functionality, "manual-untested"])[0] # total_tc_func_manual = ih.total_testcases_by_tag( # [functionality, "manual"])[0] # test_pct = percentage(total_tc_func - total_tc_func_untested, # total_tc_func) total[functionality] = total_tc_func total_virtual = ih.total_testcases_by_tag('virtual')[0] print_stat3("Total virtual test cases:", total_virtual, percentage(total_virtual, total_tc)) total_physical = ih.total_testcases_by_tag('physical')[0] print_stat3("Total physical test cases:", total_physical, percentage(total_physical, total_tc)) total_unspecified_topology = ih.total_testcases_by_tag( 'unspecified-topology')[0] print_stat3("Total unspecified-topology test cases:", total_unspecified_topology, percentage(total_unspecified_topology, total_tc)) total_generic_topology = ih.total_testcases_by_tag('generic-topology')[0] print_stat3("Total generic-topology test cases:", total_generic_topology, percentage(total_generic_topology, total_tc)) total_missing_topology = ih.total_testcases_by_tag('missing-topology')[0] print_stat3("Total missing-topology test cases:", total_missing_topology, percentage(total_missing_topology, total_tc)) total_manual = ih.total_testcases_by_tag('manual')[0] print_stat3("Total manual test cases:", total_manual, percentage(total_manual, total_tc)) total_untested = ih.total_testcases_by_tag('manual-untested')[0] print_stat3("Total manual-untested test cases:", total_untested, percentage(total_untested, total_tc)) total_tc_executable = ih.total_executable_testcases() print_stat3("Total test cases executable:", total_tc_executable, percentage(total_tc_executable, total_tc)) total_tc_automated = total_tc - total_tc_manual - total_tc_untested # total_tc_automated_executable = total_tc_executable - total_tc_manual print_stat3("Total test cases automated:", total_tc_automated, percentage(total_tc_automated, total_tc)) print "" print "Regression Run Results (Build: %s)" % (build) i = 0 for a_build in cat.aggregated_build(build): i += 1 print "\tBuild %02d: %s" % (i, a_build) print "===================================================================" total_testsuites_in_release_executed = ih.total_testsuites_executed( build_name=build)[0] print_stat("Total test suites executed:", total_testsuites_in_release_executed) if args.show_suites: suites_executed = ih.testsuites( release=ih.release_lowercase(), build=build, collection_testcases="test_cases_archive") print_suites(suites_executed) print "" print_stat( "Total test suites not executed:", total_testsuites_in_release - total_testsuites_in_release_executed) if args.show_suites: print_suites_not_executed(suites_in_release, suites_executed) print "" print "Test Summary exec% pass% auto%" print "-------------------------------------------------------------------------- ----------------------- -------------------- -------------- ----- ----- -----" total_executed = ih.total_testcases_executed(build_name=build) total_untested = ih.total_testcases_by_tag(["manual-untested"])[0] total_manual = ih.total_testcases_by_tag(["manual"])[0] test_pct = percentage(total_executed[0], total['tests']) pass_pct = percentage(total_executed[1], total_executed[0]) total_tc_automated = total['tests'] - total_untested - total_manual automation_pct = percentage(total_tc_automated, total['tests']) print_stat2( "Total test cases (total, executed, passed, failed):", (total['tests'], ) + total_executed, total_untested, total_manual, test_pct, pass_pct, automation_pct, ) if args.show_untested and total_untested != 0: print "\t----- manual-untested(%d)" % total_untested print_testcases(ih.manual_untested_by_tag()) if args.show_manual and total_manual != 0: print "\t----- manual(%d)" % total_manual print_testcases(ih.manual_by_tag()) for functionality in cat.test_types() + [ 'virtual', 'physical', 'unspecified-topology', 'generic-topology', 'missing-topology', "manual", "manual-untested" ]: total_executed = ih.total_testcases_by_tag_executed(functionality) total_untested = ih.total_testcases_by_tag( [functionality, "manual-untested"])[0] total_manual = ih.total_testcases_by_tag([functionality, "manual"])[0] test_pct = percentage(total_executed[0], total[functionality]) pass_pct = percentage(total_executed[1], total_executed[0]) if functionality in ['manual', 'manual-untested']: automation_pct = 0.0 else: total_tc_automated = total[ functionality] - total_untested - total_manual automation_pct = percentage(total_tc_automated, total[functionality]) print_stat2( "Total %s tests (total, executed, passed, failed):" % functionality, (total[functionality], ) + total_executed, total_untested, total_manual, test_pct, pass_pct, automation_pct, ) if args.show_untested and total_untested != 0: print "\t----- manual-untested(%d)" % total_untested print_testcases(ih.manual_untested_by_tag(functionality)) if args.show_manual and total_manual != 0: print "\t----- manual(%d)" % total_manual print_testcases(ih.manual_by_tag(functionality)) if not args.no_show_functional_areas: print "" print "Functional Areas exec% pass% auto%" print "-------------------------------------------------------------------------- ----------------------- -------------------- -------------- ----- ----- -----" functionality = "feature" for feature in cat.features(release=ih.release_lowercase()): total_tc = ih.total_testcases_by_tag([functionality, feature])[0] total_executed = ih.total_testcases_by_tag_executed( [functionality, feature]) total_untested = ih.total_testcases_by_tag( [functionality, feature, "manual-untested"])[0] total_manual = ih.total_testcases_by_tag( [functionality, feature, "manual"])[0] test_pct = percentage(total_executed[0], total_tc) pass_pct = percentage(total_executed[1], total_executed[0]) total_tc_automated = total_tc - total_untested - total_manual automation_pct = percentage(total_tc_automated, total_tc) print_stat2( "Total %s+%s tests (total, executed, passed, failed):" % (functionality, feature), (total_tc, ) + total_executed, total_untested, total_manual, test_pct, pass_pct, automation_pct, ) if args.show_untested and total_untested != 0: print "\t----- manual-untested(%d)" % total_untested print_testcases( ih.manual_untested_by_tag([functionality, feature])) if args.show_manual and total_manual != 0: print "\t----- manual(%d)" % total_manual print_testcases(ih.manual_by_tag([functionality, feature])) print "" print "Manual Verification Details" print "===================================================================" # test_case_cursor = cat.find_test_cases_archive_matching_build(build_name=build) test_case_cursor = ih.testcases_archive(release=ih.release_lowercase(), build=build) # print "Total test cases for build '%s': %s" % (build, test_case_cursor.count()) i = 0 pass_rate = {} details_str = "" # print "XXXX release: %s, build: %s, tc total: %s" % (ih.release_lowercase(), # build, # test_case_cursor.count()) for test_case in test_case_cursor: # if (('jira' in test_case and test_case['jira']) or # ('notes' in test_case and test_case['notes'])): if 'build_name_verified' in test_case: i += 1 if test_case['product_suite'] in product_suites: author = helpers.utf8( product_suites[test_case['product_suite']]['author']) else: author = 'unknown' if author in pass_rate: pass_rate[author][test_case['status']] += 1 else: pass_rate[author] = {'PASS': 0, 'FAIL': 0} pass_rate[author][test_case['status']] += 1 if 'build_name_verified' in test_case and test_case[ 'build_name_verified']: build_name_verified = test_case['build_name_verified'] else: build_name_verified = None if 'jira' in test_case and test_case['jira']: jira = test_case['jira'] else: jira = None if 'notes' in test_case and test_case['notes']: notes = test_case['notes'] else: notes = None if jira: jira += " - https://bigswitch.atlassian.net/browse/%s" % jira details_str += "%03d. %s %s %s\n" % ( i, author, test_case['product_suite'], test_case['name']) details_str += "\tverified in : %s\n" % build_name_verified details_str += "\tstatus : %s\n" % test_case['status'] details_str += "\tjira : %s\n" % jira details_str += "\tnotes : %s\n" % notes details_str += "" if i == 0: print "None" else: total_passes = total_fails = 0 for author, status in pass_rate.items(): print "Author: %14s Passes: %3d Fails: %3d Total: %3d" % ( author, status['PASS'], status['FAIL'], status['PASS'] + status['FAIL']) total_passes += status['PASS'] total_fails += status['FAIL'] print " %14s Passes: %3d Fails: %3d Total: %3d" % ( 'Combined', total_passes, total_fails, total_passes + total_fails) print "" print details_str
} }, { "tags": { "$nin": test_types } }] }, { "product_suite": 1, "name": 1, "tags": 1, "_id": 0 }).sort("product") tc_dict = {} for x in tc: product_suite = helpers.utf8(x['product_suite']) if product_suite not in tc_dict: tc_dict[product_suite] = [] tc_dict[product_suite].append("%s, %s" % (helpers.utf8(x['name']), x['tags'])) count = 0 for key, value in tc_dict.items(): ts = testsuites.find_one({"product_suite": key}) print "Name: %s, Source file: %s" % (ts["author"], ts["source"]) for x in ["\t%s" % x for x in value]: print x count += 1 print ""