def prog_args(): descr = """\ Update the regression tags for a build. The currently supported tags are 'daily' and 'full'. Example: % BUILD_NAME=ihplus_bcf_10G-490 ./update_regression_tags.py --tags full """ parser = argparse.ArgumentParser(prog='update_regression_tags', formatter_class=argparse.RawDescriptionHelpFormatter, description=descr) parser.add_argument('--verbose', action='store_true', default=False, help=("Print verbose output")) parser.add_argument('--build', help=("Jenkins build string," " e.g., 'bvs master #2007'")) parser.add_argument('--tags', metavar=('tag1', 'tag2'), nargs='*', help=("Regression tags," " e.g., 'full', 'daily'")) _args = parser.parse_args() # _args.build <=> env BUILD_NAME if not _args.build and 'BUILD_NAME' in os.environ: _args.build = os.environ['BUILD_NAME'] elif not _args.build: helpers.error_exit("Must specify --build option or set environment" " variable BUILD_NAME") else: os.environ['BUILD_NAME'] = _args.build return _args
def prog_args(): descr = """\ Check the database collection 'builds' for a document which matches the env BUILD_NAME. - If found, return the build name. - If not found, create a new document, then return the build name. """ parser = argparse.ArgumentParser( prog='db_chk_and_add_build_name', formatter_class=argparse.RawDescriptionHelpFormatter, description=descr) parser.add_argument('--verbose', action='store_true', default=False, help=("Print verbose output")) parser.add_argument('--build', help=("Jenkins build string," " e.g., 'bvs master #2007'")) parser.add_argument( '--regression-tags', help=("Supported regression tags are 'daily' or 'full'.")) _args = parser.parse_args() # _args.build <=> env BUILD_NAME if not _args.build and 'BUILD_NAME' in os.environ: _args.build = os.environ['BUILD_NAME'] elif not _args.build: helpers.error_exit("Must specify --build option or set environment" " variable BUILD_NAME") else: os.environ['BUILD_NAME'] = _args.build return _args
def __init__(self, build): self._aggregated_build_name = build self._cat = None self._builds = self.catalog().aggregated_build(build) if not self._builds: helpers.error_exit( "Aggregated build '%s' is not defined in catalog.yaml." % build)
def load_configs(self): try: config_file = helpers.bigrobot_configs_path() + "/testrail.yaml" config = helpers.load_config(config_file) except IOError: s = "Unable to open TestRail config file %s" % config_file helpers.error_exit(s) return config
def prog_args(): descr = """\ Display test execution stats collected for a specific build. """ parser = argparse.ArgumentParser( prog='db_collect_stats', formatter_class=argparse.RawDescriptionHelpFormatter, description=descr) parser.add_argument( '--release', help= ("Product release, e.g., 'ironhorse', 'ironhorse-plus', 'jackfrost', etc." )) parser.add_argument('--build', help=("Jenkins build string," " e.g., 'bvs master #2007'")) parser.add_argument('--show-untested', action='store_true', default=False, help=("Show the manual-untested test cases")) parser.add_argument('--show-manual', action='store_true', default=False, help=("Show the manual test cases")) parser.add_argument('--show-suites', action='store_true', default=False, help=("Show the test suites")) parser.add_argument('--show-all', action='store_true', default=False, help=("Show all stats")) parser.add_argument('--no-show-functional-areas', action='store_true', default=False, help=("Don't show test details for functional areas")) _args = parser.parse_args() # _args.build <=> env BUILD_NAME if not _args.build and 'BUILD_NAME' in os.environ: _args.build = os.environ['BUILD_NAME'] elif not _args.build: helpers.error_exit("Must specify --build option or set environment" " variable BUILD_NAME") else: os.environ['BUILD_NAME'] = _args.build # _args.release <=> env RELEASE_NAME if not _args.release and 'RELEASE_NAME' in os.environ: _args.release = os.environ['RELEASE_NAME'] elif not _args.release: helpers.error_exit("Must specify --release option or set environment" " variable RELEASE_NAME") else: os.environ['RELEASE_NAME'] = _args.release _args.release = _args.release.lower() return _args
def prog_args(): descr = """ Display test data for a specific build. Example: % RELEASE_NAME="<release>" BUILD_NAME="bvs master #3271" \\ ./db_get_test_data.py --tags manual-untested \\ --no-show-header """ parser = argparse.ArgumentParser( prog='db_get_test_data', formatter_class=argparse.RawDescriptionHelpFormatter, description=descr) parser.add_argument('--build', help=("Jenkins build string," " e.g., 'bvs master #2007'")) parser.add_argument( '--release', help= ("Product release, e.g., 'ironhorse', 'ironhorse-plus', 'jackfrost', etc." )) parser.add_argument( '--tags', metavar=('tag1', 'tag2'), nargs='*', help= 'Test case tags to include, e.g., manual-untested, scaling, ztn, etc.') parser.add_argument( '--no-show-header', action='store_true', default=False, help=("Don't show header info (build name, release, total)")) _args = parser.parse_args() # _args.build <=> env BUILD_NAME if not _args.build and 'BUILD_NAME' in os.environ: _args.build = os.environ['BUILD_NAME'] elif not _args.build: helpers.error_exit("Must specify --build option or set environment" " variable BUILD_NAME") else: os.environ['BUILD_NAME'] = _args.build # _args.release <=> env RELEASE_NAME if not _args.release and 'RELEASE_NAME' in os.environ: _args.release = os.environ['RELEASE_NAME'] elif not _args.release: helpers.error_exit("Must specify --release option or set environment" " variable RELEASE_NAME") else: os.environ['RELEASE_NAME'] = _args.release _args.release = _args.release.lower() return _args
def prog_args(): descr = """\ Given the release (RELEASE_NAME env) and build (BUILD_NAME env), print a list of failed test cases. Examples: % RELEASE_NAME="ironhorse" BUILD_NAME="bvs master bcf-2.0.0 fcs" ./dump_test_cases_failed.py % ./dump_test_cases_failed.py --release ironhorse --build "bvs master aggregated 2014 wk40" """ parser = argparse.ArgumentParser( prog='dump_test_cases_failed', formatter_class=argparse.RawDescriptionHelpFormatter, description=descr) parser.add_argument('--build', help=("Jenkins build string," " e.g., 'bvs master #2007'")) parser.add_argument( '--release', help= ("Product release, e.g., 'ironhorse', 'ironhorse-plus', 'jackfrost', etc." )) parser.add_argument('--show-tags', action='store_true', default=False, help=("Show test case tags")) _args = parser.parse_args() # _args.build <=> env BUILD_NAME if not _args.build and 'BUILD_NAME' in os.environ: _args.build = os.environ['BUILD_NAME'] elif not _args.build: helpers.error_exit("Must specify --build option or set environment" " variable BUILD_NAME") else: os.environ['BUILD_NAME'] = _args.build # _args.release <=> env RELEASE_NAME if not _args.release and 'RELEASE_NAME' in os.environ: _args.release = os.environ['RELEASE_NAME'] elif not _args.release: helpers.error_exit("Must specify --release option or set environment" " variable RELEASE_NAME") else: os.environ['RELEASE_NAME'] = _args.release _args.release = _args.release.lower() return _args
def __init__(self, build, user, overwrite): self._aggregated_build_name = build self._user = user self._is_overwrite = overwrite self._cat = None self._builds = self.catalog().aggregated_build(build) self._test_case_dict = {} if not self._builds: helpers.error_exit( "Aggregated build '%s' is not defined in the database." % build) # dictionary of test suites indexed by product_suite key. self._product_suites = {} self.load_product_suites_dict()
def prog_args(): descr = """\ Populate the QA Dashboard database with data found in the user-entered verification files (in bigrobot/data/verification/). The specified build (BUILD_NAME) must be the name of an aggregated build. Examples: -- Sanity check the data. It will not update the database. % BUILD_NAME="bvs master bcf-2.0.0 fcs" ./populate_db_user_verification.py \\ --infile ../data/verification/bvs_master_bcf-2.0.0_fcs/verification_kranti.yaml \\ --sanitize-data -- If everything looks good. Update the database. % BUILD_NAME="bvs master bcf-2.0.0 fcs" ./populate_db_user_verification.py \\ --infile ../data/verification/bvs_master_bcf-2.0.0_fcs/verification_kranti.yaml """ parser = argparse.ArgumentParser( prog='db_manual_verification', formatter_class=argparse.RawDescriptionHelpFormatter, description=descr) parser.add_argument( '--build', help=("Build name," " e.g., 'bvs master ironhorse beta2 aggregated'")) parser.add_argument('--infile', required=True, help=("Input verification file")) parser.add_argument('--sanitize-data', action='store_true', default=False, help=("Check data file for formatting errors")) _args = parser.parse_args() # _args.build <=> env BUILD_NAME if not _args.build and 'BUILD_NAME' in os.environ: _args.build = os.environ['BUILD_NAME'] elif not _args.build: helpers.error_exit("Must specify --build option or set environment" " variable BUILD_NAME") else: os.environ['BUILD_NAME'] = _args.build return _args
def prog_args(): descr = """\ Check the database collection 'aggregated_builds' for a document which matches the env BUILD_NAME. - If found, return the aggregated build name. - If not found and build_name matches a typical build name (e.g., 'bvs master #1234'), create a new document, then return the weekly aggregated build name. - If not found and build_name does not match a typical build name (e.g., a Beta build does not have a typical build name - 'bvs master #bcf-2.0.0_13'), don't create a new document. You should instead rerun command and specify the aggregated beta build name using the option --aggregated-build-name. Examples: % BUILD_NAME="bvs master #3436" ./db_chk_and_add_wk_aggregated_build.py % BUILD_NAME="bvs master #bcf-2.0.0_13" ./db_chk_and_add_wk_aggregated_build.py \\ --aggregated-build-name "bvs master bcf-2.0.0 aggregated" """ parser = argparse.ArgumentParser( prog='db_chk_and_add_wk_aggregated_build', formatter_class=argparse.RawDescriptionHelpFormatter, description=descr) parser.add_argument('--verbose', action='store_true', default=False, help=("Print verbose output")) parser.add_argument('--build', help=("Jenkins build string," " e.g., 'bvs master #2007'")) parser.add_argument('--aggregated-build-name', help=("Aggregated build name," " e.g., 'bvs master bcf-2.0.0 aggregated'")) _args = parser.parse_args() # _args.build <=> env BUILD_NAME if not _args.build and 'BUILD_NAME' in os.environ: _args.build = os.environ['BUILD_NAME'] elif not _args.build: helpers.error_exit("Must specify --build option or set environment" " variable BUILD_NAME") else: os.environ['BUILD_NAME'] = _args.build return _args
def prog_args(): descr = """\ Search for all FAILed test cases in a (aggregated) build and generate verification records for each failed test case. The specified build (BUILD_NAME) must be the name of an aggregated build. Examples: -- Generate the verification file for 'mingtao' % BUILD_NAME="bvs master bcf-2.0.0 fcs" ./create_or_update_verification_file.py --user mingtao -- Generate the verification files for all QA engineers % BUILD_NAME="bvs master bcf-2.0.0 fcs" ./create_or_update_verification_file.py --user all """ parser = argparse.ArgumentParser( prog='create_or_update_verification_file', formatter_class=argparse.RawDescriptionHelpFormatter, description=descr) parser.add_argument( '--build', help=("Build name," " e.g., 'bvs master ironhorse beta2 aggregated'")) parser.add_argument( '--user', required=True, help=("Create verification file for user (as defined in" " test catalog).")) parser.add_argument('--force-overwrite', action='store_true', default=False, help=("Overwrite the existing verification files")) _args = parser.parse_args() # _args.build <=> env BUILD_NAME if not _args.build and 'BUILD_NAME' in os.environ: _args.build = os.environ['BUILD_NAME'] elif not _args.build: helpers.error_exit("Must specify --build option or set environment" " variable BUILD_NAME") else: os.environ['BUILD_NAME'] = _args.build return _args
def prog_args(): descr = """\ Parse the Robot output.xml files and generate/populate the collections for the Test Catalog (MongoDB) database. """ parser = argparse.ArgumentParser(prog='parse_test_xml_output', formatter_class=argparse.RawDescriptionHelpFormatter, description=descr) parser.add_argument('--build', help=("Jenkins build string," " e.g., 'bvs master #2007'")) parser.add_argument('--input', required=True, help=("Input file which contains a list of Robot" " output.xml files with complete pathnames")) parser.add_argument('--output-suites', required=True, help=("JSON output file containing test suites")) parser.add_argument('--output-testcases', required=True, help=("JSON output file containing test cases")) parser.add_argument('--is-baseline', action='store_true', default=False, help=("Specify this option if generating baseline" " data")) parser.add_argument('--is-regression', action='store_true', default=False, help=("Specify this option if analyzing regression" " results")) _args = parser.parse_args() # _args.build <=> env BUILD_NAME if not _args.build and 'BUILD_NAME' in os.environ: _args.build = os.environ['BUILD_NAME'] elif not _args.build: helpers.error_exit("Must specify --build option or set environment" " variable BUILD_NAME") else: os.environ['BUILD_NAME'] = _args.build if not _args.is_baseline and not _args.is_regression: helpers.error_exit("Must specify --is-regression or --is-baseline") return _args
def prog_args(): descr = """\ Generate the BUILD_NAME string based on a BCF/BigTap version string found in a file (see --version-file) or a string (see --version-str). The version string needs to have the following format: Big Tap Controller 4.1.1 (2014.11.13.1922-b.bsc.corsair-4.1.1beta Big Cloud Fabric Appliance 3.0.0-master01-SNAPSHOT (bcf_master #4201) Big Cloud Fabric Appliance 2.0.0-master01-SNAPSHOT (ihplus_bcf #515) Returns the BUILD_NAME as followed: corsair411_bigtap_10G-1922 master_bcf_10G-4201 """ parser = argparse.ArgumentParser( prog='build_name_gen', formatter_class=argparse.RawDescriptionHelpFormatter, description=descr) parser.add_argument('--version-file', default='/var/tmp/ver.txt', help=("Version file")) parser.add_argument('--version-str', help=("Version string")) parser.add_argument( '--additional-version-descr', help=("An additional version description to add to BUILD_NAME")) parser.add_argument( '--testbed', required=True, help= ("Testbed name, e.g., common, Dell, Accton, Quanta, 10G, 40G, virtual, etc." )) _args = parser.parse_args() if _args.testbed == None: helpers.error_exit("Env BIGROBOT_PARAMS_INPUT is not found.") return _args
def prog_args(): descr = """\ Check the database collection 'build_groups' for a document which matches the BUILD_NAME group. - If found, return the build name group. - If not found, create a new document, then return the build name group. """ parser = argparse.ArgumentParser( prog='db_chk_and_add_build_name_group', formatter_class=argparse.RawDescriptionHelpFormatter, description=descr) parser.add_argument('--verbose', action='store_true', default=False, help=("Print verbose output")) parser.add_argument('--build', help=("Jenkins build string," " e.g., 'bvs master #2007'")) parser.add_argument('--createtime', help=("Build creation time," " e.g., '2015-03-06T12:54:58.130'")) parser.add_argument('--updatetime', help=("Build updated time," " e.g., '2015-03-06T12:54:58.130'")) _args = parser.parse_args() # _args.build <=> env BUILD_NAME if not _args.build and 'BUILD_NAME' in os.environ: _args.build = os.environ['BUILD_NAME'] elif not _args.build: helpers.error_exit("Must specify --build option or set environment" " variable BUILD_NAME") else: os.environ['BUILD_NAME'] = _args.build return _args
def get_version_string(args): version_str = args.version_str if version_str == None: if helpers.file_not_exists(args.version_file): helpers.error_exit("Version file '%s' is not found." % args.version_file) version_str = helpers.file_read_once(args.version_file).strip() version = None release = None build_name = None testbed = args.testbed ########################################################################## # Match BigTap build string # - The match logic is as defined in bigrobot/configs/catalog.yaml ########################################################################## re_bigtap = r'.*Big Tap Controller (\d+\.\d+\.\d+).+\d\d\d\d\.\d\d\.\d\d\.(\d+).+' match = re.match(re_bigtap, version_str) if match: version = match.group(1) build_id = match.group(2) if version in ['4.0.0', '4.0.1', '4.0.2', '4.1.0']: release = 'corsair400' elif version in ['4.1.1']: release = 'corsair411' elif version in ['4.5.0']: release = 'corsair450' else: helpers.error_exit( "Invalid BigTap version '%s' in version string '%s'." % (version, version_str)) build_name = "%s_%s_%s-%s" % (release, "bigtap", testbed, build_id) if args.additional_version_descr != None: build_name = "%s_%s" % (build_name, args.additional_version_descr) return build_name ########################################################################## # Match BCF build string # - The match logic is as defined in bigrobot/configs/catalog.yaml ########################################################################## re_bcf = r'.*Big Cloud Fabric Appliance \d+\.\d+\.\d+.+\((\w+)_(\w+) #(\d+)\).*' match = re.match(re_bcf, version_str) if match: if match.group(1) == 'bcf': release = match.group(2) else: release = match.group(1) build_id = match.group(3) if not release in ['ih', 'ihplus', 'jf', 'master']: helpers.error_exit( "Invalid BCF version '%s' in version string '%s'." % (version, version_str)) build_name = "%s_%s_%s-%s" % (release, "bcf", testbed, build_id) if args.additional_version_descr != None: build_name = "%s_%s" % (build_name, args.additional_version_descr) return build_name helpers.error_exit("Invalid version string: '%s'." % version_str)
def do_it(self): # Search for failed test cases. Verification list will be built/updated # based on the failed test cases. query = { "build_name": self.aggregated_build_name(), "status": 'FAIL', # "tags": { "$all": ['ironhorse']}, } aggr_cursor = self.catalog().find_test_cases_archive(query) fail_count = aggr_cursor.count() print("Total failed test cases in build '%s': %s" % (self.aggregated_build_name(), fail_count)) if self._user == 'all': pass elif self._user not in Authors().get().values(): helpers.error_exit( "User '%s' is not defined in the test catalog." % self._user) # Initialize test case dictionary for each author if not exist for author in Authors().get().values(): file_name = self.verification_file(author) if helpers.file_exists(file_name): # print "Loading file %s" % file_name tc_list = PseudoYAML(file_name).load_yaml_file() self._test_case_dict[author] = {} if tc_list: # Test cases found for tc in tc_list: tc_key = tc['product_suite'] + ' ' + tc['name'] self._test_case_dict[author][tc_key] = tc else: self._test_case_dict[author] = {} first_pass = {} for tc in aggr_cursor: product_suite = tc['product_suite'] tc_name = tc['name'] status = tc['status'] # build_name = build_name_last = tc['build_name'] if 'build_name_list' in tc and tc['build_name_list']: build_name_last = tc['build_name_list'][-1] author = self.author(product_suite) file_name = self.verification_file(author) new_file_name = file_name + ".new" if self._user == 'all' or self._user == author: pass # do action else: continue if new_file_name not in first_pass: if helpers.file_exists(new_file_name): print "Removing file %s" % new_file_name helpers.file_remove(new_file_name) helpers.file_copy(self.verification_header_template(), new_file_name) print "Creating file %s" % new_file_name first_pass[new_file_name] = file_name key = product_suite + ' ' + tc_name if key in self._test_case_dict[author]: if status == self._test_case_dict[author][key]['status']: self._test_case_dict[author][key]['name'] = \ sanitize_yaml_string( self._test_case_dict[author][key]['name']) self.write_entry_to_file(self._test_case_dict[author][key], new_file_name) else: helpers.file_write_append_once( new_file_name, "\n### status: %s in recent '%s'" % (status, build_name_last)) self._test_case_dict[author][key]['name'] = \ sanitize_yaml_string( self._test_case_dict[author][key]['name']) self.write_entry_to_file(self._test_case_dict[author][key], new_file_name) del self._test_case_dict[author][key] else: helpers.file_write_append_once( new_file_name, "\n### new entry in recent '%s'" % (build_name_last)) rec = { "name": sanitize_yaml_string(tc_name), "product_suite": product_suite, "status": status, } if 'jira' in tc and tc['jira']: rec['jira'] = tc['jira'] if 'jira' in tc and tc['jira']: rec['jira'] = tc['jira'] if 'notes' in tc and tc['notes']: rec['notes'] = tc['notes'] self.write_entry_to_file(rec, new_file_name) # Dump remaining test cases from YAML for author in self._test_case_dict: if self._user == 'all' or self._user == author: pass # do action else: continue file_name = self.verification_file(author) new_file_name = file_name + ".new" for key in self._test_case_dict[author]: if not helpers.file_exists(new_file_name): helpers.file_copy(self.verification_header_template(), new_file_name) tc = self._test_case_dict[author][key] if tc['status'] == 'FAIL': if tc['jira'] or (tc['notes'] and tc['notes'] != "No description."): # Contains user comments helpers.file_write_append_once( new_file_name, "\n### Previously verified. Not reported as" " FAIL in recent report so is likely PASSing.") tc['name'] = sanitize_yaml_string(tc['name']) self.write_entry_to_file(tc, new_file_name) else: # User didn't get a chance to verify it previously. # So just silently ignore it. pass else: # Contains user comments tc['name'] = sanitize_yaml_string(tc['name']) self.write_entry_to_file(tc, new_file_name)
parser.add_argument('--sanitize-data', action='store_true', default=False, help=("Check data file for formatting errors")) _args = parser.parse_args() # _args.build <=> env BUILD_NAME if not _args.build and 'BUILD_NAME' in os.environ: _args.build = os.environ['BUILD_NAME'] elif not _args.build: helpers.error_exit("Must specify --build option or set environment" " variable BUILD_NAME") else: os.environ['BUILD_NAME'] = _args.build return _args if __name__ == '__main__': args = prog_args() if not helpers.file_exists(args.infile): helpers.error_exit("File '%s' is not found." % args.infile, 1) verification_build = ManualVerificationBuild(args.build, args.infile) if args.sanitize_data: if verification_build.sanitize_verification_data(): print "Data format is correct." else: print "Error found in data format." else: verification_build.populate_db_with_verification_data()
# Determine BigRobot path(s) based on this executable (which resides in # the bin/ directory. bigrobot_path = os.path.dirname(__file__) + '/..' exscript_path = bigrobot_path + '/vendors/exscript/src' sys.path.insert(0, bigrobot_path) sys.path.insert(1, exscript_path) import autobot.helpers as helpers helpers.set_env('IS_GOBOT', 'False') helpers.set_env('AUTOBOT_LOG', './myrobot.log') if not 'BUILD_NAME' in os.environ: helpers.error_exit("Environment variable BUILD_NAME is not defined.", 1) configs = helpers.bigrobot_config_test_catalog() db_server = configs['db_server'] db_port = configs['db_port'] database = configs['database'] client = MongoClient(db_server, db_port) db = client[database] testsuites = db.test_suites def get_suites(): return testsuites.find({"build_name": os.environ['BUILD_NAME']}, { "author": 1, "source": 1, "topo_type": 1, "_id": 1})
def formatted_error_exit(msg): print "\n" helpers.error_exit(msg, 1)