def prog_args():
descr = """\
Update the regression tags for a build. The currently supported tags are
'daily' and 'full'.
Example:
% BUILD_NAME=ihplus_bcf_10G-490 ./update_regression_tags.py --tags full
"""
parser = argparse.ArgumentParser(prog='update_regression_tags',
formatter_class=argparse.RawDescriptionHelpFormatter,
description=descr)
parser.add_argument('--verbose', action='store_true',
default=False,
help=("Print verbose output"))
parser.add_argument('--build',
help=("Jenkins build string,"
" e.g., 'bvs master #2007'"))
parser.add_argument('--tags', metavar=('tag1', 'tag2'), nargs='*',
help=("Regression tags,"
" e.g., 'full', 'daily'"))
_args = parser.parse_args()
# _args.build <=> env BUILD_NAME
if not _args.build and 'BUILD_NAME' in os.environ:
_args.build = os.environ['BUILD_NAME']
elif not _args.build:
helpers.error_exit("Must specify --build option or set environment"
" variable BUILD_NAME")
else:
os.environ['BUILD_NAME'] = _args.build
return _args
def prog_args():
descr = """\
Check the database collection 'builds' for a document which matches the env
BUILD_NAME.
- If found, return the build name.
- If not found, create a new document, then return the build name.
"""
parser = argparse.ArgumentParser(
prog='db_chk_and_add_build_name',
formatter_class=argparse.RawDescriptionHelpFormatter,
description=descr)
parser.add_argument('--verbose',
action='store_true',
default=False,
help=("Print verbose output"))
parser.add_argument('--build',
help=("Jenkins build string,"
" e.g., 'bvs master #2007'"))
parser.add_argument(
'--regression-tags',
help=("Supported regression tags are 'daily' or 'full'."))
_args = parser.parse_args()
# _args.build <=> env BUILD_NAME
if not _args.build and 'BUILD_NAME' in os.environ:
_args.build = os.environ['BUILD_NAME']
elif not _args.build:
helpers.error_exit("Must specify --build option or set environment"
" variable BUILD_NAME")
else:
os.environ['BUILD_NAME'] = _args.build
return _args
def __init__(self, build):
self._aggregated_build_name = build
self._cat = None
self._builds = self.catalog().aggregated_build(build)
if not self._builds:
helpers.error_exit(
"Aggregated build '%s' is not defined in catalog.yaml." %
build)
def load_configs(self):
try:
config_file = helpers.bigrobot_configs_path() + "/testrail.yaml"
config = helpers.load_config(config_file)
except IOError:
s = "Unable to open TestRail config file %s" % config_file
helpers.error_exit(s)
return config
def prog_args():
descr = """\
Display test execution stats collected for a specific build.
"""
parser = argparse.ArgumentParser(
prog='db_collect_stats',
formatter_class=argparse.RawDescriptionHelpFormatter,
description=descr)
parser.add_argument(
'--release',
help=
("Product release, e.g., 'ironhorse', 'ironhorse-plus', 'jackfrost', etc."
))
parser.add_argument('--build',
help=("Jenkins build string,"
" e.g., 'bvs master #2007'"))
parser.add_argument('--show-untested',
action='store_true',
default=False,
help=("Show the manual-untested test cases"))
parser.add_argument('--show-manual',
action='store_true',
default=False,
help=("Show the manual test cases"))
parser.add_argument('--show-suites',
action='store_true',
default=False,
help=("Show the test suites"))
parser.add_argument('--show-all',
action='store_true',
default=False,
help=("Show all stats"))
parser.add_argument('--no-show-functional-areas',
action='store_true',
default=False,
help=("Don't show test details for functional areas"))
_args = parser.parse_args()
# _args.build <=> env BUILD_NAME
if not _args.build and 'BUILD_NAME' in os.environ:
_args.build = os.environ['BUILD_NAME']
elif not _args.build:
helpers.error_exit("Must specify --build option or set environment"
" variable BUILD_NAME")
else:
os.environ['BUILD_NAME'] = _args.build
# _args.release <=> env RELEASE_NAME
if not _args.release and 'RELEASE_NAME' in os.environ:
_args.release = os.environ['RELEASE_NAME']
elif not _args.release:
helpers.error_exit("Must specify --release option or set environment"
" variable RELEASE_NAME")
else:
os.environ['RELEASE_NAME'] = _args.release
_args.release = _args.release.lower()
return _args
def prog_args():
descr = """
Display test data for a specific build. Example:
% RELEASE_NAME="<release>" BUILD_NAME="bvs master #3271" \\
./db_get_test_data.py --tags manual-untested \\
--no-show-header
"""
parser = argparse.ArgumentParser(
prog='db_get_test_data',
formatter_class=argparse.RawDescriptionHelpFormatter,
description=descr)
parser.add_argument('--build',
help=("Jenkins build string,"
" e.g., 'bvs master #2007'"))
parser.add_argument(
'--release',
help=
("Product release, e.g., 'ironhorse', 'ironhorse-plus', 'jackfrost', etc."
))
parser.add_argument(
'--tags',
metavar=('tag1', 'tag2'),
nargs='*',
help=
'Test case tags to include, e.g., manual-untested, scaling, ztn, etc.')
parser.add_argument(
'--no-show-header',
action='store_true',
default=False,
help=("Don't show header info (build name, release, total)"))
_args = parser.parse_args()
# _args.build <=> env BUILD_NAME
if not _args.build and 'BUILD_NAME' in os.environ:
_args.build = os.environ['BUILD_NAME']
elif not _args.build:
helpers.error_exit("Must specify --build option or set environment"
" variable BUILD_NAME")
else:
os.environ['BUILD_NAME'] = _args.build
# _args.release <=> env RELEASE_NAME
if not _args.release and 'RELEASE_NAME' in os.environ:
_args.release = os.environ['RELEASE_NAME']
elif not _args.release:
helpers.error_exit("Must specify --release option or set environment"
" variable RELEASE_NAME")
else:
os.environ['RELEASE_NAME'] = _args.release
_args.release = _args.release.lower()
return _args
def prog_args():
descr = """\
Given the release (RELEASE_NAME env) and build (BUILD_NAME env), print a list
of failed test cases.
Examples:
% RELEASE_NAME="ironhorse" BUILD_NAME="bvs master bcf-2.0.0 fcs" ./dump_test_cases_failed.py
% ./dump_test_cases_failed.py --release ironhorse --build "bvs master aggregated 2014 wk40"
"""
parser = argparse.ArgumentParser(
prog='dump_test_cases_failed',
formatter_class=argparse.RawDescriptionHelpFormatter,
description=descr)
parser.add_argument('--build',
help=("Jenkins build string,"
" e.g., 'bvs master #2007'"))
parser.add_argument(
'--release',
help=
("Product release, e.g., 'ironhorse', 'ironhorse-plus', 'jackfrost', etc."
))
parser.add_argument('--show-tags',
action='store_true',
default=False,
help=("Show test case tags"))
_args = parser.parse_args()
# _args.build <=> env BUILD_NAME
if not _args.build and 'BUILD_NAME' in os.environ:
_args.build = os.environ['BUILD_NAME']
elif not _args.build:
helpers.error_exit("Must specify --build option or set environment"
" variable BUILD_NAME")
else:
os.environ['BUILD_NAME'] = _args.build
# _args.release <=> env RELEASE_NAME
if not _args.release and 'RELEASE_NAME' in os.environ:
_args.release = os.environ['RELEASE_NAME']
elif not _args.release:
helpers.error_exit("Must specify --release option or set environment"
" variable RELEASE_NAME")
else:
os.environ['RELEASE_NAME'] = _args.release
_args.release = _args.release.lower()
return _args
def __init__(self, build, user, overwrite):
self._aggregated_build_name = build
self._user = user
self._is_overwrite = overwrite
self._cat = None
self._builds = self.catalog().aggregated_build(build)
self._test_case_dict = {}
if not self._builds:
helpers.error_exit(
"Aggregated build '%s' is not defined in the database." %
build)
# dictionary of test suites indexed by product_suite key.
self._product_suites = {}
self.load_product_suites_dict()
def prog_args():
descr = """\
Populate the QA Dashboard database with data found in the user-entered
verification files (in bigrobot/data/verification/).
The specified build (BUILD_NAME) must be the name of an aggregated build.
Examples:
-- Sanity check the data. It will not update the database.
% BUILD_NAME="bvs master bcf-2.0.0 fcs" ./populate_db_user_verification.py \\
--infile ../data/verification/bvs_master_bcf-2.0.0_fcs/verification_kranti.yaml \\
--sanitize-data
-- If everything looks good. Update the database.
% BUILD_NAME="bvs master bcf-2.0.0 fcs" ./populate_db_user_verification.py \\
--infile ../data/verification/bvs_master_bcf-2.0.0_fcs/verification_kranti.yaml
"""
parser = argparse.ArgumentParser(
prog='db_manual_verification',
formatter_class=argparse.RawDescriptionHelpFormatter,
description=descr)
parser.add_argument(
'--build',
help=("Build name,"
" e.g., 'bvs master ironhorse beta2 aggregated'"))
parser.add_argument('--infile',
required=True,
help=("Input verification file"))
parser.add_argument('--sanitize-data',
action='store_true',
default=False,
help=("Check data file for formatting errors"))
_args = parser.parse_args()
# _args.build <=> env BUILD_NAME
if not _args.build and 'BUILD_NAME' in os.environ:
_args.build = os.environ['BUILD_NAME']
elif not _args.build:
helpers.error_exit("Must specify --build option or set environment"
" variable BUILD_NAME")
else:
os.environ['BUILD_NAME'] = _args.build
return _args
def prog_args():
descr = """\
Check the database collection 'aggregated_builds' for a document which
matches the env BUILD_NAME.
- If found, return the aggregated build name.
- If not found and build_name matches a typical build name (e.g., 'bvs master #1234'),
create a new document, then return the weekly aggregated build name.
- If not found and build_name does not match a typical build name (e.g., a Beta build
does not have a typical build name - 'bvs master #bcf-2.0.0_13'), don't create a
new document. You should instead rerun command and specify the aggregated beta build name
using the option --aggregated-build-name.
Examples:
% BUILD_NAME="bvs master #3436" ./db_chk_and_add_wk_aggregated_build.py
% BUILD_NAME="bvs master #bcf-2.0.0_13" ./db_chk_and_add_wk_aggregated_build.py \\
--aggregated-build-name "bvs master bcf-2.0.0 aggregated"
"""
parser = argparse.ArgumentParser(
prog='db_chk_and_add_wk_aggregated_build',
formatter_class=argparse.RawDescriptionHelpFormatter,
description=descr)
parser.add_argument('--verbose',
action='store_true',
default=False,
help=("Print verbose output"))
parser.add_argument('--build',
help=("Jenkins build string,"
" e.g., 'bvs master #2007'"))
parser.add_argument('--aggregated-build-name',
help=("Aggregated build name,"
" e.g., 'bvs master bcf-2.0.0 aggregated'"))
_args = parser.parse_args()
# _args.build <=> env BUILD_NAME
if not _args.build and 'BUILD_NAME' in os.environ:
_args.build = os.environ['BUILD_NAME']
elif not _args.build:
helpers.error_exit("Must specify --build option or set environment"
" variable BUILD_NAME")
else:
os.environ['BUILD_NAME'] = _args.build
return _args
def prog_args():
descr = """\
Search for all FAILed test cases in a (aggregated) build and generate
verification records for each failed test case.
The specified build (BUILD_NAME) must be the name of an aggregated build.
Examples:
-- Generate the verification file for 'mingtao'
% BUILD_NAME="bvs master bcf-2.0.0 fcs" ./create_or_update_verification_file.py --user mingtao
-- Generate the verification files for all QA engineers
% BUILD_NAME="bvs master bcf-2.0.0 fcs" ./create_or_update_verification_file.py --user all
"""
parser = argparse.ArgumentParser(
prog='create_or_update_verification_file',
formatter_class=argparse.RawDescriptionHelpFormatter,
description=descr)
parser.add_argument(
'--build',
help=("Build name,"
" e.g., 'bvs master ironhorse beta2 aggregated'"))
parser.add_argument(
'--user',
required=True,
help=("Create verification file for user (as defined in"
" test catalog)."))
parser.add_argument('--force-overwrite',
action='store_true',
default=False,
help=("Overwrite the existing verification files"))
_args = parser.parse_args()
# _args.build <=> env BUILD_NAME
if not _args.build and 'BUILD_NAME' in os.environ:
_args.build = os.environ['BUILD_NAME']
elif not _args.build:
helpers.error_exit("Must specify --build option or set environment"
" variable BUILD_NAME")
else:
os.environ['BUILD_NAME'] = _args.build
return _args
def prog_args():
descr = """\
Parse the Robot output.xml files and generate/populate the collections for the
Test Catalog (MongoDB) database.
"""
parser = argparse.ArgumentParser(prog='parse_test_xml_output',
formatter_class=argparse.RawDescriptionHelpFormatter,
description=descr)
parser.add_argument('--build',
help=("Jenkins build string,"
" e.g., 'bvs master #2007'"))
parser.add_argument('--input', required=True,
help=("Input file which contains a list of Robot"
" output.xml files with complete pathnames"))
parser.add_argument('--output-suites', required=True,
help=("JSON output file containing test suites"))
parser.add_argument('--output-testcases', required=True,
help=("JSON output file containing test cases"))
parser.add_argument('--is-baseline',
action='store_true', default=False,
help=("Specify this option if generating baseline"
" data"))
parser.add_argument('--is-regression',
action='store_true', default=False,
help=("Specify this option if analyzing regression"
" results"))
_args = parser.parse_args()
# _args.build <=> env BUILD_NAME
if not _args.build and 'BUILD_NAME' in os.environ:
_args.build = os.environ['BUILD_NAME']
elif not _args.build:
helpers.error_exit("Must specify --build option or set environment"
" variable BUILD_NAME")
else:
os.environ['BUILD_NAME'] = _args.build
if not _args.is_baseline and not _args.is_regression:
helpers.error_exit("Must specify --is-regression or --is-baseline")
return _args
def prog_args():
descr = """\
Generate the BUILD_NAME string based on a BCF/BigTap version string found in
a file (see --version-file) or a string (see --version-str). The version string
needs to have the following format:
Big Tap Controller 4.1.1 (2014.11.13.1922-b.bsc.corsair-4.1.1beta
Big Cloud Fabric Appliance 3.0.0-master01-SNAPSHOT (bcf_master #4201)
Big Cloud Fabric Appliance 2.0.0-master01-SNAPSHOT (ihplus_bcf #515)
Returns the BUILD_NAME as followed:
corsair411_bigtap_10G-1922
master_bcf_10G-4201
"""
parser = argparse.ArgumentParser(
prog='build_name_gen',
formatter_class=argparse.RawDescriptionHelpFormatter,
description=descr)
parser.add_argument('--version-file',
default='/var/tmp/ver.txt',
help=("Version file"))
parser.add_argument('--version-str', help=("Version string"))
parser.add_argument(
'--additional-version-descr',
help=("An additional version description to add to BUILD_NAME"))
parser.add_argument(
'--testbed',
required=True,
help=
("Testbed name, e.g., common, Dell, Accton, Quanta, 10G, 40G, virtual, etc."
))
_args = parser.parse_args()
if _args.testbed == None:
helpers.error_exit("Env BIGROBOT_PARAMS_INPUT is not found.")
return _args
def prog_args():
descr = """\
Check the database collection 'build_groups' for a document which matches the
BUILD_NAME group.
- If found, return the build name group.
- If not found, create a new document, then return the build name group.
"""
parser = argparse.ArgumentParser(
prog='db_chk_and_add_build_name_group',
formatter_class=argparse.RawDescriptionHelpFormatter,
description=descr)
parser.add_argument('--verbose',
action='store_true',
default=False,
help=("Print verbose output"))
parser.add_argument('--build',
help=("Jenkins build string,"
" e.g., 'bvs master #2007'"))
parser.add_argument('--createtime',
help=("Build creation time,"
" e.g., '2015-03-06T12:54:58.130'"))
parser.add_argument('--updatetime',
help=("Build updated time,"
" e.g., '2015-03-06T12:54:58.130'"))
_args = parser.parse_args()
# _args.build <=> env BUILD_NAME
if not _args.build and 'BUILD_NAME' in os.environ:
_args.build = os.environ['BUILD_NAME']
elif not _args.build:
helpers.error_exit("Must specify --build option or set environment"
" variable BUILD_NAME")
else:
os.environ['BUILD_NAME'] = _args.build
return _args
def get_version_string(args):
version_str = args.version_str
if version_str == None:
if helpers.file_not_exists(args.version_file):
helpers.error_exit("Version file '%s' is not found." %
args.version_file)
version_str = helpers.file_read_once(args.version_file).strip()
version = None
release = None
build_name = None
testbed = args.testbed
##########################################################################
# Match BigTap build string
# - The match logic is as defined in bigrobot/configs/catalog.yaml
##########################################################################
re_bigtap = r'.*Big Tap Controller (\d+\.\d+\.\d+).+\d\d\d\d\.\d\d\.\d\d\.(\d+).+'
match = re.match(re_bigtap, version_str)
if match:
version = match.group(1)
build_id = match.group(2)
if version in ['4.0.0', '4.0.1', '4.0.2', '4.1.0']:
release = 'corsair400'
elif version in ['4.1.1']:
release = 'corsair411'
elif version in ['4.5.0']:
release = 'corsair450'
else:
helpers.error_exit(
"Invalid BigTap version '%s' in version string '%s'." %
(version, version_str))
build_name = "%s_%s_%s-%s" % (release, "bigtap", testbed, build_id)
if args.additional_version_descr != None:
build_name = "%s_%s" % (build_name, args.additional_version_descr)
return build_name
##########################################################################
# Match BCF build string
# - The match logic is as defined in bigrobot/configs/catalog.yaml
##########################################################################
re_bcf = r'.*Big Cloud Fabric Appliance \d+\.\d+\.\d+.+\((\w+)_(\w+) #(\d+)\).*'
match = re.match(re_bcf, version_str)
if match:
if match.group(1) == 'bcf':
release = match.group(2)
else:
release = match.group(1)
build_id = match.group(3)
if not release in ['ih', 'ihplus', 'jf', 'master']:
helpers.error_exit(
"Invalid BCF version '%s' in version string '%s'." %
(version, version_str))
build_name = "%s_%s_%s-%s" % (release, "bcf", testbed, build_id)
if args.additional_version_descr != None:
build_name = "%s_%s" % (build_name, args.additional_version_descr)
return build_name
helpers.error_exit("Invalid version string: '%s'." % version_str)
def do_it(self):
# Search for failed test cases. Verification list will be built/updated
# based on the failed test cases.
query = {
"build_name": self.aggregated_build_name(),
"status": 'FAIL',
# "tags": { "$all": ['ironhorse']},
}
aggr_cursor = self.catalog().find_test_cases_archive(query)
fail_count = aggr_cursor.count()
print("Total failed test cases in build '%s': %s" %
(self.aggregated_build_name(), fail_count))
if self._user == 'all':
pass
elif self._user not in Authors().get().values():
helpers.error_exit(
"User '%s' is not defined in the test catalog." % self._user)
# Initialize test case dictionary for each author if not exist
for author in Authors().get().values():
file_name = self.verification_file(author)
if helpers.file_exists(file_name):
# print "Loading file %s" % file_name
tc_list = PseudoYAML(file_name).load_yaml_file()
self._test_case_dict[author] = {}
if tc_list:
# Test cases found
for tc in tc_list:
tc_key = tc['product_suite'] + ' ' + tc['name']
self._test_case_dict[author][tc_key] = tc
else:
self._test_case_dict[author] = {}
first_pass = {}
for tc in aggr_cursor:
product_suite = tc['product_suite']
tc_name = tc['name']
status = tc['status']
# build_name = build_name_last = tc['build_name']
if 'build_name_list' in tc and tc['build_name_list']:
build_name_last = tc['build_name_list'][-1]
author = self.author(product_suite)
file_name = self.verification_file(author)
new_file_name = file_name + ".new"
if self._user == 'all' or self._user == author:
pass # do action
else:
continue
if new_file_name not in first_pass:
if helpers.file_exists(new_file_name):
print "Removing file %s" % new_file_name
helpers.file_remove(new_file_name)
helpers.file_copy(self.verification_header_template(),
new_file_name)
print "Creating file %s" % new_file_name
first_pass[new_file_name] = file_name
key = product_suite + ' ' + tc_name
if key in self._test_case_dict[author]:
if status == self._test_case_dict[author][key]['status']:
self._test_case_dict[author][key]['name'] = \
sanitize_yaml_string(
self._test_case_dict[author][key]['name'])
self.write_entry_to_file(self._test_case_dict[author][key],
new_file_name)
else:
helpers.file_write_append_once(
new_file_name, "\n### status: %s in recent '%s'" %
(status, build_name_last))
self._test_case_dict[author][key]['name'] = \
sanitize_yaml_string(
self._test_case_dict[author][key]['name'])
self.write_entry_to_file(self._test_case_dict[author][key],
new_file_name)
del self._test_case_dict[author][key]
else:
helpers.file_write_append_once(
new_file_name,
"\n### new entry in recent '%s'" % (build_name_last))
rec = {
"name": sanitize_yaml_string(tc_name),
"product_suite": product_suite,
"status": status,
}
if 'jira' in tc and tc['jira']:
rec['jira'] = tc['jira']
if 'jira' in tc and tc['jira']:
rec['jira'] = tc['jira']
if 'notes' in tc and tc['notes']:
rec['notes'] = tc['notes']
self.write_entry_to_file(rec, new_file_name)
# Dump remaining test cases from YAML
for author in self._test_case_dict:
if self._user == 'all' or self._user == author:
pass # do action
else:
continue
file_name = self.verification_file(author)
new_file_name = file_name + ".new"
for key in self._test_case_dict[author]:
if not helpers.file_exists(new_file_name):
helpers.file_copy(self.verification_header_template(),
new_file_name)
tc = self._test_case_dict[author][key]
if tc['status'] == 'FAIL':
if tc['jira'] or (tc['notes']
and tc['notes'] != "No description."):
# Contains user comments
helpers.file_write_append_once(
new_file_name,
"\n### Previously verified. Not reported as"
" FAIL in recent report so is likely PASSing.")
tc['name'] = sanitize_yaml_string(tc['name'])
self.write_entry_to_file(tc, new_file_name)
else:
# User didn't get a chance to verify it previously.
# So just silently ignore it.
pass
else:
# Contains user comments
tc['name'] = sanitize_yaml_string(tc['name'])
self.write_entry_to_file(tc, new_file_name)
parser.add_argument('--sanitize-data',
action='store_true',
default=False,
help=("Check data file for formatting errors"))
_args = parser.parse_args()
# _args.build <=> env BUILD_NAME
if not _args.build and 'BUILD_NAME' in os.environ:
_args.build = os.environ['BUILD_NAME']
elif not _args.build:
helpers.error_exit("Must specify --build option or set environment"
" variable BUILD_NAME")
else:
os.environ['BUILD_NAME'] = _args.build
return _args
if __name__ == '__main__':
args = prog_args()
if not helpers.file_exists(args.infile):
helpers.error_exit("File '%s' is not found." % args.infile, 1)
verification_build = ManualVerificationBuild(args.build, args.infile)
if args.sanitize_data:
if verification_build.sanitize_verification_data():
print "Data format is correct."
else:
print "Error found in data format."
else:
verification_build.populate_db_with_verification_data()
# Determine BigRobot path(s) based on this executable (which resides in
# the bin/ directory.
bigrobot_path = os.path.dirname(__file__) + '/..'
exscript_path = bigrobot_path + '/vendors/exscript/src'
sys.path.insert(0, bigrobot_path)
sys.path.insert(1, exscript_path)
import autobot.helpers as helpers
helpers.set_env('IS_GOBOT', 'False')
helpers.set_env('AUTOBOT_LOG', './myrobot.log')
if not 'BUILD_NAME' in os.environ:
helpers.error_exit("Environment variable BUILD_NAME is not defined.", 1)
configs = helpers.bigrobot_config_test_catalog()
db_server = configs['db_server']
db_port = configs['db_port']
database = configs['database']
client = MongoClient(db_server, db_port)
db = client[database]
testsuites = db.test_suites
def get_suites():
return testsuites.find({"build_name": os.environ['BUILD_NAME']},
{ "author": 1, "source": 1, "topo_type": 1, "_id": 1})
def formatted_error_exit(msg):
print "\n"
helpers.error_exit(msg, 1)
