def retry_tmpfunc():
# Using develop=True to not authenticate to the server
pdc_session = PDCClient(common.conf["pdc"]["pdc_server"],
ssl_verify=True,
develop=True)
core.print_debug(pdc_session, pdc_query)
return pdc_session(**pdc_query)
def __generateDepModules_solver(self, parentdict):
deps = self.__get_module_requires()
core.print_debug("tree traverse from %s: %s" % (self.name, deps))
for dep in deps:
if dep not in parentdict:
parentdict[dep] = deps[dep]
a = PDCParser(dep, deps[dep])
a.__generateDepModules_solver(parentdict=parentdict)
def __init__(self):
config = {}
for cfgdir in [self.source_cfg, self.default_cfg, self.user_cfg]:
core.print_debug("MTF config dir search: {}".format(cfgdir))
if os.path.exists(cfgdir):
core.print_info(
"MTF config dir exists, search for {}/{}".format(
cfgdir, self.pattern))
for cfgfile in glob.glob(os.path.join(cfgdir, self.pattern)):
core.print_info("MTF config load: {}".format(cfgfile))
config.update(yaml.load(open(cfgfile)))
assert config.get("generic")
super(MTFConfParser, self).__init__(config)
def get_docker_file(dir_name=conf["docker"]["dockerfiledefaultlocation"]):
"""
Function returns full path to dockerfile.
:param dir_name: dir_name, where should be Dockerfile located
:return: full_path to Dockerfile
"""
fromenv = os.environ.get("DOCKERFILE")
if fromenv:
dockerfile = fromenv
else:
dockerfile = os.path.join(dir_name, conf["docker"]["dockerfile"])
if not os.path.exists(dockerfile):
core.print_debug("Dockerfile does not exists (you can use DOCKERFILE "
"envvar to set to another): %s" % dockerfile)
dockerfile = None
return dockerfile
def tmpfunc():
a = process.run(
"cd %s; koji download-build %s -a %s -a noarch" %
(dirname, pkgbouid, common.conf["generic"]["arch"]),
shell=True,
verbose=core.is_debug(),
ignore_status=True)
if a.exit_status == 1:
if "packages available for" in a.stdout.strip():
core.print_debug(
'UNABLE TO DOWNLOAD package (intended for other architectures, GOOD):',
a.command)
else:
raise mtfexceptions.KojiExc(
'UNABLE TO DOWNLOAD package (KOJI issue, BAD):',
a.command)
def main():
core.print_debug('verbose/debug mode')
args, unknown = cli()
if args.setup:
# mtfenvset need bash environment!
from moduleframework.mtf_environment import mtfenvset
mtfenvset()
a = AvocadoStart(args, unknown)
if args.action == 'run':
returncode = a.avocado_run()
a.show_error()
else:
# when there is any need, change general method or create specific one:
returncode = a.avocado_general(action=args.action)
exit(returncode)
def status(self, command="/bin/true"):
"""
Return status of module
:param command: which command used for do that. it could be defined inside config
:return: bool
"""
try:
command = self.info.get('status') or command
a = self.run(command,
shell=True,
ignore_bg_processes=True,
verbose=core.is_not_silent())
core.print_debug("command:", a.command, "stdout:", a.stdout,
"stderr:", a.stderr)
return True
except BaseException:
return False
def download_tagged(self, dirname):
"""
Downloads packages to directory, based on koji tags
It downloads just ARCH and noarch packages
:param dirname: string
:return: None
"""
core.print_info("DOWNLOADING ALL packages for %s_%s_%s" %
(self.name, self.stream, self.version))
for foo in process.run("koji list-tagged --quiet %s" %
self.get_pdc_info()["koji_tag"],
verbose=core.is_debug()).stdout.split("\n"):
pkgbouid = foo.strip().split(" ")[0]
if len(pkgbouid) > 4:
core.print_debug("DOWNLOADING: %s" % foo)
@timeoutlib.Retry(
attempts=common.conf["generic"]["retrycount"] * 10,
timeout=common.conf["generic"]["retrytimeout"] * 60,
delay=common.conf["generic"]["retrytimeout"],
error=mtfexceptions.KojiExc(
"RETRY: Unbale to fetch package from koji after %d attempts"
% (common.conf["generic"]["retrycount"] * 10)))
def tmpfunc():
a = process.run(
"cd %s; koji download-build %s -a %s -a noarch" %
(dirname, pkgbouid, common.conf["generic"]["arch"]),
shell=True,
verbose=core.is_debug(),
ignore_status=True)
if a.exit_status == 1:
if "packages available for" in a.stdout.strip():
core.print_debug(
'UNABLE TO DOWNLOAD package (intended for other architectures, GOOD):',
a.command)
else:
raise mtfexceptions.KojiExc(
'UNABLE TO DOWNLOAD package (KOJI issue, BAD):',
a.command)
tmpfunc()
core.print_info("DOWNLOADING finished")
def get_config(fail_without_url=True, reload=False):
"""
Read the module's configuration file.
:default: ``./config.yaml`` in the ``tests`` directory of the module's root
directory
:envvar: **CONFIG=path/to/file** overrides default value.
:return: str
"""
global __persistent_config
if not __persistent_config or reload:
cfgfile = os.environ.get('CONFIG')
if cfgfile:
if os.path.exists(cfgfile):
core.print_debug("Config file defined via envvar: %s" %
cfgfile)
else:
raise mtfexceptions.ConfigExc(
"File does not exist although defined CONFIG envvar: %s" %
cfgfile)
else:
cfgfile = "./config.yaml"
if os.path.exists(cfgfile):
core.print_debug("Using module config file: %s" % cfgfile)
else:
if fail_without_url and not get_url():
raise mtfexceptions.DefaultConfigExc(
"You have to use URL envvar for testing your images or repos"
)
cfgfile = "/usr/share/moduleframework/docs/example-config-minimal.yaml"
core.print_debug("Using default minimal config: %s" % cfgfile)
try:
with open(cfgfile, 'r') as ymlfile:
xcfg = yaml.load(ymlfile.read())
doc_name = ['modularity-testing', 'meta-test-family', 'meta-test']
if xcfg.get('document') not in doc_name:
raise mtfexceptions.ConfigExc(
"bad yaml file: item (%s)" % doc_name,
xcfg.get('document'))
if not xcfg.get('name'):
raise mtfexceptions.ConfigExc("Missing (name:) in config file")
if not xcfg.get("module"):
raise mtfexceptions.ConfigExc(
"No module in yaml config defined")
# copy rpm section to nspawn, in case not defined explicitly
# make it backward compatible
if xcfg.get("module", {}).get("rpm") and not xcfg.get(
"module", {}).get("nspawn"):
xcfg["module"]["nspawn"] = copy.deepcopy(
xcfg.get("module", {}).get("rpm"))
__persistent_config = xcfg
return xcfg
except IOError:
raise mtfexceptions.ConfigExc(
"Error: File '%s' doesn't appear to exist or it's not a YAML file. "
"Tip: If the CONFIG envvar is not set, mtf-generator looks for './config'."
% cfgfile)
else:
return __persistent_config
def get_repo(self):
# import moved here, to avoid messages when you don't need to use ODCS
from odcs.client.odcs import ODCS, AuthMech
if self.odcsauth.get("auth_mech") == AuthMech.OpenIDC:
if not self.odcsauth.get("openidc_token"):
self.odcsauth["openidc_token"] = common.get_openidc_auth()
odcs = ODCS(common.conf["odcs"]["url"], **self.odcsauth)
core.print_debug(
"ODCS Starting module composing: %s" % odcs, "%s compose for: %s" %
(self.compose_type, self.get_module_identifier()))
compose_builder = odcs.new_compose(
self.get_module_identifier(), self.compose_type,
**common.conf["odcs"]["new_compose_dict"])
timeout_time = common.conf["odcs"]["timeout"]
core.print_debug("ODCS Module compose started, timeout set to %ss" %
timeout_time)
compose_state = odcs.wait_for_compose(compose_builder["id"],
timeout=timeout_time)
core.print_debug(
"ODCS compose debug info for: %s" % self.get_module_identifier(),
compose_state)
if compose_state["state_name"] == "done":
compose = "{compose}/{arch}/os".format(
compose=compose_state["result_repo"],
arch=common.conf["generic"]["arch"])
core.print_info("ODCS Compose done, URL with repo file", compose)
return compose
else:
raise mtfexceptions.PDCExc(
"ODCS: Failed to generate compose for module: %s" %
self.get_module_identifier())
def get_helpmd_file(dir_name=conf["docker"]["dockerfiledefaultlocation"]):
"""
Function returns full path to HelpMD file.
:param dir_name: dir_name, where should be helpMD file located
:return: full_path to Dockerfile
"""
fromenv = os.environ.get("HELPMDFILE")
if fromenv:
helpmdfile = fromenv
elif os.environ.get("DOCKERFILE"):
# when DOCKERFILE is placed, search for HelpMD file in same directory
helpmdfile = os.path.join(
os.path.dirname(os.environ.get("DOCKERFILE")),
conf["docker"]["helpmdfile"])
else:
helpmdfile = os.path.join(dir_name, conf["docker"]["helpmdfile"])
if not os.path.exists(helpmdfile):
core.print_debug(
"Help MD file does not exists (you can use HELPMDFILE "
"envvar to set to another): %s" % helpmdfile)
helpmdfile = None
return helpmdfile
def __init__(self, args, unknown):
# choose between TESTS and ADDITIONAL ENVIRONMENT from options
if args.linter:
self.tests += glob.glob("{MTF_TOOLS}/{GENERIC_TEST}/*.py".format(
MTF_TOOLS=metadata_common.MetadataLoaderMTF.MTF_LINTER_PATH,
GENERIC_TEST=common.conf["generic"]["generic_tests"]))
self.tests += glob.glob("{MTF_TOOLS}/{STATIC_LINTERS}/*.py".format(
MTF_TOOLS=metadata_common.MetadataLoaderMTF.MTF_LINTER_PATH,
STATIC_LINTERS=common.conf["generic"]["static_tests"]))
self.args = args
# parse unknow options and try to find what parameter is test
while unknown:
if unknown[0] in self.A_KNOWN_PARAMS_SIMPLE:
self.additionalAvocadoArg.append(unknown[0])
unknown = unknown[1:]
elif unknown[0].startswith("-"):
if "=" in unknown[0] or len(unknown) < 2:
self.additionalAvocadoArg.append(unknown[0])
unknown = unknown[1:]
else:
self.additionalAvocadoArg += unknown[0:2]
unknown = unknown[2:]
elif glob.glob(unknown[0]):
# dereference filename via globs
testlist = glob.glob(unknown[0])
self.tests += testlist
unknown = unknown[1:]
else:
self.tests.append(unknown[0])
unknown = unknown[1:]
if self.args.metadata:
core.print_info("Using Metadata loader for tests and filtering")
metadata_tests = filtertests(backend="mtf",
location=os.getcwd(),
linters=False,
tests=[],
tags=[],
relevancy="")
tests_dict = [x[metadata_common.SOURCE] for x in metadata_tests]
self.tests += tests_dict
core.print_debug("Loaded tests via metadata file: %s" % tests_dict)
core.print_debug("tests = {0}".format(self.tests))
core.print_debug("additionalAvocadoArg = {0}".format(
self.additionalAvocadoArg))
def _tcinfo(self, testcases, header, logs=True, description=True):
"""
Parse testcases dics and print output for them in nicer format
Main purpose is to display docstrings of testcases for failures
example:
def test_some(something)
'''
This is line1.
This is line2.
:return: None
'''
self.assertTrue(False, msg="This is fail reason")
procudes line: desc -> This is line1. This is line2.
reason -> This is fail reason
:param testcases: dict of testcases
:param header: str what to print as header
:param logs: boolean if print logs for these testcases (default True)
:param description: boolean if print description = docs strings for test class + function
:return: None
"""
if testcases:
emptydelimiter = ""
harddelimiter = "------------------------"
core.print_info(emptydelimiter,
"{0} {1} {0}".format(harddelimiter, header))
for testcase in testcases:
tcname = testcase
if re.search('^[0-9]+-', testcase.get('id', "")):
tcname = testcase.get('id').split("-", 1)[1]
tcnameoutput = tcname
splitted = re.search("(.*):(.+)\.(.+)$", tcname)
if splitted:
docstrcls = []
docstrtst = []
testfile, classname, fnname = splitted.groups()
try:
testmodule = imp.load_source("test", testfile)
if getattr(testmodule, classname).__doc__:
docstrcls = getattr(
testmodule,
classname).__doc__.strip().split("\n")
if getattr(getattr(testmodule, classname),
fnname).__doc__:
docstrtst = getattr(
getattr(testmodule, classname),
fnname).__doc__.strip().split("\n")
tcnameoutput = " ".join([
x for x in docstrcls + docstrtst
if not re.search(":.*:", x)
])
# TODO: replace more whitespaces just by one - we should find better solution how to that
tcnameoutput = ' '.join(tcnameoutput.split())
except Exception as e:
core.print_debug(
"(INFO) Error happen when parsing testcase name ({})"
.format(tcname), e)
pass
core.print_info("TEST {0}: {1}".format(
testcase.get('status'), tcname))
if description and tcnameoutput != tcname and tcnameoutput and tcnameoutput.strip(
):
core.print_info(" desc -> {0}".format(tcnameoutput))
if testcase.get('fail_reason'
) and testcase.get('fail_reason') != "None":
core.print_info(" reason -> {0}".format(
testcase.get('fail_reason')))
if logs:
core.print_info(" {0}".format(testcase.get('logfile')))
core.print_info(emptydelimiter)
def cli():
# unknown options are forwarded to avocado run
args, unknown = mtfparser().parse_known_args()
if args.version:
print "0.7.7"
exit(0)
# uses additional arguments, set up variable asap, its used afterwards:
if args.debug:
os.environ['DEBUG'] = 'yes'
os.environ['AVOCADO_LOG_DEBUG'] = 'yes'
if args.config:
os.environ['CONFIG'] = args.config
if args.url:
os.environ['URL'] = args.url
if args.modulemdurl:
os.environ['MODULEMDURL'] = args.modulemdurl
core.print_debug(
"Options: linter={0}, setup={1}, action={2}, module={3}".format(
args.linter, args.setup, args.action, args.module))
core.print_debug(
"remaining options for avocado or test files: {0}".format(unknown))
# environment usage:
# read: os.environ.get('MODULE')
# write: os.environ['MODULE']
# MODULE could be from:
# 1. environment ... MODULE=docker etc
# 2. argument ... --module=docker
# 3. from config.yaml in default_module
# 4. default module stored in general mtf config yaml file
if os.environ.get('MODULE') is not None:
# environment is the highest priority because mtf uses environment (too much)
args.module = os.environ['MODULE']
if not args.module:
args.module = common.get_module_type()
os.environ['MODULE'] = args.module
if not os.environ.get('URL'):
try:
common.get_config(reload=True)
except mtfexceptions.DefaultConfigExc:
raise mtfexceptions.DefaultConfigExc(
"You have to set URL variable (via URL envar or --url parameter) in case of default config"
)
supported_modules = set(common.get_backend_list() +
common.list_modules_from_config())
if args.module in supported_modules:
# for debug purposes, to be sure about module variables or options
core.print_debug("MODULE={0}, options={1}".format(
os.environ.get('MODULE'), args.module))
else:
# TODO: what to do here? whats the defaults value for MODULE, do I know it?
raise mtfexceptions.ModuleFrameworkException(
"MODULE={0} ; we support {1}".format(os.environ.get('MODULE'),
supported_modules))
core.print_debug("MODULE={0}".format(os.environ.get('MODULE')))
return args, unknown
def __init__(self, args, unknown):
# choose between TESTS and ADDITIONAL ENVIRONMENT from options
if args.linter:
self.tests += glob.glob("{MTF_TOOLS}/{GENERIC_TEST}/*.py".format(
MTF_TOOLS=metadata_common.MetadataLoaderMTF.MTF_LINTER_PATH,
GENERIC_TEST=common.conf["generic"]["generic_tests"]))
self.tests += glob.glob("{MTF_TOOLS}/{STATIC_LINTERS}/*.py".format(
MTF_TOOLS=metadata_common.MetadataLoaderMTF.MTF_LINTER_PATH,
STATIC_LINTERS=common.conf["generic"]["static_tests"]))
self.args = args
# parse unknow options and try to find what parameter is test
while unknown:
if unknown[0] in self.A_KNOWN_PARAMS_SIMPLE:
self.additionalAvocadoArg.append(unknown[0])
unknown = unknown[1:]
elif unknown[0].startswith("-"):
if "=" in unknown[0] or len(unknown) < 2:
self.additionalAvocadoArg.append(unknown[0])
unknown = unknown[1:]
else:
self.additionalAvocadoArg += unknown[0:2]
unknown = unknown[2:]
elif glob.glob(unknown[0]):
# dereference filename via globs
testlist = glob.glob(unknown[0])
self.tests += testlist
unknown = unknown[1:]
else:
self.tests.append(unknown[0])
unknown = unknown[1:]
if self.args.metadata:
core.print_info("Using Metadata loader for tests and filtering")
metadata_tests = filtertests(backend="mtf",
location=os.getcwd(),
linters=False,
tests=[],
tags=[],
relevancy="")
tests_dict = [x[metadata_common.SOURCE] for x in metadata_tests]
self.tests += tests_dict
core.print_debug("Loaded tests via metadata file: %s" % tests_dict)
core.print_debug("tests = {0}".format(self.tests))
core.print_debug("additionalAvocadoArg = {0}".format(
self.additionalAvocadoArg))
# Advanced filtering and testcases adding based on FMF metadata, see help
if self.args.fmffilter or self.args.fmfpath:
# if fmf path is empty, use actual directory
if not self.args.fmfpath:
self.args.fmfpath = ["."]
try:
import fmf
except ImportError:
mtfexceptions.ModuleFrameworkException(
"FMF metadata format not installed on your system,"
" see fmf.readthedocs.io/en/latest/"
"for more info (how to install)")
core.print_debug(
"Using FMF metadata: path - {} and filters - {}".format(
self.args.fmfpath,
common.conf["fmf"]["filters"] + self.args.fmffilter))
for onepath in self.args.fmfpath:
tree = fmf.Tree(onepath)
for node in tree.prune(
False, common.conf["fmf"]["keys"],
common.conf["fmf"]["names"],
common.conf["fmf"]["filters"] + self.args.fmffilter):
testcase = node.show(False, common.conf["fmf"]["format"],
common.conf["fmf"]["values"])
core.print_debug("added test by FMF: {}".format(testcase))
self.tests.append(testcase)
