def retry_tmpfunc(): # Using develop=True to not authenticate to the server pdc_session = PDCClient(common.conf["pdc"]["pdc_server"], ssl_verify=True, develop=True) core.print_debug(pdc_session, pdc_query) return pdc_session(**pdc_query)
def __generateDepModules_solver(self, parentdict): deps = self.__get_module_requires() core.print_debug("tree traverse from %s: %s" % (self.name, deps)) for dep in deps: if dep not in parentdict: parentdict[dep] = deps[dep] a = PDCParser(dep, deps[dep]) a.__generateDepModules_solver(parentdict=parentdict)
def __init__(self): config = {} for cfgdir in [self.source_cfg, self.default_cfg, self.user_cfg]: core.print_debug("MTF config dir search: {}".format(cfgdir)) if os.path.exists(cfgdir): core.print_info( "MTF config dir exists, search for {}/{}".format( cfgdir, self.pattern)) for cfgfile in glob.glob(os.path.join(cfgdir, self.pattern)): core.print_info("MTF config load: {}".format(cfgfile)) config.update(yaml.load(open(cfgfile))) assert config.get("generic") super(MTFConfParser, self).__init__(config)
def get_docker_file(dir_name=conf["docker"]["dockerfiledefaultlocation"]): """ Function returns full path to dockerfile. :param dir_name: dir_name, where should be Dockerfile located :return: full_path to Dockerfile """ fromenv = os.environ.get("DOCKERFILE") if fromenv: dockerfile = fromenv else: dockerfile = os.path.join(dir_name, conf["docker"]["dockerfile"]) if not os.path.exists(dockerfile): core.print_debug("Dockerfile does not exists (you can use DOCKERFILE " "envvar to set to another): %s" % dockerfile) dockerfile = None return dockerfile
def tmpfunc(): a = process.run( "cd %s; koji download-build %s -a %s -a noarch" % (dirname, pkgbouid, common.conf["generic"]["arch"]), shell=True, verbose=core.is_debug(), ignore_status=True) if a.exit_status == 1: if "packages available for" in a.stdout.strip(): core.print_debug( 'UNABLE TO DOWNLOAD package (intended for other architectures, GOOD):', a.command) else: raise mtfexceptions.KojiExc( 'UNABLE TO DOWNLOAD package (KOJI issue, BAD):', a.command)
def main(): core.print_debug('verbose/debug mode') args, unknown = cli() if args.setup: # mtfenvset need bash environment! from moduleframework.mtf_environment import mtfenvset mtfenvset() a = AvocadoStart(args, unknown) if args.action == 'run': returncode = a.avocado_run() a.show_error() else: # when there is any need, change general method or create specific one: returncode = a.avocado_general(action=args.action) exit(returncode)
def status(self, command="/bin/true"): """ Return status of module :param command: which command used for do that. it could be defined inside config :return: bool """ try: command = self.info.get('status') or command a = self.run(command, shell=True, ignore_bg_processes=True, verbose=core.is_not_silent()) core.print_debug("command:", a.command, "stdout:", a.stdout, "stderr:", a.stderr) return True except BaseException: return False
def download_tagged(self, dirname): """ Downloads packages to directory, based on koji tags It downloads just ARCH and noarch packages :param dirname: string :return: None """ core.print_info("DOWNLOADING ALL packages for %s_%s_%s" % (self.name, self.stream, self.version)) for foo in process.run("koji list-tagged --quiet %s" % self.get_pdc_info()["koji_tag"], verbose=core.is_debug()).stdout.split("\n"): pkgbouid = foo.strip().split(" ")[0] if len(pkgbouid) > 4: core.print_debug("DOWNLOADING: %s" % foo) @timeoutlib.Retry( attempts=common.conf["generic"]["retrycount"] * 10, timeout=common.conf["generic"]["retrytimeout"] * 60, delay=common.conf["generic"]["retrytimeout"], error=mtfexceptions.KojiExc( "RETRY: Unbale to fetch package from koji after %d attempts" % (common.conf["generic"]["retrycount"] * 10))) def tmpfunc(): a = process.run( "cd %s; koji download-build %s -a %s -a noarch" % (dirname, pkgbouid, common.conf["generic"]["arch"]), shell=True, verbose=core.is_debug(), ignore_status=True) if a.exit_status == 1: if "packages available for" in a.stdout.strip(): core.print_debug( 'UNABLE TO DOWNLOAD package (intended for other architectures, GOOD):', a.command) else: raise mtfexceptions.KojiExc( 'UNABLE TO DOWNLOAD package (KOJI issue, BAD):', a.command) tmpfunc() core.print_info("DOWNLOADING finished")
def get_config(fail_without_url=True, reload=False): """ Read the module's configuration file. :default: ``./config.yaml`` in the ``tests`` directory of the module's root directory :envvar: **CONFIG=path/to/file** overrides default value. :return: str """ global __persistent_config if not __persistent_config or reload: cfgfile = os.environ.get('CONFIG') if cfgfile: if os.path.exists(cfgfile): core.print_debug("Config file defined via envvar: %s" % cfgfile) else: raise mtfexceptions.ConfigExc( "File does not exist although defined CONFIG envvar: %s" % cfgfile) else: cfgfile = "./config.yaml" if os.path.exists(cfgfile): core.print_debug("Using module config file: %s" % cfgfile) else: if fail_without_url and not get_url(): raise mtfexceptions.DefaultConfigExc( "You have to use URL envvar for testing your images or repos" ) cfgfile = "/usr/share/moduleframework/docs/example-config-minimal.yaml" core.print_debug("Using default minimal config: %s" % cfgfile) try: with open(cfgfile, 'r') as ymlfile: xcfg = yaml.load(ymlfile.read()) doc_name = ['modularity-testing', 'meta-test-family', 'meta-test'] if xcfg.get('document') not in doc_name: raise mtfexceptions.ConfigExc( "bad yaml file: item (%s)" % doc_name, xcfg.get('document')) if not xcfg.get('name'): raise mtfexceptions.ConfigExc("Missing (name:) in config file") if not xcfg.get("module"): raise mtfexceptions.ConfigExc( "No module in yaml config defined") # copy rpm section to nspawn, in case not defined explicitly # make it backward compatible if xcfg.get("module", {}).get("rpm") and not xcfg.get( "module", {}).get("nspawn"): xcfg["module"]["nspawn"] = copy.deepcopy( xcfg.get("module", {}).get("rpm")) __persistent_config = xcfg return xcfg except IOError: raise mtfexceptions.ConfigExc( "Error: File '%s' doesn't appear to exist or it's not a YAML file. " "Tip: If the CONFIG envvar is not set, mtf-generator looks for './config'." % cfgfile) else: return __persistent_config
def get_repo(self): # import moved here, to avoid messages when you don't need to use ODCS from odcs.client.odcs import ODCS, AuthMech if self.odcsauth.get("auth_mech") == AuthMech.OpenIDC: if not self.odcsauth.get("openidc_token"): self.odcsauth["openidc_token"] = common.get_openidc_auth() odcs = ODCS(common.conf["odcs"]["url"], **self.odcsauth) core.print_debug( "ODCS Starting module composing: %s" % odcs, "%s compose for: %s" % (self.compose_type, self.get_module_identifier())) compose_builder = odcs.new_compose( self.get_module_identifier(), self.compose_type, **common.conf["odcs"]["new_compose_dict"]) timeout_time = common.conf["odcs"]["timeout"] core.print_debug("ODCS Module compose started, timeout set to %ss" % timeout_time) compose_state = odcs.wait_for_compose(compose_builder["id"], timeout=timeout_time) core.print_debug( "ODCS compose debug info for: %s" % self.get_module_identifier(), compose_state) if compose_state["state_name"] == "done": compose = "{compose}/{arch}/os".format( compose=compose_state["result_repo"], arch=common.conf["generic"]["arch"]) core.print_info("ODCS Compose done, URL with repo file", compose) return compose else: raise mtfexceptions.PDCExc( "ODCS: Failed to generate compose for module: %s" % self.get_module_identifier())
def get_helpmd_file(dir_name=conf["docker"]["dockerfiledefaultlocation"]): """ Function returns full path to HelpMD file. :param dir_name: dir_name, where should be helpMD file located :return: full_path to Dockerfile """ fromenv = os.environ.get("HELPMDFILE") if fromenv: helpmdfile = fromenv elif os.environ.get("DOCKERFILE"): # when DOCKERFILE is placed, search for HelpMD file in same directory helpmdfile = os.path.join( os.path.dirname(os.environ.get("DOCKERFILE")), conf["docker"]["helpmdfile"]) else: helpmdfile = os.path.join(dir_name, conf["docker"]["helpmdfile"]) if not os.path.exists(helpmdfile): core.print_debug( "Help MD file does not exists (you can use HELPMDFILE " "envvar to set to another): %s" % helpmdfile) helpmdfile = None return helpmdfile
def __init__(self, args, unknown): # choose between TESTS and ADDITIONAL ENVIRONMENT from options if args.linter: self.tests += glob.glob("{MTF_TOOLS}/{GENERIC_TEST}/*.py".format( MTF_TOOLS=metadata_common.MetadataLoaderMTF.MTF_LINTER_PATH, GENERIC_TEST=common.conf["generic"]["generic_tests"])) self.tests += glob.glob("{MTF_TOOLS}/{STATIC_LINTERS}/*.py".format( MTF_TOOLS=metadata_common.MetadataLoaderMTF.MTF_LINTER_PATH, STATIC_LINTERS=common.conf["generic"]["static_tests"])) self.args = args # parse unknow options and try to find what parameter is test while unknown: if unknown[0] in self.A_KNOWN_PARAMS_SIMPLE: self.additionalAvocadoArg.append(unknown[0]) unknown = unknown[1:] elif unknown[0].startswith("-"): if "=" in unknown[0] or len(unknown) < 2: self.additionalAvocadoArg.append(unknown[0]) unknown = unknown[1:] else: self.additionalAvocadoArg += unknown[0:2] unknown = unknown[2:] elif glob.glob(unknown[0]): # dereference filename via globs testlist = glob.glob(unknown[0]) self.tests += testlist unknown = unknown[1:] else: self.tests.append(unknown[0]) unknown = unknown[1:] if self.args.metadata: core.print_info("Using Metadata loader for tests and filtering") metadata_tests = filtertests(backend="mtf", location=os.getcwd(), linters=False, tests=[], tags=[], relevancy="") tests_dict = [x[metadata_common.SOURCE] for x in metadata_tests] self.tests += tests_dict core.print_debug("Loaded tests via metadata file: %s" % tests_dict) core.print_debug("tests = {0}".format(self.tests)) core.print_debug("additionalAvocadoArg = {0}".format( self.additionalAvocadoArg))
def _tcinfo(self, testcases, header, logs=True, description=True): """ Parse testcases dics and print output for them in nicer format Main purpose is to display docstrings of testcases for failures example: def test_some(something) ''' This is line1. This is line2. :return: None ''' self.assertTrue(False, msg="This is fail reason") procudes line: desc -> This is line1. This is line2. reason -> This is fail reason :param testcases: dict of testcases :param header: str what to print as header :param logs: boolean if print logs for these testcases (default True) :param description: boolean if print description = docs strings for test class + function :return: None """ if testcases: emptydelimiter = "" harddelimiter = "------------------------" core.print_info(emptydelimiter, "{0} {1} {0}".format(harddelimiter, header)) for testcase in testcases: tcname = testcase if re.search('^[0-9]+-', testcase.get('id', "")): tcname = testcase.get('id').split("-", 1)[1] tcnameoutput = tcname splitted = re.search("(.*):(.+)\.(.+)$", tcname) if splitted: docstrcls = [] docstrtst = [] testfile, classname, fnname = splitted.groups() try: testmodule = imp.load_source("test", testfile) if getattr(testmodule, classname).__doc__: docstrcls = getattr( testmodule, classname).__doc__.strip().split("\n") if getattr(getattr(testmodule, classname), fnname).__doc__: docstrtst = getattr( getattr(testmodule, classname), fnname).__doc__.strip().split("\n") tcnameoutput = " ".join([ x for x in docstrcls + docstrtst if not re.search(":.*:", x) ]) # TODO: replace more whitespaces just by one - we should find better solution how to that tcnameoutput = ' '.join(tcnameoutput.split()) except Exception as e: core.print_debug( "(INFO) Error happen when parsing testcase name ({})" .format(tcname), e) pass core.print_info("TEST {0}: {1}".format( testcase.get('status'), tcname)) if description and tcnameoutput != tcname and tcnameoutput and tcnameoutput.strip( ): core.print_info(" desc -> {0}".format(tcnameoutput)) if testcase.get('fail_reason' ) and testcase.get('fail_reason') != "None": core.print_info(" reason -> {0}".format( testcase.get('fail_reason'))) if logs: core.print_info(" {0}".format(testcase.get('logfile'))) core.print_info(emptydelimiter)
def cli(): # unknown options are forwarded to avocado run args, unknown = mtfparser().parse_known_args() if args.version: print "0.7.7" exit(0) # uses additional arguments, set up variable asap, its used afterwards: if args.debug: os.environ['DEBUG'] = 'yes' os.environ['AVOCADO_LOG_DEBUG'] = 'yes' if args.config: os.environ['CONFIG'] = args.config if args.url: os.environ['URL'] = args.url if args.modulemdurl: os.environ['MODULEMDURL'] = args.modulemdurl core.print_debug( "Options: linter={0}, setup={1}, action={2}, module={3}".format( args.linter, args.setup, args.action, args.module)) core.print_debug( "remaining options for avocado or test files: {0}".format(unknown)) # environment usage: # read: os.environ.get('MODULE') # write: os.environ['MODULE'] # MODULE could be from: # 1. environment ... MODULE=docker etc # 2. argument ... --module=docker # 3. from config.yaml in default_module # 4. default module stored in general mtf config yaml file if os.environ.get('MODULE') is not None: # environment is the highest priority because mtf uses environment (too much) args.module = os.environ['MODULE'] if not args.module: args.module = common.get_module_type() os.environ['MODULE'] = args.module if not os.environ.get('URL'): try: common.get_config(reload=True) except mtfexceptions.DefaultConfigExc: raise mtfexceptions.DefaultConfigExc( "You have to set URL variable (via URL envar or --url parameter) in case of default config" ) supported_modules = set(common.get_backend_list() + common.list_modules_from_config()) if args.module in supported_modules: # for debug purposes, to be sure about module variables or options core.print_debug("MODULE={0}, options={1}".format( os.environ.get('MODULE'), args.module)) else: # TODO: what to do here? whats the defaults value for MODULE, do I know it? raise mtfexceptions.ModuleFrameworkException( "MODULE={0} ; we support {1}".format(os.environ.get('MODULE'), supported_modules)) core.print_debug("MODULE={0}".format(os.environ.get('MODULE'))) return args, unknown
def __init__(self, args, unknown): # choose between TESTS and ADDITIONAL ENVIRONMENT from options if args.linter: self.tests += glob.glob("{MTF_TOOLS}/{GENERIC_TEST}/*.py".format( MTF_TOOLS=metadata_common.MetadataLoaderMTF.MTF_LINTER_PATH, GENERIC_TEST=common.conf["generic"]["generic_tests"])) self.tests += glob.glob("{MTF_TOOLS}/{STATIC_LINTERS}/*.py".format( MTF_TOOLS=metadata_common.MetadataLoaderMTF.MTF_LINTER_PATH, STATIC_LINTERS=common.conf["generic"]["static_tests"])) self.args = args # parse unknow options and try to find what parameter is test while unknown: if unknown[0] in self.A_KNOWN_PARAMS_SIMPLE: self.additionalAvocadoArg.append(unknown[0]) unknown = unknown[1:] elif unknown[0].startswith("-"): if "=" in unknown[0] or len(unknown) < 2: self.additionalAvocadoArg.append(unknown[0]) unknown = unknown[1:] else: self.additionalAvocadoArg += unknown[0:2] unknown = unknown[2:] elif glob.glob(unknown[0]): # dereference filename via globs testlist = glob.glob(unknown[0]) self.tests += testlist unknown = unknown[1:] else: self.tests.append(unknown[0]) unknown = unknown[1:] if self.args.metadata: core.print_info("Using Metadata loader for tests and filtering") metadata_tests = filtertests(backend="mtf", location=os.getcwd(), linters=False, tests=[], tags=[], relevancy="") tests_dict = [x[metadata_common.SOURCE] for x in metadata_tests] self.tests += tests_dict core.print_debug("Loaded tests via metadata file: %s" % tests_dict) core.print_debug("tests = {0}".format(self.tests)) core.print_debug("additionalAvocadoArg = {0}".format( self.additionalAvocadoArg)) # Advanced filtering and testcases adding based on FMF metadata, see help if self.args.fmffilter or self.args.fmfpath: # if fmf path is empty, use actual directory if not self.args.fmfpath: self.args.fmfpath = ["."] try: import fmf except ImportError: mtfexceptions.ModuleFrameworkException( "FMF metadata format not installed on your system," " see fmf.readthedocs.io/en/latest/" "for more info (how to install)") core.print_debug( "Using FMF metadata: path - {} and filters - {}".format( self.args.fmfpath, common.conf["fmf"]["filters"] + self.args.fmffilter)) for onepath in self.args.fmfpath: tree = fmf.Tree(onepath) for node in tree.prune( False, common.conf["fmf"]["keys"], common.conf["fmf"]["names"], common.conf["fmf"]["filters"] + self.args.fmffilter): testcase = node.show(False, common.conf["fmf"]["format"], common.conf["fmf"]["values"]) core.print_debug("added test by FMF: {}".format(testcase)) self.tests.append(testcase)