def test_singleton_instance(self): '''Test whether we really have a singleton Config instance''' conf = config.get_config() assert isinstance(conf, config.Config) assert conf is config._config conf2 = config.get_config() assert conf2 is conf
def main(): '''Main entry point executed by runtask script''' # Preliminary initialization of logging, so all messages before regular # initialization can be logged to stream. logger.init_prior_config() log.info('Execution started at: %s', datetime.datetime.utcnow().strftime('%Y-%m-%d %H:%M:%S UTC')) log.debug('Using checkb %s', checkb.__version__) # parse cmdline parser = get_argparser() args = parser.parse_args() check_args(parser, vars(args)) log.debug('Parsed arguments: %s', args) arg_data = process_args(vars(args)) # create artifacts directory + subdirs try: artif_subdir = os.path.join(arg_data['artifactsdir'], 'checkb') file_utils.makedirs(artif_subdir) log.info("Task artifacts will be saved in: %s", arg_data['artifactsdir']) except OSError: log.error("Can't create artifacts directory %s", artif_subdir) raise # initialize logging level_stream = logging.DEBUG if args.debug else None logger.init(level_stream=level_stream) logpath = os.path.join(artif_subdir, 'checkb.log') logger.add_filehandler(level_file=logging.DEBUG, filelog_path=logpath) logger.remove_mem_handler() # start execution executor = Executor(arg_data) success = executor.execute() # finalize log.info('Task artifacts were saved in: %s', arg_data['artifactsdir']) if config.get_config().profile == config.ProfileName.PRODUCTION: log.info('External URL for task artifacts: %s/%s', config.get_config().artifacts_baseurl, arg_data['uuid']) log.info('Execution finished at: %s', datetime.datetime.utcnow().strftime('%Y-%m-%d %H:%M:%S UTC')) if not success: log.error('Some playbooks failed. Exiting with non-zero exit code.') sys.exit(0 if success else 1)
def test_testing_profile(self): '''By default we should have a testing profile''' assert os.getenv(config.PROFILE_VAR) == config.ProfileName.TESTING conf = config.get_config() print(conf) assert conf.profile == config.ProfileName.TESTING assert isinstance(conf, config.TestingConfig)
def setup(self, tmpdir, monkeypatch): '''Run this before every test invocation''' self.artifactsdir = tmpdir.mkdir('artifacts') self.taskdir = tmpdir.mkdir('taskdir') self.client_taskdir = tmpdir.mkdir('client_taskdir') self.arg_data = { 'artifactsdir': self.artifactsdir.strpath, 'taskdir': self.taskdir.strpath, 'item': 'htop-2.0.2-4.fc27', 'type': 'koji_build', 'arch': 'noarch', 'debug': False, 'local': True, 'libvirt': False, 'ssh': False, 'ssh_privkey': None, } self.playbook_name = 'tests.yml' self.playbook = self.taskdir.join(self.playbook_name) self.playbook.write(PLAYBOOK) self.ipaddr = '127.0.0.1' self.executor = executor.Executor(self.arg_data) self.playbook_vars = self.executor._create_playbook_vars( self.playbook_name) monkeypatch.setattr(config, '_config', None) self.conf = config.get_config() self.conf.client_taskdir = self.client_taskdir.strpath
def test_devel_profile_empty_config(self, tmpconffile): '''Empty file and no envvar should return development profile''' # create an empty config file tmpconffile.write('# just a comment') conf = config.get_config() assert conf.profile == config.ProfileName.DEVELOPMENT
def get_latest(cls, distro, release, flavor, arch='x86_64', imagesdir=None): """Search for the most recent image available on the system. :param distro: distro of the image (e.g. ``fedora``) :param release: release of the image (e.g. ``23``) :param flavor: flavor of the image (e.g. ``minimal``) :param imagesdir: absolute path to directory containing the images, path from config is used if None :param arch: architecture of the image :return: file:// url of the latest image available :raises CheckbImageError: if no such image for given release and flavor was found """ if not imagesdir: imagesdir = config.get_config().imagesdir latest_metadata = cls.get_latest_metadata(distro, release, flavor, arch, imagesdir) if not latest_metadata: raise exc.CheckbImageNotFoundError( 'No image for DISTRO: %s, RELEASE: %s, FLAVOR: %s, ARCH: %s in %s' % (distro, release, flavor, arch, imagesdir)) else: url = "file://" + os.path.join(imagesdir, latest_metadata['filename']) log.debug("Found image: %s" % url) return url
def _spawn_vm(self, uuid, playbook_vars): '''Spawn a virtual machine using testcloud. :param str uuid: unicode string uuid for the task being executed :param dict playbook_vars: a vars dict created by :meth:`_create_playbook_vars` :returns: str ip address of spawned vm ''' log.info('Spawning disposable client') env = image_utils.devise_environment(self.arg_data, playbook_vars) self.task_vm = vm.TestCloudMachine(uuid) retries = config.get_config().spawn_vm_retries while retries > 0: retries -= 1 try: self.task_vm.prepare(**env) self.task_vm.wait_for_port(22) log.debug('Disposable client (%s %s) ready', self.task_vm.instancename, self.task_vm.ipaddr) return self.task_vm.ipaddr except vm.TestcloudInstanceError as e: if retries <= 0: raise exc.CheckbMinionError( 'Disposable client failed ' 'to boot: %s', e) else: log.warning( 'Disposable client failed to boot, retrying: ' '%s', e) self.task_vm.teardown()
def _get_vault_secrets(self, taskdir): '''Load secrets from the Vault server and store them in a file :param str taskdir: path to the directory with test suite (on overlord) :return: a filename with decrypted secrets ''' cfg = config.get_config() secrets = {} if cfg.vault_enabled: task_repo_url = resultsdb_directive.git_origin_url(taskdir) if task_repo_url: try: session = file_utils._get_session() r = session.get( "%s/buckets" % cfg.vault_server, auth=(cfg.vault_username, cfg.vault_password), ) except requests.exceptions.RequestException as e: log.error("Connection to Vault server failed. %s", e) r = None if r and r.ok: data = r.json()['data'] valid_buckets = [] re_enabler = re.compile(r'checkb_enable\((.*?)\)') for b in data: desc = b['description'] if not desc: continue enabled_for = ', '.join(re_enabler.findall(desc)) if not task_repo_url in enabled_for: continue valid_buckets.append(b) for b in valid_buckets: secrets[b['uuid']] = b['secrets'] elif r and not r.ok: log.error("Could not get data from vault. %r, %r", r.status_code, r.reason) if config.get_config().profile == config.ProfileName.TESTING: return secrets fd, fname = tempfile.mkstemp(prefix='checkb_secrets') os.close(fd) with open(fname, 'w') as fd: fd.write(json.dumps(secrets, indent=2, sort_keys=True)) return fname
def test_profile_override_in_conf(self, tmpconffile): '''No envvar and something specified in config file should respect the config file choice''' # set a profile in config file tmpconffile.write('profile: %s' % config.ProfileName.PRODUCTION) conf = config.get_config() assert conf.profile == config.ProfileName.PRODUCTION
def test_devel_profile_no_config(self, monkeypatch): '''When there are no config files and no envvar, the profile should be set to 'development' ''' self.unset_profile(monkeypatch) self.disable_create_dirs(monkeypatch) # make sure we don't find any config files monkeypatch.setattr(config, 'CONF_DIRS', []) conf = config.get_config() assert conf.profile == config.ProfileName.DEVELOPMENT
def setup(self, monkeypatch): '''Run before every method''' conf = config.get_config() monkeypatch.setattr(conf, 'log_file_enabled', True) # remember the list of root handlers self.root_handlers = logging.getLogger().handlers # remember the level of the stream handler self.stream_level = (logger.stream_handler.level if logger.stream_handler else None)
def test_profile_envvar(self, monkeypatch, tmpconffile): '''If both envvar and config profile are specified, envvar should have the preference''' # set a profile in envvar monkeypatch.setenv(config.PROFILE_VAR, config.ProfileName.PRODUCTION) # set a profile in config file tmpconffile.write('profile: %s' % config.ProfileName.DEVELOPMENT) conf = config.get_config() assert conf.profile == config.ProfileName.PRODUCTION
def _prepare_image(self, distro, release, flavor, arch): '''Use testcloud to prepare an image for local booting :param str distro: Distro to use in image discovery :param str release: Distro's release to use in image discovery :param str flavor: base-image flavor to use in image discovery :param str arch: arch to use in image discovery :raises CheckbImageNotFoundError: when base image of the required type is not found :raises CheckbImageError: for errors in preparing the image with testcloud ''' tc_image = None try: if config.get_config().force_imageurl: img_url = config.get_config().imageurl else: log.debug( "Looking for image with DISTRO: %s, RELEASE: %s, FLAVOR: %s, ARCH: %s" % (distro, release, flavor, arch)) img_url = ImageFinder.get_latest(distro=distro, release=release, flavor=flavor, arch=arch) except exc.CheckbImageNotFoundError as e: log.error(e) raise log.debug("Preparing image {} for task {}".format(img_url, self.uuid)) try: tc_image = image.Image(img_url) # symlink the image instead of copying it to the testcloud dir, because our user only # expects image handling in checkb dirs, and we remove all minion instances # immediately after task execution anyway tc_image.prepare(copy=False) except TestcloudImageError as e: log.exception(e) raise exc.CheckbImageError( "There was an error while preparing the " "testcloud image", e) return tc_image
def test_invalid_level(self, monkeypatch): '''Invalid log level in config file should be reverted to default''' conf = config.get_config() default_level = conf.log_level_stream monkeypatch.setattr(conf, 'log_level_stream', 'INVALID') logger.init() assert logging.getLevelName(logger.stream_handler.level) != 'INVALID' assert logging.getLevelName( logger.stream_handler.level) == default_level
def get_all_filenames(cls, imagesdir=None): """Get list of images present on the system. :param imagesdir: absolute path to directory containing the images, path from config is used if None """ if not imagesdir: imagesdir = config.get_config().imagesdir return os.listdir(imagesdir)
def setup_method(self, method): self.arg_data = { 'item': 'htop-2.0.2-4.fc20', 'type': 'koji_build', 'arch': 'i686', } self.playbook_vars = { 'checkb_match_host_distro': True, 'checkb_match_host_release': True, 'checkb_match_host_arch': True, } self.cfg = config.get_config()
def test_merge_config(self): '''Test _merge_config() function''' conf = config.get_config() # let's try to override 'tmpdir' # the only exception is 'profile', it must not be overridden assert hasattr(conf, 'tmpdir') assert hasattr(conf, 'profile') old_profile = conf.profile file_config = {'tmpdir': '/a/road/to/nowhere', 'profile': 'invalid'} config._merge_config(conf, file_config) assert conf.tmpdir == '/a/road/to/nowhere' assert conf.profile == old_profile
def __init__(self, arch=None, filelist=None, resolve_baseurl=True, resolve_retry=3): ''' :param str arch: architecture for which to adjust repo URLs. By default it refers to the architecture of the current machine. It's always converted to basearch. :param filelist: list of config files to read information from. The first available config file is used. If ``None``, then the default list of locations is used. :type filelist: iterable of str :param bool resolve_baseurl: if baseurl is a known redirect, resolve it for each section during initialization. If this is ``False``, you must call :meth:`_switch_to_mirror` manually. :param int resolve_retry: how many tries to retry resolving the URL for each section in case the network request fails :raise CheckbConfigError: if no YUM repositories data is found (empty or non-existent config file). It's not raised if you specifically request no data to load (``filelist=[]``). :raise CheckbRemoteError: if url resolving fails ''' if config.get_config().profile == config.ProfileName.TESTING: resolve_baseurl = False self.arch = arch_utils.basearch(arch) self.filelist = (filelist if filelist is not None else [ os.path.join(confdir, 'yumrepoinfo.conf') for confdir in config.CONF_DIRS ]) self.resolve_retry = resolve_retry self.parser = PARSER_CLASS(defaults={'arch': self.arch}) if not self.filelist: # no data should be loaded return self._read() if not self.repos(): msg = ("No YUM repo definitions found in the following locations: " "%s" % self.filelist) log.critical(msg) raise exc.CheckbConfigError(msg) self._adjust_baseurl() # download.fp.o is a known redirect if resolve_baseurl and ('download.fedoraproject.org' in self.parser.get('DEFAULT', 'baseurl')): self._switch_to_mirror()
def setup(self, monkeypatch): '''Run this before every test invocation''' self.cd = check.CheckDetail( item='foo_bar', report_type=check.ReportType.KOJI_BUILD, outcome='NEEDS_INSPECTION', note='foo_bar note', output=["foo\nbar"], keyvals={"foo": "moo1", "bar": "moo2"}, checkname='qa.test_resultsdb_report', ) self.yaml = check.export_YAML(self.cd) self.ref_input = {'results': self.yaml} self.ref_arg_data = { 'resultsdb_job_id': 1, 'jobid': 'all/123', 'uuid': 'c25237a4-b6b3-11e4-b98a-3c970e018701', 'artifactsdir': '/some/directory/', 'task': '/taskdir', 'item': 'firefox-45.0.2-1.fc23' } self.ref_resultdata = {u'id': 1234} self.ref_jobid = 1234 self.ref_uuid = 'c25237a4-b6b3-11e4-b98a-3c970e018701' self.ref_refurl = u'http://example.com/%s' % self.cd.checkname self.ref_jobdata = {u'end_time': None, u'href': u'http://127.0.0.1/api/v2.0/jobs/%d' % self.ref_jobid, u'id': self.ref_jobid, u'name': self.cd.checkname, u'ref_url': self.ref_refurl, u'results': [], u'results_count': 0, u'start_time': None, u'status': u'SCHEDULED'} self.stub_rdb = mock.Mock(**{ 'get_testcase.return_value': {}, 'create_job.return_value': self.ref_jobdata, 'create_result.return_value': self.ref_resultdata, }) self.test_rdb = resultsdb_directive.ResultsdbDirective(self.stub_rdb) # while it appears useless, this actually sets config in several tests monkeypatch.setattr(config, '_config', None) self.conf = config.get_config() self.conf.report_to_resultsdb = True monkeypatch.setattr(configparser, 'ConfigParser', StubConfigParser)
def test_check_sanity_runtask_mode(self): '''Should crash for unknown runtask mode names''' conf = config.get_config() for attr, value in vars(config.RuntaskModeName).items(): if attr.startswith('_'): continue conf.runtask_mode = value config._check_sanity(conf) conf.runtask_mode = 'invalid runtask mode name' with pytest.raises(exc.CheckbConfigError): config._check_sanity(conf)
def __init__(self, resultsdb=None): self.resultsdb = resultsdb conf = config.get_config() self.masterurl = conf.checkb_master self.task_stepname = conf.buildbot_task_step self.execdb_server = "%s/jobs" % conf.execdb_server self.artifacts_baseurl = conf.artifacts_baseurl if self.resultsdb is None: self.resultsdb = resultsdb_api.ResultsDBapi(conf.resultsdb_server) self._ensured_testcases = []
def get_latest_metadata(cls, distro, release, flavor, arch='x86_64', imagesdir=None): """Search for the most recent image available on the system. :param distro: distro of the image (e.g. ``fedora``) :param release: release of the image (e.g. ``23``) :param flavor: flavor of the image (e.g. ``minimal``) :param imagesdir: absolute path to directory containing the images, path from config is used if None :param arch: arch of the image (e.g. 'x86_64') :return: metadata of the most recent image :rtype: dict {'date': str, 'version': str, 'release': str, 'arch': str, 'filename': str} """ if not imagesdir: imagesdir = config.get_config().imagesdir release = str(release) # The pattern is: YYMMDD_HHMM-DISTRO-RELEASE-FLAVOR-ARCH.(qcow2|raw|img) # For example: 160301_1030-fedora-23-checkb_cloud-x86_64.img pattern = re.compile( r'^([0-9]{6}_[0-9]{4})-(.*?)-(.*?)-(.*?)-(.*?)\.(qcow2|raw|img)$') images = [] for filename in cls.get_all_filenames(imagesdir): m = pattern.match(filename) if m: images.append({ 'timestamp': m.group(1), 'distro': m.group(2), 'release': m.group(3), 'flavor': m.group(4), 'arch': m.group(5), 'filename': filename }) filtered = [ i for i in images if i['distro'] == distro and i['release'] == release and i['flavor'] == flavor and i['arch'] == arch ] if not filtered: return None else: return sorted(filtered, key=lambda i: i['timestamp'])[-1]
def test_conf_file_override(self, monkeypatch, tmpconffile): '''Even if both envvar and config profile are specified, we should still respect config file option values''' # set a profile in envvar monkeypatch.setenv(config.PROFILE_VAR, config.ProfileName.PRODUCTION) # set a profile in config file tmpconffile.write(''' profile: %s cachedir: /some/path ''' % config.ProfileName.DEVELOPMENT) conf = config.get_config() assert conf.cachedir == '/some/path' assert conf.cachedir != config.ProductionConfig().cachedir
def test_staging_or_production(self, monkeypatch): '''Client should be correctly created when requested staging in config or not''' monkeypatch.setattr(config, '_config', None) conf = config.get_config() assert conf.bodhi_staging is False bu = bodhi_utils.BodhiUtils() prod_url = bu.client.base_url conf.bodhi_staging = True bu_stg = bodhi_utils.BodhiUtils() stg_url = bu_stg.client.base_url assert prod_url != stg_url assert prod_url == bodhi.client.bindings.BASE_URL assert stg_url == bodhi.client.bindings.STG_BASE_URL
def test_no_load_config(self, monkeypatch): '''We shouldn't load config files in the testing profile''' # we will do that by checking that appropriate disk-touching methods # don't get called def _search_dirs_raise(x, y): assert False, 'This must not be called' def _load_file_raise(x): assert False, 'This must not be called' monkeypatch.setattr(config, '_search_dirs', _search_dirs_raise) monkeypatch.setattr(config, '_load_file', _load_file_raise) conf = config.get_config() assert isinstance(conf, config.Config)
def __init__(self, client=None): '''Create a new BodhiUtils instance. :param client: custom :class:`Bodhi2Client` instance. If ``None``, a default Bodhi2Client instance is used. ''' self.config = config.get_config() if not client: self.client = bodhi.client.bindings.BodhiClient( staging=self.config.bodhi_staging) log.debug('Created Bodhi client to: %s', self.client.base_url) # automatically retry failed requests (HTTP 5xx and similar) self.client.retries = 10 else: self.client = client
def test_config_reporting_disabled(self): """Checks config option that disables reporting.""" conf = config.get_config() conf.report_to_resultsdb = False yaml = self.test_rdb.process(self.ref_input, self.ref_arg_data) cds = check.import_YAML(yaml) my_cd = check.import_YAML(check.export_YAML(self.cd)) # return value should be the same YAML assert len(cds) == 1 assert cds[0].__dict__ == my_cd[0].__dict__ # no call should have been made assert len(self.stub_rdb.mock_calls) == 0 config._config = None
def process(self, params, arg_data): output_data = {} valid_actions = ['download'] action = params.get('action', None) if action not in valid_actions: raise CheckbDirectiveError( '%s is not a valid command for bodhi directive' % action) if 'arch' not in params or 'target_dir' not in params: detected_args = ', '.join(params.keys()) raise exc.CheckbDirectiveError( "The bodhi directive requires 'arch' and 'target_dir' as an " "argument. Detected arguments: %s" % detected_args) # convert str to list if isinstance(params['arch'], basestring): params['arch'] = [params['arch']] if action == 'download': if 'update_id' not in params or 'arch' not in params: detected_args = ', '.join(params.keys()) raise CheckbDirectiveError( "The bodhi directive 'download' requires both 'update_id' and " "'arch' arguments. Detected arguments: %s" % detected_args) target_dir = params['target_dir'] updateid = params['update_id'] if 'all' in params['arch']: arches = config.get_config().supported_arches + ['noarch'] else: arches = params['arch'] if arches and ('noarch' not in arches): arches.append('noarch') src = params.get('src', False) log.info("getting rpms for update %s (%s) and downloading to %s", updateid, arches, target_dir) output_data['downloaded_rpms'] = self.action_download( updateid, arches, src, target_dir) return output_data
def init(level_stream=None, stream=True, syslog=False): """Initialize Checkb logging. Note: Since this touches the root logger, it should be called only when Checkb is run as the main program (through its runner), not when it is used as a library. :param int level_stream: level of stream logging as defined in :mod:`logging`. If ``None``, a default level from config file is used. :param bool stream: enable logging to process stream (stderr) :param bool syslog: enable logging to syslog """ # We import checkb.config here because import from beginning # of this module causes problems with cyclic dependencies from checkb import config conf = config.get_config() if level_stream is None: level_stream = conf.log_level_stream _create_handlers() rootlogger = logging.getLogger() sys.excepthook = _log_excepthook if stream: _set_level(stream_handler, level_stream, "log_level_stream") if stream_handler.level <= logging.DEBUG: stream_handler.setFormatter(_formatter_full) rootlogger.addHandler(stream_handler) log.debug("Stream logging enabled with level: %s", logging.getLevelName(stream_handler.level)) else: rootlogger.removeHandler(stream_handler) if syslog: _create_handlers(syslog=True) rootlogger.addHandler(syslog_handler) log.debug("Syslog logging enabled with level: %s", logging.getLevelName(syslog_handler.level)) else: rootlogger.removeHandler(syslog_handler) add_filehandler()
def _get_client_ipaddr(self): '''Get an IP address of the machine the task is going to be executed on. :returns: str ip address of the machine or None if the machine is yet to be created ''' # when running remotely, run directly over ssh, instead of using # libvirt persistent = False runtask_mode = config.get_config().runtask_mode if runtask_mode == config.RuntaskModeName.LOCAL: self.run_remotely = False elif runtask_mode == config.RuntaskModeName.LIBVIRT: self.run_remotely = True else: assert False, 'This should never occur' if self.arg_data['local']: log.debug("Forcing local execution (option --local)") self.run_remotely = False elif self.arg_data['libvirt']: log.debug("Forcing remote execution (option --libvirt)") self.run_remotely = True persistent = False elif self.arg_data['ssh']: log.debug('Forcing remote execution (option --ssh)') self.run_remotely = True persistent = True log.debug('Execution mode: %s', 'remote' if self.run_remotely else 'local') ipaddr = '127.0.0.1' if self.run_remotely: ipaddr = self.arg_data['machine'] if persistent else None return ipaddr