def osbs_for_capture(tmpdir): kwargs = { 'build_json_dir': 'inputs', 'openshift_url': OPENSHIFT_URL, 'namespace': TEST_OCP_NAMESPACE } with NamedTemporaryFile(mode="wt") as fp: config = dedent("""\ [general] build_json_dir = {build_json_dir} [default] openshift_url = {openshift_url} use_auth = false namespace = {namespace} """) fp.write(config.format(**kwargs)) fp.flush() dummy_config = Configuration(fp.name, conf_section='default') osbs = OSBS(dummy_config) setup_json_capture(osbs, osbs.os_conf, str(tmpdir)) return osbs
def cmd_build_source_container(args): if args.instance is None: conf_section = DEFAULT_CONF_SOURCE_SECTION else: conf_section = args.instance os_conf = Configuration(conf_file=args.config, conf_section=conf_section, cli_args=args) osbs = OSBS(os_conf) build_kwargs = { 'user': osbs.os_conf.get_user(), 'target': osbs.os_conf.get_koji_target(), 'scratch': args.scratch, 'signing_intent': args.signing_intent, 'sources_for_koji_build_nvr': args.sources_for_koji_build_nvr, 'sources_for_koji_build_id': args.sources_for_koji_build_id, 'component': args.component, } if args.userdata: build_kwargs['userdata'] = json.loads(args.userdata) pipeline_run = osbs.create_source_container_pipeline_run(**build_kwargs) print_output(pipeline_run, export_metadata_file=args.export_metadata_file) return_val = -1 if pipeline_run.has_succeeded(): return_val = 0 return return_val
def osbs106(openshift): with NamedTemporaryFile(mode="wt") as fp: fp.write(""" [general] build_json_dir = {build_json_dir} openshift_required_version = 1.0.6 [default] openshift_url = / registry_uri = registry.example.com sources_command = fedpkg sources vendor = Example, Inc. build_host = localhost authoritative_registry = registry.example.com distribution_scope = authoritative-source-only koji_root = http://koji.example.com/kojiroot koji_hub = http://koji.example.com/kojihub use_auth = false build_from = image:buildroot:latest """.format(build_json_dir="inputs")) fp.flush() dummy_config = Configuration(fp.name) osbs = OSBS(dummy_config, dummy_config) osbs.os = openshift return osbs
def test_create_build_config_already_running(self): config = Configuration() osbs = OSBS(config, config) build_json = { 'apiVersion': osbs.os_conf.get_openshift_api_version(), 'metadata': { 'name': 'build', 'labels': { 'git-repo-name': 'reponame', 'git-branch': 'branch', }, }, } existing_build_json = copy.deepcopy(build_json) existing_build_json['metadata']['name'] = 'existing-build' build_request = flexmock(render=lambda: build_json, is_auto_instantiated=lambda: False, scratch=False) (flexmock(osbs).should_receive('_get_existing_build_config').once(). and_return(existing_build_json)) (flexmock(osbs).should_receive( '_get_running_builds_for_build_config').once().and_return([ flexmock(status='Running', get_build_name=lambda: 'build-1'), ])) with pytest.raises(OsbsException): osbs._create_build_config_and_build(build_request)
def test_create_build_config_create(self): config = Configuration() osbs = OSBS(config, config) build_json = { 'apiVersion': osbs.os_conf.get_openshift_api_version(), 'metadata': { 'name': 'build', 'labels': { 'git-repo-name': 'reponame', 'git-branch': 'branch', }, }, } build_request = flexmock(render=lambda: build_json, is_auto_instantiated=lambda: False, scratch=False) (flexmock(osbs).should_receive( '_get_existing_build_config').once().and_return(None)) (flexmock(osbs.os).should_receive('create_build_config').with_args( json.dumps(build_json)).once().and_return( flexmock(json=lambda: {'spam': 'maps'}))) (flexmock(osbs.os).should_receive( 'start_build').with_args('build').once().and_return( flexmock(json=lambda: {'spam': 'maps'}))) build_response = osbs._create_build_config_and_build(build_request) assert build_response.json == {'spam': 'maps'}
def osbs(request, openshift): kwargs = request.param['kwargs'] or {} kwargs.setdefault('build_json_dir', 'inputs') kwargs.setdefault('additional_general', '') with NamedTemporaryFile(mode="wt") as fp: config = dedent("""\ [general] build_json_dir = {build_json_dir} {additional_general} [default] openshift_url = / flatpak_base_image = registry.fedoraproject.org/fedora:latest can_orchestrate = true use_auth = false build_from = image:buildroot:latest """) if request.param['additional_config'] is not None: config += request.param['additional_config'] config += '\n' fp.write(config.format(**kwargs)) fp.flush() dummy_config = Configuration(fp.name) osbs = OSBS(dummy_config, dummy_config) osbs.os = openshift return osbs
def get_openshift_session(workflow, fallback): config = get_openshift(workflow, fallback) namespace = get_build_json().get('metadata', {}).get('namespace', None) from osbs.api import OSBS from osbs.conf import Configuration config_kwargs = { 'verify_ssl': not config.get('insecure', False), 'namespace': namespace, 'use_auth': False, 'conf_file': None, 'openshift_url': config['url'], 'build_json_dir': config.get('build_json_dir') } if config.get('auth'): krb_keytab_path = config['auth'].get('krb_keytab_path') if krb_keytab_path: config_kwargs['kerberos_keytab'] = krb_keytab_path krb_principal = config['auth'].get('krb_principal') if krb_principal: config_kwargs['kerberos_principal'] = krb_principal krb_cache_path = config['auth'].get('krb_cache_path') if krb_cache_path: config_kwargs['kerberos_ccache'] = krb_cache_path ssl_certs_dir = config['auth'].get('ssl_certs_dir') if ssl_certs_dir: config_kwargs['client_cert'] = os.path.join(ssl_certs_dir, 'cert') config_kwargs['client_key'] = os.path.join(ssl_certs_dir, 'key') config_kwargs['use_auth'] = config['auth'].get('enable', False) osbs_conf = Configuration(**config_kwargs) return OSBS(osbs_conf, osbs_conf)
def get_openshift_session(config, namespace): from osbs.api import OSBS from osbs.conf import Configuration config_kwargs = { 'verify_ssl': not config.openshift.get('insecure', False), 'namespace': namespace, 'use_auth': False, 'conf_file': None, 'openshift_url': config.openshift['url'], } if config.openshift.get('auth'): krb_keytab_path = config.openshift['auth'].get('krb_keytab_path') if krb_keytab_path: config_kwargs['kerberos_keytab'] = krb_keytab_path krb_principal = config.openshift['auth'].get('krb_principal') if krb_principal: config_kwargs['kerberos_principal'] = krb_principal krb_cache_path = config.openshift['auth'].get('krb_cache_path') if krb_cache_path: config_kwargs['kerberos_ccache'] = krb_cache_path ssl_certs_dir = config.openshift['auth'].get('ssl_certs_dir') if ssl_certs_dir: config_kwargs['client_cert'] = os.path.join(ssl_certs_dir, 'cert') config_kwargs['client_key'] = os.path.join(ssl_certs_dir, 'key') config_kwargs['use_auth'] = config.openshift['auth'].get( 'enable', False) osbs_conf = Configuration(**config_kwargs) return OSBS(osbs_conf)
def run(self): """ run the plugin """ try: build_json = json.loads(os.environ["BUILD"]) except KeyError: self.log.error("No $BUILD env variable. Probably not running in build container") raise metadata = build_json.get("metadata", {}) labels = metadata.get("labels", {}) buildconfig = labels["buildconfig"] is_rebuild = labels.get(self.label_key) == self.label_value self.log.info("This is a rebuild? %s", is_rebuild) if not is_rebuild: # Update the BuildConfig metadata so the next Build # instantiated from it is detected as being an automated # rebuild kwargs = {} if 'namespace' in metadata: kwargs['namespace'] = metadata['namespace'] # FIXME: remove `openshift_uri` once osbs-client is released osbs_conf = Configuration(conf_file=None, openshift_uri=self.url, openshift_url=self.url, use_auth=self.use_auth, verify_ssl=self.verify_ssl) osbs = OSBS(osbs_conf, osbs_conf) labels = {self.label_key: self.label_value} osbs.set_labels_on_build_config(buildconfig, labels, **kwargs) return is_rebuild
def test_get_existing_build_config_by_name(self): build_config = { 'metadata': { 'name': 'name', 'labels': { 'git-repo-name': 'reponame', 'git-branch': 'branch', } }, } existing_build_config = copy.deepcopy(build_config) existing_build_config['_from'] = 'from-name' config = Configuration() osbs = OSBS(config, config) (flexmock( osbs.os).should_receive('get_build_config_by_labels').with_args([ ('git-repo-name', 'reponame'), ('git-branch', 'branch') ]).once().and_raise(OsbsException)) (flexmock(osbs.os).should_receive('get_build_config').with_args( 'name').once().and_return(existing_build_config)) actual_build_config = osbs._get_existing_build_config(build_config) assert actual_build_config == existing_build_config assert actual_build_config['_from'] == 'from-name'
def get_cluster_info(self, cluster, platform): kwargs = deepcopy(self.config_kwargs) kwargs['conf_section'] = cluster.name if self.osbs_client_config: kwargs['conf_file'] = os.path.join(self.osbs_client_config, 'osbs.conf') conf = Configuration(**kwargs) osbs = OSBS(conf, conf) try: current_builds = self.get_current_builds(osbs) except OsbsException as e: # If the build is canceled reraise the error if isinstance(e.cause, BuildCanceledException): raise e self.log.exception("Error occurred while listing builds on %s", cluster.name) return ClusterInfo(cluster, platform, osbs, self.UNREACHABLE_CLUSTER_LOAD) load = current_builds / cluster.max_concurrent_builds self.log.debug( 'enabled cluster %s for platform %s has load %s and active builds %s/%s', cluster.name, platform, load, current_builds, cluster.max_concurrent_builds) return ClusterInfo(cluster, platform, osbs, load)
def test_scratch_param_to_create_build(self): config = Configuration() osbs = OSBS(config, config) class MockParser(object): labels = {'Name': 'fedora23/something'} baseimage = 'fedora23/python' kwargs = { 'git_uri': TEST_GIT_URI, 'git_ref': TEST_GIT_REF, 'git_branch': TEST_GIT_BRANCH, 'user': TEST_USER, 'component': TEST_COMPONENT, 'target': TEST_TARGET, 'architecture': TEST_ARCH, 'yum_repourls': None, 'koji_task_id': None, 'scratch': True, } (flexmock(utils).should_receive('get_df_parser').with_args( TEST_GIT_URI, TEST_GIT_REF, git_branch=TEST_GIT_BRANCH).and_return(MockParser())) (flexmock(osbs).should_receive( '_create_scratch_build').once().and_return( flexmock(json=lambda: {'spam': 'maps'}))) (flexmock(osbs.os).should_receive('create_build_config').never()) (flexmock(osbs.os).should_receive('update_build_config').never()) build_response = osbs.create_build(**kwargs) assert build_response.json() == {'spam': 'maps'}
def test_create_build_config_label_mismatch(self): config = Configuration() osbs = OSBS(config, config) build_json = { 'apiVersion': osbs.os_conf.get_openshift_api_version(), 'metadata': { 'name': 'build', 'labels': { 'git-repo-name': 'reponame', 'git-branch': 'branch', }, }, } existing_build_json = copy.deepcopy(build_json) existing_build_json['metadata']['name'] = 'build' existing_build_json['metadata']['labels'][ 'git-repo-name'] = 'reponame2' existing_build_json['metadata']['labels']['git-branch'] = 'branch2' build_request = flexmock(render=lambda: build_json, is_auto_instantiated=lambda: False, scratch=False) (flexmock(osbs).should_receive('_get_existing_build_config').once(). and_return(existing_build_json)) with pytest.raises(OsbsValidationException) as exc: osbs._create_build_config_and_build(build_request) assert 'Git labels collide' in str(exc.value)
def test_verify_no_running_builds_zero(self): config = Configuration() osbs = OSBS(config, config) (flexmock(osbs).should_receive('_get_running_builds_for_build_config'). with_args('build_config_name').once().and_return([])) osbs._verify_no_running_builds('build_config_name')
def setup_osbs_api(self): metadata = get_build_json().get("metadata", {}) osbs_conf = Configuration(conf_file=None, openshift_url=self.url, use_auth=self.use_auth, verify_ssl=self.verify_ssl, build_json_dir=self.build_json_dir, namespace=metadata.get('namespace', None)) self.osbs = OSBS(osbs_conf, osbs_conf)
def __init__(self, tasker, workflow, kojihub, url, verify_ssl=True, use_auth=True, koji_ssl_certs=None, koji_proxy_user=None, koji_principal=None, koji_keytab=None, metadata_only=False, blocksize=None, target=None, poll_interval=5): """ constructor :param tasker: DockerTasker instance :param workflow: DockerBuildWorkflow instance :param kojihub: string, koji hub (xmlrpc) :param url: string, URL for OSv3 instance :param verify_ssl: bool, verify OSv3 SSL certificate? :param use_auth: bool, initiate authentication with OSv3? :param koji_ssl_certs: str, path to 'cert', 'ca', 'serverca' :param koji_proxy_user: str, user to log in as (requires hub config) :param koji_principal: str, Kerberos principal (must specify keytab) :param koji_keytab: str, keytab name (must specify principal) :param metadata_only: bool, whether to omit the 'docker save' image :param blocksize: int, blocksize to use for uploading files :param target: str, koji target :param poll_interval: int, seconds between Koji task status requests """ super(KojiPromotePlugin, self).__init__(tasker, workflow) self.kojihub = kojihub self.koji_ssl_certs = koji_ssl_certs self.koji_proxy_user = koji_proxy_user self.koji_principal = koji_principal self.koji_keytab = koji_keytab self.metadata_only = metadata_only self.blocksize = blocksize self.target = target self.poll_interval = poll_interval self.namespace = get_build_json().get('metadata', {}).get('namespace', None) osbs_conf = Configuration(conf_file=None, openshift_uri=url, use_auth=use_auth, verify_ssl=verify_ssl, namespace=self.namespace) self.osbs = OSBS(osbs_conf, osbs_conf) self.build_id = None self.pullspec_image = None
def test_create_build_config_bad_version(self): config = Configuration() osbs = OSBS(config, config) build_json = {'apiVersion': 'spam'} build_request = flexmock(render=lambda: build_json, is_auto_instantiated=lambda: False) with pytest.raises(OsbsValidationException): osbs._create_build_config_and_build(build_request)
def __init__(self, tasker, workflow, kojihub, url, build_json_dir, koji_upload_dir, verify_ssl=True, use_auth=True, koji_ssl_certs_dir=None, koji_proxy_user=None, koji_principal=None, koji_keytab=None, blocksize=None): """ constructor :param tasker: DockerTasker instance :param workflow: DockerBuildWorkflow instance :param kojihub: string, koji hub (xmlrpc) :param url: string, URL for OSv3 instance :param build_json_dir: str, path to directory with input json :param koji_upload_dir: str, path to use when uploading to hub :param verify_ssl: bool, verify OSv3 SSL certificate? :param use_auth: bool, initiate authentication with OSv3? :param koji_ssl_certs_dir: str, path to 'cert', 'ca', 'serverca' :param koji_proxy_user: str, user to log in as (requires hub config) :param koji_principal: str, Kerberos principal (must specify keytab) :param koji_keytab: str, keytab name (must specify principal) :param blocksize: int, blocksize to use for uploading files """ super(KojiUploadPlugin, self).__init__(tasker, workflow) self.kojihub = kojihub self.koji_ssl_certs_dir = koji_ssl_certs_dir self.koji_proxy_user = koji_proxy_user self.koji_principal = koji_principal self.koji_keytab = koji_keytab self.blocksize = blocksize self.build_json_dir = build_json_dir self.koji_upload_dir = koji_upload_dir self.namespace = get_build_json().get('metadata', {}).get('namespace', None) osbs_conf = Configuration(conf_file=None, openshift_uri=url, use_auth=use_auth, verify_ssl=verify_ssl, build_json_dir=self.build_json_dir, namespace=self.namespace) self.osbs = OSBS(osbs_conf, osbs_conf) self.build_id = None self.pullspec_image = None
def osbs(self): """Handler of OSBS object""" if not self._osbs: os_conf = Configuration() build_conf = Configuration() if self.opts.get('scratch'): os_conf = Configuration(conf_section='scratch') build_conf = Configuration(conf_section='scratch') self._osbs = OSBS(os_conf, build_conf) assert self._osbs return self._osbs
def run(self): metadata = get_build_json().get("metadata", {}) kwargs = {} # FIXME: remove `openshift_uri` once osbs-client is released osbs_conf = Configuration(conf_file=None, openshift_uri=self.url, openshift_url=self.url, use_auth=self.use_auth, verify_ssl=self.verify_ssl, build_json_dir=self.build_json_dir, namespace=metadata.get('namespace', None)) osbs = OSBS(osbs_conf, osbs_conf) imagestream = None try: imagestream = osbs.get_image_stream(self.imagestream) except OsbsResponseException: if self.insecure_registry is not None: kwargs['insecure_registry'] = self.insecure_registry self.log.info("Creating ImageStream %s for %s", self.imagestream, self.docker_image_repo) imagestream = osbs.create_image_stream(self.imagestream, self.docker_image_repo, **kwargs) self.log.info("Importing new tags for %s", self.imagestream) primaries = None try: primaries = self.workflow.build_result.annotations['repositories'][ 'primary'] except (TypeError, KeyError): self.log.exception( 'Unable to read primary repositories annotations') if not primaries: raise RuntimeError('Could not find primary images in workflow') failures = False for s in primaries: tag_image_name = ImageName.parse(s) tag = tag_image_name.tag try: osbs.ensure_image_stream_tag(imagestream.json(), tag) self.log.info("Imported ImageStreamTag: (%s)", tag) except OsbsResponseException: failures = True self.log.info("Could not import ImageStreamTag: (%s)", tag) if failures: raise RuntimeError( "Failed to import ImageStreamTag(s). Check logs") osbs.import_image(self.imagestream)
def get_cluster_info(self, cluster, platform): kwargs = deepcopy(self.config_kwargs) kwargs['conf_section'] = cluster.name if self.osbs_client_config: kwargs['conf_file'] = os.path.join(self.osbs_client_config, 'osbs.conf') conf = Configuration(**kwargs) osbs = OSBS(conf, conf) current_builds = self.get_current_builds(osbs) load = current_builds / cluster.max_concurrent_builds return ClusterInfo(cluster, platform, osbs, load)
def osbs(openshift, kwargs=None, additional_config=None, platform_descriptors=None): kwargs = kwargs or {} platform_descriptors = platform_descriptors or {} kwargs.setdefault('build_json_dir', 'inputs') kwargs.setdefault('registry_uri', 'registry.example.com') kwargs.setdefault('additional_general', '') with NamedTemporaryFile(mode="wt") as fp: config = dedent("""\ [general] build_json_dir = {build_json_dir} {additional_general} [default] openshift_url = / registry_uri = {registry_uri} sources_command = fedpkg sources vendor = Example, Inc. build_host = localhost authoritative_registry = registry.example.com distribution_scope = authoritative-source-only koji_root = http://koji.example.com/kojiroot koji_hub = http://koji.example.com/kojihub flatpak_base_image = registry.fedoraproject.org/fedora:latest odcs_url = https://odcs.example.com/odcs/1 pdc_url = https://pdc.example.com/rest_api/v1 use_auth = false can_orchestrate = true build_from = image:buildroot:latest """) if additional_config is not None: config += additional_config config += '\n' for platform, platform_info in platform_descriptors.items(): if not platform_info: continue config += '[platform:{0}]\n'.format(platform) for item, value in platform_info.items(): config += '{0} = {1}\n'.format(item, value) fp.write(config.format(**kwargs)) fp.flush() dummy_config = Configuration(fp.name) osbs = OSBS(dummy_config, dummy_config) osbs.os = openshift return osbs
def test_verify_no_running_builds_one(self): config = Configuration() osbs = OSBS(config, config) (flexmock(osbs).should_receive('_get_running_builds_for_build_config'). with_args('build_config_name').once().and_return([ flexmock(status='Running', get_build_name=lambda: 'build-1'), ])) with pytest.raises(OsbsException) as exc: osbs._verify_no_running_builds('build_config_name') assert str(exc.value).startswith('Build build-1 for build_config_name')
def get_plugins_with_user_data(user_params, user_data): """Get the reactor config map and derive an osbs instance from it""" from osbs.api import OSBS from osbs.conf import Configuration reactor_config_override = user_data.get('reactor_config_override') if reactor_config_override: read_yaml(json.dumps(reactor_config_override), 'schemas/config.json') osbs_conf = Configuration(build_json_dir=user_data.get('build_json_dir')) osbs = OSBS(osbs_conf, osbs_conf) return osbs.render_plugins_configuration(user_params)
def cmd_build(args): if args.instance is None: conf_section = DEFAULT_CONF_BINARY_SECTION else: conf_section = args.instance os_conf = Configuration(conf_file=args.config, conf_section=conf_section, cli_args=args) osbs = OSBS(os_conf) build_kwargs = { 'git_uri': osbs.os_conf.get_git_uri(), 'git_ref': osbs.os_conf.get_git_ref(), 'git_branch': osbs.os_conf.get_git_branch(), 'user': osbs.os_conf.get_user(), 'tag': osbs.os_conf.get_tag(), 'target': osbs.os_conf.get_koji_target(), 'yum_repourls': osbs.os_conf.get_yum_repourls(), 'dependency_replacements': osbs.os_conf.get_dependency_replacements(), 'scratch': args.scratch, 'platforms': args.platforms, 'release': args.release, 'koji_parent_build': args.koji_parent_build, 'isolated': args.isolated, 'signing_intent': args.signing_intent, 'compose_ids': args.compose_ids, 'operator_csv_modifications_url': args.operator_csv_modifications_url, } if args.userdata: build_kwargs['userdata'] = json.loads(args.userdata) if osbs.os_conf.get_flatpak(): build_kwargs['flatpak'] = True pipeline_run = osbs.create_binary_container_pipeline_run(**build_kwargs) print_output(pipeline_run, export_metadata_file=args.export_metadata_file) return_val = -1 if pipeline_run.has_succeeded(): return_val = 0 cleanup_used_resources = osbs.os_conf.get_cleanup_used_resources() if cleanup_used_resources: try: logger.info("pipeline run removed: %s", pipeline_run.remove_pipeline_run()) except OsbsResponseException: logger.error("failed to remove pipeline run %s", pipeline_run.pipeline_run_name) raise return return_val
def osbs(self): """Handler of OSBS object""" if not self._osbs: os_conf = Configuration() build_conf = Configuration() if self.opts.get('scratch'): os_conf = Configuration(conf_section='scratch') build_conf = Configuration(conf_section='scratch') self._osbs = OSBS(os_conf, build_conf) if not self._osbs: msg = 'Could not successfully instantiate `osbs`' raise ContainerError(msg) self.setup_osbs_logging() return self._osbs
def osbs_cant_orchestrate(openshift): with NamedTemporaryFile(mode="wt") as fp: fp.write(""" [general] build_json_dir = {build_json_dir} [default] openshift_url = / use_auth = false """.format(build_json_dir="inputs")) fp.flush() dummy_config = Configuration(fp.name) osbs = OSBS(dummy_config, dummy_config) osbs.os = openshift return osbs
def test_no_branch(): with NamedTemporaryFile(mode='w+') as f: f.write(""" [default] openshift_url=https://172.0.0.1:8443/ registry_uri=127.0.0.1:5000 """) f.flush() f.seek(0) with pytest.raises(OsbsException): os_conf = Configuration(conf_file=f.name, conf_section="default") osbs = OSBS(os_conf) osbs.create_binary_container_pipeline_run(git_uri="https://example.com/example.git", git_ref="master")
def osbs106(openshift): with NamedTemporaryFile(mode="wt") as fp: fp.write(""" [general] build_json_dir = {build_json_dir} openshift_required_version = 1.0.6 [default] openshift_url = / build_from = image:buildroot:latest """.format(build_json_dir="inputs")) fp.flush() dummy_config = Configuration(fp.name) osbs = OSBS(dummy_config, dummy_config) osbs.os = openshift return osbs
def osbs_binary(): with NamedTemporaryFile(mode="wt") as fp: fp.write(""" [default_binary] openshift_url = / namespace = {namespace} use_auth = false pipeline_run_path = {pipeline_run_path} reactor_config_map = rcm """.format(namespace=TEST_OCP_NAMESPACE, pipeline_run_path=TEST_PIPELINE_RUN_TEMPLATE)) # noqa E501 fp.flush() dummy_config = Configuration(fp.name, conf_section='default_binary') osbs = OSBS(dummy_config) return osbs