def greatest_component_version_by_name( component_name: str, ctx_repo_base_url: str, ctx_repo: cm.RepositoryContext=None, cache_dir: str=None, ): if not (bool(ctx_repo_base_url) ^ bool(ctx_repo)): raise ValueError('exactly one of ctx_repo_base_url, ctx_repo must be passed') if ctx_repo_base_url: logger.warning('passing ctx_repo_base_url is deprecated - pass ctx_repo instead!') ctx_repo = cm.OciRepositoryContext(baseUrl=ctx_repo_base_url) if not isinstance(ctx_repo, cm.OciRepositoryContext): raise NotImplementedError(ctx_repo) greatest_version = greatest_component_version( component_name=component_name, ctx_repo=ctx_repo, ) component_descriptor = download_component_descriptor_v2( component_name, greatest_version, ctx_repo=ctx_repo, cache_dir=cache_dir, ) return component_descriptor.component
def retrieve(name: str, version: str, ctx_base_url: str = None, out: str = None): ctx_repo = cm.OciRepositoryContext( baseUrl=ctx_base_url, componentNameMapping=cm.OciComponentNameMapping.URL_PATH, ) target_ref = product.v2._target_oci_ref_from_ctx_base_url( component_name=name, component_version=version, ctx_repo=ctx_repo, ) component_descriptor = product.v2.retrieve_component_descriptor_from_oci_ref( manifest_oci_image_ref=target_ref, absent_ok=False, ) if out: outfh = open(out, 'w') else: outfh = sys.stdout component_descriptor.to_fobj(fileobj=outfh) outfh.flush() outfh.close()
def test_component(): component = cm.Component( name='component-name', version='1.2.3', repositoryContexts=[ cm.OciRepositoryContext(baseUrl='old-ctx-url'), cm.OciRepositoryContext(baseUrl='current-ctx-url'), ], provider=None, sources=(), componentReferences=(), resources=(), labels=(), ) assert component.current_repository_ctx().baseUrl == 'current-ctx-url'
def new_from_source_descriptor( descriptor: cm.ComponentDescriptor, context_url: str, external_resources: list, local_resources: list, ): return ComponentTool.new_from_descriptor( descriptor=cm.ComponentDescriptor( meta=descriptor.meta, component=cm.Component( name=descriptor.component.name, version=descriptor.component.version, repositoryContexts=[ cm.OciRepositoryContext( baseUrl=context_url, type=cm.AccessType.OCI_REGISTRY, ), ], provider=descriptor.component.provider, sources=descriptor.component.sources, componentReferences=descriptor.component. componentReferences, externalResources=external_resources, localResources=local_resources, )))
def greatest_component_version_with_matching_minor( component_name: str, reference_version: str, ctx_repo_base_url: str=None, ctx_repo: cm.RepositoryContext=None, ignore_prerelease_versions: bool=False, ) -> str: if not (bool(ctx_repo_base_url) ^ bool(ctx_repo)): raise ValueError('exactly one of ctx_repo_base_url, ctx_repo must be passed') if ctx_repo_base_url: logger.warning('passing ctx_repo_base_url is deprecated - pass ctx_repo instead!') ctx_repo = cm.OciRepositoryContext(baseUrl=ctx_repo_base_url) if not isinstance(ctx_repo, cm.OciRepositoryContext): raise NotImplementedError(ctx_repo) oci_image_repo = _target_oci_repository_from_component_name( component_name, ctx_repo=ctx_repo, ) client = ccc.oci.oci_client() image_tags = client.tags(image_reference=oci_image_repo) return version.find_latest_version_with_matching_minor( reference_version=reference_version, versions=image_tags, ignore_prerelease_versions=ignore_prerelease_versions, )
def base_component_descriptor_v2( component_name_v2: str, component_labels: typing.Iterable[cm.Label], effective_version: str, source_labels: tuple, ctx_repository_base_url: str, commit: str, ): import gci.componentmodel as cm import version as version_util parsed_version = version_util.parse_to_semver(effective_version) if parsed_version.finalize_version() == parsed_version: # "final" version --> there will be a tag, later (XXX hardcoded hack) src_ref = f'refs/tags/{effective_version}' else: # let's hope the version contains something committish if parsed_version.build: src_ref = f'{parsed_version.prerelease}{parsed_version.build}' else: src_ref = f'{parsed_version.prerelease}' # logical names must not contain slashes or dots logical_name = component_name_v2.replace('/', '_').replace('.', '_') base_descriptor_v2 = cm.ComponentDescriptor( meta=cm.Metadata(schemaVersion=cm.SchemaVersion.V2), component=cm.Component( name=component_name_v2, version=effective_version, repositoryContexts=[ cm.OciRepositoryContext( baseUrl=ctx_repository_base_url, type=cm.AccessType.OCI_REGISTRY, ) ], provider=cm.Provider.INTERNAL, sources=[ cm.ComponentSource( name=logical_name, type=cm.SourceType.GIT, access=cm.GithubAccess( type=cm.AccessType.GITHUB, repoUrl=component_name_v2, ref=src_ref, commit=commit, ), version=effective_version, labels=source_labels, ) ], componentReferences=[], # added later resources=[], # added later labels=list(component_labels), ), ) return base_descriptor_v2
def _comp(name=name, version=version): return cm.Component( name=name, version=version, repositoryContexts=[ cm.OciRepositoryContext( baseUrl='example.com/context', type=cm.AccessType.OCI_REGISTRY, ), ], provider=cm.Provider, sources=cm.ComponentSource, componentReferences=cm.ComponentReference, resources=cm.Resource, )
def ctx_repository(self) -> cm.OciRepositoryContext: ctx_repo_name = self.raw.get('ctx_repository') # XXX hack for unittests if not self.cfg_set: return None if ctx_repo_name: ctx_repo_cfg = self.cfg_set.ctx_repository(ctx_repo_name) else: ctx_repo_cfg = self.cfg_set.ctx_repository() ctx_repo_cfg: model.ctx_repository.CtxRepositoryCfg return cm.OciRepositoryContext( baseUrl=ctx_repo_cfg.base_url(), componentNameMapping=ctx_repo_cfg.component_name_mapping(), )
def _base_component_descriptor( version: str, ctx_repository_base_url: str, commit: str, branch: str, component_name: str = 'github.com/gardenlinux/gardenlinux', ): parsed_version = version_util.parse_to_semver(version) if parsed_version.finalize_version() == parsed_version: # "final" version --> there will be a tag, later src_ref = f'refs/tags/{version}' else: src_ref = f'refs/heads/{branch}' # logical names must not contain slashes or dots logical_name = component_name.replace('/', '_').replace('.', '_') base_descriptor_v2 = cm.ComponentDescriptor( meta=cm.Metadata(schemaVersion=cm.SchemaVersion.V2), component=cm.Component( name=component_name, version=version, repositoryContexts=[ cm.OciRepositoryContext( baseUrl=ctx_repository_base_url, type=cm.AccessType.OCI_REGISTRY, ) ], provider=cm.Provider.INTERNAL, sources=[ cm.ComponentSource( name=logical_name, type=cm.SourceType.GIT, access=cm.GithubAccess( type=cm.AccessType.GITHUB, repoUrl=component_name, ref=src_ref, commit=commit, ), version=version, ) ], componentReferences=[], resources=[], # added later ), ) return base_descriptor_v2
def greatest_component_version( component_name: str, ctx_repo_base_url: str=None, ctx_repo: cm.RepositoryContext=None, ignore_prerelease_versions: bool=False, ) -> str: if not (bool(ctx_repo_base_url) ^ bool(ctx_repo)): raise ValueError('exactly one of ctx_repo_base_url, ctx_repo must be passed') if ctx_repo_base_url: logger.warning('passing ctx_repo_base_url is deprecated - pass ctx_repo instead!') ctx_repo = cm.OciRepositoryContext(baseUrl=ctx_repo_base_url) if not isinstance(ctx_repo, cm.OciRepositoryContext): raise NotImplementedError(ctx_repo) image_tags = component_versions( component_name=component_name, ctx_repo=ctx_repo, ) return version.find_latest_version(image_tags, ignore_prerelease_versions)
def component_versions( component_name: str, ctx_repo_base_url: str=None, ctx_repo: cm.RepositoryContext=None, ) -> typing.Sequence[str]: if not (bool(ctx_repo_base_url) ^ bool(ctx_repo)): raise ValueError('exactly one of ctx_repo_base_url, ctx_repo must be passed') if ctx_repo_base_url: logger.warning('passing ctx_repo_base_url is deprecated - pass ctx_repo instead!') ctx_repo = cm.OciRepositoryContext(baseUrl=ctx_repo_base_url) if not isinstance(ctx_repo, cm.OciRepositoryContext): raise NotImplementedError(ctx_repo) ctx_repo: cm.OciRepositoryContext oci_ref = _target_oci_repository_from_component_name( component_name, ctx_repo=ctx_repo, ) client = ccc.oci.oci_client() return client.tags(image_reference=oci_ref)
def setUp(self): self.tmp_dir = tempfile.TemporaryDirectory() self.render_step = step_def('update_component_deps') self.update_component_deps_trait = update_component_deps.UpdateComponentDependenciesTrait( name='update_component_dependencies', variant_name='don\'t_care', raw_dict={ 'set_dependency_version_script':'some_path', 'upstream_component_name':'don\'t_care', }, ) self.component_descriptor_trait = component_descriptor.ComponentDescriptorTrait( name='component_descriptor', variant_name='don\'t_care', raw_dict={ 'component_name': 'github.com/org/repo_name', }, ) self.component_descriptor_trait.ctx_repository = lambda: cm.OciRepositoryContext( baseUrl='dummy-base-url', ) self.main_repo = test_utils.repository() repo_dir = pathlib.Path(self.tmp_dir.name, self.main_repo.resource_name()) repo_dir.mkdir() self.job_variant = test_utils.job(self.main_repo) self.job_variant._traits_dict = { 'update_component_deps': self.update_component_deps_trait, 'component_descriptor': self.component_descriptor_trait, } self.old_cwd = os.getcwd()
def component_descriptor( name: str, version: str, ctx_repo_url: str = None, ctx_repo: cm.RepositoryContext = None, delivery_client: delivery.client.DeliveryServiceClient = None, cache_dir: str = None, validation_mode: cm.ValidationMode = cm.ValidationMode.NONE, ) -> cm.ComponentDescriptor: ''' retrieves the requested, deserialised component-descriptor, preferring delivery-service, with a fallback to the underlying oci-registry ''' if not (bool(ctx_repo_url) ^ bool(ctx_repo)): raise ValueError( 'exactly one of ctx_repo, ctx_repo_url must be passed') if ctx_repo_url: logger.warning('passing ctx_repo_url is deprecated - pass ctx_repo') ctx_repo = cm.OciRepositoryContext( baseUrl=ctx_repo_url, componentNameMapping=cm.OciComponentNameMapping.URL_PATH, ) if not isinstance(ctx_repo, cm.OciRepositoryContext): raise NotImplementedError(ctx_repo) ctx_repo: cm.OciRepositoryContext return _component_descriptor( name=name, version=version, ctx_repo=ctx_repo, delivery_client=delivery_client, cache_dir=cache_dir, validation_mode=validation_mode, )
def build_component_descriptor( version: str, committish: str, cicd_cfg_name: str, gardenlinux_epoch: str, publishing_actions: str, ctx_repository_config_name: str, branch: str, snapshot_ctx_repository_config_name: str = None, ): publishing_actions = [ glci.model.PublishingAction(action.strip()) for action in publishing_actions.split(',') ] if glci.model.PublishingAction.COMPONENT_DESCRIPTOR not in publishing_actions: print( f'publishing action {glci.model.PublishingAction.COMPONENT_DESCRIPTOR} not specified ' 'exiting now') sys.exit(0) if glci.model.PublishingAction.BUILD_ONLY in publishing_actions: print( f'publishing action {glci.model.PublishingAction.BUILD_ONLY=} specified - exiting now' ) sys.exit(0) cicd_cfg = glci.util.cicd_cfg(cfg_name=cicd_cfg_name) flavour_set = glci.util.flavour_set(flavour_set_name='all') find_releases = glci.util.preconfigured( func=glci.util.find_releases, cicd_cfg=cicd_cfg, ) releases = tuple( find_releases( flavour_set=flavour_set, version=version, build_committish=committish, gardenlinux_epoch=int(gardenlinux_epoch), prefix=glci.model.ReleaseManifest.manifest_key_prefix, )) if glci.model.PublishingAction.RELEASE not in publishing_actions: version = version_util.process_version( version_str=version, operation=version_util.SET_PRERELEASE, prerelease=committish, ) base_url = _resolve_ctx_repository_config(ctx_repository_config_name) if snapshot_ctx_repository_config_name: snapshot_repo_base_url = _resolve_ctx_repository_config( snapshot_ctx_repository_config_name) else: snapshot_repo_base_url = None component_descriptor = _base_component_descriptor( version=version, branch=branch, commit=committish, ctx_repository_base_url=base_url) component_descriptor.component.resources.extend([ virtual_machine_image_resource(release_manifest, cicd_cfg) for release_manifest in releases ]) logger.info('Generated Component-Descriptor:\n' f'{pprint.pformat(dataclasses.asdict(component_descriptor))}') if glci.model.PublishingAction.RELEASE in publishing_actions: product.v2.upload_component_descriptor_v2_to_oci_registry( component_descriptor_v2=component_descriptor, ) if snapshot_repo_base_url: if base_url != snapshot_repo_base_url: repo_ctx = cm.OciRepositoryContext( baseUrl=snapshot_repo_base_url, type=cm.AccessType.OCI_REGISTRY, ) component_descriptor.component.repositoryContexts.append(repo_ctx) # upload obeys the appended repo_ctx product.v2.upload_component_descriptor_v2_to_oci_registry( component_descriptor_v2=component_descriptor, on_exist=product.v2.UploadMode.OVERWRITE, )
def test_label_usage(): component_name = 'c' component_version = '1.2.3' sources = [ cm.ComponentSource( name='repo_aux_source', access=cm.GithubAccess(type=cm.AccessType.GITHUB, ref='refs/heads/master', repoUrl='github.com/otherOrg/otherRepo'), labels=[ cm.Label( name='cloud.gardener/cicd/source', value={'repository-classification': 'auxiliary'}, ), ], ), cm.ComponentSource( name='repo_main_source', access=cm.GithubAccess(type=cm.AccessType.GITHUB, ref='refs/heads/master', repoUrl='github.com/org/repo'), labels=[ cm.Label( name='cloud.gardener/cicd/source', value={'repository-classification': 'main'}, ), ], ), ] component_with_source_label = cm.Component( name=component_name, version=component_version, sources=sources, componentReferences=[], labels=[], repositoryContexts=[ cm.OciRepositoryContext( baseUrl= 'eu.gcr.io/sap-se-gcr-k8s-private/cnudie/gardener/landscapes', type='ociRegistry', ), ], resources=[], provider=[], ) main_source = cnudie.util.determine_main_source_for_component( component_with_source_label, ) assert main_source.labels[0].value == {'repository-classification': 'main'} assert main_source.name == 'repo_main_source' component_without_source_label = cm.Component( name=component_name, version=component_version, sources=[ cm.ComponentSource( name='repo_main_source', access=cm.GithubAccess(type=cm.AccessType.GITHUB, ref='refs/heads/master', repoUrl='github.com/org/repo'), ), cm.ComponentSource( name='repo_aux_source', access=cm.GithubAccess( type=cm.AccessType.GITHUB, ref='refs/heads/master', repoUrl='github.com/otherOrg/otherRepo'), ), ], componentReferences=[], labels=[], repositoryContexts=[ cm.OciRepositoryContext( baseUrl= 'eu.gcr.io/sap-se-gcr-k8s-private/cnudie/gardener/landscapes', type='ociRegistry', ), ], resources=[], provider=[], ) main_source = cnudie.util.determine_main_source_for_component( component_without_source_label) assert main_source.name == 'repo_main_source'
def test_determine_reference_versions(): # Case 1: No Upstream greatest_version = '2.1.1' component_name = 'example.org/foo/bar' base_url = "foo" # actual value not relevant here ctx_repo = cm.OciRepositoryContext(baseUrl=base_url) examinee = functools.partial( determine_reference_versions, component_name=component_name, ctx_repo=ctx_repo, ) with unittest.mock.patch('product.v2') as product_mock: product_mock.greatest_component_version.return_value = greatest_version # no upstream component -> expect latest version to be returned assert examinee( reference_version='2.1.0', upstream_component_name=None, ) == (greatest_version,) product_mock.greatest_component_version.assert_called_with( component_name=component_name, ctx_repo=ctx_repo, ignore_prerelease_versions=False, ) product_mock.greatest_component_version.reset_mock() assert examinee( reference_version='2.2.0', # same result, if our version is already greater upstream_component_name=None, ) == (greatest_version,) product_mock.greatest_component_version.assert_called_with( component_name=component_name, ctx_repo=ctx_repo, ignore_prerelease_versions=False, ) # Case 2: Upstream component defined examinee = functools.partial( determine_reference_versions, component_name='example.org/foo/bar', upstream_component_name='example.org/foo/bar', ctx_repo=ctx_repo, ) with unittest.mock.patch( 'concourse.steps.update_component_deps.latest_component_version_from_upstream' ) as upstream_version_mock: upstream_version = '2.2.0' UUP = update_component_deps.UpstreamUpdatePolicy upstream_version_mock.return_value = upstream_version # should return upstream version, by default (default to strict-following) assert examinee( reference_version='1.2.3', # does not matter ) == (upstream_version,) upstream_version_mock.assert_called_once_with( component_name=component_name, upstream_component_name='example.org/foo/bar', ctx_repo=ctx_repo, ignore_prerelease_versions=False, ) upstream_version_mock.reset_mock() # same behaviour if explicitly configured assert examinee( reference_version='1.2.3', # does not matter upstream_update_policy=UUP.STRICTLY_FOLLOW, ) == (upstream_version,) upstream_version_mock.assert_called_once_with( component_name=component_name, upstream_component_name='example.org/foo/bar', ctx_repo=ctx_repo, ignore_prerelease_versions=False, ) upstream_version_mock.reset_mock() with unittest.mock.patch('product.v2') as product_mock: # if not strictly following, should consider hotfix reference_version = '1.2.3' upstream_hotfix_version = '2.2.3' product_mock.greatest_component_version_with_matching_minor.return_value = \ upstream_hotfix_version assert examinee( reference_version=reference_version, # does not matter upstream_update_policy=UUP.ACCEPT_HOTFIXES, ) == (upstream_hotfix_version, upstream_version) upstream_version_mock.assert_called_once_with( component_name=component_name, upstream_component_name='example.org/foo/bar', ctx_repo=ctx_repo, ignore_prerelease_versions=False, ) product_mock.greatest_component_version_with_matching_minor.assert_called_once_with( component_name=component_name, ctx_repo=ctx_repo, reference_version=reference_version, ignore_prerelease_versions=False, )
def download_component_descriptor_v2( component_name: str, component_version: str, ctx_repo_base_url: str=None, ctx_repo: cm.RepositoryContext=None, absent_ok: bool=False, cache_dir: str=None, validation_mode: cm.ValidationMode=cm.ValidationMode.NONE, ): if not (bool(ctx_repo_base_url) ^ bool(ctx_repo)): raise ValueError('exactly one of ctx_repo_base_url, ctx_repo must be passed') if ctx_repo_base_url: logger.warning('passing ctx_repo_base_url is deprecated - pass ctx_repo instead!') ctx_repo = cm.OciRepositoryContext(baseUrl=ctx_repo_base_url) if not isinstance(ctx_repo, cm.OciRepositoryContext): raise NotImplementedError(ctx_repo) ctx_repo: cm.OciRepositoryContext target_ref = _target_oci_ref_from_ctx_base_url( component_name=component_name, component_version=component_version, ctx_repo=ctx_repo, ) if cache_dir: descriptor_path = os.path.join( cache_dir, ctx_repo.baseUrl.replace('/', '-'), f'{component_name}-{component_version}', ) if os.path.isfile(descriptor_path): return cm.ComponentDescriptor.from_dict( ci.util.parse_yaml_file(descriptor_path), validation_mode=validation_mode, ) else: base_dir = os.path.dirname(descriptor_path) os.makedirs(name=base_dir, exist_ok=True) component_descriptor = retrieve_component_descriptor_from_oci_ref( manifest_oci_image_ref=target_ref, absent_ok=absent_ok, validation_mode=validation_mode, ) if absent_ok and not component_descriptor: return None if cache_dir: try: f = tempfile.NamedTemporaryFile(mode='w', delete=False) # write to tempfile, followed by a mv to avoid collisions through concurrent # processes or threads (assuming mv is an atomic operation) yaml.dump( data=dataclasses.asdict(component_descriptor), Dumper=cm.EnumValueYamlDumper, stream=f.file, ) shutil.move(f.name, descriptor_path) except: os.unlink(f.name) raise return component_descriptor
def build_component_descriptor( version: str, committish: str, cicd_cfg_name: str, gardenlinux_epoch: str, publishing_actions: str, ctx_repository_config_name: str, branch: str, snapshot_ctx_repository_config_name: str = None, ): publishing_actions = [ glci.model.PublishingAction(action.strip()) for action in publishing_actions.split(',') ] if glci.model.PublishingAction.COMPONENT_DESCRIPTOR not in publishing_actions: print( f'{glci.model.PublishingAction.COMPONENT_DESCRIPTOR} not specified ' 'exiting now') sys.exit(0) if glci.model.PublishingAction.BUILD_ONLY in publishing_actions: print( f'{glci.model.PublishingAction.BUILD_ONLY=} specified - exiting now' ) sys.exit(0) cicd_cfg = glci.util.cicd_cfg(cfg_name=cicd_cfg_name) flavour_set = glci.util.flavour_set(flavour_set_name='all') find_releases = glci.util.preconfigured( func=glci.util.find_releases, cicd_cfg=cicd_cfg, ) # effective version is used to incorporate into component-descriptor # (may deviate from gardenlinux-versions, which are always "final") effective_version = _calculate_effective_version( version=version, publishing_actions=publishing_actions, committish=committish, ) releases = tuple( find_releases( flavour_set=flavour_set, version=version, build_committish=committish, gardenlinux_epoch=int(gardenlinux_epoch), prefix=glci.model.ReleaseManifest.manifest_key_prefix, )) releases: typing.Tuple[glci.model.OnlineReleaseManifest] base_url = _resolve_ctx_repository_config(ctx_repository_config_name) if snapshot_ctx_repository_config_name: snapshot_repo_base_url = _resolve_ctx_repository_config( snapshot_ctx_repository_config_name) else: snapshot_repo_base_url = None component_descriptor = _base_component_descriptor( version=effective_version, branch=branch, commit=committish, ctx_repository_base_url=base_url) component_descriptor.component.resources = [ virtual_machine_image_resource( release_manifest=release_manifest, cicd_cfg=cicd_cfg, effective_version=effective_version, ) for release_manifest in releases ] component_descriptor.component.resources.extend( oci_image_resources( releases=releases, effective_version=effective_version, )) logger.info('Generated Component-Descriptor:\n' f'{pprint.pformat(dataclasses.asdict(component_descriptor))}') product.v2.upload_component_descriptor_v2_to_oci_registry( component_descriptor_v2=component_descriptor, on_exist=product.v2.UploadMode.OVERWRITE, ) if snapshot_repo_base_url: if base_url != snapshot_repo_base_url: repo_ctx = cm.OciRepositoryContext( baseUrl=snapshot_repo_base_url, type=cm.AccessType.OCI_REGISTRY, ) component_descriptor.component.repositoryContexts.append(repo_ctx) # upload obeys the appended repo_ctx product.v2.upload_component_descriptor_v2_to_oci_registry( component_descriptor_v2=component_descriptor, on_exist=product.v2.UploadMode.OVERWRITE, )