def validate(self):
version.parse_to_semver(self.release_version)
existing_dir(self.repo_dir)
have_ctf = os.path.exists(self.ctf_path)
have_cd = os.path.exists(self.component_descriptor_v2_path)
if not have_ctf ^ have_cd:
ci.util.fail(
'exactly one of component-descriptor, or ctf-archive must exist'
)
elif have_cd:
self.component_descriptor_v2 = cm.ComponentDescriptor.from_dict(
ci.util.parse_yaml_file(self.component_descriptor_v2_path), )
elif have_ctf:
component_descriptors = list(
cnudie.util.component_descriptors_from_ctf_archive(
self.ctf_path, ))
if not component_descriptors:
ci.util.fail(
f'No component descriptor found in CTF archive at {self.ctf_path}'
)
if len(component_descriptors) > 1:
ci.util.fail(
f'More than one component descriptor found in CTF archive at {self.ctf_path}'
)
self.component_descriptor_v2 = component_descriptors[0]
def is_valid_semver(tag_name):
try:
version.parse_to_semver(tag_name)
return True
except ValueError:
warning('{tag} is not a valid SemVer string'.format(tag=tag_name))
return False
def determine_reference_versions(
component_name: str,
reference_version: str,
component_resolver: product.util.ComponentResolver,
component_descriptor_resolver: product.util.ComponentDescriptorResolver,
upstream_component_name: str=None,
upstream_update_policy: UpstreamUpdatePolicy=UpstreamUpdatePolicy.STRICTLY_FOLLOW,
_component: callable=_component, # allow easier mocking (for unittests)
upstream_reference_component: callable=upstream_reference_component, # allow easier mocking
) -> typing.Sequence[str]:
if upstream_component_name is None:
# no upstream component defined - look for greatest released version
return (component_resolver.latest_component_version(component_name),)
version_candidate = _component(
upstream_reference_component(
component_resolver=component_resolver,
component_descriptor_resolver=component_descriptor_resolver,
component_name=upstream_component_name,
).dependencies(), component_name).version()
version_candidate = version.parse_to_semver(version_candidate)
if upstream_update_policy is UpstreamUpdatePolicy.STRICTLY_FOLLOW:
return (str(version_candidate),)
elif upstream_update_policy is UpstreamUpdatePolicy.ACCEPT_HOTFIXES:
pass # continue
else:
raise NotImplementedError
# also consider hotfixes
hotfix_candidate = component_resolver.greatest_component_version_with_matching_minor(
component_name=component_name,
reference_version=str(reference_version),
)
hotfix_candidate = version.parse_to_semver(hotfix_candidate)
return (str(hotfix_candidate), str(version_candidate))
def validate(self):
version.parse_to_semver(self.release_version)
# either cds _XOR_ ctf must exist
have_ctf = os.path.exists(self.ctf_path)
have_cd = os.path.exists(self.component_descriptor_v2_path)
if not have_ctf ^ have_cd:
ci.util.fail(
'exactly one of component-descriptor, or ctf-archive must exist'
)
elif have_cd:
self.components = [
cm.ComponentDescriptor.from_dict(
ci.util.parse_yaml_file(
self.component_descriptor_v2_path), )
]
elif have_ctf:
self.components = tuple(
cnudie.util.component_descriptors_from_ctf_archive(
self.ctf_path, ))
if not self.components:
ci.util.fail(
f'No component descriptor found in CTF archive at {self.ctf_path=}'
)
for component_descriptor_v2 in self.components:
collections.deque(
cnudie.retrieve.components(component=component_descriptor_v2),
maxlen=0,
)
def validate(self):
existing_dir(self.repo_dir)
version.parse_to_semver(self.release_version)
if (self.release_commit_callback):
existing_file(self.release_commit_callback)
existing_file(self.repository_version_file_path)
def is_obsolete(
self,
reference_component: gci.componentmodel.Component,
):
'''returns a boolean indicating whether or not this Upgrade PR is "obsolete"
A Upgrade is considered to be obsolete, iff the following conditions hold true:
- the reference product contains a component reference with the same name
- the destination version is greater than the greatest reference component version
'''
# find matching component versions
if not isinstance(reference_component, gci.componentmodel.Component):
raise TypeError(reference_component)
if self.reference_type_name == product.v2.COMPONENT_TYPE_NAME:
reference_refs = sorted(
[
rc for rc in reference_component.componentReferences
if rc.componentName == self.ref_name
],
key=lambda r: version.parse_to_semver(r.version))
if not reference_refs:
return False # special case: we have a new reference
greatest_reference_version = version.parse_to_semver(
reference_refs[-1].version)
else:
raise NotImplementedError
# PR is obsolete if same or newer component version is already configured in reference
return greatest_reference_version >= version.parse_to_semver(
self.to_ref.version)
def is_obsolete(self, reference_component):
'''returns a boolean indicating whether or not this Upgrade PR is "obsolete"
A Upgrade is considered to be obsolete, iff the following conditions hold true:
- the reference product contains a component reference with the same name
- the destination version is greater than the greatest reference component version
'''
# find matching component versions
reference_refs = sorted(
[
rc for rc in reference_component.dependencies().references(
type_name=self.reference_type_name)
if rc.name() == self.ref_name
],
key=lambda r: version.parse_to_semver(r.version()))
if not reference_refs:
return False # special case: we have a new reference
# sorted will return the greatest version last
greatest_reference_version = version.parse_to_semver(
reference_refs[-1].version())
# PR is obsolete if same or newer component version is already configured in reference
return greatest_reference_version >= version.parse_to_semver(
self.to_ref.version())
def filter_non_semver_parseable_releases(release_name):
try:
version.parse_to_semver(release_name)
return True
except ValueError:
if warn_for_unparseable_releases:
ci.util.warning(f"Skipping release with non semver-parseable name {release_name}")
return False
def release_versions(self):
for tag_name in self.release_tags():
try:
version.parse_to_semver(tag_name)
yield tag_name
# XXX should rather return a "Version" object, containing both parsed and original
except ValueError:
pass # ignore
def filter_non_semver_parseable_releases(release_name):
try:
version.parse_to_semver(release_name)
return True
except ValueError:
if warn_for_unparseable_releases:
ci.util.warning(
f'ignoring release {release_name=} (not semver)')
return False
def outdated_draft_releases(
draft_releases: [github3.repos.release.Release],
greatest_release_version: str,
):
'''Find outdated draft releases from a list of draft releases and return them. This is achieved
by partitioning the release versions according to their joined major and minor version.
Partitions are then checked:
- if there is only a single release in a partition it is either a hotfix release
(keep corresponding release) or it is not (delete if it is not the greatest release
according to semver)
- if there are multiple releases versions in a partition, keep only the release
corresponding to greatest (according to semver)
'''
greatest_release_version_info = version.parse_to_semver(greatest_release_version)
def _has_semver_draft_prerelease_label(release_name):
version_info = version.parse_to_semver(release_name)
if version_info.prerelease != 'draft':
return False
return True
autogenerated_draft_releases = [
release for release in draft_releases
if release.name
and version.is_semver_parseable(release.name)
and _has_semver_draft_prerelease_label(release.name)
]
draft_release_version_infos = [
version.parse_to_semver(release.name)
for release in autogenerated_draft_releases
]
def _yield_outdated_version_infos_from_partition(partition):
if len(partition) == 1:
version_info = partition.pop()
if version_info < greatest_release_version_info and version_info.patch == 0:
yield version_info
else:
yield from [
version_info
for version_info in partition[1:]
]
outdated_version_infos = list()
for partition in version.partition_by_major_and_minor(draft_release_version_infos):
outdated_version_infos.extend(_yield_outdated_version_infos_from_partition(partition))
outdated_draft_releases = [
release
for release in autogenerated_draft_releases
if version.parse_to_semver(release.name) in outdated_version_infos
]
return outdated_draft_releases
def validate(self):
version.parse_to_semver(self.release_version)
existing_dir(self.repo_dir)
# check whether a release with the given version exists
try:
self.github_helper.repository.release_from_tag(
self.release_version)
except NotFoundError:
raise RuntimeError(
f'No release with tag {self.release_version} found')
def determine_upgrade_prs(
upstream_component_name: str,
upstream_update_policy: UpstreamUpdatePolicy,
upgrade_pull_requests,
ctx_repo_base_url: str,
ignore_prerelease_versions=False,
) -> typing.Iterable[typing.Tuple[
gci.componentmodel.ComponentReference, gci.componentmodel.ComponentReference, str
]]:
for greatest_component_reference in product.v2.greatest_references(
references=current_component().componentReferences,
):
for greatest_version in determine_reference_versions(
component_name=greatest_component_reference.componentName,
reference_version=greatest_component_reference.version,
upstream_component_name=upstream_component_name,
upstream_update_policy=upstream_update_policy,
repository_ctx_base_url=ctx_repo_base_url,
ignore_prerelease_versions=ignore_prerelease_versions,
):
if not greatest_version:
# if None is returned, no versions at all were found
print(
'Warning: no component versions found for '
f'{greatest_component_reference.componentName=}'
)
continue
greatest_version_semver = version.parse_to_semver(greatest_version)
print(f'{greatest_version=}, ours: {greatest_component_reference} {ctx_repo_base_url=}')
if greatest_version_semver <= version.parse_to_semver(
greatest_component_reference.version
):
logger.info(
f'skipping (outdated) {greatest_component_reference=}; '
f'our {greatest_component_reference.version=}, '
f'found: {greatest_version=}'
)
continue
elif upgrade_pr_exists(
component_reference=greatest_component_reference,
component_version=greatest_version,
upgrade_requests=upgrade_pull_requests,
):
logger.info(
'skipping upgrade (PR already exists): '
f'{greatest_component_reference=} '
f'to {greatest_version=}'
)
continue
else:
yield(greatest_component_reference, greatest_version)
def validate(self):
version.parse_to_semver(self.release_version)
existing_dir(self.repo_dir)
try:
self.component_descriptor_v2 = cnudie.util.determine_main_component(
repository_hostname=self.repository_hostname,
repository_path=self.repository_path,
component_descriptor_v2_path=self.component_descriptor_v2_path,
ctf_path=self.ctf_path,
)
except ValueError as err:
ci.util.fail(str(err))
def validate(self):
version.parse_to_semver(self.release_version)
# if a tag with the given release version exists, we cannot create another release
# pointing to it
if self.github_helper.tag_exists(tag_name=self.release_version):
raise RuntimeError(
f"Cannot create tag '{self.release_version}' for release: Tag already exists"
)
if self.component_descriptor_file_path:
existing_file(self.component_descriptor_file_path)
with open(self.component_descriptor_file_path) as f:
# TODO: Proper validation
not_empty(f.read().strip())
def greatest_release_before(self, component_name: str, version: str):
component_reference = ComponentReference.create(name=component_name,
version=version)
repo_helper = self._repository_helper(component_reference)
version = ver.parse_to_semver(version)
# greatest version comes last
versions = sorted(repo_helper.release_versions(),
key=ver.parse_to_semver)
versions = [v for v in versions if ver.parse_to_semver(v) < version]
if len(versions) == 0:
return None # no release before current was found
return versions[-1]
def validate(self):
existing_dir(self.repo_dir)
version.parse_to_semver(self.release_version)
if self.next_version_callback:
existing_file(self.next_version_callback)
existing_file(self.repository_version_file_path)
# perform version ops once to validate args
_calculate_next_cycle_dev_version(
release_version=self.release_version,
version_operation=self.version_operation,
prerelease_suffix=self.prerelease_suffix,
)
def validate(self):
version.parse_to_semver(self.release_version)
tag_template_vars = {
'VERSION': self.release_version,
}
# prepare tags to create
self.github_release_tag_formatted = self.github_release_tag[
'ref_template'].format(**tag_template_vars)
self.git_tags_formatted = [
tag_template['ref_template'].format(**tag_template_vars)
for tag_template in self.git_tags
]
def upload_and_publish_image(
s3_client,
service_principal_cfg: glci.model.AzureServicePrincipalCfg,
storage_account_cfg: glci.model.AzureStorageAccountCfg,
marketplace_cfg: glci.model.AzureMarketplaceCfg,
release: glci.model.OnlineReleaseManifest,
notification_emails: typing.Tuple[str, ...],
) -> glci.model.OnlineReleaseManifest:
'''Copies an image from S3 to an Azure Storage Account, updates the corresponding
Azure Marketplace offering and publish the offering.
'''
azure_release_artifact = util.virtual_image_artifact_for_platform('azure')
azure_release_artifact_path = release.path_by_suffix(
azure_release_artifact)
# Copy image from s3 to Azure Storage Account
target_blob_name = f"gardenlinux-az-{release.version}.vhd"
image_url = copy_image_from_s3_to_az_storage_account(
storage_account_cfg=storage_account_cfg,
s3_client=s3_client,
s3_bucket_name=azure_release_artifact_path.s3_bucket_name,
s3_object_key=azure_release_artifact_path.s3_key,
target_blob_name=target_blob_name,
)
# version _must_ (of course..) be strict semver for azure
azure_version = str(version.parse_to_semver(release.version))
# Update Marketplace offer and start publishing.
publish_operation_id = update_and_publish_marketplace_offer(
service_principal_cfg=service_principal_cfg,
marketplace_cfg=marketplace_cfg,
image_version=azure_version,
image_url=image_url,
notification_recipients=notification_emails,
)
# use anticipated URN for now
parsed_version = version_util.parse_to_semver(release.version)
published_image = glci.model.AzurePublishedImage(
transport_state=glci.model.AzureTransportState.PUBLISH,
publish_operation_id=publish_operation_id,
golive_operation_id='',
urn=generate_urn(marketplace_cfg, parsed_version),
)
return dataclasses.replace(release,
published_image_metadata=published_image)
def greatest_references(references: typing.Iterable[DependencyBase]):
'''
yields the component references from the specified iterable of ComponentReference that
have the greates version (grouped by component name).
Id est: if the sequence contains exactly one version of each contained component name,
the sequence is returned unchanged.
'''
not_none(references)
references = list(references)
for ref in references:
check_type(ref, DependencyBase)
names = [ref.name() for ref in references]
for name in names:
matching_refs = [r for r in references if r.name() == name]
if len(matching_refs) == 1:
# in case reference name was unique, do not bother sorting
# (this also works around issues from non-semver versions)
yield matching_refs[0]
continue
# there might be multiple component versions of the same name
# --> use the greatest version in that case
matching_refs = sorted(
matching_refs,
key=lambda r: ver.parse_to_semver(r.version()),
)
# greates version comes last
yield matching_refs[-1]
def find_greatest_github_release_version(
releases: [github3.repos.release.Release], ):
# currently, non-draft-releases are not created with a name by us. Use the tag name as fallback
release_versions = [
release.name if release.name else release.tag_name
for release in releases
]
def filter_non_semver_parseable_releases(release_name):
try:
version.parse_to_semver(release_name)
return True
except ValueError:
ci.util.warning(
f"Skipping release with non semver-parseable name {release_name}"
)
return False
release_versions = [
name for name in filter(filter_non_semver_parseable_releases,
release_versions)
]
release_version_infos = [
version.parse_to_semver(release_version)
for release_version in release_versions
]
return str(version.find_latest_version(release_version_infos))
def find_greatest_github_release_version(
releases: [github3.repos.release.Release],
warn_for_unparseable_releases: bool = True,
):
# currently, non-draft-releases are not created with a name by us. Use the tag name as fallback
release_versions = [
release.name if release.name else release.tag_name
for release in releases
]
def filter_non_semver_parseable_releases(release_name):
try:
version.parse_to_semver(release_name)
return True
except ValueError:
if warn_for_unparseable_releases:
ci.util.warning(
f'ignoring release {release_name=} (not semver)')
return False
release_versions = [
name for name in filter(filter_non_semver_parseable_releases,
release_versions)
]
release_version_infos = [
version.parse_to_semver(release_version)
for release_version in release_versions
]
latest_version = version.find_latest_version(release_version_infos)
if latest_version:
return str(latest_version)
else:
return None
def _guess_commit_from_ref(component: product.model.Component):
"""
heuristically guess the appropriate git-ref for the given component's version
"""
github_api = _github_api(component_name=component)
github_repo = github_api.repository(
component.github_organisation(),
component.github_repo(),
)
clogger = component_logger(component=component)
def in_repo(commit_ish):
clogger.info(f"commit-ish {commit_ish}")
try:
return github_repo.ref(commit_ish).object.sha
except github3.exceptions.NotFoundError:
pass
try:
return github_repo.commit(commit_ish).sha
except (github3.exceptions.UnprocessableEntity, github3.exceptions.NotFoundError):
return None
# first guess: component version could already be a valid "Gardener-relaxed-semver"
version_str = str(version.parse_to_semver(component))
commit = in_repo(version_str)
if commit:
return commit
# also try unmodified version-str
if (commit := in_repo(component.version())):
return commit
def greatest_references(
references: typing.Iterable[gci.componentmodel.ComponentReference],
) -> gci.componentmodel.ComponentReference:
'''
yields the component references from the specified iterable of ComponentReference that
have the greates version (grouped by component name).
Id est: if the sequence contains exactly one version of each contained component name,
the sequence is returned unchanged.
'''
references = tuple(references)
names = [r.name for r in references]
for name in names:
matching_refs = [r for r in references if r.name == name]
if len(matching_refs) == 1:
# in case reference name was unique, do not bother sorting
# (this also works around issues from non-semver versions)
yield matching_refs[0]
else:
# there might be multiple component versions of the same name
# --> use the greatest version in that case
matching_refs.sort(
key=lambda r: version.parse_to_semver(r.version))
# greates version comes last
yield matching_refs[-1]
def __init__(self, version: str):
self._version_str = str(not_none(version))
try:
self._version_semver = ver.parse_to_semver(self._version_str)
except ValueError:
self._version_semver = None
def greatest_version_before(
component_name: str,
component_version: str,
ctx_repo_base_url: str,
):
versions = component_versions(
component_name=component_name,
ctx_repo_base_url=ctx_repo_base_url,
)
versions = sorted(versions, key=version.parse_to_semver)
versions = [
v for v in versions if
version.parse_to_semver(v) < version.parse_to_semver(component_version)
]
if len(versions) == 0:
return None # no release before current was found
return versions[-1]
def base_component_descriptor_v2(
component_name_v2: str,
component_labels: typing.Iterable[cm.Label],
effective_version: str,
source_labels: tuple,
ctx_repository_base_url: str,
commit: str,
):
import gci.componentmodel as cm
import version as version_util
parsed_version = version_util.parse_to_semver(effective_version)
if parsed_version.finalize_version() == parsed_version:
# "final" version --> there will be a tag, later (XXX hardcoded hack)
src_ref = f'refs/tags/{effective_version}'
else:
# let's hope the version contains something committish
if parsed_version.build:
src_ref = f'{parsed_version.prerelease}{parsed_version.build}'
else:
src_ref = f'{parsed_version.prerelease}'
# logical names must not contain slashes or dots
logical_name = component_name_v2.replace('/', '_').replace('.', '_')
base_descriptor_v2 = cm.ComponentDescriptor(
meta=cm.Metadata(schemaVersion=cm.SchemaVersion.V2),
component=cm.Component(
name=component_name_v2,
version=effective_version,
repositoryContexts=[
cm.RepositoryContext(
baseUrl=ctx_repository_base_url,
type=cm.AccessType.OCI_REGISTRY,
)
],
provider=cm.Provider.INTERNAL,
sources=[
cm.ComponentSource(
name=logical_name,
type=cm.SourceType.GIT,
access=cm.GithubAccess(
type=cm.AccessType.GITHUB,
repoUrl=component_name_v2,
ref=src_ref,
commit=commit,
),
version=effective_version,
labels=source_labels,
)
],
componentReferences=[], # added later
resources=[], # added later
labels=list(component_labels),
),
)
return base_descriptor_v2
def validate(self):
version.parse_to_semver(self.release_version)
# either cds _XOR_ ctf must exist
have_ctf = os.path.exists(self.ctf_path)
have_cd = os.path.exists(self.component_descriptor_v2_path)
if not have_ctf ^ have_cd:
ci.util.fail(
'exactly one of component-descriptor, or ctf-archive must exist'
)
elif have_cd:
component_descriptor_v2 = cm.ComponentDescriptor.from_dict(
ci.util.parse_yaml_file(self.component_descriptor_v2_path), )
# resolve all referenced components (thus ensure all dependencies are available)
tuple(
cnudie.retrieve.components(component=component_descriptor_v2))
# TODO: more validation (e.g. check for uniqueness of names)
elif have_ctf:
# nothing to do, already uploaded in component_descriptor step.
pass
def greatest_version_before(
component_name: str,
component_version: str,
ctx_repo: cm.RepositoryContext,
):
if not isinstance(ctx_repo, cm.OciRepositoryContext):
raise NotImplementedError(ctx_repo)
versions = component_versions(
component_name=component_name,
ctx_repo=ctx_repo,
)
versions = sorted(versions, key=version.parse_to_semver)
versions = [
v for v in versions
if version.parse_to_semver(v) < version.parse_to_semver(component_version)
]
if len(versions) == 0:
return None # no release before current was found
return versions[-1]
def validate(self):
version.parse_to_semver(self.release_version)
# either cds _OR_ ctf must exist
have_ctf = os.path.exists(self.ctf_path)
have_cd = os.path.exists(self.component_descriptor_v2_path)
if have_ctf and have_cd:
ci.util.fail(
'Both CTF and Component Descriptor are defined. Only one may be defined.'
)
elif have_cd:
existing_file(self.component_descriptor_file_path)
with open(self.component_descriptor_file_path) as f:
# TODO: Proper validation
not_empty(f.read().strip())
component_descriptor_v2 = cm.ComponentDescriptor.from_dict(
ci.util.parse_yaml_file(self.component_descriptor_v2_path), )
product.v2.resolve_dependencies(
component=component_descriptor_v2.component)
# TODO: more validation (e.g. check for uniqueness of names)
elif have_ctf:
# nothing to do, already uploaded in component_descriptor step.
pass