def test_assembly_basis_event(self):
self.assertEqual(assembly_basis_event(self.releases_config, 'ART_1'),
None)
self.assertEqual(assembly_basis_event(self.releases_config, 'ART_6'),
5)
try:
assembly_basis_event(self.releases_config, 'ART_INFINITE')
self.fail('Expected ValueError on assembly infinite recursion')
except ValueError:
pass
except Exception as e:
self.fail(
f'Expected ValueError on assembly infinite recursion but got: {type(e)}: {e}'
)
def get_latest_build(self,
default: Optional[Any] = -1,
assembly: Optional[str] = None,
extra_pattern: str = '*',
build_state: BuildStates = BuildStates.COMPLETE,
component_name: Optional[str] = None,
el_target: Optional[Union[str, int]] = None,
honor_is: bool = True,
complete_before_event: Optional[int] = None):
"""
:param default: A value to return if no latest is found (if not specified, an exception will be thrown)
:param assembly: A non-default assembly name to search relative to. If not specified, runtime.assembly
will be used. If runtime.assembly is also None, the search will return true latest.
If the assembly parameter is set to '', this search will also return true latest.
:param extra_pattern: An extra glob pattern that must be matched in the middle of the
build's release field. Pattern must match release timestamp and components
like p? and git commit (up to, but not including ".assembly.<name>" release
component). e.g. "*.g<commit>.* or '*.p1.*'
:param build_state: 0=BUILDING, 1=COMPLETE, 2=DELETED, 3=FAILED, 4=CANCELED
:param component_name: If not specified, looks up builds for self component.
:param el_target: In the case of an RPM, which can build for multiple targets, you can specify
'7' for el7, '8' for el8, etc. You can also pass in a brew target that
contains '....-rhel-?..' and the number will be extracted. If you want the true
latest, leave as None.
:param honor_is: If True, and an assembly component specifies 'is', that nvr will be returned.
:param complete_before_event: If a value is specified >= 0, the search will be constrained to builds which completed before
the specified brew_event. If a value is specified < 0, the search will be conducted with no constraint on
brew event. If no value is specified, the search will be relative to the current assembly's basis event.
:return: Returns the most recent build object from koji for the specified component & assembly.
Example https://gist.github.com/jupierce/57e99b80572336e8652df3c6be7bf664
"""
if not component_name:
component_name = self.get_component_name()
builds = []
with self.runtime.pooled_koji_client_session(caching=True) as koji_api:
package_info = koji_api.getPackage(
component_name
) # e.g. {'id': 66873, 'name': 'atomic-openshift-descheduler-container'}
if not package_info:
raise IOError(
f'No brew package is defined for {component_name}')
package_id = package_info[
'id'] # we could just constrain package name using pattern glob, but providing package ID # should be a much more efficient DB query.
# listBuilds returns all builds for the package; We need to limit the query to the builds
# relevant for our major/minor.
rpm_suffix = '' # By default, find the latest RPM build - regardless of el7, el8, ...
el_ver = None
if self.meta_type == 'image':
ver_prefix = 'v' # openshift-enterprise-console-container-v4.7.0-202106032231.p0.git.d9f4379
else:
# RPMs do not have a 'v' in front of their version; images do.
ver_prefix = '' # openshift-clients-4.7.0-202106032231.p0.git.e29b355.el8
if el_target:
el_ver = isolate_el_version_in_brew_tag(el_target)
if el_ver:
rpm_suffix = f'.el{el_ver}'
else:
raise IOError(
f'Unable to determine rhel version from specified el_target: {el_target}'
)
pattern_prefix = f'{component_name}-{ver_prefix}{self.branch_major_minor()}.'
if assembly is None:
assembly = self.runtime.assembly
list_builds_kwargs = {
} # extra kwargs that will be passed to koji_api.listBuilds invocations
if complete_before_event is not None:
if complete_before_event < 0:
# By setting the parameter to None, it tells the koji wrapper to not bound the brew event.
list_builds_kwargs['completeBefore'] = None
else:
# listBuilds accepts timestamps, not brew events, so convert brew event into seconds since the epoch
complete_before_ts = koji_api.getEvent(
complete_before_event)['ts']
list_builds_kwargs['completeBefore'] = complete_before_ts
def default_return():
msg = f"No builds detected for using prefix: '{pattern_prefix}', extra_pattern: '{extra_pattern}', assembly: '{assembly}', build_state: '{build_state.name}', el_target: '{el_target}'"
if default != -1:
self.logger.info(msg)
return default
raise IOError(msg)
def latest_build_list(pattern_suffix):
# Include * after pattern_suffix to tolerate:
# 1. Matching an unspecified RPM suffix (e.g. .el7).
# 2. Other release components that might be introduced later.
builds = koji_api.listBuilds(
packageID=package_id,
state=None if build_state is None else build_state.value,
pattern=
f'{pattern_prefix}{extra_pattern}{pattern_suffix}*{rpm_suffix}',
queryOpts={
'limit': 1,
'order': '-creation_event_id'
},
**list_builds_kwargs)
# Ensure the suffix ends the string OR at least terminated by a '.' .
# This latter check ensures that 'assembly.how' doesn't not match a build from
# "assembly.howdy'.
refined = [
b for b in builds if b['nvr'].endswith(pattern_suffix)
or f'{pattern_suffix}.' in b['nvr']
]
if refined and build_state == BuildStates.COMPLETE:
# A final sanity check to see if the build is tagged with something we
# respect. There is a chance that a human may untag a build. There
# is no standard practice at present in which they should (they should just trigger
# a rebuild). If we find the latest build is not tagged appropriately, blow up
# and let a human figure out what happened.
check_nvr = refined[0]['nvr']
for i in range(2):
tags = {
tag['name']
for tag in koji_api.listTags(build=check_nvr)
}
if tags:
refined[0][
'_tags'] = tags # save tag names to dict for future use
break
# Observed that a complete build needs some time before it gets tagged. Give it some
# time if not immediately available.
time.sleep(60)
# RPMS have multiple targets, so our self.branch() isn't perfect.
# We should permit rhel-8/rhel-7/etc.
tag_prefix = self.branch().rsplit(
'-', 1)[0] + '-' # String off the rhel version.
accepted_tags = [
name for name in tags if name.startswith(tag_prefix)
]
if not accepted_tags:
self.logger.warning(
f'Expected to find at least one tag starting with {self.branch()} on latest build {check_nvr} but found [{tags}]; tagging failed after build or something has changed tags in an unexpected way'
)
return refined
if honor_is and self.config['is']:
if build_state != BuildStates.COMPLETE:
# If this component is defined by 'is', history failures, etc, do not matter.
return default_return()
# under 'is' for RPMs, we expect 'el7' and/or 'el8', etc. For images, just 'nvr'.
isd = self.config['is']
if self.meta_type == 'rpm':
if el_ver is None:
raise ValueError(
f'Expected el_target to be set when querying a pinned RPM component {self.distgit_key}'
)
is_nvr = isd[f'el{el_ver}']
if not is_nvr:
return default_return()
else:
# The image metadata (or, more likely, the currently assembly) has the image
# pinned. Return only the pinned NVR. When a child image is being rebased,
# it uses get_latest_build to find the parent NVR to use (if it is not
# included in the "-i" doozer argument). We need it to find the pinned NVR
# to place in its Dockerfile.
# Pinning also informs gen-payload when attempting to assemble a release.
is_nvr = isd.nvr
if not is_nvr:
raise ValueError(
f'Did not find nvr field in pinned Image component {self.distgit_key}'
)
# strict means raise an exception if not found.
found_build = koji_api.getBuild(is_nvr, strict=True)
# Different brew apis return different keys here; normalize to make the rest of doozer not need to change.
found_build['id'] = found_build['build_id']
return found_build
if not assembly:
# if assembly is '' (by parameter) or still None after runtime.assembly,
# we are returning true latest.
builds = latest_build_list('')
else:
basis_event = assembly_basis_event(
self.runtime.get_releases_config(), assembly=assembly)
if basis_event:
# If an assembly has a basis event, its latest images can only be sourced from
# "is:" or the stream assembly. We've already checked for "is" above.
assembly = 'stream'
# Assemblies without a basis will return assembly qualified builds for their
# latest images. This includes "stream" and "test", but could also include
# an assembly that is customer specific with its own branch.
builds = latest_build_list(f'.assembly.{assembly}')
if not builds:
if assembly != 'stream':
builds = latest_build_list('.assembly.stream')
if not builds:
# Fall back to true latest
builds = latest_build_list('')
if builds and '.assembly.' in builds[0]['release']:
# True latest belongs to another assembly. In this case, just return
# that they are no builds for this assembly.
builds = []
if not builds:
return default_return()
found_build = builds[0]
# Different brew apis return different keys here; normalize to make the rest of doozer not need to change.
found_build['id'] = found_build['build_id']
return found_build
def initialize(self, mode='none', no_group=False):
if self.initialized:
return
if self.quiet and self.verbose:
click.echo("Flags --quiet and --verbose are mutually exclusive")
exit(1)
# We could mark these as required and the click library would do this for us,
# but this seems to prevent getting help from the various commands (unless you
# specify the required parameters). This can probably be solved more cleanly, but TODO
if not no_group and self.group is None:
click.echo("Group must be specified")
exit(1)
if self.working_dir is None:
self.working_dir = tempfile.mkdtemp(".tmp", "elliott-")
# This can be set to False by operations which want the working directory to be left around
self.remove_tmp_working_dir = True
atexit.register(remove_tmp_working_dir, self)
else:
self.working_dir = os.path.abspath(self.working_dir)
if not os.path.isdir(self.working_dir):
os.makedirs(self.working_dir)
self.initialize_logging()
if no_group:
return # nothing past here should be run without a group
self.resolve_metadata()
self.group_dir = self.gitdata.data_dir
self.group_config = self.get_group_config()
if self.group_config.name != self.group:
raise IOError(
"Name in group.yml does not match group name. Someone may have copied this group without updating group.yml (make sure to check branch)"
)
if self.group_config.assemblies.enabled or self.enable_assemblies:
if re.fullmatch(r'[\w.]+', self.assembly) is None or self.assembly[
0] == '.' or self.assembly[-1] == '.':
raise ValueError(
'Assembly names may only consist of alphanumerics, ., and _, but not start or end with a dot (.).'
)
else:
# If assemblies are not enabled for the group,
# ignore this argument throughout doozer.
self.assembly = None
if self.branch is not None:
self.logger.info("Using branch from command line: %s" %
self.branch)
elif self.group_config.branch is not Missing:
self.branch = self.group_config.branch
self.logger.info("Using branch from group.yml: %s" % self.branch)
else:
self.logger.info(
"No branch specified either in group.yml or on the command line; all included images will need to specify their own."
)
# Flattens a list like like [ 'x', 'y,z' ] into [ 'x.yml', 'y.yml', 'z.yml' ]
# for later checking we need to remove from the lists, but they are tuples. Clone to list
def flatten_list(names):
if not names:
return []
# split csv values
result = []
for n in names:
result.append(
[x for x in n.replace(' ', ',').split(',') if x != ''])
# flatten result and remove dupes
return list(set([y for x in result for y in x]))
def filter_enabled(n, d):
return d.get('mode', 'enabled') == 'enabled'
exclude_keys = flatten_list(self.exclude)
image_ex = list(exclude_keys)
rpm_ex = list(exclude_keys)
image_keys = flatten_list(self.images)
rpm_keys = flatten_list(self.rpms)
filter_func = filter_enabled
replace_vars = self.group_config.vars.primitive(
) if self.group_config.vars else {}
if self.assembly:
replace_vars['runtime_assembly'] = self.assembly
image_data = {}
if mode in ['images', 'both']:
image_data = self.gitdata.load_data(
path='images',
keys=image_keys,
exclude=image_ex,
filter_funcs=None if len(image_keys) else filter_func,
replace_vars=replace_vars)
for i in image_data.values():
self.late_resolve_image(i.key, add=True, data_obj=i)
if not self.image_map:
self.logger.warning(
"No image metadata directories found for given options within: {}"
.format(self.group_dir))
if mode in ['rpms', 'both']:
rpm_data = self.gitdata.load_data(
path='rpms',
keys=rpm_keys,
exclude=rpm_ex,
replace_vars=replace_vars,
filter_funcs=None if len(rpm_keys) else filter_func)
for r in rpm_data.values():
metadata = RPMMetadata(self, r)
self.rpm_map[metadata.distgit_key] = metadata
if not self.rpm_map:
self.logger.warning(
"No rpm metadata directories found for given options within: {}"
.format(self.group_dir))
missed_include = set(image_keys) - set(image_data.keys())
if len(missed_include) > 0:
raise ElliottFatalError(
'The following images or rpms were either missing or filtered out: {}'
.format(', '.join(missed_include)))
self.assembly_basis_event = assembly_basis_event(
self.get_releases_config(), self.assembly)
if self.assembly_basis_event:
if self.brew_event:
raise ElliottFatalError(
f'Cannot run with assembly basis event {self.assembly_basis_event} and --brew-event at the same time.'
)
# If the assembly has a basis event, we constrain all brew calls to that event.
self.brew_event = self.assembly_basis_event
self.logger.warning(
f'Constraining brew event to assembly basis for {self.assembly}: {self.brew_event}'
)
self.initialized = True