def assembly_metadata_config(releases_config: Model, assembly: str,
meta_type: str, distgit_key: str,
meta_config: Model) -> Model:
"""
Returns a group member's metadata configuration based on the assembly information
and the initial file-based config.
:param releases_config: A Model for releases.yaml.
:param assembly: The name of the assembly
:param meta_type: 'rpm' or 'image'
:param distgit_key: The member's distgit_key
:param meta_config: The meta's config object
:return: Returns a computed config for the metadata (e.g. value for meta.config).
"""
if not assembly or not isinstance(releases_config, Model):
return meta_config
_check_recursion(releases_config, assembly)
target_assembly = releases_config.releases[assembly].assembly
if target_assembly.basis.assembly: # Does this assembly inherit from another?
# Recursive apply ancestor assemblies
meta_config = assembly_metadata_config(releases_config,
target_assembly.basis.assembly,
meta_type, distgit_key,
meta_config)
config_dict = meta_config.primitive()
component_list = target_assembly.members[f'{meta_type}s']
for component_entry in component_list:
if component_entry.distgit_key == '*' or component_entry.distgit_key == distgit_key and component_entry.metadata:
config_dict = merger(component_entry.metadata.primitive(),
config_dict)
return Model(dict_to_model=config_dict)
def assembly_group_config(releases_config: Model, assembly: str,
group_config: Model) -> Model:
"""
Returns a group config based on the assembly information
and the input group config.
:param releases_config: A Model for releases.yaml.
:param assembly: The name of the assembly
:param group_config: The group config to merge into a new group config (original Model will not be altered)
:param _visited: Keeps track of visited assembly definitions to prevent infinite recursion.
"""
if not assembly or not isinstance(releases_config, Model):
return group_config
_check_recursion(releases_config, assembly)
target_assembly = releases_config.releases[assembly].assembly
if target_assembly.basis.assembly: # Does this assembly inherit from another?
# Recursively apply ancestor assemblies
group_config = assembly_group_config(releases_config,
target_assembly.basis.assembly,
group_config)
target_assembly_group = target_assembly.group
if not target_assembly_group:
return group_config
return Model(dict_to_model=merger(target_assembly_group.primitive(),
group_config.primitive()))
def test_from_image_member_deps(self, assembly_metadata_config: Mock):
finder = BuildFinder(MagicMock())
finder._get_builds = MagicMock(return_value=[
{
"id": 1,
"build_id": 1,
"name": "fake1",
"nvr": "fake1-1.2.3-1.el8"
},
{
"id": 2,
"build_id": 2,
"name": "fake2",
"nvr": "fake2-1.2.3-1.el8"
},
{
"id": 3,
"build_id": 3,
"name": "fake3",
"nvr": "fake3-1.2.3-1.el8"
},
])
assembly_metadata_config.return_value = Model({
"dependencies": {
"rpms": [
{
"el8": "fake1-1.2.3-1.el8"
},
{
"el8": "fake2-1.2.3-1.el8"
},
{
"el8": "fake3-1.2.3-1.el8"
},
{
"el7": "fake2-1.2.3-1.el7"
},
{
"el7": "fake2-1.2.3-1.el7"
},
]
}
})
image_meta = Model({
"distgit_key": "fake-image",
})
actual = finder.from_image_member_deps(8, "art1", Model(), image_meta,
{})
self.assertEqual(
[b["nvr"] for b in actual.values()],
["fake1-1.2.3-1.el8", "fake2-1.2.3-1.el8", "fake3-1.2.3-1.el8"])
finder._get_builds.assert_called_once_with(
["fake1-1.2.3-1.el8", "fake2-1.2.3-1.el8", "fake3-1.2.3-1.el8"])
assembly_metadata_config.assert_called_once()
def get_releases_config(self):
if self.releases_config is not None:
return self.releases_config
load = self.gitdata.load_data(key='releases')
if load:
self.releases_config = Model(load.data)
else:
self.releases_config = Model()
return self.releases_config
def get_group_config(self):
# group.yml can contain a `vars` section which should be a
# single level dict containing keys to str.format(**dict) replace
# into the YAML content. If `vars` found, the format will be
# preformed and the YAML model will reloaded from that result
tmp_config = Model(self.gitdata.load_data(key='group').data)
replace_vars = tmp_config.vars
if replace_vars is not Missing:
try:
group_yml = yaml.safe_dump(tmp_config.primitive(),
default_flow_style=False)
tmp_config = Model(
yaml.safe_load(group_yml.format(**replace_vars)))
except KeyError as e:
raise ValueError(
'group.yml contains template key `{}` but no value was provided'
.format(e.args[0]))
return tmp_config
def __init__(self, meta_type, runtime, data_obj):
"""
:param: meta_type - a string. Index to the sub-class <'rpm'|'image'>.
:param: runtime - a Runtime object.
:param: name - a filename to load as metadata
"""
self.meta_type = meta_type
self.runtime = runtime
self.data_obj = data_obj
self.base_dir = data_obj.base_dir
self.config_filename = data_obj.filename
self.full_config_path = data_obj.path
# Some config filenames have suffixes to avoid name collisions; strip off the suffix to find the real
# distgit repo name (which must be combined with the distgit namespace).
# e.g. openshift-enterprise-mediawiki.apb.yml
# distgit_key=openshift-enterprise-mediawiki.apb
# name (repo name)=openshift-enterprise-mediawiki
self.distgit_key = data_obj.key
self.name = self.distgit_key.split('.')[
0] # Split off any '.apb' style differentiator (if present)
self.runtime.logger.debug("Loading metadata from {}".format(
self.full_config_path))
self.raw_config = Model(
data_obj.data) # Config straight from ocp-build-data
assert (self.raw_config.name is not Missing)
self.config = assembly_metadata_config(runtime.get_releases_config(),
runtime.assembly, meta_type,
self.distgit_key,
self.raw_config)
self.namespace, self._component_name = Metadata.extract_component_info(
meta_type, self.name, self.config)
self.mode = self.config.get('mode', CONFIG_MODE_DEFAULT).lower()
if self.mode not in CONFIG_MODES:
raise ValueError('Invalid mode for {}'.format(
self.config_filename))
self.enabled = (self.mode == CONFIG_MODE_DEFAULT)
self.qualified_name = "%s/%s" % (self.namespace, self.name)
self.qualified_key = "%s/%s" % (self.namespace, self.distgit_key)
# Includes information to identify the metadata being used with each log message
self.logger = logutil.EntityLoggingAdapter(
logger=self.runtime.logger, extra={'entity': self.qualified_key})
self._distgit_repo = None
def test_from_group_deps_with_art_managed_rpms(self):
finder = BuildFinder(MagicMock())
group_config = Model({
"dependencies": {
"rpms": [
{
"el8": "fake1-1.2.3-1.el8"
},
{
"el8": "fake2-1.2.3-1.el8"
},
{
"el8": "fake3-1.2.3-1.el8"
},
{
"el7": "fake2-1.2.3-1.el7"
},
{
"el7": "fake2-1.2.3-1.el7"
},
]
}
})
finder._get_builds = MagicMock(return_value=[
{
"id": 1,
"build_id": 1,
"name": "fake1",
"nvr": "fake1-1.2.3-1.el8"
},
{
"id": 2,
"build_id": 2,
"name": "fake2",
"nvr": "fake2-1.2.3-1.el8"
},
{
"id": 3,
"build_id": 3,
"name": "fake3",
"nvr": "fake3-1.2.3-1.el8"
},
])
with self.assertRaises(ValueError) as ex:
finder.from_group_deps(8, group_config,
{"fake3": MagicMock(rpm_name="fake3")})
self.assertIn("Group dependencies cannot have ART managed RPMs",
str(ex.exception))
finder._get_builds.assert_called_once()
def test_from_pinned_by_is(self, assembly_metadata_config: Mock):
finder = BuildFinder(MagicMock())
releases_config = Model()
rpm_metas = {
"fake1": MagicMock(rpm_name="fake1"),
"fake2": MagicMock(rpm_name="fake2"),
}
meta_configs = {
"fake1": Model({"is": {
"el8": "fake1-1.2.3-1.el8"
}}),
"fake2": Model({"is": {
"el8": "fake2-1.2.3-1.el8"
}}),
}
finder._get_builds = MagicMock(return_value=[
{
"id": 1,
"build_id": 1,
"name": "fake1",
"nvr": "fake1-1.2.3-1.el8"
},
{
"id": 2,
"build_id": 2,
"name": "fake2",
"nvr": "fake2-1.2.3-1.el8"
},
])
assembly_metadata_config.side_effect = lambda *args: meta_configs[args[
3]]
actual = finder.from_pinned_by_is(8, "art1", releases_config,
rpm_metas)
self.assertEqual([b["nvr"] for b in actual.values()],
["fake1-1.2.3-1.el8", "fake2-1.2.3-1.el8"])
finder._get_builds.assert_called_once()
def _assembly_field(field_name: str, releases_config: Model,
assembly: str) -> Model:
"""
:param field_name: the field name
:param releases_config: The content of releases.yml in Model form.
:param assembly: The name of the assembly to assess
Returns the a computed rhcos config model for a given assembly.
"""
if not assembly or not isinstance(releases_config, Model):
return Missing
_check_recursion(releases_config, assembly)
target_assembly = releases_config.releases[assembly].assembly
config_dict = target_assembly.get(field_name, {})
if target_assembly.basis.assembly: # Does this assembly inherit from another?
# Recursive apply ancestor assemblies
basis_rhcos_config = _assembly_field(field_name, releases_config,
target_assembly.basis.assembly)
config_dict = merger(config_dict, basis_rhcos_config.primitive())
return Model(dict_to_model=config_dict)
def test_from_group_deps(self):
finder = BuildFinder(MagicMock())
group_config = Model({
"dependencies": {
"rpms": [
{
"el8": "fake1-1.2.3-1.el8"
},
{
"el8": "fake2-1.2.3-1.el8"
},
{
"el7": "fake2-1.2.3-1.el7"
},
{
"el7": "fake2-1.2.3-1.el7"
},
]
}
})
finder._get_builds = MagicMock(return_value=[
{
"id": 1,
"build_id": 1,
"name": "fake1",
"nvr": "fake1-1.2.3-1.el8"
},
{
"id": 2,
"build_id": 2,
"name": "fake2",
"nvr": "fake2-1.2.3-1.el8"
},
])
actual = finder.from_group_deps(8, group_config, {})
self.assertEqual([b["nvr"] for b in actual.values()],
["fake1-1.2.3-1.el8", "fake2-1.2.3-1.el8"])
finder._get_builds.assert_called_once()
def test_asembly_metadata_config(self):
meta_config = Model(
dict_to_model={
'owners': ['*****@*****.**'],
'content': {
'source': {
'git': {
'url':
'[email protected]:openshift-priv/kuryr-kubernetes.git',
'branch': {
'target': 'release-4.8',
}
},
'specfile': 'openshift-kuryr-kubernetes-rhel8.spec'
}
},
'name': 'openshift-kuryr'
})
config = assembly_metadata_config(self.releases_config, 'ART_1', 'rpm',
'openshift-kuryr', meta_config)
# Ensure no loss
self.assertEqual(config.name, 'openshift-kuryr')
self.assertEqual(len(config.owners), 1)
self.assertEqual(config.owners[0], '*****@*****.**')
# Check that things were overridden
self.assertEqual(config.content.source.git.url,
'[email protected]:jupierce/kuryr-kubernetes.git')
self.assertEqual(config.content.source.git.branch.target, '1_hash')
config = assembly_metadata_config(self.releases_config, 'ART_5', 'rpm',
'openshift-kuryr', meta_config)
# Ensure no loss
self.assertEqual(config.name, 'openshift-kuryr')
self.assertEqual(len(config.owners), 1)
self.assertEqual(config.owners[0], '*****@*****.**')
# Check that things were overridden
self.assertEqual(config.content.source.git.url,
'[email protected]:jupierce/kuryr-kubernetes.git')
self.assertEqual(config.content.source.git.branch.target, '2_hash')
config = assembly_metadata_config(self.releases_config, 'ART_6', 'rpm',
'openshift-kuryr', meta_config)
# Ensure no loss
self.assertEqual(config.name, 'openshift-kuryr')
self.assertEqual(len(config.owners), 1)
self.assertEqual(config.owners[0], '*****@*****.**')
# Check that things were overridden. 6 changes branches for all rpms
self.assertEqual(config.content.source.git.url,
'[email protected]:jupierce/kuryr-kubernetes.git')
self.assertEqual(config.content.source.git.branch.target, 'customer_6')
config = assembly_metadata_config(self.releases_config, 'ART_8',
'image', 'openshift-kuryr',
meta_config)
# Ensure no loss
self.assertEqual(config.name, 'openshift-kuryr')
self.assertEqual(config.content.source.git.url,
'[email protected]:jupierce/kuryr-kubernetes.git')
self.assertEqual(config.content.source.git.branch.target, '1_hash')
# Ensure that 'is' comes from ART_8 and not ART_7
self.assertEqual(config['is'], 'kuryr-nvr2')
# Ensure that 'dependencies' were accumulate
self.assertEqual(len(config.dependencies.rpms), 2)
try:
assembly_metadata_config(self.releases_config, 'ART_INFINITE',
'rpm', 'openshift-kuryr', meta_config)
self.fail('Expected ValueError on assembly infinite recursion')
except ValueError:
pass
except Exception as e:
self.fail(
f'Expected ValueError on assembly infinite recursion but got: {type(e)}: {e}'
)
def test_assembly_group_config(self):
group_config = Model(dict_to_model={
'arches': ['x86_64'],
'advisories': {
'image': 1,
'extras': 1,
}
})
config = assembly_group_config(self.releases_config, 'ART_1',
group_config)
self.assertEqual(len(config.arches), 3)
config = assembly_group_config(self.releases_config, 'ART_2',
group_config)
self.assertEqual(len(config.arches), 2)
# 3 inherits from 2 an only overrides advisory value
config = assembly_group_config(self.releases_config, 'ART_3',
group_config)
self.assertEqual(len(config.arches), 2)
self.assertEqual(config.advisories.image, 31)
self.assertEqual(
config.advisories.extras,
1) # Extras never override, so should be from group_config
# 4 inherits from 3, but sets "advsories!"
config = assembly_group_config(self.releases_config, 'ART_4',
group_config)
self.assertEqual(len(config.arches), 2)
self.assertEqual(config.advisories.image, 41)
self.assertEqual(config.advisories.extras, Missing)
# 5 inherits from 4, but sets "advsories!" (overriding 4's !) and "arches!"
config = assembly_group_config(self.releases_config, 'ART_5',
group_config)
self.assertEqual(len(config.arches), 1)
self.assertEqual(config.advisories.image, 51)
config = assembly_group_config(self.releases_config, 'not_defined',
group_config)
self.assertEqual(len(config.arches), 1)
config = assembly_group_config(self.releases_config, 'ART_7',
group_config)
self.assertEqual(len(config.dependencies.rpms), 1)
config = assembly_group_config(self.releases_config, 'ART_8',
group_config)
self.assertEqual(len(config.dependencies.rpms), 2)
try:
assembly_group_config(self.releases_config, 'ART_INFINITE',
group_config)
self.fail('Expected ValueError on assembly infinite recursion')
except ValueError:
pass
except Exception as e:
self.fail(
f'Expected ValueError on assembly infinite recursion but got: {type(e)}: {e}'
)
def setUp(self) -> None:
releases_yml = """
releases:
ART_1:
assembly:
members:
rpms:
- distgit_key: openshift-kuryr
metadata: # changes to make the metadata
content:
source:
git:
url: [email protected]:jupierce/kuryr-kubernetes.git
branch:
target: 1_hash
group:
arches:
- x86_64
- ppc64le
- s390x
advisories:
image: 11
extras: 12
ART_2:
assembly:
basis:
brew_event: 5
members:
rpms:
- distgit_key: openshift-kuryr
metadata: # changes to make the metadata
content:
source:
git:
url: [email protected]:jupierce/kuryr-kubernetes.git
branch:
target: 2_hash
group:
arches:
- x86_64
- s390x
advisories:
image: 21
ART_3:
assembly:
basis:
assembly: ART_2
group:
advisories:
image: 31
ART_4:
assembly:
basis:
assembly: ART_3
group:
advisories!:
image: 41
ART_5:
assembly:
basis:
assembly: ART_4
group:
arches!:
- s390x
advisories!:
image: 51
ART_6:
assembly:
basis:
assembly: ART_5
members:
rpms:
- distgit_key: '*'
metadata:
content:
source:
git:
branch:
target: customer_6
ART_7:
assembly:
basis:
brew_event: 5
members:
images:
- distgit_key: openshift-kuryr
metadata:
content:
source:
git:
url: [email protected]:jupierce/kuryr-kubernetes.git
branch:
target: 1_hash
is: kuryr-nvr
dependencies:
rpms:
- el7: some-nvr-1
non_gc_tag: some-tag-1
group:
dependencies:
rpms:
- el7: some-nvr-3
non_gc_tag: some-tag-3
rhcos:
machine-os-content:
images:
x86_64: registry.example.com/rhcos-x86_64:test
dependencies:
rpms:
- el7: some-nvr-4
non_gc_tag: some-tag-4
- el8: some-nvr-5
non_gc_tag: some-tag-4
ART_8:
assembly:
basis:
assembly: ART_7
members:
images:
- distgit_key: openshift-kuryr
metadata:
is: kuryr-nvr2
dependencies:
rpms:
- el7: some-nvr-2
non_gc_tag: some-tag-2
group:
dependencies:
rpms:
- el7: some-nvr-4
non_gc_tag: some-tag-4
rhcos:
machine-os-content:
images: {}
dependencies:
rpms:
- el8: some-nvr-6
non_gc_tag: some-tag-6
ART_INFINITE:
assembly:
basis:
assembly: ART_INFINITE
members:
rpms:
- distgit_key: '*'
metadata:
content:
source:
git:
branch:
target: customer_6
"""
self.releases_config = Model(
dict_to_model=yaml.safe_load(releases_yml))
async def run(self):
self.working_dir.mkdir(parents=True, exist_ok=True)
build_data_repo = self.working_dir / "ocp-build-data-push"
shutil.rmtree(build_data_repo, ignore_errors=True)
shutil.rmtree(self.elliott_working_dir, ignore_errors=True)
shutil.rmtree(self.doozer_working_dir, ignore_errors=True)
release_config = None
group_config = await self.load_group_config()
if self.assembly != "stream":
releases_config = await self.load_releases_config()
release_config = releases_config.get("releases",
{}).get(self.assembly, {})
if not release_config:
raise ValueError(
f"Assembly {self.assembly} is not defined in releases.yml for group {self.group_name}."
)
group_config = assembly_group_config(
Model(releases_config), self.assembly,
Model(group_config)).primitive()
asssembly_type = release_config.get("assembly",
{}).get("type", "standard")
if asssembly_type == "standard":
self.release_name = self.assembly
self.release_version = tuple(
map(int, self.release_name.split(".", 2)))
elif asssembly_type == "custom":
self.release_name = f"{self.release_version[0]}.{self.release_version[1]}.0-assembly.{self.assembly}"
elif asssembly_type == "candidate":
self.release_name = f"{self.release_version[0]}.{self.release_version[1]}.0-{self.assembly}"
nightlies = release_config.get("assembly", {}).get(
"basis", {}).get("reference_releases", {}).values()
self.candidate_nightlies = self.parse_nighties(nightlies)
if release_config and asssembly_type != "standard":
_LOGGER.warning("No need to check Blocker Bugs for assembly %s",
self.assembly)
else:
_LOGGER.info("Checking Blocker Bugs for release %s...",
self.release_name)
self.check_blockers()
advisories = {}
if self.default_advisories:
advisories = group_config.get("advisories", {})
else:
_LOGGER.info("Creating advisories for release %s...",
self.release_name)
if release_config:
advisories = group_config.get("advisories", {}).copy()
if self.release_version[2] == 0: # GA release
if advisories.get("rpm", 0) <= 0:
advisories["rpm"] = self.create_advisory(
"RHEA", "rpm", "ga")
if advisories.get("image", 0) <= 0:
advisories["image"] = self.create_advisory(
"RHEA", "image", "ga")
else: # z-stream release
if advisories.get("rpm", 0) <= 0:
advisories["rpm"] = self.create_advisory(
"RHBA", "rpm", "standard")
if advisories.get("image", 0) <= 0:
advisories["image"] = self.create_advisory(
"RHBA", "image", "standard")
if self.release_version[0] > 3:
if advisories.get("extras", 0) <= 0:
advisories["extras"] = self.create_advisory(
"RHBA", "image", "extras")
if advisories.get("metadata", 0) <= 0:
advisories["metadata"] = self.create_advisory(
"RHBA", "image", "metadata")
_LOGGER.info("Ensuring JIRA ticket for release %s...",
self.release_name)
jira_issue_key = group_config.get("release_jira")
jira_template_vars = {
"release_name": self.release_name,
"x": self.release_version[0],
"y": self.release_version[1],
"z": self.release_version[2],
"release_date": self.release_date,
"advisories": advisories,
"candidate_nightlies": self.candidate_nightlies,
}
if jira_issue_key:
_LOGGER.info("Reusing existing release JIRA %s", jira_issue_key)
jira_issue = self._jira_client.get_issue(jira_issue_key)
subtasks = [
self._jira_client.get_issue(subtask.key)
for subtask in jira_issue.fields.subtasks
]
self.update_release_jira(jira_issue, subtasks, jira_template_vars)
else:
_LOGGER.info("Creating a release JIRA...")
jira_issues = self.create_release_jira(jira_template_vars)
jira_issue = jira_issues[0] if jira_issues else None
jira_issue_key = jira_issue.key if jira_issue else None
_LOGGER.info("Updating ocp-build-data...")
build_data_changed = await self.update_build_data(
advisories, jira_issue_key)
_LOGGER.info("Sweep builds into the the advisories...")
for kind, advisory in advisories.items():
if not advisory:
continue
if kind == "rpm":
self.sweep_builds("rpm", advisory)
elif kind == "image":
self.sweep_builds("image",
advisory,
only_payload=self.release_version[0] >= 4)
elif kind == "extras":
self.sweep_builds("image", advisory, only_non_payload=True)
elif kind == "metadata":
await self.build_and_attach_bundles(advisory)
# bugs should be swept after builds to have validation
# for only those bugs to be attached which have corresponding brew builds
# attached to the advisory
# currently for rpm advisory and cves only
_LOGGER.info("Sweep bugs into the the advisories...")
self.sweep_bugs(check_builds=True)
_LOGGER.info("Adding placeholder bugs...")
for kind, advisory in advisories.items():
bug_ids = get_bug_ids(advisory)
if not bug_ids: # Only create placeholder bug if the advisory has no attached bugs
_LOGGER.info("Create placeholder bug for %s advisory %s...",
kind, advisory)
self.create_and_attach_placeholder_bug(kind, advisory)
# Verify the swept builds match the nightlies
if self.release_version[0] < 4:
_LOGGER.info("Don't verify payloads for OCP3 releases")
else:
_LOGGER.info("Verify the swept builds match the nightlies...")
for _, payload in self.candidate_nightlies.items():
self.verify_payload(payload, advisories["image"])
if build_data_changed or self.candidate_nightlies:
_LOGGER.info("Sending a notification to QE and multi-arch QE...")
if self.dry_run:
jira_issue_link = "https://jira.example.com/browse/FOO-1"
else:
jira_issue_link = jira_issue.permalink()
self.send_notification_email(advisories, jira_issue_link)
# Move advisories to QE
for kind, advisory in advisories.items():
try:
if kind == "metadata":
# Verify attached operators
await self.verify_attached_operators(
advisories["image"], advisories["extras"],
advisories["metadata"])
self.change_advisory_state(advisory, "QE")
except CalledProcessError as ex:
_LOGGER.warning(
f"Unable to move {kind} advisory {advisory} to QE: {ex}")