def __init__(self, pyxis=None):
super().__init__()
if pyxis:
self._pyxis = pyxis
else:
if not conf.pyxis_server_url:
raise ValueError("'pyxis_server_url' parameter should be set")
self._pyxis = Pyxis(conf.pyxis_server_url)
def __init__(self, pyxis=None):
super().__init__()
if pyxis:
self._pyxis = pyxis
else:
if not conf.pyxis_server_url:
raise ValueError("'PYXIS_SERVER_URL' parameter should be set")
self._pyxis = Pyxis(conf.pyxis_server_url)
if not conf.freshmaker_root_url or "://" not in conf.freshmaker_root_url:
raise ValueError(
"'FRESHMAKER_ROOT_URL' parameter should be set to "
"a valid URL")
def setUp(self):
super().setUp()
self.patcher = helpers.Patcher('freshmaker.pyxis.')
self.fake_server_url = 'https://pyxis.localhost/'
self.px = Pyxis(self.fake_server_url)
self.response = create_autospec(requests.Response)
self.response.status_code = HTTPStatus.OK
self.bad_requests_response = {
"detail": [
"Unable to parse the filter from URL.",
"Please verify the 'Field Name' in the RSQL Expression.",
"Please visit the following end-point for more details:",
" /v1/docs/filtering-language"
],
"status":
400,
"title":
"Bad Request",
"type":
"about:blank"
}
self.empty_response_page = {
"data": [],
"page": 0,
"page_size": 100,
"total": 0
}
self.indices = [{
"_id": "1",
"created_by": "meteor",
"creation_date": "2020-01-01T09:32:31.692000+00:00",
"last_update_date": "2020-01-01T09:32:31.692000+00:00",
"last_updated_by": "meteor",
"ocp_version": "4.5",
"organization": "org",
"path": "path/to/registry:v4.5"
}, {
"_id": "2",
"created_by": "meteor",
"creation_date": "2020-01-01T09:32:38.486000+00:00",
"last_update_date": "2020-01-01T09:32:38.486000+00:00",
"last_updated_by": "meteor",
"ocp_version": "4.6",
"organization": "org",
"path": "path/to/registry:v4.6"
}, {
"_id": "2",
"created_by": "meteor",
"creation_date": "2020-01-01T09:32:38.486000+00:00",
"last_update_date": "2020-01-01T09:32:38.486000+00:00",
"last_updated_by": "meteor",
"ocp_version": "4.6",
"organization": "org",
"path": ""
}]
self.bundles = [
{
"channel_name":
"streams-1.5.x",
"related_images": [{
"image": "registry/amq7/amq-streams-r-operator@sha256:111",
"name": "strimzi-cluster-operator",
"digest": "sha256:111"
}, {
"image": "registry/amq7/amq-streams-kafka-24-r@sha256:222",
"name": "strimzi-kafka-24",
"digest": "sha256:222"
}, {
"image": "registry/amq7/amq-streams-kafka-25-r@sha256:333",
"name": "strimzi-kafka-25",
"digest": "sha256:333"
}, {
"image": "registry/amq7/amq-streams-bridge-r@sha256:444",
"name": "strimzi-bridge",
"digest": "sha256:444"
}],
"version":
"1.5.3"
},
{
"channel_name":
"streams-1.5.x",
"related_images": [{
"image": "registry/amq7/amq-streams-r-operator@sha256:555",
"name": "strimzi-cluster-operator",
"digest": "sha256:555"
}, {
"image": "registry/amq7/amq-streams-kafka-24-r@sha256:666",
"name": "strimzi-kafka-24",
"digest": "sha256:666"
}, {
"image": "registry/amq7/amq-streams-kafka-25-r@sha256:777",
"name": "strimzi-kafka-25",
"digest": "sha256:777"
}, {
"image": "registry/amq7/amq-streams-bridge-r@sha256:888",
"name": "strimzi-bridge",
"digest": "sha256:888"
}],
"version":
"1.5.4"
},
{
"channel_name":
"stable",
"related_images": [{
"image": "registry/amq7/amq--operator@sha256:999",
"name": "strimzi-cluster-operator",
"digest": "sha256:999"
}, {
"image": "registry/amq7/kafka-24-r@sha256:aaa",
"name": "strimzi-kafka-24",
"digest": "sha256:aaa"
}, {
"image": "registry/amq7/kafka-25-r@sha256:bbb",
"name": "strimzi-kafka-25",
"digest": "sha256:bbb"
}, {
"image": "registry/amq7/amq-streams-bridge-r@sha256:ccc",
"name": "strimzi-bridge",
"digest": "sha256:ccc"
}],
"version":
"1.5.3"
},
{
"channel_name":
"stable",
"related_images": [{
"image": "registry/tracing/j-operator:1.13.2",
"name": "j-1.13.2-annotation",
"digest": "sha256:fff"
}, {
"image": "registry/tracing/j-operator:1.13.2",
"name": "j-operator",
"digest": "sha256:ffff"
}],
"version":
"1.5.2"
},
{
"channel_name":
"quay-v3.3",
"related_images": [{
"image": "registry/quay/quay-operator@sha256:ddd",
"name": "quay-operator-annotation",
"digest": "sha256:ddd"
}, {
"image":
"registry/quay/quay-security-r-operator@sha256:eee",
"name": "container-security-operator",
"digest": "sha256:eee"
}],
"version":
"3.3.1"
},
]
self.images = [{
"brew": {
"build": "s2i-1-2",
"completion_date": "2020-08-12T11:31:39+00:00",
"nvra": "s2i-1-2.ppc64le",
"package": "s2i-core-container"
},
"repositories": [{
"manifest_list_digest": "sha256:1111",
"published": False,
"registry": "reg1",
"repository": "repo1",
"tags": [{
"name": "tag0"
}]
}, {
"manifest_list_digest": "sha256:1112",
"published": True,
"registry": "reg2",
"repository": "repo2",
"tags": [{
"name": "tag1"
}, {
"name": "tag2"
}]
}]
}, {
"brew": {
"build": "s2i-1-2",
"completion_date": "2020-08-12T11:31:39+00:00",
"nvra": "s2i-1-2.s390x",
"package": "s2i-core-container"
},
"repositories": [{
"manifest_list_digest": "sha256:2222",
"published": True,
"registry": "reg2",
"repository": "repo2",
"tags": [{
"name": "tag2"
}]
}]
}, {
"brew": {
"build": "s2i-1-2",
"completion_date": "2020-08-12T11:31:39+00:00",
"nvra": "s2i-1-2.amd64",
"package": "s2i-core-container"
},
"repositories": [{
"manifest_list_digest": "sha256:3333",
"published": True,
"registry": "reg3",
"repository": "repo3",
"tags": [{
"name": "latest"
}]
}]
}, {
"brew": {
"build": "s2i-1-2",
"completion_date": "2020-08-12T11:31:39+00:00",
"nvra": "s2i-1-2.arm64",
"package": "s2i-core-container"
},
"repositories": [{
"manifest_list_digest": "sha256:4444",
"published": True,
"registry": "reg4",
"repository": "repo4",
"tags": [{
"name": "tag1"
}]
}]
}]
class TestQueryPyxis(helpers.FreshmakerTestCase):
def setUp(self):
super().setUp()
self.patcher = helpers.Patcher('freshmaker.pyxis.')
self.fake_server_url = 'https://pyxis.localhost/'
self.px = Pyxis(self.fake_server_url)
self.response = create_autospec(requests.Response)
self.response.status_code = HTTPStatus.OK
self.bad_requests_response = {
"detail": [
"Unable to parse the filter from URL.",
"Please verify the 'Field Name' in the RSQL Expression.",
"Please visit the following end-point for more details:",
" /v1/docs/filtering-language"
],
"status":
400,
"title":
"Bad Request",
"type":
"about:blank"
}
self.empty_response_page = {
"data": [],
"page": 0,
"page_size": 100,
"total": 0
}
self.indices = [{
"_id": "1",
"created_by": "meteor",
"creation_date": "2020-01-01T09:32:31.692000+00:00",
"last_update_date": "2020-01-01T09:32:31.692000+00:00",
"last_updated_by": "meteor",
"ocp_version": "4.5",
"organization": "org",
"path": "path/to/registry:v4.5"
}, {
"_id": "2",
"created_by": "meteor",
"creation_date": "2020-01-01T09:32:38.486000+00:00",
"last_update_date": "2020-01-01T09:32:38.486000+00:00",
"last_updated_by": "meteor",
"ocp_version": "4.6",
"organization": "org",
"path": "path/to/registry:v4.6"
}, {
"_id": "2",
"created_by": "meteor",
"creation_date": "2020-01-01T09:32:38.486000+00:00",
"last_update_date": "2020-01-01T09:32:38.486000+00:00",
"last_updated_by": "meteor",
"ocp_version": "4.6",
"organization": "org",
"path": ""
}]
self.bundles = [
{
"channel_name":
"streams-1.5.x",
"related_images": [{
"image": "registry/amq7/amq-streams-r-operator@sha256:111",
"name": "strimzi-cluster-operator",
"digest": "sha256:111"
}, {
"image": "registry/amq7/amq-streams-kafka-24-r@sha256:222",
"name": "strimzi-kafka-24",
"digest": "sha256:222"
}, {
"image": "registry/amq7/amq-streams-kafka-25-r@sha256:333",
"name": "strimzi-kafka-25",
"digest": "sha256:333"
}, {
"image": "registry/amq7/amq-streams-bridge-r@sha256:444",
"name": "strimzi-bridge",
"digest": "sha256:444"
}],
"version":
"1.5.3"
},
{
"channel_name":
"streams-1.5.x",
"related_images": [{
"image": "registry/amq7/amq-streams-r-operator@sha256:555",
"name": "strimzi-cluster-operator",
"digest": "sha256:555"
}, {
"image": "registry/amq7/amq-streams-kafka-24-r@sha256:666",
"name": "strimzi-kafka-24",
"digest": "sha256:666"
}, {
"image": "registry/amq7/amq-streams-kafka-25-r@sha256:777",
"name": "strimzi-kafka-25",
"digest": "sha256:777"
}, {
"image": "registry/amq7/amq-streams-bridge-r@sha256:888",
"name": "strimzi-bridge",
"digest": "sha256:888"
}],
"version":
"1.5.4"
},
{
"channel_name":
"stable",
"related_images": [{
"image": "registry/amq7/amq--operator@sha256:999",
"name": "strimzi-cluster-operator",
"digest": "sha256:999"
}, {
"image": "registry/amq7/kafka-24-r@sha256:aaa",
"name": "strimzi-kafka-24",
"digest": "sha256:aaa"
}, {
"image": "registry/amq7/kafka-25-r@sha256:bbb",
"name": "strimzi-kafka-25",
"digest": "sha256:bbb"
}, {
"image": "registry/amq7/amq-streams-bridge-r@sha256:ccc",
"name": "strimzi-bridge",
"digest": "sha256:ccc"
}],
"version":
"1.5.3"
},
{
"channel_name":
"stable",
"related_images": [{
"image": "registry/tracing/j-operator:1.13.2",
"name": "j-1.13.2-annotation",
"digest": "sha256:fff"
}, {
"image": "registry/tracing/j-operator:1.13.2",
"name": "j-operator",
"digest": "sha256:ffff"
}],
"version":
"1.5.2"
},
{
"channel_name":
"quay-v3.3",
"related_images": [{
"image": "registry/quay/quay-operator@sha256:ddd",
"name": "quay-operator-annotation",
"digest": "sha256:ddd"
}, {
"image":
"registry/quay/quay-security-r-operator@sha256:eee",
"name": "container-security-operator",
"digest": "sha256:eee"
}],
"version":
"3.3.1"
},
]
self.images = [{
"brew": {
"build": "s2i-1-2",
"completion_date": "2020-08-12T11:31:39+00:00",
"nvra": "s2i-1-2.ppc64le",
"package": "s2i-core-container"
},
"repositories": [{
"manifest_list_digest": "sha256:1111",
"published": False,
"registry": "reg1",
"repository": "repo1",
"tags": [{
"name": "tag0"
}]
}, {
"manifest_list_digest": "sha256:1112",
"published": True,
"registry": "reg2",
"repository": "repo2",
"tags": [{
"name": "tag1"
}, {
"name": "tag2"
}]
}]
}, {
"brew": {
"build": "s2i-1-2",
"completion_date": "2020-08-12T11:31:39+00:00",
"nvra": "s2i-1-2.s390x",
"package": "s2i-core-container"
},
"repositories": [{
"manifest_list_digest": "sha256:2222",
"published": True,
"registry": "reg2",
"repository": "repo2",
"tags": [{
"name": "tag2"
}]
}]
}, {
"brew": {
"build": "s2i-1-2",
"completion_date": "2020-08-12T11:31:39+00:00",
"nvra": "s2i-1-2.amd64",
"package": "s2i-core-container"
},
"repositories": [{
"manifest_list_digest": "sha256:3333",
"published": True,
"registry": "reg3",
"repository": "repo3",
"tags": [{
"name": "latest"
}]
}]
}, {
"brew": {
"build": "s2i-1-2",
"completion_date": "2020-08-12T11:31:39+00:00",
"nvra": "s2i-1-2.arm64",
"package": "s2i-core-container"
},
"repositories": [{
"manifest_list_digest": "sha256:4444",
"published": True,
"registry": "reg4",
"repository": "repo4",
"tags": [{
"name": "tag1"
}]
}]
}]
def tearDown(self):
super().tearDown()
self.patcher.unpatch_all()
@staticmethod
def copy_call_args(mock):
"""
Copy args of Mock to another Mock so we can check call args if we call
mock with mutable args and change it between calls
"""
new_mock = Mock()
def side_effect(*args, **kwargs):
args = deepcopy(args)
kwargs = deepcopy(kwargs)
return new_mock(*args, **kwargs)
mock.side_effect = side_effect
return new_mock
@patch('freshmaker.pyxis.HTTPKerberosAuth')
@patch('freshmaker.pyxis.requests.get')
def test_make_request(self, get, auth):
get.return_value = self.response
test_params = {'key1': 'val1'}
self.px._make_request('test', test_params)
get_url = self.fake_server_url + 'v1/test'
self.response.json.assert_called_once()
test_params['page_size'] = "100"
get.assert_called_once_with(get_url,
params=test_params,
auth=auth(),
timeout=conf.net_timeout)
@patch('freshmaker.pyxis.HTTPKerberosAuth')
@patch('freshmaker.pyxis.requests.get')
def test_make_request_error(self, get, auth):
get.return_value = self.response
self.response.ok = False
self.response.json.side_effect = ValueError
self.response.json.text = 'test message'
self.response.request = Mock()
self.response.request.url = 'test/url'
with self.assertRaises(PyxisRequestError, msg='test message'):
self.px._make_request('test', {})
@patch('freshmaker.pyxis.HTTPKerberosAuth')
@patch('freshmaker.pyxis.Pyxis._make_request')
def test_pagination(self, request, auth):
my_request = self.copy_call_args(request)
my_request.side_effect = [{
"page": 0,
"data": ["fake_data1"]
}, {
"page": 1,
"data": ["fake_data2"]
}, {
"page": 2,
"data": []
}]
test_params = {'include': ['total', 'field1']}
entity = 'test'
auth.return_value = 1
self.px._pagination(entity, test_params)
self.assertEqual(request.call_count, 3)
default_params = {'page_size': '100', 'include': ['total', 'field1']}
calls = [
call('test', params={
**default_params, 'page': 0
}),
call('test', params={
**default_params, 'page': 1
}),
call('test', params={
**default_params, 'page': 2
})
]
my_request.assert_has_calls(calls)
@patch.object(conf, 'pyxis_index_image_organization', new='org')
@patch('freshmaker.pyxis.Pyxis._pagination')
def test_get_operator_indices(self, page):
self.px.get_operator_indices()
page.assert_called_once_with('operators/indices',
{'filter': 'organization==org'})
@patch.object(conf,
"product_pages_api_url",
new="http://pp.example.com/api")
@patch("freshmaker.pyxis.Pyxis._pagination")
def test_get_operator_indices_with_unreleased_filtered_out(self, page):
pp_mock_data = [{
"url":
"http://pp.example.com/api/releases/openshift-4.5/schedule-tasks",
"json": [{
"name": "GA",
"date_finish": "2020-02-05"
}]
}, {
"url":
"http://pp.example.com/api/releases/openshift-4.6/schedule-tasks",
"json": [{
"name": "GA",
"date_finish": "2020-05-23"
}]
}, {
"url":
"http://pp.example.com/api/releases/openshift-4.8/schedule-tasks",
"json": [{
"name": "GA",
"date_finish": "2021-08-12"
}]
}]
page.return_value = self.indices + [{
"_id": "3",
"created_by": "meteor",
"creation_date": "2020-11-01T08:23:28.253000+00:00",
"last_update_date": "2020-11-01T08:23:28.253000+00:00",
"last_updated_by": "meteor",
"ocp_version": "4.8",
"organization": "org",
"path": ""
}]
now = datetime(year=2020, month=12, day=15, hour=0, minute=0, second=0)
with requests_mock.Mocker() as http:
for data in pp_mock_data:
http.get(data["url"], json=data["json"])
with freeze_time(now):
indices = self.px.get_operator_indices()
assert len(indices) == 3
assert "4.8" not in [i["ocp_version"] for i in indices]
@patch('freshmaker.pyxis.Pyxis._pagination')
def test_get_bundles_per_index_image(self, page):
page_copy = self.copy_call_args(page)
page_copy.side_effect = [self.bundles[0:2], self.bundles[2:]]
out = self.px._get_bundles_per_index_image(self.indices)
expected_out = {
"path/to/registry:v4.5": self.bundles[0:2],
"path/to/registry:v4.6": self.bundles[2:]
}
self.assertEqual(out, expected_out)
page.assert_has_calls([
call(
'operators/bundles', {
'include':
'data.channel_name,data.version,'
'data.related_images,data.bundle_path_digest,'
'data.bundle_path',
'filter':
'source_index_container_path==path/to/registry:v4.6'
}),
call(
'operators/bundles', {
'include':
'data.channel_name,data.version,'
'data.related_images,data.bundle_path_digest,'
'data.bundle_path',
'filter':
'source_index_container_path==path/to/registry:v4.6'
})
])
@patch('freshmaker.pyxis.Pyxis._get_bundles_per_index_image')
def test_get_latest_bundles(self, get_bundles_per_indices):
get_bundles_per_indices.return_value = {
"path/to/registry:v4.5": self.bundles[0:2],
"path/to/registry:v4.6": self.bundles[2:]
}
out = self.px.get_latest_bundles(self.indices)
# we expect 0 and 3 bundles to be filtered out, because of older versions
expected_out = self.bundles[1:3] + self.bundles[4:]
self.assertEqual(out, expected_out)
get_bundles_per_indices.assert_called_once()
@patch('freshmaker.pyxis.Pyxis._get_bundles_per_index_image')
def test_get_latest_bundles_invalid_version(self, get_bundles_per_indices):
# set invalid version
for bundle in self.bundles:
bundle['version'] = "InvalidVersion"
get_bundles_per_indices.return_value = {
"path/to/registry:v4.5": self.bundles[0:2],
"path/to/registry:v4.6": self.bundles[2:]
}
with self.assertLogs("freshmaker", level="WARNING"):
bundles = self.px.get_latest_bundles(self.indices)
self.assertEqual(bundles, [])
get_bundles_per_indices.assert_called_once()
@patch('freshmaker.pyxis.Pyxis._pagination')
def test_get_digests_by_nvrs(self, page):
page.return_value = self.images
digests = self.px.get_digests_by_nvrs(['s2i-1-2'])
expected_digests = {
'sha256:1112', 'sha256:4444', 'sha256:3333', 'sha256:2222'
}
self.assertEqual(digests, expected_digests)
page.assert_called_once_with(
'images/nvr/s2i-1-2', {'include': 'data.brew,data.repositories'})
def test_filter_bundles_by_related_image_digests(self):
digests = {'sha256:111', 'sha256:bbb', 'sha256:ddd'}
new_bundles = self.px.filter_bundles_by_related_image_digests(
digests, self.bundles)
expected_bundles = [self.bundles[0], self.bundles[2], self.bundles[4]]
self.assertListEqual(new_bundles, expected_bundles)
@patch('freshmaker.pyxis.Pyxis._pagination')
def test_get_images_by_digests(self, page):
my_pagination = self.copy_call_args(page)
my_pagination.return_value = [self.images[0], self.images[1]]
images = self.px.get_images_by_digests(['sha256:111', 'sha256:222'])
my_pagination.assert_called_once_with(
'images', {
'include':
'data.brew,data.repositories',
'filter':
'repositories.manifest_list_digest=in='
'(sha256:111,sha256:222)'
})
self.assertEqual(page.call_count, 1)
self.assertEqual(self.images[0:2], images)
@patch('freshmaker.pyxis.Pyxis._pagination')
def test_add_repositories_info(self, page):
reg_repo_info = {
('reg1', 'rep1'): {
'nvrs': {'nvr1', 'nvr2'}
},
('reg1', 'rep2'): {
'nvrs': {'nvr3'}
},
('reg2', 'rep1'): {
'nvrs': {'nvr4', 'nvr5'}
},
('reg3', 'rep3'): {
'nvrs': {'nvr6', 'nvr7'}
},
}
page.return_value = [{
'auto_rebuild_tags': ['tag1', 'tag2'],
'registry': 'reg1',
'repository': 'rep1'
}, {
'auto_rebuild_tags': ['latest'],
'registry': 'reg1',
'repository': 'rep2'
}, {
'auto_rebuild_tags': ['tag1', 'tag2'],
'registry': 'reg2',
'repository': 'rep1'
}, {
'registry': 'reg3',
'repository': 'rep3'
}]
self.px._add_repositories_info(reg_repo_info)
params = {
'include':
'data.auto_rebuild_tags,data.registry,data.repository',
'filter':
'(registry==reg1;repository==rep1),'
'(registry==reg1;repository==rep2),'
'(registry==reg2;repository==rep1),'
'(registry==reg3;repository==rep3)'
}
page.assert_called_with('repositories', params)
expected_info = {
('reg1', 'rep1'): {
'nvrs': {'nvr1', 'nvr2'},
'auto_rebuild_tags': {'tag1', 'tag2'}
},
('reg1', 'rep2'): {
'nvrs': {'nvr3'},
'auto_rebuild_tags': {'latest'}
},
('reg2', 'rep1'): {
'nvrs': {'nvr4', 'nvr5'},
'auto_rebuild_tags': {'tag1', 'tag2'}
},
}
self.assertEqual(reg_repo_info, expected_info)
@patch('freshmaker.pyxis.Pyxis._pagination')
def test_filter_auto_rebuild_nvrs(self, page):
my_page = self.copy_call_args(page)
reg_repo_info = {
('reg1', 'rep1'): {
'nvrs': {'nvr1', 'nvr2'},
'auto_rebuild_tags': ['tag1', 'tag2']
},
('reg1', 'rep2'): {
'nvrs': {'nvr3'},
'auto_rebuild_tags': ['latest']
},
('reg2', 'rep1'): {
'nvrs': {'nvr4', 'nvr5'},
'auto_rebuild_tags': ['tag1', 'tag2']
},
('reg3', 'rep3'): {
'auto_rebuild_tags': ['tag1', 'tag2']
}
}
my_page.side_effect = [[{
'brew': {
'build': 'nvr1'
}
}, {
'brew': {
'build': 'nvr2'
}
}], [], [], [{
'brew': {
'build': 'nvr4'
}
}], [{
'brew': {
'build': 'nvr5'
}
}], []]
ret = self.px._filter_auto_rebuild_nvrs(reg_repo_info)
expected = {'nvr1', 'nvr2', 'nvr4', 'nvr5'}
self.assertEqual(ret, expected)
self.assertEqual(my_page.call_count, 5)
# using .join() method because it's unclear how set will be ordered
my_page.assert_has_calls([
call(
'repositories/registry/reg1/repository/rep1/tag/tag1', {
'include': 'data.brew.build',
'filter': f'brew.build=in=({",".join({"nvr1", "nvr2"})})'
}),
call(
'repositories/registry/reg1/repository/rep1/tag/tag2', {
'include': 'data.brew.build',
'filter': f'brew.build=in=({",".join({"nvr1", "nvr2"})})'
}),
call('repositories/registry/reg1/repository/rep2/tag/latest', {
'include': 'data.brew.build',
'filter': 'brew.build=in=(nvr3)'
}),
call(
'repositories/registry/reg2/repository/rep1/tag/tag1', {
'include': 'data.brew.build',
'filter': f'brew.build=in=({",".join({"nvr4","nvr5"})})'
}),
call(
'repositories/registry/reg2/repository/rep1/tag/tag2', {
'include': 'data.brew.build',
'filter': f'brew.build=in=({",".join({"nvr4","nvr5"})})'
}),
])
@patch('freshmaker.pyxis.Pyxis._filter_auto_rebuild_nvrs')
@patch('freshmaker.pyxis.Pyxis._add_repositories_info')
def test_get_auto_rebuild_tagged_images(self, info, tag_filter):
tag_filter.return_value = {'nvr1', 'nvr2', 'nvr4', 'nvr5'}
# change nvr for the second image to have more variations to test
self.images[1]['brew']['build'] = 'nvr1'
ret = self.px.get_auto_rebuild_tagged_images(self.images)
info.assert_called_once_with({
('reg1', 'repo1'): {
'nvrs': {'s2i-1-2'}
},
('reg2', 'repo2'): {
'nvrs': {'s2i-1-2', 'nvr1'}
},
('reg3', 'repo3'): {
'nvrs': {'s2i-1-2'}
},
('reg4', 'repo4'): {
'nvrs': {'s2i-1-2'}
},
})
expected_ret = {'nvr1', 'nvr2', 'nvr4', 'nvr5'}
self.assertEqual(ret, expected_ret)
@patch('freshmaker.pyxis.Pyxis._filter_auto_rebuild_nvrs')
@patch('freshmaker.pyxis.Pyxis._add_repositories_info')
def test_no_nvr_and_repos(self, info, tag_filter):
del self.images[0]['brew']['build']
del self.images[1]['repositories']
self.images[1]['brew']['build'] = 'nvr1'
with self.assertLogs('freshmaker', level='WARNING') as log:
self.px.get_auto_rebuild_tagged_images(self.images)
self.assertTrue('One of bundle images doesn\'t have brew.build' in
log.output[0])
self.assertTrue('Bundle image nvr1 doesn\'t have repositories set'
in log.output[1])
class HandleBotasAdvisory(ContainerBuildHandler):
"""
Handles event that was created by transition of an advisory filed by
BOTAS to SHIPPED_LIVE state
"""
name = "HandleBotasAdvisory"
def __init__(self, pyxis=None):
super().__init__()
if pyxis:
self._pyxis = pyxis
else:
if not conf.pyxis_server_url:
raise ValueError("'PYXIS_SERVER_URL' parameter should be set")
self._pyxis = Pyxis(conf.pyxis_server_url)
if not conf.freshmaker_root_url or "://" not in conf.freshmaker_root_url:
raise ValueError(
"'FRESHMAKER_ROOT_URL' parameter should be set to "
"a valid URL")
def can_handle(self, event):
if (isinstance(event, BotasErrataShippedEvent)
and 'docker' in event.advisory.content_types):
return True
return False
def handle(self, event):
if event.dry_run:
self.force_dry_run()
self.event = event
db_event = Event.get_or_create_from_event(db.session, event)
self.set_context(db_event)
# Check if event is allowed by internal policies
if not self.event.is_allowed(self):
msg = ("This image rebuild is not allowed by internal policy. "
f"message_id: {event.msg_id}")
db_event.transition(EventState.SKIPPED, msg)
self.log_info(msg)
return []
# Mapping of original build nvrs to rebuilt nvrs in advisory
nvrs_mapping = self._create_original_to_rebuilt_nvrs_map()
original_nvrs = nvrs_mapping.keys()
self.log_info(
"Orignial nvrs of build in the advisory #{0} are: {1}".format(
event.advisory.errata_id, " ".join(original_nvrs)))
# Get image manifest_list_digest for all original images, manifest_list_digest is used
# in pullspecs in bundle's related images
original_digests_by_nvr = {}
original_nvrs_by_digest = {}
for nvr in original_nvrs:
digest = self._pyxis.get_manifest_list_digest_by_nvr(nvr)
if digest:
original_digests_by_nvr[nvr] = digest
original_nvrs_by_digest[digest] = nvr
else:
log.warning(
f"Image manifest_list_digest not found for original image {nvr} in Pyxis, "
"skip this image")
if not original_digests_by_nvr:
msg = f"None of the original images have digests in Pyxis: {','.join(original_nvrs)}"
log.warning(msg)
db_event.transition(EventState.SKIPPED, msg)
return []
# Get image manifest_list_digest for all rebuilt images, manifest_list_digest is used
# in pullspecs of bundle's related images
rebuilt_digests_by_nvr = {}
rebuilt_nvrs = nvrs_mapping.values()
for nvr in rebuilt_nvrs:
digest = self._pyxis.get_manifest_list_digest_by_nvr(nvr)
if digest:
rebuilt_digests_by_nvr[nvr] = digest
else:
log.warning(
f"Image manifest_list_digest not found for rebuilt image {nvr} in Pyxis, "
"skip this image")
if not rebuilt_digests_by_nvr:
msg = f"None of the rebuilt images have digests in Pyxis: {','.join(rebuilt_nvrs)}"
log.warning(msg)
db_event.transition(EventState.SKIPPED, msg)
return []
index_images = self._pyxis.get_operator_indices()
# get latest bundle images per channel per index image filtered
# by the highest semantic version
all_bundles = self._pyxis.get_latest_bundles(index_images)
# A mapping of digests to bundle metadata. This metadata is used to
# for the CSV metadata updates.
bundle_mds_by_digest = {}
# get bundle digests for original images
bundle_digests_by_related_nvr = {}
for image_nvr, image_digest in original_digests_by_nvr.items():
bundles = self._pyxis.get_bundles_by_related_image_digest(
image_digest, all_bundles)
if not bundles:
log.info(
f"No latest bundle image with the related image of {image_nvr}"
)
continue
for bundle in bundles:
bundle_digest = bundle['bundle_path_digest']
bundle_mds_by_digest[bundle_digest] = bundle
bundle_digests_by_related_nvr.setdefault(
image_nvr, []).append(bundle_digest)
if not bundle_digests_by_related_nvr:
msg = "None of the original images have related bundles, skip."
log.warning(msg)
db_event.transition(EventState.SKIPPED, msg)
return []
# Mapping of bundle digest to bundle data
# {
# digest: {
# "images": [image_amd64, image_aarch64],
# "nvr": NVR,
# "auto_rebuild": True/False,
# "osbs_pinning": True/False,
# "pullspecs": [...],
# }
# }
bundles_by_digest = {}
default_bundle_data = {
'images': [],
'nvr': None,
'auto_rebuild': False,
'osbs_pinning': False,
# CSV modifications for the rebuilt bundle image
'pullspecs': [],
'append': {},
'update': {},
}
# Get images for each bundle digest, a bundle digest can have multiple images
# with different arches.
for digest in bundle_mds_by_digest:
bundles = self._pyxis.get_images_by_digest(digest)
# If no bundle image found, just skip this bundle digest
if not bundles:
continue
bundles_by_digest.setdefault(digest,
copy.deepcopy(default_bundle_data))
bundles_by_digest[digest]['nvr'] = bundles[0]['brew']['build']
bundles_by_digest[digest]['images'] = bundles
# Unauthenticated koji session to fetch build info of bundles
koji_api = KojiService(conf.koji_profile)
# For each bundle, check whether it should be rebuilt by comparing the
# auto_rebuild_tags of repository and bundle's tags
for digest, bundle_data in bundles_by_digest.items():
bundle_nvr = bundle_data['nvr']
# Images are for different arches, just check against the first image
image = bundle_data['images'][0]
if self.image_has_auto_rebuild_tag(image):
bundle_data['auto_rebuild'] = True
# Fetch buildinfo
buildinfo = koji_api.get_build(bundle_nvr)
related_images = (buildinfo.get('extra', {}).get('image', {}).get(
'operator_manifests', {}).get('related_images', {}))
bundle_data['osbs_pinning'] = related_images.get(
'created_by_osbs', False)
# Save the original pullspecs
bundle_data['pullspecs'] = related_images.get('pullspecs', [])
# Digests of bundles to be rebuilt
to_rebuild_digests = set()
# Now for each bundle, replace the original digest with rebuilt
# digest (override pullspecs)
for digest, bundle_data in bundles_by_digest.items():
# Override pullspecs only when auto_rebuild is enabled and OSBS-pinning
# mechanism is used.
if not (bundle_data['auto_rebuild']
and bundle_data['osbs_pinning']):
continue
csv_name = bundle_mds_by_digest[digest]['csv_name']
version = bundle_mds_by_digest[digest]['version']
bundle_data.update(self._get_csv_updates(csv_name, version))
for pullspec in bundle_data['pullspecs']:
# A pullspec item example:
# {
# 'new': 'registry.exampe.io/repo/example-operator@sha256:<sha256-value>'
# 'original': 'registry.example.io/repo/example-operator:v2.2.0',
# 'pinned': True
# }
# A pullspec path is in format of "registry/repository@digest"
pullspec_elems = pullspec.get('new').split('@')
old_digest = pullspec_elems[1]
if old_digest not in original_nvrs_by_digest:
# This related image is not one of the original images
continue
# This related image is one of our original images
old_nvr = original_nvrs_by_digest[old_digest]
new_nvr = nvrs_mapping[old_nvr]
new_digest = rebuilt_digests_by_nvr[new_nvr]
# Replace the old digest with new digest
pullspec_elems[1] = new_digest
new_pullspec = '@'.join(pullspec_elems)
pullspec['new'] = new_pullspec
# Always set pinned to True when it was replaced by Freshmaker
# since it indicates that the pullspec was modified from the
# original pullspec
pullspec['pinned'] = True
# Once a pullspec in this bundle has been overrided, add this bundle
# to rebuild list
to_rebuild_digests.add(digest)
if not to_rebuild_digests:
msg = f"No bundle images to rebuild for advisory {event.advisory.name}"
self.log_info(msg)
db_event.transition(EventState.SKIPPED, msg)
db.session.commit()
return []
builds = self._prepare_builds(db_event, bundles_by_digest,
to_rebuild_digests)
# Reset context to db_event.
self.set_context(db_event)
self.start_to_build_images(builds)
msg = f"Advisory {db_event.search_key}: Rebuilding " \
f"{len(db_event.builds.all())} bundle images."
db_event.transition(EventState.BUILDING, msg)
return []
@classmethod
def _get_csv_updates(cls, csv_name, version):
"""
Determine the CSV updates required for the bundle image.
:param str csv_name: the name field in the bundle's ClusterServiceVersion file
:param str version: the version of the bundle image being rebuilt
:return: a dictionary of the CSV updates needed
:rtype: dict
"""
csv_modifications = {}
# Make sure that OLM will skip the version being rebuilt when upgrading to the rebuilt
# version
csv_modifications['append'] = {
'spec': {
'skips': [version],
}
}
new_version, fm_suffix = cls._get_rebuild_bundle_version(version)
new_csv_name = cls._get_csv_name(csv_name, version, new_version,
fm_suffix)
csv_modifications['update'] = {
'metadata': {
# Update the name of the CSV to something uniquely identify the rebuild
'name': new_csv_name,
# Declare that this rebuild is a substitute of the bundle being rebuilt
'substitutes-for': version,
},
'spec': {
# Update the version of the rebuild to be unique and a newer version than the
# the version of the bundle being rebuilt
'version': new_version,
}
}
return csv_modifications
@classmethod
def _get_rebuild_bundle_version(cls, version):
"""
Get a bundle version for the Freshmaker rebuild of the bundle image.
Examples:
1.2.3 => 1.2.3+0.$timestamp.patched (no build ID and not a rebuild)
1.2.3+48273 => 1.2.3+48273.0.$timestamp.patched (build ID and not a rebuild)
1.2.3+48273.0.1616457250.patched => 1.2.3+48273.0.$timestamp.patched (build ID and a rebuild)
:param str version: the version of the bundle image being rebuilt
:return: a tuple of the bundle version of the Freshmaker rebuild of the bundle image and
the suffix that was added by Freshmaker
:rtype: tuple(str, str)
"""
parsed_version = semver.VersionInfo.parse(version)
# Strip off the microseconds of the timestamp
timestamp = int(datetime.utcnow().timestamp())
new_fm_suffix = f'0.{timestamp}.patched'
if parsed_version.build:
# Check if the bundle was a Freshmaker rebuild
fm_suffix_search = re.search(r'(?P<fm_suffix>0\.\d+\.patched)$',
parsed_version.build)
if fm_suffix_search:
fm_suffix = fm_suffix_search.groupdict()['fm_suffix']
# Get the build without the Freshmaker suffix. This may include a build ID
# from the original build before Freshmaker rebuilt it or be empty.
build_wo_fm_suffix = parsed_version.build[:-len(fm_suffix)]
new_build = f"{build_wo_fm_suffix}{new_fm_suffix}"
else:
# This was not previously rebuilt by Freshmaker so just append the suffix
# to the existing build ID with '.' separating it.
new_build = f"{parsed_version.build}.{new_fm_suffix}"
else:
# If there is existing build ID, then make the Freshmaker suffix the build ID
new_build = new_fm_suffix
new_version = str(parsed_version.replace(build=new_build))
return new_version, new_fm_suffix
@staticmethod
def _get_csv_name(csv_name, version, rebuild_version, fm_suffix):
"""
Get a bundle CSV name for the Freshmaker rebuild of the bundle image.
:param str csv_name: the name of the ClusterServiceVersion (CSV) file of the bundle image
:param str version: the version of the bundle image being rebuilt
:param str rebuild_version: the new version being assigned by Freshmaker for the rebuild
:param str fm_suffix: the portion of rebuild_version that was generated by Freshmaker
:return: the bundle ClusterServiceVersion (CSV) name of the Freshmaker rebuild of the bundle
image
:rtype: str
"""
if version in csv_name:
return csv_name.replace(version, rebuild_version)
else:
return f'{csv_name}.{fm_suffix}'
def get_published_original_nvr(self, rebuilt_nvr):
"""
Search for an original build, that has been built and published to a
repository, and get original_nvr from it
:param str rebuilt_nvr: rebuilt NVR to look build by
:rtype: str or None
:return: original NVR from the first published FM build for given NVR
"""
original_nvr = None
# artifact build should be only one in database, or raise an error
artifact_build = db.session.query(ArtifactBuild).filter(
ArtifactBuild.rebuilt_nvr == rebuilt_nvr,
ArtifactBuild.type == ArtifactType.IMAGE.value,
).one_or_none()
# recursively search for original artifact build
if artifact_build is not None:
original_nvr = artifact_build.original_nvr
# check if image is published
request_params = {'include': 'data.repositories', 'page_size': 1}
images = self._pyxis._pagination(f'images/nvr/{original_nvr}',
request_params)
if not images:
return None
# stop recursion if the image is published in some repo
if any(repo['published']
for repo in images[0].get('repositories')):
return original_nvr
next_nvr = self.get_published_original_nvr(original_nvr)
if next_nvr is not None:
original_nvr = next_nvr
return original_nvr
def image_has_auto_rebuild_tag(self, image):
""" Check if image has a tag enabled for auto rebuild.
:param dict image: Dict representation of an image entity in Pyxis.
:rtype: bool
:return: True if image has a tag enabled for auto rebuild in repository, otherwise False.
"""
for repo in image['repositories']:
# Skip unpublished repository
if not repo['published']:
continue
auto_rebuild_tags = self._pyxis.get_auto_rebuild_tags(
repo['registry'], repo['repository'])
tags = [t['name'] for t in repo.get('tags', [])]
if set(auto_rebuild_tags) & set(tags):
return True
return False
def _create_original_to_rebuilt_nvrs_map(self):
"""
Creates mapping of original build NVRs to rebuilt NVRs in advisory.
Including NVRs of the builds from the blocking advisories
:rtype: dict
:return: map of the original NVRs as keys and rebuilt NVRs as values
"""
nvrs_mapping = {}
# Get builds from all blocking advisories
blocking_advisories_builds = \
Errata().get_blocking_advisories_builds(self.event.advisory.errata_id)
# Get builds NVRs from the advisory attached to the message/event and
# then get original NVR for every build
for product_info in self.event.advisory.builds.values():
for build in product_info['builds']:
# Search for the first build that triggered the chain of rebuilds
# for every shipped NVR to get original NVR from it
original_nvr = self.get_published_original_nvr(build['nvr'])
if original_nvr is None:
continue
nvrs_mapping[original_nvr] = build['nvr']
build_nvr = parse_nvr(build['nvr'])
# Check builds from blocking advisories and add to the mapping
# all of them, that have overlapping package names
for block_build in blocking_advisories_builds:
block_build_nvr = parse_nvr(block_build)
if block_build_nvr['name'] == build_nvr['name'] and \
block_build_nvr['version'] == build_nvr['version']:
nvrs_mapping[block_build] = build['nvr']
return nvrs_mapping
def _prepare_builds(self, db_event, bundles_by_digest, to_rebuild_digests):
"""
Prepare models.ArtifactBuild instance for every bundle that will be
rebuilt
:param models.Event db_event: database event that will contain builds
:param dict bundles_by_digest: mapping of bundle digest to bundle data
:param list to_rebuild_digests: digests of bundles to rebuild
:return: builds that already in database and ready to be submitted to brew
:rtype: list
"""
builds = []
csv_mod_url = conf.freshmaker_root_url + "/api/2/pullspec_overrides/{}"
for digest in to_rebuild_digests:
bundle = bundles_by_digest[digest]
# Reset context to db_event for each iteration before
# the ArtifactBuild is created.
self.set_context(db_event)
rebuild_reason = RebuildReason.DIRECTLY_AFFECTED.value
bundle_name = koji.parse_NVR(bundle["nvr"])["name"]
build = self.record_build(db_event,
bundle_name,
ArtifactType.IMAGE,
state=ArtifactBuildState.PLANNED.value,
original_nvr=bundle["nvr"],
rebuild_reason=rebuild_reason)
# Set context to particular build so logging shows this build
# in case of error.
self.set_context(build)
build.transition(ArtifactBuildState.PLANNED.value, "")
additional_data = ContainerImage.get_additional_data_from_koji(
bundle["nvr"])
build.build_args = json.dumps({
"repository":
additional_data["repository"],
"commit":
additional_data["commit"],
"target":
additional_data["target"],
"branch":
additional_data["git_branch"],
"arches":
additional_data["arches"],
"operator_csv_modifications_url":
csv_mod_url.format(build.id),
})
build.bundle_pullspec_overrides = {
"append": bundle["append"],
"pullspecs": bundle["pullspecs"],
"update": bundle["update"],
}
db.session.commit()
builds.append(build)
return builds
class HandleBotasAdvisory(ContainerBuildHandler):
"""
Handles event that was created by transition of an advisory filed by
BOTAS to SHIPPED_LIVE state
"""
name = "HandleBotasAdvisory"
# This prefix should be added to event reason, when skipping the event.
# Because Release Driver checks event's reason for certain prefixes,
# to determine if there is an error in bundles processing.
_no_bundle_prefix = "No bundles to rebuild: "
def __init__(self, pyxis=None):
super().__init__()
if pyxis:
self._pyxis = pyxis
else:
if not conf.pyxis_server_url:
raise ValueError("'PYXIS_SERVER_URL' parameter should be set")
self._pyxis = Pyxis(conf.pyxis_server_url)
if not conf.freshmaker_root_url or "://" not in conf.freshmaker_root_url:
raise ValueError(
"'FRESHMAKER_ROOT_URL' parameter should be set to "
"a valid URL")
# Currently processed event
self.event = None
def can_handle(self, event):
if (isinstance(event, BotasErrataShippedEvent)
and 'docker' in event.advisory.content_types):
return True
# This handler can handle manual bundle rebuilds too
if isinstance(event, ManualBundleRebuild):
return True
return False
def handle(self, event):
if event.dry_run:
self.force_dry_run()
self.event = event
db_event = Event.get_or_create_from_event(db.session, event)
self.set_context(db_event)
# Check if event is allowed by internal policies
if not self.event.is_allowed(self):
msg = ("This image rebuild is not allowed by internal policy. "
f"message_id: {event.msg_id}")
db_event.transition(EventState.SKIPPED, msg)
self.log_info(msg)
return []
if isinstance(event, ManualBundleRebuild):
bundles_to_rebuild = self._handle_manual_rebuild(db_event)
else:
bundles_to_rebuild = self._handle_auto_rebuild(db_event)
if not bundles_to_rebuild:
return []
builds = self._prepare_builds(db_event, bundles_to_rebuild)
# Reset context to db_event.
self.set_context(db_event)
self.start_to_build_images(builds)
msg = f"Advisory {db_event.search_key}: Rebuilding " \
f"{len(db_event.builds.all())} bundle images."
db_event.transition(EventState.BUILDING, msg)
return []
def _handle_auto_rebuild(self, db_event):
"""
Handle auto rebuild for an advisory created by Botas
:param db_event: database event that represent rebuild event
:rtype: list
:return: list of advisories that should be rebuilt
"""
# Mapping of original build nvrs to rebuilt nvrs in advisory
nvrs_mapping = self._create_original_to_rebuilt_nvrs_map()
original_nvrs = nvrs_mapping.keys()
self.log_info(
"Orignial nvrs of build in the advisory #{0} are: {1}".format(
self.event.advisory.errata_id, " ".join(original_nvrs)))
# Get image manifest_list_digest for all original images, manifest_list_digest is used
# in pullspecs in bundle's related images
original_digests_by_nvr = {}
original_nvrs_by_digest = {}
for nvr in original_nvrs:
digest = self._pyxis.get_manifest_list_digest_by_nvr(nvr)
if digest:
original_digests_by_nvr[nvr] = digest
original_nvrs_by_digest[digest] = nvr
else:
log.warning(
f"Image manifest_list_digest not found for original image {nvr} in Pyxis, "
"skip this image")
if not original_digests_by_nvr:
msg = f"None of the original images have digests in Pyxis: {','.join(original_nvrs)}"
log.warning(msg)
db_event.transition(EventState.SKIPPED, msg)
return []
# Get image manifest_list_digest for all rebuilt images, manifest_list_digest is used
# in pullspecs of bundle's related images
rebuilt_digests_by_nvr = {}
rebuilt_nvrs = nvrs_mapping.values()
for nvr in rebuilt_nvrs:
# Don't require that the manifest list digest be published in this case because
# there's a delay from after an advisory is shipped and when the published repositories
# entry is populated
digest = self._pyxis.get_manifest_list_digest_by_nvr(
nvr, must_be_published=False)
if digest:
rebuilt_digests_by_nvr[nvr] = digest
else:
log.warning(
f"Image manifest_list_digest not found for rebuilt image {nvr} in Pyxis, "
"skip this image")
if not rebuilt_digests_by_nvr:
msg = f"None of the rebuilt images have digests in Pyxis: {','.join(rebuilt_nvrs)}"
log.warning(msg)
db_event.transition(EventState.SKIPPED, msg)
return []
index_images = self._pyxis.get_operator_indices()
# get latest bundle images per channel per index image filtered
# by the highest semantic version
all_bundles = self._pyxis.get_latest_bundles(index_images)
self.log_debug(
"There are %d bundles that are latest in a channel in the found index images",
len(all_bundles),
)
# A mapping of digests to bundle metadata. This metadata is used to
# for the CSV metadata updates.
bundle_mds_by_digest = {}
# get bundle digests for original images
bundle_digests_by_related_nvr = {}
for image_nvr, image_digest in original_digests_by_nvr.items():
bundles = self._pyxis.get_bundles_by_related_image_digest(
image_digest, all_bundles)
if not bundles:
log.info(
f"No latest bundle image with the related image of {image_nvr}"
)
continue
for bundle in bundles:
bundle_digest = bundle['bundle_path_digest']
bundle_mds_by_digest[bundle_digest] = bundle
bundle_digests_by_related_nvr.setdefault(
image_nvr, []).append(bundle_digest)
if not bundle_digests_by_related_nvr:
msg = "None of the original images have related bundles, skip."
log.warning(msg)
db_event.transition(EventState.SKIPPED, msg)
return []
self.log_info("Found %d bundles with relevant related images",
len(bundle_digests_by_related_nvr))
# Mapping of bundle digest to bundle data
# {
# digest: {
# "images": [image_amd64, image_aarch64],
# "nvr": NVR,
# "auto_rebuild": True/False,
# "osbs_pinning": True/False,
# "pullspecs": [...],
# }
# }
bundles_by_digest = {}
default_bundle_data = {
'images': [],
'nvr': None,
'auto_rebuild': False,
'osbs_pinning': False,
# CSV modifications for the rebuilt bundle image
'pullspec_replacements': [],
'update': {},
}
# Get images for each bundle digest, a bundle digest can have multiple images
# with different arches.
for digest in bundle_mds_by_digest:
bundles = self._pyxis.get_images_by_digest(digest)
# If no bundle image found, just skip this bundle digest
if not bundles:
self.log_warn(
'The bundle digest %r was not found in Pyxis. Skipping.',
digest)
continue
bundles_by_digest.setdefault(digest,
copy.deepcopy(default_bundle_data))
bundles_by_digest[digest]['nvr'] = bundles[0]['brew']['build']
bundles_by_digest[digest]['images'] = bundles
# Unauthenticated koji session to fetch build info of bundles
koji_api = KojiService(conf.koji_profile)
# For each bundle, check whether it should be rebuilt by comparing the
# auto_rebuild_tags of repository and bundle's tags
for digest, bundle_data in bundles_by_digest.items():
bundle_nvr = bundle_data['nvr']
# Images are for different arches, just check against the first image
image = bundle_data['images'][0]
if self.image_has_auto_rebuild_tag(image):
bundle_data['auto_rebuild'] = True
# Fetch buildinfo
buildinfo = koji_api.get_build(bundle_nvr)
related_images = (buildinfo.get('extra', {}).get('image', {}).get(
'operator_manifests', {}).get('related_images', {}))
bundle_data['osbs_pinning'] = related_images.get(
'created_by_osbs', False)
# Save the original pullspecs
bundle_data['pullspec_replacements'] = related_images.get(
'pullspecs', [])
# Digests of bundles to be rebuilt
to_rebuild_digests = set()
# Now for each bundle, replace the original digest with rebuilt
# digest (override pullspecs)
for digest, bundle_data in bundles_by_digest.items():
# Override pullspecs only when auto_rebuild is enabled and OSBS-pinning
# mechanism is used.
if not (bundle_data['auto_rebuild']
and bundle_data['osbs_pinning']):
self.log_info(
'The bundle %r does not have auto-rebuild tags (%r) and/or OSBS pinning (%r)',
bundle_data['nvr'],
bundle_data['auto_rebuild'],
bundle_data['osbs_pinning'],
)
continue
csv_name = bundle_mds_by_digest[digest]['csv_name']
version = bundle_mds_by_digest[digest]['version']
bundle_data.update(self._get_csv_updates(csv_name, version))
for pullspec in bundle_data['pullspec_replacements']:
# A pullspec item example:
# {
# 'new': 'registry.exampe.io/repo/example-operator@sha256:<sha256-value>',
# 'original': 'registry.example.io/repo/example-operator:v2.2.0',
# 'pinned': True,
# # value used for internal purpose during manual rebuilds, it's an old pullspec that was replaced
# '_old': 'registry.exampe.io/repo/example-operator@sha256:<previous-sha256-value>,
# }
# A pullspec path is in format of "registry/repository@digest"
pullspec_elems = pullspec.get('new').split('@')
old_digest = pullspec_elems[1]
if old_digest not in original_nvrs_by_digest:
# This related image is not one of the original images
continue
# This related image is one of our original images
old_nvr = original_nvrs_by_digest[old_digest]
new_nvr = nvrs_mapping[old_nvr]
new_digest = rebuilt_digests_by_nvr[new_nvr]
# save pullspec that image had before rebuild
pullspec['_old'] = pullspec.get('new')
# Replace the old digest with new digest
pullspec_elems[1] = new_digest
new_pullspec = '@'.join(pullspec_elems)
pullspec['new'] = new_pullspec
# Always set pinned to True when it was replaced by Freshmaker
# since it indicates that the pullspec was modified from the
# original pullspec
pullspec['pinned'] = True
# Once a pullspec in this bundle has been overrided, add this bundle
# to rebuild list
self.log_info(
'Changing pullspec %r to %r in the bundle %r',
pullspec['_old'],
pullspec['new'],
bundle_data['nvr'],
)
to_rebuild_digests.add(digest)
if not to_rebuild_digests:
msg = self._no_bundle_prefix + "No bundle images to rebuild for " \
f"advisory {self.event.advisory.name}"
self.log_info(msg)
db_event.transition(EventState.SKIPPED, msg)
db.session.commit()
return []
bundles_to_rebuild = list(
map(lambda x: bundles_by_digest[x], to_rebuild_digests))
return bundles_to_rebuild
def _handle_manual_rebuild(self, db_event):
"""
Handle manual rebuild submitted by Release Driver for an advisory created by Botas
:param db_event: database event that represents a rebuild event
:rtype: list
:return: list of advisories that should be rebuilt
"""
old_to_new_pullspec_map = self._get_pullspecs_mapping()
if not old_to_new_pullspec_map:
msg = self._no_bundle_prefix + 'None of the bundle images have ' \
'applicable pullspecs to replace'
log.warning(msg)
db_event.transition(EventState.SKIPPED, msg)
return []
# Unauthenticated koji session to fetch build info of bundles
koji_api = KojiService(conf.koji_profile)
rebuild_nvr_to_pullspecs_map = dict()
# compare replaced pullspecs with pullspecs in 'container_images' and
# create map for bundles that should be rebuilt with their nvrs
for container_image_nvr in self.event.container_images:
artifact_build = db.session.query(ArtifactBuild).filter(
ArtifactBuild.rebuilt_nvr == container_image_nvr,
ArtifactBuild.type == ArtifactType.IMAGE.value,
).one_or_none()
pullspecs = []
# Try to find build in FM database, if it's not there check in Brew
if artifact_build:
pullspecs = artifact_build.bundle_pullspec_overrides[
"pullspec_replacements"]
else:
# Fetch buildinfo from Koji
buildinfo = koji_api.get_build(container_image_nvr)
# Get the original pullspecs
pullspecs = (buildinfo.get('extra', {}).get('image', {}).get(
'operator_manifests', {}).get('related_images',
{}).get('pullspecs', []))
for pullspec in pullspecs:
if pullspec.get('new') not in old_to_new_pullspec_map:
continue
# use newer pullspecs in the image
pullspec['new'] = old_to_new_pullspec_map[pullspec['new']]
rebuild_nvr_to_pullspecs_map[container_image_nvr] = pullspecs
if not rebuild_nvr_to_pullspecs_map:
msg = self._no_bundle_prefix + 'None of the container images have ' \
'applicable pullspecs from the input bundle images'
log.info(msg)
db_event.transition(EventState.SKIPPED, msg)
return []
# list with metadata about every bundle to do rebuild
to_rebuild_bundles = []
# fill 'append' and 'update' fields for bundles to rebuild
for nvr, pullspecs in rebuild_nvr_to_pullspecs_map.items():
bundle_digest = self._pyxis.get_manifest_list_digest_by_nvr(nvr)
if bundle_digest is not None:
bundles = self._pyxis.get_bundles_by_digest(bundle_digest)
temp_bundle = bundles[0]
csv_updates = (self._get_csv_updates(temp_bundle['csv_name'],
temp_bundle['version']))
to_rebuild_bundles.append({
'nvr': nvr,
'update': csv_updates['update'],
'pullspec_replacements': pullspecs,
})
else:
log.warning('Can\'t find manifest_list_digest for bundle '
f'"{nvr}" in Pyxis')
if not to_rebuild_bundles:
msg = 'Can\'t find digests for any of the bundles to rebuild'
log.warning(msg)
db_event.transition(EventState.FAILED, msg)
return []
return to_rebuild_bundles
def _get_pullspecs_mapping(self):
"""
Get map of all replaced pullspecs from 'bundle_images' provided in an event.
:rtype: dict
:return: map of all '_old' pullspecs that was replaced by 'new'
pullspecs in previous Freshmaker rebuilds
"""
old_to_new_pullspec_map = dict()
for bundle_nvr in self.event.bundle_images:
artifact_build = db.session.query(ArtifactBuild).filter(
ArtifactBuild.rebuilt_nvr == bundle_nvr,
ArtifactBuild.type == ArtifactType.IMAGE.value,
).one_or_none()
if artifact_build is None:
log.warning(
f'Can\'t find build for a bundle image "{bundle_nvr}"')
continue
pullspec_overrides = artifact_build.bundle_pullspec_overrides
for pullspec in pullspec_overrides['pullspec_replacements']:
old_pullspec = pullspec.get('_old', None)
if old_pullspec is None:
continue
old_to_new_pullspec_map[old_pullspec] = pullspec['new']
return old_to_new_pullspec_map
@classmethod
def _get_csv_updates(cls, csv_name, version):
"""
Determine the CSV updates required for the bundle image.
:param str csv_name: the name field in the bundle's ClusterServiceVersion file
:param str version: the version of the bundle image being rebuilt
:return: a dictionary of the CSV updates needed
:rtype: dict
"""
csv_modifications = {}
new_version, fm_suffix = cls._get_rebuild_bundle_version(version)
new_csv_name = cls._get_csv_name(csv_name, version, new_version,
fm_suffix)
csv_modifications['update'] = {
'metadata': {
# Update the name of the CSV to something uniquely identify the rebuild
'name': new_csv_name,
# Declare that this rebuild is a substitute of the bundle being rebuilt
'annotations': {
'olm.substitutesFor': version
}
},
'spec': {
# Update the version of the rebuild to be unique and a newer version than the
# the version of the bundle being rebuilt
'version': new_version,
}
}
return csv_modifications
@classmethod
def _get_rebuild_bundle_version(cls, version):
"""
Get a bundle version for the Freshmaker rebuild of the bundle image.
Examples:
1.2.3 => 1.2.3+0.$timestamp.p (no build ID and not a rebuild)
1.2.3+48273 => 1.2.3+48273.0.$timestamp.p (build ID and not a rebuild)
1.2.3+48273.0.1616457250.p => 1.2.3+48273.0.$timestamp.p (build ID and a rebuild)
:param str version: the version of the bundle image being rebuilt
:return: a tuple of the bundle version of the Freshmaker rebuild of the bundle image and
the suffix that was added by Freshmaker
:rtype: tuple(str, str)
"""
parsed_version = semver.VersionInfo.parse(version)
# Strip off the microseconds of the timestamp
timestamp = int(datetime.utcnow().timestamp())
new_fm_suffix = f'0.{timestamp}.p'
if parsed_version.build:
# Check if the bundle was a Freshmaker rebuild. Include .patched
# for backwards compatibility with the old suffix.
fm_suffix_search = re.search(
r'(?P<fm_suffix>0\.\d+\.(?:p|patched))$', parsed_version.build)
if fm_suffix_search:
fm_suffix = fm_suffix_search.groupdict()['fm_suffix']
# Get the build without the Freshmaker suffix. This may include a build ID
# from the original build before Freshmaker rebuilt it or be empty.
build_wo_fm_suffix = parsed_version.build[:-len(fm_suffix)]
new_build = f"{build_wo_fm_suffix}{new_fm_suffix}"
else:
# This was not previously rebuilt by Freshmaker so just append the suffix
# to the existing build ID with '.' separating it.
new_build = f"{parsed_version.build}.{new_fm_suffix}"
else:
# If there is existing build ID, then make the Freshmaker suffix the build ID
new_build = new_fm_suffix
# Don't use the replace method in order to support semver 2.8.1
new_version_dict = parsed_version._asdict()
new_version_dict["build"] = new_build
new_version = str(semver.VersionInfo(**new_version_dict))
return new_version, new_fm_suffix
@staticmethod
def _get_csv_name(csv_name, version, rebuild_version, fm_suffix):
"""
Get a bundle CSV name for the Freshmaker rebuild of the bundle image.
:param str csv_name: the name of the ClusterServiceVersion (CSV) file of the bundle image
:param str version: the version of the bundle image being rebuilt
:param str rebuild_version: the new version being assigned by Freshmaker for the rebuild
:param str fm_suffix: the portion of rebuild_version that was generated by Freshmaker
:return: the bundle ClusterServiceVersion (CSV) name of the Freshmaker rebuild of the bundle
image
:rtype: str
"""
# The CSV name must be in the format of a valid DNS name, which means the + from the
# build ID must be replaced. In the event this was a previous Freshmaker rebuild, version
# may have a build ID that would be the DNS safe version in the CSV name.
dns_safe_version = version.replace('+', '-')
if dns_safe_version in csv_name:
dns_safe_rebuild_version = rebuild_version.replace('+', '-')
return csv_name.replace(dns_safe_version, dns_safe_rebuild_version)
else:
return f'{csv_name}.{fm_suffix}'
def get_published_original_nvr(self, rebuilt_nvr):
"""
Search for an original build, that has been built and published to a
repository, and get original_nvr from it
:param str rebuilt_nvr: rebuilt NVR to look build by
:rtype: str or None
:return: original NVR from the first published FM build for given NVR
"""
original_nvr = None
# artifact build should be only one in database, or raise an error
artifact_build = db.session.query(ArtifactBuild).filter(
ArtifactBuild.rebuilt_nvr == rebuilt_nvr,
ArtifactBuild.type == ArtifactType.IMAGE.value,
).one_or_none()
# recursively search for original artifact build
if artifact_build is not None:
original_nvr = artifact_build.original_nvr
# check if image is published
request_params = {'include': 'data.repositories', 'page_size': 1}
images = self._pyxis._pagination(f'images/nvr/{original_nvr}',
request_params)
if not images:
return None
# stop recursion if the image is published in some repo
if any(repo['published']
for repo in images[0].get('repositories')):
return original_nvr
next_nvr = self.get_published_original_nvr(original_nvr)
if next_nvr is not None:
original_nvr = next_nvr
return original_nvr
def image_has_auto_rebuild_tag(self, image):
""" Check if image has a tag enabled for auto rebuild.
:param dict image: Dict representation of an image entity in Pyxis.
:rtype: bool
:return: True if image has a tag enabled for auto rebuild in repository, otherwise False.
"""
for repo in image['repositories']:
# Skip unpublished repository
if not repo['published']:
continue
auto_rebuild_tags = self._pyxis.get_auto_rebuild_tags(
repo['registry'], repo['repository'])
tags = [t['name'] for t in repo.get('tags', [])]
if set(auto_rebuild_tags) & set(tags):
return True
# It'd be more efficient to do this check first, but the exceptions are edge cases
# (e.g. testing) and it's best to not use it unless absolutely necessary
nvr = image['brew']['build']
parsed_nvr = parse_nvr(nvr)
nv = f'{parsed_nvr["name"]}-{parsed_nvr["version"]}'
if nv in conf.bundle_autorebuild_tag_exceptions:
self.log_info(
'The bundle %r has an exception for being tagged with an auto-rebuild tag',
nvr)
return True
return False
def _create_original_to_rebuilt_nvrs_map(self):
"""
Creates mapping of original build NVRs to rebuilt NVRs in advisory.
Including NVRs of the builds from the blocking advisories
:rtype: dict
:return: map of the original NVRs as keys and rebuilt NVRs as values
"""
nvrs_mapping = {}
# Get builds from all blocking advisories
blocking_advisories_builds = \
Errata().get_blocking_advisories_builds(self.event.advisory.errata_id)
# Get builds NVRs from the advisory attached to the message/event and
# then get original NVR for every build
for product_info in self.event.advisory.builds.values():
for build in product_info['builds']:
# Each build is a one key/value pair, and key is the build NVR
build_nvr = next(iter(build))
# Search for the first build that triggered the chain of rebuilds
# for every shipped NVR to get original NVR from it
original_nvr = self.get_published_original_nvr(build_nvr)
if original_nvr is None:
continue
nvrs_mapping[original_nvr] = build_nvr
parsed_build_nvr = parse_nvr(build_nvr)
# Check builds from blocking advisories and add to the mapping
# all of them, that have overlapping package names
for block_build in blocking_advisories_builds:
block_build_nvr = parse_nvr(block_build)
if block_build_nvr['name'] == parsed_build_nvr['name'] and \
block_build_nvr['version'] == parsed_build_nvr['version']:
nvrs_mapping[block_build] = build_nvr
return nvrs_mapping
def _prepare_builds(self, db_event, to_rebuild_bundles):
"""
Prepare models.ArtifactBuild instance for every bundle that will be
rebuilt
:param models.Event db_event: database event that will contain builds
:param list to_rebuild_bundles: bundles to rebuild
:return: builds that already in database and ready to be submitted to brew
:rtype: list
"""
builds = []
csv_mod_url = conf.freshmaker_root_url + "/api/2/pullspec_overrides/{}"
for bundle in to_rebuild_bundles:
# Reset context to db_event for each iteration before
# the ArtifactBuild is created.
self.set_context(db_event)
rebuild_reason = RebuildReason.DIRECTLY_AFFECTED.value
bundle_name = koji.parse_NVR(bundle["nvr"])["name"]
build = self.record_build(db_event,
bundle_name,
ArtifactType.IMAGE,
state=ArtifactBuildState.PLANNED.value,
original_nvr=bundle["nvr"],
rebuild_reason=rebuild_reason)
# Set context to particular build so logging shows this build
# in case of error.
self.set_context(build)
build.transition(ArtifactBuildState.PLANNED.value, "")
additional_data = ContainerImage.get_additional_data_from_koji(
bundle["nvr"])
build.build_args = json.dumps({
"repository":
additional_data["repository"],
"commit":
additional_data["commit"],
"target":
additional_data["target"],
"branch":
additional_data["git_branch"],
"arches":
additional_data["arches"],
# The build system always enforces that bundle images build from
# "scratch", so there is no parent image. See:
# https://osbs.readthedocs.io/en/latest/users.html?#operator-manifest-bundle-builds
"original_parent":
None,
"operator_csv_modifications_url":
csv_mod_url.format(build.id),
})
build.bundle_pullspec_overrides = {
"pullspec_replacements": bundle["pullspec_replacements"],
"update": bundle["update"],
}
db.session.commit()
builds.append(build)
return builds
class HandleBotasAdvisory(ContainerBuildHandler):
"""
Handles event that was created by transition of an advisory filed by
BOTAS to SHIPPED_LIVE state
"""
name = "HandleBotasAdvisory"
def __init__(self, pyxis=None):
super().__init__()
if pyxis:
self._pyxis = pyxis
else:
if not conf.pyxis_server_url:
raise ValueError("'pyxis_server_url' parameter should be set")
self._pyxis = Pyxis(conf.pyxis_server_url)
def can_handle(self, event):
if (isinstance(event, BotasErrataShippedEvent) and
'docker' in event.advisory.content_types):
return True
return False
def handle(self, event):
if event.dry_run:
self.force_dry_run()
self.event = event
db_event = Event.get_or_create_from_event(db.session, event)
self.set_context(db_event)
# Check if event is allowed by internal policies
if not self.event.is_allowed(self):
msg = ("This image rebuild is not allowed by internal policy. "
f"message_id: {event.msg_id}")
db_event.transition(EventState.SKIPPED, msg)
self.log_info(msg)
return []
# Get builds NVRs from the advisory attached to the message/event and
# then get original NVR for every build
# Mapping of original build nvrs to rebuilt nvrs in advisory
nvrs_mapping = {}
for product_info in event.advisory.builds.values():
for build in product_info['builds']:
# Search for the first build that triggered the chain of rebuilds
# for every shipped NVR to get original NVR from it
original_nvr = self.get_published_original_nvr(build['nvr'])
if original_nvr is None:
continue
nvrs_mapping[original_nvr] = build['nvr']
original_nvrs = nvrs_mapping.keys()
self.log_info(
"Orignial nvrs of build in the advisory #{0} are: {1}".format(
event.advisory.errata_id, " ".join(original_nvrs)))
# Get image manifest_list_digest for all original images, manifest_list_digest is used
# in pullspecs in bundle's related images
original_digests_by_nvr = {}
original_nvrs_by_digest = {}
for nvr in original_nvrs:
digest = self._pyxis.get_manifest_list_digest_by_nvr(nvr)
if digest:
original_digests_by_nvr[nvr] = digest
original_nvrs_by_digest[digest] = nvr
else:
log.warning(
f"Image manifest_list_digest not found for original image {nvr} in Pyxis, "
"skip this image"
)
if not original_digests_by_nvr:
msg = f"None of the original images have digests in Pyxis: {','.join(original_nvrs)}"
log.warning(msg)
db_event.transition(EventState.SKIPPED, msg)
return []
# Get image manifest_list_digest for all rebuilt images, manifest_list_digest is used
# in pullspecs of bundle's related images
rebuilt_digests_by_nvr = {}
rebuilt_nvrs = nvrs_mapping.values()
for nvr in rebuilt_nvrs:
digest = self._pyxis.get_manifest_list_digest_by_nvr(nvr)
if digest:
rebuilt_digests_by_nvr[nvr] = digest
else:
log.warning(
f"Image manifest_list_digest not found for rebuilt image {nvr} in Pyxis, "
"skip this image"
)
if not rebuilt_digests_by_nvr:
msg = f"None of the rebuilt images have digests in Pyxis: {','.join(rebuilt_nvrs)}"
log.warning(msg)
db_event.transition(EventState.SKIPPED, msg)
return []
index_images = self._pyxis.get_operator_indices()
# get latest bundle images per channel per index image filtered
# by the highest semantic version
all_bundles = self._pyxis.get_latest_bundles(index_images)
# A set of unique bundle digests
bundle_digests = set()
# get bundle digests for original images
bundle_digests_by_related_nvr = {}
for image_nvr, image_digest in original_digests_by_nvr.items():
bundles = self._pyxis.get_bundles_by_related_image_digest(
image_digest, all_bundles
)
if not bundles:
log.info(f"No latest bundle image with the related image of {image_nvr}")
continue
for bundle in bundles:
bundle_digest = bundle['bundle_path_digest']
bundle_digests.add(bundle_digest)
bundle_digests_by_related_nvr.setdefault(image_nvr, []).append(bundle_digest)
if not bundle_digests_by_related_nvr:
msg = "None of the original images have related bundles, skip."
log.warning(msg)
db_event.transition(EventState.SKIPPED, msg)
return []
# Mapping of bundle digest to bundle data
# {
# digest: {
# "images": [image_amd64, image_aarch64],
# "nvr": NVR,
# "auto_rebuild": True/False,
# "osbs_pinning": True/False,
# "pullspecs": [...],
# }
# }
bundles_by_digest = {}
default_bundle_data = {
'images': [],
'nvr': None,
'auto_rebuild': False,
'osbs_pinning': False,
'pullspecs': [],
}
# Get images for each bundle digest, a bundle digest can have multiple images
# with different arches.
for digest in bundle_digests:
bundles = self._pyxis.get_images_by_digest(digest)
# If no bundle image found, just skip this bundle digest
if not bundles:
continue
bundles_by_digest.setdefault(digest, copy.deepcopy(default_bundle_data))
bundles_by_digest[digest]['nvr'] = bundles[0]['brew']['build']
bundles_by_digest[digest]['images'] = bundles
# Unauthenticated koji session to fetch build info of bundles
koji_api = KojiService(conf.koji_profile)
# For each bundle, check whether it should be rebuilt by comparing the
# auto_rebuild_tags of repository and bundle's tags
for digest, bundle_data in bundles_by_digest.items():
bundle_nvr = bundle_data['nvr']
# Images are for different arches, just check against the first image
image = bundle_data['images'][0]
if self.image_has_auto_rebuild_tag(image):
bundle_data['auto_rebuild'] = True
# Fetch buildinfo
buildinfo = koji_api.get_build(bundle_nvr)
related_images = (
buildinfo.get("extra", {})
.get("image", {})
.get("operator_manifests", {})
.get("related_images", {})
)
bundle_data['osbs_pinning'] = related_images.get('created_by_osbs', False)
# Save the original pullspecs
bundle_data['pullspecs'] = related_images.get('pullspecs', [])
# Digests of bundles to be rebuilt
to_rebuild_digests = set()
# Now for each bundle, replace the original digest with rebuilt
# digest (override pullspecs)
for digest, bundle_data in bundles_by_digest.items():
# Override pullspecs only when auto_rebuild is enabled and OSBS-pinning
# mechanism is used.
if not (bundle_data['auto_rebuild'] and bundle_data['osbs_pinning']):
continue
for pullspec in bundle_data['pullspecs']:
# A pullspec item example:
# {
# 'new': 'registry.exampe.io/repo/example-operator@sha256:<sha256-value>'
# 'original': 'registry.example.io/repo/example-operator:v2.2.0',
# 'pinned': True
# }
# If related image is not pinned by OSBS, skip
if not pullspec.get('pinned', False):
continue
# A pullspec path is in format of "registry/repository@digest"
pullspec_elems = pullspec.get('new').split('@')
old_digest = pullspec_elems[1]
if old_digest not in original_nvrs_by_digest:
# This related image is not one of the original images
continue
# This related image is one of our original images
old_nvr = original_nvrs_by_digest[old_digest]
new_nvr = nvrs_mapping[old_nvr]
new_digest = rebuilt_digests_by_nvr[new_nvr]
# Replace the old digest with new digest
pullspec_elems[1] = new_digest
new_pullspec = '@'.join(pullspec_elems)
pullspec['new'] = new_pullspec
# Once a pullspec in this bundle has been overrided, add this bundle
# to rebuild list
to_rebuild_digests.add(digest)
# Skip that event because we can't proceed with processing it.
# TODO
# Now when we have bundle images' nvrs we can procceed with rebuilding it
msg = f"Skipping the rebuild of {len(to_rebuild_digests)} bundle images " \
"due to being blocked on further implementation for now."
db_event.transition(EventState.SKIPPED, msg)
return []
def get_published_original_nvr(self, rebuilt_nvr):
"""
Search for an original build, that has been built and published to a
repository, and get original_nvr from it
:param str rebuilt_nvr: rebuilt NVR to look build by
:rtype: str or None
:return: original NVR from the first published FM build for given NVR
"""
original_nvr = None
# artifact build should be only one in database, or raise an error
artifact_build = db.session.query(ArtifactBuild).filter(
ArtifactBuild.rebuilt_nvr == rebuilt_nvr,
ArtifactBuild.type == ArtifactType.IMAGE.value,
).one_or_none()
# recursively search for original artifact build
if artifact_build is not None:
original_nvr = artifact_build.original_nvr
# check if image is published
request_params = {'include': 'data.repositories',
'page_size': 1}
images = self._pyxis._pagination(f'images/nvr/{original_nvr}',
request_params)
if not images:
return None
# stop recursion if the image is published in some repo
if any(repo['published'] for repo in images[0].get('repositories')):
return original_nvr
next_nvr = self.get_published_original_nvr(original_nvr)
if next_nvr is not None:
original_nvr = next_nvr
return original_nvr
def image_has_auto_rebuild_tag(self, image):
""" Check if image has a tag enabled for auto rebuild.
:param dict image: Dict representation of an image entity in Pyxis.
:rtype: bool
:return: True if image has a tag enabled for auto rebuild in repository, otherwise False.
"""
for repo in image['repositories']:
# Skip unpublished repository
if not repo['published']:
continue
auto_rebuild_tags = self._pyxis.get_auto_rebuild_tags(
repo['registry'], repo['repository']
)
tags = [t['name'] for t in repo.get('tags', [])]
if set(auto_rebuild_tags) & set(tags):
return True
return False
class HandleBotasAdvisory(ContainerBuildHandler):
"""
Handles event that was created by transition of an advisory filed by
BOTAS to SHIPPED_LIVE state
"""
name = "HandleBotasAdvisory"
def __init__(self, pyxis=None):
super().__init__()
if pyxis:
self._pyxis = pyxis
else:
if not conf.pyxis_server_url:
raise ValueError("'pyxis_server_url' parameter should be set")
self._pyxis = Pyxis(conf.pyxis_server_url)
def can_handle(self, event):
if (isinstance(event, BotasErrataShippedEvent)
and 'docker' in event.advisory.content_types):
return True
return False
def handle(self, event):
if event.dry_run:
self.force_dry_run()
self.event = event
db_event = Event.get_or_create_from_event(db.session, event)
self.set_context(db_event)
# Check if event is allowed by internal policies
if not self.event.is_allowed(self):
msg = ("This image rebuild is not allowed by internal policy. "
f"message_id: {event.msg_id}")
db_event.transition(EventState.SKIPPED, msg)
self.log_info(msg)
return []
# Get builds NVRs from the advisory attached to the message/event and
# then get original NVR for every build
original_nvrs = set()
for product_info in event.advisory.builds.values():
for build in product_info['builds']:
# Search for the first build that triggered the chain of rebuilds
# for every shipped NVR to get original NVR from it
original_nvr = self.get_published_original_nvr(build['nvr'])
if original_nvr is None:
continue
original_nvrs.add(original_nvr)
self.log_info(
"Orignial nvrs of build in the advisory #{0} are: {1}".format(
event.advisory.errata_id, " ".join(original_nvrs)))
# Get images by nvrs and then get their digests
original_images_digests = self._pyxis.get_digests_by_nvrs(
original_nvrs)
if not original_images_digests:
msg = f"There are no digests for NVRs: {','.join(original_nvrs)}"
log.warning(msg)
db_event.transition(EventState.SKIPPED, msg)
return []
index_images = self._pyxis.get_operator_indices()
# get latest bundle images per channel per index image filtered
# by the highest semantic version
all_bundles = self._pyxis.get_latest_bundles(index_images)
bundles = self._pyxis.filter_bundles_by_related_image_digests(
original_images_digests, all_bundles)
bundle_digests = set()
for bundle in bundles:
if not bundle.get('bundle_path_digest'):
log.warning("Bundle %s doesn't have 'bundle_path_digests' set",
bundle['bundle_path'])
continue
bundle_digests.add(bundle['bundle_path_digest'])
bundle_images = self._pyxis.get_images_by_digests(bundle_digests)
# Filter image nvrs that don't have or never had auto_rebuild tag
# in repos, where image is published
auto_rebuild_nvrs = self._pyxis.get_auto_rebuild_tagged_images(
bundle_images)
# get NVRs only of those bundles, which have OSBS pinning
bundles_nvrs = self._filter_bundles_by_pinned_related_images(
auto_rebuild_nvrs)
# Skip that event because we can't proceed with processing it.
# TODO
# Now when we have bundle images' nvrs we can procceed with rebuilding it
msg = f"Skipping the rebuild of {len(bundles_nvrs)} bundle images " \
"due to being blocked on further implementation for now."
db_event.transition(EventState.SKIPPED, msg)
return []
def _filter_bundles_by_pinned_related_images(self, bundle_image_nvrs):
"""
If the digests were not pinned by OSBS, the bundle image nvr
will be filtered out.
There is no need in checking pinning for every of related images,
because we already know that digest points to the manifest list,
because of previous filtering.
:param set bundle_image_nvrs: NVRs of operator bundles
:return: list of NVRs of bundle images that have at least one
original related image that was rebuilt
"""
ret_bundle_images_nvrs = set()
with koji_service(conf.koji_profile,
log,
dry_run=self.dry_run,
login=False) as session:
for nvr in bundle_image_nvrs:
build = session.get_build(nvr)
if not build:
log.error("Could not find the build %s in Koji", nvr)
continue
related_images = (build.get("build", {}).get("extra", {}).get(
"image", {}).get("operator_manifests",
{}).get("related_images", {}))
# Skip the bundle if the related images section was not populated by OSBS
if related_images.get("created_by_osbs") is not True:
continue
ret_bundle_images_nvrs.add(nvr)
return ret_bundle_images_nvrs
def get_published_original_nvr(self, rebuilt_nvr):
"""
Search for an original build, that has been built and published to a
repository, and get original_nvr from it
:param str rebuilt_nvr: rebuilt NVR to look build by
:rtype: str or None
:return: original NVR from the first published FM build for given NVR
"""
original_nvr = None
# artifact build should be only one in database, or raise an error
artifact_build = db.session.query(ArtifactBuild).filter(
ArtifactBuild.rebuilt_nvr == rebuilt_nvr,
ArtifactBuild.type == ArtifactType.IMAGE.value,
).one_or_none()
# recursively search for original artifact build
if artifact_build is not None:
original_nvr = artifact_build.original_nvr
# check if image is published
request_params = {'include': 'data.repositories', 'page_size': 1}
images = self._pyxis._pagination(f'images/nvr/{original_nvr}',
request_params)
if not images:
return None
# stop recursion if the image is published in some repo
if any(repo['published']
for repo in images[0].get('repositories')):
return original_nvr
next_nvr = self.get_published_original_nvr(original_nvr)
if next_nvr is not None:
original_nvr = next_nvr
return original_nvr
class TestQueryPyxis(helpers.FreshmakerTestCase):
def setUp(self):
super().setUp()
self.patcher = helpers.Patcher(
'freshmaker.pyxis.')
self.fake_server_url = 'https://pyxis.localhost/'
self.px = Pyxis(self.fake_server_url)
self.response = create_autospec(requests.Response)
self.response.status_code = HTTPStatus.OK
self.bad_requests_response = {
"detail": [
"Unable to parse the filter from URL.",
"Please verify the 'Field Name' in the RSQL Expression.",
"Please visit the following end-point for more details:",
" /v1/docs/filtering-language"
],
"status": 400,
"title": "Bad Request",
"type": "about:blank"
}
self.empty_response_page = {
"data": [],
"page": 0,
"page_size": 100,
"total": 0
}
self.indices = [
{
"_id": "1",
"created_by": "meteor",
"creation_date": "2020-01-01T09:32:31.692000+00:00",
"last_update_date": "2020-01-01T09:32:31.692000+00:00",
"last_updated_by": "meteor",
"ocp_version": "4.5",
"organization": "org",
"path": "path/to/registry:v4.5"
},
{
"_id": "2",
"created_by": "meteor",
"creation_date": "2020-01-01T09:32:38.486000+00:00",
"last_update_date": "2020-01-01T09:32:38.486000+00:00",
"last_updated_by": "meteor",
"ocp_version": "4.6",
"organization": "org",
"path": "path/to/registry:v4.6"
},
{
"_id": "2",
"created_by": "meteor",
"creation_date": "2020-01-01T09:32:38.486000+00:00",
"last_update_date": "2020-01-01T09:32:38.486000+00:00",
"last_updated_by": "meteor",
"ocp_version": "4.6",
"organization": "org",
"path": ""
}
]
self.bundles = [
{
"channel_name": "streams-1.5.x",
"related_images": [
{
"image": "registry/amq7/amq-streams-r-operator@sha256:111",
"name": "strimzi-cluster-operator",
"digest": "sha256:111"
},
{
"image": "registry/amq7/amq-streams-kafka-24-r@sha256:222",
"name": "strimzi-kafka-24",
"digest": "sha256:222"
},
{
"image": "registry/amq7/amq-streams-kafka-25-r@sha256:333",
"name": "strimzi-kafka-25",
"digest": "sha256:333"
},
{
"image": "registry/amq7/amq-streams-bridge-r@sha256:444",
"name": "strimzi-bridge",
"digest": "sha256:444"
}
],
"version": "1.5.3"
},
{
"channel_name": "streams-1.5.x",
"related_images": [
{
"image": "registry/amq7/amq-streams-r-operator@sha256:555",
"name": "strimzi-cluster-operator",
"digest": "sha256:555"
},
{
"image": "registry/amq7/amq-streams-kafka-24-r@sha256:666",
"name": "strimzi-kafka-24",
"digest": "sha256:666"
},
{
"image": "registry/amq7/amq-streams-kafka-25-r@sha256:777",
"name": "strimzi-kafka-25",
"digest": "sha256:777"
},
{
"image": "registry/amq7/amq-streams-bridge-r@sha256:888",
"name": "strimzi-bridge",
"digest": "sha256:888"
}
],
"version": "1.5.4"
},
{
"channel_name": "stable",
"related_images": [
{
"image": "registry/amq7/amq--operator@sha256:999",
"name": "strimzi-cluster-operator",
"digest": "sha256:999"
},
{
"image": "registry/amq7/kafka-24-r@sha256:aaa",
"name": "strimzi-kafka-24",
"digest": "sha256:aaa"
},
{
"image": "registry/amq7/kafka-25-r@sha256:bbb",
"name": "strimzi-kafka-25",
"digest": "sha256:bbb"
},
{
"image": "registry/amq7/amq-streams-bridge-r@sha256:ccc",
"name": "strimzi-bridge",
"digest": "sha256:ccc"
}
],
"version": "1.5.3"
},
{
"channel_name": "stable",
"related_images": [
{
"image": "registry/tracing/j-operator:1.13.2",
"name": "j-1.13.2-annotation",
"digest": "sha256:fff"
},
{
"image": "registry/tracing/j-operator:1.13.2",
"name": "j-operator",
"digest": "sha256:ffff"
}
],
"version": "1.5.2"
},
{
"channel_name": "quay-v3.3",
"related_images": [
{
"image": "registry/quay/quay-operator@sha256:ddd",
"name": "quay-operator-annotation",
"digest": "sha256:ddd"
},
{
"image": "registry/quay/quay-security-r-operator@sha256:eee",
"name": "container-security-operator",
"digest": "sha256:eee"
}
],
"version": "3.3.1"
},
]
self.images = [
{
"brew": {
"build": "s2i-1-2",
"completion_date": "2020-08-12T11:31:39+00:00",
"nvra": "s2i-1-2.ppc64le",
"package": "s2i-core-container"
},
"repositories": [
{
"manifest_list_digest": "sha256:1111",
"published": False,
"registry": "reg1",
"repository": "repo1",
"tags": [{"name": "tag0"}]
},
{
"manifest_list_digest": "sha256:1112",
"published": True,
"registry": "reg2",
"repository": "repo2",
"tags": [{"name": "tag1"}, {"name": "tag2"}]
}
]
},
{
"brew": {
"build": "s2i-1-2",
"completion_date": "2020-08-12T11:31:39+00:00",
"nvra": "s2i-1-2.s390x",
"package": "s2i-core-container"
},
"repositories": [
{
"manifest_list_digest": "sha256:2222",
"published": True,
"registry": "reg2",
"repository": "repo2",
"tags": [{"name": "tag2"}]
}
]
},
{
"brew": {
"build": "s2i-1-2",
"completion_date": "2020-08-12T11:31:39+00:00",
"nvra": "s2i-1-2.amd64",
"package": "s2i-core-container"
},
"repositories": [
{
"manifest_list_digest": "sha256:3333",
"published": True,
"registry": "reg3",
"repository": "repo3",
"tags": [{"name": "latest"}]
}
]
},
{
"brew": {
"build": "s2i-1-2",
"completion_date": "2020-08-12T11:31:39+00:00",
"nvra": "s2i-1-2.arm64",
"package": "s2i-core-container"
},
"repositories": [
{
"manifest_list_digest": "sha256:4444",
"published": True,
"registry": "reg4",
"repository": "repo4",
"tags": [{"name": "tag1"}]
}
]
}
]
def tearDown(self):
super().tearDown()
self.patcher.unpatch_all()
@staticmethod
def copy_call_args(mock):
"""
Copy args of Mock to another Mock so we can check call args if we call
mock with mutable args and change it between calls
"""
new_mock = Mock()
def side_effect(*args, **kwargs):
args = deepcopy(args)
kwargs = deepcopy(kwargs)
return new_mock(*args, **kwargs)
mock.side_effect = side_effect
return new_mock
@patch('freshmaker.pyxis.HTTPKerberosAuth')
@patch('freshmaker.pyxis.requests.get')
def test_make_request(self, get, auth):
get.return_value = self.response
test_params = {'key1': 'val1'}
self.px._make_request('test', test_params)
get_url = self.fake_server_url + 'v1/test'
self.response.json.assert_called_once()
test_params['page_size'] = "100"
get.assert_called_once_with(get_url, params=test_params, auth=auth(),
timeout=conf.net_timeout)
@patch('freshmaker.pyxis.HTTPKerberosAuth')
@patch('freshmaker.pyxis.requests.get')
def test_make_request_error(self, get, auth):
get.return_value = self.response
self.response.ok = False
self.response.json.side_effect = ValueError
self.response.json.text = 'test message'
self.response.request = Mock()
self.response.request.url = 'test/url'
with self.assertRaises(PyxisRequestError, msg='test message'):
self.px._make_request('test', {})
@patch('freshmaker.pyxis.HTTPKerberosAuth')
@patch('freshmaker.pyxis.Pyxis._make_request')
def test_pagination(self, request, auth):
my_request = self.copy_call_args(request)
my_request.side_effect = [
{"page": 0, "data": ["fake_data1"]},
{"page": 1, "data": ["fake_data2"]},
{"page": 2, "data": []}
]
test_params = {'include': ['total', 'field1']}
entity = 'test'
auth.return_value = 1
self.px._pagination(entity, test_params)
self.assertEqual(request.call_count, 3)
default_params = {'page_size': '100', 'include': ['total', 'field1']}
calls = [call('test', params={**default_params, 'page': 0}),
call('test', params={**default_params, 'page': 1}),
call('test', params={**default_params, 'page': 2})
]
my_request.assert_has_calls(calls)
@patch.object(conf, 'pyxis_index_image_organization', new='org')
@patch('freshmaker.pyxis.Pyxis._pagination')
def test_get_operator_indices(self, page):
self.px.get_operator_indices()
page.assert_called_once_with(
'operators/indices', {'filter': 'organization==org'})
@patch.object(conf, "product_pages_api_url", new="http://pp.example.com/api")
@patch("freshmaker.pyxis.Pyxis._pagination")
def test_get_operator_indices_with_unreleased_filtered_out(self, page):
pp_mock_data = [
{
"url": "http://pp.example.com/api/releases/openshift-4.5/schedule-tasks",
"json": [{"name": "GA", "date_finish": "2020-02-05"}]
},
{
"url": "http://pp.example.com/api/releases/openshift-4.6/schedule-tasks",
"json": [{"name": "GA", "date_finish": "2020-05-23"}]
},
{
"url": "http://pp.example.com/api/releases/openshift-4.8/schedule-tasks",
"json": [{"name": "GA", "date_finish": "2021-08-12"}]
}
]
page.return_value = self.indices + [
{
"_id": "3",
"created_by": "meteor",
"creation_date": "2020-11-01T08:23:28.253000+00:00",
"last_update_date": "2020-11-01T08:23:28.253000+00:00",
"last_updated_by": "meteor",
"ocp_version": "4.8",
"organization": "org",
"path": ""
}
]
now = datetime(year=2020, month=12, day=15, hour=0, minute=0, second=0)
with requests_mock.Mocker() as http:
for data in pp_mock_data:
http.get(data["url"], json=data["json"])
with freeze_time(now):
indices = self.px.get_operator_indices()
assert len(indices) == 3
assert "4.8" not in [i["ocp_version"] for i in indices]
@patch('freshmaker.pyxis.Pyxis._pagination')
def test_get_latest_bundles(self, page):
page_copy = self.copy_call_args(page)
# Ensure this one is ignored
bad_version_bundle = {
"channel_name": "test-v2.3",
"related_images": [
{
"image": "registry/quay/quay-operator@sha256:ddd",
"name": "quay-operator-annotation",
"digest": "sha256:ddd"
},
{
"image": "registry/quay/quay-security-r-operator@sha256:eee",
"name": "container-security-operator",
"digest": "sha256:eee"
}
],
"version": "version_me"
}
page_copy.side_effect = [self.bundles[:3] + [bad_version_bundle], []]
out = self.px.get_latest_bundles(self.indices)
expected_out = self.bundles[:3]
self.assertEqual(out, expected_out)
page_copy.assert_has_calls([
call('operators/bundles',
{'include': 'data.channel_name,data.version,'
'data.related_images,data.bundle_path_digest,'
'data.bundle_path,data.csv_name',
'filter': 'latest_in_channel==true%20and%20'
'source_index_container_path==path/to/registry:v4.5'}),
call('operators/bundles',
{'include': 'data.channel_name,data.version,'
'data.related_images,data.bundle_path_digest,'
'data.bundle_path,data.csv_name',
'filter': 'latest_in_channel==true%20and%20'
'source_index_container_path==path/to/registry:v4.6'}),
])
@patch('freshmaker.pyxis.Pyxis._pagination')
def test_get_manifest_list_digest_by_nvr(self, page):
page.return_value = self.images
digest = self.px.get_manifest_list_digest_by_nvr('s2i-1-2')
expected_digest = 'sha256:1112'
self.assertEqual(digest, expected_digest)
page.assert_called_once_with(
'images/nvr/s2i-1-2',
{'include': 'data.brew,data.repositories'}
)
def test_get_bundles_by_related_image_digest(self):
digest = 'sha256:111'
new_bundles = self.px.get_bundles_by_related_image_digest(
digest, self.bundles)
expected_bundles = [self.bundles[0]]
self.assertListEqual(new_bundles, expected_bundles)
@patch('freshmaker.pyxis.requests.get')
def test_get_images_by_digest(self, mock_get):
image_1 = {
'brew': {
'build': 'foo-operator-2.1-2',
'nvra': 'foo-operator-2.1-2.amd64',
'package': 'foo',
},
'repositories': [
{
'content_advisory_ids': [],
'manifest_list_digest': 'sha256:12345',
'manifest_schema2_digest': 'sha256:23456',
'published': True,
'registry': 'registry.example.com',
'repository': 'foo/foo-operator-bundle',
'tags': [{'name': '2'}, {'name': '2.1'}],
}
],
}
fake_responses = [Mock(ok=True), Mock(ok=True)]
fake_responses[0].json.return_value = {'data': [image_1]}
fake_responses[1].json.return_value = {'data': []}
mock_get.side_effect = fake_responses
digest = 'sha256:23456'
images = self.px.get_images_by_digest(digest)
self.assertListEqual(images, [image_1])
@patch('freshmaker.pyxis.requests.get')
def test_get_auto_rebuild_tags(self, mock_get):
mock_get.return_value = Mock(ok=True)
mock_get.return_value.json.return_value = {
'_links': {},
'auto_rebuild_tags': [
'2.3',
'latest'
]
}
tags = self.px.get_auto_rebuild_tags('registry.example.com', 'foo/foo-operator-bundle')
self.assertListEqual(tags, ['2.3', 'latest'])