def get_operator_artifact_type(operatorArtifactString):
"""get_operator_artifact_type takes a yaml string and determines if it is
one of the expected bundle types: ClusterServiceVersion,
CustomResourceDefinition, or Package.
:param operatorArtifactString: Yaml string to type check
"""
try:
operatorArtifact = safe_load(operatorArtifactString)
except MarkedYAMLError:
msg = "Courier requires valid input YAML files"
logger.error(msg)
raise OpCourierBadYaml(msg)
else:
artifact_type, artifact_name = None, None
if isinstance(operatorArtifact, dict):
if "packageName" in operatorArtifact:
artifact_type = "Package"
artifact_name = operatorArtifact['packageName']
elif operatorArtifact.get("kind") in ("ClusterServiceVersion",
"CustomResourceDefinition"):
artifact_type = operatorArtifact["kind"]
artifact_name = operatorArtifact['metadata']['name']
if artifact_type is not None:
logger.info('Parsed %s: %s', artifact_type, artifact_name)
return artifact_type
msg = 'Courier requires valid CSV, CRD, and Package files'
logger.error(msg)
raise OpCourierBadArtifact(msg)
def get_folder_semver(folder_path: str):
for item in os.listdir(folder_path):
item_path = os.path.join(folder_path, item)
if not os.path.isfile(item_path) or not is_yaml_file(item_path):
continue
with open(item_path, 'r') as f:
file_content = f.read()
if identify.get_operator_artifact_type(
file_content) == 'ClusterServiceVersion':
try:
csv_version = safe_load(file_content)['spec']['version']
except MarkedYAMLError:
msg = f'{item} is not a valid YAML file.'
logger.error(msg)
raise OpCourierBadYaml(msg)
except KeyError:
msg = f'{item} is not a valid CSV file as "spec.version" ' \
f'field is required'
logger.error(msg)
raise OpCourierBadBundle(msg, {})
return csv_version
return None
def get_operator_artifact_type(operatorArtifactString):
"""get_operator_artifact_type takes a yaml string and determines if it is
one of the expected bundle types: ClusterServiceVersion,
CustomResourceDefinition, or Package.
:param operatorArtifactString: Yaml string to type check
"""
try:
operatorArtifact = safe_load(operatorArtifactString)
except MarkedYAMLError:
msg = "Courier requires valid input YAML files"
logger.error(msg)
raise OpCourierBadYaml(msg)
else:
artifact_type = None
if isinstance(operatorArtifact, dict):
if "packageName" in operatorArtifact:
artifact_type = PKG_STR
elif operatorArtifact.get("kind") in {CRD_STR, CSV_STR}:
artifact_type = operatorArtifact["kind"]
if artifact_type is not None:
return artifact_type
msg = 'Courier requires valid CSV, CRD, and Package files'
logger.error(msg)
raise OpCourierBadArtifact(msg)
def get_operator_artifact_type(operatorArtifactString):
"""get_operator_artifact_type takes a yaml string and determines if it is
one of the expected bundle types.
:param operatorArtifactString: Yaml string to type check
"""
# Default to unknown file unless identified
artifact_type = UNKNOWN_FILE
try:
operatorArtifact = safe_load(operatorArtifactString)
except MarkedYAMLError:
msg = "Courier requires valid input YAML files"
logger.error(msg)
raise OpCourierBadYaml(msg)
else:
if isinstance(operatorArtifact, dict):
if "packageName" in operatorArtifact:
artifact_type = PKG_STR
elif operatorArtifact.get("kind") in {CRD_STR, CSV_STR}:
artifact_type = operatorArtifact["kind"]
return artifact_type
class TestQuayOrganization:
"""Tests for QuayOrganization class"""
org = "org"
cnr_token = "cnr_token"
repo = "repo"
version = "0.0.1"
source_dir = "/not/important/dir"
@pytest.mark.usefixtures('mocked_op_courier_push')
def test_push_operator_manifest(self):
"""Test for pushing operator manifest"""
qo = QuayOrganization(self.org, self.cnr_token)
qo.push_operator_manifest(self.repo, self.version, self.source_dir)
@pytest.mark.parametrize('courier_exception', [
OpCourierError('a'),
OpCourierQuayError('b'),
OpCourierQuayCommunicationError('c'),
OpCourierValueError('d')
])
def test_generic_courier_error(self, courier_exception, caplog,
op_courier_push_raising):
"""Test that all the courier exceptions meant to be handled
in a generic way are in fact handled that way"""
expected_dict = {
'status': 500,
'error': 'QuayCourierError',
'message': f'Failed to push manifest: {courier_exception}',
'quay_response': {}
}
self._test_courier_exception(courier_exception, QuayCourierError,
expected_dict, caplog,
op_courier_push_raising)
@pytest.mark.parametrize('courier_exception',
[OpCourierBadYaml('Bad yaml.')])
def test_courier_invalid_files_error(self, courier_exception, caplog,
op_courier_push_raising):
"""Test that the proper exception is raised when courier reports
an error while building bundle (invalid yaml/artifact)"""
expected_dict = {
'status': 400,
'error': 'PackageValidationError',
'message': f'Failed to push manifest: {courier_exception}',
'validation_info': {}
}
self._test_courier_exception(courier_exception, PackageValidationError,
expected_dict, caplog,
op_courier_push_raising)
def test_courier_invalid_bundle_error(self, caplog,
op_courier_push_raising):
"""Test that the proper exception is raised when courier reports
a validation error after building bundle"""
validation_info = {'errors': ['this one', 'this one too']}
error = OpCourierBadBundle('Bad bundle.', validation_info)
expected_dict = {
'status': 400,
'error': 'PackageValidationError',
'message': f'Failed to push manifest: {error}',
'validation_info': validation_info
}
self._test_courier_exception(error, PackageValidationError,
expected_dict, caplog,
op_courier_push_raising)
def test_courier_quay_authorization_error(self, caplog,
op_courier_push_raising):
"""Test that the proper exception is raised when courier reports
a Quay authorization error"""
error_response = {'error': 'something with authorization'}
error = OpCourierQuayErrorResponse('Quay error.', 403, error_response)
expected_dict = {
'status': 403,
'error': 'QuayAuthorizationError',
'message': f'Failed to push manifest: {error}',
'quay_response': error_response
}
self._test_courier_exception(error, QuayAuthorizationError,
expected_dict, caplog,
op_courier_push_raising)
def _test_courier_exception(self, courier_exception,
expected_omps_exception, expected_dict, caplog,
op_courier_push_raising):
qo = QuayOrganization(self.org, self.cnr_token)
with op_courier_push_raising(courier_exception):
with pytest.raises(expected_omps_exception) as exc_info, \
caplog.at_level(logging.ERROR):
qo.push_operator_manifest(self.repo, self.version,
self.source_dir)
e = exc_info.value
assert e.to_dict() == expected_dict
assert any('Operator courier call failed' in message
for message in caplog.messages)
@pytest.mark.usefixtures('mocked_op_courier_push')
def test_push_operator_manifest_publish_repo(self):
"""Organizations marked as public will try to publish new
repositories"""
qo = QuayOrganization(self.org,
self.cnr_token,
oauth_token='random',
public=True)
(flexmock(qo).should_receive('publish_repo').and_return(None).once())
qo.push_operator_manifest(self.repo, self.version, self.source_dir)
@pytest.mark.usefixtures('mocked_op_courier_push')
def test_push_operator_manifest_publish_repo_no_public(self):
"""Make repos won't be published for non-public organizations"""
qo = QuayOrganization(self.org,
self.cnr_token,
oauth_token='random',
public=False)
(flexmock(qo).should_receive('publish_repo').and_return(None).never())
qo.push_operator_manifest(self.repo, self.version, self.source_dir)
@pytest.mark.usefixtures('mocked_op_courier_push')
def test_push_operator_manifest_publish_repo_no_oauth(self, caplog):
"""Make sure that proper warning msg is logged"""
qo = QuayOrganization(self.org, self.cnr_token, public=True)
(flexmock(qo).should_receive('publish_repo').and_return(None).never())
caplog.clear()
with caplog.at_level(logging.ERROR):
qo.push_operator_manifest(self.repo, self.version, self.source_dir)
messages = (rec.message for rec in caplog.records)
assert any('Oauth access is not configured' in m for m in messages)
def test_get_latest_release_version(self):
"""Test getting the latest release version"""
org = "test_org"
repo = "test_repo"
with requests_mock.Mocker() as m:
m.get(f'/cnr/api/v1/packages?namespace={org}',
json=[
{
'name': 'org/something_else',
'releases': ["2.0.0"],
},
{
'name': f'{org}/{repo}',
'releases': ["1.2.0", "1.1.0", "1.0.0"]
},
])
qo = QuayOrganization(org, "token")
latest = qo.get_latest_release_version(repo)
assert str(latest) == "1.2.0"
def test_get_latest_release_version_not_found(self):
"""Test if proper exception is raised when no package is not found"""
org = "test_org"
repo = "test_repo"
with requests_mock.Mocker() as m:
m.get(f'/cnr/api/v1/packages?namespace={org}',
json=[{
'name': 'org/something_else',
'releases': ["2.0.0"],
}])
qo = QuayOrganization(org, "token")
with pytest.raises(QuayPackageNotFound):
qo.get_latest_release_version(repo)
def test_get_latest_release_version_invalid_version_only(self):
"""Test if proper exception is raised when packages only with invalid
version are available
Invalid versions should be ignored, thus QuayPackageNotFound
should be raised as may assume that OMPS haven't managed that packages
previously
"""
org = "test_org"
repo = "test_repo"
with requests_mock.Mocker() as m:
m.get(f'/cnr/api/v1/packages?namespace={org}',
json=[
{
'name': f'{org}/{repo}',
'releases': ["1.0.0-invalid"]
},
])
qo = QuayOrganization(org, "token")
with pytest.raises(QuayPackageNotFound):
qo.get_latest_release_version(repo)
def test_get_releases_raw(self):
"""Test if all release are returned from quay.io, including format that
is OMPS invalid"""
org = "test_org"
repo = "test_repo"
with requests_mock.Mocker() as m:
m.get(f'/cnr/api/v1/packages?namespace={org}',
json=[
{
'name': 'org/something_else',
'releases': ["2.0.0"],
},
{
'name': f'{org}/{repo}',
'releases': ["1.2.0", "1.0.1-random", "1.0.0"]
},
])
qo = QuayOrganization(org, "token")
releases = qo.get_releases_raw(repo)
assert sorted(releases) == ["1.0.0", "1.0.1-random", "1.2.0"]
@pytest.mark.parametrize('error_code, expected_exc_type',
[(403, QuayAuthorizationError),
(500, QuayPackageError)])
def test_get_releases_raw_errors(self, error_code, expected_exc_type):
"""Test that the proper exceptions are raised for various kinds
of HTTP errors"""
org = "test_org"
repo = "test_repo"
qo = QuayOrganization(org, TOKEN)
with requests_mock.Mocker() as m:
m.get(f'/cnr/api/v1/packages?namespace={org}',
status_code=error_code)
with pytest.raises(expected_exc_type):
qo.get_releases_raw(repo)
def test_get_releases(self):
"""Test if only proper releases are used and returned"""
org = "test_org"
repo = "test_repo"
qo = QuayOrganization(org, TOKEN)
(flexmock(qo).should_receive('get_releases_raw').and_return(
["1.0.0", "1.0.1-random", "1.2.0"]))
expected = [ReleaseVersion.from_str(v) for v in ["1.0.0", "1.2.0"]]
assert qo.get_releases(repo) == expected
def test_delete_release(self):
"""Test of deleting releases"""
org = "test_org"
repo = "test_repo"
version = '1.2.3'
qo = QuayOrganization(org, TOKEN)
with requests_mock.Mocker() as m:
m.delete(f'/cnr/api/v1/packages/{org}/{repo}/{version}/helm', )
qo.delete_release(repo, version)
@pytest.mark.parametrize('code,exc_class', [
(requests.codes.not_found, QuayPackageNotFound),
(requests.codes.method_not_allowed, QuayPackageError),
(requests.codes.internal_server_error, QuayPackageError),
])
def test_delete_release_quay_error(self, code, exc_class):
"""Test of error handling from quay errors"""
org = "test_org"
repo = "test_repo"
version = '1.2.3'
qo = QuayOrganization(org, TOKEN)
with requests_mock.Mocker() as m:
m.delete(f'/cnr/api/v1/packages/{org}/{repo}/{version}/helm',
status_code=code)
with pytest.raises(exc_class):
qo.delete_release(repo, version)
def test_publish_repo(self):
"""Test publishing repository"""
org = 'testorg'
repo = 'testrepo'
qo = QuayOrganization(org, TOKEN, oauth_token='randomtoken')
with requests_mock.Mocker() as m:
m.post(
f'/api/v1/repository/{org}/{repo}/changevisibility',
status_code=requests.codes.ok,
)
qo.publish_repo(repo)
def test_publish_repo_error(self):
"""Test if publishing repository raises proper exception"""
org = 'testorg'
repo = 'testrepo'
qo = QuayOrganization(org, TOKEN, oauth_token='randomtoken')
with requests_mock.Mocker() as m:
m.post(
f'/api/v1/repository/{org}/{repo}/changevisibility',
status_code=requests.codes.server_error,
)
with pytest.raises(QuayPackageError):
qo.publish_repo(repo)
@pytest.mark.parametrize('enabled', [True, False])
def test_registry_replacing_enabled(self, enabled):
"""Test if property returns correct value"""
if enabled:
replace_conf = [{'old': 'reg_old', 'new': 'reg_new'}]
else:
replace_conf = None
org = 'testorg'
qo = QuayOrganization(org, TOKEN, replace_registry_conf=replace_conf)
assert qo.registry_replacing_enabled == enabled
@pytest.mark.parametrize('text,expected', [
('Registry reg_old will be replaced',
'Registry reg_new will be replaced'),
(
'Registry nope will not be replaced',
'Registry nope will not be replaced',
),
])
def test_replace_registries(self, text, expected):
"""Test if registries are replaced properly"""
replace_conf = [{'old': 'reg_old', 'new': 'reg_new'}]
org = 'testorg'
qo = QuayOrganization(org, TOKEN, replace_registry_conf=replace_conf)
assert qo.replace_registries(text) == expected
def test_replace_registries_unconfigured(self):
"""Test if replace operation returns unchanged text"""
org = 'testorg'
qo = QuayOrganization(org, TOKEN)
text = 'text'
res = qo.replace_registries(text)
assert res == text
assert id(res) == id(text)
@pytest.mark.parametrize('text,expected', [(
'Registry reg_old will be replaced using a regexp: reg_old',
'Registry reg_old will be replaced using a regexp: reg_new',
)])
def test_regexp_replace_registries(self, text, expected):
"""Test if registries are replaced properly with regexp"""
replace_conf = [{'old': 'reg_old$', 'new': 'reg_new', 'regexp': True}]
org = 'testorg'
qo = QuayOrganization(org, TOKEN, replace_registry_conf=replace_conf)
assert qo.replace_registries(text) == expected
def parse_manifest_folder(manifest_path: str, folder_semver: str,
csv_paths: list, crd_dict: Dict[str, Tuple[str,
str]]):
"""
Parse the version folder of the bundle and collect information of CSV and CRDs
in the bundle
:param manifest_path: The path of the manifest folder containing bundle files
:param folder_semver: The semantic version of the current folder
:param csv_paths: A list of CSV file paths inside version folders
:param crd_dict: dict that contains CRD info collected from different version folders,
where the key is the CRD name, and the value is a tuple where the first element is
the version of the bundle, and the second is the path of the CRD file
"""
logger.info('Parsing folder %s for operator version %s',
os.path.basename(manifest_path), folder_semver)
contains_csv = False
for item in os.listdir(manifest_path):
item_path = os.path.join(manifest_path, item)
if not os.path.isfile(item_path):
logger.warning('Ignoring %s as it is not a regular file.', item)
continue
if not is_yaml_file(item_path):
logger.warning(
'Ignoring %s as the file does not end with .yaml or .yml',
item_path)
continue
with open(item_path, 'r') as f:
file_content = f.read()
yaml_type = identify.get_operator_artifact_type(file_content)
if yaml_type == 'ClusterServiceVersion':
contains_csv = True
csv_paths.append(item_path)
elif yaml_type == 'CustomResourceDefinition':
try:
crd_name = safe_load(file_content)['metadata']['name']
except MarkedYAMLError:
msg = "Courier requires valid input YAML files"
logger.error(msg)
raise OpCourierBadYaml(msg)
except KeyError:
msg = f'{item} is not a valid CRD file as "metadata.name" ' \
f'field is required'
logger.error(msg)
raise OpCourierBadBundle(msg, {})
# create new CRD type entry if not found in dict
if crd_name not in crd_dict:
crd_dict[crd_name] = (folder_semver, item_path)
# update the CRD type entry with the file with the newest version
elif semver.compare(folder_semver, crd_dict[crd_name][0]) > 0:
crd_dict[crd_name] = (crd_dict[crd_name][0], item_path)
if not contains_csv:
msg = 'This version directory does not contain any valid CSV file.'
logger.error(msg)
raise OpCourierBadBundle(msg, {})