def _run_mirror(self, ocp_release, dest_ocp_release, dest_ocp_art_dev):
# Checking if the image is already there
if self._is_image_there(dest_ocp_release):
LOG.info(f'Image {ocp_release} already in '
f'the mirror. Skipping.')
return
LOG.info(f'Mirroring {ocp_release} to {dest_ocp_art_dev} '
f'to_release {dest_ocp_release}')
if self.dry_run:
return
# Creating a new, bare, OC client since we don't
# want to run this against any cluster or via
# a jump host
oc_cli = OC(server='',
token='',
jh=None,
settings=None,
init_projects=False,
init_api_resources=False)
oc_cli.release_mirror(from_release=ocp_release,
to=dest_ocp_art_dev,
to_release=dest_ocp_release,
dockerconfig=self.registry_creds)
def test_get_owned_pods(self, oc_get_obj_root_owner, oc_get):
owner_body = {'kind': 'ownerkind', 'metadata': {'name': 'ownername'}}
owner_resource = OR(owner_body, '', '')
oc_get.return_value = {
'items': [
{
'metadata': {
'name':
'pod1',
'ownerReferences': [{
'controller': True,
'kind': 'ownerkind',
'name': 'ownername'
}]
}
},
{
'metadata': {
'name':
'pod2',
'ownerReferences': [{
'controller': True,
'kind': 'notownerkind',
'name': 'notownername'
}]
}
},
{
'metadata': {
'name':
'pod3',
'ownerReferences': [{
'controller': True,
'kind': 'ownerkind',
'name': 'notownername'
}]
}
},
]
}
oc_get_obj_root_owner.side_effect = [
owner_resource.body, {
'kind': 'notownerkind',
'metadata': {
'name': 'notownername'
},
}, {
'kind': 'ownerkind',
'metadata': {
'name': 'notownername'
}
}
]
oc = OC('server', 'token', local=True)
pods = oc.get_owned_pods('namespace', owner_resource)
self.assertEqual(len(pods), 1)
self.assertEqual(pods[0]['metadata']['name'], 'pod1')
def test_no_owner(self):
obj = {
'metadata': {
'name': 'pod1',
}
}
oc = OC('server', 'token', local=True)
result_obj = oc.get_obj_root_owner('namespace', obj)
self.assertEqual(result_obj, obj)
def test_validate_pod_ready_all_good(oc_get):
oc_get.return_value = {
'status': {
'containerStatuses': [{
'name': 'container1',
'ready': True,
}, {
'name': 'container2',
'ready': True
}]
}
}
oc = OC('server', 'token', local=True)
oc.validate_pod_ready('namespace', 'podname')
def test_owner(self, oc_get):
obj = {
'metadata': {
'name':
'pod1',
'ownerReferences': [{
'controller': True,
'kind': 'ownerkind',
'name': 'ownername'
}]
}
}
owner_obj = {'kind': 'ownerkind', 'metadata': {'name': 'ownername'}}
oc_get.side_effect = [owner_obj]
oc = OC('server', 'token', local=True)
result_owner_obj = oc.get_obj_root_owner('namespace', obj)
self.assertEqual(result_owner_obj, owner_obj)
def test_no_owner(self):
obj = {
"metadata": {
"name": "pod1",
}
}
oc = OC("cluster", "server", "token", local=True)
result_obj = oc.get_obj_root_owner("namespace", obj)
self.assertEqual(result_obj, obj)
def _run_mirror(self, ocp_release, dest_ocp_release, dest_ocp_art_dev):
# Checking if the image is already there
if self._is_image_there(dest_ocp_release):
LOG.debug(f'Image {ocp_release} already in '
f'the mirror. Skipping.')
return
LOG.info(f'Mirroring {ocp_release} to {dest_ocp_art_dev} '
f'to_release {dest_ocp_release}')
if self.dry_run:
return
# Creating a new, bare, OC client since we don't
# want to run this against any cluster or via
# a jump host
oc_cli = OC('cluster', None, None, local=True)
oc_cli.release_mirror(from_release=ocp_release,
to=dest_ocp_art_dev,
to_release=dest_ocp_release,
dockerconfig=self.registry_creds)
def test_ok(self):
pods = [{
'spec': {
'volumes': [{
'persistentVolumeClaim': {
'claimName': 'cm'
}
}]
}
}]
oc = OC('cluster', 'server', 'token', local=True)
owned_pvc_names = oc.get_pod_owned_pvc_names(pods)
self.assertEqual(len(owned_pvc_names), 1)
self.assertEqual(list(owned_pvc_names)[0], 'cm')
def test_ok(self):
pods = [{
"spec": {
"volumes": [{
"persistentVolumeClaim": {
"claimName": "cm"
}
}]
}
}]
oc = OC("cluster", "server", "token", local=True)
owned_pvc_names = oc.get_pod_owned_pvc_names(pods)
self.assertEqual(len(owned_pvc_names), 1)
self.assertEqual(list(owned_pvc_names)[0], "cm")
def test_controller_false_return_obj(self):
"""Returns obj if all ownerReferences have controller set to false"""
obj = {
"metadata": {
"name": "pod1",
"ownerReferences": [{
"controller": False
}]
}
}
oc = OC("cluster", "server", "token", local=True)
result_obj = oc.get_obj_root_owner("namespace", obj)
self.assertEqual(result_obj, obj)
def test_controller_false_return_obj(self):
"""Returns obj if all ownerReferences have controller set to false
"""
obj = {
'metadata': {
'name': 'pod1',
'ownerReferences': [{
'controller': False
}]
}
}
oc = OC('cluster', 'server', 'token', local=True)
result_obj = oc.get_obj_root_owner('namespace', obj)
self.assertEqual(result_obj, obj)
def test_validate_pod_ready_all_good(self, oc_get):
oc_get.return_value = {
"status": {
"containerStatuses": [
{
"name": "container1",
"ready": True,
},
{
"name": "container2",
"ready": True
},
]
}
}
oc = OC("cluster", "server", "token", local=True)
oc.validate_pod_ready("namespace", "podname")
def test_validate_pod_ready_one_missing(self, oc_get):
oc_get.return_value = {
'status': {
'containerStatuses': [{
'name': 'container1',
'ready': True,
}, {
'name': 'container2',
'ready': False
}]
}
}
oc = OC('server', 'token', local=True)
with self.assertRaises(PodNotReadyError):
# Bypass the retry stuff
oc.validate_pod_ready.__wrapped__(oc, 'namespace', 'podname')
def test_ok(self):
size = "100Gi"
resource = {
"spec": {
"volumeClaimTemplates": [{
"spec": {
"resources": {
"requests": {
"storage": size
}
}
}
}]
}
}
oc = OC("cluster", "server", "token", local=True)
result = oc.get_storage(resource)
self.assertEqual(result, size)
def test_ok(self):
size = "100Gi"
resource = {
'spec': {
'volumeClaimTemplates': [{
'spec': {
'resources': {
'requests': {
'storage': size
}
}
}
}]
}
}
oc = OC('cluster', 'server', 'token', local=True)
result = oc.get_storage(resource)
self.assertEqual(result, size)
def test_owner(self, oc_get):
obj = {
"metadata": {
"name":
"pod1",
"ownerReferences": [{
"controller": True,
"kind": "ownerkind",
"name": "ownername"
}],
}
}
owner_obj = {"kind": "ownerkind", "metadata": {"name": "ownername"}}
oc_get.side_effect = [owner_obj]
oc = OC("cluster", "server", "token", local=True)
result_owner_obj = oc.get_obj_root_owner("namespace", obj)
self.assertEqual(result_owner_obj, owner_obj)
def test_validate_pod_ready_one_missing(self, oc_get):
oc_get.return_value = {
"status": {
"containerStatuses": [
{
"name": "container1",
"ready": True,
},
{
"name": "container2",
"ready": False
},
]
}
}
oc = OC("cluster", "server", "token", local=True)
with self.assertRaises(PodNotReadyError):
# Bypass the retry stuff
oc.validate_pod_ready.__wrapped__(oc, "namespace", "podname")
def test_controller_true_allow_false_ref_not_found_raise(self, oc_get):
"""Throws an exception if controller is true, allow_not_found false,
but referenced object does not exist
"""
obj = {
'metadata': {
'name':
'pod1',
'ownerReferences': [{
'controller': True,
'kind': 'ownerkind',
'name': 'ownername'
}]
}
}
oc_get.side_effect = StatusCodeError
oc = OC('cluster', 'server', 'token', local=True)
with self.assertRaises(StatusCodeError):
oc.get_obj_root_owner('namespace', obj, allow_not_found=False)
def test_controller_true_allow_false_ref_not_found_raise(self, oc_get):
"""Throws an exception if controller is true, allow_not_found false,
but referenced object does not exist
"""
obj = {
"metadata": {
"name":
"pod1",
"ownerReferences": [{
"controller": True,
"kind": "ownerkind",
"name": "ownername"
}],
}
}
oc_get.side_effect = StatusCodeError
oc = OC("cluster", "server", "token", local=True)
with self.assertRaises(StatusCodeError):
oc.get_obj_root_owner("namespace", obj, allow_not_found=False)
def test_oc_native(self, mock_request):
mock_request.side_effect = request
oc = OC('cluster', 'server', 'token', init_projects=True, local=True)
expected = fixture['api_kind_version']
self.assertEqual(oc.api_kind_version, expected)
expected = fixture['projects']
self.assertEqual(oc.projects, expected)
kind = 'Project.config.openshift.io'
_k, group_version = oc._parse_kind(kind)
self.assertEqual(group_version, 'config.openshift.io/v1')
kind = 'Test'
with self.assertRaises(StatusCodeError):
oc._parse_kind(kind)
kind = 'Project.test.io'
with self.assertRaises(StatusCodeError):
oc._parse_kind(kind)
def test_oc_native(self, mock_request):
mock_request.side_effect = request
oc = OC("cluster", "server", "token", init_projects=True, local=True)
expected = fixture["api_kind_version"]
self.assertEqual(oc.api_kind_version, expected)
expected = fixture["projects"]
self.assertEqual(oc.projects, expected)
kind = "Project.config.openshift.io"
_k, group_version = oc._parse_kind(kind)
self.assertEqual(group_version, "config.openshift.io/v1")
kind = "Test"
with self.assertRaises(StatusCodeError):
oc._parse_kind(kind)
kind = "Project.test.io"
with self.assertRaises(StatusCodeError):
oc._parse_kind(kind)
def test_controller_false_return_controller(self, oc_get):
"""Returns owner if all ownerReferences have controller set to false
and allow_not_controller is set to True
"""
obj = {
"metadata": {
"name":
"pod1",
"ownerReferences": [{
"controller": False,
"kind": "ownerkind",
"name": "ownername"
}],
}
}
owner_obj = {"kind": "ownerkind", "metadata": {"name": "ownername"}}
oc_get.side_effect = [owner_obj]
oc = OC("cluster", "server", "token", local=True)
result_obj = oc.get_obj_root_owner("namespace",
obj,
allow_not_controller=True)
self.assertEqual(result_obj, owner_obj)
def test_controller_false_return_controller(self, oc_get):
"""Returns owner if all ownerReferences have controller set to false
and allow_not_controller is set to True
"""
obj = {
'metadata': {
'name':
'pod1',
'ownerReferences': [{
'controller': False,
'kind': 'ownerkind',
'name': 'ownername'
}]
}
}
owner_obj = {'kind': 'ownerkind', 'metadata': {'name': 'ownername'}}
oc_get.side_effect = [owner_obj]
oc = OC('cluster', 'server', 'token', local=True)
result_obj = oc.get_obj_root_owner('namespace',
obj,
allow_not_controller=True)
self.assertEqual(result_obj, owner_obj)
def test_cont_true_allow_true_ref_not_found_return_obj(self, oc_get):
"""Returns obj if controller is true, allow_not_found is true,
but referenced object does not exist '{}'
"""
obj = {
"metadata": {
"name":
"pod1",
"ownerReferences": [{
"controller": True,
"kind": "ownerkind",
"name": "ownername"
}],
}
}
owner_obj = {}
oc_get.side_effect = [owner_obj]
oc = OC("cluster", "server", "token", local=True)
result_obj = oc.get_obj_root_owner("namespace",
obj,
allow_not_found=True)
self.assertEqual(result_obj, obj)
def test_cont_true_allow_true_ref_not_found_return_obj(self, oc_get):
"""Returns obj if controller is true, allow_not_found is true,
but referenced object does not exist '{}'
"""
obj = {
'metadata': {
'name':
'pod1',
'ownerReferences': [{
'controller': True,
'kind': 'ownerkind',
'name': 'ownername'
}]
}
}
owner_obj = {}
oc_get.side_effect = [owner_obj]
oc = OC('cluster', 'server', 'token', local=True)
result_obj = oc.get_obj_root_owner('namespace',
obj,
allow_not_found=True)
self.assertEqual(result_obj, obj)
def _process_template(self, options):
saas_file_name = options['saas_file_name']
resource_template_name = options['resource_template_name']
image_auth = options['image_auth']
url = options['url']
path = options['path']
provider = options['provider']
target = options['target']
github = options['github']
target_ref = target['ref']
target_promotion = target.get('promotion') or {}
resources = None
html_url = None
commit_sha = None
if provider == 'openshift-template':
hash_length = options['hash_length']
parameters = options['parameters']
environment = target['namespace']['environment']
environment_parameters = self._collect_parameters(environment)
target_parameters = self._collect_parameters(target)
consolidated_parameters = {}
consolidated_parameters.update(environment_parameters)
consolidated_parameters.update(parameters)
consolidated_parameters.update(target_parameters)
for replace_key, replace_value in consolidated_parameters.items():
if not isinstance(replace_value, str):
continue
replace_pattern = '${' + replace_key + '}'
for k, v in consolidated_parameters.items():
if not isinstance(v, str):
continue
if replace_pattern in v:
consolidated_parameters[k] = \
v.replace(replace_pattern, replace_value)
get_file_contents_options = {
'url': url,
'path': path,
'ref': target_ref,
'github': github
}
try:
template, html_url, commit_sha = \
self._get_file_contents(get_file_contents_options)
except Exception as e:
logging.error(f"[{url}/{path}:{target_ref}] " +
f"error fetching template: {str(e)}")
return None, None, None
# add IMAGE_TAG only if it is unspecified
image_tag = consolidated_parameters.get('IMAGE_TAG')
if not image_tag:
sha_substring = commit_sha[:hash_length]
# IMAGE_TAG takes one of two forms:
# - If saas file attribute 'use_channel_in_image_tag' is true,
# it is {CHANNEL}-{SHA}
# - Otherwise it is just {SHA}
if self._get_saas_file_attribute("use_channel_in_image_tag"):
try:
channel = consolidated_parameters["CHANNEL"]
except KeyError:
logging.error(
f"[{saas_file_name}/{resource_template_name}] " +
f"{html_url}: CHANNEL is required when " +
"'use_channel_in_image_tag' is true.")
return None, None, None
image_tag = f"{channel}-{sha_substring}"
else:
image_tag = sha_substring
consolidated_parameters['IMAGE_TAG'] = image_tag
# This relies on IMAGE_TAG already being calculated.
need_repo_digest = self._parameter_value_needed(
"REPO_DIGEST", consolidated_parameters, template)
need_image_digest = self._parameter_value_needed(
"IMAGE_DIGEST", consolidated_parameters, template)
if need_repo_digest or need_image_digest:
try:
logging.debug("Generating REPO_DIGEST.")
registry_image = consolidated_parameters["REGISTRY_IMG"]
except KeyError as e:
logging.error(
f"[{saas_file_name}/{resource_template_name}] " +
f"{html_url}: error generating REPO_DIGEST. " +
"Is REGISTRY_IMG missing? " + f"{str(e)}")
return None, None, None
try:
image_uri = f"{registry_image}:{image_tag}"
img = Image(image_uri, **image_auth)
if need_repo_digest:
consolidated_parameters["REPO_DIGEST"] = img.url_digest
if need_image_digest:
consolidated_parameters["IMAGE_DIGEST"] = img.digest
except (rqexc.ConnectionError, rqexc.HTTPError) as e:
logging.error(
f"[{saas_file_name}/{resource_template_name}] " +
f"{html_url}: error generating REPO_DIGEST for " +
f"{image_uri}: {str(e)}")
return None, None, None
oc = OC('server', 'token', local=True)
try:
resources = oc.process(template, consolidated_parameters)
except StatusCodeError as e:
logging.error(
f"[{saas_file_name}/{resource_template_name}] " +
f"{html_url}: error processing template: {str(e)}")
elif provider == 'directory':
get_directory_contents_options = {
'url': url,
'path': path,
'ref': target_ref,
'github': github
}
try:
resources, html_url, commit_sha = \
self._get_directory_contents(
get_directory_contents_options)
except Exception as e:
logging.error(f"[{url}/{path}:{target_ref}] " +
f"error fetching directory: {str(e)}")
return None, None, None
else:
logging.error(f"[{saas_file_name}/{resource_template_name}] " +
f"unknown provider: {provider}")
target_promotion['commit_sha'] = commit_sha
return resources, html_url, target_promotion
def _process_template(self, options):
saas_file_name = options['saas_file_name']
resource_template_name = options['resource_template_name']
url = options['url']
path = options['path']
provider = options['provider']
target = options['target']
github = options['github']
target_ref = target['ref']
target_promotion = target.get('promotion') or {}
resources = None
html_url = None
commit_sha = None
if provider == 'openshift-template':
hash_length = options['hash_length']
parameters = options['parameters']
environment = target['namespace']['environment']
environment_parameters = self._collect_parameters(environment)
target_parameters = self._collect_parameters(target)
consolidated_parameters = {}
consolidated_parameters.update(environment_parameters)
consolidated_parameters.update(parameters)
consolidated_parameters.update(target_parameters)
for replace_key, replace_value in consolidated_parameters.items():
if not isinstance(replace_value, str):
continue
replace_pattern = '${' + replace_key + '}'
for k, v in consolidated_parameters.items():
if not isinstance(v, str):
continue
if replace_pattern in v:
consolidated_parameters[k] = \
v.replace(replace_pattern, replace_value)
get_file_contents_options = {
'url': url,
'path': path,
'ref': target_ref,
'github': github
}
try:
template, html_url, commit_sha = \
self._get_file_contents(get_file_contents_options)
except Exception as e:
logging.error(f"[{url}/{path}:{target_ref}] " +
f"error fetching template: {str(e)}")
return None, None
if "IMAGE_TAG" not in consolidated_parameters:
template_parameters = template.get('parameters')
if template_parameters is not None:
for template_parameter in template_parameters:
if template_parameter['name'] == 'IMAGE_TAG':
# add IMAGE_TAG only if it is required
image_tag = commit_sha[:hash_length]
consolidated_parameters['IMAGE_TAG'] = image_tag
oc = OC('server', 'token', local=True)
try:
resources = oc.process(template, consolidated_parameters)
except StatusCodeError as e:
logging.error(
f"[{saas_file_name}/{resource_template_name}] " +
f"{html_url}: error processing template: {str(e)}")
elif provider == 'directory':
get_directory_contents_options = {
'url': url,
'path': path,
'ref': target_ref,
'github': github
}
try:
resources, html_url, commit_sha = \
self._get_directory_contents(
get_directory_contents_options)
except Exception as e:
logging.error(f"[{url}/{path}:{target_ref}] " +
f"error fetching directory: {str(e)}")
return None, None
else:
logging.error(f"[{saas_file_name}/{resource_template_name}] " +
f"unknown provider: {provider}")
target_promotion['commit_sha'] = commit_sha
return resources, html_url, target_promotion
def test_no_volumes(self):
pods = [{'spec': {'volumes': []}}]
oc = OC('cluster', 'server', 'token', local=True)
owned_pvc_names = oc.get_pod_owned_pvc_names(pods)
self.assertEqual(len(owned_pvc_names), 0)
def test_other_volumes(self):
pods = [{'spec': {'volumes': [{'configMap': {'name': 'cm'}}]}}]
oc = OC('cluster', 'server', 'token', local=True)
owned_pvc_names = oc.get_pod_owned_pvc_names(pods)
self.assertEqual(len(owned_pvc_names), 0)
def test_none(self):
resource = {'spec': {'what': 'ever'}}
oc = OC('cluster', 'server', 'token', local=True)
storage = oc.get_storage(resource)
self.assertIsNone(storage)