def test_update_manifests_no_overrides_and_values_valid(self):
with open(self.base_manifest) as f:
documents = list(yaml.safe_load_all(f.read()))
ovr = Override(documents)
ovr.update_manifests()
# no updates since no overrides and values provided
self.assertEqual(documents, ovr.documents)
def test_convert_array_to_dict_invalid(self):
data_path = ['a', 'b', 'c']
new_value = ""
ovr = Override(self.base_manifest).array_to_dict(data_path, new_value)
self.assertIsNone(ovr)
ovr = Override(self.base_manifest).array_to_dict([], new_value)
self.assertIsNone(ovr)
def test_find_document_type_valid(self):
ovr = Override(self.base_documents)
test_group = ovr.find_document_type('chart_group')
self.assertEqual(test_group, const.DOCUMENT_GROUP)
test_chart = ovr.find_document_type('chart')
self.assertEqual(test_chart, const.DOCUMENT_CHART)
test_manifest = ovr.find_document_type('manifest')
self.assertEqual(test_manifest, const.DOCUMENT_MANIFEST)
def on_post(self, req, resp):
# Load data from request and get options
if req.content_type == 'application/x-yaml':
data = list(self.req_yaml(req))
if type(data[0]) is list:
documents = list(data[0])
else:
documents = data
elif req.content_type == 'application/json':
self.logger.debug("Applying manifest based on reference.")
req_body = self.req_json(req)
doc_ref = req_body.get('hrefs', None)
if not doc_ref:
self.logger.info("Request did not contain 'hrefs'.")
resp.status = falcon.HTTP_400
return
data = ReferenceResolver.resolve_reference(doc_ref)
documents = list()
for d in data:
documents.extend(list(yaml.safe_load_all(d.decode())))
if req_body.get('overrides', None):
overrides = Override(documents,
overrides=req_body.get('overrides'))
documents = overrides.update_manifests()
else:
self.error(req.context,
"Unknown content-type %s" % req.content_type)
# TODO(fmontei): Use falcon.<Relevant API Exception Class> instead.
return self.return_error(
resp,
falcon.HTTP_415,
message="Request must be in application/x-yaml"
"or application/json")
try:
with self.get_tiller(req, resp) as tiller:
msg = self.handle(req, documents, tiller)
resp.body = json.dumps({
'message': msg,
})
resp.content_type = 'application/json'
resp.status = falcon.HTTP_200
except exceptions.ManifestException as e:
self.return_error(resp, falcon.HTTP_400, message=str(e))
except LockException as e:
self.return_error(resp, falcon.HTTP_409, message=str(e))
except Exception as e:
self.logger.exception('Caught unexpected exception')
err_message = 'Failed to apply manifest: {}'.format(e)
self.error(req.context, err_message)
self.return_error(resp, falcon.HTTP_500, message=err_message)
def test_find_document_type_valid(self):
with open(self.base_manifest) as f:
documents = list(yaml.safe_load_all(f.read()))
ovr = Override(documents)
test_group = ovr.find_document_type('chart_group')
self.assertEqual(test_group, const.DOCUMENT_GROUP)
test_chart = ovr.find_document_type('chart')
self.assertEqual(test_chart, const.DOCUMENT_CHART)
test_manifest = ovr.find_document_type('manifest')
self.assertEqual(test_manifest, const.DOCUMENT_MANIFEST)
def test_set_list_valid(self):
expected = "{}/templates/override-{}-expected.yaml".format(
self.basepath, '03')
with open(self.base_manifest) as f, open(expected) as e:
doc_obj = list(yaml.safe_load_all(f.read()))
doc_path = ['manifest', 'simple-armada']
override = ('manifest:simple-armada:chart_groups=\
blog-group3,blog-group4',)
ovr = Override(doc_obj, override)
ovr.update_manifests()
ovr_doc = ovr.find_manifest_document(doc_path)
expect_doc = list(yaml.load_all(e.read()))[0]
self.assertEqual(expect_doc, ovr_doc)
def test_update_manifests_with_values_valid(self):
original = "{}/templates/override-{}.yaml".format(self.basepath, '01')
values_yaml = "{}/templates/override-{}-expected.yaml".format(
self.basepath, '01')
with open(original) as f, open(values_yaml) as g:
original_documents = list(yaml.safe_load_all(f.read()))
documents_copy = copy.deepcopy(original_documents)
values_documents = list(yaml.safe_load_all(g.read()))
ovr = Override(original_documents, None, [values_yaml])
ovr.update_manifests()
# updating values changed the original document
self.assertNotEqual(original_documents, documents_copy)
# verifying that these documents have the same value now
self.assertEqual(original_documents, values_documents)
def test_update_chart_group_document_keys_not_removed_with_override(self):
with open(self.base_manifest) as f:
documents = list(yaml.safe_load_all(f.read()))
documents_modified = copy.deepcopy(documents)
del documents_modified[1]['data']['sequenced']
# verify both doc have different values for data
self.assertNotEqual(documents[1], documents_modified[1])
ovr = Override(documents)
# update with document values with the modified ones
ovr.update_chart_group_document(documents_modified[1])
self.assertIn('sequenced', ovr.documents[1]['data'])
self.assertNotEqual(ovr.documents[1], documents_modified[1])
def test_update_armada_manifest_keys_not_removed_with_override(self):
with open(self.base_manifest) as f:
documents = list(yaml.safe_load_all(f.read()))
documents_modified = copy.deepcopy(documents)
del documents_modified[2]['data']['release_prefix']
# verify both doc have different values for data
self.assertNotEqual(documents[2], documents_modified[2])
ovr = Override(documents)
# update with document values from base_manifest
ovr.update_armada_manifest(documents_modified[2])
self.assertIn('release_prefix', ovr.documents[2]['data'])
self.assertNotEqual(ovr.documents[2], documents_modified[2])
def test_update_dictionary_valid(self):
expected = "{}/templates/override-{}-expected.yaml".format(
self.basepath, '01')
merge = "{}/templates/override-{}.yaml".format(self.basepath, '01')
with open(self.base_manifest) as f, open(expected) as e, open(
merge) as m:
merging_values = list(yaml.safe_load_all(m.read()))
documents = list(yaml.safe_load_all(f.read()))
doc_path = ['chart', 'blog-1']
ovr = Override(documents)
ovr.update_document(merging_values)
ovr_doc = ovr.find_manifest_document(doc_path)
expect_doc = list(yaml.load_all(e.read()))[0]
self.assertEqual(ovr_doc, expect_doc)
def test_convert_array_to_dict_valid(self):
data_path = ['a', 'b', 'c']
new_value = "dev"
expected_dict = {'a': {'b': {'c': 'dev'}}}
ovr = Override(self.base_manifest).array_to_dict(data_path, new_value)
self.assertEqual(ovr, expected_dict)
def test_update_manifests_invalid(self):
with open(self.base_manifest) as f:
original_documents = list(yaml.safe_load_all(f.read()))
override = ('manifest:simple-armada:release_prefix=' '\overridden', )
ovr = Override(original_documents, override)
self.assertRaises(json.decoder.JSONDecodeError, ovr.update_manifests)
def test_find_manifest_document_valid(self):
doc_path = ['chart', 'blog-1']
doc_obj = list(yaml.safe_load_all(self.base_documents))
ovr = Override(doc_obj).find_manifest_document(doc_path)
expected_doc = yaml.safe_load_all("""
---
schema: armada/Chart/v1
metadata:
schema: metadata/Document/v1
name: blog-1
data:
chart_name: blog-1
release: blog-1
namespace: default
values: {}
source:
type: git
location: https://github.com/namespace/hello-world-chart
subpath: .
reference: master
dependencies: []
""")
self.assertEqual(ovr, expected_doc)
def __init__(self,
documents,
disable_update_pre=False,
disable_update_post=False,
enable_chart_cleanup=False,
dry_run=False,
set_ovr=None,
force_wait=False,
timeout=0,
tiller_host=None,
tiller_port=None,
tiller_namespace=None,
values=None,
target_manifest=None,
k8s_wait_attempts=1,
k8s_wait_attempt_sleep=1):
'''
Initialize the Armada engine and establish a connection to Tiller.
:param List[dict] documents: Armada documents.
:param bool disable_update_pre: Disable pre-update Tiller operations.
:param bool disable_update_post: Disable post-update Tiller
operations.
:param bool enable_chart_cleanup: Clean up unmanaged charts.
:param bool dry_run: Run charts without installing them.
:param bool force_wait: Force Tiller to wait until all charts are
deployed, rather than using each chart's specified wait policy.
:param int timeout: Specifies overall time in seconds that Tiller
should wait for charts until timing out.
:param str tiller_host: Tiller host IP. Default is None.
:param int tiller_port: Tiller host port. Default is
``CONF.tiller_port``.
:param str tiller_namespace: Tiller host namespace. Default is
``CONF.tiller_namespace``.
:param str target_manifest: The target manifest to run. Useful for
specifying which manifest to run when multiple are available.
:param int k8s_wait_attempts: The number of times to attempt waiting
for pods to become ready.
:param int k8s_wait_attempt_sleep: The time in seconds to sleep
between attempts.
'''
tiller_port = tiller_port or CONF.tiller_port
tiller_namespace = tiller_namespace or CONF.tiller_namespace
self.disable_update_pre = disable_update_pre
self.disable_update_post = disable_update_post
self.enable_chart_cleanup = enable_chart_cleanup
self.dry_run = dry_run
self.force_wait = force_wait
self.timeout = timeout
self.tiller = Tiller(tiller_host=tiller_host,
tiller_port=tiller_port,
tiller_namespace=tiller_namespace)
self.documents = Override(documents, overrides=set_ovr,
values=values).update_manifests()
self.k8s_wait_attempts = k8s_wait_attempts
self.k8s_wait_attempt_sleep = k8s_wait_attempt_sleep
self.manifest = Manifest(
self.documents, target_manifest=target_manifest).get_manifest()
def test_update_chart_group_document_valid(self):
with open(self.base_manifest) as f:
documents = list(yaml.safe_load_all(f.read()))
documents_modified = copy.deepcopy(documents)
documents_modified[1]['data']['sequenced'] = True
# starting out, both doc have different values for data
self.assertNotEqual(documents[1], documents_modified[1])
ovr = Override(documents)
# update with document values with the modified ones
ovr.update_chart_group_document(documents_modified[1])
# after the update, both documents are equal
self.assertEqual(ovr.documents[1]['data']['sequenced'],
documents_modified[1]['data']['sequenced'])
self.assertEqual(ovr.documents[1], documents_modified[1])
def test_update_manifests_invalid_override_format(self):
with open(self.base_manifest) as f:
original_documents = list(yaml.safe_load_all(f.read()))
original_documents[-1]['data']['test'] = {'foo': 'bar'}
override = ('manifest:simple-armada:test=' '{"foo": "bar"}', )
ovr = Override(original_documents, override, [])
self.assertRaises(json.decoder.JSONDecodeError, ovr.update_manifests)
def test_update_armada_manifest_valid(self):
with open(self.base_manifest) as f:
documents = list(yaml.safe_load_all(f.read()))
documents_modified = copy.deepcopy(documents)
documents_modified[2]['data']['release_prefix'] = 'armada-modified'
# starting out, both doc have different values for data
self.assertNotEqual(documents[2], documents_modified[2])
ovr = Override(documents)
# update with document values with the modified ones
ovr.update_armada_manifest(documents_modified[2])
# after the update, both documents are equal
self.assertEqual(ovr.documents[2]['data']['release_prefix'],
documents_modified[2]['data']['release_prefix'])
self.assertEqual(ovr.documents[2], documents_modified[2])
def test_load_yaml_file_invalid(self):
missing_yaml = "{}/templates/non_existing_yaml.yaml". \
format(self.basepath)
with open(self.base_manifest) as f:
documents = list(yaml.safe_load_all(f.read()))
ovr = Override(documents)
self.assertRaises(override_exceptions.InvalidOverrideFileException,
ovr._load_yaml_file, missing_yaml)
def test_update_manifests_invalid_dic_override(self):
missing_yaml = "{}/templates/non_existing_yaml.yaml". \
format(self.basepath)
with open(self.base_manifest):
ovr = Override(missing_yaml)
self.assertRaises(
override_exceptions.InvalidOverrideValueException,
ovr.update_manifests)
def test_update_manifests_invalid_override(self):
with open(self.base_manifest) as f:
original_documents = list(yaml.safe_load_all(f.read()))
override = ('manifest:simple-armada:name=' 'overridden', )
ovr = Override(original_documents, override)
self.assertRaises(override_exceptions.InvalidOverrideValueException,
ovr.update_manifests)
def test_update_dictionary_valid(self):
manifest_change = """
---
schema: armada/Chart/v1
metadata:
schema: metadata/Document/v1
name: blog-1
data:
chart_name: blog-1
release: blog-1
namespace: blog-blog
values: {}
source:
type: dev
"""
expected_document = """
---
schema: armada/Chart/v1
metadata:
schema: metadata/Document/v1
name: blog-1
data:
chart_name: blog-1
release: blog-1
namespace: blog-blog
values: {}
source:
type: dev
location: https://github.com/namespace/hello-world-chart
subpath: .
reference: master
dependencies: []
---
schema: armada/ChartGroup/v1
metadata:
schema: metadata/Document/v1
name: blog-group
data:
description: Deploys Simple Service
sequenced: False
chart_group:
- blog-1
---
schema: armada/Manifest/v1
metadata:
schema: metadata/Document/v1
name: simple-armada
data:
release_prefix: armada
chart_groups:
- blog-group
"""
merging_values = yaml.load_all(manifest_change)
ovr = Override(self.base_documents).update_document(merging_values)
expect_doc = yaml.load_all(expected_document)
self.assertEqual(ovr, expect_doc)
def test_set_list_valid(self):
expected = "{}/templates/override-{}-expected.yaml".format(
self.basepath, '03')
with open(self.base_manifest) as f, open(expected) as e:
documents = list(yaml.safe_load_all(f.read()))
doc_path = ['manifest', 'simple-armada']
override = ('manifest:simple-armada:chart_groups=\
blog-group3,blog-group4', )
ovr = Override(documents, override)
ovr.update_manifests()
ovr_doc = ovr.find_manifest_document(doc_path)
target_docs = list(yaml.load_all(e.read()))
expected_doc = [
x for x in target_docs
if x.get('schema') == 'armada/Manifest/v1'
][0]
self.assertEqual(expected_doc.get('data'), ovr_doc.get('data'))
def __init__(self,
documents,
tiller,
disable_update_pre=False,
disable_update_post=False,
enable_chart_cleanup=False,
dry_run=False,
set_ovr=None,
force_wait=False,
timeout=None,
values=None,
target_manifest=None,
k8s_wait_attempts=1,
k8s_wait_attempt_sleep=1):
'''
Initialize the Armada engine and establish a connection to Tiller.
:param List[dict] documents: Armada documents.
:param tiller: Tiller instance to use.
:param bool disable_update_pre: Disable pre-update Tiller operations.
:param bool disable_update_post: Disable post-update Tiller
operations.
:param bool enable_chart_cleanup: Clean up unmanaged charts.
:param bool dry_run: Run charts without installing them.
:param bool force_wait: Force Tiller to wait until all charts are
deployed, rather than using each chart's specified wait policy.
:param int timeout: Specifies overall time in seconds that Tiller
should wait for charts until timing out.
:param str target_manifest: The target manifest to run. Useful for
specifying which manifest to run when multiple are available.
:param int k8s_wait_attempts: The number of times to attempt waiting
for pods to become ready.
:param int k8s_wait_attempt_sleep: The time in seconds to sleep
between attempts.
'''
self.enable_chart_cleanup = enable_chart_cleanup
self.dry_run = dry_run
self.force_wait = force_wait
self.tiller = tiller
try:
self.documents = Override(documents,
overrides=set_ovr,
values=values).update_manifests()
except (validate_exceptions.InvalidManifestException,
override_exceptions.InvalidOverrideValueException):
raise
self.manifest = Manifest(
self.documents, target_manifest=target_manifest).get_manifest()
self.chart_cache = {}
self.chart_deploy = ChartDeploy(disable_update_pre,
disable_update_post, self.dry_run,
k8s_wait_attempts,
k8s_wait_attempt_sleep, timeout,
self.tiller)
def test_find_manifest_document_valid(self):
expected = "{}/templates/override-{}-expected.yaml".format(
self.basepath, '02')
with open(self.base_manifest) as f, open(expected) as e:
doc_path = ['chart', 'blog-1']
documents = list(yaml.safe_load_all(f.read()))
ovr = Override(documents).find_manifest_document(doc_path)
expected_doc = list(yaml.safe_load_all(e.read()))[0]
self.assertEqual(ovr, expected_doc)
def pre_flight_ops(self):
'''
Perform a series of checks and operations to ensure proper deployment
'''
# Ensure tiller is available and manifest is valid
if not self.tiller.tiller_status():
raise tiller_exceptions.TillerServicesUnavailableException()
if not lint.validate_armada_documents(self.documents):
raise lint_exceptions.InvalidManifestException()
# Override manifest values if --set flag is used
if self.overrides or self.values:
self.documents = Override(self.documents,
overrides=self.overrides,
values=self.values).update_manifests()
# Get config and validate
self.config = self.get_armada_manifest()
if not lint.validate_armada_object(self.config):
raise lint_exceptions.InvalidArmadaObjectException()
# Purge known releases that have failed and are in the current yaml
prefix = self.config.get(const.KEYWORD_ARMADA).get(
const.KEYWORD_PREFIX)
failed_releases = self.get_releases_by_status(const.STATUS_FAILED)
for release in failed_releases:
for group in self.config.get(const.KEYWORD_ARMADA).get(
const.KEYWORD_GROUPS):
for ch in group.get(const.KEYWORD_CHARTS):
ch_release_name = release_prefix(
prefix,
ch.get('chart').get('chart_name'))
if release[0] == ch_release_name:
LOG.info(
'Purging failed release %s '
'before deployment', release[0])
self.tiller.uninstall_release(release[0])
# Clone the chart sources
#
# We only support a git source type right now, which can also
# handle git:// local paths as well
repos = {}
for group in self.config.get(const.KEYWORD_ARMADA).get(
const.KEYWORD_GROUPS):
for ch in group.get(const.KEYWORD_CHARTS):
self.tag_cloned_repo(ch, repos)
for dep in ch.get('chart').get('dependencies'):
self.tag_cloned_repo(dep, repos)
def test_update_manifests_with_values_and_overrides_valid(self):
values_yaml = "{}/templates/override-{}-expected.yaml".format(
self.basepath, '01')
comparison_yaml = "{}/templates/override-{}-expected.yaml".format(
self.basepath, '03')
with open(self.base_manifest) as f, open(values_yaml) as g:
original_documents = list(yaml.safe_load_all(f.read()))
documents_copy = copy.deepcopy(original_documents)
values_documents = list(yaml.safe_load_all(g.read()))
override = ('manifest:simple-armada:release_prefix=' 'overridden', )
# Case 1: Checking if primitive gets updated.
ovr = Override(original_documents, override, [values_yaml])
ovr.update_manifests()
# updating values changed the original document
self.assertNotEqual(original_documents, documents_copy)
# since overrides done, these documents aren't same anymore
self.assertNotEqual(original_documents, values_documents)
target_doc = [
x for x in ovr.documents
if x.get('metadata').get('name') == 'simple-armada'
][0]
self.assertEqual('overridden', target_doc['data']['release_prefix'])
override = ('manifest:simple-armada:chart_groups='
'blog-group3,blog-group4', )
# Case 2: Checking if list gets updated.
ovr = Override(original_documents, override, [values_yaml])
ovr.update_manifests()
# updating values changed the original document
self.assertNotEqual(original_documents, documents_copy)
# since overrides done, these documents aren't same anymore
self.assertNotEqual(original_documents, values_documents)
with open(comparison_yaml) as c:
comparison_documents = list(yaml.safe_load_all(c.read()))
# verifying that the override is correct
self.assertEqual(original_documents[2]['data']['chart_groups'],
comparison_documents[0]['data']['chart_groups'])
def test_update_chart_document_valid(self):
with open(self.base_manifest) as f:
documents = list(yaml.safe_load_all(f.read()))
documents_modified = copy.deepcopy(documents)
# Case 1: Checking if primitives get updated
documents_modified[0]['data']['chart_name'] = 'modified'
# starting out, both doc have different values for data
self.assertNotEqual(documents[0], documents_modified[0])
ovr = Override(documents)
# update with document values with the modified ones
ovr.update_chart_document(documents_modified[0])
# after the update, both documents are equal
self.assertEqual(ovr.documents[0]['data']['chart_name'],
documents_modified[0]['data']['chart_name'])
self.assertEqual(ovr.documents[0], documents_modified[0])
# Case 2: Checking if dictionaries get updated
documents_modified[0]['data']['values'] = {'foo': 'bar'}
ovr.update_chart_document(documents_modified[0])
# after the update, both documents are equal
self.assertEqual(ovr.documents[0]['data']['values'],
documents_modified[0]['data']['values'])
self.assertEqual(ovr.documents[0], documents_modified[0])
# Case 3: Checking if lists get updated
documents_modified[0]['data']['dependencies'] = ['foo', 'bar']
ovr.update_chart_document(documents_modified[0])
# after the update, both documents are equal
self.assertEqual(['foo', 'bar'],
ovr.documents[0]['data']['dependencies'])
self.assertEqual(documents_modified[0]['data']['dependencies'],
ovr.documents[0]['data']['dependencies'])
self.assertEqual(ovr.documents[0], documents_modified[0])
def test_find_document_type_invalid(self):
with open(self.base_manifest) as f:
documents = list(yaml.safe_load_all(f.read()))
ovr = Override(documents)
self.assertRaises(ValueError, ovr.find_document_type,
'non_existing_document')
def test_find_document_type_invalid(self):
with self.assertRaises(Exception):
with open(self.base_manifest) as f:
doc_obj = list(yaml.safe_load_all(f.read()))
ovr = Override(doc_obj)
ovr.find_document_type('charts')
def test_load_yaml_file(self):
with open(self.base_manifest) as f:
documents = list(yaml.safe_load_all(f.read()))
ovr = Override(documents)
value = ovr._load_yaml_file(self.base_manifest)
self.assertIsInstance(value, list)
