def test_update_manifests_no_overrides_and_values_valid(self): with open(self.base_manifest) as f: documents = list(yaml.safe_load_all(f.read())) ovr = Override(documents) ovr.update_manifests() # no updates since no overrides and values provided self.assertEqual(documents, ovr.documents)
def test_convert_array_to_dict_invalid(self): data_path = ['a', 'b', 'c'] new_value = "" ovr = Override(self.base_manifest).array_to_dict(data_path, new_value) self.assertIsNone(ovr) ovr = Override(self.base_manifest).array_to_dict([], new_value) self.assertIsNone(ovr)
def test_find_document_type_valid(self): ovr = Override(self.base_documents) test_group = ovr.find_document_type('chart_group') self.assertEqual(test_group, const.DOCUMENT_GROUP) test_chart = ovr.find_document_type('chart') self.assertEqual(test_chart, const.DOCUMENT_CHART) test_manifest = ovr.find_document_type('manifest') self.assertEqual(test_manifest, const.DOCUMENT_MANIFEST)
def on_post(self, req, resp): # Load data from request and get options if req.content_type == 'application/x-yaml': data = list(self.req_yaml(req)) if type(data[0]) is list: documents = list(data[0]) else: documents = data elif req.content_type == 'application/json': self.logger.debug("Applying manifest based on reference.") req_body = self.req_json(req) doc_ref = req_body.get('hrefs', None) if not doc_ref: self.logger.info("Request did not contain 'hrefs'.") resp.status = falcon.HTTP_400 return data = ReferenceResolver.resolve_reference(doc_ref) documents = list() for d in data: documents.extend(list(yaml.safe_load_all(d.decode()))) if req_body.get('overrides', None): overrides = Override(documents, overrides=req_body.get('overrides')) documents = overrides.update_manifests() else: self.error(req.context, "Unknown content-type %s" % req.content_type) # TODO(fmontei): Use falcon.<Relevant API Exception Class> instead. return self.return_error( resp, falcon.HTTP_415, message="Request must be in application/x-yaml" "or application/json") try: with self.get_tiller(req, resp) as tiller: msg = self.handle(req, documents, tiller) resp.body = json.dumps({ 'message': msg, }) resp.content_type = 'application/json' resp.status = falcon.HTTP_200 except exceptions.ManifestException as e: self.return_error(resp, falcon.HTTP_400, message=str(e)) except LockException as e: self.return_error(resp, falcon.HTTP_409, message=str(e)) except Exception as e: self.logger.exception('Caught unexpected exception') err_message = 'Failed to apply manifest: {}'.format(e) self.error(req.context, err_message) self.return_error(resp, falcon.HTTP_500, message=err_message)
def test_find_document_type_valid(self): with open(self.base_manifest) as f: documents = list(yaml.safe_load_all(f.read())) ovr = Override(documents) test_group = ovr.find_document_type('chart_group') self.assertEqual(test_group, const.DOCUMENT_GROUP) test_chart = ovr.find_document_type('chart') self.assertEqual(test_chart, const.DOCUMENT_CHART) test_manifest = ovr.find_document_type('manifest') self.assertEqual(test_manifest, const.DOCUMENT_MANIFEST)
def test_set_list_valid(self): expected = "{}/templates/override-{}-expected.yaml".format( self.basepath, '03') with open(self.base_manifest) as f, open(expected) as e: doc_obj = list(yaml.safe_load_all(f.read())) doc_path = ['manifest', 'simple-armada'] override = ('manifest:simple-armada:chart_groups=\ blog-group3,blog-group4',) ovr = Override(doc_obj, override) ovr.update_manifests() ovr_doc = ovr.find_manifest_document(doc_path) expect_doc = list(yaml.load_all(e.read()))[0] self.assertEqual(expect_doc, ovr_doc)
def test_update_manifests_with_values_valid(self): original = "{}/templates/override-{}.yaml".format(self.basepath, '01') values_yaml = "{}/templates/override-{}-expected.yaml".format( self.basepath, '01') with open(original) as f, open(values_yaml) as g: original_documents = list(yaml.safe_load_all(f.read())) documents_copy = copy.deepcopy(original_documents) values_documents = list(yaml.safe_load_all(g.read())) ovr = Override(original_documents, None, [values_yaml]) ovr.update_manifests() # updating values changed the original document self.assertNotEqual(original_documents, documents_copy) # verifying that these documents have the same value now self.assertEqual(original_documents, values_documents)
def test_update_chart_group_document_keys_not_removed_with_override(self): with open(self.base_manifest) as f: documents = list(yaml.safe_load_all(f.read())) documents_modified = copy.deepcopy(documents) del documents_modified[1]['data']['sequenced'] # verify both doc have different values for data self.assertNotEqual(documents[1], documents_modified[1]) ovr = Override(documents) # update with document values with the modified ones ovr.update_chart_group_document(documents_modified[1]) self.assertIn('sequenced', ovr.documents[1]['data']) self.assertNotEqual(ovr.documents[1], documents_modified[1])
def test_update_armada_manifest_keys_not_removed_with_override(self): with open(self.base_manifest) as f: documents = list(yaml.safe_load_all(f.read())) documents_modified = copy.deepcopy(documents) del documents_modified[2]['data']['release_prefix'] # verify both doc have different values for data self.assertNotEqual(documents[2], documents_modified[2]) ovr = Override(documents) # update with document values from base_manifest ovr.update_armada_manifest(documents_modified[2]) self.assertIn('release_prefix', ovr.documents[2]['data']) self.assertNotEqual(ovr.documents[2], documents_modified[2])
def test_update_dictionary_valid(self): expected = "{}/templates/override-{}-expected.yaml".format( self.basepath, '01') merge = "{}/templates/override-{}.yaml".format(self.basepath, '01') with open(self.base_manifest) as f, open(expected) as e, open( merge) as m: merging_values = list(yaml.safe_load_all(m.read())) documents = list(yaml.safe_load_all(f.read())) doc_path = ['chart', 'blog-1'] ovr = Override(documents) ovr.update_document(merging_values) ovr_doc = ovr.find_manifest_document(doc_path) expect_doc = list(yaml.load_all(e.read()))[0] self.assertEqual(ovr_doc, expect_doc)
def test_convert_array_to_dict_valid(self): data_path = ['a', 'b', 'c'] new_value = "dev" expected_dict = {'a': {'b': {'c': 'dev'}}} ovr = Override(self.base_manifest).array_to_dict(data_path, new_value) self.assertEqual(ovr, expected_dict)
def test_update_manifests_invalid(self): with open(self.base_manifest) as f: original_documents = list(yaml.safe_load_all(f.read())) override = ('manifest:simple-armada:release_prefix=' '\overridden', ) ovr = Override(original_documents, override) self.assertRaises(json.decoder.JSONDecodeError, ovr.update_manifests)
def test_find_manifest_document_valid(self): doc_path = ['chart', 'blog-1'] doc_obj = list(yaml.safe_load_all(self.base_documents)) ovr = Override(doc_obj).find_manifest_document(doc_path) expected_doc = yaml.safe_load_all(""" --- schema: armada/Chart/v1 metadata: schema: metadata/Document/v1 name: blog-1 data: chart_name: blog-1 release: blog-1 namespace: default values: {} source: type: git location: https://github.com/namespace/hello-world-chart subpath: . reference: master dependencies: [] """) self.assertEqual(ovr, expected_doc)
def __init__(self, documents, disable_update_pre=False, disable_update_post=False, enable_chart_cleanup=False, dry_run=False, set_ovr=None, force_wait=False, timeout=0, tiller_host=None, tiller_port=None, tiller_namespace=None, values=None, target_manifest=None, k8s_wait_attempts=1, k8s_wait_attempt_sleep=1): ''' Initialize the Armada engine and establish a connection to Tiller. :param List[dict] documents: Armada documents. :param bool disable_update_pre: Disable pre-update Tiller operations. :param bool disable_update_post: Disable post-update Tiller operations. :param bool enable_chart_cleanup: Clean up unmanaged charts. :param bool dry_run: Run charts without installing them. :param bool force_wait: Force Tiller to wait until all charts are deployed, rather than using each chart's specified wait policy. :param int timeout: Specifies overall time in seconds that Tiller should wait for charts until timing out. :param str tiller_host: Tiller host IP. Default is None. :param int tiller_port: Tiller host port. Default is ``CONF.tiller_port``. :param str tiller_namespace: Tiller host namespace. Default is ``CONF.tiller_namespace``. :param str target_manifest: The target manifest to run. Useful for specifying which manifest to run when multiple are available. :param int k8s_wait_attempts: The number of times to attempt waiting for pods to become ready. :param int k8s_wait_attempt_sleep: The time in seconds to sleep between attempts. ''' tiller_port = tiller_port or CONF.tiller_port tiller_namespace = tiller_namespace or CONF.tiller_namespace self.disable_update_pre = disable_update_pre self.disable_update_post = disable_update_post self.enable_chart_cleanup = enable_chart_cleanup self.dry_run = dry_run self.force_wait = force_wait self.timeout = timeout self.tiller = Tiller(tiller_host=tiller_host, tiller_port=tiller_port, tiller_namespace=tiller_namespace) self.documents = Override(documents, overrides=set_ovr, values=values).update_manifests() self.k8s_wait_attempts = k8s_wait_attempts self.k8s_wait_attempt_sleep = k8s_wait_attempt_sleep self.manifest = Manifest( self.documents, target_manifest=target_manifest).get_manifest()
def test_update_chart_group_document_valid(self): with open(self.base_manifest) as f: documents = list(yaml.safe_load_all(f.read())) documents_modified = copy.deepcopy(documents) documents_modified[1]['data']['sequenced'] = True # starting out, both doc have different values for data self.assertNotEqual(documents[1], documents_modified[1]) ovr = Override(documents) # update with document values with the modified ones ovr.update_chart_group_document(documents_modified[1]) # after the update, both documents are equal self.assertEqual(ovr.documents[1]['data']['sequenced'], documents_modified[1]['data']['sequenced']) self.assertEqual(ovr.documents[1], documents_modified[1])
def test_update_manifests_invalid_override_format(self): with open(self.base_manifest) as f: original_documents = list(yaml.safe_load_all(f.read())) original_documents[-1]['data']['test'] = {'foo': 'bar'} override = ('manifest:simple-armada:test=' '{"foo": "bar"}', ) ovr = Override(original_documents, override, []) self.assertRaises(json.decoder.JSONDecodeError, ovr.update_manifests)
def test_update_armada_manifest_valid(self): with open(self.base_manifest) as f: documents = list(yaml.safe_load_all(f.read())) documents_modified = copy.deepcopy(documents) documents_modified[2]['data']['release_prefix'] = 'armada-modified' # starting out, both doc have different values for data self.assertNotEqual(documents[2], documents_modified[2]) ovr = Override(documents) # update with document values with the modified ones ovr.update_armada_manifest(documents_modified[2]) # after the update, both documents are equal self.assertEqual(ovr.documents[2]['data']['release_prefix'], documents_modified[2]['data']['release_prefix']) self.assertEqual(ovr.documents[2], documents_modified[2])
def test_load_yaml_file_invalid(self): missing_yaml = "{}/templates/non_existing_yaml.yaml". \ format(self.basepath) with open(self.base_manifest) as f: documents = list(yaml.safe_load_all(f.read())) ovr = Override(documents) self.assertRaises(override_exceptions.InvalidOverrideFileException, ovr._load_yaml_file, missing_yaml)
def test_update_manifests_invalid_dic_override(self): missing_yaml = "{}/templates/non_existing_yaml.yaml". \ format(self.basepath) with open(self.base_manifest): ovr = Override(missing_yaml) self.assertRaises( override_exceptions.InvalidOverrideValueException, ovr.update_manifests)
def test_update_manifests_invalid_override(self): with open(self.base_manifest) as f: original_documents = list(yaml.safe_load_all(f.read())) override = ('manifest:simple-armada:name=' 'overridden', ) ovr = Override(original_documents, override) self.assertRaises(override_exceptions.InvalidOverrideValueException, ovr.update_manifests)
def test_update_dictionary_valid(self): manifest_change = """ --- schema: armada/Chart/v1 metadata: schema: metadata/Document/v1 name: blog-1 data: chart_name: blog-1 release: blog-1 namespace: blog-blog values: {} source: type: dev """ expected_document = """ --- schema: armada/Chart/v1 metadata: schema: metadata/Document/v1 name: blog-1 data: chart_name: blog-1 release: blog-1 namespace: blog-blog values: {} source: type: dev location: https://github.com/namespace/hello-world-chart subpath: . reference: master dependencies: [] --- schema: armada/ChartGroup/v1 metadata: schema: metadata/Document/v1 name: blog-group data: description: Deploys Simple Service sequenced: False chart_group: - blog-1 --- schema: armada/Manifest/v1 metadata: schema: metadata/Document/v1 name: simple-armada data: release_prefix: armada chart_groups: - blog-group """ merging_values = yaml.load_all(manifest_change) ovr = Override(self.base_documents).update_document(merging_values) expect_doc = yaml.load_all(expected_document) self.assertEqual(ovr, expect_doc)
def test_set_list_valid(self): expected = "{}/templates/override-{}-expected.yaml".format( self.basepath, '03') with open(self.base_manifest) as f, open(expected) as e: documents = list(yaml.safe_load_all(f.read())) doc_path = ['manifest', 'simple-armada'] override = ('manifest:simple-armada:chart_groups=\ blog-group3,blog-group4', ) ovr = Override(documents, override) ovr.update_manifests() ovr_doc = ovr.find_manifest_document(doc_path) target_docs = list(yaml.load_all(e.read())) expected_doc = [ x for x in target_docs if x.get('schema') == 'armada/Manifest/v1' ][0] self.assertEqual(expected_doc.get('data'), ovr_doc.get('data'))
def __init__(self, documents, tiller, disable_update_pre=False, disable_update_post=False, enable_chart_cleanup=False, dry_run=False, set_ovr=None, force_wait=False, timeout=None, values=None, target_manifest=None, k8s_wait_attempts=1, k8s_wait_attempt_sleep=1): ''' Initialize the Armada engine and establish a connection to Tiller. :param List[dict] documents: Armada documents. :param tiller: Tiller instance to use. :param bool disable_update_pre: Disable pre-update Tiller operations. :param bool disable_update_post: Disable post-update Tiller operations. :param bool enable_chart_cleanup: Clean up unmanaged charts. :param bool dry_run: Run charts without installing them. :param bool force_wait: Force Tiller to wait until all charts are deployed, rather than using each chart's specified wait policy. :param int timeout: Specifies overall time in seconds that Tiller should wait for charts until timing out. :param str target_manifest: The target manifest to run. Useful for specifying which manifest to run when multiple are available. :param int k8s_wait_attempts: The number of times to attempt waiting for pods to become ready. :param int k8s_wait_attempt_sleep: The time in seconds to sleep between attempts. ''' self.enable_chart_cleanup = enable_chart_cleanup self.dry_run = dry_run self.force_wait = force_wait self.tiller = tiller try: self.documents = Override(documents, overrides=set_ovr, values=values).update_manifests() except (validate_exceptions.InvalidManifestException, override_exceptions.InvalidOverrideValueException): raise self.manifest = Manifest( self.documents, target_manifest=target_manifest).get_manifest() self.chart_cache = {} self.chart_deploy = ChartDeploy(disable_update_pre, disable_update_post, self.dry_run, k8s_wait_attempts, k8s_wait_attempt_sleep, timeout, self.tiller)
def test_find_manifest_document_valid(self): expected = "{}/templates/override-{}-expected.yaml".format( self.basepath, '02') with open(self.base_manifest) as f, open(expected) as e: doc_path = ['chart', 'blog-1'] documents = list(yaml.safe_load_all(f.read())) ovr = Override(documents).find_manifest_document(doc_path) expected_doc = list(yaml.safe_load_all(e.read()))[0] self.assertEqual(ovr, expected_doc)
def pre_flight_ops(self): ''' Perform a series of checks and operations to ensure proper deployment ''' # Ensure tiller is available and manifest is valid if not self.tiller.tiller_status(): raise tiller_exceptions.TillerServicesUnavailableException() if not lint.validate_armada_documents(self.documents): raise lint_exceptions.InvalidManifestException() # Override manifest values if --set flag is used if self.overrides or self.values: self.documents = Override(self.documents, overrides=self.overrides, values=self.values).update_manifests() # Get config and validate self.config = self.get_armada_manifest() if not lint.validate_armada_object(self.config): raise lint_exceptions.InvalidArmadaObjectException() # Purge known releases that have failed and are in the current yaml prefix = self.config.get(const.KEYWORD_ARMADA).get( const.KEYWORD_PREFIX) failed_releases = self.get_releases_by_status(const.STATUS_FAILED) for release in failed_releases: for group in self.config.get(const.KEYWORD_ARMADA).get( const.KEYWORD_GROUPS): for ch in group.get(const.KEYWORD_CHARTS): ch_release_name = release_prefix( prefix, ch.get('chart').get('chart_name')) if release[0] == ch_release_name: LOG.info( 'Purging failed release %s ' 'before deployment', release[0]) self.tiller.uninstall_release(release[0]) # Clone the chart sources # # We only support a git source type right now, which can also # handle git:// local paths as well repos = {} for group in self.config.get(const.KEYWORD_ARMADA).get( const.KEYWORD_GROUPS): for ch in group.get(const.KEYWORD_CHARTS): self.tag_cloned_repo(ch, repos) for dep in ch.get('chart').get('dependencies'): self.tag_cloned_repo(dep, repos)
def test_update_manifests_with_values_and_overrides_valid(self): values_yaml = "{}/templates/override-{}-expected.yaml".format( self.basepath, '01') comparison_yaml = "{}/templates/override-{}-expected.yaml".format( self.basepath, '03') with open(self.base_manifest) as f, open(values_yaml) as g: original_documents = list(yaml.safe_load_all(f.read())) documents_copy = copy.deepcopy(original_documents) values_documents = list(yaml.safe_load_all(g.read())) override = ('manifest:simple-armada:release_prefix=' 'overridden', ) # Case 1: Checking if primitive gets updated. ovr = Override(original_documents, override, [values_yaml]) ovr.update_manifests() # updating values changed the original document self.assertNotEqual(original_documents, documents_copy) # since overrides done, these documents aren't same anymore self.assertNotEqual(original_documents, values_documents) target_doc = [ x for x in ovr.documents if x.get('metadata').get('name') == 'simple-armada' ][0] self.assertEqual('overridden', target_doc['data']['release_prefix']) override = ('manifest:simple-armada:chart_groups=' 'blog-group3,blog-group4', ) # Case 2: Checking if list gets updated. ovr = Override(original_documents, override, [values_yaml]) ovr.update_manifests() # updating values changed the original document self.assertNotEqual(original_documents, documents_copy) # since overrides done, these documents aren't same anymore self.assertNotEqual(original_documents, values_documents) with open(comparison_yaml) as c: comparison_documents = list(yaml.safe_load_all(c.read())) # verifying that the override is correct self.assertEqual(original_documents[2]['data']['chart_groups'], comparison_documents[0]['data']['chart_groups'])
def test_update_chart_document_valid(self): with open(self.base_manifest) as f: documents = list(yaml.safe_load_all(f.read())) documents_modified = copy.deepcopy(documents) # Case 1: Checking if primitives get updated documents_modified[0]['data']['chart_name'] = 'modified' # starting out, both doc have different values for data self.assertNotEqual(documents[0], documents_modified[0]) ovr = Override(documents) # update with document values with the modified ones ovr.update_chart_document(documents_modified[0]) # after the update, both documents are equal self.assertEqual(ovr.documents[0]['data']['chart_name'], documents_modified[0]['data']['chart_name']) self.assertEqual(ovr.documents[0], documents_modified[0]) # Case 2: Checking if dictionaries get updated documents_modified[0]['data']['values'] = {'foo': 'bar'} ovr.update_chart_document(documents_modified[0]) # after the update, both documents are equal self.assertEqual(ovr.documents[0]['data']['values'], documents_modified[0]['data']['values']) self.assertEqual(ovr.documents[0], documents_modified[0]) # Case 3: Checking if lists get updated documents_modified[0]['data']['dependencies'] = ['foo', 'bar'] ovr.update_chart_document(documents_modified[0]) # after the update, both documents are equal self.assertEqual(['foo', 'bar'], ovr.documents[0]['data']['dependencies']) self.assertEqual(documents_modified[0]['data']['dependencies'], ovr.documents[0]['data']['dependencies']) self.assertEqual(ovr.documents[0], documents_modified[0])
def test_find_document_type_invalid(self): with open(self.base_manifest) as f: documents = list(yaml.safe_load_all(f.read())) ovr = Override(documents) self.assertRaises(ValueError, ovr.find_document_type, 'non_existing_document')
def test_find_document_type_invalid(self): with self.assertRaises(Exception): with open(self.base_manifest) as f: doc_obj = list(yaml.safe_load_all(f.read())) ovr = Override(doc_obj) ovr.find_document_type('charts')
def test_load_yaml_file(self): with open(self.base_manifest) as f: documents = list(yaml.safe_load_all(f.read())) ovr = Override(documents) value = ovr._load_yaml_file(self.base_manifest) self.assertIsInstance(value, list)