def invoke(self): tiller = Tiller(tiller_host=self.tiller_host, tiller_port=self.tiller_port) known_release_names = [release[0] for release in tiller.list_charts()] if self.releases: target_releases = [ r.strip() for r in self.releases.split(',') if r.strip() in known_release_names ] if not target_releases: self.logger.info("There's no release to delete.") return if not self.ctx.obj.get('api', False): for r in target_releases: self.logger.info("Deleting release %s", r) tiller.uninstall_release(r, purge=self.purge) else: raise NotImplementedError() if self.manifest: target_releases = [] with open(self.manifest) as f: documents = yaml.safe_load_all(f.read()) try: armada_obj = Manifest(documents).get_manifest() prefix = armada_obj.get(const.KEYWORD_ARMADA).get( const.KEYWORD_PREFIX) for group in armada_obj.get(const.KEYWORD_ARMADA).get( const.KEYWORD_GROUPS): for ch in group.get(const.KEYWORD_CHARTS): release_name = release_prefix( prefix, ch.get('chart').get('chart_name')) if release_name in known_release_names: target_releases.append(release_name) except yaml.YAMLError as e: mark = e.problem_mark self.logger.info( "While parsing the manifest file, %s. " "Error position: (%s:%s)", e.problem, mark.line + 1, mark.column + 1) if not target_releases: self.logger.info("There's no release to delete.") return if not self.ctx.obj.get('api', False): for r in target_releases: self.logger.info("Deleting release %s", r) tiller.uninstall_release(r, purge=self.purge) else: raise NotImplementedError()
def on_post(self, req, resp): try: opts = req.params tiller = Tiller(tiller_host=opts.get('tiller_host', None), tiller_port=opts.get('tiller_port', None)) documents = self.req_yaml(req) armada_obj = Manifest(documents).get_manifest() prefix = armada_obj.get(const.KEYWORD_ARMADA).get( const.KEYWORD_PREFIX) known_releases = [release[0] for release in tiller.list_charts()] message = { 'tests': { 'passed': [], 'skipped': [], 'failed': [] } } for group in armada_obj.get(const.KEYWORD_ARMADA).get( const.KEYWORD_GROUPS): for ch in group.get(const.KEYWORD_CHARTS): release_name = release_prefix( prefix, ch.get('chart').get('chart_name')) if release_name in known_releases: self.logger.info('RUNNING: %s tests', release_name) resp = tiller.testing_release(release_name) if not resp: continue test_status = getattr( resp.info.status, 'last_test_suite_run', 'FAILED') if test_status.results[0].status: self.logger.info("PASSED: %s", release_name) message['test']['passed'].append(release_name) else: self.logger.info("FAILED: %s", release_name) message['test']['failed'].append(release_name) else: self.logger.info( 'Release %s not found - SKIPPING', release_name) message['test']['skipped'].append(release_name) resp.status = falcon.HTTP_200 resp.body = json.dumps(message) resp.content_type = 'application/json' except Exception as e: err_message = 'Failed to test manifest: {}'.format(e) self.error(req.context, err_message) self.return_error( resp, falcon.HTTP_500, message=err_message)
def pre_flight_ops(self): """Perform a series of checks and operations to ensure proper deployment. """ LOG.info("Performing pre-flight operations.") # Ensure Tiller is available and manifest is valid if not self.tiller.tiller_status(): raise tiller_exceptions.TillerServicesUnavailableException() valid, details = validate.validate_armada_documents(self.documents) if details: for msg in details: if msg.get('error', False): LOG.error(msg.get('message', 'Unknown validation error.')) else: LOG.debug(msg.get('message', 'Validation succeeded.')) if not valid: raise validate_exceptions.InvalidManifestException( error_messages=details) result, msg_list = validate.validate_armada_manifests(self.documents) if not result: raise validate_exceptions.InvalidArmadaObjectException( details=','.join([m.get('message') for m in msg_list])) # Purge known releases that have failed and are in the current yaml manifest_data = self.manifest.get(KEYWORD_ARMADA, {}) prefix = manifest_data.get(KEYWORD_PREFIX, '') failed_releases = self.get_releases_by_status(STATUS_FAILED) for release in failed_releases: for group in manifest_data.get(KEYWORD_GROUPS, []): for ch in group.get(KEYWORD_CHARTS, []): ch_release_name = release_prefix( prefix, ch.get('chart', {}).get('chart_name')) if release[0] == ch_release_name: LOG.info( 'Purging failed release %s ' 'before deployment', release[0]) self.tiller.uninstall_release(release[0]) # Clone the chart sources # # We only support a git source type right now, which can also # handle git:// local paths as well repos = {} for group in manifest_data.get(KEYWORD_GROUPS, []): for ch in group.get(KEYWORD_CHARTS, []): self.tag_cloned_repo(ch, repos) for dep in ch.get('chart', {}).get('dependencies', []): self.tag_cloned_repo(dep, repos)
def pre_flight_ops(self): ''' Perform a series of checks and operations to ensure proper deployment ''' # Ensure tiller is available and manifest is valid if not self.tiller.tiller_status(): raise tiller_exceptions.TillerServicesUnavailableException() if not lint.validate_armada_documents(self.documents): raise lint_exceptions.InvalidManifestException() # Override manifest values if --set flag is used if self.overrides or self.values: self.documents = Override(self.documents, overrides=self.overrides, values=self.values).update_manifests() # Get config and validate self.config = self.get_armada_manifest() if not lint.validate_armada_object(self.config): raise lint_exceptions.InvalidArmadaObjectException() # Purge known releases that have failed and are in the current yaml prefix = self.config.get(const.KEYWORD_ARMADA).get( const.KEYWORD_PREFIX) failed_releases = self.get_releases_by_status(const.STATUS_FAILED) for release in failed_releases: for group in self.config.get(const.KEYWORD_ARMADA).get( const.KEYWORD_GROUPS): for ch in group.get(const.KEYWORD_CHARTS): ch_release_name = release_prefix( prefix, ch.get('chart').get('chart_name')) if release[0] == ch_release_name: LOG.info( 'Purging failed release %s ' 'before deployment', release[0]) self.tiller.uninstall_release(release[0]) # Clone the chart sources # # We only support a git source type right now, which can also # handle git:// local paths as well repos = {} for group in self.config.get(const.KEYWORD_ARMADA).get( const.KEYWORD_GROUPS): for ch in group.get(const.KEYWORD_CHARTS): self.tag_cloned_repo(ch, repos) for dep in ch.get('chart').get('dependencies'): self.tag_cloned_repo(dep, repos)
def testService(args): tiller = Tiller(tiller_host=args.tiller_host, tiller_port=args.tiller_port) known_release_names = [release[0] for release in tiller.list_charts()] if args.release: LOG.info("RUNNING: %s tests", args.release) resp = tiller.testing_release(args.release) if not resp: LOG.info("FAILED: %s", args.release) return test_status = getattr(resp.info.status, 'last_test_suite_run', 'FAILED') if test_status.results[0].status: LOG.info("PASSED: %s", args.release) else: LOG.info("FAILED: %s", args.release) if args.file: documents = yaml.safe_load_all(open(args.file).read()) armada_obj = Manifest(documents).get_manifest() prefix = armada_obj.get(const.KEYWORD_ARMADA).get(const.KEYWORD_PREFIX) for group in armada_obj.get(const.KEYWORD_ARMADA).get( const.KEYWORD_GROUPS): for ch in group.get(const.KEYWORD_CHARTS): release_name = release_prefix( prefix, ch.get('chart').get('chart_name')) if release_name in known_release_names: LOG.info('RUNNING: %s tests', release_name) resp = tiller.testing_release(release_name) if not resp: continue test_status = getattr(resp.info.status, 'last_test_suite_run', 'FAILED') if test_status.results[0].status: LOG.info("PASSED: %s", release_name) else: LOG.info("FAILED: %s", release_name) else: LOG.info('Release %s not found - SKIPPING', release_name)
def chart_cleanup(self, prefix, charts): ''' :params charts - list of yaml charts :params known_release - list of releases in tiller :result - will remove any chart that is not present in yaml ''' valid_charts = [] for gchart in charts: for chart in gchart.get('chart_group'): valid_charts.append(release_prefix( prefix, chart.get('chart').get('name'))) actual_charts = [x.name for x in self.list_releases()] chart_diff = list(set(actual_charts) - set(valid_charts)) for chart in chart_diff: if chart.startswith(prefix): LOG.debug("Release: %s will be removed", chart) self.uninstall_release(chart)
def invoke(self): tiller = Tiller( tiller_host=self.tiller_host, tiller_port=self.tiller_port) known_release_names = [release[0] for release in tiller.list_charts()] if self.release: if not self.ctx.obj.get('api', False): self.logger.info("RUNNING: %s tests", self.release) resp = tiller.testing_release(self.release) if not resp: self.logger.info("FAILED: %s", self.release) return test_status = getattr(resp.info.status, 'last_test_suite_run', 'FAILED') if test_status.results[0].status: self.logger.info("PASSED: %s", self.release) else: self.logger.info("FAILED: %s", self.release) else: client = self.ctx.obj.get('CLIENT') resp = client.get_test_release(release=self.release) self.logger.info(resp.get('result')) self.logger.info(resp.get('message')) if self.file: if not self.ctx.obj.get('api', False): documents = yaml.safe_load_all(open(self.file).read()) armada_obj = Manifest(documents).get_manifest() prefix = armada_obj.get(const.KEYWORD_ARMADA).get( const.KEYWORD_PREFIX) for group in armada_obj.get(const.KEYWORD_ARMADA).get( const.KEYWORD_GROUPS): for ch in group.get(const.KEYWORD_CHARTS): release_name = release_prefix( prefix, ch.get('chart').get('chart_name')) if release_name in known_release_names: self.logger.info('RUNNING: %s tests', release_name) resp = tiller.testing_release(release_name) if not resp: continue test_status = getattr( resp.info.status, 'last_test_suite_run', 'FAILED') if test_status.results[0].status: self.logger.info("PASSED: %s", release_name) else: self.logger.info("FAILED: %s", release_name) else: self.logger.info( 'Release %s not found - SKIPPING', release_name) else: client = self.ctx.obj.get('CLIENT') with open(self.filename, 'r') as f: resp = client.get_test_manifest(manifest=f.read()) for test in resp.get('tests'): self.logger.info('Test State: %s', test) for item in test.get('tests').get(test): self.logger.info(item) self.logger.info(resp)
def test_release_prefix_int_int(self): expected = '4-4' prefix, chart = (4, 4) assert rel.release_prefix(prefix, chart) == expected
def test_release_prefix_int_string(self): expected = 'armada-4' prefix, chart = ('armada', 4) assert rel.release_prefix(prefix, chart) == expected
def test_release_prefix_pass(self): expected = 'armada-test' prefix, chart = ('armada', 'test') assert rel.release_prefix(prefix, chart) == expected
def sync(self): ''' Syncronize Helm with the Armada Config(s) ''' msg = {'install': [], 'upgrade': [], 'diff': []} # TODO: (gardlt) we need to break up this func into # a more cleaner format LOG.info("Performing Pre-Flight Operations") self.pre_flight_ops() # extract known charts on tiller right now known_releases = self.tiller.list_charts() prefix = self.config.get(const.KEYWORD_ARMADA).get( const.KEYWORD_PREFIX) if known_releases is None: raise armada_exceptions.KnownReleasesException() for release in known_releases: LOG.debug("Release %s, Version %s found on tiller", release[0], release[1]) for entry in self.config[const.KEYWORD_ARMADA][const.KEYWORD_GROUPS]: chart_wait = self.wait desc = entry.get('description', 'A Chart Group') chart_group = entry.get(const.KEYWORD_CHARTS, []) test_charts = entry.get('test_charts', False) if entry.get('sequenced', False) or test_charts: chart_wait = True LOG.info('Deploying: %s', desc) for gchart in chart_group: chart = dotify(gchart['chart']) values = gchart.get('chart').get('values', {}) wait_values = gchart.get('chart').get('wait', {}) test_chart = gchart.get('chart').get('test', False) pre_actions = {} post_actions = {} if chart.release is None: continue if test_chart: chart_wait = True # retrieve appropriate timeout value if 'wait' is specified chart_timeout = self.timeout if chart_wait: if chart_timeout == DEFAULT_TIMEOUT: chart_timeout = getattr(chart, 'timeout', chart_timeout) chartbuilder = ChartBuilder(chart) protoc_chart = chartbuilder.get_helm_chart() # determine install or upgrade by examining known releases LOG.debug("RELEASE: %s", chart.release) deployed_releases = [x[0] for x in known_releases] prefix_chart = release_prefix(prefix, chart.release) if prefix_chart in deployed_releases: # indicate to the end user what path we are taking LOG.info("Upgrading release %s", chart.release) # extract the installed chart and installed values from the # latest release so we can compare to the intended state LOG.info("Checking Pre/Post Actions") apply_chart, apply_values = self.find_release_chart( known_releases, prefix_chart) LOG.info("Checking Pre/Post Actions") upgrade = gchart.get('chart', {}).get('upgrade', False) if upgrade: if not self.disable_update_pre and upgrade.get( 'pre', False): pre_actions = getattr(chart.upgrade, 'pre', {}) if not self.disable_update_post and upgrade.get( 'post', False): post_actions = getattr(chart.upgrade, 'post', {}) # show delta for both the chart templates and the chart # values # TODO(alanmeadows) account for .files differences # once we support those upgrade_diff = self.show_diff(chart, apply_chart, apply_values, chartbuilder.dump(), values, msg) if not upgrade_diff: LOG.info("There are no updates found in this chart") continue # do actual update LOG.info('wait: %s', chart_wait) self.tiller.update_release( protoc_chart, prefix_chart, chart.namespace, pre_actions=pre_actions, post_actions=post_actions, dry_run=self.dry_run, disable_hooks=chart.upgrade.no_hooks, values=yaml.safe_dump(values), wait=chart_wait, timeout=chart_timeout) if chart_wait: # TODO(gardlt): after v0.7.1 depricate timeout values if not wait_values.get('timeout', None): wait_values['timeout'] = chart_timeout self.tiller.k8s.wait_until_ready( release=prefix_chart, labels=wait_values.get('labels', ''), namespace=chart.namespace, timeout=wait_values.get('timeout', DEFAULT_TIMEOUT)) msg['upgrade'].append(prefix_chart) # process install else: LOG.info("Installing release %s", chart.release) self.tiller.install_release(protoc_chart, prefix_chart, chart.namespace, dry_run=self.dry_run, values=yaml.safe_dump(values), wait=chart_wait, timeout=chart_timeout) if chart_wait: if not wait_values.get('timeout', None): wait_values['timeout'] = chart_timeout self.tiller.k8s.wait_until_ready( release=prefix_chart, labels=wait_values.get('labels', ''), namespace=chart.namespace, timeout=wait_values.get('timeout', 3600)) msg['install'].append(prefix_chart) LOG.debug("Cleaning up chart source in %s", chartbuilder.source_directory) if test_charts or test_chart: LOG.info('Testing: %s', prefix_chart) resp = self.tiller.testing_release(prefix_chart) test_status = getattr(resp.info.status, 'last_test_suite_run', 'FAILED') LOG.info("Test INFO: %s", test_status) if resp: LOG.info("PASSED: %s", prefix_chart) else: LOG.info("FAILED: %s", prefix_chart) self.tiller.k8s.wait_until_ready(timeout=chart_timeout) LOG.info("Performing Post-Flight Operations") self.post_flight_ops() if self.enable_chart_cleanup: self.tiller.chart_cleanup( prefix, self.config[const.KEYWORD_ARMADA][const.KEYWORD_GROUPS]) return msg
def test_release_prefix_int_int(self): expected = '4-4' prefix, chart = (4, 4) assert rel.release_prefix(prefix, chart) == expected
def test_release_prefix_int_string(self): expected = 'armada-4' prefix, chart = ('armada', 4) assert rel.release_prefix(prefix, chart) == expected
def test_release_prefix_pass(self): expected = 'armada-test' prefix, chart = ('armada', 'test') assert rel.release_prefix(prefix, chart) == expected
def on_post(self, req, resp): # TODO(fmontei): Validation Content-Type is application/x-yaml. target_manifest = req.get_param('target_manifest', None) try: tiller = Tiller( tiller_host=req.get_param('tiller_host'), tiller_port=req.get_param_as_int( 'tiller_port') or CONF.tiller_port, tiller_namespace=req.get_param( 'tiller_namespace', default=CONF.tiller_namespace)) # TODO(fmontei): Provide more sensible exception(s) here. except Exception: err_message = 'Failed to initialize Tiller handler.' self.error(req.context, err_message) return self.return_error( resp, falcon.HTTP_500, message=err_message) try: documents = self.req_yaml(req, default=[]) except yaml.YAMLError: err_message = 'Documents must be valid YAML.' return self.return_error( resp, falcon.HTTP_400, message=err_message) is_valid = self._validate_documents(req, resp, documents) if not is_valid: return resp armada_obj = Manifest( documents, target_manifest=target_manifest).get_manifest() prefix = armada_obj.get(const.KEYWORD_ARMADA).get( const.KEYWORD_PREFIX) known_releases = [release[0] for release in tiller.list_charts()] message = { 'tests': { 'passed': [], 'skipped': [], 'failed': [] } } for group in armada_obj.get(const.KEYWORD_ARMADA).get( const.KEYWORD_GROUPS): for ch in group.get(const.KEYWORD_CHARTS): release_name = release_prefix( prefix, ch.get('chart').get('chart_name')) if release_name in known_releases: self.logger.info('RUNNING: %s tests', release_name) resp = tiller.testing_release(release_name) if not resp: continue test_status = getattr( resp.info.status, 'last_test_suite_run', 'FAILED') if test_status.results[0].status: self.logger.info("PASSED: %s", release_name) message['test']['passed'].append(release_name) else: self.logger.info("FAILED: %s", release_name) message['test']['failed'].append(release_name) else: self.logger.info( 'Release %s not found - SKIPPING', release_name) message['test']['skipped'].append(release_name) resp.status = falcon.HTTP_200 resp.body = json.dumps(message) resp.content_type = 'application/json'
def sync(self): ''' Synchronize Helm with the Armada Config(s) ''' msg = {'install': [], 'upgrade': [], 'diff': []} # TODO: (gardlt) we need to break up this func into # a more cleaner format self.pre_flight_ops() # extract known charts on tiller right now known_releases = self.tiller.list_charts() manifest_data = self.manifest.get(KEYWORD_ARMADA, {}) prefix = manifest_data.get(KEYWORD_PREFIX, '') for chartgroup in manifest_data.get(KEYWORD_GROUPS, []): cg_name = chartgroup.get('name', '<missing name>') cg_desc = chartgroup.get('description', '<missing description>') LOG.info('Processing ChartGroup: %s (%s)', cg_name, cg_desc) cg_sequenced = chartgroup.get('sequenced', False) cg_test_all_charts = chartgroup.get('test_charts', False) namespaces_seen = set() tests_to_run = [] cg_charts = chartgroup.get(KEYWORD_CHARTS, []) # Track largest Chart timeout to stop the ChartGroup at the end cg_max_timeout = 0 for chart_entry in cg_charts: chart = chart_entry.get('chart', {}) namespace = chart.get('namespace') release = chart.get('release') values = chart.get('values', {}) pre_actions = {} post_actions = {} wait_timeout = self.timeout wait_labels = {} release_name = release_prefix(prefix, release) # Retrieve appropriate timeout value if wait_timeout <= 0: # TODO(MarshM): chart's `data.timeout` should be deprecated chart_timeout = chart.get('timeout', 0) # Favor data.wait.timeout over data.timeout, until removed wait_values = chart.get('wait', {}) wait_timeout = wait_values.get('timeout', chart_timeout) wait_labels = wait_values.get('labels', {}) this_chart_should_wait = (cg_sequenced or self.force_wait or wait_timeout > 0 or len(wait_labels) > 0) if this_chart_should_wait and wait_timeout <= 0: LOG.warn('No Chart timeout specified, using default: %ss', DEFAULT_CHART_TIMEOUT) wait_timeout = DEFAULT_CHART_TIMEOUT # Track namespaces + labels touched namespaces_seen.add((namespace, tuple(wait_labels.items()))) # Naively take largest timeout to apply at end # TODO(MarshM) better handling of timeout/timer cg_max_timeout = max(wait_timeout, cg_max_timeout) # Chart test policy can override ChartGroup, if specified test_this_chart = chart.get('test', cg_test_all_charts) chartbuilder = ChartBuilder(chart) protoc_chart = chartbuilder.get_helm_chart() deployed_releases = [x[0] for x in known_releases] # Begin Chart timeout deadline deadline = time.time() + wait_timeout # TODO(mark-burnett): It may be more robust to directly call # tiller status to decide whether to install/upgrade rather # than checking for list membership. if release_name in deployed_releases: # indicate to the end user what path we are taking LOG.info("Upgrading release %s in namespace %s", release_name, namespace) # extract the installed chart and installed values from the # latest release so we can compare to the intended state apply_chart, apply_values = self.find_release_chart( known_releases, release_name) upgrade = chart.get('upgrade', {}) disable_hooks = upgrade.get('no_hooks', False) LOG.info("Checking Pre/Post Actions") if upgrade: upgrade_pre = upgrade.get('pre', {}) upgrade_post = upgrade.get('post', {}) if not self.disable_update_pre and upgrade_pre: pre_actions = upgrade_pre if not self.disable_update_post and upgrade_post: post_actions = upgrade_post # Show delta for both the chart templates and the chart # values # TODO(alanmeadows) account for .files differences # once we support those LOG.info('Checking upgrade chart diffs.') upgrade_diff = self.show_diff(chart, apply_chart, apply_values, chartbuilder.dump(), values, msg) if not upgrade_diff: LOG.info("There are no updates found in this chart") continue # TODO(MarshM): Add tiller dry-run before upgrade and # consider deadline impacts # do actual update timer = int(round(deadline - time.time())) LOG.info('Beginning Upgrade, wait=%s, timeout=%ss', this_chart_should_wait, timer) tiller_result = self.tiller.update_release( protoc_chart, release_name, namespace, pre_actions=pre_actions, post_actions=post_actions, dry_run=self.dry_run, disable_hooks=disable_hooks, values=yaml.safe_dump(values), wait=this_chart_should_wait, timeout=timer) if this_chart_should_wait: self.tiller.k8s.wait_until_ready( release=release_name, labels=wait_labels, namespace=namespace, k8s_wait_attempts=self.k8s_wait_attempts, k8s_wait_attempt_sleep=self.k8s_wait_attempt_sleep, timeout=timer) LOG.info('Upgrade completed with results from Tiller: %s', tiller_result.__dict__) msg['upgrade'].append(release_name) # process install else: LOG.info("Installing release %s in namespace %s", release_name, namespace) timer = int(round(deadline - time.time())) LOG.info('Beginning Install, wait=%s, timeout=%ss', this_chart_should_wait, timer) tiller_result = self.tiller.install_release( protoc_chart, release_name, namespace, dry_run=self.dry_run, values=yaml.safe_dump(values), wait=this_chart_should_wait, timeout=timer) if this_chart_should_wait: self.tiller.k8s.wait_until_ready( release=release_name, labels=wait_labels, namespace=namespace, k8s_wait_attempts=self.k8s_wait_attempts, k8s_wait_attempt_sleep=self.k8s_wait_attempt_sleep, timeout=timer) LOG.info('Install completed with results from Tiller: %s', tiller_result.__dict__) msg['install'].append(release_name) # Sequenced ChartGroup should run tests after each Chart timer = int(round(deadline - time.time())) if test_this_chart and cg_sequenced: LOG.info('Running sequenced test, timeout remaining: %ss.', timer) if timer <= 0: reason = ('Timeout expired before testing sequenced ' 'release %s' % release_name) LOG.error(reason) raise ArmadaTimeoutException(reason) self._test_chart(release_name, timer) # Un-sequenced ChartGroup should run tests at the end elif test_this_chart: # Keeping track of time remaining tests_to_run.append((release_name, timer)) # End of Charts in ChartGroup LOG.info('All Charts applied.') # After all Charts are applied, we should wait for the entire # ChartGroup to become healthy by looking at the namespaces seen # TODO(MarshM): Need to restrict to only releases we processed # TODO(MarshM): Need to determine a better timeout # (not cg_max_timeout) if cg_max_timeout <= 0: cg_max_timeout = DEFAULT_CHART_TIMEOUT deadline = time.time() + cg_max_timeout for (ns, labels) in namespaces_seen: labels_dict = dict(labels) timer = int(round(deadline - time.time())) LOG.info( 'Final wait for healthy namespace (%s), label=(%s), ' 'timeout remaining: %ss.', ns, labels_dict, timer) if timer <= 0: reason = ('Timeout expired waiting on namespace: %s, ' 'label: %s' % (ns, labels_dict)) LOG.error(reason) raise ArmadaTimeoutException(reason) self.tiller.k8s.wait_until_ready( namespace=ns, labels=labels_dict, k8s_wait_attempts=self.k8s_wait_attempts, k8s_wait_attempt_sleep=self.k8s_wait_attempt_sleep, timeout=timer) # After entire ChartGroup is healthy, run any pending tests for (test, test_timer) in tests_to_run: self._test_chart(test, test_timer) LOG.info("Performing Post-Flight Operations") self.post_flight_ops() if self.enable_chart_cleanup: self.tiller.chart_cleanup( prefix, self.manifest[KEYWORD_ARMADA][KEYWORD_GROUPS]) return msg