def handle(self, tiller): known_release_names = [release[0] for release in tiller.list_charts()] if self.releases: target_releases = [ r.strip() for r in self.releases.split(',') if r.strip() in known_release_names ] if not target_releases: self.logger.info("There's no release to delete.") return if not self.ctx.obj.get('api', False): for r in target_releases: self.logger.info("Deleting release %s", r) tiller.delete_release(r, purge=self.purge) else: raise NotImplementedError() if self.manifest: target_deletes = [] with open(self.manifest) as f: documents = list(yaml.safe_load_all(f.read())) try: armada_obj = Manifest(documents).get_manifest() prefix = armada_obj.get(const.KEYWORD_DATA).get( const.KEYWORD_PREFIX) for group in armada_obj.get(const.KEYWORD_DATA).get( const.KEYWORD_GROUPS): for ch in group.get(const.KEYWORD_DATA).get( const.KEYWORD_CHARTS): chart = ch.get(const.KEYWORD_DATA) release_name = release_prefixer( prefix, chart.get('release')) if release_name in known_release_names: target_deletes.append((chart, release_name)) except yaml.YAMLError as e: mark = e.problem_mark self.logger.info( "While parsing the manifest file, %s. " "Error position: (%s:%s)", e.problem, mark.line + 1, mark.column + 1) if not target_deletes: self.logger.info("There's no release to delete.") return if not self.ctx.obj.get('api', False): for chart, release in target_deletes: chart_delete = ChartDelete(chart, release, tiller, purge=self.purge) chart_delete.delete() else: raise NotImplementedError()
def handle(self, req, resp, tiller): try: documents = self.req_yaml(req, default=[]) except yaml.YAMLError: err_message = 'Documents must be valid YAML.' return self.return_error( resp, falcon.HTTP_400, message=err_message) target_manifest = req.get_param('target_manifest', None) is_valid = self._validate_documents(req, resp, documents) if not is_valid: return armada_obj = Manifest( documents, target_manifest=target_manifest).get_manifest() prefix = armada_obj.get(const.KEYWORD_ARMADA).get(const.KEYWORD_PREFIX) known_releases = [release[0] for release in tiller.list_charts()] message = {'tests': {'passed': [], 'skipped': [], 'failed': []}} for group in armada_obj.get(const.KEYWORD_ARMADA).get( const.KEYWORD_GROUPS): for ch in group.get(const.KEYWORD_CHARTS): chart = ch['chart'] release_name = release_prefixer(prefix, chart.get('release')) if release_name in known_releases: cleanup = req.get_param_as_bool('cleanup') enable_all = req.get_param_as_bool('enable_all') cg_test_charts = group.get('test_charts') test_values = chart.get('test', {}) test_handler = Test( release_name, tiller, cg_test_charts=cg_test_charts, cleanup=cleanup, enable_all=enable_all, test_values=test_values) if test_handler.test_enabled: success = test_handler.test_release_for_success() if success: message['test']['passed'].append(release_name) else: message['test']['failed'].append(release_name) else: self.logger.info('Release %s not found - SKIPPING', release_name) message['test']['skipped'].append(release_name) resp.status = falcon.HTTP_200 resp.body = json.dumps(message) resp.content_type = 'application/json'
def handle(self, req, resp, helm): try: documents = self.req_yaml(req, default=[]) except yaml.YAMLError: err_message = 'Documents must be valid YAML.' return self.return_error(resp, falcon.HTTP_400, message=err_message) target_manifest = req.get_param('target_manifest', None) is_valid = self._validate_documents(req, resp, documents) if not is_valid: return armada_obj = Manifest(documents, target_manifest=target_manifest).get_manifest() prefix = armada_obj[const.KEYWORD_DATA][const.KEYWORD_PREFIX] release_ids = helm.list_release_ids() message = {'tests': {'passed': [], 'skipped': [], 'failed': []}} for group in armada_obj.get(const.KEYWORD_DATA).get( const.KEYWORD_GROUPS): for ch in group.get(const.KEYWORD_CHARTS): chart = ch['chart'] release_id = helm.HelmReleaseId( ch['namespace'], release_prefixer(prefix, ch['release'])) if release_id in release_ids: enable_all = req.get_param_as_bool('enable_all') cg_test_charts = group.get('test_charts') test_handler = Test(chart, release_id, helm, cg_test_charts=cg_test_charts, enable_all=enable_all) if test_handler.test_enabled: success = test_handler.test_release_for_success() if success: message['test']['passed'].append(release_id) else: message['test']['failed'].append(release_id) else: self.logger.info('Release %s not found - SKIPPING', release_id) message['test']['skipped'].append(release_id) resp.status = falcon.HTTP_200 resp.text = json.dumps(message) resp.content_type = 'application/json'
def handle(self, helm): release_ids = helm.list_release_ids() if self.release: if not self.ctx.obj.get('api', False): release_id = HelmReleaseId(self.namespace, self.release) test_handler = Test({}, release_id, helm) test_handler.test_release_for_success() else: client = self.ctx.obj.get('CLIENT') resp = client.get_test_release(release=self.release) self.logger.info(resp.get('result')) self.logger.info(resp.get('message')) if self.file: if not self.ctx.obj.get('api', False): documents = list(yaml.safe_load_all(open(self.file).read())) armada_obj = Manifest( documents, target_manifest=self.target_manifest).get_manifest() prefix = armada_obj.get(const.KEYWORD_DATA).get( const.KEYWORD_PREFIX) for group in armada_obj.get(const.KEYWORD_DATA).get( const.KEYWORD_GROUPS): for ch in group.get(const.KEYWORD_CHARTS): chart = ch['chart'] release_id = HelmReleaseId( chart['namespace'], release_prefixer(prefix, chart['release'])) if release_id in release_ids: test_handler = Test(chart, release_id, helm, enable_all=self.enable_all) if test_handler.test_enabled: test_handler.test_release_for_success() else: self.logger.info('Release %s not found - SKIPPING', release_id) else: client = self.ctx.obj.get('CLIENT') with open(self.filename, 'r') as f: resp = client.get_test_manifest(manifest=f.read()) for test in resp.get('tests'): self.logger.info('Test State: %s', test) for item in test.get('tests').get(test): self.logger.info(item) self.logger.info(resp)
def _chart_cleanup(self, prefix, charts, msg): LOG.info('Processing chart cleanup to remove unspecified releases.') valid_releases = [] for gchart in charts: for chart in gchart.get(const.KEYWORD_CHARTS, []): valid_releases.append( release_prefixer(prefix, chart.get('chart', {}).get('release'))) actual_releases = [x.name for x in self.tiller.list_releases()] release_diff = list(set(actual_releases) - set(valid_releases)) for release in release_diff: if release.startswith(prefix): LOG.info('Purging release %s as part of chart cleanup.', release) self.tiller.uninstall_release(release) msg['purge'].append(release)
def _chart_cleanup(self, prefix, chart_groups, msg): LOG.info('Processing chart cleanup to remove unspecified releases.') valid_release_ids = [] for group in chart_groups: group_data = group.get(const.KEYWORD_DATA, {}) for chart in group_data.get(const.KEYWORD_CHARTS, []): chart_data = chart.get(const.KEYWORD_DATA, {}) valid_release_ids.append( HelmReleaseId( chart_data['namespace'], release_prefixer(prefix, chart_data['release']))) actual_release_ids = self.helm.list_release_ids() release_diff = list(set(actual_release_ids) - set(valid_release_ids)) for release_id in release_diff: if release_id.name.startswith(prefix): LOG.info('Purging release %s as part of chart cleanup.', release_id) self.helm.uninstall_release(release_id) msg['purge'].append('{}'.format(release_id))
def _do_test(mock_test, mock_chartbuilder, MockChartDownload, mock_post_flight): MockChartDownload.return_value.get_chart.side_effect = \ set_source_dir # Instantiate Armada object. yaml_documents = list(yaml.safe_load_all(TEST_YAML)) m_helm = mock.MagicMock() armada_obj = armada.Armada(yaml_documents, m_helm) prefix = armada_obj.manifest['data']['release_prefix'] def release_metadata(release_id, **kwargs): try: return next(r for r in known_releases if release_id.name == r['name'] and release_id.namespace == r['namespace']) except StopIteration: return None m_helm.release_metadata.side_effect = release_metadata armada_obj.chart_deploy.get_diff = mock.Mock() cg = armada_obj.manifest['data']['chart_groups'][0] chart_group = cg['data'] charts = chart_group['chart_group'] cg_test_all_charts = chart_group.get('test_charts') mock_test_release = mock_test.return_value.test_release if test_failure_to_run: mock_test_release.side_effect = Exception('test failed to run') else: if not test_success: mock_test_release.side_effect = Exception('test failed') mock_test.return_value.timeout = const.DEFAULT_TEST_TIMEOUT # Stub out irrelevant methods called by `armada.sync()`. mock_chartbuilder.get_helm_chart.return_value = None # Simulate chart diff, upgrade should only happen if non-empty. armada_obj.chart_deploy.get_diff.return_value = diff armada_obj.sync() expected_install_release_calls = [] expected_upgrade_release_calls = [] expected_uninstall_release_calls = [] expected_test_constructor_calls = [] for c in charts: chart = c['data'] release = chart['release'] release_name = release_prefixer(prefix, release) release_id = helm.HelmReleaseId(chart['namespace'], release_name) source_dir = chart['source_dir'] source_directory = os.path.join(*source_dir) # Simplified check because the actual code uses logical-or's # multiple conditions, so this is enough. native_wait_enabled = (chart['wait'].get('native', {}).get( 'enabled', True)) if release_name not in [x['name'] for x in known_releases]: expected_install_release_calls.append( mock.call(source_directory, release_id, values=chart['values'], wait=native_wait_enabled, timeout=mock.ANY)) else: target_release = None for known_release in known_releases: if known_release['name'] == release_name: target_release = known_release break if target_release: status = get_release_status(target_release) if status == helm.STATUS_FAILED: protected = chart.get('protected', {}) if not protected: expected_uninstall_release_calls.append( mock.call( release_id, purge=True, timeout=const.DEFAULT_DELETE_TIMEOUT)) expected_install_release_calls.append( mock.call(source_directory, release_id, values=chart['values'], wait=native_wait_enabled, timeout=mock.ANY)) else: p_continue = protected.get( 'continue_processing', False) if p_continue: continue else: if chart_group['sequenced']: break if status == helm.STATUS_DEPLOYED: if diff: upgrade = chart.get('upgrade', {}) disable_hooks = upgrade.get('no_hooks', False) options = upgrade.get('options', {}) force = options.get('force', False) expected_upgrade_release_calls.append( mock.call(source_directory, release_id, disable_hooks=disable_hooks, force=force, values=chart['values'], wait=native_wait_enabled, timeout=mock.ANY)) expected_test_constructor_calls.append( mock.call(chart, release_id, m_helm, cg_test_charts=cg_test_all_charts)) any_order = not chart_group['sequenced'] # Verify that at least 1 release is either installed or updated. self.assertTrue( len(expected_install_release_calls) >= 1 or len(expected_upgrade_release_calls) >= 1) # Verify that the expected number of non-deployed releases are # installed with expected arguments. self.assertEqual(len(expected_install_release_calls), m_helm.install_release.call_count) m_helm.install_release.assert_has_calls( expected_install_release_calls, any_order=any_order) # Verify that the expected number of deployed releases are # updated with expected arguments. self.assertEqual(len(expected_upgrade_release_calls), m_helm.upgrade_release.call_count) m_helm.upgrade_release.assert_has_calls( expected_upgrade_release_calls, any_order=any_order) # Verify that the expected number of deployed releases are # uninstalled with expected arguments. self.assertEqual(len(expected_uninstall_release_calls), m_helm.uninstall_release.call_count) m_helm.uninstall_release.assert_has_calls( expected_uninstall_release_calls, any_order=any_order) # Verify that the expected number of deployed releases are # tested with expected arguments. self.assertEqual(len(expected_test_constructor_calls), mock_test.call_count) mock_test.assert_has_calls(expected_test_constructor_calls, any_order=True)
def test_release_prefix_int_int(self): expected = '4-4' prefix, release = (4, 4) assert rel.release_prefixer(prefix, release) == expected
def test_release_prefix_int_string(self): expected = 'armada-4' prefix, release = ('armada', 4) assert rel.release_prefixer(prefix, release) == expected
def test_release_prefix_pass(self): expected = 'armada-test' prefix, release = ('armada', 'test') assert rel.release_prefixer(prefix, release) == expected
def on_post(self, req, resp): # TODO(fmontei): Validation Content-Type is application/x-yaml. target_manifest = req.get_param('target_manifest', None) try: tiller = Tiller(tiller_host=req.get_param('tiller_host'), tiller_port=req.get_param_as_int('tiller_port') or CONF.tiller_port, tiller_namespace=req.get_param( 'tiller_namespace', default=CONF.tiller_namespace)) # TODO(fmontei): Provide more sensible exception(s) here. except Exception: err_message = 'Failed to initialize Tiller handler.' self.error(req.context, err_message) return self.return_error(resp, falcon.HTTP_500, message=err_message) try: documents = self.req_yaml(req, default=[]) except yaml.YAMLError: err_message = 'Documents must be valid YAML.' return self.return_error(resp, falcon.HTTP_400, message=err_message) is_valid = self._validate_documents(req, resp, documents) if not is_valid: return resp armada_obj = Manifest(documents, target_manifest=target_manifest).get_manifest() prefix = armada_obj.get(const.KEYWORD_ARMADA).get(const.KEYWORD_PREFIX) known_releases = [release[0] for release in tiller.list_charts()] message = {'tests': {'passed': [], 'skipped': [], 'failed': []}} for group in armada_obj.get(const.KEYWORD_ARMADA).get( const.KEYWORD_GROUPS): for ch in group.get(const.KEYWORD_CHARTS): chart = ch['chart'] release_name = release_prefixer(prefix, chart.get('release')) cleanup = req.get_param_as_bool('cleanup') if cleanup is None: test_chart_override = chart.get('test', {}) if isinstance(test_chart_override, bool): self.logger.warn( 'Boolean value for chart `test` key is deprecated ' 'and will be removed. Use `test.enabled` instead.') # Use old default value. cleanup = True else: cleanup = test_chart_override.get('options', {}).get( 'cleanup', False) if release_name in known_releases: self.logger.info('RUNNING: %s tests', release_name) success = test_release_for_success(tiller, release_name, cleanup=cleanup) if success: self.logger.info("PASSED: %s", release_name) message['test']['passed'].append(release_name) else: self.logger.info("FAILED: %s", release_name) message['test']['failed'].append(release_name) else: self.logger.info('Release %s not found - SKIPPING', release_name) message['test']['skipped'].append(release_name) resp.status = falcon.HTTP_200 resp.body = json.dumps(message) resp.content_type = 'application/json'
def _execute(self, ch, cg_test_all_charts, prefix): manifest_name = self.manifest['metadata']['name'] chart = ch[const.KEYWORD_DATA] chart_name = ch['metadata']['name'] namespace = chart.get('namespace') release = chart.get('release') release_name = r.release_prefixer(prefix, release) release_id = helm.HelmReleaseId(namespace, release_name) source_dir = chart['source_dir'] source_directory = os.path.join(*source_dir) LOG.info('Processing Chart, release=%s', release_id) result = {} chart_wait = ChartWait( self.helm.k8s, release_id, ch, k8s_wait_attempts=self.k8s_wait_attempts, k8s_wait_attempt_sleep=self.k8s_wait_attempt_sleep, timeout=self.timeout) wait_timeout = chart_wait.get_timeout() # Begin Chart timeout deadline deadline = time.time() + wait_timeout old_release = self.helm.release_metadata(release_id) action = metrics.ChartDeployAction.NOOP def noop(): pass deploy = noop # Resolve action values = chart.get('values', {}) pre_actions = {} status = None if old_release: status = r.get_release_status(old_release) native_wait_enabled = chart_wait.is_native_enabled() chartbuilder = ChartBuilder.from_chart_doc(ch, self.helm) if status == helm.STATUS_DEPLOYED: # indicate to the end user what path we are taking LOG.info("Existing release %s found", release_id) # extract the installed chart and installed values from the # latest release so we can compare to the intended state old_chart = old_release['chart'] old_values = old_release.get('config', {}) upgrade = chart.get('upgrade', {}) options = upgrade.get('options', {}) # TODO: Remove when v1 doc support is removed. schema_info = get_schema_info(ch['schema']) if schema_info.version < 2: no_hooks_location = upgrade else: no_hooks_location = options disable_hooks = no_hooks_location.get('no_hooks', False) force = options.get('force', False) if upgrade: upgrade_pre = upgrade.get('pre', {}) upgrade_post = upgrade.get('post', {}) if not self.disable_update_pre and upgrade_pre: pre_actions = upgrade_pre if not self.disable_update_post and upgrade_post: LOG.warning('Post upgrade actions are ignored by Armada' 'and will not affect deployment.') LOG.info('Checking for updates to chart release inputs.') new_chart = chartbuilder.get_helm_chart(release_id, values) diff = self.get_diff(old_chart, old_values, new_chart, values) if not diff: LOG.info("Found no updates to chart release inputs") else: action = metrics.ChartDeployAction.UPGRADE LOG.info("Found updates to chart release inputs") def upgrade(): # do actual update timer = int(round(deadline - time.time())) PreUpdateActions(self.helm.k8s).execute( pre_actions, release, namespace, chart, disable_hooks, values, timer) LOG.info("Upgrading release=%s, wait=%s, " "timeout=%ss", release_id, native_wait_enabled, timer) self.helm.upgrade_release(source_directory, release_id, disable_hooks=disable_hooks, values=values, wait=native_wait_enabled, timeout=timer, force=force) LOG.info('Upgrade completed') result['upgrade'] = release_id deploy = upgrade else: def install(): timer = int(round(deadline - time.time())) LOG.info("Installing release=%s, wait=%s, " "timeout=%ss", release_id, native_wait_enabled, timer) self.helm.install_release(source_directory, release_id, values=values, wait=native_wait_enabled, timeout=timer) LOG.info('Install completed') result['install'] = release_id # Check for release with status other than DEPLOYED if status: if status != helm.STATUS_FAILED: LOG.warn( 'Unexpected release status encountered ' 'release=%s, status=%s', release_id, status) # Make best effort to determine whether a deployment is # likely pending, by checking if the last deployment # was started within the timeout window of the chart. last_deployment_age = r.get_last_deployment_age( old_release) likely_pending = last_deployment_age <= wait_timeout if likely_pending: # We don't take any deploy action and wait for the # to get deployed. deploy = noop deadline = deadline - last_deployment_age else: # Release is likely stuck in an unintended # state. Log and continue on with remediation steps # below. LOG.info( 'Old release %s likely stuck in status %s, ' '(last deployment age=%ss) >= ' '(chart wait timeout=%ss)', release, status, last_deployment_age, wait_timeout) res = self.purge_release(chart, release_id, status, manifest_name, chart_name, result) if isinstance(res, dict): if 'protected' in res: return res action = metrics.ChartDeployAction.INSTALL deploy = install else: # The chart is in Failed state, hence we purge # the chart and attempt to install it again. res = self.purge_release(chart, release_id, status, manifest_name, chart_name, result) if isinstance(res, dict): if 'protected' in res: return res action = metrics.ChartDeployAction.INSTALL deploy = install if status is None: action = metrics.ChartDeployAction.INSTALL deploy = install # Deploy with metrics.CHART_DEPLOY.get_context(wait_timeout, manifest_name, chart_name, action.get_label_value()): deploy() # Wait timer = int(round(deadline - time.time())) chart_wait.wait(timer) # Test just_deployed = ('install' in result) or ('upgrade' in result) last_test_passed = old_release and r.get_last_test_result(old_release) test_handler = Test(chart, release_id, self.helm, cg_test_charts=cg_test_all_charts) run_test = test_handler.test_enabled and (just_deployed or not last_test_passed) if run_test: with metrics.CHART_TEST.get_context(test_handler.timeout, manifest_name, chart_name): self._test_chart(test_handler) return result
def invoke(self): tiller = Tiller( tiller_host=self.tiller_host, tiller_port=self.tiller_port, tiller_namespace=self.tiller_namespace) known_release_names = [release[0] for release in tiller.list_charts()] if self.release: if not self.ctx.obj.get('api', False): self.logger.info("RUNNING: %s tests", self.release) success = test_release_for_success( tiller, self.release, cleanup=self.cleanup) if success: self.logger.info("PASSED: %s", self.release) else: self.logger.info("FAILED: %s", self.release) else: client = self.ctx.obj.get('CLIENT') query = { 'tiller_host': self.tiller_host, 'tiller_port': self.tiller_port, 'tiller_namespace': self.tiller_namespace } resp = client.get_test_release( release=self.release, query=query) self.logger.info(resp.get('result')) self.logger.info(resp.get('message')) if self.file: if not self.ctx.obj.get('api', False): documents = list(yaml.safe_load_all(open(self.file).read())) armada_obj = Manifest( documents, target_manifest=self.target_manifest).get_manifest() prefix = armada_obj.get(const.KEYWORD_ARMADA).get( const.KEYWORD_PREFIX) for group in armada_obj.get(const.KEYWORD_ARMADA).get( const.KEYWORD_GROUPS): for ch in group.get(const.KEYWORD_CHARTS): chart = ch['chart'] release_name = release_prefixer( prefix, chart.get('release')) if release_name in known_release_names: cleanup = self.cleanup if cleanup is None: test_chart_override = chart.get('test', {}) if isinstance(test_chart_override, bool): self.logger.warn( 'Boolean value for chart `test` key is' ' deprecated and support for this will' ' be removed. Use `test.enabled` ' 'instead.') # Use old default value. cleanup = True else: cleanup = test_chart_override.get( 'options', {}).get('cleanup', False) self.logger.info('RUNNING: %s tests', release_name) success = test_release_for_success( tiller, release_name, cleanup=cleanup) if success: self.logger.info("PASSED: %s", release_name) else: self.logger.info("FAILED: %s", release_name) else: self.logger.info('Release %s not found - SKIPPING', release_name) else: client = self.ctx.obj.get('CLIENT') query = { 'tiller_host': self.tiller_host, 'tiller_port': self.tiller_port, 'tiller_namespace': self.tiller_namespace } with open(self.filename, 'r') as f: resp = client.get_test_manifest( manifest=f.read(), query=query) for test in resp.get('tests'): self.logger.info('Test State: %s', test) for item in test.get('tests').get(test): self.logger.info(item) self.logger.info(resp)
def _execute(self, ch, cg_test_all_charts, prefix, known_releases): manifest_name = self.manifest['metadata']['name'] chart = ch[const.KEYWORD_DATA] chart_name = ch['metadata']['name'] namespace = chart.get('namespace') release = chart.get('release') release_name = r.release_prefixer(prefix, release) LOG.info('Processing Chart, release=%s', release_name) result = {} chart_wait = ChartWait( self.tiller.k8s, release_name, ch, namespace, k8s_wait_attempts=self.k8s_wait_attempts, k8s_wait_attempt_sleep=self.k8s_wait_attempt_sleep, timeout=self.timeout) wait_timeout = chart_wait.get_timeout() # Begin Chart timeout deadline deadline = time.time() + wait_timeout old_release = self.find_chart_release(known_releases, release_name) action = metrics.ChartDeployAction.NOOP def noop(): pass deploy = noop # Resolve action values = chart.get('values', {}) pre_actions = {} post_actions = {} status = None if old_release: status = r.get_release_status(old_release) native_wait_enabled = chart_wait.is_native_enabled() chartbuilder = ChartBuilder.from_chart_doc(ch) new_chart = chartbuilder.get_helm_chart() if status == const.STATUS_DEPLOYED: # indicate to the end user what path we are taking LOG.info("Existing release %s found in namespace %s", release_name, namespace) # extract the installed chart and installed values from the # latest release so we can compare to the intended state old_chart = old_release.chart old_values_string = old_release.config.raw upgrade = chart.get('upgrade', {}) options = upgrade.get('options', {}) # TODO: Remove when v1 doc support is removed. schema_info = get_schema_info(ch['schema']) if schema_info.version < 2: no_hooks_location = upgrade else: no_hooks_location = options disable_hooks = no_hooks_location.get('no_hooks', False) force = options.get('force', False) recreate_pods = options.get('recreate_pods', False) if upgrade: upgrade_pre = upgrade.get('pre', {}) upgrade_post = upgrade.get('post', {}) if not self.disable_update_pre and upgrade_pre: pre_actions = upgrade_pre if not self.disable_update_post and upgrade_post: LOG.warning('Post upgrade actions are ignored by Armada' 'and will not affect deployment.') post_actions = upgrade_post try: old_values = yaml.safe_load(old_values_string) except yaml.YAMLError: chart_desc = '{} (previously deployed)'.format( old_chart.metadata.name) raise armada_exceptions.\ InvalidOverrideValuesYamlException(chart_desc) LOG.info('Checking for updates to chart release inputs.') diff = self.get_diff(old_chart, old_values, new_chart, values) if not diff: LOG.info("Found no updates to chart release inputs") else: action = metrics.ChartDeployAction.UPGRADE LOG.info("Found updates to chart release inputs") LOG.debug("%s", diff) result['diff'] = {chart['release']: str(diff)} def upgrade(): # do actual update timer = int(round(deadline - time.time())) LOG.info( "Upgrading release %s in namespace %s, wait=%s, " "timeout=%ss", release_name, namespace, native_wait_enabled, timer) tiller_result = self.tiller.update_release( new_chart, release_name, namespace, pre_actions=pre_actions, post_actions=post_actions, disable_hooks=disable_hooks, values=yaml.safe_dump(values), wait=native_wait_enabled, timeout=timer, force=force, recreate_pods=recreate_pods) LOG.info('Upgrade completed with results from Tiller: %s', tiller_result.__dict__) result['upgrade'] = release_name deploy = upgrade else: # Check for release with status other than DEPLOYED if status: if status != const.STATUS_FAILED: LOG.warn( 'Unexpected release status encountered ' 'release=%s, status=%s', release_name, status) # Make best effort to determine whether a deployment is # likely pending, by checking if the last deployment # was started within the timeout window of the chart. last_deployment_age = r.get_last_deployment_age( old_release) likely_pending = last_deployment_age <= wait_timeout if likely_pending: # Give up if a deployment is likely pending, we do not # want to have multiple operations going on for the # same release at the same time. raise armada_exceptions.\ DeploymentLikelyPendingException( release_name, status, last_deployment_age, wait_timeout) else: # Release is likely stuck in an unintended (by tiller) # state. Log and continue on with remediation steps # below. LOG.info( 'Old release %s likely stuck in status %s, ' '(last deployment age=%ss) >= ' '(chart wait timeout=%ss)', release, status, last_deployment_age, wait_timeout) protected = chart.get('protected', {}) if protected: p_continue = protected.get('continue_processing', False) if p_continue: LOG.warn( 'Release %s is `protected`, ' 'continue_processing=True. Operator must ' 'handle %s release manually.', release_name, status) result['protected'] = release_name return result else: LOG.error( 'Release %s is `protected`, ' 'continue_processing=False.', release_name) raise armada_exceptions.ProtectedReleaseException( release_name, status) else: # Purge the release with metrics.CHART_DELETE.get_context( manifest_name, chart_name): LOG.info('Purging release %s with status %s', release_name, status) chart_delete = ChartDelete(chart, release_name, self.tiller) chart_delete.delete() result['purge'] = release_name action = metrics.ChartDeployAction.INSTALL def install(): timer = int(round(deadline - time.time())) LOG.info( "Installing release %s in namespace %s, wait=%s, " "timeout=%ss", release_name, namespace, native_wait_enabled, timer) tiller_result = self.tiller.install_release( new_chart, release_name, namespace, values=yaml.safe_dump(values), wait=native_wait_enabled, timeout=timer) LOG.info('Install completed with results from Tiller: %s', tiller_result.__dict__) result['install'] = release_name deploy = install # Deploy with metrics.CHART_DEPLOY.get_context(wait_timeout, manifest_name, chart_name, action.get_label_value()): deploy() # Wait timer = int(round(deadline - time.time())) chart_wait.wait(timer) # Test just_deployed = ('install' in result) or ('upgrade' in result) last_test_passed = old_release and r.get_last_test_result(old_release) test_handler = Test(chart, release_name, self.tiller, cg_test_charts=cg_test_all_charts) run_test = test_handler.test_enabled and (just_deployed or not last_test_passed) if run_test: with metrics.CHART_TEST.get_context(test_handler.timeout, manifest_name, chart_name): self._test_chart(release_name, test_handler) return result
def execute(self, chart, cg_test_all_charts, prefix, known_releases): namespace = chart.get('namespace') release = chart.get('release') release_name = r.release_prefixer(prefix, release) LOG.info('Processing Chart, release=%s', release_name) values = chart.get('values', {}) pre_actions = {} post_actions = {} result = {} protected = chart.get('protected', {}) p_continue = protected.get('continue_processing', False) old_release = self.find_chart_release(known_releases, release_name) status = None if old_release: status = r.get_release_status(old_release) if status not in [const.STATUS_FAILED, const.STATUS_DEPLOYED]: raise armada_exceptions.UnexpectedReleaseStatusException( release_name, status) chart_wait = ChartWait( self.tiller.k8s, release_name, chart, namespace, k8s_wait_attempts=self.k8s_wait_attempts, k8s_wait_attempt_sleep=self.k8s_wait_attempt_sleep, timeout=self.timeout) native_wait_enabled = chart_wait.is_native_enabled() # Begin Chart timeout deadline deadline = time.time() + chart_wait.get_timeout() chartbuilder = ChartBuilder(chart) new_chart = chartbuilder.get_helm_chart() # Check for existing FAILED release, and purge if status == const.STATUS_FAILED: LOG.info('Purging FAILED release %s before deployment.', release_name) if protected: if p_continue: LOG.warn( 'Release %s is `protected`, ' 'continue_processing=True. Operator must ' 'handle FAILED release manually.', release_name) result['protected'] = release_name return result else: LOG.error( 'Release %s is `protected`, ' 'continue_processing=False.', release_name) raise armada_exceptions.ProtectedReleaseException( release_name) else: # Purge the release self.tiller.uninstall_release(release_name) result['purge'] = release_name # TODO(mark-burnett): It may be more robust to directly call # tiller status to decide whether to install/upgrade rather # than checking for list membership. if status == const.STATUS_DEPLOYED: # indicate to the end user what path we are taking LOG.info("Existing release %s found in namespace %s", release_name, namespace) # extract the installed chart and installed values from the # latest release so we can compare to the intended state old_chart = old_release.chart old_values_string = old_release.config.raw upgrade = chart.get('upgrade', {}) disable_hooks = upgrade.get('no_hooks', False) options = upgrade.get('options', {}) force = options.get('force', False) recreate_pods = options.get('recreate_pods', False) if upgrade: upgrade_pre = upgrade.get('pre', {}) upgrade_post = upgrade.get('post', {}) if not self.disable_update_pre and upgrade_pre: pre_actions = upgrade_pre if not self.disable_update_post and upgrade_post: LOG.warning('Post upgrade actions are ignored by Armada' 'and will not affect deployment.') post_actions = upgrade_post try: old_values = yaml.safe_load(old_values_string) except yaml.YAMLError: chart_desc = '{} (previously deployed)'.format( old_chart.metadata.name) raise armada_exceptions.\ InvalidOverrideValuesYamlException(chart_desc) LOG.info('Checking for updates to chart release inputs.') diff = self.get_diff(old_chart, old_values, new_chart, values) if not diff: LOG.info("Found no updates to chart release inputs") else: LOG.info("Found updates to chart release inputs") LOG.debug("%s", diff) result['diff'] = {chart['release']: str(diff)} # TODO(MarshM): Add tiller dry-run before upgrade and # consider deadline impacts # do actual update timer = int(round(deadline - time.time())) LOG.info( "Upgrading release %s in namespace %s, wait=%s, " "timeout=%ss", release_name, namespace, native_wait_enabled, timer) tiller_result = self.tiller.update_release( new_chart, release_name, namespace, pre_actions=pre_actions, post_actions=post_actions, disable_hooks=disable_hooks, values=yaml.safe_dump(values), wait=native_wait_enabled, timeout=timer, force=force, recreate_pods=recreate_pods) LOG.info('Upgrade completed with results from Tiller: %s', tiller_result.__dict__) result['upgrade'] = release_name else: timer = int(round(deadline - time.time())) LOG.info( "Installing release %s in namespace %s, wait=%s, " "timeout=%ss", release_name, namespace, native_wait_enabled, timer) tiller_result = self.tiller.install_release( new_chart, release_name, namespace, values=yaml.safe_dump(values), wait=native_wait_enabled, timeout=timer) LOG.info('Install completed with results from Tiller: %s', tiller_result.__dict__) result['install'] = release_name # Wait timer = int(round(deadline - time.time())) chart_wait.wait(timer) # Test just_deployed = ('install' in result) or ('upgrade' in result) last_test_passed = old_release and r.get_last_test_result(old_release) test_values = chart.get('test') test_handler = Test(release_name, self.tiller, cg_test_charts=cg_test_all_charts, test_values=test_values) run_test = test_handler.test_enabled and (just_deployed or not last_test_passed) if run_test: timer = int(round(deadline - time.time())) self._test_chart(release_name, timer, test_handler) return result
def handle(self, tiller): known_release_names = [release[0] for release in tiller.list_charts()] if self.release: if not self.ctx.obj.get('api', False): test_handler = Test(self.release, tiller, cleanup=self.cleanup) test_handler.test_release_for_success() else: client = self.ctx.obj.get('CLIENT') query = { 'tiller_host': self.tiller_host, 'tiller_port': self.tiller_port, 'tiller_namespace': self.tiller_namespace } resp = client.get_test_release(release=self.release, query=query) self.logger.info(resp.get('result')) self.logger.info(resp.get('message')) if self.file: if not self.ctx.obj.get('api', False): documents = list(yaml.safe_load_all(open(self.file).read())) armada_obj = Manifest( documents, target_manifest=self.target_manifest).get_manifest() prefix = armada_obj.get(const.KEYWORD_ARMADA).get( const.KEYWORD_PREFIX) for group in armada_obj.get(const.KEYWORD_ARMADA).get( const.KEYWORD_GROUPS): for ch in group.get(const.KEYWORD_CHARTS): chart = ch['chart'] release_name = release_prefixer( prefix, chart.get('release')) if release_name in known_release_names: test_values = chart.get('test', {}) test_handler = Test(release_name, tiller, cleanup=self.cleanup, enable_all=self.enable_all, test_values=test_values) if test_handler.test_enabled: test_handler.test_release_for_success() else: self.logger.info('Release %s not found - SKIPPING', release_name) else: client = self.ctx.obj.get('CLIENT') query = { 'tiller_host': self.tiller_host, 'tiller_port': self.tiller_port, 'tiller_namespace': self.tiller_namespace } with open(self.filename, 'r') as f: resp = client.get_test_manifest(manifest=f.read(), query=query) for test in resp.get('tests'): self.logger.info('Test State: %s', test) for item in test.get('tests').get(test): self.logger.info(item) self.logger.info(resp)
def _do_test(mock_test, mock_chartbuilder, mock_pre_flight, mock_post_flight): # Instantiate Armada object. yaml_documents = list(yaml.safe_load_all(TEST_YAML)) m_tiller = mock.MagicMock() m_tiller.list_releases.return_value = known_releases armada_obj = armada.Armada(yaml_documents, m_tiller) armada_obj.chart_deploy.get_diff = mock.Mock() chart_group = armada_obj.manifest['armada']['chart_groups'][0] charts = chart_group['chart_group'] cg_test_all_charts = chart_group.get('test_charts') mock_test_release = mock_test.return_value.test_release_for_success if test_failure_to_run: def fail(tiller, release, timeout=None, cleanup=False): status = AttrDict( **{'info': AttrDict(**{'Description': 'Failed'})}) raise tiller_exceptions.ReleaseException( release, status, 'Test') mock_test_release.side_effect = fail else: mock_test_release.return_value = test_success # Stub out irrelevant methods called by `armada.sync()`. mock_chartbuilder.get_source_path.return_value = None mock_chartbuilder.get_helm_chart.return_value = None # Simulate chart diff, upgrade should only happen if non-empty. armada_obj.chart_deploy.get_diff.return_value = diff armada_obj.sync() expected_install_release_calls = [] expected_update_release_calls = [] expected_uninstall_release_calls = [] expected_test_constructor_calls = [] for c in charts: chart = c['chart'] release = chart['release'] prefix = armada_obj.manifest['armada']['release_prefix'] release_name = release_prefixer(prefix, release) # Simplified check because the actual code uses logical-or's # multiple conditions, so this is enough. native_wait_enabled = (chart['wait'].get('native', {}).get( 'enabled', True)) if release_name not in [x.name for x in known_releases]: expected_install_release_calls.append( mock.call( mock_chartbuilder().get_helm_chart(), "{}-{}".format( armada_obj.manifest['armada'] ['release_prefix'], chart['release']), chart['namespace'], values=yaml.safe_dump(chart['values']), wait=native_wait_enabled, timeout=mock.ANY)) else: target_release = None for known_release in known_releases: if known_release.name == release_name: target_release = known_release break if target_release: status = get_release_status(target_release) if status == const.STATUS_FAILED: protected = chart.get('protected', {}) if not protected: expected_uninstall_release_calls.append( mock.call( release_name, purge=True, timeout=const.DEFAULT_DELETE_TIMEOUT)) expected_install_release_calls.append( mock.call( mock_chartbuilder().get_helm_chart(), "{}-{}".format( armada_obj.manifest['armada'] ['release_prefix'], chart['release']), chart['namespace'], values=yaml.safe_dump(chart['values']), wait=native_wait_enabled, timeout=mock.ANY)) else: p_continue = protected.get( 'continue_processing', False) if p_continue: continue else: if chart_group['sequenced']: break if status == const.STATUS_DEPLOYED: if diff: upgrade = chart.get('upgrade', {}) disable_hooks = upgrade.get('no_hooks', False) options = upgrade.get('options', {}) force = options.get('force', False) recreate_pods = options.get( 'recreate_pods', False) expected_update_release_calls.append( mock.call( mock_chartbuilder().get_helm_chart(), "{}-{}".format( armada_obj.manifest['armada'] ['release_prefix'], chart['release']), chart['namespace'], pre_actions={}, post_actions={}, disable_hooks=disable_hooks, force=force, recreate_pods=recreate_pods, values=yaml.safe_dump(chart['values']), wait=native_wait_enabled, timeout=mock.ANY)) expected_test_constructor_calls.append( mock.call( chart, release_name, m_tiller, cg_test_charts=cg_test_all_charts)) any_order = not chart_group['sequenced'] # Verify that at least 1 release is either installed or updated. self.assertTrue( len(expected_install_release_calls) >= 1 or len(expected_update_release_calls) >= 1) # Verify that the expected number of non-deployed releases are # installed with expected arguments. self.assertEqual( len(expected_install_release_calls), m_tiller.install_release.call_count) m_tiller.install_release.assert_has_calls( expected_install_release_calls, any_order=any_order) # Verify that the expected number of deployed releases are # updated with expected arguments. self.assertEqual( len(expected_update_release_calls), m_tiller.update_release.call_count) m_tiller.update_release.assert_has_calls( expected_update_release_calls, any_order=any_order) # Verify that the expected number of deployed releases are # uninstalled with expected arguments. self.assertEqual( len(expected_uninstall_release_calls), m_tiller.uninstall_release.call_count) m_tiller.uninstall_release.assert_has_calls( expected_uninstall_release_calls, any_order=any_order) # Verify that the expected number of deployed releases are # tested with expected arguments. self.assertEqual( len(expected_test_constructor_calls), mock_test.call_count) mock_test.assert_has_calls( expected_test_constructor_calls, any_order=True)
def _do_test(mock_test_release_for_success, mock_tiller, mock_chartbuilder, mock_pre_flight, mock_post_flight): # Instantiate Armada object. yaml_documents = list(yaml.safe_load_all(TEST_YAML)) armada_obj = armada.Armada(yaml_documents) armada_obj.get_diff = mock.Mock() chart_group = armada_obj.manifest['armada']['chart_groups'][0] charts = chart_group['chart_group'] cg_test_all_charts = chart_group.get('test_charts', True) m_tiller = mock_tiller.return_value m_tiller.list_charts.return_value = known_releases if test_failure_to_run: def fail(tiller, release, timeout=None, cleanup=False): status = AttrDict( **{'info': AttrDict(**{'Description': 'Failed'})}) raise tiller_exceptions.ReleaseException( release, status, 'Test') mock_test_release_for_success.side_effect = fail else: mock_test_release_for_success.return_value = test_success # Stub out irrelevant methods called by `armada.sync()`. mock_chartbuilder.get_source_path.return_value = None mock_chartbuilder.get_helm_chart.return_value = None # Simulate chart diff, upgrade should only happen if non-empty. armada_obj.get_diff.return_value = diff armada_obj.sync() expected_install_release_calls = [] expected_update_release_calls = [] expected_uninstall_release_calls = [] expected_test_release_for_success_calls = [] for c in charts: chart = c['chart'] release = chart['release'] prefix = armada_obj.manifest['armada']['release_prefix'] release_name = release_prefixer(prefix, release) # Simplified check because the actual code uses logical-or's # multiple conditions, so this is enough. this_chart_should_wait = chart['wait']['timeout'] > 0 expected_apply = True if release_name not in [x[0] for x in known_releases]: expected_install_release_calls.append( mock.call(mock_chartbuilder().get_helm_chart(), "{}-{}".format( armada_obj.manifest['armada'] ['release_prefix'], chart['release']), chart['namespace'], values=yaml.safe_dump(chart['values']), wait=this_chart_should_wait, timeout=chart['wait']['timeout'])) else: target_release = None for known_release in known_releases: if known_release[0] == release_name: target_release = known_release break if target_release: status = target_release[4] if status == const.STATUS_FAILED: protected = chart.get('protected', {}) if not protected: expected_uninstall_release_calls.append( mock.call(release_name)) expected_install_release_calls.append( mock.call( mock_chartbuilder().get_helm_chart(), "{}-{}".format( armada_obj.manifest['armada'] ['release_prefix'], chart['release']), chart['namespace'], values=yaml.safe_dump(chart['values']), wait=this_chart_should_wait, timeout=chart['wait']['timeout'])) else: p_continue = protected.get( 'continue_processing', False) if p_continue: continue else: break if status == const.STATUS_DEPLOYED: if not diff: expected_apply = False else: upgrade = chart.get('upgrade', {}) disable_hooks = upgrade.get('no_hooks', False) force = upgrade.get('force', False) recreate_pods = upgrade.get( 'recreate_pods', False) expected_update_release_calls.append( mock.call( mock_chartbuilder().get_helm_chart(), "{}-{}".format( armada_obj.manifest['armada'] ['release_prefix'], chart['release']), chart['namespace'], pre_actions={}, post_actions={}, disable_hooks=disable_hooks, force=force, recreate_pods=recreate_pods, values=yaml.safe_dump(chart['values']), wait=this_chart_should_wait, timeout=chart['wait']['timeout'])) test_chart_override = chart.get('test') # Use old default value when not using newer `test` key test_cleanup = True if test_chart_override is None: test_this_chart = cg_test_all_charts elif isinstance(test_chart_override, bool): test_this_chart = test_chart_override else: test_this_chart = test_chart_override.get('enabled', True) test_cleanup = test_chart_override.get('options', {}).get( 'cleanup', False) if test_this_chart and expected_apply: expected_test_release_for_success_calls.append( mock.call(m_tiller, release_name, timeout=mock.ANY, cleanup=test_cleanup)) # Verify that at least 1 release is either installed or updated. self.assertTrue( len(expected_install_release_calls) >= 1 or len(expected_update_release_calls) >= 1) # Verify that the expected number of non-deployed releases are # installed with expected arguments. self.assertEqual(len(expected_install_release_calls), m_tiller.install_release.call_count) m_tiller.install_release.assert_has_calls( expected_install_release_calls) # Verify that the expected number of deployed releases are # updated with expected arguments. self.assertEqual(len(expected_update_release_calls), m_tiller.update_release.call_count) m_tiller.update_release.assert_has_calls( expected_update_release_calls) # Verify that the expected number of deployed releases are # uninstalled with expected arguments. self.assertEqual(len(expected_uninstall_release_calls), m_tiller.uninstall_release.call_count) m_tiller.uninstall_release.assert_has_calls( expected_uninstall_release_calls) # Verify that the expected number of deployed releases are # tested with expected arguments. self.assertEqual(len(expected_test_release_for_success_calls), mock_test_release_for_success.call_count) mock_test_release_for_success.assert_has_calls( expected_test_release_for_success_calls)
def sync(self): ''' Synchronize Helm with the Armada Config(s) ''' if self.dry_run: LOG.info('Armada is in DRY RUN mode, no changes being made.') msg = { 'install': [], 'upgrade': [], 'diff': [], 'purge': [], 'protected': [] } # TODO: (gardlt) we need to break up this func into # a more cleaner format self.pre_flight_ops() # extract known charts on tiller right now deployed_releases, failed_releases = self._get_releases_by_status() manifest_data = self.manifest.get(const.KEYWORD_ARMADA, {}) prefix = manifest_data.get(const.KEYWORD_PREFIX) for chartgroup in manifest_data.get(const.KEYWORD_GROUPS, []): cg_name = chartgroup.get('name', '<missing name>') cg_desc = chartgroup.get('description', '<missing description>') cg_sequenced = chartgroup.get('sequenced', False) LOG.info('Processing ChartGroup: %s (%s), sequenced=%s', cg_name, cg_desc, cg_sequenced) # TODO(MarshM): Deprecate the `test_charts` key cg_test_all_charts = chartgroup.get('test_charts') if isinstance(cg_test_all_charts, bool): LOG.warn('The ChartGroup `test_charts` key is deprecated, ' 'and support for this will be removed. See the ' 'Chart `test` key for more information.') else: # This key defaults to True. Individual charts must # explicitly disable helm tests if they choose cg_test_all_charts = True ns_label_set = set() tests_to_run = [] cg_charts = chartgroup.get(const.KEYWORD_CHARTS, []) # Track largest Chart timeout to stop the ChartGroup at the end cg_max_timeout = 0 for chart_entry in cg_charts: chart = chart_entry.get('chart', {}) namespace = chart.get('namespace') release = chart.get('release') release_name = release_prefixer(prefix, release) LOG.info('Processing Chart, release=%s', release_name) values = chart.get('values', {}) pre_actions = {} post_actions = {} protected = chart.get('protected', {}) p_continue = protected.get('continue_processing', False) # Check for existing FAILED release, and purge if release_name in [rel[0] for rel in failed_releases]: LOG.info('Purging FAILED release %s before deployment.', release_name) if protected: if p_continue: LOG.warn( 'Release %s is `protected`, ' 'continue_processing=True. Operator must ' 'handle FAILED release manually.', release_name) msg['protected'].append(release_name) continue else: LOG.error( 'Release %s is `protected`, ' 'continue_processing=False.', release_name) raise armada_exceptions.ProtectedReleaseException( release_name) else: # Purge the release self.tiller.uninstall_release(release_name) msg['purge'].append(release_name) # NOTE(MarshM): Calculating `wait_timeout` is unfortunately # overly complex. The order of precedence is currently: # 1) User provided override via API/CLI (default 0 if not # provided by client/user). # 2) Chart's `data.wait.timeout`, or... # 3) Chart's `data.timeout` (deprecated). # 4) const.DEFAULT_CHART_TIMEOUT, if nothing is ever # specified, for use in waiting for final ChartGroup # health and helm tests, but ignored for the actual # install/upgrade of the Chart. # NOTE(MarshM): Not defining a timeout has a side effect of # allowing Armada to install charts with a circular # dependency defined between components. # TODO(MarshM): Deprecated, remove the following block deprecated_timeout = chart.get('timeout', None) if isinstance(deprecated_timeout, int): LOG.warn('The `timeout` key is deprecated and support ' 'for this will be removed soon. Use ' '`wait.timeout` instead.') wait_values = chart.get('wait', {}) wait_labels = wait_values.get('labels', {}) wait_timeout = self.timeout if wait_timeout <= 0: wait_timeout = wait_values.get('timeout', wait_timeout) # TODO(MarshM): Deprecated, remove the following check if wait_timeout <= 0: wait_timeout = deprecated_timeout or wait_timeout # Determine wait logic # NOTE(Dan Kim): Conditions to wait are below : # 1) set sequenced=True in chart group # 2) set force_wait param # 3) add Chart's `data.wait.timeout` # --timeout param will do not set wait=True, it just change # max timeout of chart's deployment. (default: 900) this_chart_should_wait = (cg_sequenced or self.force_wait or (bool(wait_values) and (wait_timeout > 0))) # If there is still no timeout, we need to use a default # (item 4 in note above) if wait_timeout <= 0: LOG.warn('No Chart timeout specified, using default: %ss', const.DEFAULT_CHART_TIMEOUT) wait_timeout = const.DEFAULT_CHART_TIMEOUT # Naively take largest timeout to apply at end # TODO(MarshM) better handling of timeout/timer cg_max_timeout = max(wait_timeout, cg_max_timeout) test_chart_override = chart.get('test') # Use old default value when not using newer `test` key test_cleanup = True if test_chart_override is None: test_this_chart = cg_test_all_charts elif isinstance(test_chart_override, bool): LOG.warn('Boolean value for chart `test` key is' ' deprecated and support for this will' ' be removed. Use `test.enabled` ' 'instead.') test_this_chart = test_chart_override else: # NOTE: helm tests are enabled by default test_this_chart = test_chart_override.get('enabled', True) test_cleanup = test_chart_override.get('options', {}).get( 'cleanup', False) chartbuilder = ChartBuilder(chart) new_chart = chartbuilder.get_helm_chart() # Begin Chart timeout deadline deadline = time.time() + wait_timeout # TODO(mark-burnett): It may be more robust to directly call # tiller status to decide whether to install/upgrade rather # than checking for list membership. if release_name in [rel[0] for rel in deployed_releases]: # indicate to the end user what path we are taking LOG.info("Upgrading release %s in namespace %s", release_name, namespace) # extract the installed chart and installed values from the # latest release so we can compare to the intended state old_chart, old_values_string = self.find_release_chart( deployed_releases, release_name) upgrade = chart.get('upgrade', {}) disable_hooks = upgrade.get('no_hooks', False) force = upgrade.get('force', False) recreate_pods = upgrade.get('recreate_pods', False) LOG.info("Checking Pre/Post Actions") if upgrade: upgrade_pre = upgrade.get('pre', {}) upgrade_post = upgrade.get('post', {}) if not self.disable_update_pre and upgrade_pre: pre_actions = upgrade_pre if not self.disable_update_post and upgrade_post: post_actions = upgrade_post try: old_values = yaml.safe_load(old_values_string) except yaml.YAMLError: chart_desc = '{} (previously deployed)'.format( old_chart.metadata.name) raise armada_exceptions.\ InvalidOverrideValuesYamlException(chart_desc) LOG.info('Checking for updates to chart release inputs.') diff = self.get_diff(old_chart, old_values, new_chart, values) if not diff: LOG.info("Found no updates to chart release inputs") continue LOG.info("Found updates to chart release inputs") LOG.debug("%s", diff) msg['diff'].append({chart['release']: str(diff)}) # TODO(MarshM): Add tiller dry-run before upgrade and # consider deadline impacts # do actual update timer = int(round(deadline - time.time())) LOG.info('Beginning Upgrade, wait=%s, timeout=%ss', this_chart_should_wait, timer) tiller_result = self.tiller.update_release( new_chart, release_name, namespace, pre_actions=pre_actions, post_actions=post_actions, disable_hooks=disable_hooks, values=yaml.safe_dump(values), wait=this_chart_should_wait, timeout=timer, force=force, recreate_pods=recreate_pods) if this_chart_should_wait: self._wait_until_ready(release_name, wait_labels, namespace, timer) # Track namespace+labels touched by upgrade ns_label_set.add((namespace, tuple(wait_labels.items()))) LOG.info('Upgrade completed with results from Tiller: %s', tiller_result.__dict__) msg['upgrade'].append(release_name) # process install else: LOG.info("Installing release %s in namespace %s", release_name, namespace) timer = int(round(deadline - time.time())) LOG.info('Beginning Install, wait=%s, timeout=%ss', this_chart_should_wait, timer) tiller_result = self.tiller.install_release( new_chart, release_name, namespace, values=yaml.safe_dump(values), wait=this_chart_should_wait, timeout=timer) if this_chart_should_wait: self._wait_until_ready(release_name, wait_labels, namespace, timer) # Track namespace+labels touched by install ns_label_set.add((namespace, tuple(wait_labels.items()))) LOG.info('Install completed with results from Tiller: %s', tiller_result.__dict__) msg['install'].append(release_name) # Keeping track of time remaining timer = int(round(deadline - time.time())) test_chart_args = (release_name, timer, test_cleanup) if test_this_chart: # Sequenced ChartGroup should run tests after each Chart if cg_sequenced: LOG.info( 'Running sequenced test, timeout remaining: ' '%ss.', timer) self._test_chart(*test_chart_args) # Un-sequenced ChartGroup should run tests at the end else: tests_to_run.append( functools.partial(self._test_chart, *test_chart_args)) # End of Charts in ChartGroup LOG.info('All Charts applied in ChartGroup %s.', cg_name) # After all Charts are applied, we should wait for the entire # ChartGroup to become healthy by looking at the namespaces seen # TODO(MarshM): Need to determine a better timeout # (not cg_max_timeout) if cg_max_timeout <= 0: cg_max_timeout = const.DEFAULT_CHART_TIMEOUT deadline = time.time() + cg_max_timeout for (ns, labels) in ns_label_set: labels_dict = dict(labels) timer = int(round(deadline - time.time())) LOG.info( 'Final ChartGroup wait for healthy namespace=%s, ' 'labels=(%s), timeout remaining: %ss.', ns, labels_dict, timer) if timer <= 0: reason = ('Timeout expired waiting on namespace: %s, ' 'labels: (%s)' % (ns, labels_dict)) LOG.error(reason) raise armada_exceptions.ArmadaTimeoutException(reason) self._wait_until_ready(release_name=None, wait_labels=labels_dict, namespace=ns, timeout=timer) # After entire ChartGroup is healthy, run any pending tests for callback in tests_to_run: callback() self.post_flight_ops() if self.enable_chart_cleanup: self._chart_cleanup( prefix, self.manifest[const.KEYWORD_ARMADA][const.KEYWORD_GROUPS], msg) LOG.info('Done applying manifest.') return msg