def test_dependency_resolution(self): # Main chart directory and files. chart_dir = self.useFixture(fixtures.TempDir()) self.addCleanup(shutil.rmtree, chart_dir.path) self._write_temporary_file_contents(chart_dir.path, 'Chart.yaml', self.chart_yaml) chart_doc = yaml.safe_load(self.chart_doc_yaml) chart_doc['data']['source_dir'] = (chart_dir.path, '') # Dependency chart directory and files. dep_chart_dir = self.useFixture(fixtures.TempDir()) self.addCleanup(shutil.rmtree, dep_chart_dir.path) self._write_temporary_file_contents(dep_chart_dir.path, 'Chart.yaml', self.dep_chart_yaml) dep_chart_doc = yaml.safe_load(self.dep_chart_doc_yaml) dep_chart_doc['data']['source_dir'] = (dep_chart_dir.path, '') # Add dependency chart_doc['data']['dependencies'] = [dep_chart_doc] # Mock helm cli call helm_mock = mock.Mock() helm_mock.show_chart.return_value = yaml.safe_load(self.dep_chart_yaml) ChartBuilder.from_chart_doc(chart_doc, helm_mock) expected_symlink_path = Path( chart_dir.path).joinpath('charts').joinpath('dependency-chart') self.assertTrue(expected_symlink_path.is_symlink()) self.assertEqual(dep_chart_dir.path, str(expected_symlink_path.resolve()))
def test_get_files(self): """Validates that ``get_files()`` ignores 'Chart.yaml', 'values.yaml' and 'templates' subfolder and all the files contained therein. """ # Create a temporary directory that represents a chart source directory # with various files, including 'Chart.yaml' and 'values.yaml' which # should be ignored by `get_files()`. chart_dir = self.useFixture(fixtures.TempDir()) self.addCleanup(shutil.rmtree, chart_dir.path) for filename in ['foo', 'bar', 'Chart.yaml', 'values.yaml']: self._write_temporary_file_contents(chart_dir.path, filename, "") # Create a template directory -- 'templates' -- nested inside the chart # directory which should also be ignored. templates_subdir = self._make_temporary_subdirectory( chart_dir.path, 'templates') for filename in ['template%d' % x for x in range(3)]: self._write_temporary_file_contents(templates_subdir, filename, "") chartbuilder = ChartBuilder(self._get_test_chart(chart_dir)) expected_files = ('[type_url: "%s"\n, type_url: "%s"\n]' % ('./bar', './foo')) # Validate that only 'foo' and 'bar' are returned. actual_files = sorted(chartbuilder.get_files(), key=lambda x: x.type_url) self.assertEqual(expected_files, repr(actual_files).strip())
def test_get_basic_helm_chart(self): # Before ChartBuilder is executed the `source_dir` points to a # directory that was either clone or unpacked from a tarball... pretend # that that logic has already been performed. chart_dir = self.useFixture(fixtures.TempDir()) self.addCleanup(shutil.rmtree, chart_dir.path) self._write_temporary_file_contents(chart_dir.path, 'Chart.yaml', self.chart_yaml) ch = yaml.safe_load(self.chart_stream) ch['data']['source_dir'] = (chart_dir.path, '') test_chart = ch chartbuilder = ChartBuilder(test_chart) helm_chart = chartbuilder.get_helm_chart() expected = inspect.cleandoc(""" metadata { name: "hello-world-chart" version: "0.1.0" description: "A sample Helm chart for Kubernetes" } values { } """).strip() self.assertIsInstance(helm_chart, Chart) self.assertTrue(hasattr(helm_chart, 'metadata')) self.assertTrue(hasattr(helm_chart, 'values')) self.assertEqual(expected, repr(helm_chart).strip())
def test_get_helm_chart_with_files(self): # Create a chart directory with some test files. chart_dir = self.useFixture(fixtures.TempDir()) self.addCleanup(shutil.rmtree, chart_dir.path) # Chart.yaml is mandatory for `ChartBuilder.get_metadata`. self._write_temporary_file_contents(chart_dir.path, 'Chart.yaml', self.chart_yaml) self._write_temporary_file_contents(chart_dir.path, 'foo', "foobar") self._write_temporary_file_contents(chart_dir.path, 'bar', "bazqux") # Also create a nested directory and verify that files from it are also # added. nested_dir = self._make_temporary_subdirectory(chart_dir.path, 'nested') self._write_temporary_file_contents(nested_dir, 'nested0', "random") ch = yaml.safe_load(self.chart_stream) ch['data']['source_dir'] = (chart_dir.path, '') test_chart = ch chartbuilder = ChartBuilder(test_chart) helm_chart = chartbuilder.get_helm_chart() expected_files = ('[type_url: "%s"\nvalue: "bazqux"\n, ' 'type_url: "%s"\nvalue: "foobar"\n, ' 'type_url: "%s"\nvalue: "random"\n]' % ('./bar', './foo', 'nested/nested0')) self.assertIsInstance(helm_chart, Chart) self.assertTrue(hasattr(helm_chart, 'metadata')) self.assertTrue(hasattr(helm_chart, 'values')) self.assertTrue(hasattr(helm_chart, 'files')) actual_files = sorted(helm_chart.files, key=lambda x: x.value) self.assertEqual(expected_files, repr(actual_files).strip())
def test_dump(self): # Validate base case. chart_dir = self.useFixture(fixtures.TempDir()) self.addCleanup(shutil.rmtree, chart_dir.path) self._write_temporary_file_contents(chart_dir.path, 'Chart.yaml', self.chart_yaml) ch = yaml.safe_load(self.chart_stream) ch['data']['source_dir'] = (chart_dir.path, '') test_chart = ch chartbuilder = ChartBuilder.from_chart_doc(test_chart) self.assertRegex( repr(chartbuilder.dump()), 'hello-world-chart.*A sample Helm chart for Kubernetes.*') # Validate recursive case (with dependencies). dep_chart_dir = self.useFixture(fixtures.TempDir()) self.addCleanup(shutil.rmtree, dep_chart_dir.path) self._write_temporary_file_contents(dep_chart_dir.path, 'Chart.yaml', self.dependency_chart_yaml) dep_ch = yaml.safe_load(self.dependency_chart_stream) dep_ch['data']['source_dir'] = (dep_chart_dir.path, '') dependency_chart = dep_ch test_chart['data']['dependencies'] = [dependency_chart] chartbuilder = ChartBuilder.from_chart_doc(test_chart) re = inspect.cleandoc(""" hello-world-chart.*A sample Helm chart for Kubernetes.* dependency-chart.*Another sample Helm chart for Kubernetes.* """).replace('\n', '').strip() self.assertRegex(repr(chartbuilder.dump()), re)
def test_get_helm_chart_includes_only_relevant_files(self): chart_dir = self.useFixture(fixtures.TempDir()) self.addCleanup(shutil.rmtree, chart_dir.path) templates_subdir = self._make_temporary_subdirectory( chart_dir.path, 'templates') charts_subdir = self._make_temporary_subdirectory( chart_dir.path, 'charts') templates_nested_subdir = self._make_temporary_subdirectory( templates_subdir, 'bin') charts_nested_subdir = self._make_temporary_subdirectory( charts_subdir, 'extra') self._write_temporary_file_contents(chart_dir.path, 'Chart.yaml', self.chart_yaml) self._write_temporary_file_contents(chart_dir.path, 'foo', "foobar") self._write_temporary_file_contents(chart_dir.path, 'bar', "bazqux") # Files to ignore within top-level directory. files_to_ignore = ['Chart.yaml', 'values.yaml', 'values.toml'] for file in files_to_ignore: self._write_temporary_file_contents(chart_dir.path, file, "") file_to_ignore = 'file_to_ignore' # Files to ignore within templates/ subdirectory. self._write_temporary_file_contents( templates_subdir, file_to_ignore, "") # Files to ignore within charts/ subdirectory. self._write_temporary_file_contents( charts_subdir, file_to_ignore, "") # Files to ignore within templates/bin subdirectory. self._write_temporary_file_contents( templates_nested_subdir, file_to_ignore, "") # Files to ignore within charts/extra subdirectory. self._write_temporary_file_contents( charts_nested_subdir, file_to_ignore, "") # Files to **include** within charts/ subdirectory. self._write_temporary_file_contents( charts_subdir, '.prov', "xyzzy") ch = yaml.safe_load(self.chart_stream)['chart'] ch['source_dir'] = (chart_dir.path, '') test_chart = dotify(ch) chartbuilder = ChartBuilder(test_chart) helm_chart = chartbuilder.get_helm_chart() expected_files = ('[type_url: "%s"\nvalue: "bazqux"\n, ' 'type_url: "%s"\nvalue: "foobar"\n, ' 'type_url: "%s"\nvalue: "xyzzy"\n]' % ('./bar', './foo', 'charts/.prov')) # Validate that only relevant files are included, that the ignored # files are present. self.assertIsInstance(helm_chart, Chart) self.assertTrue(hasattr(helm_chart, 'metadata')) self.assertTrue(hasattr(helm_chart, 'values')) self.assertTrue(hasattr(helm_chart, 'files')) actual_files = sorted(helm_chart.files, key=lambda x: x.value) self.assertEqual(expected_files, repr(actual_files).strip())
def test_get_files_with_unicode_characters(self): chart_dir = self.useFixture(fixtures.TempDir()) self.addCleanup(shutil.rmtree, chart_dir.path) for filename in ['foo', 'bar', 'Chart.yaml', 'values.yaml']: self._write_temporary_file_contents( chart_dir.path, filename, "DIRC^@^@^@^B^@^@^@×Z®<86>F.1") chartbuilder = ChartBuilder(self._get_test_chart(chart_dir)) chartbuilder.get_files()
def test_chart_source_clone(self, mock_os, mock_dot): from supermutes.dot import dotify import yaml mock_dot.dotify.return_value = dotify(yaml.load(self.chart_stream)) mock_os.path.join.return_value = self.chart_stream ChartBuilder.source_clone = mock.Mock(return_value='path') chartbuilder = ChartBuilder(self.chart_stream) resp = chartbuilder.get_metadata() self.assertIsNotNone(resp) self.assertIsInstance(resp, basestring)
def test_source_clone(self): # Create a temporary directory with Chart.yaml that contains data # from ``self.chart_yaml``. chart_dir = self.useFixture(fixtures.TempDir()) self.addCleanup(shutil.rmtree, chart_dir.path) self._write_temporary_file_contents(chart_dir.path, 'Chart.yaml', self.chart_yaml) chartbuilder = ChartBuilder(self._get_test_chart(chart_dir)) # Validate response type is :class:`hapi.chart.metadata_pb2.Metadata` resp = chartbuilder.get_metadata() self.assertIsInstance(resp, Metadata)
def test_get_files_fails_once_to_read_binary_file_passes(self): chart_dir = self.useFixture(fixtures.TempDir()) self.addCleanup(shutil.rmtree, chart_dir.path) files = ['foo', 'bar'] for filename in files: self._write_temporary_file_contents( chart_dir.path, filename, "DIRC^@^@^@^B^@^@^@×Z®<86>F.1") chartbuilder = ChartBuilder(self._get_test_chart(chart_dir)) side_effects = [self.exc_to_raise, "", ""] with mock.patch("builtins.open", mock.mock_open(read_data="")) \ as mock_file: mock_file.return_value.read.side_effect = side_effects chartbuilder.get_files()
def test_get_metadata_with_incorrect_file_invalid(self): chart_dir = self.useFixture(fixtures.TempDir()) self.addCleanup(shutil.rmtree, chart_dir.path) chartbuilder = ChartBuilder(self._get_test_chart(chart_dir)) self.assertRaises(chartbuilder_exceptions.MetadataLoadException, chartbuilder.get_metadata)
def test_chartbuilder_source_clone(self): chart = dotify(self.chart_stream) ChartBuilder.source_clone = mock.Mock(return_value='path') chartbuilder = ChartBuilder(chart) resp = getattr(chartbuilder, 'source_directory', None) self.assertIsNotNone(resp) self.assertIsInstance(resp, basestring)
def test_get_helm_chart_success(self): chart_dir = self.useFixture(fixtures.TempDir()) self.addCleanup(shutil.rmtree, chart_dir.path) helm_mock = mock.Mock() helm_mock.upgrade_release.return_value = {"chart": mock.sentinel.chart} chartbuilder = ChartBuilder.from_chart_doc( self._get_test_chart(chart_dir.path), helm_mock) release_id = mock.Mock() values = mock.Mock() actual_chart = chartbuilder.get_helm_chart(release_id, values) self.assertIs(mock.sentinel.chart, actual_chart)
def test_get_helm_chart_with_values(self): chart_dir = self.useFixture(fixtures.TempDir()) self.addCleanup(shutil.rmtree, chart_dir.path) self._write_temporary_file_contents(chart_dir.path, 'Chart.yaml', self.chart_yaml) self._write_temporary_file_contents(chart_dir.path, 'values.yaml', self.chart_value) ch = yaml.safe_load(self.chart_stream) ch['data']['source_dir'] = (chart_dir.path, '') test_chart = ch chartbuilder = ChartBuilder(test_chart) helm_chart = chartbuilder.get_helm_chart() self.assertIsInstance(helm_chart, Chart) self.assertTrue(hasattr(helm_chart, 'metadata')) self.assertTrue(hasattr(helm_chart, 'values')) self.assertTrue(hasattr(helm_chart.values, 'raw')) self.assertEqual(self.chart_value, helm_chart.values.raw)
def test_get_helm_chart_fail(self): chart_dir = self.useFixture(fixtures.TempDir()) self.addCleanup(shutil.rmtree, chart_dir.path) helm_mock = mock.Mock() helm_mock.upgrade_release.side_effect = Exception() chartbuilder = ChartBuilder.from_chart_doc( self._get_test_chart(chart_dir.path), helm_mock) def test(): release_id = mock.Mock() values = mock.Mock() chartbuilder.get_helm_chart(release_id, values) self.assertRaises(chartbuilder_exceptions.HelmChartBuildException, test)
def test_get_files_always_fails_to_read_binary_file_raises_exc(self): chart_dir = self.useFixture(fixtures.TempDir()) self.addCleanup(shutil.rmtree, chart_dir.path) for filename in ['foo', 'bar', 'Chart.yaml', 'values.yaml']: self._write_temporary_file_contents( chart_dir.path, filename, "DIRC^@^@^@^B^@^@^@×Z®<86>F.1") chartbuilder = ChartBuilder(self._get_test_chart(chart_dir)) # Confirm it failed for both encodings. error_re = (r'.*A str exception occurred while trying to read file:' r'.*Details:\n.*\(encoding=utf-8\).*\n\(encoding=latin1\)') with mock.patch("builtins.open", mock.mock_open(read_data="")) \ as mock_file: mock_file.return_value.read.side_effect = self.exc_to_raise self.assertRaisesRegexp(chartbuilder_exceptions.FilesLoadException, error_re, chartbuilder.get_files)
def test_get_helm_chart_with_dependencies(self): # Main chart directory and files. chart_dir = self.useFixture(fixtures.TempDir()) self.addCleanup(shutil.rmtree, chart_dir.path) self._write_temporary_file_contents(chart_dir.path, 'Chart.yaml', self.chart_yaml) ch = yaml.safe_load(self.chart_stream) ch['data']['source_dir'] = (chart_dir.path, '') # Dependency chart directory and files. dep_chart_dir = self.useFixture(fixtures.TempDir()) self.addCleanup(shutil.rmtree, dep_chart_dir.path) self._write_temporary_file_contents(dep_chart_dir.path, 'Chart.yaml', self.dependency_chart_yaml) dep_ch = yaml.safe_load(self.dependency_chart_stream) dep_ch['data']['source_dir'] = (dep_chart_dir.path, '') main_chart = ch dependency_chart = dep_ch main_chart['data']['dependencies'] = [dependency_chart] chartbuilder = ChartBuilder(main_chart) helm_chart = chartbuilder.get_helm_chart() expected_dependency = inspect.cleandoc(""" metadata { name: "dependency-chart" version: "0.1.0" description: "Another sample Helm chart for Kubernetes" } values { } """).strip() expected = inspect.cleandoc(""" metadata { name: "hello-world-chart" version: "0.1.0" description: "A sample Helm chart for Kubernetes" } dependencies { metadata { name: "dependency-chart" version: "0.1.0" description: "Another sample Helm chart for Kubernetes" } values { } } values { } """).strip() # Validate the main chart. self.assertIsInstance(helm_chart, Chart) self.assertTrue(hasattr(helm_chart, 'metadata')) self.assertTrue(hasattr(helm_chart, 'values')) self.assertEqual(expected, repr(helm_chart).strip()) # Validate the dependency chart. self.assertTrue(hasattr(helm_chart, 'dependencies')) self.assertEqual(1, len(helm_chart.dependencies)) dep_helm_chart = helm_chart.dependencies[0] self.assertIsInstance(dep_helm_chart, Chart) self.assertTrue(hasattr(dep_helm_chart, 'metadata')) self.assertTrue(hasattr(dep_helm_chart, 'values')) self.assertEqual(expected_dependency, repr(dep_helm_chart).strip())
def _execute(self, ch, cg_test_all_charts, prefix, known_releases): manifest_name = self.manifest['metadata']['name'] chart = ch[const.KEYWORD_DATA] chart_name = ch['metadata']['name'] namespace = chart.get('namespace') release = chart.get('release') release_name = r.release_prefixer(prefix, release) LOG.info('Processing Chart, release=%s', release_name) result = {} chart_wait = ChartWait( self.tiller.k8s, release_name, ch, namespace, k8s_wait_attempts=self.k8s_wait_attempts, k8s_wait_attempt_sleep=self.k8s_wait_attempt_sleep, timeout=self.timeout) wait_timeout = chart_wait.get_timeout() # Begin Chart timeout deadline deadline = time.time() + wait_timeout old_release = self.find_chart_release(known_releases, release_name) action = metrics.ChartDeployAction.NOOP def noop(): pass deploy = noop # Resolve action values = chart.get('values', {}) pre_actions = {} post_actions = {} status = None if old_release: status = r.get_release_status(old_release) native_wait_enabled = chart_wait.is_native_enabled() chartbuilder = ChartBuilder.from_chart_doc(ch) new_chart = chartbuilder.get_helm_chart() if status == const.STATUS_DEPLOYED: # indicate to the end user what path we are taking LOG.info("Existing release %s found in namespace %s", release_name, namespace) # extract the installed chart and installed values from the # latest release so we can compare to the intended state old_chart = old_release.chart old_values_string = old_release.config.raw upgrade = chart.get('upgrade', {}) options = upgrade.get('options', {}) # TODO: Remove when v1 doc support is removed. schema_info = get_schema_info(ch['schema']) if schema_info.version < 2: no_hooks_location = upgrade else: no_hooks_location = options disable_hooks = no_hooks_location.get('no_hooks', False) force = options.get('force', False) recreate_pods = options.get('recreate_pods', False) if upgrade: upgrade_pre = upgrade.get('pre', {}) upgrade_post = upgrade.get('post', {}) if not self.disable_update_pre and upgrade_pre: pre_actions = upgrade_pre if not self.disable_update_post and upgrade_post: LOG.warning('Post upgrade actions are ignored by Armada' 'and will not affect deployment.') post_actions = upgrade_post try: old_values = yaml.safe_load(old_values_string) except yaml.YAMLError: chart_desc = '{} (previously deployed)'.format( old_chart.metadata.name) raise armada_exceptions.\ InvalidOverrideValuesYamlException(chart_desc) LOG.info('Checking for updates to chart release inputs.') diff = self.get_diff(old_chart, old_values, new_chart, values) if not diff: LOG.info("Found no updates to chart release inputs") else: action = metrics.ChartDeployAction.UPGRADE LOG.info("Found updates to chart release inputs") LOG.debug("%s", diff) result['diff'] = {chart['release']: str(diff)} def upgrade(): # do actual update timer = int(round(deadline - time.time())) LOG.info( "Upgrading release %s in namespace %s, wait=%s, " "timeout=%ss", release_name, namespace, native_wait_enabled, timer) tiller_result = self.tiller.update_release( new_chart, release_name, namespace, pre_actions=pre_actions, post_actions=post_actions, disable_hooks=disable_hooks, values=yaml.safe_dump(values), wait=native_wait_enabled, timeout=timer, force=force, recreate_pods=recreate_pods) LOG.info('Upgrade completed with results from Tiller: %s', tiller_result.__dict__) result['upgrade'] = release_name deploy = upgrade else: # Check for release with status other than DEPLOYED if status: if status != const.STATUS_FAILED: LOG.warn( 'Unexpected release status encountered ' 'release=%s, status=%s', release_name, status) # Make best effort to determine whether a deployment is # likely pending, by checking if the last deployment # was started within the timeout window of the chart. last_deployment_age = r.get_last_deployment_age( old_release) likely_pending = last_deployment_age <= wait_timeout if likely_pending: # Give up if a deployment is likely pending, we do not # want to have multiple operations going on for the # same release at the same time. raise armada_exceptions.\ DeploymentLikelyPendingException( release_name, status, last_deployment_age, wait_timeout) else: # Release is likely stuck in an unintended (by tiller) # state. Log and continue on with remediation steps # below. LOG.info( 'Old release %s likely stuck in status %s, ' '(last deployment age=%ss) >= ' '(chart wait timeout=%ss)', release, status, last_deployment_age, wait_timeout) protected = chart.get('protected', {}) if protected: p_continue = protected.get('continue_processing', False) if p_continue: LOG.warn( 'Release %s is `protected`, ' 'continue_processing=True. Operator must ' 'handle %s release manually.', release_name, status) result['protected'] = release_name return result else: LOG.error( 'Release %s is `protected`, ' 'continue_processing=False.', release_name) raise armada_exceptions.ProtectedReleaseException( release_name, status) else: # Purge the release with metrics.CHART_DELETE.get_context( manifest_name, chart_name): LOG.info('Purging release %s with status %s', release_name, status) chart_delete = ChartDelete(chart, release_name, self.tiller) chart_delete.delete() result['purge'] = release_name action = metrics.ChartDeployAction.INSTALL def install(): timer = int(round(deadline - time.time())) LOG.info( "Installing release %s in namespace %s, wait=%s, " "timeout=%ss", release_name, namespace, native_wait_enabled, timer) tiller_result = self.tiller.install_release( new_chart, release_name, namespace, values=yaml.safe_dump(values), wait=native_wait_enabled, timeout=timer) LOG.info('Install completed with results from Tiller: %s', tiller_result.__dict__) result['install'] = release_name deploy = install # Deploy with metrics.CHART_DEPLOY.get_context(wait_timeout, manifest_name, chart_name, action.get_label_value()): deploy() # Wait timer = int(round(deadline - time.time())) chart_wait.wait(timer) # Test just_deployed = ('install' in result) or ('upgrade' in result) last_test_passed = old_release and r.get_last_test_result(old_release) test_handler = Test(chart, release_name, self.tiller, cg_test_charts=cg_test_all_charts) run_test = test_handler.test_enabled and (just_deployed or not last_test_passed) if run_test: with metrics.CHART_TEST.get_context(test_handler.timeout, manifest_name, chart_name): self._test_chart(release_name, test_handler) return result
def _execute(self, ch, cg_test_all_charts, prefix): manifest_name = self.manifest['metadata']['name'] chart = ch[const.KEYWORD_DATA] chart_name = ch['metadata']['name'] namespace = chart.get('namespace') release = chart.get('release') release_name = r.release_prefixer(prefix, release) release_id = helm.HelmReleaseId(namespace, release_name) source_dir = chart['source_dir'] source_directory = os.path.join(*source_dir) LOG.info('Processing Chart, release=%s', release_id) result = {} chart_wait = ChartWait( self.helm.k8s, release_id, ch, k8s_wait_attempts=self.k8s_wait_attempts, k8s_wait_attempt_sleep=self.k8s_wait_attempt_sleep, timeout=self.timeout) wait_timeout = chart_wait.get_timeout() # Begin Chart timeout deadline deadline = time.time() + wait_timeout old_release = self.helm.release_metadata(release_id) action = metrics.ChartDeployAction.NOOP def noop(): pass deploy = noop # Resolve action values = chart.get('values', {}) pre_actions = {} status = None if old_release: status = r.get_release_status(old_release) native_wait_enabled = chart_wait.is_native_enabled() chartbuilder = ChartBuilder.from_chart_doc(ch, self.helm) if status == helm.STATUS_DEPLOYED: # indicate to the end user what path we are taking LOG.info("Existing release %s found", release_id) # extract the installed chart and installed values from the # latest release so we can compare to the intended state old_chart = old_release['chart'] old_values = old_release.get('config', {}) upgrade = chart.get('upgrade', {}) options = upgrade.get('options', {}) # TODO: Remove when v1 doc support is removed. schema_info = get_schema_info(ch['schema']) if schema_info.version < 2: no_hooks_location = upgrade else: no_hooks_location = options disable_hooks = no_hooks_location.get('no_hooks', False) force = options.get('force', False) if upgrade: upgrade_pre = upgrade.get('pre', {}) upgrade_post = upgrade.get('post', {}) if not self.disable_update_pre and upgrade_pre: pre_actions = upgrade_pre if not self.disable_update_post and upgrade_post: LOG.warning('Post upgrade actions are ignored by Armada' 'and will not affect deployment.') LOG.info('Checking for updates to chart release inputs.') new_chart = chartbuilder.get_helm_chart(release_id, values) diff = self.get_diff(old_chart, old_values, new_chart, values) if not diff: LOG.info("Found no updates to chart release inputs") else: action = metrics.ChartDeployAction.UPGRADE LOG.info("Found updates to chart release inputs") def upgrade(): # do actual update timer = int(round(deadline - time.time())) PreUpdateActions(self.helm.k8s).execute( pre_actions, release, namespace, chart, disable_hooks, values, timer) LOG.info("Upgrading release=%s, wait=%s, " "timeout=%ss", release_id, native_wait_enabled, timer) self.helm.upgrade_release(source_directory, release_id, disable_hooks=disable_hooks, values=values, wait=native_wait_enabled, timeout=timer, force=force) LOG.info('Upgrade completed') result['upgrade'] = release_id deploy = upgrade else: def install(): timer = int(round(deadline - time.time())) LOG.info("Installing release=%s, wait=%s, " "timeout=%ss", release_id, native_wait_enabled, timer) self.helm.install_release(source_directory, release_id, values=values, wait=native_wait_enabled, timeout=timer) LOG.info('Install completed') result['install'] = release_id # Check for release with status other than DEPLOYED if status: if status != helm.STATUS_FAILED: LOG.warn( 'Unexpected release status encountered ' 'release=%s, status=%s', release_id, status) # Make best effort to determine whether a deployment is # likely pending, by checking if the last deployment # was started within the timeout window of the chart. last_deployment_age = r.get_last_deployment_age( old_release) likely_pending = last_deployment_age <= wait_timeout if likely_pending: # We don't take any deploy action and wait for the # to get deployed. deploy = noop deadline = deadline - last_deployment_age else: # Release is likely stuck in an unintended # state. Log and continue on with remediation steps # below. LOG.info( 'Old release %s likely stuck in status %s, ' '(last deployment age=%ss) >= ' '(chart wait timeout=%ss)', release, status, last_deployment_age, wait_timeout) res = self.purge_release(chart, release_id, status, manifest_name, chart_name, result) if isinstance(res, dict): if 'protected' in res: return res action = metrics.ChartDeployAction.INSTALL deploy = install else: # The chart is in Failed state, hence we purge # the chart and attempt to install it again. res = self.purge_release(chart, release_id, status, manifest_name, chart_name, result) if isinstance(res, dict): if 'protected' in res: return res action = metrics.ChartDeployAction.INSTALL deploy = install if status is None: action = metrics.ChartDeployAction.INSTALL deploy = install # Deploy with metrics.CHART_DEPLOY.get_context(wait_timeout, manifest_name, chart_name, action.get_label_value()): deploy() # Wait timer = int(round(deadline - time.time())) chart_wait.wait(timer) # Test just_deployed = ('install' in result) or ('upgrade' in result) last_test_passed = old_release and r.get_last_test_result(old_release) test_handler = Test(chart, release_id, self.helm, cg_test_charts=cg_test_all_charts) run_test = test_handler.test_enabled and (just_deployed or not last_test_passed) if run_test: with metrics.CHART_TEST.get_context(test_handler.timeout, manifest_name, chart_name): self._test_chart(test_handler) return result
def sync(self): ''' Synchronize Helm with the Armada Config(s) ''' msg = {'install': [], 'upgrade': [], 'diff': []} # TODO: (gardlt) we need to break up this func into # a more cleaner format self.pre_flight_ops() # extract known charts on tiller right now known_releases = self.tiller.list_charts() manifest_data = self.manifest.get(KEYWORD_ARMADA, {}) prefix = manifest_data.get(KEYWORD_PREFIX, '') for chartgroup in manifest_data.get(KEYWORD_GROUPS, []): cg_name = chartgroup.get('name', '<missing name>') cg_desc = chartgroup.get('description', '<missing description>') LOG.info('Processing ChartGroup: %s (%s)', cg_name, cg_desc) cg_sequenced = chartgroup.get('sequenced', False) cg_test_all_charts = chartgroup.get('test_charts', False) namespaces_seen = set() tests_to_run = [] cg_charts = chartgroup.get(KEYWORD_CHARTS, []) # Track largest Chart timeout to stop the ChartGroup at the end cg_max_timeout = 0 for chart_entry in cg_charts: chart = chart_entry.get('chart', {}) namespace = chart.get('namespace') release = chart.get('release') values = chart.get('values', {}) pre_actions = {} post_actions = {} wait_timeout = self.timeout wait_labels = {} release_name = release_prefix(prefix, release) # Retrieve appropriate timeout value if wait_timeout <= 0: # TODO(MarshM): chart's `data.timeout` should be deprecated chart_timeout = chart.get('timeout', 0) # Favor data.wait.timeout over data.timeout, until removed wait_values = chart.get('wait', {}) wait_timeout = wait_values.get('timeout', chart_timeout) wait_labels = wait_values.get('labels', {}) this_chart_should_wait = (cg_sequenced or self.force_wait or wait_timeout > 0 or len(wait_labels) > 0) if this_chart_should_wait and wait_timeout <= 0: LOG.warn('No Chart timeout specified, using default: %ss', DEFAULT_CHART_TIMEOUT) wait_timeout = DEFAULT_CHART_TIMEOUT # Track namespaces + labels touched namespaces_seen.add((namespace, tuple(wait_labels.items()))) # Naively take largest timeout to apply at end # TODO(MarshM) better handling of timeout/timer cg_max_timeout = max(wait_timeout, cg_max_timeout) # Chart test policy can override ChartGroup, if specified test_this_chart = chart.get('test', cg_test_all_charts) chartbuilder = ChartBuilder(chart) protoc_chart = chartbuilder.get_helm_chart() deployed_releases = [x[0] for x in known_releases] # Begin Chart timeout deadline deadline = time.time() + wait_timeout # TODO(mark-burnett): It may be more robust to directly call # tiller status to decide whether to install/upgrade rather # than checking for list membership. if release_name in deployed_releases: # indicate to the end user what path we are taking LOG.info("Upgrading release %s in namespace %s", release_name, namespace) # extract the installed chart and installed values from the # latest release so we can compare to the intended state apply_chart, apply_values = self.find_release_chart( known_releases, release_name) upgrade = chart.get('upgrade', {}) disable_hooks = upgrade.get('no_hooks', False) LOG.info("Checking Pre/Post Actions") if upgrade: upgrade_pre = upgrade.get('pre', {}) upgrade_post = upgrade.get('post', {}) if not self.disable_update_pre and upgrade_pre: pre_actions = upgrade_pre if not self.disable_update_post and upgrade_post: post_actions = upgrade_post # Show delta for both the chart templates and the chart # values # TODO(alanmeadows) account for .files differences # once we support those LOG.info('Checking upgrade chart diffs.') upgrade_diff = self.show_diff(chart, apply_chart, apply_values, chartbuilder.dump(), values, msg) if not upgrade_diff: LOG.info("There are no updates found in this chart") continue # TODO(MarshM): Add tiller dry-run before upgrade and # consider deadline impacts # do actual update timer = int(round(deadline - time.time())) LOG.info('Beginning Upgrade, wait=%s, timeout=%ss', this_chart_should_wait, timer) tiller_result = self.tiller.update_release( protoc_chart, release_name, namespace, pre_actions=pre_actions, post_actions=post_actions, dry_run=self.dry_run, disable_hooks=disable_hooks, values=yaml.safe_dump(values), wait=this_chart_should_wait, timeout=timer) if this_chart_should_wait: self.tiller.k8s.wait_until_ready( release=release_name, labels=wait_labels, namespace=namespace, k8s_wait_attempts=self.k8s_wait_attempts, k8s_wait_attempt_sleep=self.k8s_wait_attempt_sleep, timeout=timer) LOG.info('Upgrade completed with results from Tiller: %s', tiller_result.__dict__) msg['upgrade'].append(release_name) # process install else: LOG.info("Installing release %s in namespace %s", release_name, namespace) timer = int(round(deadline - time.time())) LOG.info('Beginning Install, wait=%s, timeout=%ss', this_chart_should_wait, timer) tiller_result = self.tiller.install_release( protoc_chart, release_name, namespace, dry_run=self.dry_run, values=yaml.safe_dump(values), wait=this_chart_should_wait, timeout=timer) if this_chart_should_wait: self.tiller.k8s.wait_until_ready( release=release_name, labels=wait_labels, namespace=namespace, k8s_wait_attempts=self.k8s_wait_attempts, k8s_wait_attempt_sleep=self.k8s_wait_attempt_sleep, timeout=timer) LOG.info('Install completed with results from Tiller: %s', tiller_result.__dict__) msg['install'].append(release_name) # Sequenced ChartGroup should run tests after each Chart timer = int(round(deadline - time.time())) if test_this_chart and cg_sequenced: LOG.info('Running sequenced test, timeout remaining: %ss.', timer) if timer <= 0: reason = ('Timeout expired before testing sequenced ' 'release %s' % release_name) LOG.error(reason) raise ArmadaTimeoutException(reason) self._test_chart(release_name, timer) # Un-sequenced ChartGroup should run tests at the end elif test_this_chart: # Keeping track of time remaining tests_to_run.append((release_name, timer)) # End of Charts in ChartGroup LOG.info('All Charts applied.') # After all Charts are applied, we should wait for the entire # ChartGroup to become healthy by looking at the namespaces seen # TODO(MarshM): Need to restrict to only releases we processed # TODO(MarshM): Need to determine a better timeout # (not cg_max_timeout) if cg_max_timeout <= 0: cg_max_timeout = DEFAULT_CHART_TIMEOUT deadline = time.time() + cg_max_timeout for (ns, labels) in namespaces_seen: labels_dict = dict(labels) timer = int(round(deadline - time.time())) LOG.info( 'Final wait for healthy namespace (%s), label=(%s), ' 'timeout remaining: %ss.', ns, labels_dict, timer) if timer <= 0: reason = ('Timeout expired waiting on namespace: %s, ' 'label: %s' % (ns, labels_dict)) LOG.error(reason) raise ArmadaTimeoutException(reason) self.tiller.k8s.wait_until_ready( namespace=ns, labels=labels_dict, k8s_wait_attempts=self.k8s_wait_attempts, k8s_wait_attempt_sleep=self.k8s_wait_attempt_sleep, timeout=timer) # After entire ChartGroup is healthy, run any pending tests for (test, test_timer) in tests_to_run: self._test_chart(test, test_timer) LOG.info("Performing Post-Flight Operations") self.post_flight_ops() if self.enable_chart_cleanup: self.tiller.chart_cleanup( prefix, self.manifest[KEYWORD_ARMADA][KEYWORD_GROUPS]) return msg
def execute(self, chart, cg_test_all_charts, prefix, known_releases): namespace = chart.get('namespace') release = chart.get('release') release_name = r.release_prefixer(prefix, release) LOG.info('Processing Chart, release=%s', release_name) values = chart.get('values', {}) pre_actions = {} post_actions = {} result = {} protected = chart.get('protected', {}) p_continue = protected.get('continue_processing', False) old_release = self.find_chart_release(known_releases, release_name) status = None if old_release: status = r.get_release_status(old_release) if status not in [const.STATUS_FAILED, const.STATUS_DEPLOYED]: raise armada_exceptions.UnexpectedReleaseStatusException( release_name, status) chart_wait = ChartWait( self.tiller.k8s, release_name, chart, namespace, k8s_wait_attempts=self.k8s_wait_attempts, k8s_wait_attempt_sleep=self.k8s_wait_attempt_sleep, timeout=self.timeout) native_wait_enabled = chart_wait.is_native_enabled() # Begin Chart timeout deadline deadline = time.time() + chart_wait.get_timeout() chartbuilder = ChartBuilder(chart) new_chart = chartbuilder.get_helm_chart() # Check for existing FAILED release, and purge if status == const.STATUS_FAILED: LOG.info('Purging FAILED release %s before deployment.', release_name) if protected: if p_continue: LOG.warn( 'Release %s is `protected`, ' 'continue_processing=True. Operator must ' 'handle FAILED release manually.', release_name) result['protected'] = release_name return result else: LOG.error( 'Release %s is `protected`, ' 'continue_processing=False.', release_name) raise armada_exceptions.ProtectedReleaseException( release_name) else: # Purge the release self.tiller.uninstall_release(release_name) result['purge'] = release_name # TODO(mark-burnett): It may be more robust to directly call # tiller status to decide whether to install/upgrade rather # than checking for list membership. if status == const.STATUS_DEPLOYED: # indicate to the end user what path we are taking LOG.info("Existing release %s found in namespace %s", release_name, namespace) # extract the installed chart and installed values from the # latest release so we can compare to the intended state old_chart = old_release.chart old_values_string = old_release.config.raw upgrade = chart.get('upgrade', {}) disable_hooks = upgrade.get('no_hooks', False) options = upgrade.get('options', {}) force = options.get('force', False) recreate_pods = options.get('recreate_pods', False) if upgrade: upgrade_pre = upgrade.get('pre', {}) upgrade_post = upgrade.get('post', {}) if not self.disable_update_pre and upgrade_pre: pre_actions = upgrade_pre if not self.disable_update_post and upgrade_post: LOG.warning('Post upgrade actions are ignored by Armada' 'and will not affect deployment.') post_actions = upgrade_post try: old_values = yaml.safe_load(old_values_string) except yaml.YAMLError: chart_desc = '{} (previously deployed)'.format( old_chart.metadata.name) raise armada_exceptions.\ InvalidOverrideValuesYamlException(chart_desc) LOG.info('Checking for updates to chart release inputs.') diff = self.get_diff(old_chart, old_values, new_chart, values) if not diff: LOG.info("Found no updates to chart release inputs") else: LOG.info("Found updates to chart release inputs") LOG.debug("%s", diff) result['diff'] = {chart['release']: str(diff)} # TODO(MarshM): Add tiller dry-run before upgrade and # consider deadline impacts # do actual update timer = int(round(deadline - time.time())) LOG.info( "Upgrading release %s in namespace %s, wait=%s, " "timeout=%ss", release_name, namespace, native_wait_enabled, timer) tiller_result = self.tiller.update_release( new_chart, release_name, namespace, pre_actions=pre_actions, post_actions=post_actions, disable_hooks=disable_hooks, values=yaml.safe_dump(values), wait=native_wait_enabled, timeout=timer, force=force, recreate_pods=recreate_pods) LOG.info('Upgrade completed with results from Tiller: %s', tiller_result.__dict__) result['upgrade'] = release_name else: timer = int(round(deadline - time.time())) LOG.info( "Installing release %s in namespace %s, wait=%s, " "timeout=%ss", release_name, namespace, native_wait_enabled, timer) tiller_result = self.tiller.install_release( new_chart, release_name, namespace, values=yaml.safe_dump(values), wait=native_wait_enabled, timeout=timer) LOG.info('Install completed with results from Tiller: %s', tiller_result.__dict__) result['install'] = release_name # Wait timer = int(round(deadline - time.time())) chart_wait.wait(timer) # Test just_deployed = ('install' in result) or ('upgrade' in result) last_test_passed = old_release and r.get_last_test_result(old_release) test_values = chart.get('test') test_handler = Test(release_name, self.tiller, cg_test_charts=cg_test_all_charts, test_values=test_values) run_test = test_handler.test_enabled and (just_deployed or not last_test_passed) if run_test: timer = int(round(deadline - time.time())) self._test_chart(release_name, timer, test_handler) return result
def sync(self): ''' Syncronize Helm with the Armada Config(s) ''' msg = {'install': [], 'upgrade': [], 'diff': []} # TODO: (gardlt) we need to break up this func into # a more cleaner format LOG.info("Performing Pre-Flight Operations") self.pre_flight_ops() # extract known charts on tiller right now known_releases = self.tiller.list_charts() prefix = self.config.get(const.KEYWORD_ARMADA).get( const.KEYWORD_PREFIX) if known_releases is None: raise armada_exceptions.KnownReleasesException() for release in known_releases: LOG.debug("Release %s, Version %s found on tiller", release[0], release[1]) for entry in self.config[const.KEYWORD_ARMADA][const.KEYWORD_GROUPS]: chart_wait = self.wait desc = entry.get('description', 'A Chart Group') chart_group = entry.get(const.KEYWORD_CHARTS, []) test_charts = entry.get('test_charts', False) if entry.get('sequenced', False) or test_charts: chart_wait = True LOG.info('Deploying: %s', desc) for gchart in chart_group: chart = dotify(gchart['chart']) values = gchart.get('chart').get('values', {}) wait_values = gchart.get('chart').get('wait', {}) test_chart = gchart.get('chart').get('test', False) pre_actions = {} post_actions = {} if chart.release is None: continue if test_chart: chart_wait = True # retrieve appropriate timeout value if 'wait' is specified chart_timeout = self.timeout if chart_wait: if chart_timeout == DEFAULT_TIMEOUT: chart_timeout = getattr(chart, 'timeout', chart_timeout) chartbuilder = ChartBuilder(chart) protoc_chart = chartbuilder.get_helm_chart() # determine install or upgrade by examining known releases LOG.debug("RELEASE: %s", chart.release) deployed_releases = [x[0] for x in known_releases] prefix_chart = release_prefix(prefix, chart.release) if prefix_chart in deployed_releases: # indicate to the end user what path we are taking LOG.info("Upgrading release %s", chart.release) # extract the installed chart and installed values from the # latest release so we can compare to the intended state LOG.info("Checking Pre/Post Actions") apply_chart, apply_values = self.find_release_chart( known_releases, prefix_chart) LOG.info("Checking Pre/Post Actions") upgrade = gchart.get('chart', {}).get('upgrade', False) if upgrade: if not self.disable_update_pre and upgrade.get( 'pre', False): pre_actions = getattr(chart.upgrade, 'pre', {}) if not self.disable_update_post and upgrade.get( 'post', False): post_actions = getattr(chart.upgrade, 'post', {}) # show delta for both the chart templates and the chart # values # TODO(alanmeadows) account for .files differences # once we support those upgrade_diff = self.show_diff(chart, apply_chart, apply_values, chartbuilder.dump(), values, msg) if not upgrade_diff: LOG.info("There are no updates found in this chart") continue # do actual update LOG.info('wait: %s', chart_wait) self.tiller.update_release( protoc_chart, prefix_chart, chart.namespace, pre_actions=pre_actions, post_actions=post_actions, dry_run=self.dry_run, disable_hooks=chart.upgrade.no_hooks, values=yaml.safe_dump(values), wait=chart_wait, timeout=chart_timeout) if chart_wait: # TODO(gardlt): after v0.7.1 depricate timeout values if not wait_values.get('timeout', None): wait_values['timeout'] = chart_timeout self.tiller.k8s.wait_until_ready( release=prefix_chart, labels=wait_values.get('labels', ''), namespace=chart.namespace, timeout=wait_values.get('timeout', DEFAULT_TIMEOUT)) msg['upgrade'].append(prefix_chart) # process install else: LOG.info("Installing release %s", chart.release) self.tiller.install_release(protoc_chart, prefix_chart, chart.namespace, dry_run=self.dry_run, values=yaml.safe_dump(values), wait=chart_wait, timeout=chart_timeout) if chart_wait: if not wait_values.get('timeout', None): wait_values['timeout'] = chart_timeout self.tiller.k8s.wait_until_ready( release=prefix_chart, labels=wait_values.get('labels', ''), namespace=chart.namespace, timeout=wait_values.get('timeout', 3600)) msg['install'].append(prefix_chart) LOG.debug("Cleaning up chart source in %s", chartbuilder.source_directory) if test_charts or test_chart: LOG.info('Testing: %s', prefix_chart) resp = self.tiller.testing_release(prefix_chart) test_status = getattr(resp.info.status, 'last_test_suite_run', 'FAILED') LOG.info("Test INFO: %s", test_status) if resp: LOG.info("PASSED: %s", prefix_chart) else: LOG.info("FAILED: %s", prefix_chart) self.tiller.k8s.wait_until_ready(timeout=chart_timeout) LOG.info("Performing Post-Flight Operations") self.post_flight_ops() if self.enable_chart_cleanup: self.tiller.chart_cleanup( prefix, self.config[const.KEYWORD_ARMADA][const.KEYWORD_GROUPS]) return msg
def sync(self): ''' Synchronize Helm with the Armada Config(s) ''' if self.dry_run: LOG.info('Armada is in DRY RUN mode, no changes being made.') msg = { 'install': [], 'upgrade': [], 'diff': [], 'purge': [], 'protected': [] } # TODO: (gardlt) we need to break up this func into # a more cleaner format self.pre_flight_ops() # extract known charts on tiller right now deployed_releases, failed_releases = self._get_releases_by_status() manifest_data = self.manifest.get(const.KEYWORD_ARMADA, {}) prefix = manifest_data.get(const.KEYWORD_PREFIX) for chartgroup in manifest_data.get(const.KEYWORD_GROUPS, []): cg_name = chartgroup.get('name', '<missing name>') cg_desc = chartgroup.get('description', '<missing description>') cg_sequenced = chartgroup.get('sequenced', False) LOG.info('Processing ChartGroup: %s (%s), sequenced=%s', cg_name, cg_desc, cg_sequenced) # TODO(MarshM): Deprecate the `test_charts` key cg_test_all_charts = chartgroup.get('test_charts') if isinstance(cg_test_all_charts, bool): LOG.warn('The ChartGroup `test_charts` key is deprecated, ' 'and support for this will be removed. See the ' 'Chart `test` key for more information.') else: # This key defaults to True. Individual charts must # explicitly disable helm tests if they choose cg_test_all_charts = True ns_label_set = set() tests_to_run = [] cg_charts = chartgroup.get(const.KEYWORD_CHARTS, []) # Track largest Chart timeout to stop the ChartGroup at the end cg_max_timeout = 0 for chart_entry in cg_charts: chart = chart_entry.get('chart', {}) namespace = chart.get('namespace') release = chart.get('release') release_name = release_prefixer(prefix, release) LOG.info('Processing Chart, release=%s', release_name) values = chart.get('values', {}) pre_actions = {} post_actions = {} protected = chart.get('protected', {}) p_continue = protected.get('continue_processing', False) # Check for existing FAILED release, and purge if release_name in [rel[0] for rel in failed_releases]: LOG.info('Purging FAILED release %s before deployment.', release_name) if protected: if p_continue: LOG.warn( 'Release %s is `protected`, ' 'continue_processing=True. Operator must ' 'handle FAILED release manually.', release_name) msg['protected'].append(release_name) continue else: LOG.error( 'Release %s is `protected`, ' 'continue_processing=False.', release_name) raise armada_exceptions.ProtectedReleaseException( release_name) else: # Purge the release self.tiller.uninstall_release(release_name) msg['purge'].append(release_name) # NOTE(MarshM): Calculating `wait_timeout` is unfortunately # overly complex. The order of precedence is currently: # 1) User provided override via API/CLI (default 0 if not # provided by client/user). # 2) Chart's `data.wait.timeout`, or... # 3) Chart's `data.timeout` (deprecated). # 4) const.DEFAULT_CHART_TIMEOUT, if nothing is ever # specified, for use in waiting for final ChartGroup # health and helm tests, but ignored for the actual # install/upgrade of the Chart. # NOTE(MarshM): Not defining a timeout has a side effect of # allowing Armada to install charts with a circular # dependency defined between components. # TODO(MarshM): Deprecated, remove the following block deprecated_timeout = chart.get('timeout', None) if isinstance(deprecated_timeout, int): LOG.warn('The `timeout` key is deprecated and support ' 'for this will be removed soon. Use ' '`wait.timeout` instead.') wait_values = chart.get('wait', {}) wait_labels = wait_values.get('labels', {}) wait_timeout = self.timeout if wait_timeout <= 0: wait_timeout = wait_values.get('timeout', wait_timeout) # TODO(MarshM): Deprecated, remove the following check if wait_timeout <= 0: wait_timeout = deprecated_timeout or wait_timeout # Determine wait logic # NOTE(Dan Kim): Conditions to wait are below : # 1) set sequenced=True in chart group # 2) set force_wait param # 3) add Chart's `data.wait.timeout` # --timeout param will do not set wait=True, it just change # max timeout of chart's deployment. (default: 900) this_chart_should_wait = (cg_sequenced or self.force_wait or (bool(wait_values) and (wait_timeout > 0))) # If there is still no timeout, we need to use a default # (item 4 in note above) if wait_timeout <= 0: LOG.warn('No Chart timeout specified, using default: %ss', const.DEFAULT_CHART_TIMEOUT) wait_timeout = const.DEFAULT_CHART_TIMEOUT # Naively take largest timeout to apply at end # TODO(MarshM) better handling of timeout/timer cg_max_timeout = max(wait_timeout, cg_max_timeout) test_chart_override = chart.get('test') # Use old default value when not using newer `test` key test_cleanup = True if test_chart_override is None: test_this_chart = cg_test_all_charts elif isinstance(test_chart_override, bool): LOG.warn('Boolean value for chart `test` key is' ' deprecated and support for this will' ' be removed. Use `test.enabled` ' 'instead.') test_this_chart = test_chart_override else: # NOTE: helm tests are enabled by default test_this_chart = test_chart_override.get('enabled', True) test_cleanup = test_chart_override.get('options', {}).get( 'cleanup', False) chartbuilder = ChartBuilder(chart) new_chart = chartbuilder.get_helm_chart() # Begin Chart timeout deadline deadline = time.time() + wait_timeout # TODO(mark-burnett): It may be more robust to directly call # tiller status to decide whether to install/upgrade rather # than checking for list membership. if release_name in [rel[0] for rel in deployed_releases]: # indicate to the end user what path we are taking LOG.info("Upgrading release %s in namespace %s", release_name, namespace) # extract the installed chart and installed values from the # latest release so we can compare to the intended state old_chart, old_values_string = self.find_release_chart( deployed_releases, release_name) upgrade = chart.get('upgrade', {}) disable_hooks = upgrade.get('no_hooks', False) force = upgrade.get('force', False) recreate_pods = upgrade.get('recreate_pods', False) LOG.info("Checking Pre/Post Actions") if upgrade: upgrade_pre = upgrade.get('pre', {}) upgrade_post = upgrade.get('post', {}) if not self.disable_update_pre and upgrade_pre: pre_actions = upgrade_pre if not self.disable_update_post and upgrade_post: post_actions = upgrade_post try: old_values = yaml.safe_load(old_values_string) except yaml.YAMLError: chart_desc = '{} (previously deployed)'.format( old_chart.metadata.name) raise armada_exceptions.\ InvalidOverrideValuesYamlException(chart_desc) LOG.info('Checking for updates to chart release inputs.') diff = self.get_diff(old_chart, old_values, new_chart, values) if not diff: LOG.info("Found no updates to chart release inputs") continue LOG.info("Found updates to chart release inputs") LOG.debug("%s", diff) msg['diff'].append({chart['release']: str(diff)}) # TODO(MarshM): Add tiller dry-run before upgrade and # consider deadline impacts # do actual update timer = int(round(deadline - time.time())) LOG.info('Beginning Upgrade, wait=%s, timeout=%ss', this_chart_should_wait, timer) tiller_result = self.tiller.update_release( new_chart, release_name, namespace, pre_actions=pre_actions, post_actions=post_actions, disable_hooks=disable_hooks, values=yaml.safe_dump(values), wait=this_chart_should_wait, timeout=timer, force=force, recreate_pods=recreate_pods) if this_chart_should_wait: self._wait_until_ready(release_name, wait_labels, namespace, timer) # Track namespace+labels touched by upgrade ns_label_set.add((namespace, tuple(wait_labels.items()))) LOG.info('Upgrade completed with results from Tiller: %s', tiller_result.__dict__) msg['upgrade'].append(release_name) # process install else: LOG.info("Installing release %s in namespace %s", release_name, namespace) timer = int(round(deadline - time.time())) LOG.info('Beginning Install, wait=%s, timeout=%ss', this_chart_should_wait, timer) tiller_result = self.tiller.install_release( new_chart, release_name, namespace, values=yaml.safe_dump(values), wait=this_chart_should_wait, timeout=timer) if this_chart_should_wait: self._wait_until_ready(release_name, wait_labels, namespace, timer) # Track namespace+labels touched by install ns_label_set.add((namespace, tuple(wait_labels.items()))) LOG.info('Install completed with results from Tiller: %s', tiller_result.__dict__) msg['install'].append(release_name) # Keeping track of time remaining timer = int(round(deadline - time.time())) test_chart_args = (release_name, timer, test_cleanup) if test_this_chart: # Sequenced ChartGroup should run tests after each Chart if cg_sequenced: LOG.info( 'Running sequenced test, timeout remaining: ' '%ss.', timer) self._test_chart(*test_chart_args) # Un-sequenced ChartGroup should run tests at the end else: tests_to_run.append( functools.partial(self._test_chart, *test_chart_args)) # End of Charts in ChartGroup LOG.info('All Charts applied in ChartGroup %s.', cg_name) # After all Charts are applied, we should wait for the entire # ChartGroup to become healthy by looking at the namespaces seen # TODO(MarshM): Need to determine a better timeout # (not cg_max_timeout) if cg_max_timeout <= 0: cg_max_timeout = const.DEFAULT_CHART_TIMEOUT deadline = time.time() + cg_max_timeout for (ns, labels) in ns_label_set: labels_dict = dict(labels) timer = int(round(deadline - time.time())) LOG.info( 'Final ChartGroup wait for healthy namespace=%s, ' 'labels=(%s), timeout remaining: %ss.', ns, labels_dict, timer) if timer <= 0: reason = ('Timeout expired waiting on namespace: %s, ' 'labels: (%s)' % (ns, labels_dict)) LOG.error(reason) raise armada_exceptions.ArmadaTimeoutException(reason) self._wait_until_ready(release_name=None, wait_labels=labels_dict, namespace=ns, timeout=timer) # After entire ChartGroup is healthy, run any pending tests for callback in tests_to_run: callback() self.post_flight_ops() if self.enable_chart_cleanup: self._chart_cleanup( prefix, self.manifest[const.KEYWORD_ARMADA][const.KEYWORD_GROUPS], msg) LOG.info('Done applying manifest.') return msg