def test_dependency_resolution(self):
# Main chart directory and files.
chart_dir = self.useFixture(fixtures.TempDir())
self.addCleanup(shutil.rmtree, chart_dir.path)
self._write_temporary_file_contents(chart_dir.path, 'Chart.yaml',
self.chart_yaml)
chart_doc = yaml.safe_load(self.chart_doc_yaml)
chart_doc['data']['source_dir'] = (chart_dir.path, '')
# Dependency chart directory and files.
dep_chart_dir = self.useFixture(fixtures.TempDir())
self.addCleanup(shutil.rmtree, dep_chart_dir.path)
self._write_temporary_file_contents(dep_chart_dir.path, 'Chart.yaml',
self.dep_chart_yaml)
dep_chart_doc = yaml.safe_load(self.dep_chart_doc_yaml)
dep_chart_doc['data']['source_dir'] = (dep_chart_dir.path, '')
# Add dependency
chart_doc['data']['dependencies'] = [dep_chart_doc]
# Mock helm cli call
helm_mock = mock.Mock()
helm_mock.show_chart.return_value = yaml.safe_load(self.dep_chart_yaml)
ChartBuilder.from_chart_doc(chart_doc, helm_mock)
expected_symlink_path = Path(
chart_dir.path).joinpath('charts').joinpath('dependency-chart')
self.assertTrue(expected_symlink_path.is_symlink())
self.assertEqual(dep_chart_dir.path,
str(expected_symlink_path.resolve()))
def test_get_files(self):
"""Validates that ``get_files()`` ignores 'Chart.yaml', 'values.yaml'
and 'templates' subfolder and all the files contained therein.
"""
# Create a temporary directory that represents a chart source directory
# with various files, including 'Chart.yaml' and 'values.yaml' which
# should be ignored by `get_files()`.
chart_dir = self.useFixture(fixtures.TempDir())
self.addCleanup(shutil.rmtree, chart_dir.path)
for filename in ['foo', 'bar', 'Chart.yaml', 'values.yaml']:
self._write_temporary_file_contents(chart_dir.path, filename, "")
# Create a template directory -- 'templates' -- nested inside the chart
# directory which should also be ignored.
templates_subdir = self._make_temporary_subdirectory(
chart_dir.path, 'templates')
for filename in ['template%d' % x for x in range(3)]:
self._write_temporary_file_contents(templates_subdir, filename, "")
chartbuilder = ChartBuilder(self._get_test_chart(chart_dir))
expected_files = ('[type_url: "%s"\n, type_url: "%s"\n]' %
('./bar', './foo'))
# Validate that only 'foo' and 'bar' are returned.
actual_files = sorted(chartbuilder.get_files(),
key=lambda x: x.type_url)
self.assertEqual(expected_files, repr(actual_files).strip())
def test_get_basic_helm_chart(self):
# Before ChartBuilder is executed the `source_dir` points to a
# directory that was either clone or unpacked from a tarball... pretend
# that that logic has already been performed.
chart_dir = self.useFixture(fixtures.TempDir())
self.addCleanup(shutil.rmtree, chart_dir.path)
self._write_temporary_file_contents(chart_dir.path, 'Chart.yaml',
self.chart_yaml)
ch = yaml.safe_load(self.chart_stream)
ch['data']['source_dir'] = (chart_dir.path, '')
test_chart = ch
chartbuilder = ChartBuilder(test_chart)
helm_chart = chartbuilder.get_helm_chart()
expected = inspect.cleandoc("""
metadata {
name: "hello-world-chart"
version: "0.1.0"
description: "A sample Helm chart for Kubernetes"
}
values {
}
""").strip()
self.assertIsInstance(helm_chart, Chart)
self.assertTrue(hasattr(helm_chart, 'metadata'))
self.assertTrue(hasattr(helm_chart, 'values'))
self.assertEqual(expected, repr(helm_chart).strip())
def test_get_helm_chart_with_files(self):
# Create a chart directory with some test files.
chart_dir = self.useFixture(fixtures.TempDir())
self.addCleanup(shutil.rmtree, chart_dir.path)
# Chart.yaml is mandatory for `ChartBuilder.get_metadata`.
self._write_temporary_file_contents(chart_dir.path, 'Chart.yaml',
self.chart_yaml)
self._write_temporary_file_contents(chart_dir.path, 'foo', "foobar")
self._write_temporary_file_contents(chart_dir.path, 'bar', "bazqux")
# Also create a nested directory and verify that files from it are also
# added.
nested_dir = self._make_temporary_subdirectory(chart_dir.path,
'nested')
self._write_temporary_file_contents(nested_dir, 'nested0', "random")
ch = yaml.safe_load(self.chart_stream)
ch['data']['source_dir'] = (chart_dir.path, '')
test_chart = ch
chartbuilder = ChartBuilder(test_chart)
helm_chart = chartbuilder.get_helm_chart()
expected_files = ('[type_url: "%s"\nvalue: "bazqux"\n, '
'type_url: "%s"\nvalue: "foobar"\n, '
'type_url: "%s"\nvalue: "random"\n]' %
('./bar', './foo', 'nested/nested0'))
self.assertIsInstance(helm_chart, Chart)
self.assertTrue(hasattr(helm_chart, 'metadata'))
self.assertTrue(hasattr(helm_chart, 'values'))
self.assertTrue(hasattr(helm_chart, 'files'))
actual_files = sorted(helm_chart.files, key=lambda x: x.value)
self.assertEqual(expected_files, repr(actual_files).strip())
def test_dump(self):
# Validate base case.
chart_dir = self.useFixture(fixtures.TempDir())
self.addCleanup(shutil.rmtree, chart_dir.path)
self._write_temporary_file_contents(chart_dir.path, 'Chart.yaml',
self.chart_yaml)
ch = yaml.safe_load(self.chart_stream)
ch['data']['source_dir'] = (chart_dir.path, '')
test_chart = ch
chartbuilder = ChartBuilder.from_chart_doc(test_chart)
self.assertRegex(
repr(chartbuilder.dump()),
'hello-world-chart.*A sample Helm chart for Kubernetes.*')
# Validate recursive case (with dependencies).
dep_chart_dir = self.useFixture(fixtures.TempDir())
self.addCleanup(shutil.rmtree, dep_chart_dir.path)
self._write_temporary_file_contents(dep_chart_dir.path, 'Chart.yaml',
self.dependency_chart_yaml)
dep_ch = yaml.safe_load(self.dependency_chart_stream)
dep_ch['data']['source_dir'] = (dep_chart_dir.path, '')
dependency_chart = dep_ch
test_chart['data']['dependencies'] = [dependency_chart]
chartbuilder = ChartBuilder.from_chart_doc(test_chart)
re = inspect.cleandoc("""
hello-world-chart.*A sample Helm chart for Kubernetes.*
dependency-chart.*Another sample Helm chart for Kubernetes.*
""").replace('\n', '').strip()
self.assertRegex(repr(chartbuilder.dump()), re)
def test_get_helm_chart_includes_only_relevant_files(self):
chart_dir = self.useFixture(fixtures.TempDir())
self.addCleanup(shutil.rmtree, chart_dir.path)
templates_subdir = self._make_temporary_subdirectory(
chart_dir.path, 'templates')
charts_subdir = self._make_temporary_subdirectory(
chart_dir.path, 'charts')
templates_nested_subdir = self._make_temporary_subdirectory(
templates_subdir, 'bin')
charts_nested_subdir = self._make_temporary_subdirectory(
charts_subdir, 'extra')
self._write_temporary_file_contents(chart_dir.path, 'Chart.yaml',
self.chart_yaml)
self._write_temporary_file_contents(chart_dir.path, 'foo', "foobar")
self._write_temporary_file_contents(chart_dir.path, 'bar', "bazqux")
# Files to ignore within top-level directory.
files_to_ignore = ['Chart.yaml', 'values.yaml', 'values.toml']
for file in files_to_ignore:
self._write_temporary_file_contents(chart_dir.path, file, "")
file_to_ignore = 'file_to_ignore'
# Files to ignore within templates/ subdirectory.
self._write_temporary_file_contents(
templates_subdir, file_to_ignore, "")
# Files to ignore within charts/ subdirectory.
self._write_temporary_file_contents(
charts_subdir, file_to_ignore, "")
# Files to ignore within templates/bin subdirectory.
self._write_temporary_file_contents(
templates_nested_subdir, file_to_ignore, "")
# Files to ignore within charts/extra subdirectory.
self._write_temporary_file_contents(
charts_nested_subdir, file_to_ignore, "")
# Files to **include** within charts/ subdirectory.
self._write_temporary_file_contents(
charts_subdir, '.prov', "xyzzy")
ch = yaml.safe_load(self.chart_stream)['chart']
ch['source_dir'] = (chart_dir.path, '')
test_chart = dotify(ch)
chartbuilder = ChartBuilder(test_chart)
helm_chart = chartbuilder.get_helm_chart()
expected_files = ('[type_url: "%s"\nvalue: "bazqux"\n, '
'type_url: "%s"\nvalue: "foobar"\n, '
'type_url: "%s"\nvalue: "xyzzy"\n]' %
('./bar', './foo', 'charts/.prov'))
# Validate that only relevant files are included, that the ignored
# files are present.
self.assertIsInstance(helm_chart, Chart)
self.assertTrue(hasattr(helm_chart, 'metadata'))
self.assertTrue(hasattr(helm_chart, 'values'))
self.assertTrue(hasattr(helm_chart, 'files'))
actual_files = sorted(helm_chart.files, key=lambda x: x.value)
self.assertEqual(expected_files, repr(actual_files).strip())
def test_get_files_with_unicode_characters(self):
chart_dir = self.useFixture(fixtures.TempDir())
self.addCleanup(shutil.rmtree, chart_dir.path)
for filename in ['foo', 'bar', 'Chart.yaml', 'values.yaml']:
self._write_temporary_file_contents(
chart_dir.path, filename, "DIRC^@^@^@^B^@^@^@×Z®<86>F.1")
chartbuilder = ChartBuilder(self._get_test_chart(chart_dir))
chartbuilder.get_files()
def test_chart_source_clone(self, mock_os, mock_dot):
from supermutes.dot import dotify
import yaml
mock_dot.dotify.return_value = dotify(yaml.load(self.chart_stream))
mock_os.path.join.return_value = self.chart_stream
ChartBuilder.source_clone = mock.Mock(return_value='path')
chartbuilder = ChartBuilder(self.chart_stream)
resp = chartbuilder.get_metadata()
self.assertIsNotNone(resp)
self.assertIsInstance(resp, basestring)
def test_chart_source_clone(self, mock_os, mock_dot):
from supermutes.dot import dotify
import yaml
mock_dot.dotify.return_value = dotify(yaml.load(self.chart_stream))
mock_os.path.join.return_value = self.chart_stream
ChartBuilder.source_clone = mock.Mock(return_value='path')
chartbuilder = ChartBuilder(self.chart_stream)
resp = chartbuilder.get_metadata()
self.assertIsNotNone(resp)
self.assertIsInstance(resp, basestring)
def test_source_clone(self):
# Create a temporary directory with Chart.yaml that contains data
# from ``self.chart_yaml``.
chart_dir = self.useFixture(fixtures.TempDir())
self.addCleanup(shutil.rmtree, chart_dir.path)
self._write_temporary_file_contents(chart_dir.path, 'Chart.yaml',
self.chart_yaml)
chartbuilder = ChartBuilder(self._get_test_chart(chart_dir))
# Validate response type is :class:`hapi.chart.metadata_pb2.Metadata`
resp = chartbuilder.get_metadata()
self.assertIsInstance(resp, Metadata)
def test_get_files_fails_once_to_read_binary_file_passes(self):
chart_dir = self.useFixture(fixtures.TempDir())
self.addCleanup(shutil.rmtree, chart_dir.path)
files = ['foo', 'bar']
for filename in files:
self._write_temporary_file_contents(
chart_dir.path, filename, "DIRC^@^@^@^B^@^@^@×Z®<86>F.1")
chartbuilder = ChartBuilder(self._get_test_chart(chart_dir))
side_effects = [self.exc_to_raise, "", ""]
with mock.patch("builtins.open", mock.mock_open(read_data="")) \
as mock_file:
mock_file.return_value.read.side_effect = side_effects
chartbuilder.get_files()
def test_get_metadata_with_incorrect_file_invalid(self):
chart_dir = self.useFixture(fixtures.TempDir())
self.addCleanup(shutil.rmtree, chart_dir.path)
chartbuilder = ChartBuilder(self._get_test_chart(chart_dir))
self.assertRaises(chartbuilder_exceptions.MetadataLoadException,
chartbuilder.get_metadata)
def test_chartbuilder_source_clone(self):
chart = dotify(self.chart_stream)
ChartBuilder.source_clone = mock.Mock(return_value='path')
chartbuilder = ChartBuilder(chart)
resp = getattr(chartbuilder, 'source_directory', None)
self.assertIsNotNone(resp)
self.assertIsInstance(resp, basestring)
def test_get_helm_chart_success(self):
chart_dir = self.useFixture(fixtures.TempDir())
self.addCleanup(shutil.rmtree, chart_dir.path)
helm_mock = mock.Mock()
helm_mock.upgrade_release.return_value = {"chart": mock.sentinel.chart}
chartbuilder = ChartBuilder.from_chart_doc(
self._get_test_chart(chart_dir.path), helm_mock)
release_id = mock.Mock()
values = mock.Mock()
actual_chart = chartbuilder.get_helm_chart(release_id, values)
self.assertIs(mock.sentinel.chart, actual_chart)
def test_get_helm_chart_with_values(self):
chart_dir = self.useFixture(fixtures.TempDir())
self.addCleanup(shutil.rmtree, chart_dir.path)
self._write_temporary_file_contents(chart_dir.path, 'Chart.yaml',
self.chart_yaml)
self._write_temporary_file_contents(chart_dir.path, 'values.yaml',
self.chart_value)
ch = yaml.safe_load(self.chart_stream)
ch['data']['source_dir'] = (chart_dir.path, '')
test_chart = ch
chartbuilder = ChartBuilder(test_chart)
helm_chart = chartbuilder.get_helm_chart()
self.assertIsInstance(helm_chart, Chart)
self.assertTrue(hasattr(helm_chart, 'metadata'))
self.assertTrue(hasattr(helm_chart, 'values'))
self.assertTrue(hasattr(helm_chart.values, 'raw'))
self.assertEqual(self.chart_value, helm_chart.values.raw)
def test_get_helm_chart_fail(self):
chart_dir = self.useFixture(fixtures.TempDir())
self.addCleanup(shutil.rmtree, chart_dir.path)
helm_mock = mock.Mock()
helm_mock.upgrade_release.side_effect = Exception()
chartbuilder = ChartBuilder.from_chart_doc(
self._get_test_chart(chart_dir.path), helm_mock)
def test():
release_id = mock.Mock()
values = mock.Mock()
chartbuilder.get_helm_chart(release_id, values)
self.assertRaises(chartbuilder_exceptions.HelmChartBuildException,
test)
def test_get_files_always_fails_to_read_binary_file_raises_exc(self):
chart_dir = self.useFixture(fixtures.TempDir())
self.addCleanup(shutil.rmtree, chart_dir.path)
for filename in ['foo', 'bar', 'Chart.yaml', 'values.yaml']:
self._write_temporary_file_contents(
chart_dir.path, filename, "DIRC^@^@^@^B^@^@^@×Z®<86>F.1")
chartbuilder = ChartBuilder(self._get_test_chart(chart_dir))
# Confirm it failed for both encodings.
error_re = (r'.*A str exception occurred while trying to read file:'
r'.*Details:\n.*\(encoding=utf-8\).*\n\(encoding=latin1\)')
with mock.patch("builtins.open", mock.mock_open(read_data="")) \
as mock_file:
mock_file.return_value.read.side_effect = self.exc_to_raise
self.assertRaisesRegexp(chartbuilder_exceptions.FilesLoadException,
error_re, chartbuilder.get_files)
def test_get_helm_chart_with_dependencies(self):
# Main chart directory and files.
chart_dir = self.useFixture(fixtures.TempDir())
self.addCleanup(shutil.rmtree, chart_dir.path)
self._write_temporary_file_contents(chart_dir.path, 'Chart.yaml',
self.chart_yaml)
ch = yaml.safe_load(self.chart_stream)
ch['data']['source_dir'] = (chart_dir.path, '')
# Dependency chart directory and files.
dep_chart_dir = self.useFixture(fixtures.TempDir())
self.addCleanup(shutil.rmtree, dep_chart_dir.path)
self._write_temporary_file_contents(dep_chart_dir.path, 'Chart.yaml',
self.dependency_chart_yaml)
dep_ch = yaml.safe_load(self.dependency_chart_stream)
dep_ch['data']['source_dir'] = (dep_chart_dir.path, '')
main_chart = ch
dependency_chart = dep_ch
main_chart['data']['dependencies'] = [dependency_chart]
chartbuilder = ChartBuilder(main_chart)
helm_chart = chartbuilder.get_helm_chart()
expected_dependency = inspect.cleandoc("""
metadata {
name: "dependency-chart"
version: "0.1.0"
description: "Another sample Helm chart for Kubernetes"
}
values {
}
""").strip()
expected = inspect.cleandoc("""
metadata {
name: "hello-world-chart"
version: "0.1.0"
description: "A sample Helm chart for Kubernetes"
}
dependencies {
metadata {
name: "dependency-chart"
version: "0.1.0"
description: "Another sample Helm chart for Kubernetes"
}
values {
}
}
values {
}
""").strip()
# Validate the main chart.
self.assertIsInstance(helm_chart, Chart)
self.assertTrue(hasattr(helm_chart, 'metadata'))
self.assertTrue(hasattr(helm_chart, 'values'))
self.assertEqual(expected, repr(helm_chart).strip())
# Validate the dependency chart.
self.assertTrue(hasattr(helm_chart, 'dependencies'))
self.assertEqual(1, len(helm_chart.dependencies))
dep_helm_chart = helm_chart.dependencies[0]
self.assertIsInstance(dep_helm_chart, Chart)
self.assertTrue(hasattr(dep_helm_chart, 'metadata'))
self.assertTrue(hasattr(dep_helm_chart, 'values'))
self.assertEqual(expected_dependency, repr(dep_helm_chart).strip())
def _execute(self, ch, cg_test_all_charts, prefix, known_releases):
manifest_name = self.manifest['metadata']['name']
chart = ch[const.KEYWORD_DATA]
chart_name = ch['metadata']['name']
namespace = chart.get('namespace')
release = chart.get('release')
release_name = r.release_prefixer(prefix, release)
LOG.info('Processing Chart, release=%s', release_name)
result = {}
chart_wait = ChartWait(
self.tiller.k8s,
release_name,
ch,
namespace,
k8s_wait_attempts=self.k8s_wait_attempts,
k8s_wait_attempt_sleep=self.k8s_wait_attempt_sleep,
timeout=self.timeout)
wait_timeout = chart_wait.get_timeout()
# Begin Chart timeout deadline
deadline = time.time() + wait_timeout
old_release = self.find_chart_release(known_releases, release_name)
action = metrics.ChartDeployAction.NOOP
def noop():
pass
deploy = noop
# Resolve action
values = chart.get('values', {})
pre_actions = {}
post_actions = {}
status = None
if old_release:
status = r.get_release_status(old_release)
native_wait_enabled = chart_wait.is_native_enabled()
chartbuilder = ChartBuilder.from_chart_doc(ch)
new_chart = chartbuilder.get_helm_chart()
if status == const.STATUS_DEPLOYED:
# indicate to the end user what path we are taking
LOG.info("Existing release %s found in namespace %s", release_name,
namespace)
# extract the installed chart and installed values from the
# latest release so we can compare to the intended state
old_chart = old_release.chart
old_values_string = old_release.config.raw
upgrade = chart.get('upgrade', {})
options = upgrade.get('options', {})
# TODO: Remove when v1 doc support is removed.
schema_info = get_schema_info(ch['schema'])
if schema_info.version < 2:
no_hooks_location = upgrade
else:
no_hooks_location = options
disable_hooks = no_hooks_location.get('no_hooks', False)
force = options.get('force', False)
recreate_pods = options.get('recreate_pods', False)
if upgrade:
upgrade_pre = upgrade.get('pre', {})
upgrade_post = upgrade.get('post', {})
if not self.disable_update_pre and upgrade_pre:
pre_actions = upgrade_pre
if not self.disable_update_post and upgrade_post:
LOG.warning('Post upgrade actions are ignored by Armada'
'and will not affect deployment.')
post_actions = upgrade_post
try:
old_values = yaml.safe_load(old_values_string)
except yaml.YAMLError:
chart_desc = '{} (previously deployed)'.format(
old_chart.metadata.name)
raise armada_exceptions.\
InvalidOverrideValuesYamlException(chart_desc)
LOG.info('Checking for updates to chart release inputs.')
diff = self.get_diff(old_chart, old_values, new_chart, values)
if not diff:
LOG.info("Found no updates to chart release inputs")
else:
action = metrics.ChartDeployAction.UPGRADE
LOG.info("Found updates to chart release inputs")
LOG.debug("%s", diff)
result['diff'] = {chart['release']: str(diff)}
def upgrade():
# do actual update
timer = int(round(deadline - time.time()))
LOG.info(
"Upgrading release %s in namespace %s, wait=%s, "
"timeout=%ss", release_name, namespace,
native_wait_enabled, timer)
tiller_result = self.tiller.update_release(
new_chart,
release_name,
namespace,
pre_actions=pre_actions,
post_actions=post_actions,
disable_hooks=disable_hooks,
values=yaml.safe_dump(values),
wait=native_wait_enabled,
timeout=timer,
force=force,
recreate_pods=recreate_pods)
LOG.info('Upgrade completed with results from Tiller: %s',
tiller_result.__dict__)
result['upgrade'] = release_name
deploy = upgrade
else:
# Check for release with status other than DEPLOYED
if status:
if status != const.STATUS_FAILED:
LOG.warn(
'Unexpected release status encountered '
'release=%s, status=%s', release_name, status)
# Make best effort to determine whether a deployment is
# likely pending, by checking if the last deployment
# was started within the timeout window of the chart.
last_deployment_age = r.get_last_deployment_age(
old_release)
likely_pending = last_deployment_age <= wait_timeout
if likely_pending:
# Give up if a deployment is likely pending, we do not
# want to have multiple operations going on for the
# same release at the same time.
raise armada_exceptions.\
DeploymentLikelyPendingException(
release_name, status, last_deployment_age,
wait_timeout)
else:
# Release is likely stuck in an unintended (by tiller)
# state. Log and continue on with remediation steps
# below.
LOG.info(
'Old release %s likely stuck in status %s, '
'(last deployment age=%ss) >= '
'(chart wait timeout=%ss)', release, status,
last_deployment_age, wait_timeout)
protected = chart.get('protected', {})
if protected:
p_continue = protected.get('continue_processing', False)
if p_continue:
LOG.warn(
'Release %s is `protected`, '
'continue_processing=True. Operator must '
'handle %s release manually.', release_name,
status)
result['protected'] = release_name
return result
else:
LOG.error(
'Release %s is `protected`, '
'continue_processing=False.', release_name)
raise armada_exceptions.ProtectedReleaseException(
release_name, status)
else:
# Purge the release
with metrics.CHART_DELETE.get_context(
manifest_name, chart_name):
LOG.info('Purging release %s with status %s',
release_name, status)
chart_delete = ChartDelete(chart, release_name,
self.tiller)
chart_delete.delete()
result['purge'] = release_name
action = metrics.ChartDeployAction.INSTALL
def install():
timer = int(round(deadline - time.time()))
LOG.info(
"Installing release %s in namespace %s, wait=%s, "
"timeout=%ss", release_name, namespace,
native_wait_enabled, timer)
tiller_result = self.tiller.install_release(
new_chart,
release_name,
namespace,
values=yaml.safe_dump(values),
wait=native_wait_enabled,
timeout=timer)
LOG.info('Install completed with results from Tiller: %s',
tiller_result.__dict__)
result['install'] = release_name
deploy = install
# Deploy
with metrics.CHART_DEPLOY.get_context(wait_timeout, manifest_name,
chart_name,
action.get_label_value()):
deploy()
# Wait
timer = int(round(deadline - time.time()))
chart_wait.wait(timer)
# Test
just_deployed = ('install' in result) or ('upgrade' in result)
last_test_passed = old_release and r.get_last_test_result(old_release)
test_handler = Test(chart,
release_name,
self.tiller,
cg_test_charts=cg_test_all_charts)
run_test = test_handler.test_enabled and (just_deployed
or not last_test_passed)
if run_test:
with metrics.CHART_TEST.get_context(test_handler.timeout,
manifest_name, chart_name):
self._test_chart(release_name, test_handler)
return result
def _execute(self, ch, cg_test_all_charts, prefix):
manifest_name = self.manifest['metadata']['name']
chart = ch[const.KEYWORD_DATA]
chart_name = ch['metadata']['name']
namespace = chart.get('namespace')
release = chart.get('release')
release_name = r.release_prefixer(prefix, release)
release_id = helm.HelmReleaseId(namespace, release_name)
source_dir = chart['source_dir']
source_directory = os.path.join(*source_dir)
LOG.info('Processing Chart, release=%s', release_id)
result = {}
chart_wait = ChartWait(
self.helm.k8s,
release_id,
ch,
k8s_wait_attempts=self.k8s_wait_attempts,
k8s_wait_attempt_sleep=self.k8s_wait_attempt_sleep,
timeout=self.timeout)
wait_timeout = chart_wait.get_timeout()
# Begin Chart timeout deadline
deadline = time.time() + wait_timeout
old_release = self.helm.release_metadata(release_id)
action = metrics.ChartDeployAction.NOOP
def noop():
pass
deploy = noop
# Resolve action
values = chart.get('values', {})
pre_actions = {}
status = None
if old_release:
status = r.get_release_status(old_release)
native_wait_enabled = chart_wait.is_native_enabled()
chartbuilder = ChartBuilder.from_chart_doc(ch, self.helm)
if status == helm.STATUS_DEPLOYED:
# indicate to the end user what path we are taking
LOG.info("Existing release %s found", release_id)
# extract the installed chart and installed values from the
# latest release so we can compare to the intended state
old_chart = old_release['chart']
old_values = old_release.get('config', {})
upgrade = chart.get('upgrade', {})
options = upgrade.get('options', {})
# TODO: Remove when v1 doc support is removed.
schema_info = get_schema_info(ch['schema'])
if schema_info.version < 2:
no_hooks_location = upgrade
else:
no_hooks_location = options
disable_hooks = no_hooks_location.get('no_hooks', False)
force = options.get('force', False)
if upgrade:
upgrade_pre = upgrade.get('pre', {})
upgrade_post = upgrade.get('post', {})
if not self.disable_update_pre and upgrade_pre:
pre_actions = upgrade_pre
if not self.disable_update_post and upgrade_post:
LOG.warning('Post upgrade actions are ignored by Armada'
'and will not affect deployment.')
LOG.info('Checking for updates to chart release inputs.')
new_chart = chartbuilder.get_helm_chart(release_id, values)
diff = self.get_diff(old_chart, old_values, new_chart, values)
if not diff:
LOG.info("Found no updates to chart release inputs")
else:
action = metrics.ChartDeployAction.UPGRADE
LOG.info("Found updates to chart release inputs")
def upgrade():
# do actual update
timer = int(round(deadline - time.time()))
PreUpdateActions(self.helm.k8s).execute(
pre_actions, release, namespace, chart, disable_hooks,
values, timer)
LOG.info("Upgrading release=%s, wait=%s, "
"timeout=%ss", release_id, native_wait_enabled,
timer)
self.helm.upgrade_release(source_directory,
release_id,
disable_hooks=disable_hooks,
values=values,
wait=native_wait_enabled,
timeout=timer,
force=force)
LOG.info('Upgrade completed')
result['upgrade'] = release_id
deploy = upgrade
else:
def install():
timer = int(round(deadline - time.time()))
LOG.info("Installing release=%s, wait=%s, "
"timeout=%ss", release_id, native_wait_enabled, timer)
self.helm.install_release(source_directory,
release_id,
values=values,
wait=native_wait_enabled,
timeout=timer)
LOG.info('Install completed')
result['install'] = release_id
# Check for release with status other than DEPLOYED
if status:
if status != helm.STATUS_FAILED:
LOG.warn(
'Unexpected release status encountered '
'release=%s, status=%s', release_id, status)
# Make best effort to determine whether a deployment is
# likely pending, by checking if the last deployment
# was started within the timeout window of the chart.
last_deployment_age = r.get_last_deployment_age(
old_release)
likely_pending = last_deployment_age <= wait_timeout
if likely_pending:
# We don't take any deploy action and wait for the
# to get deployed.
deploy = noop
deadline = deadline - last_deployment_age
else:
# Release is likely stuck in an unintended
# state. Log and continue on with remediation steps
# below.
LOG.info(
'Old release %s likely stuck in status %s, '
'(last deployment age=%ss) >= '
'(chart wait timeout=%ss)', release, status,
last_deployment_age, wait_timeout)
res = self.purge_release(chart, release_id, status,
manifest_name, chart_name,
result)
if isinstance(res, dict):
if 'protected' in res:
return res
action = metrics.ChartDeployAction.INSTALL
deploy = install
else:
# The chart is in Failed state, hence we purge
# the chart and attempt to install it again.
res = self.purge_release(chart, release_id, status,
manifest_name, chart_name, result)
if isinstance(res, dict):
if 'protected' in res:
return res
action = metrics.ChartDeployAction.INSTALL
deploy = install
if status is None:
action = metrics.ChartDeployAction.INSTALL
deploy = install
# Deploy
with metrics.CHART_DEPLOY.get_context(wait_timeout, manifest_name,
chart_name,
action.get_label_value()):
deploy()
# Wait
timer = int(round(deadline - time.time()))
chart_wait.wait(timer)
# Test
just_deployed = ('install' in result) or ('upgrade' in result)
last_test_passed = old_release and r.get_last_test_result(old_release)
test_handler = Test(chart,
release_id,
self.helm,
cg_test_charts=cg_test_all_charts)
run_test = test_handler.test_enabled and (just_deployed
or not last_test_passed)
if run_test:
with metrics.CHART_TEST.get_context(test_handler.timeout,
manifest_name, chart_name):
self._test_chart(test_handler)
return result
def sync(self):
'''
Synchronize Helm with the Armada Config(s)
'''
msg = {'install': [], 'upgrade': [], 'diff': []}
# TODO: (gardlt) we need to break up this func into
# a more cleaner format
self.pre_flight_ops()
# extract known charts on tiller right now
known_releases = self.tiller.list_charts()
manifest_data = self.manifest.get(KEYWORD_ARMADA, {})
prefix = manifest_data.get(KEYWORD_PREFIX, '')
for chartgroup in manifest_data.get(KEYWORD_GROUPS, []):
cg_name = chartgroup.get('name', '<missing name>')
cg_desc = chartgroup.get('description', '<missing description>')
LOG.info('Processing ChartGroup: %s (%s)', cg_name, cg_desc)
cg_sequenced = chartgroup.get('sequenced', False)
cg_test_all_charts = chartgroup.get('test_charts', False)
namespaces_seen = set()
tests_to_run = []
cg_charts = chartgroup.get(KEYWORD_CHARTS, [])
# Track largest Chart timeout to stop the ChartGroup at the end
cg_max_timeout = 0
for chart_entry in cg_charts:
chart = chart_entry.get('chart', {})
namespace = chart.get('namespace')
release = chart.get('release')
values = chart.get('values', {})
pre_actions = {}
post_actions = {}
wait_timeout = self.timeout
wait_labels = {}
release_name = release_prefix(prefix, release)
# Retrieve appropriate timeout value
if wait_timeout <= 0:
# TODO(MarshM): chart's `data.timeout` should be deprecated
chart_timeout = chart.get('timeout', 0)
# Favor data.wait.timeout over data.timeout, until removed
wait_values = chart.get('wait', {})
wait_timeout = wait_values.get('timeout', chart_timeout)
wait_labels = wait_values.get('labels', {})
this_chart_should_wait = (cg_sequenced or self.force_wait
or wait_timeout > 0
or len(wait_labels) > 0)
if this_chart_should_wait and wait_timeout <= 0:
LOG.warn('No Chart timeout specified, using default: %ss',
DEFAULT_CHART_TIMEOUT)
wait_timeout = DEFAULT_CHART_TIMEOUT
# Track namespaces + labels touched
namespaces_seen.add((namespace, tuple(wait_labels.items())))
# Naively take largest timeout to apply at end
# TODO(MarshM) better handling of timeout/timer
cg_max_timeout = max(wait_timeout, cg_max_timeout)
# Chart test policy can override ChartGroup, if specified
test_this_chart = chart.get('test', cg_test_all_charts)
chartbuilder = ChartBuilder(chart)
protoc_chart = chartbuilder.get_helm_chart()
deployed_releases = [x[0] for x in known_releases]
# Begin Chart timeout deadline
deadline = time.time() + wait_timeout
# TODO(mark-burnett): It may be more robust to directly call
# tiller status to decide whether to install/upgrade rather
# than checking for list membership.
if release_name in deployed_releases:
# indicate to the end user what path we are taking
LOG.info("Upgrading release %s in namespace %s",
release_name, namespace)
# extract the installed chart and installed values from the
# latest release so we can compare to the intended state
apply_chart, apply_values = self.find_release_chart(
known_releases, release_name)
upgrade = chart.get('upgrade', {})
disable_hooks = upgrade.get('no_hooks', False)
LOG.info("Checking Pre/Post Actions")
if upgrade:
upgrade_pre = upgrade.get('pre', {})
upgrade_post = upgrade.get('post', {})
if not self.disable_update_pre and upgrade_pre:
pre_actions = upgrade_pre
if not self.disable_update_post and upgrade_post:
post_actions = upgrade_post
# Show delta for both the chart templates and the chart
# values
# TODO(alanmeadows) account for .files differences
# once we support those
LOG.info('Checking upgrade chart diffs.')
upgrade_diff = self.show_diff(chart, apply_chart,
apply_values,
chartbuilder.dump(), values,
msg)
if not upgrade_diff:
LOG.info("There are no updates found in this chart")
continue
# TODO(MarshM): Add tiller dry-run before upgrade and
# consider deadline impacts
# do actual update
timer = int(round(deadline - time.time()))
LOG.info('Beginning Upgrade, wait=%s, timeout=%ss',
this_chart_should_wait, timer)
tiller_result = self.tiller.update_release(
protoc_chart,
release_name,
namespace,
pre_actions=pre_actions,
post_actions=post_actions,
dry_run=self.dry_run,
disable_hooks=disable_hooks,
values=yaml.safe_dump(values),
wait=this_chart_should_wait,
timeout=timer)
if this_chart_should_wait:
self.tiller.k8s.wait_until_ready(
release=release_name,
labels=wait_labels,
namespace=namespace,
k8s_wait_attempts=self.k8s_wait_attempts,
k8s_wait_attempt_sleep=self.k8s_wait_attempt_sleep,
timeout=timer)
LOG.info('Upgrade completed with results from Tiller: %s',
tiller_result.__dict__)
msg['upgrade'].append(release_name)
# process install
else:
LOG.info("Installing release %s in namespace %s",
release_name, namespace)
timer = int(round(deadline - time.time()))
LOG.info('Beginning Install, wait=%s, timeout=%ss',
this_chart_should_wait, timer)
tiller_result = self.tiller.install_release(
protoc_chart,
release_name,
namespace,
dry_run=self.dry_run,
values=yaml.safe_dump(values),
wait=this_chart_should_wait,
timeout=timer)
if this_chart_should_wait:
self.tiller.k8s.wait_until_ready(
release=release_name,
labels=wait_labels,
namespace=namespace,
k8s_wait_attempts=self.k8s_wait_attempts,
k8s_wait_attempt_sleep=self.k8s_wait_attempt_sleep,
timeout=timer)
LOG.info('Install completed with results from Tiller: %s',
tiller_result.__dict__)
msg['install'].append(release_name)
# Sequenced ChartGroup should run tests after each Chart
timer = int(round(deadline - time.time()))
if test_this_chart and cg_sequenced:
LOG.info('Running sequenced test, timeout remaining: %ss.',
timer)
if timer <= 0:
reason = ('Timeout expired before testing sequenced '
'release %s' % release_name)
LOG.error(reason)
raise ArmadaTimeoutException(reason)
self._test_chart(release_name, timer)
# Un-sequenced ChartGroup should run tests at the end
elif test_this_chart:
# Keeping track of time remaining
tests_to_run.append((release_name, timer))
# End of Charts in ChartGroup
LOG.info('All Charts applied.')
# After all Charts are applied, we should wait for the entire
# ChartGroup to become healthy by looking at the namespaces seen
# TODO(MarshM): Need to restrict to only releases we processed
# TODO(MarshM): Need to determine a better timeout
# (not cg_max_timeout)
if cg_max_timeout <= 0:
cg_max_timeout = DEFAULT_CHART_TIMEOUT
deadline = time.time() + cg_max_timeout
for (ns, labels) in namespaces_seen:
labels_dict = dict(labels)
timer = int(round(deadline - time.time()))
LOG.info(
'Final wait for healthy namespace (%s), label=(%s), '
'timeout remaining: %ss.', ns, labels_dict, timer)
if timer <= 0:
reason = ('Timeout expired waiting on namespace: %s, '
'label: %s' % (ns, labels_dict))
LOG.error(reason)
raise ArmadaTimeoutException(reason)
self.tiller.k8s.wait_until_ready(
namespace=ns,
labels=labels_dict,
k8s_wait_attempts=self.k8s_wait_attempts,
k8s_wait_attempt_sleep=self.k8s_wait_attempt_sleep,
timeout=timer)
# After entire ChartGroup is healthy, run any pending tests
for (test, test_timer) in tests_to_run:
self._test_chart(test, test_timer)
LOG.info("Performing Post-Flight Operations")
self.post_flight_ops()
if self.enable_chart_cleanup:
self.tiller.chart_cleanup(
prefix, self.manifest[KEYWORD_ARMADA][KEYWORD_GROUPS])
return msg
def execute(self, chart, cg_test_all_charts, prefix, known_releases):
namespace = chart.get('namespace')
release = chart.get('release')
release_name = r.release_prefixer(prefix, release)
LOG.info('Processing Chart, release=%s', release_name)
values = chart.get('values', {})
pre_actions = {}
post_actions = {}
result = {}
protected = chart.get('protected', {})
p_continue = protected.get('continue_processing', False)
old_release = self.find_chart_release(known_releases, release_name)
status = None
if old_release:
status = r.get_release_status(old_release)
if status not in [const.STATUS_FAILED, const.STATUS_DEPLOYED]:
raise armada_exceptions.UnexpectedReleaseStatusException(
release_name, status)
chart_wait = ChartWait(
self.tiller.k8s,
release_name,
chart,
namespace,
k8s_wait_attempts=self.k8s_wait_attempts,
k8s_wait_attempt_sleep=self.k8s_wait_attempt_sleep,
timeout=self.timeout)
native_wait_enabled = chart_wait.is_native_enabled()
# Begin Chart timeout deadline
deadline = time.time() + chart_wait.get_timeout()
chartbuilder = ChartBuilder(chart)
new_chart = chartbuilder.get_helm_chart()
# Check for existing FAILED release, and purge
if status == const.STATUS_FAILED:
LOG.info('Purging FAILED release %s before deployment.',
release_name)
if protected:
if p_continue:
LOG.warn(
'Release %s is `protected`, '
'continue_processing=True. Operator must '
'handle FAILED release manually.', release_name)
result['protected'] = release_name
return result
else:
LOG.error(
'Release %s is `protected`, '
'continue_processing=False.', release_name)
raise armada_exceptions.ProtectedReleaseException(
release_name)
else:
# Purge the release
self.tiller.uninstall_release(release_name)
result['purge'] = release_name
# TODO(mark-burnett): It may be more robust to directly call
# tiller status to decide whether to install/upgrade rather
# than checking for list membership.
if status == const.STATUS_DEPLOYED:
# indicate to the end user what path we are taking
LOG.info("Existing release %s found in namespace %s", release_name,
namespace)
# extract the installed chart and installed values from the
# latest release so we can compare to the intended state
old_chart = old_release.chart
old_values_string = old_release.config.raw
upgrade = chart.get('upgrade', {})
disable_hooks = upgrade.get('no_hooks', False)
options = upgrade.get('options', {})
force = options.get('force', False)
recreate_pods = options.get('recreate_pods', False)
if upgrade:
upgrade_pre = upgrade.get('pre', {})
upgrade_post = upgrade.get('post', {})
if not self.disable_update_pre and upgrade_pre:
pre_actions = upgrade_pre
if not self.disable_update_post and upgrade_post:
LOG.warning('Post upgrade actions are ignored by Armada'
'and will not affect deployment.')
post_actions = upgrade_post
try:
old_values = yaml.safe_load(old_values_string)
except yaml.YAMLError:
chart_desc = '{} (previously deployed)'.format(
old_chart.metadata.name)
raise armada_exceptions.\
InvalidOverrideValuesYamlException(chart_desc)
LOG.info('Checking for updates to chart release inputs.')
diff = self.get_diff(old_chart, old_values, new_chart, values)
if not diff:
LOG.info("Found no updates to chart release inputs")
else:
LOG.info("Found updates to chart release inputs")
LOG.debug("%s", diff)
result['diff'] = {chart['release']: str(diff)}
# TODO(MarshM): Add tiller dry-run before upgrade and
# consider deadline impacts
# do actual update
timer = int(round(deadline - time.time()))
LOG.info(
"Upgrading release %s in namespace %s, wait=%s, "
"timeout=%ss", release_name, namespace,
native_wait_enabled, timer)
tiller_result = self.tiller.update_release(
new_chart,
release_name,
namespace,
pre_actions=pre_actions,
post_actions=post_actions,
disable_hooks=disable_hooks,
values=yaml.safe_dump(values),
wait=native_wait_enabled,
timeout=timer,
force=force,
recreate_pods=recreate_pods)
LOG.info('Upgrade completed with results from Tiller: %s',
tiller_result.__dict__)
result['upgrade'] = release_name
else:
timer = int(round(deadline - time.time()))
LOG.info(
"Installing release %s in namespace %s, wait=%s, "
"timeout=%ss", release_name, namespace, native_wait_enabled,
timer)
tiller_result = self.tiller.install_release(
new_chart,
release_name,
namespace,
values=yaml.safe_dump(values),
wait=native_wait_enabled,
timeout=timer)
LOG.info('Install completed with results from Tiller: %s',
tiller_result.__dict__)
result['install'] = release_name
# Wait
timer = int(round(deadline - time.time()))
chart_wait.wait(timer)
# Test
just_deployed = ('install' in result) or ('upgrade' in result)
last_test_passed = old_release and r.get_last_test_result(old_release)
test_values = chart.get('test')
test_handler = Test(release_name,
self.tiller,
cg_test_charts=cg_test_all_charts,
test_values=test_values)
run_test = test_handler.test_enabled and (just_deployed
or not last_test_passed)
if run_test:
timer = int(round(deadline - time.time()))
self._test_chart(release_name, timer, test_handler)
return result
def sync(self):
'''
Syncronize Helm with the Armada Config(s)
'''
msg = {'install': [], 'upgrade': [], 'diff': []}
# TODO: (gardlt) we need to break up this func into
# a more cleaner format
LOG.info("Performing Pre-Flight Operations")
self.pre_flight_ops()
# extract known charts on tiller right now
known_releases = self.tiller.list_charts()
prefix = self.config.get(const.KEYWORD_ARMADA).get(
const.KEYWORD_PREFIX)
if known_releases is None:
raise armada_exceptions.KnownReleasesException()
for release in known_releases:
LOG.debug("Release %s, Version %s found on tiller", release[0],
release[1])
for entry in self.config[const.KEYWORD_ARMADA][const.KEYWORD_GROUPS]:
chart_wait = self.wait
desc = entry.get('description', 'A Chart Group')
chart_group = entry.get(const.KEYWORD_CHARTS, [])
test_charts = entry.get('test_charts', False)
if entry.get('sequenced', False) or test_charts:
chart_wait = True
LOG.info('Deploying: %s', desc)
for gchart in chart_group:
chart = dotify(gchart['chart'])
values = gchart.get('chart').get('values', {})
wait_values = gchart.get('chart').get('wait', {})
test_chart = gchart.get('chart').get('test', False)
pre_actions = {}
post_actions = {}
if chart.release is None:
continue
if test_chart:
chart_wait = True
# retrieve appropriate timeout value if 'wait' is specified
chart_timeout = self.timeout
if chart_wait:
if chart_timeout == DEFAULT_TIMEOUT:
chart_timeout = getattr(chart, 'timeout',
chart_timeout)
chartbuilder = ChartBuilder(chart)
protoc_chart = chartbuilder.get_helm_chart()
# determine install or upgrade by examining known releases
LOG.debug("RELEASE: %s", chart.release)
deployed_releases = [x[0] for x in known_releases]
prefix_chart = release_prefix(prefix, chart.release)
if prefix_chart in deployed_releases:
# indicate to the end user what path we are taking
LOG.info("Upgrading release %s", chart.release)
# extract the installed chart and installed values from the
# latest release so we can compare to the intended state
LOG.info("Checking Pre/Post Actions")
apply_chart, apply_values = self.find_release_chart(
known_releases, prefix_chart)
LOG.info("Checking Pre/Post Actions")
upgrade = gchart.get('chart', {}).get('upgrade', False)
if upgrade:
if not self.disable_update_pre and upgrade.get(
'pre', False):
pre_actions = getattr(chart.upgrade, 'pre', {})
if not self.disable_update_post and upgrade.get(
'post', False):
post_actions = getattr(chart.upgrade, 'post', {})
# show delta for both the chart templates and the chart
# values
# TODO(alanmeadows) account for .files differences
# once we support those
upgrade_diff = self.show_diff(chart, apply_chart,
apply_values,
chartbuilder.dump(), values,
msg)
if not upgrade_diff:
LOG.info("There are no updates found in this chart")
continue
# do actual update
LOG.info('wait: %s', chart_wait)
self.tiller.update_release(
protoc_chart,
prefix_chart,
chart.namespace,
pre_actions=pre_actions,
post_actions=post_actions,
dry_run=self.dry_run,
disable_hooks=chart.upgrade.no_hooks,
values=yaml.safe_dump(values),
wait=chart_wait,
timeout=chart_timeout)
if chart_wait:
# TODO(gardlt): after v0.7.1 depricate timeout values
if not wait_values.get('timeout', None):
wait_values['timeout'] = chart_timeout
self.tiller.k8s.wait_until_ready(
release=prefix_chart,
labels=wait_values.get('labels', ''),
namespace=chart.namespace,
timeout=wait_values.get('timeout',
DEFAULT_TIMEOUT))
msg['upgrade'].append(prefix_chart)
# process install
else:
LOG.info("Installing release %s", chart.release)
self.tiller.install_release(protoc_chart,
prefix_chart,
chart.namespace,
dry_run=self.dry_run,
values=yaml.safe_dump(values),
wait=chart_wait,
timeout=chart_timeout)
if chart_wait:
if not wait_values.get('timeout', None):
wait_values['timeout'] = chart_timeout
self.tiller.k8s.wait_until_ready(
release=prefix_chart,
labels=wait_values.get('labels', ''),
namespace=chart.namespace,
timeout=wait_values.get('timeout', 3600))
msg['install'].append(prefix_chart)
LOG.debug("Cleaning up chart source in %s",
chartbuilder.source_directory)
if test_charts or test_chart:
LOG.info('Testing: %s', prefix_chart)
resp = self.tiller.testing_release(prefix_chart)
test_status = getattr(resp.info.status,
'last_test_suite_run', 'FAILED')
LOG.info("Test INFO: %s", test_status)
if resp:
LOG.info("PASSED: %s", prefix_chart)
else:
LOG.info("FAILED: %s", prefix_chart)
self.tiller.k8s.wait_until_ready(timeout=chart_timeout)
LOG.info("Performing Post-Flight Operations")
self.post_flight_ops()
if self.enable_chart_cleanup:
self.tiller.chart_cleanup(
prefix,
self.config[const.KEYWORD_ARMADA][const.KEYWORD_GROUPS])
return msg
def sync(self):
'''
Synchronize Helm with the Armada Config(s)
'''
if self.dry_run:
LOG.info('Armada is in DRY RUN mode, no changes being made.')
msg = {
'install': [],
'upgrade': [],
'diff': [],
'purge': [],
'protected': []
}
# TODO: (gardlt) we need to break up this func into
# a more cleaner format
self.pre_flight_ops()
# extract known charts on tiller right now
deployed_releases, failed_releases = self._get_releases_by_status()
manifest_data = self.manifest.get(const.KEYWORD_ARMADA, {})
prefix = manifest_data.get(const.KEYWORD_PREFIX)
for chartgroup in manifest_data.get(const.KEYWORD_GROUPS, []):
cg_name = chartgroup.get('name', '<missing name>')
cg_desc = chartgroup.get('description', '<missing description>')
cg_sequenced = chartgroup.get('sequenced', False)
LOG.info('Processing ChartGroup: %s (%s), sequenced=%s', cg_name,
cg_desc, cg_sequenced)
# TODO(MarshM): Deprecate the `test_charts` key
cg_test_all_charts = chartgroup.get('test_charts')
if isinstance(cg_test_all_charts, bool):
LOG.warn('The ChartGroup `test_charts` key is deprecated, '
'and support for this will be removed. See the '
'Chart `test` key for more information.')
else:
# This key defaults to True. Individual charts must
# explicitly disable helm tests if they choose
cg_test_all_charts = True
ns_label_set = set()
tests_to_run = []
cg_charts = chartgroup.get(const.KEYWORD_CHARTS, [])
# Track largest Chart timeout to stop the ChartGroup at the end
cg_max_timeout = 0
for chart_entry in cg_charts:
chart = chart_entry.get('chart', {})
namespace = chart.get('namespace')
release = chart.get('release')
release_name = release_prefixer(prefix, release)
LOG.info('Processing Chart, release=%s', release_name)
values = chart.get('values', {})
pre_actions = {}
post_actions = {}
protected = chart.get('protected', {})
p_continue = protected.get('continue_processing', False)
# Check for existing FAILED release, and purge
if release_name in [rel[0] for rel in failed_releases]:
LOG.info('Purging FAILED release %s before deployment.',
release_name)
if protected:
if p_continue:
LOG.warn(
'Release %s is `protected`, '
'continue_processing=True. Operator must '
'handle FAILED release manually.',
release_name)
msg['protected'].append(release_name)
continue
else:
LOG.error(
'Release %s is `protected`, '
'continue_processing=False.', release_name)
raise armada_exceptions.ProtectedReleaseException(
release_name)
else:
# Purge the release
self.tiller.uninstall_release(release_name)
msg['purge'].append(release_name)
# NOTE(MarshM): Calculating `wait_timeout` is unfortunately
# overly complex. The order of precedence is currently:
# 1) User provided override via API/CLI (default 0 if not
# provided by client/user).
# 2) Chart's `data.wait.timeout`, or...
# 3) Chart's `data.timeout` (deprecated).
# 4) const.DEFAULT_CHART_TIMEOUT, if nothing is ever
# specified, for use in waiting for final ChartGroup
# health and helm tests, but ignored for the actual
# install/upgrade of the Chart.
# NOTE(MarshM): Not defining a timeout has a side effect of
# allowing Armada to install charts with a circular
# dependency defined between components.
# TODO(MarshM): Deprecated, remove the following block
deprecated_timeout = chart.get('timeout', None)
if isinstance(deprecated_timeout, int):
LOG.warn('The `timeout` key is deprecated and support '
'for this will be removed soon. Use '
'`wait.timeout` instead.')
wait_values = chart.get('wait', {})
wait_labels = wait_values.get('labels', {})
wait_timeout = self.timeout
if wait_timeout <= 0:
wait_timeout = wait_values.get('timeout', wait_timeout)
# TODO(MarshM): Deprecated, remove the following check
if wait_timeout <= 0:
wait_timeout = deprecated_timeout or wait_timeout
# Determine wait logic
# NOTE(Dan Kim): Conditions to wait are below :
# 1) set sequenced=True in chart group
# 2) set force_wait param
# 3) add Chart's `data.wait.timeout`
# --timeout param will do not set wait=True, it just change
# max timeout of chart's deployment. (default: 900)
this_chart_should_wait = (cg_sequenced or self.force_wait
or (bool(wait_values) and
(wait_timeout > 0)))
# If there is still no timeout, we need to use a default
# (item 4 in note above)
if wait_timeout <= 0:
LOG.warn('No Chart timeout specified, using default: %ss',
const.DEFAULT_CHART_TIMEOUT)
wait_timeout = const.DEFAULT_CHART_TIMEOUT
# Naively take largest timeout to apply at end
# TODO(MarshM) better handling of timeout/timer
cg_max_timeout = max(wait_timeout, cg_max_timeout)
test_chart_override = chart.get('test')
# Use old default value when not using newer `test` key
test_cleanup = True
if test_chart_override is None:
test_this_chart = cg_test_all_charts
elif isinstance(test_chart_override, bool):
LOG.warn('Boolean value for chart `test` key is'
' deprecated and support for this will'
' be removed. Use `test.enabled` '
'instead.')
test_this_chart = test_chart_override
else:
# NOTE: helm tests are enabled by default
test_this_chart = test_chart_override.get('enabled', True)
test_cleanup = test_chart_override.get('options', {}).get(
'cleanup', False)
chartbuilder = ChartBuilder(chart)
new_chart = chartbuilder.get_helm_chart()
# Begin Chart timeout deadline
deadline = time.time() + wait_timeout
# TODO(mark-burnett): It may be more robust to directly call
# tiller status to decide whether to install/upgrade rather
# than checking for list membership.
if release_name in [rel[0] for rel in deployed_releases]:
# indicate to the end user what path we are taking
LOG.info("Upgrading release %s in namespace %s",
release_name, namespace)
# extract the installed chart and installed values from the
# latest release so we can compare to the intended state
old_chart, old_values_string = self.find_release_chart(
deployed_releases, release_name)
upgrade = chart.get('upgrade', {})
disable_hooks = upgrade.get('no_hooks', False)
force = upgrade.get('force', False)
recreate_pods = upgrade.get('recreate_pods', False)
LOG.info("Checking Pre/Post Actions")
if upgrade:
upgrade_pre = upgrade.get('pre', {})
upgrade_post = upgrade.get('post', {})
if not self.disable_update_pre and upgrade_pre:
pre_actions = upgrade_pre
if not self.disable_update_post and upgrade_post:
post_actions = upgrade_post
try:
old_values = yaml.safe_load(old_values_string)
except yaml.YAMLError:
chart_desc = '{} (previously deployed)'.format(
old_chart.metadata.name)
raise armada_exceptions.\
InvalidOverrideValuesYamlException(chart_desc)
LOG.info('Checking for updates to chart release inputs.')
diff = self.get_diff(old_chart, old_values, new_chart,
values)
if not diff:
LOG.info("Found no updates to chart release inputs")
continue
LOG.info("Found updates to chart release inputs")
LOG.debug("%s", diff)
msg['diff'].append({chart['release']: str(diff)})
# TODO(MarshM): Add tiller dry-run before upgrade and
# consider deadline impacts
# do actual update
timer = int(round(deadline - time.time()))
LOG.info('Beginning Upgrade, wait=%s, timeout=%ss',
this_chart_should_wait, timer)
tiller_result = self.tiller.update_release(
new_chart,
release_name,
namespace,
pre_actions=pre_actions,
post_actions=post_actions,
disable_hooks=disable_hooks,
values=yaml.safe_dump(values),
wait=this_chart_should_wait,
timeout=timer,
force=force,
recreate_pods=recreate_pods)
if this_chart_should_wait:
self._wait_until_ready(release_name, wait_labels,
namespace, timer)
# Track namespace+labels touched by upgrade
ns_label_set.add((namespace, tuple(wait_labels.items())))
LOG.info('Upgrade completed with results from Tiller: %s',
tiller_result.__dict__)
msg['upgrade'].append(release_name)
# process install
else:
LOG.info("Installing release %s in namespace %s",
release_name, namespace)
timer = int(round(deadline - time.time()))
LOG.info('Beginning Install, wait=%s, timeout=%ss',
this_chart_should_wait, timer)
tiller_result = self.tiller.install_release(
new_chart,
release_name,
namespace,
values=yaml.safe_dump(values),
wait=this_chart_should_wait,
timeout=timer)
if this_chart_should_wait:
self._wait_until_ready(release_name, wait_labels,
namespace, timer)
# Track namespace+labels touched by install
ns_label_set.add((namespace, tuple(wait_labels.items())))
LOG.info('Install completed with results from Tiller: %s',
tiller_result.__dict__)
msg['install'].append(release_name)
# Keeping track of time remaining
timer = int(round(deadline - time.time()))
test_chart_args = (release_name, timer, test_cleanup)
if test_this_chart:
# Sequenced ChartGroup should run tests after each Chart
if cg_sequenced:
LOG.info(
'Running sequenced test, timeout remaining: '
'%ss.', timer)
self._test_chart(*test_chart_args)
# Un-sequenced ChartGroup should run tests at the end
else:
tests_to_run.append(
functools.partial(self._test_chart,
*test_chart_args))
# End of Charts in ChartGroup
LOG.info('All Charts applied in ChartGroup %s.', cg_name)
# After all Charts are applied, we should wait for the entire
# ChartGroup to become healthy by looking at the namespaces seen
# TODO(MarshM): Need to determine a better timeout
# (not cg_max_timeout)
if cg_max_timeout <= 0:
cg_max_timeout = const.DEFAULT_CHART_TIMEOUT
deadline = time.time() + cg_max_timeout
for (ns, labels) in ns_label_set:
labels_dict = dict(labels)
timer = int(round(deadline - time.time()))
LOG.info(
'Final ChartGroup wait for healthy namespace=%s, '
'labels=(%s), timeout remaining: %ss.', ns, labels_dict,
timer)
if timer <= 0:
reason = ('Timeout expired waiting on namespace: %s, '
'labels: (%s)' % (ns, labels_dict))
LOG.error(reason)
raise armada_exceptions.ArmadaTimeoutException(reason)
self._wait_until_ready(release_name=None,
wait_labels=labels_dict,
namespace=ns,
timeout=timer)
# After entire ChartGroup is healthy, run any pending tests
for callback in tests_to_run:
callback()
self.post_flight_ops()
if self.enable_chart_cleanup:
self._chart_cleanup(
prefix,
self.manifest[const.KEYWORD_ARMADA][const.KEYWORD_GROUPS], msg)
LOG.info('Done applying manifest.')
return msg
