def handle(self, tiller):
known_release_names = [release[0] for release in tiller.list_charts()]
if self.releases:
target_releases = [
r.strip() for r in self.releases.split(',')
if r.strip() in known_release_names
]
if not target_releases:
self.logger.info("There's no release to delete.")
return
if not self.ctx.obj.get('api', False):
for r in target_releases:
self.logger.info("Deleting release %s", r)
tiller.delete_release(r, purge=self.purge)
else:
raise NotImplementedError()
if self.manifest:
target_deletes = []
with open(self.manifest) as f:
documents = list(yaml.safe_load_all(f.read()))
try:
armada_obj = Manifest(documents).get_manifest()
prefix = armada_obj.get(const.KEYWORD_DATA).get(
const.KEYWORD_PREFIX)
for group in armada_obj.get(const.KEYWORD_DATA).get(
const.KEYWORD_GROUPS):
for ch in group.get(const.KEYWORD_DATA).get(
const.KEYWORD_CHARTS):
chart = ch.get(const.KEYWORD_DATA)
release_name = release_prefixer(
prefix, chart.get('release'))
if release_name in known_release_names:
target_deletes.append((chart, release_name))
except yaml.YAMLError as e:
mark = e.problem_mark
self.logger.info(
"While parsing the manifest file, %s. "
"Error position: (%s:%s)", e.problem, mark.line + 1,
mark.column + 1)
if not target_deletes:
self.logger.info("There's no release to delete.")
return
if not self.ctx.obj.get('api', False):
for chart, release in target_deletes:
chart_delete = ChartDelete(chart,
release,
tiller,
purge=self.purge)
chart_delete.delete()
else:
raise NotImplementedError()
def handle(self, req, resp, tiller):
try:
documents = self.req_yaml(req, default=[])
except yaml.YAMLError:
err_message = 'Documents must be valid YAML.'
return self.return_error(
resp, falcon.HTTP_400, message=err_message)
target_manifest = req.get_param('target_manifest', None)
is_valid = self._validate_documents(req, resp, documents)
if not is_valid:
return
armada_obj = Manifest(
documents, target_manifest=target_manifest).get_manifest()
prefix = armada_obj.get(const.KEYWORD_ARMADA).get(const.KEYWORD_PREFIX)
known_releases = [release[0] for release in tiller.list_charts()]
message = {'tests': {'passed': [], 'skipped': [], 'failed': []}}
for group in armada_obj.get(const.KEYWORD_ARMADA).get(
const.KEYWORD_GROUPS):
for ch in group.get(const.KEYWORD_CHARTS):
chart = ch['chart']
release_name = release_prefixer(prefix, chart.get('release'))
if release_name in known_releases:
cleanup = req.get_param_as_bool('cleanup')
enable_all = req.get_param_as_bool('enable_all')
cg_test_charts = group.get('test_charts')
test_values = chart.get('test', {})
test_handler = Test(
release_name,
tiller,
cg_test_charts=cg_test_charts,
cleanup=cleanup,
enable_all=enable_all,
test_values=test_values)
if test_handler.test_enabled:
success = test_handler.test_release_for_success()
if success:
message['test']['passed'].append(release_name)
else:
message['test']['failed'].append(release_name)
else:
self.logger.info('Release %s not found - SKIPPING',
release_name)
message['test']['skipped'].append(release_name)
resp.status = falcon.HTTP_200
resp.body = json.dumps(message)
resp.content_type = 'application/json'
def handle(self, req, resp, helm):
try:
documents = self.req_yaml(req, default=[])
except yaml.YAMLError:
err_message = 'Documents must be valid YAML.'
return self.return_error(resp,
falcon.HTTP_400,
message=err_message)
target_manifest = req.get_param('target_manifest', None)
is_valid = self._validate_documents(req, resp, documents)
if not is_valid:
return
armada_obj = Manifest(documents,
target_manifest=target_manifest).get_manifest()
prefix = armada_obj[const.KEYWORD_DATA][const.KEYWORD_PREFIX]
release_ids = helm.list_release_ids()
message = {'tests': {'passed': [], 'skipped': [], 'failed': []}}
for group in armada_obj.get(const.KEYWORD_DATA).get(
const.KEYWORD_GROUPS):
for ch in group.get(const.KEYWORD_CHARTS):
chart = ch['chart']
release_id = helm.HelmReleaseId(
ch['namespace'], release_prefixer(prefix, ch['release']))
if release_id in release_ids:
enable_all = req.get_param_as_bool('enable_all')
cg_test_charts = group.get('test_charts')
test_handler = Test(chart,
release_id,
helm,
cg_test_charts=cg_test_charts,
enable_all=enable_all)
if test_handler.test_enabled:
success = test_handler.test_release_for_success()
if success:
message['test']['passed'].append(release_id)
else:
message['test']['failed'].append(release_id)
else:
self.logger.info('Release %s not found - SKIPPING',
release_id)
message['test']['skipped'].append(release_id)
resp.status = falcon.HTTP_200
resp.text = json.dumps(message)
resp.content_type = 'application/json'
def handle(self, helm):
release_ids = helm.list_release_ids()
if self.release:
if not self.ctx.obj.get('api', False):
release_id = HelmReleaseId(self.namespace, self.release)
test_handler = Test({}, release_id, helm)
test_handler.test_release_for_success()
else:
client = self.ctx.obj.get('CLIENT')
resp = client.get_test_release(release=self.release)
self.logger.info(resp.get('result'))
self.logger.info(resp.get('message'))
if self.file:
if not self.ctx.obj.get('api', False):
documents = list(yaml.safe_load_all(open(self.file).read()))
armada_obj = Manifest(
documents,
target_manifest=self.target_manifest).get_manifest()
prefix = armada_obj.get(const.KEYWORD_DATA).get(
const.KEYWORD_PREFIX)
for group in armada_obj.get(const.KEYWORD_DATA).get(
const.KEYWORD_GROUPS):
for ch in group.get(const.KEYWORD_CHARTS):
chart = ch['chart']
release_id = HelmReleaseId(
chart['namespace'],
release_prefixer(prefix, chart['release']))
if release_id in release_ids:
test_handler = Test(chart,
release_id,
helm,
enable_all=self.enable_all)
if test_handler.test_enabled:
test_handler.test_release_for_success()
else:
self.logger.info('Release %s not found - SKIPPING',
release_id)
else:
client = self.ctx.obj.get('CLIENT')
with open(self.filename, 'r') as f:
resp = client.get_test_manifest(manifest=f.read())
for test in resp.get('tests'):
self.logger.info('Test State: %s', test)
for item in test.get('tests').get(test):
self.logger.info(item)
self.logger.info(resp)
def _chart_cleanup(self, prefix, charts, msg):
LOG.info('Processing chart cleanup to remove unspecified releases.')
valid_releases = []
for gchart in charts:
for chart in gchart.get(const.KEYWORD_CHARTS, []):
valid_releases.append(
release_prefixer(prefix,
chart.get('chart', {}).get('release')))
actual_releases = [x.name for x in self.tiller.list_releases()]
release_diff = list(set(actual_releases) - set(valid_releases))
for release in release_diff:
if release.startswith(prefix):
LOG.info('Purging release %s as part of chart cleanup.',
release)
self.tiller.uninstall_release(release)
msg['purge'].append(release)
def _chart_cleanup(self, prefix, chart_groups, msg):
LOG.info('Processing chart cleanup to remove unspecified releases.')
valid_release_ids = []
for group in chart_groups:
group_data = group.get(const.KEYWORD_DATA, {})
for chart in group_data.get(const.KEYWORD_CHARTS, []):
chart_data = chart.get(const.KEYWORD_DATA, {})
valid_release_ids.append(
HelmReleaseId(
chart_data['namespace'],
release_prefixer(prefix, chart_data['release'])))
actual_release_ids = self.helm.list_release_ids()
release_diff = list(set(actual_release_ids) - set(valid_release_ids))
for release_id in release_diff:
if release_id.name.startswith(prefix):
LOG.info('Purging release %s as part of chart cleanup.',
release_id)
self.helm.uninstall_release(release_id)
msg['purge'].append('{}'.format(release_id))
def _do_test(mock_test, mock_chartbuilder, MockChartDownload,
mock_post_flight):
MockChartDownload.return_value.get_chart.side_effect = \
set_source_dir
# Instantiate Armada object.
yaml_documents = list(yaml.safe_load_all(TEST_YAML))
m_helm = mock.MagicMock()
armada_obj = armada.Armada(yaml_documents, m_helm)
prefix = armada_obj.manifest['data']['release_prefix']
def release_metadata(release_id, **kwargs):
try:
return next(r for r in known_releases
if release_id.name == r['name']
and release_id.namespace == r['namespace'])
except StopIteration:
return None
m_helm.release_metadata.side_effect = release_metadata
armada_obj.chart_deploy.get_diff = mock.Mock()
cg = armada_obj.manifest['data']['chart_groups'][0]
chart_group = cg['data']
charts = chart_group['chart_group']
cg_test_all_charts = chart_group.get('test_charts')
mock_test_release = mock_test.return_value.test_release
if test_failure_to_run:
mock_test_release.side_effect = Exception('test failed to run')
else:
if not test_success:
mock_test_release.side_effect = Exception('test failed')
mock_test.return_value.timeout = const.DEFAULT_TEST_TIMEOUT
# Stub out irrelevant methods called by `armada.sync()`.
mock_chartbuilder.get_helm_chart.return_value = None
# Simulate chart diff, upgrade should only happen if non-empty.
armada_obj.chart_deploy.get_diff.return_value = diff
armada_obj.sync()
expected_install_release_calls = []
expected_upgrade_release_calls = []
expected_uninstall_release_calls = []
expected_test_constructor_calls = []
for c in charts:
chart = c['data']
release = chart['release']
release_name = release_prefixer(prefix, release)
release_id = helm.HelmReleaseId(chart['namespace'],
release_name)
source_dir = chart['source_dir']
source_directory = os.path.join(*source_dir)
# Simplified check because the actual code uses logical-or's
# multiple conditions, so this is enough.
native_wait_enabled = (chart['wait'].get('native', {}).get(
'enabled', True))
if release_name not in [x['name'] for x in known_releases]:
expected_install_release_calls.append(
mock.call(source_directory,
release_id,
values=chart['values'],
wait=native_wait_enabled,
timeout=mock.ANY))
else:
target_release = None
for known_release in known_releases:
if known_release['name'] == release_name:
target_release = known_release
break
if target_release:
status = get_release_status(target_release)
if status == helm.STATUS_FAILED:
protected = chart.get('protected', {})
if not protected:
expected_uninstall_release_calls.append(
mock.call(
release_id,
purge=True,
timeout=const.DEFAULT_DELETE_TIMEOUT))
expected_install_release_calls.append(
mock.call(source_directory,
release_id,
values=chart['values'],
wait=native_wait_enabled,
timeout=mock.ANY))
else:
p_continue = protected.get(
'continue_processing', False)
if p_continue:
continue
else:
if chart_group['sequenced']:
break
if status == helm.STATUS_DEPLOYED:
if diff:
upgrade = chart.get('upgrade', {})
disable_hooks = upgrade.get('no_hooks', False)
options = upgrade.get('options', {})
force = options.get('force', False)
expected_upgrade_release_calls.append(
mock.call(source_directory,
release_id,
disable_hooks=disable_hooks,
force=force,
values=chart['values'],
wait=native_wait_enabled,
timeout=mock.ANY))
expected_test_constructor_calls.append(
mock.call(chart,
release_id,
m_helm,
cg_test_charts=cg_test_all_charts))
any_order = not chart_group['sequenced']
# Verify that at least 1 release is either installed or updated.
self.assertTrue(
len(expected_install_release_calls) >= 1
or len(expected_upgrade_release_calls) >= 1)
# Verify that the expected number of non-deployed releases are
# installed with expected arguments.
self.assertEqual(len(expected_install_release_calls),
m_helm.install_release.call_count)
m_helm.install_release.assert_has_calls(
expected_install_release_calls, any_order=any_order)
# Verify that the expected number of deployed releases are
# updated with expected arguments.
self.assertEqual(len(expected_upgrade_release_calls),
m_helm.upgrade_release.call_count)
m_helm.upgrade_release.assert_has_calls(
expected_upgrade_release_calls, any_order=any_order)
# Verify that the expected number of deployed releases are
# uninstalled with expected arguments.
self.assertEqual(len(expected_uninstall_release_calls),
m_helm.uninstall_release.call_count)
m_helm.uninstall_release.assert_has_calls(
expected_uninstall_release_calls, any_order=any_order)
# Verify that the expected number of deployed releases are
# tested with expected arguments.
self.assertEqual(len(expected_test_constructor_calls),
mock_test.call_count)
mock_test.assert_has_calls(expected_test_constructor_calls,
any_order=True)
def test_release_prefix_int_int(self):
expected = '4-4'
prefix, release = (4, 4)
assert rel.release_prefixer(prefix, release) == expected
def test_release_prefix_int_string(self):
expected = 'armada-4'
prefix, release = ('armada', 4)
assert rel.release_prefixer(prefix, release) == expected
def test_release_prefix_pass(self):
expected = 'armada-test'
prefix, release = ('armada', 'test')
assert rel.release_prefixer(prefix, release) == expected
def on_post(self, req, resp):
# TODO(fmontei): Validation Content-Type is application/x-yaml.
target_manifest = req.get_param('target_manifest', None)
try:
tiller = Tiller(tiller_host=req.get_param('tiller_host'),
tiller_port=req.get_param_as_int('tiller_port')
or CONF.tiller_port,
tiller_namespace=req.get_param(
'tiller_namespace',
default=CONF.tiller_namespace))
# TODO(fmontei): Provide more sensible exception(s) here.
except Exception:
err_message = 'Failed to initialize Tiller handler.'
self.error(req.context, err_message)
return self.return_error(resp,
falcon.HTTP_500,
message=err_message)
try:
documents = self.req_yaml(req, default=[])
except yaml.YAMLError:
err_message = 'Documents must be valid YAML.'
return self.return_error(resp,
falcon.HTTP_400,
message=err_message)
is_valid = self._validate_documents(req, resp, documents)
if not is_valid:
return resp
armada_obj = Manifest(documents,
target_manifest=target_manifest).get_manifest()
prefix = armada_obj.get(const.KEYWORD_ARMADA).get(const.KEYWORD_PREFIX)
known_releases = [release[0] for release in tiller.list_charts()]
message = {'tests': {'passed': [], 'skipped': [], 'failed': []}}
for group in armada_obj.get(const.KEYWORD_ARMADA).get(
const.KEYWORD_GROUPS):
for ch in group.get(const.KEYWORD_CHARTS):
chart = ch['chart']
release_name = release_prefixer(prefix, chart.get('release'))
cleanup = req.get_param_as_bool('cleanup')
if cleanup is None:
test_chart_override = chart.get('test', {})
if isinstance(test_chart_override, bool):
self.logger.warn(
'Boolean value for chart `test` key is deprecated '
'and will be removed. Use `test.enabled` instead.')
# Use old default value.
cleanup = True
else:
cleanup = test_chart_override.get('options', {}).get(
'cleanup', False)
if release_name in known_releases:
self.logger.info('RUNNING: %s tests', release_name)
success = test_release_for_success(tiller,
release_name,
cleanup=cleanup)
if success:
self.logger.info("PASSED: %s", release_name)
message['test']['passed'].append(release_name)
else:
self.logger.info("FAILED: %s", release_name)
message['test']['failed'].append(release_name)
else:
self.logger.info('Release %s not found - SKIPPING',
release_name)
message['test']['skipped'].append(release_name)
resp.status = falcon.HTTP_200
resp.body = json.dumps(message)
resp.content_type = 'application/json'
def _execute(self, ch, cg_test_all_charts, prefix):
manifest_name = self.manifest['metadata']['name']
chart = ch[const.KEYWORD_DATA]
chart_name = ch['metadata']['name']
namespace = chart.get('namespace')
release = chart.get('release')
release_name = r.release_prefixer(prefix, release)
release_id = helm.HelmReleaseId(namespace, release_name)
source_dir = chart['source_dir']
source_directory = os.path.join(*source_dir)
LOG.info('Processing Chart, release=%s', release_id)
result = {}
chart_wait = ChartWait(
self.helm.k8s,
release_id,
ch,
k8s_wait_attempts=self.k8s_wait_attempts,
k8s_wait_attempt_sleep=self.k8s_wait_attempt_sleep,
timeout=self.timeout)
wait_timeout = chart_wait.get_timeout()
# Begin Chart timeout deadline
deadline = time.time() + wait_timeout
old_release = self.helm.release_metadata(release_id)
action = metrics.ChartDeployAction.NOOP
def noop():
pass
deploy = noop
# Resolve action
values = chart.get('values', {})
pre_actions = {}
status = None
if old_release:
status = r.get_release_status(old_release)
native_wait_enabled = chart_wait.is_native_enabled()
chartbuilder = ChartBuilder.from_chart_doc(ch, self.helm)
if status == helm.STATUS_DEPLOYED:
# indicate to the end user what path we are taking
LOG.info("Existing release %s found", release_id)
# extract the installed chart and installed values from the
# latest release so we can compare to the intended state
old_chart = old_release['chart']
old_values = old_release.get('config', {})
upgrade = chart.get('upgrade', {})
options = upgrade.get('options', {})
# TODO: Remove when v1 doc support is removed.
schema_info = get_schema_info(ch['schema'])
if schema_info.version < 2:
no_hooks_location = upgrade
else:
no_hooks_location = options
disable_hooks = no_hooks_location.get('no_hooks', False)
force = options.get('force', False)
if upgrade:
upgrade_pre = upgrade.get('pre', {})
upgrade_post = upgrade.get('post', {})
if not self.disable_update_pre and upgrade_pre:
pre_actions = upgrade_pre
if not self.disable_update_post and upgrade_post:
LOG.warning('Post upgrade actions are ignored by Armada'
'and will not affect deployment.')
LOG.info('Checking for updates to chart release inputs.')
new_chart = chartbuilder.get_helm_chart(release_id, values)
diff = self.get_diff(old_chart, old_values, new_chart, values)
if not diff:
LOG.info("Found no updates to chart release inputs")
else:
action = metrics.ChartDeployAction.UPGRADE
LOG.info("Found updates to chart release inputs")
def upgrade():
# do actual update
timer = int(round(deadline - time.time()))
PreUpdateActions(self.helm.k8s).execute(
pre_actions, release, namespace, chart, disable_hooks,
values, timer)
LOG.info("Upgrading release=%s, wait=%s, "
"timeout=%ss", release_id, native_wait_enabled,
timer)
self.helm.upgrade_release(source_directory,
release_id,
disable_hooks=disable_hooks,
values=values,
wait=native_wait_enabled,
timeout=timer,
force=force)
LOG.info('Upgrade completed')
result['upgrade'] = release_id
deploy = upgrade
else:
def install():
timer = int(round(deadline - time.time()))
LOG.info("Installing release=%s, wait=%s, "
"timeout=%ss", release_id, native_wait_enabled, timer)
self.helm.install_release(source_directory,
release_id,
values=values,
wait=native_wait_enabled,
timeout=timer)
LOG.info('Install completed')
result['install'] = release_id
# Check for release with status other than DEPLOYED
if status:
if status != helm.STATUS_FAILED:
LOG.warn(
'Unexpected release status encountered '
'release=%s, status=%s', release_id, status)
# Make best effort to determine whether a deployment is
# likely pending, by checking if the last deployment
# was started within the timeout window of the chart.
last_deployment_age = r.get_last_deployment_age(
old_release)
likely_pending = last_deployment_age <= wait_timeout
if likely_pending:
# We don't take any deploy action and wait for the
# to get deployed.
deploy = noop
deadline = deadline - last_deployment_age
else:
# Release is likely stuck in an unintended
# state. Log and continue on with remediation steps
# below.
LOG.info(
'Old release %s likely stuck in status %s, '
'(last deployment age=%ss) >= '
'(chart wait timeout=%ss)', release, status,
last_deployment_age, wait_timeout)
res = self.purge_release(chart, release_id, status,
manifest_name, chart_name,
result)
if isinstance(res, dict):
if 'protected' in res:
return res
action = metrics.ChartDeployAction.INSTALL
deploy = install
else:
# The chart is in Failed state, hence we purge
# the chart and attempt to install it again.
res = self.purge_release(chart, release_id, status,
manifest_name, chart_name, result)
if isinstance(res, dict):
if 'protected' in res:
return res
action = metrics.ChartDeployAction.INSTALL
deploy = install
if status is None:
action = metrics.ChartDeployAction.INSTALL
deploy = install
# Deploy
with metrics.CHART_DEPLOY.get_context(wait_timeout, manifest_name,
chart_name,
action.get_label_value()):
deploy()
# Wait
timer = int(round(deadline - time.time()))
chart_wait.wait(timer)
# Test
just_deployed = ('install' in result) or ('upgrade' in result)
last_test_passed = old_release and r.get_last_test_result(old_release)
test_handler = Test(chart,
release_id,
self.helm,
cg_test_charts=cg_test_all_charts)
run_test = test_handler.test_enabled and (just_deployed
or not last_test_passed)
if run_test:
with metrics.CHART_TEST.get_context(test_handler.timeout,
manifest_name, chart_name):
self._test_chart(test_handler)
return result
def invoke(self):
tiller = Tiller(
tiller_host=self.tiller_host,
tiller_port=self.tiller_port,
tiller_namespace=self.tiller_namespace)
known_release_names = [release[0] for release in tiller.list_charts()]
if self.release:
if not self.ctx.obj.get('api', False):
self.logger.info("RUNNING: %s tests", self.release)
success = test_release_for_success(
tiller, self.release, cleanup=self.cleanup)
if success:
self.logger.info("PASSED: %s", self.release)
else:
self.logger.info("FAILED: %s", self.release)
else:
client = self.ctx.obj.get('CLIENT')
query = {
'tiller_host': self.tiller_host,
'tiller_port': self.tiller_port,
'tiller_namespace': self.tiller_namespace
}
resp = client.get_test_release(
release=self.release, query=query)
self.logger.info(resp.get('result'))
self.logger.info(resp.get('message'))
if self.file:
if not self.ctx.obj.get('api', False):
documents = list(yaml.safe_load_all(open(self.file).read()))
armada_obj = Manifest(
documents,
target_manifest=self.target_manifest).get_manifest()
prefix = armada_obj.get(const.KEYWORD_ARMADA).get(
const.KEYWORD_PREFIX)
for group in armada_obj.get(const.KEYWORD_ARMADA).get(
const.KEYWORD_GROUPS):
for ch in group.get(const.KEYWORD_CHARTS):
chart = ch['chart']
release_name = release_prefixer(
prefix, chart.get('release'))
if release_name in known_release_names:
cleanup = self.cleanup
if cleanup is None:
test_chart_override = chart.get('test', {})
if isinstance(test_chart_override, bool):
self.logger.warn(
'Boolean value for chart `test` key is'
' deprecated and support for this will'
' be removed. Use `test.enabled` '
'instead.')
# Use old default value.
cleanup = True
else:
cleanup = test_chart_override.get(
'options', {}).get('cleanup', False)
self.logger.info('RUNNING: %s tests', release_name)
success = test_release_for_success(
tiller, release_name, cleanup=cleanup)
if success:
self.logger.info("PASSED: %s", release_name)
else:
self.logger.info("FAILED: %s", release_name)
else:
self.logger.info('Release %s not found - SKIPPING',
release_name)
else:
client = self.ctx.obj.get('CLIENT')
query = {
'tiller_host': self.tiller_host,
'tiller_port': self.tiller_port,
'tiller_namespace': self.tiller_namespace
}
with open(self.filename, 'r') as f:
resp = client.get_test_manifest(
manifest=f.read(), query=query)
for test in resp.get('tests'):
self.logger.info('Test State: %s', test)
for item in test.get('tests').get(test):
self.logger.info(item)
self.logger.info(resp)
def _execute(self, ch, cg_test_all_charts, prefix, known_releases):
manifest_name = self.manifest['metadata']['name']
chart = ch[const.KEYWORD_DATA]
chart_name = ch['metadata']['name']
namespace = chart.get('namespace')
release = chart.get('release')
release_name = r.release_prefixer(prefix, release)
LOG.info('Processing Chart, release=%s', release_name)
result = {}
chart_wait = ChartWait(
self.tiller.k8s,
release_name,
ch,
namespace,
k8s_wait_attempts=self.k8s_wait_attempts,
k8s_wait_attempt_sleep=self.k8s_wait_attempt_sleep,
timeout=self.timeout)
wait_timeout = chart_wait.get_timeout()
# Begin Chart timeout deadline
deadline = time.time() + wait_timeout
old_release = self.find_chart_release(known_releases, release_name)
action = metrics.ChartDeployAction.NOOP
def noop():
pass
deploy = noop
# Resolve action
values = chart.get('values', {})
pre_actions = {}
post_actions = {}
status = None
if old_release:
status = r.get_release_status(old_release)
native_wait_enabled = chart_wait.is_native_enabled()
chartbuilder = ChartBuilder.from_chart_doc(ch)
new_chart = chartbuilder.get_helm_chart()
if status == const.STATUS_DEPLOYED:
# indicate to the end user what path we are taking
LOG.info("Existing release %s found in namespace %s", release_name,
namespace)
# extract the installed chart and installed values from the
# latest release so we can compare to the intended state
old_chart = old_release.chart
old_values_string = old_release.config.raw
upgrade = chart.get('upgrade', {})
options = upgrade.get('options', {})
# TODO: Remove when v1 doc support is removed.
schema_info = get_schema_info(ch['schema'])
if schema_info.version < 2:
no_hooks_location = upgrade
else:
no_hooks_location = options
disable_hooks = no_hooks_location.get('no_hooks', False)
force = options.get('force', False)
recreate_pods = options.get('recreate_pods', False)
if upgrade:
upgrade_pre = upgrade.get('pre', {})
upgrade_post = upgrade.get('post', {})
if not self.disable_update_pre and upgrade_pre:
pre_actions = upgrade_pre
if not self.disable_update_post and upgrade_post:
LOG.warning('Post upgrade actions are ignored by Armada'
'and will not affect deployment.')
post_actions = upgrade_post
try:
old_values = yaml.safe_load(old_values_string)
except yaml.YAMLError:
chart_desc = '{} (previously deployed)'.format(
old_chart.metadata.name)
raise armada_exceptions.\
InvalidOverrideValuesYamlException(chart_desc)
LOG.info('Checking for updates to chart release inputs.')
diff = self.get_diff(old_chart, old_values, new_chart, values)
if not diff:
LOG.info("Found no updates to chart release inputs")
else:
action = metrics.ChartDeployAction.UPGRADE
LOG.info("Found updates to chart release inputs")
LOG.debug("%s", diff)
result['diff'] = {chart['release']: str(diff)}
def upgrade():
# do actual update
timer = int(round(deadline - time.time()))
LOG.info(
"Upgrading release %s in namespace %s, wait=%s, "
"timeout=%ss", release_name, namespace,
native_wait_enabled, timer)
tiller_result = self.tiller.update_release(
new_chart,
release_name,
namespace,
pre_actions=pre_actions,
post_actions=post_actions,
disable_hooks=disable_hooks,
values=yaml.safe_dump(values),
wait=native_wait_enabled,
timeout=timer,
force=force,
recreate_pods=recreate_pods)
LOG.info('Upgrade completed with results from Tiller: %s',
tiller_result.__dict__)
result['upgrade'] = release_name
deploy = upgrade
else:
# Check for release with status other than DEPLOYED
if status:
if status != const.STATUS_FAILED:
LOG.warn(
'Unexpected release status encountered '
'release=%s, status=%s', release_name, status)
# Make best effort to determine whether a deployment is
# likely pending, by checking if the last deployment
# was started within the timeout window of the chart.
last_deployment_age = r.get_last_deployment_age(
old_release)
likely_pending = last_deployment_age <= wait_timeout
if likely_pending:
# Give up if a deployment is likely pending, we do not
# want to have multiple operations going on for the
# same release at the same time.
raise armada_exceptions.\
DeploymentLikelyPendingException(
release_name, status, last_deployment_age,
wait_timeout)
else:
# Release is likely stuck in an unintended (by tiller)
# state. Log and continue on with remediation steps
# below.
LOG.info(
'Old release %s likely stuck in status %s, '
'(last deployment age=%ss) >= '
'(chart wait timeout=%ss)', release, status,
last_deployment_age, wait_timeout)
protected = chart.get('protected', {})
if protected:
p_continue = protected.get('continue_processing', False)
if p_continue:
LOG.warn(
'Release %s is `protected`, '
'continue_processing=True. Operator must '
'handle %s release manually.', release_name,
status)
result['protected'] = release_name
return result
else:
LOG.error(
'Release %s is `protected`, '
'continue_processing=False.', release_name)
raise armada_exceptions.ProtectedReleaseException(
release_name, status)
else:
# Purge the release
with metrics.CHART_DELETE.get_context(
manifest_name, chart_name):
LOG.info('Purging release %s with status %s',
release_name, status)
chart_delete = ChartDelete(chart, release_name,
self.tiller)
chart_delete.delete()
result['purge'] = release_name
action = metrics.ChartDeployAction.INSTALL
def install():
timer = int(round(deadline - time.time()))
LOG.info(
"Installing release %s in namespace %s, wait=%s, "
"timeout=%ss", release_name, namespace,
native_wait_enabled, timer)
tiller_result = self.tiller.install_release(
new_chart,
release_name,
namespace,
values=yaml.safe_dump(values),
wait=native_wait_enabled,
timeout=timer)
LOG.info('Install completed with results from Tiller: %s',
tiller_result.__dict__)
result['install'] = release_name
deploy = install
# Deploy
with metrics.CHART_DEPLOY.get_context(wait_timeout, manifest_name,
chart_name,
action.get_label_value()):
deploy()
# Wait
timer = int(round(deadline - time.time()))
chart_wait.wait(timer)
# Test
just_deployed = ('install' in result) or ('upgrade' in result)
last_test_passed = old_release and r.get_last_test_result(old_release)
test_handler = Test(chart,
release_name,
self.tiller,
cg_test_charts=cg_test_all_charts)
run_test = test_handler.test_enabled and (just_deployed
or not last_test_passed)
if run_test:
with metrics.CHART_TEST.get_context(test_handler.timeout,
manifest_name, chart_name):
self._test_chart(release_name, test_handler)
return result
def execute(self, chart, cg_test_all_charts, prefix, known_releases):
namespace = chart.get('namespace')
release = chart.get('release')
release_name = r.release_prefixer(prefix, release)
LOG.info('Processing Chart, release=%s', release_name)
values = chart.get('values', {})
pre_actions = {}
post_actions = {}
result = {}
protected = chart.get('protected', {})
p_continue = protected.get('continue_processing', False)
old_release = self.find_chart_release(known_releases, release_name)
status = None
if old_release:
status = r.get_release_status(old_release)
if status not in [const.STATUS_FAILED, const.STATUS_DEPLOYED]:
raise armada_exceptions.UnexpectedReleaseStatusException(
release_name, status)
chart_wait = ChartWait(
self.tiller.k8s,
release_name,
chart,
namespace,
k8s_wait_attempts=self.k8s_wait_attempts,
k8s_wait_attempt_sleep=self.k8s_wait_attempt_sleep,
timeout=self.timeout)
native_wait_enabled = chart_wait.is_native_enabled()
# Begin Chart timeout deadline
deadline = time.time() + chart_wait.get_timeout()
chartbuilder = ChartBuilder(chart)
new_chart = chartbuilder.get_helm_chart()
# Check for existing FAILED release, and purge
if status == const.STATUS_FAILED:
LOG.info('Purging FAILED release %s before deployment.',
release_name)
if protected:
if p_continue:
LOG.warn(
'Release %s is `protected`, '
'continue_processing=True. Operator must '
'handle FAILED release manually.', release_name)
result['protected'] = release_name
return result
else:
LOG.error(
'Release %s is `protected`, '
'continue_processing=False.', release_name)
raise armada_exceptions.ProtectedReleaseException(
release_name)
else:
# Purge the release
self.tiller.uninstall_release(release_name)
result['purge'] = release_name
# TODO(mark-burnett): It may be more robust to directly call
# tiller status to decide whether to install/upgrade rather
# than checking for list membership.
if status == const.STATUS_DEPLOYED:
# indicate to the end user what path we are taking
LOG.info("Existing release %s found in namespace %s", release_name,
namespace)
# extract the installed chart and installed values from the
# latest release so we can compare to the intended state
old_chart = old_release.chart
old_values_string = old_release.config.raw
upgrade = chart.get('upgrade', {})
disable_hooks = upgrade.get('no_hooks', False)
options = upgrade.get('options', {})
force = options.get('force', False)
recreate_pods = options.get('recreate_pods', False)
if upgrade:
upgrade_pre = upgrade.get('pre', {})
upgrade_post = upgrade.get('post', {})
if not self.disable_update_pre and upgrade_pre:
pre_actions = upgrade_pre
if not self.disable_update_post and upgrade_post:
LOG.warning('Post upgrade actions are ignored by Armada'
'and will not affect deployment.')
post_actions = upgrade_post
try:
old_values = yaml.safe_load(old_values_string)
except yaml.YAMLError:
chart_desc = '{} (previously deployed)'.format(
old_chart.metadata.name)
raise armada_exceptions.\
InvalidOverrideValuesYamlException(chart_desc)
LOG.info('Checking for updates to chart release inputs.')
diff = self.get_diff(old_chart, old_values, new_chart, values)
if not diff:
LOG.info("Found no updates to chart release inputs")
else:
LOG.info("Found updates to chart release inputs")
LOG.debug("%s", diff)
result['diff'] = {chart['release']: str(diff)}
# TODO(MarshM): Add tiller dry-run before upgrade and
# consider deadline impacts
# do actual update
timer = int(round(deadline - time.time()))
LOG.info(
"Upgrading release %s in namespace %s, wait=%s, "
"timeout=%ss", release_name, namespace,
native_wait_enabled, timer)
tiller_result = self.tiller.update_release(
new_chart,
release_name,
namespace,
pre_actions=pre_actions,
post_actions=post_actions,
disable_hooks=disable_hooks,
values=yaml.safe_dump(values),
wait=native_wait_enabled,
timeout=timer,
force=force,
recreate_pods=recreate_pods)
LOG.info('Upgrade completed with results from Tiller: %s',
tiller_result.__dict__)
result['upgrade'] = release_name
else:
timer = int(round(deadline - time.time()))
LOG.info(
"Installing release %s in namespace %s, wait=%s, "
"timeout=%ss", release_name, namespace, native_wait_enabled,
timer)
tiller_result = self.tiller.install_release(
new_chart,
release_name,
namespace,
values=yaml.safe_dump(values),
wait=native_wait_enabled,
timeout=timer)
LOG.info('Install completed with results from Tiller: %s',
tiller_result.__dict__)
result['install'] = release_name
# Wait
timer = int(round(deadline - time.time()))
chart_wait.wait(timer)
# Test
just_deployed = ('install' in result) or ('upgrade' in result)
last_test_passed = old_release and r.get_last_test_result(old_release)
test_values = chart.get('test')
test_handler = Test(release_name,
self.tiller,
cg_test_charts=cg_test_all_charts,
test_values=test_values)
run_test = test_handler.test_enabled and (just_deployed
or not last_test_passed)
if run_test:
timer = int(round(deadline - time.time()))
self._test_chart(release_name, timer, test_handler)
return result
def handle(self, tiller):
known_release_names = [release[0] for release in tiller.list_charts()]
if self.release:
if not self.ctx.obj.get('api', False):
test_handler = Test(self.release, tiller, cleanup=self.cleanup)
test_handler.test_release_for_success()
else:
client = self.ctx.obj.get('CLIENT')
query = {
'tiller_host': self.tiller_host,
'tiller_port': self.tiller_port,
'tiller_namespace': self.tiller_namespace
}
resp = client.get_test_release(release=self.release,
query=query)
self.logger.info(resp.get('result'))
self.logger.info(resp.get('message'))
if self.file:
if not self.ctx.obj.get('api', False):
documents = list(yaml.safe_load_all(open(self.file).read()))
armada_obj = Manifest(
documents,
target_manifest=self.target_manifest).get_manifest()
prefix = armada_obj.get(const.KEYWORD_ARMADA).get(
const.KEYWORD_PREFIX)
for group in armada_obj.get(const.KEYWORD_ARMADA).get(
const.KEYWORD_GROUPS):
for ch in group.get(const.KEYWORD_CHARTS):
chart = ch['chart']
release_name = release_prefixer(
prefix, chart.get('release'))
if release_name in known_release_names:
test_values = chart.get('test', {})
test_handler = Test(release_name,
tiller,
cleanup=self.cleanup,
enable_all=self.enable_all,
test_values=test_values)
if test_handler.test_enabled:
test_handler.test_release_for_success()
else:
self.logger.info('Release %s not found - SKIPPING',
release_name)
else:
client = self.ctx.obj.get('CLIENT')
query = {
'tiller_host': self.tiller_host,
'tiller_port': self.tiller_port,
'tiller_namespace': self.tiller_namespace
}
with open(self.filename, 'r') as f:
resp = client.get_test_manifest(manifest=f.read(),
query=query)
for test in resp.get('tests'):
self.logger.info('Test State: %s', test)
for item in test.get('tests').get(test):
self.logger.info(item)
self.logger.info(resp)
def _do_test(mock_test, mock_chartbuilder, mock_pre_flight,
mock_post_flight):
# Instantiate Armada object.
yaml_documents = list(yaml.safe_load_all(TEST_YAML))
m_tiller = mock.MagicMock()
m_tiller.list_releases.return_value = known_releases
armada_obj = armada.Armada(yaml_documents, m_tiller)
armada_obj.chart_deploy.get_diff = mock.Mock()
chart_group = armada_obj.manifest['armada']['chart_groups'][0]
charts = chart_group['chart_group']
cg_test_all_charts = chart_group.get('test_charts')
mock_test_release = mock_test.return_value.test_release_for_success
if test_failure_to_run:
def fail(tiller, release, timeout=None, cleanup=False):
status = AttrDict(
**{'info': AttrDict(**{'Description': 'Failed'})})
raise tiller_exceptions.ReleaseException(
release, status, 'Test')
mock_test_release.side_effect = fail
else:
mock_test_release.return_value = test_success
# Stub out irrelevant methods called by `armada.sync()`.
mock_chartbuilder.get_source_path.return_value = None
mock_chartbuilder.get_helm_chart.return_value = None
# Simulate chart diff, upgrade should only happen if non-empty.
armada_obj.chart_deploy.get_diff.return_value = diff
armada_obj.sync()
expected_install_release_calls = []
expected_update_release_calls = []
expected_uninstall_release_calls = []
expected_test_constructor_calls = []
for c in charts:
chart = c['chart']
release = chart['release']
prefix = armada_obj.manifest['armada']['release_prefix']
release_name = release_prefixer(prefix, release)
# Simplified check because the actual code uses logical-or's
# multiple conditions, so this is enough.
native_wait_enabled = (chart['wait'].get('native', {}).get(
'enabled', True))
if release_name not in [x.name for x in known_releases]:
expected_install_release_calls.append(
mock.call(
mock_chartbuilder().get_helm_chart(),
"{}-{}".format(
armada_obj.manifest['armada']
['release_prefix'], chart['release']),
chart['namespace'],
values=yaml.safe_dump(chart['values']),
wait=native_wait_enabled,
timeout=mock.ANY))
else:
target_release = None
for known_release in known_releases:
if known_release.name == release_name:
target_release = known_release
break
if target_release:
status = get_release_status(target_release)
if status == const.STATUS_FAILED:
protected = chart.get('protected', {})
if not protected:
expected_uninstall_release_calls.append(
mock.call(
release_name,
purge=True,
timeout=const.DEFAULT_DELETE_TIMEOUT))
expected_install_release_calls.append(
mock.call(
mock_chartbuilder().get_helm_chart(),
"{}-{}".format(
armada_obj.manifest['armada']
['release_prefix'],
chart['release']),
chart['namespace'],
values=yaml.safe_dump(chart['values']),
wait=native_wait_enabled,
timeout=mock.ANY))
else:
p_continue = protected.get(
'continue_processing', False)
if p_continue:
continue
else:
if chart_group['sequenced']:
break
if status == const.STATUS_DEPLOYED:
if diff:
upgrade = chart.get('upgrade', {})
disable_hooks = upgrade.get('no_hooks', False)
options = upgrade.get('options', {})
force = options.get('force', False)
recreate_pods = options.get(
'recreate_pods', False)
expected_update_release_calls.append(
mock.call(
mock_chartbuilder().get_helm_chart(),
"{}-{}".format(
armada_obj.manifest['armada']
['release_prefix'],
chart['release']),
chart['namespace'],
pre_actions={},
post_actions={},
disable_hooks=disable_hooks,
force=force,
recreate_pods=recreate_pods,
values=yaml.safe_dump(chart['values']),
wait=native_wait_enabled,
timeout=mock.ANY))
expected_test_constructor_calls.append(
mock.call(
chart,
release_name,
m_tiller,
cg_test_charts=cg_test_all_charts))
any_order = not chart_group['sequenced']
# Verify that at least 1 release is either installed or updated.
self.assertTrue(
len(expected_install_release_calls) >= 1 or
len(expected_update_release_calls) >= 1)
# Verify that the expected number of non-deployed releases are
# installed with expected arguments.
self.assertEqual(
len(expected_install_release_calls),
m_tiller.install_release.call_count)
m_tiller.install_release.assert_has_calls(
expected_install_release_calls, any_order=any_order)
# Verify that the expected number of deployed releases are
# updated with expected arguments.
self.assertEqual(
len(expected_update_release_calls),
m_tiller.update_release.call_count)
m_tiller.update_release.assert_has_calls(
expected_update_release_calls, any_order=any_order)
# Verify that the expected number of deployed releases are
# uninstalled with expected arguments.
self.assertEqual(
len(expected_uninstall_release_calls),
m_tiller.uninstall_release.call_count)
m_tiller.uninstall_release.assert_has_calls(
expected_uninstall_release_calls, any_order=any_order)
# Verify that the expected number of deployed releases are
# tested with expected arguments.
self.assertEqual(
len(expected_test_constructor_calls), mock_test.call_count)
mock_test.assert_has_calls(
expected_test_constructor_calls, any_order=True)
def _do_test(mock_test_release_for_success, mock_tiller,
mock_chartbuilder, mock_pre_flight, mock_post_flight):
# Instantiate Armada object.
yaml_documents = list(yaml.safe_load_all(TEST_YAML))
armada_obj = armada.Armada(yaml_documents)
armada_obj.get_diff = mock.Mock()
chart_group = armada_obj.manifest['armada']['chart_groups'][0]
charts = chart_group['chart_group']
cg_test_all_charts = chart_group.get('test_charts', True)
m_tiller = mock_tiller.return_value
m_tiller.list_charts.return_value = known_releases
if test_failure_to_run:
def fail(tiller, release, timeout=None, cleanup=False):
status = AttrDict(
**{'info': AttrDict(**{'Description': 'Failed'})})
raise tiller_exceptions.ReleaseException(
release, status, 'Test')
mock_test_release_for_success.side_effect = fail
else:
mock_test_release_for_success.return_value = test_success
# Stub out irrelevant methods called by `armada.sync()`.
mock_chartbuilder.get_source_path.return_value = None
mock_chartbuilder.get_helm_chart.return_value = None
# Simulate chart diff, upgrade should only happen if non-empty.
armada_obj.get_diff.return_value = diff
armada_obj.sync()
expected_install_release_calls = []
expected_update_release_calls = []
expected_uninstall_release_calls = []
expected_test_release_for_success_calls = []
for c in charts:
chart = c['chart']
release = chart['release']
prefix = armada_obj.manifest['armada']['release_prefix']
release_name = release_prefixer(prefix, release)
# Simplified check because the actual code uses logical-or's
# multiple conditions, so this is enough.
this_chart_should_wait = chart['wait']['timeout'] > 0
expected_apply = True
if release_name not in [x[0] for x in known_releases]:
expected_install_release_calls.append(
mock.call(mock_chartbuilder().get_helm_chart(),
"{}-{}".format(
armada_obj.manifest['armada']
['release_prefix'], chart['release']),
chart['namespace'],
values=yaml.safe_dump(chart['values']),
wait=this_chart_should_wait,
timeout=chart['wait']['timeout']))
else:
target_release = None
for known_release in known_releases:
if known_release[0] == release_name:
target_release = known_release
break
if target_release:
status = target_release[4]
if status == const.STATUS_FAILED:
protected = chart.get('protected', {})
if not protected:
expected_uninstall_release_calls.append(
mock.call(release_name))
expected_install_release_calls.append(
mock.call(
mock_chartbuilder().get_helm_chart(),
"{}-{}".format(
armada_obj.manifest['armada']
['release_prefix'],
chart['release']),
chart['namespace'],
values=yaml.safe_dump(chart['values']),
wait=this_chart_should_wait,
timeout=chart['wait']['timeout']))
else:
p_continue = protected.get(
'continue_processing', False)
if p_continue:
continue
else:
break
if status == const.STATUS_DEPLOYED:
if not diff:
expected_apply = False
else:
upgrade = chart.get('upgrade', {})
disable_hooks = upgrade.get('no_hooks', False)
force = upgrade.get('force', False)
recreate_pods = upgrade.get(
'recreate_pods', False)
expected_update_release_calls.append(
mock.call(
mock_chartbuilder().get_helm_chart(),
"{}-{}".format(
armada_obj.manifest['armada']
['release_prefix'],
chart['release']),
chart['namespace'],
pre_actions={},
post_actions={},
disable_hooks=disable_hooks,
force=force,
recreate_pods=recreate_pods,
values=yaml.safe_dump(chart['values']),
wait=this_chart_should_wait,
timeout=chart['wait']['timeout']))
test_chart_override = chart.get('test')
# Use old default value when not using newer `test` key
test_cleanup = True
if test_chart_override is None:
test_this_chart = cg_test_all_charts
elif isinstance(test_chart_override, bool):
test_this_chart = test_chart_override
else:
test_this_chart = test_chart_override.get('enabled', True)
test_cleanup = test_chart_override.get('options', {}).get(
'cleanup', False)
if test_this_chart and expected_apply:
expected_test_release_for_success_calls.append(
mock.call(m_tiller,
release_name,
timeout=mock.ANY,
cleanup=test_cleanup))
# Verify that at least 1 release is either installed or updated.
self.assertTrue(
len(expected_install_release_calls) >= 1
or len(expected_update_release_calls) >= 1)
# Verify that the expected number of non-deployed releases are
# installed with expected arguments.
self.assertEqual(len(expected_install_release_calls),
m_tiller.install_release.call_count)
m_tiller.install_release.assert_has_calls(
expected_install_release_calls)
# Verify that the expected number of deployed releases are
# updated with expected arguments.
self.assertEqual(len(expected_update_release_calls),
m_tiller.update_release.call_count)
m_tiller.update_release.assert_has_calls(
expected_update_release_calls)
# Verify that the expected number of deployed releases are
# uninstalled with expected arguments.
self.assertEqual(len(expected_uninstall_release_calls),
m_tiller.uninstall_release.call_count)
m_tiller.uninstall_release.assert_has_calls(
expected_uninstall_release_calls)
# Verify that the expected number of deployed releases are
# tested with expected arguments.
self.assertEqual(len(expected_test_release_for_success_calls),
mock_test_release_for_success.call_count)
mock_test_release_for_success.assert_has_calls(
expected_test_release_for_success_calls)
def sync(self):
'''
Synchronize Helm with the Armada Config(s)
'''
if self.dry_run:
LOG.info('Armada is in DRY RUN mode, no changes being made.')
msg = {
'install': [],
'upgrade': [],
'diff': [],
'purge': [],
'protected': []
}
# TODO: (gardlt) we need to break up this func into
# a more cleaner format
self.pre_flight_ops()
# extract known charts on tiller right now
deployed_releases, failed_releases = self._get_releases_by_status()
manifest_data = self.manifest.get(const.KEYWORD_ARMADA, {})
prefix = manifest_data.get(const.KEYWORD_PREFIX)
for chartgroup in manifest_data.get(const.KEYWORD_GROUPS, []):
cg_name = chartgroup.get('name', '<missing name>')
cg_desc = chartgroup.get('description', '<missing description>')
cg_sequenced = chartgroup.get('sequenced', False)
LOG.info('Processing ChartGroup: %s (%s), sequenced=%s', cg_name,
cg_desc, cg_sequenced)
# TODO(MarshM): Deprecate the `test_charts` key
cg_test_all_charts = chartgroup.get('test_charts')
if isinstance(cg_test_all_charts, bool):
LOG.warn('The ChartGroup `test_charts` key is deprecated, '
'and support for this will be removed. See the '
'Chart `test` key for more information.')
else:
# This key defaults to True. Individual charts must
# explicitly disable helm tests if they choose
cg_test_all_charts = True
ns_label_set = set()
tests_to_run = []
cg_charts = chartgroup.get(const.KEYWORD_CHARTS, [])
# Track largest Chart timeout to stop the ChartGroup at the end
cg_max_timeout = 0
for chart_entry in cg_charts:
chart = chart_entry.get('chart', {})
namespace = chart.get('namespace')
release = chart.get('release')
release_name = release_prefixer(prefix, release)
LOG.info('Processing Chart, release=%s', release_name)
values = chart.get('values', {})
pre_actions = {}
post_actions = {}
protected = chart.get('protected', {})
p_continue = protected.get('continue_processing', False)
# Check for existing FAILED release, and purge
if release_name in [rel[0] for rel in failed_releases]:
LOG.info('Purging FAILED release %s before deployment.',
release_name)
if protected:
if p_continue:
LOG.warn(
'Release %s is `protected`, '
'continue_processing=True. Operator must '
'handle FAILED release manually.',
release_name)
msg['protected'].append(release_name)
continue
else:
LOG.error(
'Release %s is `protected`, '
'continue_processing=False.', release_name)
raise armada_exceptions.ProtectedReleaseException(
release_name)
else:
# Purge the release
self.tiller.uninstall_release(release_name)
msg['purge'].append(release_name)
# NOTE(MarshM): Calculating `wait_timeout` is unfortunately
# overly complex. The order of precedence is currently:
# 1) User provided override via API/CLI (default 0 if not
# provided by client/user).
# 2) Chart's `data.wait.timeout`, or...
# 3) Chart's `data.timeout` (deprecated).
# 4) const.DEFAULT_CHART_TIMEOUT, if nothing is ever
# specified, for use in waiting for final ChartGroup
# health and helm tests, but ignored for the actual
# install/upgrade of the Chart.
# NOTE(MarshM): Not defining a timeout has a side effect of
# allowing Armada to install charts with a circular
# dependency defined between components.
# TODO(MarshM): Deprecated, remove the following block
deprecated_timeout = chart.get('timeout', None)
if isinstance(deprecated_timeout, int):
LOG.warn('The `timeout` key is deprecated and support '
'for this will be removed soon. Use '
'`wait.timeout` instead.')
wait_values = chart.get('wait', {})
wait_labels = wait_values.get('labels', {})
wait_timeout = self.timeout
if wait_timeout <= 0:
wait_timeout = wait_values.get('timeout', wait_timeout)
# TODO(MarshM): Deprecated, remove the following check
if wait_timeout <= 0:
wait_timeout = deprecated_timeout or wait_timeout
# Determine wait logic
# NOTE(Dan Kim): Conditions to wait are below :
# 1) set sequenced=True in chart group
# 2) set force_wait param
# 3) add Chart's `data.wait.timeout`
# --timeout param will do not set wait=True, it just change
# max timeout of chart's deployment. (default: 900)
this_chart_should_wait = (cg_sequenced or self.force_wait
or (bool(wait_values) and
(wait_timeout > 0)))
# If there is still no timeout, we need to use a default
# (item 4 in note above)
if wait_timeout <= 0:
LOG.warn('No Chart timeout specified, using default: %ss',
const.DEFAULT_CHART_TIMEOUT)
wait_timeout = const.DEFAULT_CHART_TIMEOUT
# Naively take largest timeout to apply at end
# TODO(MarshM) better handling of timeout/timer
cg_max_timeout = max(wait_timeout, cg_max_timeout)
test_chart_override = chart.get('test')
# Use old default value when not using newer `test` key
test_cleanup = True
if test_chart_override is None:
test_this_chart = cg_test_all_charts
elif isinstance(test_chart_override, bool):
LOG.warn('Boolean value for chart `test` key is'
' deprecated and support for this will'
' be removed. Use `test.enabled` '
'instead.')
test_this_chart = test_chart_override
else:
# NOTE: helm tests are enabled by default
test_this_chart = test_chart_override.get('enabled', True)
test_cleanup = test_chart_override.get('options', {}).get(
'cleanup', False)
chartbuilder = ChartBuilder(chart)
new_chart = chartbuilder.get_helm_chart()
# Begin Chart timeout deadline
deadline = time.time() + wait_timeout
# TODO(mark-burnett): It may be more robust to directly call
# tiller status to decide whether to install/upgrade rather
# than checking for list membership.
if release_name in [rel[0] for rel in deployed_releases]:
# indicate to the end user what path we are taking
LOG.info("Upgrading release %s in namespace %s",
release_name, namespace)
# extract the installed chart and installed values from the
# latest release so we can compare to the intended state
old_chart, old_values_string = self.find_release_chart(
deployed_releases, release_name)
upgrade = chart.get('upgrade', {})
disable_hooks = upgrade.get('no_hooks', False)
force = upgrade.get('force', False)
recreate_pods = upgrade.get('recreate_pods', False)
LOG.info("Checking Pre/Post Actions")
if upgrade:
upgrade_pre = upgrade.get('pre', {})
upgrade_post = upgrade.get('post', {})
if not self.disable_update_pre and upgrade_pre:
pre_actions = upgrade_pre
if not self.disable_update_post and upgrade_post:
post_actions = upgrade_post
try:
old_values = yaml.safe_load(old_values_string)
except yaml.YAMLError:
chart_desc = '{} (previously deployed)'.format(
old_chart.metadata.name)
raise armada_exceptions.\
InvalidOverrideValuesYamlException(chart_desc)
LOG.info('Checking for updates to chart release inputs.')
diff = self.get_diff(old_chart, old_values, new_chart,
values)
if not diff:
LOG.info("Found no updates to chart release inputs")
continue
LOG.info("Found updates to chart release inputs")
LOG.debug("%s", diff)
msg['diff'].append({chart['release']: str(diff)})
# TODO(MarshM): Add tiller dry-run before upgrade and
# consider deadline impacts
# do actual update
timer = int(round(deadline - time.time()))
LOG.info('Beginning Upgrade, wait=%s, timeout=%ss',
this_chart_should_wait, timer)
tiller_result = self.tiller.update_release(
new_chart,
release_name,
namespace,
pre_actions=pre_actions,
post_actions=post_actions,
disable_hooks=disable_hooks,
values=yaml.safe_dump(values),
wait=this_chart_should_wait,
timeout=timer,
force=force,
recreate_pods=recreate_pods)
if this_chart_should_wait:
self._wait_until_ready(release_name, wait_labels,
namespace, timer)
# Track namespace+labels touched by upgrade
ns_label_set.add((namespace, tuple(wait_labels.items())))
LOG.info('Upgrade completed with results from Tiller: %s',
tiller_result.__dict__)
msg['upgrade'].append(release_name)
# process install
else:
LOG.info("Installing release %s in namespace %s",
release_name, namespace)
timer = int(round(deadline - time.time()))
LOG.info('Beginning Install, wait=%s, timeout=%ss',
this_chart_should_wait, timer)
tiller_result = self.tiller.install_release(
new_chart,
release_name,
namespace,
values=yaml.safe_dump(values),
wait=this_chart_should_wait,
timeout=timer)
if this_chart_should_wait:
self._wait_until_ready(release_name, wait_labels,
namespace, timer)
# Track namespace+labels touched by install
ns_label_set.add((namespace, tuple(wait_labels.items())))
LOG.info('Install completed with results from Tiller: %s',
tiller_result.__dict__)
msg['install'].append(release_name)
# Keeping track of time remaining
timer = int(round(deadline - time.time()))
test_chart_args = (release_name, timer, test_cleanup)
if test_this_chart:
# Sequenced ChartGroup should run tests after each Chart
if cg_sequenced:
LOG.info(
'Running sequenced test, timeout remaining: '
'%ss.', timer)
self._test_chart(*test_chart_args)
# Un-sequenced ChartGroup should run tests at the end
else:
tests_to_run.append(
functools.partial(self._test_chart,
*test_chart_args))
# End of Charts in ChartGroup
LOG.info('All Charts applied in ChartGroup %s.', cg_name)
# After all Charts are applied, we should wait for the entire
# ChartGroup to become healthy by looking at the namespaces seen
# TODO(MarshM): Need to determine a better timeout
# (not cg_max_timeout)
if cg_max_timeout <= 0:
cg_max_timeout = const.DEFAULT_CHART_TIMEOUT
deadline = time.time() + cg_max_timeout
for (ns, labels) in ns_label_set:
labels_dict = dict(labels)
timer = int(round(deadline - time.time()))
LOG.info(
'Final ChartGroup wait for healthy namespace=%s, '
'labels=(%s), timeout remaining: %ss.', ns, labels_dict,
timer)
if timer <= 0:
reason = ('Timeout expired waiting on namespace: %s, '
'labels: (%s)' % (ns, labels_dict))
LOG.error(reason)
raise armada_exceptions.ArmadaTimeoutException(reason)
self._wait_until_ready(release_name=None,
wait_labels=labels_dict,
namespace=ns,
timeout=timer)
# After entire ChartGroup is healthy, run any pending tests
for callback in tests_to_run:
callback()
self.post_flight_ops()
if self.enable_chart_cleanup:
self._chart_cleanup(
prefix,
self.manifest[const.KEYWORD_ARMADA][const.KEYWORD_GROUPS], msg)
LOG.info('Done applying manifest.')
return msg
