def handle(self, tiller):
known_release_names = [release[0] for release in tiller.list_charts()]
if self.releases:
target_releases = [
r.strip() for r in self.releases.split(',')
if r.strip() in known_release_names
]
if not target_releases:
self.logger.info("There's no release to delete.")
return
if not self.ctx.obj.get('api', False):
for r in target_releases:
self.logger.info("Deleting release %s", r)
tiller.delete_release(r, purge=self.purge)
else:
raise NotImplementedError()
if self.manifest:
target_deletes = []
with open(self.manifest) as f:
documents = list(yaml.safe_load_all(f.read()))
try:
armada_obj = Manifest(documents).get_manifest()
prefix = armada_obj.get(const.KEYWORD_DATA).get(
const.KEYWORD_PREFIX)
for group in armada_obj.get(const.KEYWORD_DATA).get(
const.KEYWORD_GROUPS):
for ch in group.get(const.KEYWORD_DATA).get(
const.KEYWORD_CHARTS):
chart = ch.get(const.KEYWORD_DATA)
release_name = release_prefixer(
prefix, chart.get('release'))
if release_name in known_release_names:
target_deletes.append((chart, release_name))
except yaml.YAMLError as e:
mark = e.problem_mark
self.logger.info(
"While parsing the manifest file, %s. "
"Error position: (%s:%s)", e.problem, mark.line + 1,
mark.column + 1)
if not target_deletes:
self.logger.info("There's no release to delete.")
return
if not self.ctx.obj.get('api', False):
for chart, release in target_deletes:
chart_delete = ChartDelete(chart,
release,
tiller,
purge=self.purge)
chart_delete.delete()
else:
raise NotImplementedError()
def purge_release(self, chart, release_id, status, manifest_name,
chart_name, result):
protected = chart.get('protected', {})
if protected:
p_continue = protected.get('continue_processing', False)
if p_continue:
LOG.warn(
'Release %s is `protected`, '
'continue_processing=True. Operator must '
'handle %s release manually.', release_id, status)
result['protected'] = release_id
return result
else:
LOG.error(
'Release %s is `protected`, '
'continue_processing=False.', release_id)
raise armada_exceptions.ProtectedReleaseException(
release_id, status)
else:
# Purge the release
with metrics.CHART_DELETE.get_context(manifest_name, chart_name):
LOG.info('Purging release %s with status %s', release_id,
status)
chart_delete = ChartDelete(chart, release_id, self.helm)
chart_delete.delete()
result['purge'] = release_id
def execute(self, chart, cg_test_all_charts, prefix, known_releases):
namespace = chart.get('namespace')
release = chart.get('release')
release_name = r.release_prefixer(prefix, release)
LOG.info('Processing Chart, release=%s', release_name)
values = chart.get('values', {})
pre_actions = {}
post_actions = {}
result = {}
old_release = self.find_chart_release(known_releases, release_name)
status = None
if old_release:
status = r.get_release_status(old_release)
chart_wait = ChartWait(
self.tiller.k8s,
release_name,
chart,
namespace,
k8s_wait_attempts=self.k8s_wait_attempts,
k8s_wait_attempt_sleep=self.k8s_wait_attempt_sleep,
timeout=self.timeout)
native_wait_enabled = chart_wait.is_native_enabled()
# Begin Chart timeout deadline
deadline = time.time() + chart_wait.get_timeout()
chartbuilder = ChartBuilder(chart)
new_chart = chartbuilder.get_helm_chart()
# TODO(mark-burnett): It may be more robust to directly call
# tiller status to decide whether to install/upgrade rather
# than checking for list membership.
if status == const.STATUS_DEPLOYED:
# indicate to the end user what path we are taking
LOG.info("Existing release %s found in namespace %s", release_name,
namespace)
# extract the installed chart and installed values from the
# latest release so we can compare to the intended state
old_chart = old_release.chart
old_values_string = old_release.config.raw
upgrade = chart.get('upgrade', {})
disable_hooks = upgrade.get('no_hooks', False)
options = upgrade.get('options', {})
force = options.get('force', False)
recreate_pods = options.get('recreate_pods', False)
if upgrade:
upgrade_pre = upgrade.get('pre', {})
upgrade_post = upgrade.get('post', {})
if not self.disable_update_pre and upgrade_pre:
pre_actions = upgrade_pre
if not self.disable_update_post and upgrade_post:
LOG.warning('Post upgrade actions are ignored by Armada'
'and will not affect deployment.')
post_actions = upgrade_post
try:
old_values = yaml.safe_load(old_values_string)
except yaml.YAMLError:
chart_desc = '{} (previously deployed)'.format(
old_chart.metadata.name)
raise armada_exceptions.\
InvalidOverrideValuesYamlException(chart_desc)
LOG.info('Checking for updates to chart release inputs.')
diff = self.get_diff(old_chart, old_values, new_chart, values)
if not diff:
LOG.info("Found no updates to chart release inputs")
else:
LOG.info("Found updates to chart release inputs")
LOG.debug("%s", diff)
result['diff'] = {chart['release']: str(diff)}
# TODO(MarshM): Add tiller dry-run before upgrade and
# consider deadline impacts
# do actual update
timer = int(round(deadline - time.time()))
LOG.info(
"Upgrading release %s in namespace %s, wait=%s, "
"timeout=%ss", release_name, namespace,
native_wait_enabled, timer)
tiller_result = self.tiller.update_release(
new_chart,
release_name,
namespace,
pre_actions=pre_actions,
post_actions=post_actions,
disable_hooks=disable_hooks,
values=yaml.safe_dump(values),
wait=native_wait_enabled,
timeout=timer,
force=force,
recreate_pods=recreate_pods)
LOG.info('Upgrade completed with results from Tiller: %s',
tiller_result.__dict__)
result['upgrade'] = release_name
else:
# Check for release with status other than DEPLOYED
if status:
if status != const.STATUS_FAILED:
LOG.warn(
'Unexpected release status encountered '
'release=%s, status=%s', release_name, status)
# Make best effort to determine whether a deployment is
# likely pending, by checking if the last deployment
# was started within the timeout window of the chart.
last_deployment_age = r.get_last_deployment_age(
old_release)
wait_timeout = chart_wait.get_timeout()
likely_pending = last_deployment_age <= wait_timeout
if likely_pending:
# Give up if a deployment is likely pending, we do not
# want to have multiple operations going on for the
# same release at the same time.
raise armada_exceptions.\
DeploymentLikelyPendingException(
release_name, status, last_deployment_age,
wait_timeout)
else:
# Release is likely stuck in an unintended (by tiller)
# state. Log and continue on with remediation steps
# below.
LOG.info(
'Old release %s likely stuck in status %s, '
'(last deployment age=%ss) >= '
'(chart wait timeout=%ss)', release, status,
last_deployment_age, wait_timeout)
protected = chart.get('protected', {})
if protected:
p_continue = protected.get('continue_processing', False)
if p_continue:
LOG.warn(
'Release %s is `protected`, '
'continue_processing=True. Operator must '
'handle %s release manually.', release_name,
status)
result['protected'] = release_name
return result
else:
LOG.error(
'Release %s is `protected`, '
'continue_processing=False.', release_name)
raise armada_exceptions.ProtectedReleaseException(
release_name, status)
else:
# Purge the release
LOG.info('Purging release %s with status %s', release_name,
status)
chart_delete = ChartDelete(chart, release_name,
self.tiller)
chart_delete.delete()
result['purge'] = release_name
timer = int(round(deadline - time.time()))
LOG.info(
"Installing release %s in namespace %s, wait=%s, "
"timeout=%ss", release_name, namespace, native_wait_enabled,
timer)
tiller_result = self.tiller.install_release(
new_chart,
release_name,
namespace,
values=yaml.safe_dump(values),
wait=native_wait_enabled,
timeout=timer)
LOG.info('Install completed with results from Tiller: %s',
tiller_result.__dict__)
result['install'] = release_name
# Wait
timer = int(round(deadline - time.time()))
chart_wait.wait(timer)
# Test
just_deployed = ('install' in result) or ('upgrade' in result)
last_test_passed = old_release and r.get_last_test_result(old_release)
test_handler = Test(chart,
release_name,
self.tiller,
cg_test_charts=cg_test_all_charts)
run_test = test_handler.test_enabled and (just_deployed
or not last_test_passed)
if run_test:
self._test_chart(release_name, test_handler)
return result
def _execute(self, ch, cg_test_all_charts, prefix, known_releases):
manifest_name = self.manifest['metadata']['name']
chart = ch[const.KEYWORD_DATA]
chart_name = ch['metadata']['name']
namespace = chart.get('namespace')
release = chart.get('release')
release_name = r.release_prefixer(prefix, release)
LOG.info('Processing Chart, release=%s', release_name)
result = {}
chart_wait = ChartWait(
self.tiller.k8s,
release_name,
ch,
namespace,
k8s_wait_attempts=self.k8s_wait_attempts,
k8s_wait_attempt_sleep=self.k8s_wait_attempt_sleep,
timeout=self.timeout)
wait_timeout = chart_wait.get_timeout()
# Begin Chart timeout deadline
deadline = time.time() + wait_timeout
old_release = self.find_chart_release(known_releases, release_name)
action = metrics.ChartDeployAction.NOOP
def noop():
pass
deploy = noop
# Resolve action
values = chart.get('values', {})
pre_actions = {}
post_actions = {}
status = None
if old_release:
status = r.get_release_status(old_release)
native_wait_enabled = chart_wait.is_native_enabled()
chartbuilder = ChartBuilder.from_chart_doc(ch)
new_chart = chartbuilder.get_helm_chart()
if status == const.STATUS_DEPLOYED:
# indicate to the end user what path we are taking
LOG.info("Existing release %s found in namespace %s", release_name,
namespace)
# extract the installed chart and installed values from the
# latest release so we can compare to the intended state
old_chart = old_release.chart
old_values_string = old_release.config.raw
upgrade = chart.get('upgrade', {})
options = upgrade.get('options', {})
# TODO: Remove when v1 doc support is removed.
schema_info = get_schema_info(ch['schema'])
if schema_info.version < 2:
no_hooks_location = upgrade
else:
no_hooks_location = options
disable_hooks = no_hooks_location.get('no_hooks', False)
force = options.get('force', False)
recreate_pods = options.get('recreate_pods', False)
if upgrade:
upgrade_pre = upgrade.get('pre', {})
upgrade_post = upgrade.get('post', {})
if not self.disable_update_pre and upgrade_pre:
pre_actions = upgrade_pre
if not self.disable_update_post and upgrade_post:
LOG.warning('Post upgrade actions are ignored by Armada'
'and will not affect deployment.')
post_actions = upgrade_post
try:
old_values = yaml.safe_load(old_values_string)
except yaml.YAMLError:
chart_desc = '{} (previously deployed)'.format(
old_chart.metadata.name)
raise armada_exceptions.\
InvalidOverrideValuesYamlException(chart_desc)
LOG.info('Checking for updates to chart release inputs.')
diff = self.get_diff(old_chart, old_values, new_chart, values)
if not diff:
LOG.info("Found no updates to chart release inputs")
else:
action = metrics.ChartDeployAction.UPGRADE
LOG.info("Found updates to chart release inputs")
LOG.debug("%s", diff)
result['diff'] = {chart['release']: str(diff)}
def upgrade():
# do actual update
timer = int(round(deadline - time.time()))
LOG.info(
"Upgrading release %s in namespace %s, wait=%s, "
"timeout=%ss", release_name, namespace,
native_wait_enabled, timer)
tiller_result = self.tiller.update_release(
new_chart,
release_name,
namespace,
pre_actions=pre_actions,
post_actions=post_actions,
disable_hooks=disable_hooks,
values=yaml.safe_dump(values),
wait=native_wait_enabled,
timeout=timer,
force=force,
recreate_pods=recreate_pods)
LOG.info('Upgrade completed with results from Tiller: %s',
tiller_result.__dict__)
result['upgrade'] = release_name
deploy = upgrade
else:
# Check for release with status other than DEPLOYED
if status:
if status != const.STATUS_FAILED:
LOG.warn(
'Unexpected release status encountered '
'release=%s, status=%s', release_name, status)
# Make best effort to determine whether a deployment is
# likely pending, by checking if the last deployment
# was started within the timeout window of the chart.
last_deployment_age = r.get_last_deployment_age(
old_release)
likely_pending = last_deployment_age <= wait_timeout
if likely_pending:
# Give up if a deployment is likely pending, we do not
# want to have multiple operations going on for the
# same release at the same time.
raise armada_exceptions.\
DeploymentLikelyPendingException(
release_name, status, last_deployment_age,
wait_timeout)
else:
# Release is likely stuck in an unintended (by tiller)
# state. Log and continue on with remediation steps
# below.
LOG.info(
'Old release %s likely stuck in status %s, '
'(last deployment age=%ss) >= '
'(chart wait timeout=%ss)', release, status,
last_deployment_age, wait_timeout)
protected = chart.get('protected', {})
if protected:
p_continue = protected.get('continue_processing', False)
if p_continue:
LOG.warn(
'Release %s is `protected`, '
'continue_processing=True. Operator must '
'handle %s release manually.', release_name,
status)
result['protected'] = release_name
return result
else:
LOG.error(
'Release %s is `protected`, '
'continue_processing=False.', release_name)
raise armada_exceptions.ProtectedReleaseException(
release_name, status)
else:
# Purge the release
with metrics.CHART_DELETE.get_context(
manifest_name, chart_name):
LOG.info('Purging release %s with status %s',
release_name, status)
chart_delete = ChartDelete(chart, release_name,
self.tiller)
chart_delete.delete()
result['purge'] = release_name
action = metrics.ChartDeployAction.INSTALL
def install():
timer = int(round(deadline - time.time()))
LOG.info(
"Installing release %s in namespace %s, wait=%s, "
"timeout=%ss", release_name, namespace,
native_wait_enabled, timer)
tiller_result = self.tiller.install_release(
new_chart,
release_name,
namespace,
values=yaml.safe_dump(values),
wait=native_wait_enabled,
timeout=timer)
LOG.info('Install completed with results from Tiller: %s',
tiller_result.__dict__)
result['install'] = release_name
deploy = install
# Deploy
with metrics.CHART_DEPLOY.get_context(wait_timeout, manifest_name,
chart_name,
action.get_label_value()):
deploy()
# Wait
timer = int(round(deadline - time.time()))
chart_wait.wait(timer)
# Test
just_deployed = ('install' in result) or ('upgrade' in result)
last_test_passed = old_release and r.get_last_test_result(old_release)
test_handler = Test(chart,
release_name,
self.tiller,
cg_test_charts=cg_test_all_charts)
run_test = test_handler.test_enabled and (just_deployed
or not last_test_passed)
if run_test:
with metrics.CHART_TEST.get_context(test_handler.timeout,
manifest_name, chart_name):
self._test_chart(release_name, test_handler)
return result