def update_document(self, ovr):
ovr_schema_info = schema.get_schema_info(ovr.get('schema'))
if ovr_schema_info:
for doc in self.documents:
schema_info = schema.get_schema_info(doc.get('schema'))
if schema_info:
if schema_info == ovr_schema_info:
if doc['metadata']['name'] == ovr['metadata']['name']:
data = doc.get('data', {})
ovr_data = ovr.get('data', {})
self.update(data, ovr_data)
return
def _find_documents(self, target_manifest=None):
"""Returns the chart documents, chart group documents,
and Armada manifest
If multiple documents with schema "armada/Manifest/v1" are provided,
specify ``target_manifest`` to select the target one.
:param str target_manifest: The target manifest to use when multiple
documents with "armada/Manifest/v1" are contained in
``documents``. Default is None.
:returns: Tuple of chart documents, chart groups, and manifests
found in ``self.documents``
:rtype: tuple
"""
charts = []
groups = []
manifests = []
for document in self.documents:
schema_info = schema.get_schema_info(document.get('schema'))
if not schema_info:
continue
if schema_info.type == schema.TYPE_CHART:
charts.append(document)
if schema_info.type == schema.TYPE_CHARTGROUP:
groups.append(document)
if schema_info.type == schema.TYPE_MANIFEST:
manifest_name = document.get('metadata', {}).get('name')
if target_manifest:
if manifest_name == target_manifest:
manifests.append(document)
else:
manifests.append(document)
return charts, groups, manifests
def from_chart_doc(cls, chart):
'''
Returns a ChartBuilder defined by an Armada Chart doc.
:param chart: Armada Chart doc for which to build the Helm chart.
'''
name = chart['metadata']['name']
chart_data = chart[const.KEYWORD_DATA]
source_dir = chart_data.get('source_dir')
source_directory = os.path.join(*source_dir)
dependencies = chart_data.get('dependencies')
# TODO: Remove when v1 doc support is removed.
schema_info = get_schema_info(chart['schema'])
if schema_info.version < 2:
fix_tpl_name = False
else:
fix_tpl_name = True
if dependencies is not None:
dependency_builders = []
for chart_dep in dependencies:
builder = ChartBuilder.from_chart_doc(chart_dep)
dependency_builders.append(builder)
return cls(name,
source_directory,
dependency_builders,
fix_tpl_name=fix_tpl_name)
return cls.from_source(name,
source_directory,
fix_tpl_name=fix_tpl_name)
def find_manifest_document(self, doc_path):
for doc in self.documents:
schema_info = schema.get_schema_info(doc.get('schema'))
if schema_info.type == self.find_document_type(
doc_path[0]) and doc.get('metadata',
{}).get('name') == doc_path[1]:
return doc
raise override_exceptions.UnknownDocumentOverrideException(
doc_path[0], doc_path[1])
def validate_armada_document(document):
"""Validates a document ingested by Armada by subjecting it to JSON schema
validation.
:param dict dictionary: The document to validate.
:returns: A tuple of (bool, list[dict]) where the first value
indicates whether the validation succeeded or failed and
the second value is the validation details with a minimum
keyset of (message(str), error(bool))
:rtype: tuple.
:raises TypeError: If ``document`` is not of type ``dict``.
"""
if not isinstance(document, dict):
raise TypeError('The provided input "%s" must be a dictionary.' %
document)
schema = document.get('schema', '<missing>')
document_name = document.get('metadata', {}).get('name', None)
details = []
LOG.debug('Validating document [%s] %s', schema, document_name)
schema_info = sch.get_schema_info(schema)
if schema_info:
try:
validator = jsonschema.Draft4Validator(schema_info.data)
for error in validator.iter_errors(document.get('data')):
error_message = "Invalid document [%s] %s: %s." % \
(schema, document_name, error.message)
vmsg = ValidationMessage(message=error_message,
error=True,
name='ARM100',
level='Error',
schema=schema,
doc_name=document_name)
LOG.info('ValidationMessage: %s', vmsg.get_output_json())
details.append(vmsg.get_output())
except jsonschema.SchemaError as e:
error_message = ('The built-in Armada JSON schema %s is invalid. '
'Details: %s.' % (e.schema, e.message))
vmsg = ValidationMessage(message=error_message,
error=True,
name='ARM000',
level='Error',
diagnostic='Armada is misconfigured.')
LOG.error('ValidationMessage: %s', vmsg.get_output_json())
details.append(vmsg.get_output())
if len([x for x in details if x.get('error', False)]) > 0:
return False, details
return True, details
def wait(self, timeout):
'''
:param timeout: time before disconnecting ``Watch`` stream
'''
min_ready_msg = ', min_ready={}'.format(
self.min_ready.source) if isinstance(self, ControllerWait) else ''
LOG.info(
"Waiting for resource type=%s, namespace=%s labels=%s "
"required=%s%s for %ss", self.resource_type,
self.chart_wait.release_id.namespace, self.label_selector,
self.required, min_ready_msg, timeout)
if not self.label_selector:
LOG.warn(
'"label_selector" not specified, waiting with no labels '
'may cause unintended consequences.')
# Track the overall deadline for timing out during waits
deadline = time.time() + timeout
schema_info = get_schema_info(self.chart_wait.chart['schema'])
# TODO: Remove when v1 doc support is removed.
if schema_info.version < 2:
# NOTE(mark-burnett): Attempt to wait multiple times without
# modification, in case new resources appear after our watch exits.
successes = 0
while True:
modified = self._wait(deadline)
if modified is None:
break
if modified:
successes = 0
LOG.debug('Found modified resources: %s', sorted(modified))
else:
successes += 1
LOG.debug('Found no modified resources.')
if successes >= self.chart_wait.k8s_wait_attempts:
return
LOG.debug(
'Continuing to wait: %s consecutive attempts without '
'modified resources of %s required.', successes,
self.chart_wait.k8s_wait_attempts)
time.sleep(self.chart_wait.k8s_wait_attempt_sleep)
else:
self._wait(deadline)
def validate_armada_manifests(documents):
"""Validate each Armada manifest found in the document set.
:param documents: List of Armada documents to validate
:type documents: :func: `list[dict]`.
"""
messages = []
all_valid = True
for document in documents:
doc_schema = document.get('schema')
if doc_schema:
schema_info = sch.get_schema_info(doc_schema)
if schema_info and schema_info.type == sch.TYPE_MANIFEST:
target = document.get('metadata').get('name')
# TODO(MarshM) explore: why does this pass 'documents'?
manifest = Manifest(documents, target_manifest=target)
is_valid, details = _validate_armada_manifest(manifest)
all_valid = all_valid and is_valid
messages.extend(details)
return all_valid, messages
def get_exclude_reason(self, resource):
pod = resource
# Exclude helm test pods
# TODO: Possibly exclude other helm hook pods/jobs (besides tests)?
if is_test_pod(pod):
return 'helm test pod'
if pod.status.phase == 'Evicted':
return "pod was evicted"
schema_info = get_schema_info(self.chart_wait.chart['schema'])
# TODO: Remove when v1 doc support is removed.
if schema_info.version < 2:
# Exclude job pods
if has_owner(pod, 'Job'):
return 'owned by job'
else:
# Exclude all pods with an owner (only include raw pods)
if has_owner(pod):
return 'owned by another resource'
return None
def get_exclude_reason(self, resource):
pod = resource
# Exclude helm test pods
# TODO: Possibly exclude other helm hook pods/jobs (besides tests)?
if is_test_pod(pod):
return 'helm test pod'
schema_info = get_schema_info(self.chart_wait.chart['schema'])
# TODO: Remove when v1 doc support is removed.
if schema_info.version < 2:
# Exclude job pods
if has_owner(pod, 'Job'):
return 'owned by job'
else:
# Exclude all pods with an owner (only include raw pods)
# TODO: In helm 3, all resources will likely have the release CR as
# an owner, so this will need to be updated to not exclude pods
# directly owned by the release.
if has_owner(pod):
return 'owned by another resource'
return None
def __init__(
self, k8s, release_id, chart, k8s_wait_attempts,
k8s_wait_attempt_sleep, timeout):
self.k8s = k8s
self.release_id = release_id
self.chart = chart
chart_data = self.chart[const.KEYWORD_DATA]
self.chart_data = chart_data
self.wait_config = self.chart_data.get('wait', {})
self.k8s_wait_attempts = max(k8s_wait_attempts, 1)
self.k8s_wait_attempt_sleep = max(k8s_wait_attempt_sleep, 1)
schema_info = get_schema_info(self.chart['schema'])
resources = self.wait_config.get('resources')
if isinstance(resources, list):
# Explicit resource config list provided.
resources_list = resources
else:
# TODO: Remove when v1 doc support is removed.
if schema_info.version < 2:
resources_list = [
{
'type': 'job',
'required': False
}, {
'type': 'pod'
}
]
else:
resources_list = self.get_resources_list(resources)
chart_labels = get_wait_labels(self.chart_data)
for resource_config in resources_list:
# Use chart labels as base labels for each config.
labels = dict(chart_labels)
resource_labels = resource_config.get('labels', {})
# Merge in any resource-specific labels.
if resource_labels:
labels.update(resource_labels)
resource_config['labels'] = labels
LOG.debug('Resolved `wait.resources` list: %s', resources_list)
self.waits = [self.get_resource_wait(conf) for conf in resources_list]
# Calculate timeout
wait_timeout = timeout
if wait_timeout is None:
wait_timeout = self.wait_config.get('timeout')
# TODO: Remove when v1 doc support is removed.
deprecated_timeout = self.chart_data.get('timeout')
if deprecated_timeout is not None:
LOG.warn(
'The `timeout` key is deprecated and support '
'for this will be removed soon. Use '
'`wait.timeout` instead.')
if wait_timeout is None:
wait_timeout = deprecated_timeout
if wait_timeout is None:
LOG.info(
'No Chart timeout specified, using default: %ss',
const.DEFAULT_CHART_TIMEOUT)
wait_timeout = const.DEFAULT_CHART_TIMEOUT
self.timeout = wait_timeout
# Determine whether to enable native wait.
native = self.wait_config.get('native', {})
# TODO: Remove when v1 doc support is removed.
default_native = schema_info.version < 2
self.native_enabled = native.get('enabled', default_native)
def delete_resources(self,
resource_type,
resource_labels,
namespace,
wait=False,
timeout=const.DEFAULT_TILLER_TIMEOUT):
'''
Delete resources matching provided resource type, labels, and
namespace.
:param resource_type: type of resource e.g. job, pod, etc.
:param resource_labels: labels for selecting the resources
:param namespace: namespace of resources
'''
timeout = self._check_timeout(wait, timeout)
label_selector = ''
if resource_labels is not None:
label_selector = label_selectors(resource_labels)
LOG.debug(
"Deleting resources in namespace %s matching "
"selectors (%s).", namespace, label_selector)
handled = False
if resource_type == 'job':
get_jobs = self.k8s.get_namespace_job(
namespace, label_selector=label_selector)
for jb in get_jobs.items:
jb_name = jb.metadata.name
if self.dry_run:
LOG.info(
'Skipping delete job during `dry-run`, would '
'have deleted job %s in namespace=%s.', jb_name,
namespace)
continue
LOG.info("Deleting job %s in namespace: %s", jb_name,
namespace)
self.k8s.delete_job_action(jb_name, namespace, timeout=timeout)
handled = True
# TODO: Remove when v1 doc support is removed.
chart = get_current_chart()
schema_info = schema.get_schema_info(chart['schema'])
job_implies_cronjob = schema_info.version < 2
implied_cronjob = resource_type == 'job' and job_implies_cronjob
if resource_type == 'cronjob' or implied_cronjob:
get_jobs = self.k8s.get_namespace_cron_job(
namespace, label_selector=label_selector)
for jb in get_jobs.items:
jb_name = jb.metadata.name
# TODO: Remove when v1 doc support is removed.
if implied_cronjob:
LOG.warn("Deleting cronjobs via `type: job` is "
"deprecated, use `type: cronjob` instead")
if self.dry_run:
LOG.info(
'Skipping delete cronjob during `dry-run`, would '
'have deleted cronjob %s in namespace=%s.', jb_name,
namespace)
continue
LOG.info("Deleting cronjob %s in namespace: %s", jb_name,
namespace)
self.k8s.delete_cron_job_action(jb_name, namespace)
handled = True
if resource_type == 'pod':
release_pods = self.k8s.get_namespace_pod(
namespace, label_selector=label_selector)
for pod in release_pods.items:
pod_name = pod.metadata.name
if self.dry_run:
LOG.info(
'Skipping delete pod during `dry-run`, would '
'have deleted pod %s in namespace=%s.', pod_name,
namespace)
continue
LOG.info("Deleting pod %s in namespace: %s", pod_name,
namespace)
self.k8s.delete_pod_action(pod_name, namespace)
if wait:
self.k8s.wait_for_pod_redeployment(pod_name, namespace)
handled = True
if not handled:
LOG.error('No resources found with labels=%s type=%s namespace=%s',
resource_labels, resource_type, namespace)
def _execute(self, ch, cg_test_all_charts, prefix):
manifest_name = self.manifest['metadata']['name']
chart = ch[const.KEYWORD_DATA]
chart_name = ch['metadata']['name']
namespace = chart.get('namespace')
release = chart.get('release')
release_name = r.release_prefixer(prefix, release)
release_id = helm.HelmReleaseId(namespace, release_name)
source_dir = chart['source_dir']
source_directory = os.path.join(*source_dir)
LOG.info('Processing Chart, release=%s', release_id)
result = {}
chart_wait = ChartWait(
self.helm.k8s,
release_id,
ch,
k8s_wait_attempts=self.k8s_wait_attempts,
k8s_wait_attempt_sleep=self.k8s_wait_attempt_sleep,
timeout=self.timeout)
wait_timeout = chart_wait.get_timeout()
# Begin Chart timeout deadline
deadline = time.time() + wait_timeout
old_release = self.helm.release_metadata(release_id)
action = metrics.ChartDeployAction.NOOP
def noop():
pass
deploy = noop
# Resolve action
values = chart.get('values', {})
pre_actions = {}
status = None
if old_release:
status = r.get_release_status(old_release)
native_wait_enabled = chart_wait.is_native_enabled()
chartbuilder = ChartBuilder.from_chart_doc(ch, self.helm)
if status == helm.STATUS_DEPLOYED:
# indicate to the end user what path we are taking
LOG.info("Existing release %s found", release_id)
# extract the installed chart and installed values from the
# latest release so we can compare to the intended state
old_chart = old_release['chart']
old_values = old_release.get('config', {})
upgrade = chart.get('upgrade', {})
options = upgrade.get('options', {})
# TODO: Remove when v1 doc support is removed.
schema_info = get_schema_info(ch['schema'])
if schema_info.version < 2:
no_hooks_location = upgrade
else:
no_hooks_location = options
disable_hooks = no_hooks_location.get('no_hooks', False)
force = options.get('force', False)
if upgrade:
upgrade_pre = upgrade.get('pre', {})
upgrade_post = upgrade.get('post', {})
if not self.disable_update_pre and upgrade_pre:
pre_actions = upgrade_pre
if not self.disable_update_post and upgrade_post:
LOG.warning('Post upgrade actions are ignored by Armada'
'and will not affect deployment.')
LOG.info('Checking for updates to chart release inputs.')
new_chart = chartbuilder.get_helm_chart(release_id, values)
diff = self.get_diff(old_chart, old_values, new_chart, values)
if not diff:
LOG.info("Found no updates to chart release inputs")
else:
action = metrics.ChartDeployAction.UPGRADE
LOG.info("Found updates to chart release inputs")
def upgrade():
# do actual update
timer = int(round(deadline - time.time()))
PreUpdateActions(self.helm.k8s).execute(
pre_actions, release, namespace, chart, disable_hooks,
values, timer)
LOG.info("Upgrading release=%s, wait=%s, "
"timeout=%ss", release_id, native_wait_enabled,
timer)
self.helm.upgrade_release(source_directory,
release_id,
disable_hooks=disable_hooks,
values=values,
wait=native_wait_enabled,
timeout=timer,
force=force)
LOG.info('Upgrade completed')
result['upgrade'] = release_id
deploy = upgrade
else:
def install():
timer = int(round(deadline - time.time()))
LOG.info("Installing release=%s, wait=%s, "
"timeout=%ss", release_id, native_wait_enabled, timer)
self.helm.install_release(source_directory,
release_id,
values=values,
wait=native_wait_enabled,
timeout=timer)
LOG.info('Install completed')
result['install'] = release_id
# Check for release with status other than DEPLOYED
if status:
if status != helm.STATUS_FAILED:
LOG.warn(
'Unexpected release status encountered '
'release=%s, status=%s', release_id, status)
# Make best effort to determine whether a deployment is
# likely pending, by checking if the last deployment
# was started within the timeout window of the chart.
last_deployment_age = r.get_last_deployment_age(
old_release)
likely_pending = last_deployment_age <= wait_timeout
if likely_pending:
# We don't take any deploy action and wait for the
# to get deployed.
deploy = noop
deadline = deadline - last_deployment_age
else:
# Release is likely stuck in an unintended
# state. Log and continue on with remediation steps
# below.
LOG.info(
'Old release %s likely stuck in status %s, '
'(last deployment age=%ss) >= '
'(chart wait timeout=%ss)', release, status,
last_deployment_age, wait_timeout)
res = self.purge_release(chart, release_id, status,
manifest_name, chart_name,
result)
if isinstance(res, dict):
if 'protected' in res:
return res
action = metrics.ChartDeployAction.INSTALL
deploy = install
else:
# The chart is in Failed state, hence we purge
# the chart and attempt to install it again.
res = self.purge_release(chart, release_id, status,
manifest_name, chart_name, result)
if isinstance(res, dict):
if 'protected' in res:
return res
action = metrics.ChartDeployAction.INSTALL
deploy = install
if status is None:
action = metrics.ChartDeployAction.INSTALL
deploy = install
# Deploy
with metrics.CHART_DEPLOY.get_context(wait_timeout, manifest_name,
chart_name,
action.get_label_value()):
deploy()
# Wait
timer = int(round(deadline - time.time()))
chart_wait.wait(timer)
# Test
just_deployed = ('install' in result) or ('upgrade' in result)
last_test_passed = old_release and r.get_last_test_result(old_release)
test_handler = Test(chart,
release_id,
self.helm,
cg_test_charts=cg_test_all_charts)
run_test = test_handler.test_enabled and (just_deployed
or not last_test_passed)
if run_test:
with metrics.CHART_TEST.get_context(test_handler.timeout,
manifest_name, chart_name):
self._test_chart(test_handler)
return result
def _execute(self, ch, cg_test_all_charts, prefix, known_releases):
manifest_name = self.manifest['metadata']['name']
chart = ch[const.KEYWORD_DATA]
chart_name = ch['metadata']['name']
namespace = chart.get('namespace')
release = chart.get('release')
release_name = r.release_prefixer(prefix, release)
LOG.info('Processing Chart, release=%s', release_name)
result = {}
chart_wait = ChartWait(
self.tiller.k8s,
release_name,
ch,
namespace,
k8s_wait_attempts=self.k8s_wait_attempts,
k8s_wait_attempt_sleep=self.k8s_wait_attempt_sleep,
timeout=self.timeout)
wait_timeout = chart_wait.get_timeout()
# Begin Chart timeout deadline
deadline = time.time() + wait_timeout
old_release = self.find_chart_release(known_releases, release_name)
action = metrics.ChartDeployAction.NOOP
def noop():
pass
deploy = noop
# Resolve action
values = chart.get('values', {})
pre_actions = {}
post_actions = {}
status = None
if old_release:
status = r.get_release_status(old_release)
native_wait_enabled = chart_wait.is_native_enabled()
chartbuilder = ChartBuilder.from_chart_doc(ch)
new_chart = chartbuilder.get_helm_chart()
if status == const.STATUS_DEPLOYED:
# indicate to the end user what path we are taking
LOG.info("Existing release %s found in namespace %s", release_name,
namespace)
# extract the installed chart and installed values from the
# latest release so we can compare to the intended state
old_chart = old_release.chart
old_values_string = old_release.config.raw
upgrade = chart.get('upgrade', {})
options = upgrade.get('options', {})
# TODO: Remove when v1 doc support is removed.
schema_info = get_schema_info(ch['schema'])
if schema_info.version < 2:
no_hooks_location = upgrade
else:
no_hooks_location = options
disable_hooks = no_hooks_location.get('no_hooks', False)
force = options.get('force', False)
recreate_pods = options.get('recreate_pods', False)
if upgrade:
upgrade_pre = upgrade.get('pre', {})
upgrade_post = upgrade.get('post', {})
if not self.disable_update_pre and upgrade_pre:
pre_actions = upgrade_pre
if not self.disable_update_post and upgrade_post:
LOG.warning('Post upgrade actions are ignored by Armada'
'and will not affect deployment.')
post_actions = upgrade_post
try:
old_values = yaml.safe_load(old_values_string)
except yaml.YAMLError:
chart_desc = '{} (previously deployed)'.format(
old_chart.metadata.name)
raise armada_exceptions.\
InvalidOverrideValuesYamlException(chart_desc)
LOG.info('Checking for updates to chart release inputs.')
diff = self.get_diff(old_chart, old_values, new_chart, values)
if not diff:
LOG.info("Found no updates to chart release inputs")
else:
action = metrics.ChartDeployAction.UPGRADE
LOG.info("Found updates to chart release inputs")
LOG.debug("%s", diff)
result['diff'] = {chart['release']: str(diff)}
def upgrade():
# do actual update
timer = int(round(deadline - time.time()))
LOG.info(
"Upgrading release %s in namespace %s, wait=%s, "
"timeout=%ss", release_name, namespace,
native_wait_enabled, timer)
tiller_result = self.tiller.update_release(
new_chart,
release_name,
namespace,
pre_actions=pre_actions,
post_actions=post_actions,
disable_hooks=disable_hooks,
values=yaml.safe_dump(values),
wait=native_wait_enabled,
timeout=timer,
force=force,
recreate_pods=recreate_pods)
LOG.info('Upgrade completed with results from Tiller: %s',
tiller_result.__dict__)
result['upgrade'] = release_name
deploy = upgrade
else:
# Check for release with status other than DEPLOYED
if status:
if status != const.STATUS_FAILED:
LOG.warn(
'Unexpected release status encountered '
'release=%s, status=%s', release_name, status)
# Make best effort to determine whether a deployment is
# likely pending, by checking if the last deployment
# was started within the timeout window of the chart.
last_deployment_age = r.get_last_deployment_age(
old_release)
likely_pending = last_deployment_age <= wait_timeout
if likely_pending:
# Give up if a deployment is likely pending, we do not
# want to have multiple operations going on for the
# same release at the same time.
raise armada_exceptions.\
DeploymentLikelyPendingException(
release_name, status, last_deployment_age,
wait_timeout)
else:
# Release is likely stuck in an unintended (by tiller)
# state. Log and continue on with remediation steps
# below.
LOG.info(
'Old release %s likely stuck in status %s, '
'(last deployment age=%ss) >= '
'(chart wait timeout=%ss)', release, status,
last_deployment_age, wait_timeout)
protected = chart.get('protected', {})
if protected:
p_continue = protected.get('continue_processing', False)
if p_continue:
LOG.warn(
'Release %s is `protected`, '
'continue_processing=True. Operator must '
'handle %s release manually.', release_name,
status)
result['protected'] = release_name
return result
else:
LOG.error(
'Release %s is `protected`, '
'continue_processing=False.', release_name)
raise armada_exceptions.ProtectedReleaseException(
release_name, status)
else:
# Purge the release
with metrics.CHART_DELETE.get_context(
manifest_name, chart_name):
LOG.info('Purging release %s with status %s',
release_name, status)
chart_delete = ChartDelete(chart, release_name,
self.tiller)
chart_delete.delete()
result['purge'] = release_name
action = metrics.ChartDeployAction.INSTALL
def install():
timer = int(round(deadline - time.time()))
LOG.info(
"Installing release %s in namespace %s, wait=%s, "
"timeout=%ss", release_name, namespace,
native_wait_enabled, timer)
tiller_result = self.tiller.install_release(
new_chart,
release_name,
namespace,
values=yaml.safe_dump(values),
wait=native_wait_enabled,
timeout=timer)
LOG.info('Install completed with results from Tiller: %s',
tiller_result.__dict__)
result['install'] = release_name
deploy = install
# Deploy
with metrics.CHART_DEPLOY.get_context(wait_timeout, manifest_name,
chart_name,
action.get_label_value()):
deploy()
# Wait
timer = int(round(deadline - time.time()))
chart_wait.wait(timer)
# Test
just_deployed = ('install' in result) or ('upgrade' in result)
last_test_passed = old_release and r.get_last_test_result(old_release)
test_handler = Test(chart,
release_name,
self.tiller,
cg_test_charts=cg_test_all_charts)
run_test = test_handler.test_enabled and (just_deployed
or not last_test_passed)
if run_test:
with metrics.CHART_TEST.get_context(test_handler.timeout,
manifest_name, chart_name):
self._test_chart(release_name, test_handler)
return result
