def update_document(self, ovr): ovr_schema_info = schema.get_schema_info(ovr.get('schema')) if ovr_schema_info: for doc in self.documents: schema_info = schema.get_schema_info(doc.get('schema')) if schema_info: if schema_info == ovr_schema_info: if doc['metadata']['name'] == ovr['metadata']['name']: data = doc.get('data', {}) ovr_data = ovr.get('data', {}) self.update(data, ovr_data) return
def _find_documents(self, target_manifest=None): """Returns the chart documents, chart group documents, and Armada manifest If multiple documents with schema "armada/Manifest/v1" are provided, specify ``target_manifest`` to select the target one. :param str target_manifest: The target manifest to use when multiple documents with "armada/Manifest/v1" are contained in ``documents``. Default is None. :returns: Tuple of chart documents, chart groups, and manifests found in ``self.documents`` :rtype: tuple """ charts = [] groups = [] manifests = [] for document in self.documents: schema_info = schema.get_schema_info(document.get('schema')) if not schema_info: continue if schema_info.type == schema.TYPE_CHART: charts.append(document) if schema_info.type == schema.TYPE_CHARTGROUP: groups.append(document) if schema_info.type == schema.TYPE_MANIFEST: manifest_name = document.get('metadata', {}).get('name') if target_manifest: if manifest_name == target_manifest: manifests.append(document) else: manifests.append(document) return charts, groups, manifests
def from_chart_doc(cls, chart): ''' Returns a ChartBuilder defined by an Armada Chart doc. :param chart: Armada Chart doc for which to build the Helm chart. ''' name = chart['metadata']['name'] chart_data = chart[const.KEYWORD_DATA] source_dir = chart_data.get('source_dir') source_directory = os.path.join(*source_dir) dependencies = chart_data.get('dependencies') # TODO: Remove when v1 doc support is removed. schema_info = get_schema_info(chart['schema']) if schema_info.version < 2: fix_tpl_name = False else: fix_tpl_name = True if dependencies is not None: dependency_builders = [] for chart_dep in dependencies: builder = ChartBuilder.from_chart_doc(chart_dep) dependency_builders.append(builder) return cls(name, source_directory, dependency_builders, fix_tpl_name=fix_tpl_name) return cls.from_source(name, source_directory, fix_tpl_name=fix_tpl_name)
def find_manifest_document(self, doc_path): for doc in self.documents: schema_info = schema.get_schema_info(doc.get('schema')) if schema_info.type == self.find_document_type( doc_path[0]) and doc.get('metadata', {}).get('name') == doc_path[1]: return doc raise override_exceptions.UnknownDocumentOverrideException( doc_path[0], doc_path[1])
def validate_armada_document(document): """Validates a document ingested by Armada by subjecting it to JSON schema validation. :param dict dictionary: The document to validate. :returns: A tuple of (bool, list[dict]) where the first value indicates whether the validation succeeded or failed and the second value is the validation details with a minimum keyset of (message(str), error(bool)) :rtype: tuple. :raises TypeError: If ``document`` is not of type ``dict``. """ if not isinstance(document, dict): raise TypeError('The provided input "%s" must be a dictionary.' % document) schema = document.get('schema', '<missing>') document_name = document.get('metadata', {}).get('name', None) details = [] LOG.debug('Validating document [%s] %s', schema, document_name) schema_info = sch.get_schema_info(schema) if schema_info: try: validator = jsonschema.Draft4Validator(schema_info.data) for error in validator.iter_errors(document.get('data')): error_message = "Invalid document [%s] %s: %s." % \ (schema, document_name, error.message) vmsg = ValidationMessage(message=error_message, error=True, name='ARM100', level='Error', schema=schema, doc_name=document_name) LOG.info('ValidationMessage: %s', vmsg.get_output_json()) details.append(vmsg.get_output()) except jsonschema.SchemaError as e: error_message = ('The built-in Armada JSON schema %s is invalid. ' 'Details: %s.' % (e.schema, e.message)) vmsg = ValidationMessage(message=error_message, error=True, name='ARM000', level='Error', diagnostic='Armada is misconfigured.') LOG.error('ValidationMessage: %s', vmsg.get_output_json()) details.append(vmsg.get_output()) if len([x for x in details if x.get('error', False)]) > 0: return False, details return True, details
def wait(self, timeout): ''' :param timeout: time before disconnecting ``Watch`` stream ''' min_ready_msg = ', min_ready={}'.format( self.min_ready.source) if isinstance(self, ControllerWait) else '' LOG.info( "Waiting for resource type=%s, namespace=%s labels=%s " "required=%s%s for %ss", self.resource_type, self.chart_wait.release_id.namespace, self.label_selector, self.required, min_ready_msg, timeout) if not self.label_selector: LOG.warn( '"label_selector" not specified, waiting with no labels ' 'may cause unintended consequences.') # Track the overall deadline for timing out during waits deadline = time.time() + timeout schema_info = get_schema_info(self.chart_wait.chart['schema']) # TODO: Remove when v1 doc support is removed. if schema_info.version < 2: # NOTE(mark-burnett): Attempt to wait multiple times without # modification, in case new resources appear after our watch exits. successes = 0 while True: modified = self._wait(deadline) if modified is None: break if modified: successes = 0 LOG.debug('Found modified resources: %s', sorted(modified)) else: successes += 1 LOG.debug('Found no modified resources.') if successes >= self.chart_wait.k8s_wait_attempts: return LOG.debug( 'Continuing to wait: %s consecutive attempts without ' 'modified resources of %s required.', successes, self.chart_wait.k8s_wait_attempts) time.sleep(self.chart_wait.k8s_wait_attempt_sleep) else: self._wait(deadline)
def validate_armada_manifests(documents): """Validate each Armada manifest found in the document set. :param documents: List of Armada documents to validate :type documents: :func: `list[dict]`. """ messages = [] all_valid = True for document in documents: doc_schema = document.get('schema') if doc_schema: schema_info = sch.get_schema_info(doc_schema) if schema_info and schema_info.type == sch.TYPE_MANIFEST: target = document.get('metadata').get('name') # TODO(MarshM) explore: why does this pass 'documents'? manifest = Manifest(documents, target_manifest=target) is_valid, details = _validate_armada_manifest(manifest) all_valid = all_valid and is_valid messages.extend(details) return all_valid, messages
def get_exclude_reason(self, resource): pod = resource # Exclude helm test pods # TODO: Possibly exclude other helm hook pods/jobs (besides tests)? if is_test_pod(pod): return 'helm test pod' if pod.status.phase == 'Evicted': return "pod was evicted" schema_info = get_schema_info(self.chart_wait.chart['schema']) # TODO: Remove when v1 doc support is removed. if schema_info.version < 2: # Exclude job pods if has_owner(pod, 'Job'): return 'owned by job' else: # Exclude all pods with an owner (only include raw pods) if has_owner(pod): return 'owned by another resource' return None
def get_exclude_reason(self, resource): pod = resource # Exclude helm test pods # TODO: Possibly exclude other helm hook pods/jobs (besides tests)? if is_test_pod(pod): return 'helm test pod' schema_info = get_schema_info(self.chart_wait.chart['schema']) # TODO: Remove when v1 doc support is removed. if schema_info.version < 2: # Exclude job pods if has_owner(pod, 'Job'): return 'owned by job' else: # Exclude all pods with an owner (only include raw pods) # TODO: In helm 3, all resources will likely have the release CR as # an owner, so this will need to be updated to not exclude pods # directly owned by the release. if has_owner(pod): return 'owned by another resource' return None
def __init__( self, k8s, release_id, chart, k8s_wait_attempts, k8s_wait_attempt_sleep, timeout): self.k8s = k8s self.release_id = release_id self.chart = chart chart_data = self.chart[const.KEYWORD_DATA] self.chart_data = chart_data self.wait_config = self.chart_data.get('wait', {}) self.k8s_wait_attempts = max(k8s_wait_attempts, 1) self.k8s_wait_attempt_sleep = max(k8s_wait_attempt_sleep, 1) schema_info = get_schema_info(self.chart['schema']) resources = self.wait_config.get('resources') if isinstance(resources, list): # Explicit resource config list provided. resources_list = resources else: # TODO: Remove when v1 doc support is removed. if schema_info.version < 2: resources_list = [ { 'type': 'job', 'required': False }, { 'type': 'pod' } ] else: resources_list = self.get_resources_list(resources) chart_labels = get_wait_labels(self.chart_data) for resource_config in resources_list: # Use chart labels as base labels for each config. labels = dict(chart_labels) resource_labels = resource_config.get('labels', {}) # Merge in any resource-specific labels. if resource_labels: labels.update(resource_labels) resource_config['labels'] = labels LOG.debug('Resolved `wait.resources` list: %s', resources_list) self.waits = [self.get_resource_wait(conf) for conf in resources_list] # Calculate timeout wait_timeout = timeout if wait_timeout is None: wait_timeout = self.wait_config.get('timeout') # TODO: Remove when v1 doc support is removed. deprecated_timeout = self.chart_data.get('timeout') if deprecated_timeout is not None: LOG.warn( 'The `timeout` key is deprecated and support ' 'for this will be removed soon. Use ' '`wait.timeout` instead.') if wait_timeout is None: wait_timeout = deprecated_timeout if wait_timeout is None: LOG.info( 'No Chart timeout specified, using default: %ss', const.DEFAULT_CHART_TIMEOUT) wait_timeout = const.DEFAULT_CHART_TIMEOUT self.timeout = wait_timeout # Determine whether to enable native wait. native = self.wait_config.get('native', {}) # TODO: Remove when v1 doc support is removed. default_native = schema_info.version < 2 self.native_enabled = native.get('enabled', default_native)
def delete_resources(self, resource_type, resource_labels, namespace, wait=False, timeout=const.DEFAULT_TILLER_TIMEOUT): ''' Delete resources matching provided resource type, labels, and namespace. :param resource_type: type of resource e.g. job, pod, etc. :param resource_labels: labels for selecting the resources :param namespace: namespace of resources ''' timeout = self._check_timeout(wait, timeout) label_selector = '' if resource_labels is not None: label_selector = label_selectors(resource_labels) LOG.debug( "Deleting resources in namespace %s matching " "selectors (%s).", namespace, label_selector) handled = False if resource_type == 'job': get_jobs = self.k8s.get_namespace_job( namespace, label_selector=label_selector) for jb in get_jobs.items: jb_name = jb.metadata.name if self.dry_run: LOG.info( 'Skipping delete job during `dry-run`, would ' 'have deleted job %s in namespace=%s.', jb_name, namespace) continue LOG.info("Deleting job %s in namespace: %s", jb_name, namespace) self.k8s.delete_job_action(jb_name, namespace, timeout=timeout) handled = True # TODO: Remove when v1 doc support is removed. chart = get_current_chart() schema_info = schema.get_schema_info(chart['schema']) job_implies_cronjob = schema_info.version < 2 implied_cronjob = resource_type == 'job' and job_implies_cronjob if resource_type == 'cronjob' or implied_cronjob: get_jobs = self.k8s.get_namespace_cron_job( namespace, label_selector=label_selector) for jb in get_jobs.items: jb_name = jb.metadata.name # TODO: Remove when v1 doc support is removed. if implied_cronjob: LOG.warn("Deleting cronjobs via `type: job` is " "deprecated, use `type: cronjob` instead") if self.dry_run: LOG.info( 'Skipping delete cronjob during `dry-run`, would ' 'have deleted cronjob %s in namespace=%s.', jb_name, namespace) continue LOG.info("Deleting cronjob %s in namespace: %s", jb_name, namespace) self.k8s.delete_cron_job_action(jb_name, namespace) handled = True if resource_type == 'pod': release_pods = self.k8s.get_namespace_pod( namespace, label_selector=label_selector) for pod in release_pods.items: pod_name = pod.metadata.name if self.dry_run: LOG.info( 'Skipping delete pod during `dry-run`, would ' 'have deleted pod %s in namespace=%s.', pod_name, namespace) continue LOG.info("Deleting pod %s in namespace: %s", pod_name, namespace) self.k8s.delete_pod_action(pod_name, namespace) if wait: self.k8s.wait_for_pod_redeployment(pod_name, namespace) handled = True if not handled: LOG.error('No resources found with labels=%s type=%s namespace=%s', resource_labels, resource_type, namespace)
def _execute(self, ch, cg_test_all_charts, prefix): manifest_name = self.manifest['metadata']['name'] chart = ch[const.KEYWORD_DATA] chart_name = ch['metadata']['name'] namespace = chart.get('namespace') release = chart.get('release') release_name = r.release_prefixer(prefix, release) release_id = helm.HelmReleaseId(namespace, release_name) source_dir = chart['source_dir'] source_directory = os.path.join(*source_dir) LOG.info('Processing Chart, release=%s', release_id) result = {} chart_wait = ChartWait( self.helm.k8s, release_id, ch, k8s_wait_attempts=self.k8s_wait_attempts, k8s_wait_attempt_sleep=self.k8s_wait_attempt_sleep, timeout=self.timeout) wait_timeout = chart_wait.get_timeout() # Begin Chart timeout deadline deadline = time.time() + wait_timeout old_release = self.helm.release_metadata(release_id) action = metrics.ChartDeployAction.NOOP def noop(): pass deploy = noop # Resolve action values = chart.get('values', {}) pre_actions = {} status = None if old_release: status = r.get_release_status(old_release) native_wait_enabled = chart_wait.is_native_enabled() chartbuilder = ChartBuilder.from_chart_doc(ch, self.helm) if status == helm.STATUS_DEPLOYED: # indicate to the end user what path we are taking LOG.info("Existing release %s found", release_id) # extract the installed chart and installed values from the # latest release so we can compare to the intended state old_chart = old_release['chart'] old_values = old_release.get('config', {}) upgrade = chart.get('upgrade', {}) options = upgrade.get('options', {}) # TODO: Remove when v1 doc support is removed. schema_info = get_schema_info(ch['schema']) if schema_info.version < 2: no_hooks_location = upgrade else: no_hooks_location = options disable_hooks = no_hooks_location.get('no_hooks', False) force = options.get('force', False) if upgrade: upgrade_pre = upgrade.get('pre', {}) upgrade_post = upgrade.get('post', {}) if not self.disable_update_pre and upgrade_pre: pre_actions = upgrade_pre if not self.disable_update_post and upgrade_post: LOG.warning('Post upgrade actions are ignored by Armada' 'and will not affect deployment.') LOG.info('Checking for updates to chart release inputs.') new_chart = chartbuilder.get_helm_chart(release_id, values) diff = self.get_diff(old_chart, old_values, new_chart, values) if not diff: LOG.info("Found no updates to chart release inputs") else: action = metrics.ChartDeployAction.UPGRADE LOG.info("Found updates to chart release inputs") def upgrade(): # do actual update timer = int(round(deadline - time.time())) PreUpdateActions(self.helm.k8s).execute( pre_actions, release, namespace, chart, disable_hooks, values, timer) LOG.info("Upgrading release=%s, wait=%s, " "timeout=%ss", release_id, native_wait_enabled, timer) self.helm.upgrade_release(source_directory, release_id, disable_hooks=disable_hooks, values=values, wait=native_wait_enabled, timeout=timer, force=force) LOG.info('Upgrade completed') result['upgrade'] = release_id deploy = upgrade else: def install(): timer = int(round(deadline - time.time())) LOG.info("Installing release=%s, wait=%s, " "timeout=%ss", release_id, native_wait_enabled, timer) self.helm.install_release(source_directory, release_id, values=values, wait=native_wait_enabled, timeout=timer) LOG.info('Install completed') result['install'] = release_id # Check for release with status other than DEPLOYED if status: if status != helm.STATUS_FAILED: LOG.warn( 'Unexpected release status encountered ' 'release=%s, status=%s', release_id, status) # Make best effort to determine whether a deployment is # likely pending, by checking if the last deployment # was started within the timeout window of the chart. last_deployment_age = r.get_last_deployment_age( old_release) likely_pending = last_deployment_age <= wait_timeout if likely_pending: # We don't take any deploy action and wait for the # to get deployed. deploy = noop deadline = deadline - last_deployment_age else: # Release is likely stuck in an unintended # state. Log and continue on with remediation steps # below. LOG.info( 'Old release %s likely stuck in status %s, ' '(last deployment age=%ss) >= ' '(chart wait timeout=%ss)', release, status, last_deployment_age, wait_timeout) res = self.purge_release(chart, release_id, status, manifest_name, chart_name, result) if isinstance(res, dict): if 'protected' in res: return res action = metrics.ChartDeployAction.INSTALL deploy = install else: # The chart is in Failed state, hence we purge # the chart and attempt to install it again. res = self.purge_release(chart, release_id, status, manifest_name, chart_name, result) if isinstance(res, dict): if 'protected' in res: return res action = metrics.ChartDeployAction.INSTALL deploy = install if status is None: action = metrics.ChartDeployAction.INSTALL deploy = install # Deploy with metrics.CHART_DEPLOY.get_context(wait_timeout, manifest_name, chart_name, action.get_label_value()): deploy() # Wait timer = int(round(deadline - time.time())) chart_wait.wait(timer) # Test just_deployed = ('install' in result) or ('upgrade' in result) last_test_passed = old_release and r.get_last_test_result(old_release) test_handler = Test(chart, release_id, self.helm, cg_test_charts=cg_test_all_charts) run_test = test_handler.test_enabled and (just_deployed or not last_test_passed) if run_test: with metrics.CHART_TEST.get_context(test_handler.timeout, manifest_name, chart_name): self._test_chart(test_handler) return result
def _execute(self, ch, cg_test_all_charts, prefix, known_releases): manifest_name = self.manifest['metadata']['name'] chart = ch[const.KEYWORD_DATA] chart_name = ch['metadata']['name'] namespace = chart.get('namespace') release = chart.get('release') release_name = r.release_prefixer(prefix, release) LOG.info('Processing Chart, release=%s', release_name) result = {} chart_wait = ChartWait( self.tiller.k8s, release_name, ch, namespace, k8s_wait_attempts=self.k8s_wait_attempts, k8s_wait_attempt_sleep=self.k8s_wait_attempt_sleep, timeout=self.timeout) wait_timeout = chart_wait.get_timeout() # Begin Chart timeout deadline deadline = time.time() + wait_timeout old_release = self.find_chart_release(known_releases, release_name) action = metrics.ChartDeployAction.NOOP def noop(): pass deploy = noop # Resolve action values = chart.get('values', {}) pre_actions = {} post_actions = {} status = None if old_release: status = r.get_release_status(old_release) native_wait_enabled = chart_wait.is_native_enabled() chartbuilder = ChartBuilder.from_chart_doc(ch) new_chart = chartbuilder.get_helm_chart() if status == const.STATUS_DEPLOYED: # indicate to the end user what path we are taking LOG.info("Existing release %s found in namespace %s", release_name, namespace) # extract the installed chart and installed values from the # latest release so we can compare to the intended state old_chart = old_release.chart old_values_string = old_release.config.raw upgrade = chart.get('upgrade', {}) options = upgrade.get('options', {}) # TODO: Remove when v1 doc support is removed. schema_info = get_schema_info(ch['schema']) if schema_info.version < 2: no_hooks_location = upgrade else: no_hooks_location = options disable_hooks = no_hooks_location.get('no_hooks', False) force = options.get('force', False) recreate_pods = options.get('recreate_pods', False) if upgrade: upgrade_pre = upgrade.get('pre', {}) upgrade_post = upgrade.get('post', {}) if not self.disable_update_pre and upgrade_pre: pre_actions = upgrade_pre if not self.disable_update_post and upgrade_post: LOG.warning('Post upgrade actions are ignored by Armada' 'and will not affect deployment.') post_actions = upgrade_post try: old_values = yaml.safe_load(old_values_string) except yaml.YAMLError: chart_desc = '{} (previously deployed)'.format( old_chart.metadata.name) raise armada_exceptions.\ InvalidOverrideValuesYamlException(chart_desc) LOG.info('Checking for updates to chart release inputs.') diff = self.get_diff(old_chart, old_values, new_chart, values) if not diff: LOG.info("Found no updates to chart release inputs") else: action = metrics.ChartDeployAction.UPGRADE LOG.info("Found updates to chart release inputs") LOG.debug("%s", diff) result['diff'] = {chart['release']: str(diff)} def upgrade(): # do actual update timer = int(round(deadline - time.time())) LOG.info( "Upgrading release %s in namespace %s, wait=%s, " "timeout=%ss", release_name, namespace, native_wait_enabled, timer) tiller_result = self.tiller.update_release( new_chart, release_name, namespace, pre_actions=pre_actions, post_actions=post_actions, disable_hooks=disable_hooks, values=yaml.safe_dump(values), wait=native_wait_enabled, timeout=timer, force=force, recreate_pods=recreate_pods) LOG.info('Upgrade completed with results from Tiller: %s', tiller_result.__dict__) result['upgrade'] = release_name deploy = upgrade else: # Check for release with status other than DEPLOYED if status: if status != const.STATUS_FAILED: LOG.warn( 'Unexpected release status encountered ' 'release=%s, status=%s', release_name, status) # Make best effort to determine whether a deployment is # likely pending, by checking if the last deployment # was started within the timeout window of the chart. last_deployment_age = r.get_last_deployment_age( old_release) likely_pending = last_deployment_age <= wait_timeout if likely_pending: # Give up if a deployment is likely pending, we do not # want to have multiple operations going on for the # same release at the same time. raise armada_exceptions.\ DeploymentLikelyPendingException( release_name, status, last_deployment_age, wait_timeout) else: # Release is likely stuck in an unintended (by tiller) # state. Log and continue on with remediation steps # below. LOG.info( 'Old release %s likely stuck in status %s, ' '(last deployment age=%ss) >= ' '(chart wait timeout=%ss)', release, status, last_deployment_age, wait_timeout) protected = chart.get('protected', {}) if protected: p_continue = protected.get('continue_processing', False) if p_continue: LOG.warn( 'Release %s is `protected`, ' 'continue_processing=True. Operator must ' 'handle %s release manually.', release_name, status) result['protected'] = release_name return result else: LOG.error( 'Release %s is `protected`, ' 'continue_processing=False.', release_name) raise armada_exceptions.ProtectedReleaseException( release_name, status) else: # Purge the release with metrics.CHART_DELETE.get_context( manifest_name, chart_name): LOG.info('Purging release %s with status %s', release_name, status) chart_delete = ChartDelete(chart, release_name, self.tiller) chart_delete.delete() result['purge'] = release_name action = metrics.ChartDeployAction.INSTALL def install(): timer = int(round(deadline - time.time())) LOG.info( "Installing release %s in namespace %s, wait=%s, " "timeout=%ss", release_name, namespace, native_wait_enabled, timer) tiller_result = self.tiller.install_release( new_chart, release_name, namespace, values=yaml.safe_dump(values), wait=native_wait_enabled, timeout=timer) LOG.info('Install completed with results from Tiller: %s', tiller_result.__dict__) result['install'] = release_name deploy = install # Deploy with metrics.CHART_DEPLOY.get_context(wait_timeout, manifest_name, chart_name, action.get_label_value()): deploy() # Wait timer = int(round(deadline - time.time())) chart_wait.wait(timer) # Test just_deployed = ('install' in result) or ('upgrade' in result) last_test_passed = old_release and r.get_last_test_result(old_release) test_handler = Test(chart, release_name, self.tiller, cg_test_charts=cg_test_all_charts) run_test = test_handler.test_enabled and (just_deployed or not last_test_passed) if run_test: with metrics.CHART_TEST.get_context(test_handler.timeout, manifest_name, chart_name): self._test_chart(release_name, test_handler) return result