def calculate_range( self, repository_branch: str, ) -> str: repo = self.git_helper.repo branch_head = self.git_helper.fetch_head(ref=repository_branch) if not branch_head: fail( f'could not determine branch head of {repository_branch} branch' ) range_start = _.head( self.reachable_release_tags_from_commit( repo=repo, commit=branch_head, ), ) try: # better readable range_end by describing head commit range_end = repo.git.describe(branch_head, tags=True) except GitError as err: warning( 'failed to describe branch head, maybe the repository has no tags? ' f'GitError: {err}. Falling back to branch head commit hash.') range_end = branch_head.hexsha commit_range = f'{range_start}..{range_end}' return commit_range
def http_checker(*args, **kwargs): result = function(*args, **kwargs) if result.status_code < 200 or result.status_code >= 300: url = kwargs.get('url', None) warning('{c} - {m}: {u}'.format(c=result.status_code, m=result.content, u=url)) result.raise_for_status() return result
def calculate_range( repository_branch: str, git_helper: GitHelper, github_helper: GitHubRepositoryHelper, ) -> str: repo = git_helper.repo branch_head = git_helper.fetch_head(ref=repository_branch) if not branch_head: fail('could not determine branch head of {branch} branch'.format( branch=repository_branch)) range_start = _.head( reachable_release_tags_from_commit(github_helper, repo, branch_head)) try: # better readable range_end by describing head commit range_end = repo.git.describe(branch_head, tags=True) except GitError as err: warning( 'failed to describe branch head, maybe the repository has no tags? ' 'GitError: {err}. Falling back to branch head commit hash.'.format( err=err)) range_end = branch_head.hexsha commit_range = "{start}..{end}".format(start=range_start, end=range_end) return commit_range
def _process_definition_descriptor(self, definition_descriptor): if definition_descriptor.exception: return DeployResult( definition_descriptor=definition_descriptor, deploy_status=DeployStatus.SKIPPED, error_details=definition_descriptor.exception, ) preprocessed = self.descriptor_preprocessor.process_definition_descriptor( definition_descriptor ) result = self.definition_renderer.render(preprocessed) if self._pipeline_name_conflict( definition_descriptor=result.definition_descriptor, ): # early exit upon pipeline name conflict pipeline_name = result.definition_descriptor.pipeline_name warning(f'duplicate pipeline name: {pipeline_name}') return DeployResult( definition_descriptor=definition_descriptor, deploy_status=DeployStatus.SKIPPED, error_details=f'duplicate pipeline name: {pipeline_name}', ) if result.render_status == RenderStatus.SUCCEEDED: deploy_result = self.definition_deployer.deploy(result.definition_descriptor) else: deploy_result = DeployResult( definition_descriptor=definition_descriptor, deploy_status=DeployStatus.SKIPPED, error_details=result.error_details, ) return deploy_result
def _cfg_element(self, cfg_type_name: str, cfg_name: str): cfg_type = self._cfg_type(cfg_type_name=cfg_type_name) # retrieve model class c'tor - search module and sub-modules # TODO: switch to fully-qualified type names own_module = sys.modules[__name__] submodule_names = [ own_module.__name__ + '.' + m.name for m in pkgutil.iter_modules(own_module.__path__) ] for module_name in [__name__] + submodule_names: submodule_name = module_name.split('.')[-1] if module_name != __name__: module = getattr(__import__(module_name), submodule_name) else: module = sys.modules[submodule_name] # skip if module does not define our type if not hasattr(module, cfg_type.cfg_type()): continue # if type is defined, validate element_type = getattr(module, cfg_type.cfg_type()) if not type(element_type) == type: raise ValueError() # found it (write to cache as part of crazy workaround for kaniko) self._cfg_type_cache[cfg_type_name] = element_type break else: # workaround for kaniko, which will purge our poor modules on multi-stage-builds if cfg_type_name in self._cfg_type_cache: element_type = self._cfg_type_cache[cfg_type_name] else: print(f'{self._cfg_type_cache=}') raise ValueError( f'failed to find cfg type: {cfg_type.cfg_type()=}') # for now, let's assume all of our model element types are subtypes of NamedModelElement # (with the exception of ConfigurationSet) configs = self._configs(cfg_type.cfg_type_name()) if cfg_name not in configs: raise ConfigElementNotFoundError( 'no such cfg element: {cn}. Known: {es}'.format( cn=cfg_name, es=', '.join(configs.keys()))) kwargs = {'raw_dict': configs[cfg_name]} if element_type == ConfigurationSet: kwargs.update({'cfg_name': cfg_name, 'cfg_factory': self}) else: kwargs['name'] = cfg_name element_instance = element_type(**kwargs) try: element_instance.validate() except ModelValidationError as mve: warning(f'validation error for {cfg_name} - ignored: {mve}') return element_instance
def destroy_concourse_landscape(config_name: str, release_name: str): # Fetch concourse and kubernetes config config_factory = global_ctx().cfg_factory() config_set = config_factory.cfg_set(cfg_name=config_name) concourse_cfg = config_set.concourse() kubernetes_config_name = concourse_cfg.kubernetes_cluster_config() kubernetes_config = config_factory.kubernetes(kubernetes_config_name) context = kube_ctx context.set_kubecfg(kubernetes_config.kubeconfig()) # Delete helm release helm_cmd_path = ensure_helm_setup() KUBECONFIG_FILE_NAME = 'kubecfg' helm_env = os.environ.copy() helm_env['KUBECONFIG'] = KUBECONFIG_FILE_NAME with tempfile.TemporaryDirectory() as temp_dir: with open(os.path.join(temp_dir, KUBECONFIG_FILE_NAME), 'w') as f: yaml.dump(kubernetes_config.kubeconfig(), f) try: subprocess.run([helm_cmd_path, "delete", release_name], env=helm_env, check=True, cwd=temp_dir) except CalledProcessError: # ignore sporadic connection timeouts from infrastructure warning( "Connection to K8s cluster lost. Continue with deleting namespace {ns}" .format(ns=release_name)) # delete namespace namespace_helper = context.namespace_helper() namespace_helper.delete_namespace(namespace=release_name)
def is_valid_semver(tag_name): try: version.parse_to_semver(tag_name) return True except ValueError: warning('{tag} is not a valid SemVer string'.format(tag=tag_name)) return False
def extract_release_notes( reference_type: ReferenceType, text: str, user_login: str, current_component: gci.componentmodel.Component, source_component=None, reference_id: str = None, ) -> typing.List[ReleaseNote]: """ Keyword arguments: reference_type -- type of reference_id, either pull request or commit reference_id -- reference identifier, could be a pull request number or commit hash text -- release note text user_login -- github user_login, used for referencing the user in the release note via @<user_login> cn_current_repo -- component name of the current repository """ release_notes = list() if not text: return release_notes CATEGORY_IDS = _ \ .chain(CATEGORIES) \ .map(lambda category: category.identifiers) \ .flatten() \ .join('|') \ .value() TARGET_GROUP_IDS = _ \ .chain(TARGET_GROUPS) \ .map(lambda target_group: target_group.identifiers) \ .flatten() \ .join('|') \ .value() r = re.compile( rf"``` *(?P<category>{CATEGORY_IDS}) (?P<target_group>{TARGET_GROUP_IDS})" r"( (?P<source_repo>\S+/\S+/\S+)(( (?P<reference_type>#|\$)(?P<reference_id>\S+))?" r"( @(?P<user>\S+))?)( .*?)?|( .*?)?)\r?\n(?P<text>.*?)\n```", re.MULTILINE | re.DOTALL) for m in r.finditer(text): code_block = m.groupdict() try: rls_note_block = create_release_note_block( code_block=code_block, reference_type=reference_type, reference_id=reference_id, user_login=user_login, current_component=current_component, source_component=source_component, ) if not rls_note_block: continue release_notes.append(rls_note_block) except ModelValidationError as e: warning( f'an exception occurred while extracting release notes: {e}') continue return release_notes
def create_release_note_block( code_block: dict, reference_type: ReferenceType, user_login: str, current_component: gci.componentmodel.Component, source_component: gci.componentmodel.Component = None, reference_id: str = None, ) -> ReleaseNoteBlock: text = _.trim(code_block.get('text')) if not text or 'none' == text.lower(): return None category = code_block.get('category') target_group = code_block.get('target_group') source_repo = code_block.get('source_repo') if source_component: reference_id = code_block.get('reference_id') reference_type = reference_type_for_type_identifier( code_block.get('reference_type')) user_login = code_block.get('user') elif source_repo: try: # try to fetch cd for the parsed source repo. The actual version does not matter, # we're only interested in the components GithubAccess (we assume it does not # change). ctx_repo_url = current_component.current_repository_ctx().baseUrl source_component = cnudie.retrieve.component_descriptor( name=source_repo, version=product.v2.greatest_component_version( component_name=source_repo, ctx_repo_base_url=ctx_repo_url, ), ctx_repo_url=ctx_repo_url, ).component except requests.exceptions.HTTPError: warning( f'Unable to retrieve component descriptor for source repository {source_repo}' ) return None reference_type = reference_type_for_type_identifier( code_block.get('reference_type')) reference_id = code_block.get('reference_id') user_login = code_block.get('user') else: source_component = current_component return ReleaseNoteBlock( category_id=category, target_group_id=target_group, text=text, reference_type=reference_type, reference_id=reference_id, user_login=user_login, source_component=source_component, current_component=current_component, )
def _notify_broken_definition_owners(self, failed_descriptor): definition_descriptor = failed_descriptor.definition_descriptor main_repo = definition_descriptor.main_repo github_cfg = ccc.github.github_cfg_for_hostname(main_repo['hostname'], self._cfg_set) github_api = ccc.github.github_api(github_cfg) repo_owner, repo_name = main_repo['path'].split('/') repo_helper = ccc.github.github_repo_helper( host=main_repo['hostname'], org=repo_owner, repo=repo_name, branch=main_repo['branch'], ) codeowners_enumerator = CodeownersEnumerator() codeowners_resolver = CodeOwnerEntryResolver(github_api=github_api) recipients = set(codeowners_resolver.resolve_email_addresses( codeowners_enumerator.enumerate_remote_repo(github_repo_helper=repo_helper) )) # in case no codeowners are available, resort to using the committer if not recipients: head_commit = repo_helper.repository.commit(main_repo['branch']) user_ids = { user_info.get('login') for user_info in (head_commit.committer, head_commit.author) if user_info and user_info.get('login') } for user_id in user_ids: user = github_api.user(user_id) if user.email: recipients.add(user.email) # if there are still no recipients available print a warning if not recipients: warning(textwrap.dedent( f""" Unable to determine recipient for pipeline '{definition_descriptor.pipeline_name}' found in branch '{main_repo['branch']}' ({main_repo['path']}). Please make sure that CODEOWNERS and committers have exposed a public e-mail address in their profile. """ )) else: info(f'Sending notification e-mail to {recipients} ({main_repo["path"]})') email_cfg = self._cfg_set.email("ses_gardener_cloud_sap") _send_mail( email_cfg=email_cfg, recipients=recipients, subject='Your pipeline definition in {repo} is erroneous'.format( repo=main_repo['path'], ), mail_template=( f"The pipeline definition for pipeline '{definition_descriptor.pipeline_name}' " f" on branch '{main_repo['branch']}' contains errors.\n\n" f"Error details:\n{str(failed_descriptor.error_details)}" ) )
def retrieve_image(image_reference: str): try: container.registry.retrieve_container_image( image_reference=image_reference) info(f'downloaded {image_reference}') except Exception: warning(f'failed to retrieve {image_reference}') import traceback traceback.print_exc()
def __init__(self, raw:dict, concourse_api, name:str): self.concourse_api = concourse_api self.name = name self.raw = raw['config'] resources = self.raw.get('resources', None) if not resources: warning('Pipeline did not contain resource definitions: {p}'.format(p=name)) raise ValueError() self.resources = [PipelineConfigResource(r, self) for r in resources]
def _display_info(dry_run: bool, operation: str, **kwargs): info("Concourse will be {o} using helm with the following arguments".format(o=operation)) max_leng = max(map(len, kwargs.keys())) for k, v in kwargs.items(): key_str = k.ljust(max_leng) info("{k}: {v}".format(k=key_str, v=v)) if dry_run: warning("this was a --dry-run. Set the --no-dry-run flag to actually deploy")
def post_to_slack( release_notes: ReleaseNote, github_repository_name: str, slack_cfg_name: str, slack_channel: str, release_version: str, max_msg_size_bytes: int = 20000, ): # slack can't auto link pull requests, commits or users # hence we force the link generation when building the markdown string release_notes_md_links = release_notes.to_markdown( force_link_generation=True) # XXX slack imposes a maximum msg size # https://api.slack.com/changelog/2018-04-truncating-really-long-messages# slack_cfg = ctx().cfg_factory().slack(slack_cfg_name) slack_helper = SlackHelper(slack_cfg) idx = 0 i = 0 try: while True: title = f'[{github_repository_name}:{release_version} released' # abort on last if idx + max_msg_size_bytes > len(release_notes_md_links): did_split = i > 0 if did_split: title += ' - final]' else: title += ']' msg = release_notes_md_links[idx:] yield slack_helper.post_to_slack(channel=slack_channel, title=title, message=msg) break # post part title += f' - part {i} ]' msg = release_notes_md_links[idx:idx + max_msg_size_bytes] yield slack_helper.post_to_slack(channel=slack_channel, title=title, message=msg) i += 1 idx += max_msg_size_bytes except RuntimeError as e: warning(e)
def deploy(self, definition_descriptor): try: with open(os.path.join(self.base_dir, definition_descriptor.pipeline_name), 'w') as f: f.write(definition_descriptor.pipeline) return DeployResult( definition_descriptor=definition_descriptor, deploy_status=DeployStatus.SUCCEEDED, ) except Exception as e: warning(e) return DeployResult( definition_descriptor=definition_descriptor, deploy_status=DeployStatus.FAILED, )
def _revert(self, steps): # attempt to revert each step. Raise an exception if not all reverts succeeded. all_reverted = True for step in steps: step_name = step.name() info(f"Reverting step {step_name}") try: step.revert() except BaseException as e: all_reverted = False warning(f"An error occured while reverting step '{step_name}': {e}") traceback.print_exc() if not all_reverted: raise RuntimeError("Unable to revert all steps.")
def _post_with_retry(self, client, retries=5, **kwargs): try: response = client.files_upload(**kwargs) return response except slack.errors.SlackApiError as sae: error_code = sae.response.get('error') if retries < 1: raise sae # no retries left (or none requested) if error_code == 'markdown_conversion_failed_because_of_read_failed': warning(f'received {error_code} - retrying {retries}') return self._post_with_retry(client=client, retries=retries - 1, **kwargs) else: raise sae # only retry for known sporadic err
def _resolve_team_members(self, github_team_name: str): not_none(github_team_name) org_name, team_name = github_team_name.split( '/') # always of form 'org/name' organisation = self.github_api.organization(org_name) # unfortunately, we have to look-up the team (no api to retrieve it by name) team_or_none = _first( filter(lambda team: team.name == team_name, organisation.teams())) if not team_or_none: warning('failed to lookup team {t}'.format(t=team_name)) return [] for member in map(self.github_api.user, team_or_none.members()): if member.email: yield member.email else: warning(f'no email found for GitHub user {member}')
def execute(self): executed_steps = list() for step in self._steps: step_name = step.name() info(f"Applying step '{step_name}'") executed_steps.append(step) try: output = step.apply() self._context.set_step_output(step_name, output) except BaseException as e: warning(f"An error occured while applying step '{step_name}': {e}") traceback.print_exc() # revert the changes attempted, in reverse order self._revert(reversed(executed_steps)) # do not execute apply for remaining steps return False return True
def reachable_release_tags_from_commit( self, repo: git.Repo, commit: git.objects.Commit) -> typing.List[str]: '''Returns a list of release-tags whose tagged commits are ancestors of the given commit. The returned list is sorted in descending order, putting the greatest reachable tag first. ''' tags = self.release_tags() visited = set() queue = list() queue.append(commit) visited.add(commit.hexsha) reachable_tags = list() while queue: commit = queue.pop(0) if commit.hexsha in tags: reachable_tags.append(tags[commit.hexsha]) not_visited_parents = _.filter( commit.parents, lambda parent_commit: parent_commit.hexsha not in visited) if not_visited_parents: queue.extend(not_visited_parents) visited |= set( _.map(not_visited_parents, lambda commit: commit.hexsha)) reachable_tags.sort(key=lambda t: version.parse_to_semver(t), reverse=True) if not reachable_tags: warning('no release tag found, falling back to root commit') root_commits = repo.iter_commits(rev=commit, max_parents=0) root_commit = next(root_commits, None) if not root_commit: fail( f'could not determine root commit from rev {commit.hexsha}' ) if next(root_commits, None): fail( 'cannot determine range for release notes. Repository has multiple root ' 'commits. Specify range via commit_range parameter.') reachable_tags.append(root_commit.hexsha) return reachable_tags
def retrieve_scan_result( self, resource: gci.componentmodel.Resource, component: gci.componentmodel.Component, group_id: int=None, ): metadata = self._metadata( resource=resource, component=component, omit_version=True, # omit version when searching for existing app # (only one component version must exist per group by our chosen definition) ) if not group_id: group_id = self._group_id existing_products = self._api.list_apps( group_id=group_id, custom_attribs=metadata ) if len(existing_products) == 0: return None # no result existed yet if len(existing_products) > 1: warning(f"found more than one product for image '{resource.access.imageReference}'") products_to_rm = existing_products[1:] for p in products_to_rm: self._api.delete_product(p.product_id()) info( f'deleted product {p.display_name()} ' f'with product_id: {p.product_id()}' ) # use first (or only) match (we already printed a warning if we found more than one) product = existing_products[0] product_id = product.product_id() # update upload name to reflect new component version (if changed) upload_name = self._upload_name(resource, component) self._update_product_name(product_id, upload_name) # retrieve existing product's details (list of products contained only subset of data) product = self._api.scan_result(product_id=product_id) return product
def resolve_email_addresses(self, codeowners_entries): ''' returns a generator yielding the resolved email addresses for the given iterable of github codeowners entries. ''' for codeowner_entry in codeowners_entries: if '@' not in codeowner_entry: warning(f'invalid codeowners-entry: {codeowner_entry}') continue if not codeowner_entry.startswith('@'): yield codeowner_entry # plain email address elif '/' not in codeowner_entry: email_addr = self._determine_email_address(codeowner_entry[1:]) if email_addr: yield email_addr else: continue else: yield from self._resolve_team_members(codeowner_entry[1:])
def increment(self, method=None, url=None, response=None, error=None, _pool=None, _stacktrace=None): # super().increment will either raise an exception indicating that no retry is to # be performed or return a new, modified instance of this class retry = super().increment(method, url, response, error, _pool, _stacktrace) # Use the Retry history to determine the number of retries. num_retries = len(self.history) if self.history else 0 # Retrieve host from underlying connection pool and host = _pool.host warning( f'HTTP request (host: {host}, url: {url}, method: {method}) unsuccessful. ' f'Retries so far: {num_retries}. Retrying ...') return retry
def deploy(self, definition_descriptor): pipeline_definition = definition_descriptor.pipeline pipeline_name = definition_descriptor.pipeline_name try: api = client.from_cfg( concourse_cfg=definition_descriptor.concourse_target_cfg, team_name=definition_descriptor.concourse_target_team, ) response = api.set_pipeline( name=pipeline_name, pipeline_definition=pipeline_definition ) info( 'Deployed pipeline: ' + pipeline_name + ' to team: ' + definition_descriptor.concourse_target_team ) if self.unpause_pipelines: info(f'Unpausing pipeline {pipeline_name}') api.unpause_pipeline(pipeline_name=pipeline_name) if self.expose_pipelines: api.expose_pipeline(pipeline_name=pipeline_name) deploy_status = DeployStatus.SUCCEEDED if response is concourse.client.model.SetPipelineResult.CREATED: deploy_status |= DeployStatus.CREATED elif response is concourse.client.model.SetPipelineResult.UPDATED: pass else: raise NotImplementedError return DeployResult( definition_descriptor=definition_descriptor, deploy_status=deploy_status, ) except Exception as e: import traceback traceback.print_exc() warning(e) return DeployResult( definition_descriptor=definition_descriptor, deploy_status=DeployStatus.FAILED, error_details=traceback.format_exc(), )
def render(self, definition_descriptor): try: definition_descriptor = self._render(definition_descriptor) info('rendered pipeline {pn}'.format(pn=definition_descriptor.pipeline_name)) return RenderResult( definition_descriptor, render_status=RenderStatus.SUCCEEDED, ) except Exception: warning( f"erroneous pipeline definition '{definition_descriptor.pipeline_name}' " f"in repository '{definition_descriptor.main_repo.get('path')}' on branch " f"'{definition_descriptor.main_repo.get('branch')}'" ) traceback.print_exc() return RenderResult( definition_descriptor, render_status=RenderStatus.FAILED, error_details=traceback.format_exc(), )
def sync_org_webhooks(whd_deployment_cfg: WebhookDispatcherDeploymentConfig,): '''Syncs required organization webhooks for a given webhook dispatcher instance''' for organization_name, github_api, webhook_url in \ _enumerate_required_org_webhooks(whd_deployment_cfg=whd_deployment_cfg): webhook_syncer = github.webhook.GithubWebHookSyncer(github_api) failed_hooks = 0 try: webhook_syncer.create_or_update_org_hook( organization_name=organization_name, webhook_url=webhook_url, skip_ssl_validation=False, ) info(f'Created/updated organization hook for organization "{organization_name}"') except Exception as e: failed_hooks += 1 warning(f'org: {organization_name} - error: {e}') if failed_hooks != 0: warning('Some webhooks could not be set - for more details see above.')
def reachable_release_tags_from_commit(github_helper: GitHubRepositoryHelper, repo: git.Repo, commit: git.objects.Commit) -> [str]: tags = release_tags(github_helper, repo) visited = set() queue = list() queue.append(commit) visited.add(commit.hexsha) reachable_tags = list() while queue: commit = queue.pop(0) if commit.hexsha in tags: reachable_tags.append(tags[commit.hexsha]) not_visited_parents = _.filter( commit.parents, lambda parent_commit: parent_commit.hexsha not in visited) if not_visited_parents: queue.extend(not_visited_parents) visited |= set( _.map(not_visited_parents, lambda commit: commit.hexsha)) reachable_tags.sort(key=lambda t: version.parse_to_semver(t), reverse=True) if not reachable_tags: warning('no release tag found, falling back to root commit') root_commits = repo.iter_commits(rev=commit, max_parents=0) root_commit = next(root_commits, None) if not root_commit: fail('could not determine root commit from rev {rev}'.format( rev=commit.hexsha)) if next(root_commits, None): fail( 'cannot determine range for release notes. Repository has multiple root commits. ' 'Specify range via commit_range parameter.') reachable_tags.append(root_commit.hexsha) return reachable_tags
def resurrect_pods( namespace: str, concourse_client, kubernetes_client, ): ''' concourse pods tend to crash and need to be pruned to help with the self-healing ''' info(f'Check for not running concourse workers') worker_list = concourse_client.list_workers() pruned_workers = list() for worker in worker_list: worker_name = worker.name() info(f'Worker {worker_name}: {worker.state()}') if worker.state() != "running": warning(f'Prune worker {worker_name} and restart pod') pruned_workers.append(worker_name) concourse_client.prune_worker(worker_name) kubernetes_client.pod_helper().delete_pod(name=worker_name, namespace=namespace) return pruned_workers
def _determine_repository_branches( self, repository, ): try: branch_cfg = self._branch_cfg_or_none(repository=repository) except LintingError as e: # some linting errors (and possibly warnings) present. Print warning and continue warning(e) return if not branch_cfg: # fallback for components w/o branch_cfg: use default branch try: default_branch = repository.default_branch except Exception: default_branch = 'master' yield (default_branch, None) return for branch in repository.branches(): cfg_entry = branch_cfg.cfg_entry_for_branch(branch.name) if cfg_entry: yield (branch.name, cfg_entry)
def _cfg_element(self, cfg_type_name: str, cfg_name: str): cfg_type = self._cfg_types().get(cfg_type_name, None) if not cfg_type: raise ValueError('unknown cfg_type: ' + str(cfg_type_name)) # retrieve model class c'tor - search module and sub-modules # TODO: switch to fully-qualified type names own_module = sys.modules[__name__] # python3.5 returns a three-tuple; python3.6+ returns a ModuleInfo if sys.version_info.minor <= 5: class ModuleInfo(object): def __init__(self, module_tuple): self.path, self.name, _ = module_tuple def to_module_info(mi): return ModuleInfo(mi) else: def to_module_info(mi): return mi submodule_names = [ own_module.__name__ + '.' + m.name for m in map(to_module_info, pkgutil.iter_modules(own_module.__path__)) ] for module_name in [__name__] + submodule_names: submodule_name = module_name.split('.')[-1] if module_name != __name__: module = getattr(__import__(module_name), submodule_name) else: module = sys.modules[submodule_name] # skip if module does not define our type if not hasattr(module, cfg_type.cfg_type()): continue # if type is defined, validate element_type = getattr(module, cfg_type.cfg_type()) if not type(element_type) == type: raise ValueError() # found it break else: raise ValueError('failed to find cfg type: ' + str(cfg_type.cfg_type())) # for now, let's assume all of our model element types are subtypes of NamedModelElement # (with the exception of ConfigurationSet) configs = self._configs(cfg_type.cfg_type_name()) if cfg_name not in configs: raise ConfigElementNotFoundError('no such cfg element: {cn}. Known: {es}'.format( cn=cfg_name, es=', '.join(configs.keys()) ) ) kwargs = {'raw_dict': configs[cfg_name]} if element_type == ConfigurationSet: kwargs.update({'cfg_name': cfg_name, 'cfg_factory': self}) else: kwargs['name'] = cfg_name element_instance = element_type(**kwargs) try: element_instance.validate() except ModelValidationError as mve: warning(f'validation error for {cfg_name} - ignored: {mve}') return element_instance