def cfg_entry_for_branch(self, branch): matching_entries = [ entry for entry in self.cfg_entries() if entry.branch_matches(branch) ] if not matching_entries: return None # merge entries effective_branch_cfg = {} # order by optional attribute 'index' (random order if omitted) def entry_with_idx(idx, entry): if entry.index(): idx = int(entry.index()) return (idx, entry) indexed_entries = [ entry_with_idx(idx, entry) for idx, entry in enumerate(matching_entries) ] for _, entry in sorted(indexed_entries, key=lambda i: i[0]): effective_branch_cfg = merge_dicts(effective_branch_cfg, entry.raw) return BranchCfgEntry(name='merged', raw_dict=effective_branch_cfg)
def _render(self, definition_descriptor): effective_definition = definition_descriptor.pipeline_definition # handle inheritance for override in definition_descriptor.override_definitions: effective_definition = merge_dicts(effective_definition, override) template_name = definition_descriptor.template_name() template_contents = self.template_retriever.template_contents( template_name) pipeline_name = definition_descriptor.pipeline_name pipeline_definition = RawPipelineDefinitionDescriptor( name=pipeline_name, base_definition=effective_definition.get('base_definition', {}), variants=effective_definition.get('jobs', {}), template=template_name, ) factory = DefinitionFactory( raw_definition_descriptor=pipeline_definition, cfg_set=self.cfg_set, ) pipeline_metadata = dict() pipeline_metadata['definition'] = factory.create_pipeline_definition() pipeline_metadata['name'] = pipeline_definition.name pipeline_metadata[ 'target_team'] = definition_descriptor.concourse_target_team generated_model = pipeline_metadata.get('definition') if bg := effective_definition.get('background_image'): pipeline_metadata['background_image'] = bg
def test_merge_dicts_with_merge_retains_order(self): left = {1: [3, 1, 0]} right = {1: [1, 2, 4]} merged = examinee.merge_dicts(left, right, list_semantics='merge') self.assertEqual( merged, {1: [3, 1, 0, 2, 4]}, )
def _scan_repository_for_definitions( self, repository, github_cfg, org_name, ) -> RawPipelineDefinitionDescriptor: for branch_name, cfg_entry in self._determine_repository_branches( repository=repository): try: definitions = repository.file_contents( path='.ci/pipeline_definitions', ref=branch_name) except NotFoundError: continue # no pipeline definition for this branch repo_hostname = urlparse(github_cfg.http_url()).hostname override_definitions = cfg_entry.override_definitions( ) if cfg_entry else {} verbose('from repo: ' + repository.name + ':' + branch_name) try: decoded_definitions = definitions.decoded.decode('utf-8') info( f'Linting pipeline_definitions for {repository} on branch {branch_name}' ) lint_yaml(decoded_definitions) definitions = load_yaml(decoded_definitions) except BaseException as e: repo_path = f'{org_name}/{repository.name}' yield DefinitionDescriptor( pipeline_name='<invalid YAML>', pipeline_definition={}, main_repo={ 'path': repo_path, 'branch': branch_name, 'hostname': repo_hostname }, concourse_target_cfg=self.cfg_set.concourse(), concourse_target_team=self.job_mapping.team_name(), override_definitions=(), exception=e, ) return # nothing else to yield in case parsing failed # handle inheritance definitions = merge_dicts(definitions, override_definitions) yield from self._wrap_into_descriptors( repo_path='/'.join([org_name, repository.name]), repo_hostname=repo_hostname, branch=branch_name, raw_definitions=definitions, override_definitions=override_definitions, )
def _render(self, definition_descriptor): effective_definition = definition_descriptor.pipeline_definition # handle inheritance for override in definition_descriptor.override_definitions: effective_definition = merge_dicts(effective_definition, override) template_name = definition_descriptor.template_name() template_contents = self.template_retriever.template_contents( template_name) pipeline_name = definition_descriptor.pipeline_name pipeline_definition = RawPipelineDefinitionDescriptor( name=pipeline_name, base_definition=effective_definition.get('base_definition', {}), variants=effective_definition.get('jobs', {}), template=template_name, ) factory = DefinitionFactory( raw_definition_descriptor=pipeline_definition) pipeline_metadata = dict() pipeline_metadata['definition'] = factory.create_pipeline_definition() pipeline_metadata['name'] = pipeline_definition.name pipeline_metadata[ 'target_team'] = definition_descriptor.concourse_target_team generated_model = pipeline_metadata.get('definition') # determine pipeline name (if there is main-repo, append the configured branch name) for variant in pipeline_metadata.get('definition').variants(): # hack: take the first "main_repository" we find if not variant.has_main_repository(): continue main_repo = variant.main_repository() pipeline_metadata['pipeline_name'] = '-'.join( [pipeline_definition.name, main_repo.branch()]) break else: # fallback in case no main_repository was found pipeline_metadata['pipeline_name'] = pipeline_definition.name main_repo = None t = mako.template.Template(template_contents, lookup=self.lookup) definition_descriptor.pipeline = t.render( instance_args=generated_model, config_set=self.cfg_set, pipeline=pipeline_metadata, ) return definition_descriptor
def test_merge_dicts_simple(self): left = {1: {2: 3}} right = {1: {4: 5}, 6: 7} merged = examinee.merge_dicts(left, right) self.assertEqual(merged, { 1: { 2: 3, 4: 5 }, 6: 7, })
def test_merge_dicts_three_way_merge(self): first = {1: [3, 1, 0]} second = {1: [1, 2, 4]} third = {1: [1, 2, 5], 2: [1, 2, 3]} merged = examinee.merge_dicts(first, second, third, list_semantics='merge') self.assertEqual(merged, { 1: [3, 1, 0, 2, 4, 5], 2: [1, 2, 3], })
def _create_variants_dict(self, raw_definition_descriptor): variants_dict = normalise_to_dict(deepcopy(raw_definition_descriptor.jobs)) base_dict = deepcopy(raw_definition_descriptor.base_definition) merged_variants = {} for variant_name, variant_args in variants_dict.items(): # optimisation: if there are no variant-specific arguments, we do not need to merge if variant_args: merged_variants[variant_name] = merge_dicts(base_dict, variant_args) else: merged_variants[variant_name] = deepcopy(base_dict) return merged_variants
def _render(self, definition_descriptor): effective_definition = definition_descriptor.pipeline_definition # handle inheritance for override in definition_descriptor.override_definitions: effective_definition = merge_dicts(effective_definition, override) template_name = definition_descriptor.template_name() template_contents = self.template_retriever.template_contents( template_name) pipeline_name = definition_descriptor.pipeline_name pipeline_definition = RawPipelineDefinitionDescriptor( name=pipeline_name, base_definition=effective_definition.get('base_definition', {}), jobs=effective_definition.get('jobs', {}), template=template_name, ) factory = DefinitionFactory( raw_definition_descriptor=pipeline_definition, cfg_set=self.cfg_set, ) pipeline_metadata = { 'definition': factory.create_pipeline_definition(), 'name': pipeline_definition.name, 'target_team': definition_descriptor.concourse_target_team, 'secret_cfg': definition_descriptor.secret_cfg, 'job_mapping': definition_descriptor.job_mapping, 'render_origin': self.render_origin.value, 'cc_utils_version': _cc_utils_version(), 'pipeline_definition_committish': definition_descriptor.pipeline_definition_committish, } # also pass pipeline name if this was rendered by a replication job. Will be printed # in the meta-step later if (self.render_origin is RenderOrigin.PIPELINE_REPLICATION and (pipeline_name := os.environ.get('PIPELINE_NAME'))): pipeline_metadata['replication_pipeline_name'] = pipeline_name
def test_merge_dicts_does_not_modify_args(self): from copy import deepcopy first = {1: {2: 3}} second = {1: {4: 5}, 6: 7} first_arg = deepcopy(first) second_arg = deepcopy(second) merged = examinee.merge_dicts(first_arg, second_arg) self.assertEqual(merged, { 1: { 2: 3, 4: 5 }, 6: 7, }) self.assertEqual(first, first_arg) self.assertEqual(second, second_arg)
def _scan_repository_for_definitions( self, repository, github_cfg, org_name, job_mapping: model.concourse.JobMapping=None, target_team: str=None, secret_cfg=None, branch: str=None, ) -> RawPipelineDefinitionDescriptor: repo_hostname = urlparse(github_cfg.http_url()).hostname repo_path = f'{org_name}/{repository.name}' if not target_team: target_team = self.job_mapping.team_name() try: branches_and_cfg_entries = [ i for i in self._determine_repository_branches( repository=repository, branch=branch, ) ] except (yaml.scanner.ScannerError, yaml.parser.ParserError) as e: yield DefinitionDescriptor( pipeline_name='<invalid YAML>', pipeline_definition=None, main_repo={'path': repo_path, 'branch': 'refs/meta/ci', 'hostname': repo_hostname}, concourse_target_cfg=self.cfg_set.concourse(), concourse_target_team=target_team, override_definitions=(), exception=e, secret_cfg=secret_cfg, ) return # nothing else to yield in case parsing the branch cfg failed for branch_name, cfg_entry in branches_and_cfg_entries: try: definitions = repository.file_contents( path='.ci/pipeline_definitions', ref=branch_name ) pipeline_definition_committish = repository.ref( ref=f'heads/{branch_name}' ).object.sha except NotFoundError: continue # no pipeline definition for this branch override_definitions = cfg_entry.override_definitions() if cfg_entry else {} try: decoded_definitions = definitions.decoded.decode('utf-8') lint_yaml(decoded_definitions) definitions = load_yaml(decoded_definitions) except BaseException as e: yield DefinitionDescriptor( pipeline_name='<invalid YAML>', pipeline_definition={}, main_repo={'path': repo_path, 'branch': branch_name, 'hostname': repo_hostname}, concourse_target_cfg=self.cfg_set.concourse(), concourse_target_team=target_team, override_definitions=(), exception=e, secret_cfg=secret_cfg, ) return # nothing else to yield in case parsing failed # handle inheritance definitions = merge_dicts(definitions, override_definitions) # hacky: only set from GithubRepositoryDefinitionEnumerator target_team = getattr(self, '_target_team', None) yield from self._wrap_into_descriptors( repo_path='/'.join([org_name, repository.name]), repo_hostname=repo_hostname, branch=branch_name, raw_definitions=definitions, override_definitions=override_definitions, target_team=target_team, secret_cfg=secret_cfg, job_mapping=job_mapping, pipeline_definition_committish=pipeline_definition_committish, )