def cfg_entry_for_branch(self, branch):
matching_entries = [
entry for entry in self.cfg_entries()
if entry.branch_matches(branch)
]
if not matching_entries:
return None
# merge entries
effective_branch_cfg = {}
# order by optional attribute 'index' (random order if omitted)
def entry_with_idx(idx, entry):
if entry.index():
idx = int(entry.index())
return (idx, entry)
indexed_entries = [
entry_with_idx(idx, entry)
for idx, entry in enumerate(matching_entries)
]
for _, entry in sorted(indexed_entries, key=lambda i: i[0]):
effective_branch_cfg = merge_dicts(effective_branch_cfg, entry.raw)
return BranchCfgEntry(name='merged', raw_dict=effective_branch_cfg)
def _render(self, definition_descriptor):
effective_definition = definition_descriptor.pipeline_definition
# handle inheritance
for override in definition_descriptor.override_definitions:
effective_definition = merge_dicts(effective_definition, override)
template_name = definition_descriptor.template_name()
template_contents = self.template_retriever.template_contents(
template_name)
pipeline_name = definition_descriptor.pipeline_name
pipeline_definition = RawPipelineDefinitionDescriptor(
name=pipeline_name,
base_definition=effective_definition.get('base_definition', {}),
variants=effective_definition.get('jobs', {}),
template=template_name,
)
factory = DefinitionFactory(
raw_definition_descriptor=pipeline_definition,
cfg_set=self.cfg_set,
)
pipeline_metadata = dict()
pipeline_metadata['definition'] = factory.create_pipeline_definition()
pipeline_metadata['name'] = pipeline_definition.name
pipeline_metadata[
'target_team'] = definition_descriptor.concourse_target_team
generated_model = pipeline_metadata.get('definition')
if bg := effective_definition.get('background_image'):
pipeline_metadata['background_image'] = bg
def test_merge_dicts_with_merge_retains_order(self):
left = {1: [3, 1, 0]}
right = {1: [1, 2, 4]}
merged = examinee.merge_dicts(left, right, list_semantics='merge')
self.assertEqual(
merged,
{1: [3, 1, 0, 2, 4]},
)
def _scan_repository_for_definitions(
self,
repository,
github_cfg,
org_name,
) -> RawPipelineDefinitionDescriptor:
for branch_name, cfg_entry in self._determine_repository_branches(
repository=repository):
try:
definitions = repository.file_contents(
path='.ci/pipeline_definitions', ref=branch_name)
except NotFoundError:
continue # no pipeline definition for this branch
repo_hostname = urlparse(github_cfg.http_url()).hostname
override_definitions = cfg_entry.override_definitions(
) if cfg_entry else {}
verbose('from repo: ' + repository.name + ':' + branch_name)
try:
decoded_definitions = definitions.decoded.decode('utf-8')
info(
f'Linting pipeline_definitions for {repository} on branch {branch_name}'
)
lint_yaml(decoded_definitions)
definitions = load_yaml(decoded_definitions)
except BaseException as e:
repo_path = f'{org_name}/{repository.name}'
yield DefinitionDescriptor(
pipeline_name='<invalid YAML>',
pipeline_definition={},
main_repo={
'path': repo_path,
'branch': branch_name,
'hostname': repo_hostname
},
concourse_target_cfg=self.cfg_set.concourse(),
concourse_target_team=self.job_mapping.team_name(),
override_definitions=(),
exception=e,
)
return # nothing else to yield in case parsing failed
# handle inheritance
definitions = merge_dicts(definitions, override_definitions)
yield from self._wrap_into_descriptors(
repo_path='/'.join([org_name, repository.name]),
repo_hostname=repo_hostname,
branch=branch_name,
raw_definitions=definitions,
override_definitions=override_definitions,
)
def _render(self, definition_descriptor):
effective_definition = definition_descriptor.pipeline_definition
# handle inheritance
for override in definition_descriptor.override_definitions:
effective_definition = merge_dicts(effective_definition, override)
template_name = definition_descriptor.template_name()
template_contents = self.template_retriever.template_contents(
template_name)
pipeline_name = definition_descriptor.pipeline_name
pipeline_definition = RawPipelineDefinitionDescriptor(
name=pipeline_name,
base_definition=effective_definition.get('base_definition', {}),
variants=effective_definition.get('jobs', {}),
template=template_name,
)
factory = DefinitionFactory(
raw_definition_descriptor=pipeline_definition)
pipeline_metadata = dict()
pipeline_metadata['definition'] = factory.create_pipeline_definition()
pipeline_metadata['name'] = pipeline_definition.name
pipeline_metadata[
'target_team'] = definition_descriptor.concourse_target_team
generated_model = pipeline_metadata.get('definition')
# determine pipeline name (if there is main-repo, append the configured branch name)
for variant in pipeline_metadata.get('definition').variants():
# hack: take the first "main_repository" we find
if not variant.has_main_repository():
continue
main_repo = variant.main_repository()
pipeline_metadata['pipeline_name'] = '-'.join(
[pipeline_definition.name,
main_repo.branch()])
break
else:
# fallback in case no main_repository was found
pipeline_metadata['pipeline_name'] = pipeline_definition.name
main_repo = None
t = mako.template.Template(template_contents, lookup=self.lookup)
definition_descriptor.pipeline = t.render(
instance_args=generated_model,
config_set=self.cfg_set,
pipeline=pipeline_metadata,
)
return definition_descriptor
def test_merge_dicts_simple(self):
left = {1: {2: 3}}
right = {1: {4: 5}, 6: 7}
merged = examinee.merge_dicts(left, right)
self.assertEqual(merged, {
1: {
2: 3,
4: 5
},
6: 7,
})
def test_merge_dicts_three_way_merge(self):
first = {1: [3, 1, 0]}
second = {1: [1, 2, 4]}
third = {1: [1, 2, 5], 2: [1, 2, 3]}
merged = examinee.merge_dicts(first,
second,
third,
list_semantics='merge')
self.assertEqual(merged, {
1: [3, 1, 0, 2, 4, 5],
2: [1, 2, 3],
})
def _create_variants_dict(self, raw_definition_descriptor):
variants_dict = normalise_to_dict(deepcopy(raw_definition_descriptor.jobs))
base_dict = deepcopy(raw_definition_descriptor.base_definition)
merged_variants = {}
for variant_name, variant_args in variants_dict.items():
# optimisation: if there are no variant-specific arguments, we do not need to merge
if variant_args:
merged_variants[variant_name] = merge_dicts(base_dict, variant_args)
else:
merged_variants[variant_name] = deepcopy(base_dict)
return merged_variants
def _render(self, definition_descriptor):
effective_definition = definition_descriptor.pipeline_definition
# handle inheritance
for override in definition_descriptor.override_definitions:
effective_definition = merge_dicts(effective_definition, override)
template_name = definition_descriptor.template_name()
template_contents = self.template_retriever.template_contents(
template_name)
pipeline_name = definition_descriptor.pipeline_name
pipeline_definition = RawPipelineDefinitionDescriptor(
name=pipeline_name,
base_definition=effective_definition.get('base_definition', {}),
jobs=effective_definition.get('jobs', {}),
template=template_name,
)
factory = DefinitionFactory(
raw_definition_descriptor=pipeline_definition,
cfg_set=self.cfg_set,
)
pipeline_metadata = {
'definition':
factory.create_pipeline_definition(),
'name':
pipeline_definition.name,
'target_team':
definition_descriptor.concourse_target_team,
'secret_cfg':
definition_descriptor.secret_cfg,
'job_mapping':
definition_descriptor.job_mapping,
'render_origin':
self.render_origin.value,
'cc_utils_version':
_cc_utils_version(),
'pipeline_definition_committish':
definition_descriptor.pipeline_definition_committish,
}
# also pass pipeline name if this was rendered by a replication job. Will be printed
# in the meta-step later
if (self.render_origin is RenderOrigin.PIPELINE_REPLICATION
and (pipeline_name := os.environ.get('PIPELINE_NAME'))):
pipeline_metadata['replication_pipeline_name'] = pipeline_name
def test_merge_dicts_does_not_modify_args(self):
from copy import deepcopy
first = {1: {2: 3}}
second = {1: {4: 5}, 6: 7}
first_arg = deepcopy(first)
second_arg = deepcopy(second)
merged = examinee.merge_dicts(first_arg, second_arg)
self.assertEqual(merged, {
1: {
2: 3,
4: 5
},
6: 7,
})
self.assertEqual(first, first_arg)
self.assertEqual(second, second_arg)
def _scan_repository_for_definitions(
self,
repository,
github_cfg,
org_name,
job_mapping: model.concourse.JobMapping=None,
target_team: str=None,
secret_cfg=None,
branch: str=None,
) -> RawPipelineDefinitionDescriptor:
repo_hostname = urlparse(github_cfg.http_url()).hostname
repo_path = f'{org_name}/{repository.name}'
if not target_team:
target_team = self.job_mapping.team_name()
try:
branches_and_cfg_entries = [
i for i in self._determine_repository_branches(
repository=repository,
branch=branch,
)
]
except (yaml.scanner.ScannerError, yaml.parser.ParserError) as e:
yield DefinitionDescriptor(
pipeline_name='<invalid YAML>',
pipeline_definition=None,
main_repo={'path': repo_path, 'branch': 'refs/meta/ci', 'hostname': repo_hostname},
concourse_target_cfg=self.cfg_set.concourse(),
concourse_target_team=target_team,
override_definitions=(),
exception=e,
secret_cfg=secret_cfg,
)
return # nothing else to yield in case parsing the branch cfg failed
for branch_name, cfg_entry in branches_and_cfg_entries:
try:
definitions = repository.file_contents(
path='.ci/pipeline_definitions',
ref=branch_name
)
pipeline_definition_committish = repository.ref(
ref=f'heads/{branch_name}'
).object.sha
except NotFoundError:
continue # no pipeline definition for this branch
override_definitions = cfg_entry.override_definitions() if cfg_entry else {}
try:
decoded_definitions = definitions.decoded.decode('utf-8')
lint_yaml(decoded_definitions)
definitions = load_yaml(decoded_definitions)
except BaseException as e:
yield DefinitionDescriptor(
pipeline_name='<invalid YAML>',
pipeline_definition={},
main_repo={'path': repo_path, 'branch': branch_name, 'hostname': repo_hostname},
concourse_target_cfg=self.cfg_set.concourse(),
concourse_target_team=target_team,
override_definitions=(),
exception=e,
secret_cfg=secret_cfg,
)
return # nothing else to yield in case parsing failed
# handle inheritance
definitions = merge_dicts(definitions, override_definitions)
# hacky: only set from GithubRepositoryDefinitionEnumerator
target_team = getattr(self, '_target_team', None)
yield from self._wrap_into_descriptors(
repo_path='/'.join([org_name, repository.name]),
repo_hostname=repo_hostname,
branch=branch_name,
raw_definitions=definitions,
override_definitions=override_definitions,
target_team=target_team,
secret_cfg=secret_cfg,
job_mapping=job_mapping,
pipeline_definition_committish=pipeline_definition_committish,
)
