def _scan_repository_for_definitions(
self,
repository,
github_cfg,
org_name,
) -> RawPipelineDefinitionDescriptor:
for branch_name, cfg_entry in self._determine_repository_branches(
repository=repository):
try:
definitions = repository.file_contents(
path='.ci/pipeline_definitions', ref=branch_name)
except NotFoundError:
continue # no pipeline definition for this branch
repo_hostname = urlparse(github_cfg.http_url()).hostname
override_definitions = cfg_entry.override_definitions(
) if cfg_entry else {}
verbose('from repo: ' + repository.name + ':' + branch_name)
try:
decoded_definitions = definitions.decoded.decode('utf-8')
info(
f'Linting pipeline_definitions for {repository} on branch {branch_name}'
)
lint_yaml(decoded_definitions)
definitions = load_yaml(decoded_definitions)
except BaseException as e:
repo_path = f'{org_name}/{repository.name}'
yield DefinitionDescriptor(
pipeline_name='<invalid YAML>',
pipeline_definition={},
main_repo={
'path': repo_path,
'branch': branch_name,
'hostname': repo_hostname
},
concourse_target_cfg=self.cfg_set.concourse(),
concourse_target_team=self.job_mapping.team_name(),
override_definitions=(),
exception=e,
)
return # nothing else to yield in case parsing failed
# handle inheritance
definitions = merge_dicts(definitions, override_definitions)
yield from self._wrap_into_descriptors(
repo_path='/'.join([org_name, repository.name]),
repo_hostname=repo_hostname,
branch=branch_name,
raw_definitions=definitions,
override_definitions=override_definitions,
)
def _branch_cfg_or_none(
self,
repository,
):
try:
branch_cfg = repository.file_contents(
path='branch.cfg',
ref='refs/meta/ci',
).decoded.decode('utf-8')
return BranchCfg(raw_dict=load_yaml(branch_cfg))
except NotFoundError:
return None # no branch cfg present
def _scan_repository_for_definitions(
self,
repository,
github_cfg,
org_name,
job_mapping: model.concourse.JobMapping=None,
target_team: str=None,
secret_cfg=None,
branch: str=None,
) -> RawPipelineDefinitionDescriptor:
repo_hostname = urlparse(github_cfg.http_url()).hostname
repo_path = f'{org_name}/{repository.name}'
if not target_team:
target_team = self.job_mapping.team_name()
try:
branches_and_cfg_entries = [
i for i in self._determine_repository_branches(
repository=repository,
branch=branch,
)
]
except (yaml.scanner.ScannerError, yaml.parser.ParserError) as e:
yield DefinitionDescriptor(
pipeline_name='<invalid YAML>',
pipeline_definition=None,
main_repo={'path': repo_path, 'branch': 'refs/meta/ci', 'hostname': repo_hostname},
concourse_target_cfg=self.cfg_set.concourse(),
concourse_target_team=target_team,
override_definitions=(),
exception=e,
secret_cfg=secret_cfg,
)
return # nothing else to yield in case parsing the branch cfg failed
for branch_name, cfg_entry in branches_and_cfg_entries:
try:
definitions = repository.file_contents(
path='.ci/pipeline_definitions',
ref=branch_name
)
pipeline_definition_committish = repository.ref(
ref=f'heads/{branch_name}'
).object.sha
except NotFoundError:
continue # no pipeline definition for this branch
override_definitions = cfg_entry.override_definitions() if cfg_entry else {}
try:
decoded_definitions = definitions.decoded.decode('utf-8')
lint_yaml(decoded_definitions)
definitions = load_yaml(decoded_definitions)
except BaseException as e:
yield DefinitionDescriptor(
pipeline_name='<invalid YAML>',
pipeline_definition={},
main_repo={'path': repo_path, 'branch': branch_name, 'hostname': repo_hostname},
concourse_target_cfg=self.cfg_set.concourse(),
concourse_target_team=target_team,
override_definitions=(),
exception=e,
secret_cfg=secret_cfg,
)
return # nothing else to yield in case parsing failed
# handle inheritance
definitions = merge_dicts(definitions, override_definitions)
# hacky: only set from GithubRepositoryDefinitionEnumerator
target_team = getattr(self, '_target_team', None)
yield from self._wrap_into_descriptors(
repo_path='/'.join([org_name, repository.name]),
repo_hostname=repo_hostname,
branch=branch_name,
raw_definitions=definitions,
override_definitions=override_definitions,
target_team=target_team,
secret_cfg=secret_cfg,
job_mapping=job_mapping,
pipeline_definition_committish=pipeline_definition_committish,
)