def test_urljoin(self):
self.assertEqual('foo/bar', examinee.urljoin('foo/bar'))
# leading/trailing slashes should be preserved
self.assertEqual('//foo/bar//', examinee.urljoin('//foo/bar//'))
self.assertEqual('xxx://foo.bar/abc/def/',
examinee.urljoin('xxx://foo.bar', 'abc', 'def/'))
# leading/trailing slashes for "inner" parts should be slurped
self.assertEqual('gnu://foo.bar/abc/def/',
examinee.urljoin('gnu://foo.bar/', '/abc/', '/def/'))
def retrieve_secrets(self):
if self.cache_file and os.path.isfile(self.cache_file):
with open(self.cache_file, 'rb') as f:
if self.secret:
raw_data = _decrypt_cipher_text(
encrypted_cipher_text=f.read(),
secret=self.secret,
)
return json.loads(raw_data)
else:
return json.load(f)
request_url = urljoin(self.url, self.concourse_secret_name)
response = requests.get(request_url)
# pylint: disable=no-member
if not response.status_code == requests.codes.ok:
# pylint: enable=no-member
raise RuntimeError('secrets_server sent {d}: {m}'.format(
d=response.status_code, m=response.content))
if self.cache_file:
with open(self.cache_file, 'wb') as f:
if self.secret:
f.write(response.content)
else:
json.dump(response.json(), f)
if self.secret:
raw_data = _decrypt_cipher_text(
encrypted_cipher_text=response.content,
secret=self.secret,
)
return json.loads(raw_data)
else:
return response.json()
def from_github_repo_url(repo_url):
parsed = urllib.parse.urlparse(repo_url)
if parsed.scheme:
component_name = repo_url = urljoin(*parsed[1:3])
else:
component_name = repo_url
return ComponentName(name=component_name)
def apps(self, group_id, custom_attribs={}):
url = self._api_url('apps')
if group_id:
url = urljoin(url, str(group_id))
search_query = ' '.join(['meta:' + str(k) + '=' + str(v) for k,v in custom_attribs.items()])
if search_query:
url += '?' + urlencode({'q': search_query})
return url
def retrieve_secrets(self):
if self.cache_file and os.path.isfile(self.cache_file):
with open(self.cache_file) as f:
return json.load(f)
request_url = urljoin(self.url, self.concourse_secret_name)
response = requests.get(request_url)
# pylint: disable=no-member
if not response.status_code == requests.codes.ok:
# pylint: enable=no-member
raise RuntimeError('secrets_server sent {d}: {m}'.format(
d=response.status_code, m=response.content))
if self.cache_file:
with open(self.cache_file, 'w') as f:
json.dump(response.json(), f)
return response.json()
def clone_into(
target_directory: str,
github_cfg: GithubConfig,
github_repo_path: str,
checkout_branch: str = None,
) -> 'GitHelper':
protocol = github_cfg.preferred_protocol()
if protocol is Protocol.SSH:
cmd_env, tmp_id = _ssh_auth_env(github_cfg=github_cfg)
url = urljoin(github_cfg.ssh_url(), github_repo_path)
elif protocol is Protocol.HTTPS:
url = url_with_credentials(github_cfg, github_repo_path)
else:
raise NotImplementedError
args = ['--quiet']
if checkout_branch is not None:
args += ['--branch', checkout_branch, '--single-branch']
args += [url, target_directory]
repo = git.Git()
if protocol is Protocol.SSH:
with repo.custom_environment(**cmd_env):
repo.clone(*args)
else:
repo.clone(*args)
if protocol is Protocol.SSH:
os.unlink(tmp_id.name)
return GitHelper(
repo=target_directory,
github_cfg=github_cfg,
github_repo_path=github_repo_path,
)
def _authenticated_remote(self):
protocol = self.github_cfg.preferred_protocol()
if protocol is Protocol.SSH:
url = urljoin(self.github_cfg.ssh_url(), self.github_repo_path)
cmd_env, tmp_id = _ssh_auth_env(github_cfg=self.github_cfg)
elif protocol is Protocol.HTTPS:
url = url_with_credentials(self.github_cfg, self.github_repo_path)
cmd_env = os.environ
else:
raise NotImplementedError
remote = git.remote.Remote.add(
repo=self.repo,
name=random_str(),
url=url,
)
logger.info(f'autenticated {remote.name=} using {protocol=}')
try:
yield (cmd_env, remote)
finally:
self.repo.delete_remote(remote)
if protocol is Protocol.SSH:
os.unlink(tmp_id.name)
def _notify_broken_definition_owners(self, failed_descriptor):
definition_descriptor = failed_descriptor.definition_descriptor
main_repo = definition_descriptor.main_repo
repo_owner, repo_name = main_repo['path'].split('/')
repo_url = urljoin(main_repo['hostname'], repo_owner, repo_name)
github_cfg = ccc.github.github_cfg_for_repo_url(
repo_url, self._cfg_set)
github_api = ccc.github.github_api(github_cfg)
repo_helper = ccc.github.github_repo_helper(
host=main_repo['hostname'],
org=repo_owner,
repo=repo_name,
branch=main_repo['branch'],
)
recipients = set(
github.codeowners.resolve_email_addresses(
codeowners_entries=github.codeowners.
enumerate_codeowners_from_remote_repo(
repo=repo_helper.repository),
github_api=github_api,
))
# in case no codeowners are available, resort to using the committer
if not recipients:
head_commit = repo_helper.repository.commit(main_repo['branch'])
user_ids = {
user_info.get('login')
for user_info in (head_commit.committer, head_commit.author)
if user_info and user_info.get('login')
}
for user_id in user_ids:
user = github_api.user(user_id)
if user.email:
recipients.add(user.email)
# if there are still no recipients available print a warning
if not recipients:
logger.warning(
textwrap.dedent(f"""
Unable to determine recipient for pipeline '{definition_descriptor.pipeline_name}'
found in branch '{main_repo['branch']}' ({main_repo['path']}). Please make sure that
CODEOWNERS and committers have exposed a public e-mail address in their profile.
"""))
else:
logger.info(
f'Sending notification e-mail to {recipients} ({main_repo["path"]})'
)
email_cfg = self._cfg_set.email("ses_gardener_cloud_sap")
_send_mail(
email_cfg=email_cfg,
recipients=recipients,
subject='Your pipeline definition in {repo} is erroneous'.
format(repo=main_repo['path'], ),
mail_template=textwrap.dedent(f'''
The pipeline definition for {definition_descriptor.pipeline_name=}
on {main_repo["branch"]=} failed to be rendered.
Error details:
{str(failed_descriptor.error_details)}
'''),
)
def _url(self, *parts):
return urljoin(self._base_url, *parts)
def remote_settings_git(self, project_id: int):
return urljoin(self.scan_by_id(project_id), 'sourceCode',
'remoteSettings', 'git')
def upload_zipped_source(self, project_id: int):
return urljoin(str(self.project_by_id(project_id)), 'sourceCode',
'attachments')
def scan_statistics(self, scan_id: int):
return urljoin(self.scan(), str(scan_id), 'resultsStatistics')
def scan_by_id(self, scan_id: int):
return urljoin(self.scan(), str(scan_id))
def project_by_id(self, project_id: int):
return urljoin(self.projects(), str(project_id))
def _api_url(self, *parts, **kwargs):
return urljoin(self.base_url, 'cxrestapi', *parts)
def _web_url(self, *parts):
return urljoin(self.base_url, 'CxWebClient', *parts)
def web_ui_scan_history(self, scan_id: int):
query = urllib.parse.urlencode({'id': scan_id, 'ProjectState': 'true'})
return urljoin(self._web_url(), 'projectscans.aspx?' + query)
def web_ui_scan_viewer(self, scan_id: int, project_id: int):
query = urllib.parse.urlencode({'scanId': scan_id, 'ProjectID': project_id})
return urljoin(self._web_url(), 'ViewerMain.aspx?' + query)
def upload_image_ref(image_reference):
image_name, tag = image_reference.rsplit(':', 1)
mangled_reference = ':'.join((image_name.replace('.', '_'), tag))
return urljoin(upload_registry_prefix, mangled_reference)
