def create_ingress(self, namespace: str, ingress: V1beta1Ingress):
'''Create an ingress in a given namespace. Raises an `ApiException` if such an ingress
already exists.'''
not_empty(namespace)
not_none(ingress)
self.extensions_v1beta1_api.create_namespaced_ingress(namespace=namespace, body=ingress)
def __init__(
self,
repository_url: str,
cfg_set,
target_team: str = None,
):
if not '://' in repository_url:
repository_url = 'x://' + repository_url
self._repository_url = urlparse(not_none(repository_url))
self._repo_host = self._repository_url.hostname
self.cfg_set = not_none(cfg_set)
self._target_team = target_team
concourse_cfg = cfg_set.concourse()
job_mapping_set = cfg_set.job_mapping(
concourse_cfg.job_mapping_cfg_name())
org_name, repo_name = self._repository_url.path.lstrip('/').split('/')
if target_team:
return # if tgt-team is explicitly configured, we do not need to look it up
for job_mapping in job_mapping_set.job_mappings().values():
for org in job_mapping.github_organisations():
if org.org_name() != org_name:
continue
if not org.repository_matches(repo_name):
continue
github_cfg = cfg_set.github(org.github_cfg_name())
if github_cfg.matches_hostname(self._repo_host):
self.job_mapping = job_mapping
return
else:
raise JobMappingNotFoundError(
f'could not find matching job-mapping for org {org}')
def __init__(
self,
pipeline_name,
pipeline_definition,
main_repo,
concourse_target_cfg: model.concourse.ConcourseConfig,
concourse_target_team,
secret_cfg,
job_mapping: model.concourse.JobMapping=None,
override_definitions=[{},],
exception=None,
pipeline_definition_committish: str=None,
):
try:
self.pipeline_name = not_empty(pipeline_name)
self.pipeline_definition = not_none(pipeline_definition)
self.main_repo = not_none(main_repo)
self.concourse_target_cfg = not_none(concourse_target_cfg)
self.concourse_target_team = not_none(concourse_target_team)
self.override_definitions = not_none(override_definitions)
self.secret_cfg = secret_cfg
self.job_mapping = job_mapping
self.pipeline_definition_committish = pipeline_definition_committish
except Exception as e:
raise ValueError(
f'{e=} missing value: {pipeline_name=} {pipeline_definition=} {main_repo=} '
f'{concourse_target_cfg=} {concourse_target_team=} {override_definitions=}'
)
self.exception = exception
def __init__(
self,
repository_url: str,
cfg_set,
):
self._repository_url = urlparse(not_none(repository_url))
self._repo_host = self._repository_url.hostname
self.cfg_set = not_none(cfg_set)
concourse_cfg = cfg_set.concourse()
job_mapping_set = cfg_set.job_mapping(
concourse_cfg.job_mapping_cfg_name())
org_name, repo_name = self._repository_url.path.lstrip('/').split('/')
for job_mapping in job_mapping_set.job_mappings().values():
for org in job_mapping.github_organisations():
if org.org_name() != org_name:
continue
if not org.repository_matches(repo_name):
continue
github_cfg = cfg_set.github(org.github_cfg_name())
if github_cfg.matches_hostname(self._repo_host):
self.job_mapping = job_mapping
return
else:
raise JobMappingNotFoundError(
f'could not find matching job-mapping for org {org}')
def create_deployment(self, namespace: str, deployment: V1Deployment):
'''Create a deployment in a given namespace. Raises an `ApiException` if such a deployment
already exists.'''
not_empty(namespace)
not_none(deployment)
self.apps_api.create_namespaced_deployment(namespace=namespace, body=deployment)
def validate_component_name(name: str):
not_none(name)
if len(name) == 0:
raise InvalidComponentReferenceError(
'Component name must not be empty')
# valid component names are fully qualified github repository URLs without a schema
# (e.g. github.com/example_org/example_name)
if urllib.parse.urlparse(name).scheme:
raise InvalidComponentReferenceError(
'Component name must not contain schema')
# prepend dummy schema so that urlparse will parse away the hostname
parsed = urllib.parse.urlparse('dummy://' + name)
if not parsed.hostname:
raise InvalidComponentReferenceError(name)
path_parts = parsed.path.strip('/').split('/')
if not len(path_parts) == 2:
raise InvalidComponentReferenceError(
'Component name must end with github repository path')
return name
def __init__(
self,
pipeline_name,
pipeline_definition,
main_repo,
concourse_target_cfg,
concourse_target_team,
secret_cfg,
override_definitions=[
{},
],
exception=None,
):
try:
self.pipeline_name = not_empty(pipeline_name)
self.pipeline_definition = not_none(pipeline_definition)
self.main_repo = not_none(main_repo)
self.concourse_target_cfg = not_none(concourse_target_cfg)
self.concourse_target_team = not_none(concourse_target_team)
self.override_definitions = not_none(override_definitions)
self.secret_cfg = secret_cfg
except Exception as e:
raise ValueError(
f'{e=} missing value: {pipeline_name=} {pipeline_definition=} {main_repo=} '
f'{concourse_target_cfg=} {concourse_target_team=} {override_definitions=}'
)
self.exception = exception
def merge_products(left_product, right_product):
not_none(left_product)
not_none(right_product)
# start with a copy of left_product
merged = ComponentDescriptor.from_dict(deepcopy(dict(left_product.raw.items())))
for component in right_product.components():
existing_component = merged.component(component)
if existing_component:
# it is acceptable to add an existing component iff it is identical
if existing_component.raw == component.raw:
continue # skip
else:
raise ValueError(
'conflicting component definitions: {c1}, {c2}'.format(
c1=':'.join((existing_component.name(), existing_component.version())),
c2=':'.join((component.name(), component.version())),
)
)
merged.add_component(component)
# merge overwrites
for component_overwrite in right_product.component_overwrites():
# only one overwrite per component is allowed
for co in left_product.component_overwrites():
if co.declaring_component == component_overwrite.declaring_component():
raise ValueError(f'overwrite redefinition: {co}')
merged._add_component_overwrite(component_overwrite=component_overwrite)
return merged
def greatest_references(references: typing.Iterable[DependencyBase]):
'''
yields the component references from the specified iterable of ComponentReference that
have the greates version (grouped by component name).
Id est: if the sequence contains exactly one version of each contained component name,
the sequence is returned unchanged.
'''
not_none(references)
references = list(references)
for ref in references:
check_type(ref, DependencyBase)
names = [
ref.name() for ref
in references
]
for name in names:
matching_refs = [r for r in references if r.name() == name]
if len(matching_refs) == 1:
# in case reference name was unique, do not bother sorting
# (this also works around issues from non-semver versions)
yield matching_refs[0]
continue
# there might be multiple component versions of the same name
# --> use the greatest version in that case
matching_refs = sorted(
matching_refs,
key=lambda r: semver.parse_version_info(r.version()),
)
# greates version comes last
yield matching_refs[-1]
def create_image_pull_secret(
credentials: GcrCredentials,
image_pull_secret_name: str,
namespace: str,
):
"""Create an image pull secret in the K8s cluster to allow pods to download images from gcr"""
not_none(credentials)
not_empty(image_pull_secret_name)
not_empty(namespace)
ctx = kube_ctx
namespace_helper = ctx.namespace_helper()
namespace_helper.create_if_absent(namespace)
secret_helper = ctx.secret_helper()
if not secret_helper.get_secret(image_pull_secret_name, namespace):
secret_helper.create_gcr_secret(
namespace=namespace,
name=image_pull_secret_name,
password=credentials.passwd(),
user_name=credentials.username(),
email=credentials.email(),
server_url=credentials.host(),
)
service_account_helper = ctx.service_account_helper()
service_account_helper.patch_image_pull_secret_into_service_account(
name="default",
namespace=namespace,
image_pull_secret_name=image_pull_secret_name)
def create_tls_secret(tls_config: TlsConfig,
tls_secret_name: str,
namespace: str,
basic_auth_cred: BasicAuthCred = None):
""" Creates a secret with the configured TLS certificates in the K8s cluster.
Optionally adds credentials for Basic Authentication"""
not_none(tls_config)
not_empty(tls_secret_name)
not_empty(namespace)
ctx = kube_ctx
namespace_helper = ctx.namespace_helper()
namespace_helper.create_if_absent(namespace)
secret_helper = ctx.secret_helper()
if not secret_helper.get_secret(tls_secret_name, namespace):
data = {
'tls.key': tls_config.private_key(),
'tls.crt': tls_config.certificate(),
}
if basic_auth_cred:
ht = HtpasswdFile()
ht.set_password(basic_auth_cred.user, basic_auth_cred.password)
data['auth'] = ht.to_string().decode('utf-8')
secret_helper.put_secret(
name=tls_secret_name,
data=data,
namespace=namespace,
)
def add_triage(
self,
triage: Triage,
scope: TriageScope=None,
product_id=None,
group_id=None,
component_version=None,
):
'''
adds an existing Protecode triage to a specified target. The existing triage is usually
retrieved from an already uploaded product (which is represented by `AnalysisResult`).
This method is offered to support "transporting" existing triages.
Note that - depending on the effective target scope, the `product_id`, `group_id` formal
parameters are either required or forbidden.
Note that Protecode will only accept triages for matching (component, vulnerabilities,
version) tuples. In particular, triages for different component versions will be silently
ignored. Explicitly pass `component_version` of target protecode app (/product) to force
Protecode into accepting the given triage.
@param triage: the triage to "copy"
@param scope: if given, overrides the triage's scope
@param product_id: target product_id. required iff scope in FN, FH, R
@param group_id: target group_id. required iff scope is G(ROUP)
@param component_version: overwrite target component version
'''
# if no scope is set, use the one from passed triage
scope = scope if scope else triage.scope()
# depending on the scope, different arguments are required
if scope == TriageScope.ACCOUNT_WIDE:
pass
elif scope in (TriageScope.FILE_NAME, TriageScope.FILE_HASH, TriageScope.RESULT):
not_none(product_id)
elif scope == TriageScope.GROUP:
not_none(group_id)
else:
raise NotImplementedError()
if not component_version:
component_version = triage.component_version()
# "copy" data from existing triage
triage_dict = {
'component': triage.component_name(),
'version': component_version,
'vulns': [triage.vulnerability_id()],
'scope': triage.scope().value,
'reason': triage.reason(),
'description': triage.description(),
}
if product_id:
triage_dict['product_id'] = product_id
if group_id:
triage_dict['group_id'] = group_id
return self.add_triage_raw(triage_dict=triage_dict)
def create(name: str, variant_name: str, args_dict: dict):
if name not in TRAITS:
raise ModelValidationError('no such trait: ' + str(name))
not_none(args_dict)
ctor = TRAITS[name]
return ctor(name=name, variant_name=variant_name, raw_dict=args_dict)
def create_service(self, namespace: str, service: V1Service):
'''Create a service in a given namespace. Raises an `ApiException` if such a Service
already exists.
'''
not_empty(namespace)
not_none(service)
self.core_api.create_namespaced_service(namespace=namespace, body=service)
def __init__(self, repo, github_cfg, github_repo_path):
not_none(repo)
if not isinstance(repo, git.Repo):
# assume it's a file path if it's not already a git.Repo
repo = git.Repo(str(repo))
self.repo = repo
self.github_cfg = github_cfg
self.github_repo_path = github_repo_path
def __init__(
self,
definition_descriptor,
deploy_status,
error_details=None,
):
self.definition_descriptor = not_none(definition_descriptor)
self.deploy_status = not_none(deploy_status)
self.error_details = error_details
def __init__(
self,
job_mapping,
cfg_set,
repository_filter: callable=None,
):
self.job_mapping = not_none(job_mapping)
self.cfg_set = not_none(cfg_set)
self.repository_filter = repository_filter
def create_or_update_config_map(self, namespace: str, name: str, data: dict):
not_empty(namespace)
not_empty(name)
not_none(data)
if self.read_config_map(namespace=namespace, name=name):
self.replace_config_map(namespace=namespace, name=name, data=data)
else:
self.create_config_map(namespace=namespace, name=name, data=data)
def __init__(
self,
definition_descriptor: DefinitionDescriptor,
render_status,
error_details=None,
):
self.definition_descriptor = not_none(definition_descriptor)
self.render_status = not_none(render_status)
self.error_details = error_details
def add_triage(self,
triage: Triage,
scope: TriageScope = None,
product_id=None,
group_id=None):
'''
adds an existing Protecode triage to a specified target. The existing triage is usually
retrieved from an already uploaded product (which is represented by `AnalysisResult`).
This method is offered to support "transporting" existing triages.
Note that - depending on the effective target scope, the `product_id`, `group_id` formal
parameters are either required or forbidden.
@param triage: the triage to "copy"
@param scope: if given, overrides the triage's scope
@param product_id: target product_id. required iff scope in FN, FH, R
@param group_id: target group_id. required iff scope is G(ROUP)
'''
url = self._routes.triage()
# if no scope is set, use the one from passed triage
scope = scope if scope else triage.scope()
# depending on the scope, different arguments are required
if scope == TriageScope.ACCOUNT_WIDE:
none(product_id)
none(group_id)
elif scope in (TriageScope.FILE_NAME, TriageScope.FILE_HASH,
TriageScope.RESULT):
not_none(product_id)
none(group_id)
elif scope == TriageScope.GROUP:
none(product_id)
not_none(group_id)
else:
raise NotImplementedError()
# "copy" data from existing triage
triage_dict = {
'component': triage.component_name(),
'version': triage.component_version(),
'vulns': [triage.vulnerability_id()],
'scope': triage.scope().value,
'reason': triage.reason(),
'description': triage.description(),
}
if product_id:
triage_dict['product_id'] = product_id
if group_id:
triage_dict['group_id'] = group_id
self._put(
url=url,
json=triage_dict,
)
def __init__(self, api_routes, basic_credentials, tls_verify=False):
self._routes = not_none(api_routes)
self._credentials = not_none(basic_credentials)
self._auth = (basic_credentials.username(), basic_credentials.passwd())
self._tls_verify = tls_verify
self._session_id = None
self._session = requests.Session()
mount_default_adapter(session=self._session, )
self._csrf_token = None
def set_kubecfg(self, kubeconfig_dict: dict):
not_none(kubeconfig_dict)
configuration = kubernetes.client.Configuration()
cfg_loader = KubeConfigLoader(dict(kubeconfig_dict))
cfg_loader.load_and_set(configuration)
# pylint: disable=no-member
kubernetes.client.Configuration.set_default(configuration)
# pylint: enable=no-member
self.kubeconfig = configuration
def _send_mail(
email_cfg: EmailConfig,
recipients: typing.Iterable[str],
mail_template: str,
subject: str,
replace_tokens: dict={},
cc_recipients: typing.Iterable[str]=[],
mimetype='text',
):
not_none(email_cfg)
not_empty(recipients)
not_none(mail_template)
not_empty(subject)
# create body from template
mail_body = mailer.create_body(
mail_template=mail_template,
replace_tokens=replace_tokens,
)
recipients = {r.lower() for r in recipients}
cc_recipients = {r.lower() for r in cc_recipients}
sender_name = email_cfg.sender_name() # used for the header
sender = sender_name # passed as from-address to the mailserver
if email_cfg.use_tls():
smtp_server = smtplib.SMTP_SSL(email_cfg.smtp_host())
else:
smtp_server = smtplib.SMTP(email_cfg.smtp_host())
if email_cfg.has_credentials():
credentials = email_cfg.credentials()
sender = credentials.username()
sender_name = email_cfg.sender_name() or sender
smtp_server.login(user=credentials.username(), password=credentials.passwd())
# create mail envelope
mail = mailer.create_mail(
subject=subject,
sender=sender_name,
recipients=recipients,
cc_recipients=cc_recipients,
text=mail_body,
mimetype=mimetype,
)
recipients.update(cc_recipients)
mailer.send_mail(
smtp_server=smtp_server,
msg=mail,
sender=sender,
recipients=recipients
)
def __init__(
self,
githubrepobranch: GitHubRepoBranch,
github_helper: GitHubRepositoryHelper,
release_version: str,
repo_dir: str,
):
self.githubrepobranch = not_none(githubrepobranch)
self.github_helper = not_none(github_helper)
self.release_version = not_empty(release_version)
self.repo_dir = os.path.abspath(not_empty(repo_dir))
def __init__(self,
name,
base_definition,
jobs,
exception=None,
template='default'):
self.name = not_none(name)
self.base_definition = ensure_dict(base_definition, allow_empty=True)
self.jobs = ensure_dict(jobs, allow_empty=False)
self.template = not_none(template)
self.exception = exception
def __init__(
self,
type_name,
base_name,
qualifier=None,
logical_name=None,
):
self._type_name = not_none(type_name)
self._base_name = not_none(base_name)
self._qualifier = qualifier if qualifier else ''
self._logical_name = logical_name
def determine_email_address(
github_user_name: str,
github_api: GitHub,
) -> typing.Optional[str]:
not_none(github_user_name)
try:
user = github_api.user(github_user_name)
except NotFoundError:
logger.warning(f'failed to lookup {github_user_name=} {github_api._github_url=}')
return None
return user.email
def create_config_map(self, namespace: str, name: str, data: dict):
not_empty(namespace)
not_empty(name)
not_none(data)
self.core_api.create_namespaced_config_map(
namespace=namespace,
body=V1ConfigMap(
data=data,
metadata=V1ObjectMeta(name=name, namespace=namespace),
),
)
def set_kubecfg(self, kubeconfig_dict: typing.Union[dict, model.kubernetes.KubernetesConfig]):
not_none(kubeconfig_dict)
if isinstance(kubeconfig_dict, model.kubernetes.KubernetesConfig):
kubeconfig_dict = kubeconfig_dict.kubeconfig()
configuration = kubernetes.client.Configuration()
cfg_loader = KubeConfigLoader(dict(kubeconfig_dict))
cfg_loader.load_and_set(configuration)
# pylint: disable=no-member
kubernetes.client.Configuration.set_default(configuration)
# pylint: enable=no-member
self.kubeconfig = configuration
def __init__(
self,
status: UploadStatus,
component: Component,
result: AnalysisResult,
):
self.status = not_none(status)
self.component = not_none(component)
if result:
self.result = result
else:
self.result = None