def validate_component_name(name: str):
not_none(name)
if len(name) == 0:
raise InvalidComponentReferenceError(
'Component name must not be empty')
# valid component names are fully qualified github repository URLs without a schema
# (e.g. github.com/example_org/example_name)
if urllib.parse.urlparse(name).scheme:
raise InvalidComponentReferenceError(
'Component name must not contain schema')
# prepend dummy schema so that urlparse will parse away the hostname
parsed = urllib.parse.urlparse('dummy://' + name)
if not parsed.hostname:
raise InvalidComponentReferenceError(name)
path_parts = parsed.path.strip('/').split('/')
if not len(path_parts) == 2:
raise InvalidComponentReferenceError(
'Component name must end with github repository path')
return name
def _pr_to_upgrade_pull_request(self, pull_request):
util.not_none(pull_request)
match = self.PR_TITLE_PATTERN.fullmatch(pull_request.title)
if match is None:
raise ValueError(
"PR-title '{t}' did not match title-schema".format(
t=pull_request.title))
reference_type_name = match.group(1)
if not reference_type_name:
# backwards compatibility hack
reference_type_name = 'component'
reference_type = product.model.reference_type(reference_type_name)
ref_name = match.group(2)
from_version = match.group(3)
to_version = match.group(4)
from_ref = reference_type.create(name=ref_name, version=from_version)
to_ref = reference_type.create(name=ref_name, version=to_version)
return UpgradePullRequest(
pull_request=pull_request,
from_ref=from_ref,
to_ref=to_ref,
)
def create_image_pull_secret(
credentials: GcrCredentials,
image_pull_secret_name: str,
namespace: str,
):
"""Create an image pull secret in the K8s cluster to allow pods to download images from gcr"""
not_none(credentials)
not_empty(image_pull_secret_name)
not_empty(namespace)
ctx = kube_ctx
namespace_helper = ctx.namespace_helper()
namespace_helper.create_if_absent(namespace)
secret_helper = ctx.secret_helper()
if not secret_helper.get_secret(image_pull_secret_name, namespace):
secret_helper.create_gcr_secret(
namespace=namespace,
name=image_pull_secret_name,
password=credentials.passwd(),
user_name=credentials.username(),
email=credentials.email(),
server_url=credentials.host(),
)
service_account_helper = ctx.service_account_helper()
service_account_helper.patch_image_pull_secret_into_service_account(
name="default",
namespace=namespace,
image_pull_secret_name=image_pull_secret_name
)
def create_ingress(self, namespace: str, ingress: V1beta1Ingress):
'''Create an ingress in a given namespace. Raises an `ApiException` if such an ingress
already exists.'''
not_empty(namespace)
not_none(ingress)
self.extensions_v1beta1_api.create_namespaced_ingress(namespace=namespace, body=ingress)
def merge_products(left_product, right_product):
not_none(left_product)
not_none(right_product)
# start with a copy of left_product
merged = ComponentDescriptor.from_dict(
deepcopy(dict(left_product.raw.items())))
for component in right_product.components():
existing_component = merged.component(component)
if existing_component:
# it is acceptable to add an existing component iff it is identical
if existing_component.raw == component.raw:
continue # skip
else:
raise ValueError(
'conflicting component definitions: {c1}, {c2}'.format(
c1=':'.join((existing_component.name(),
existing_component.version())),
c2=':'.join((component.name(), component.version())),
))
merged.add_component(component)
# merge overwrites
for component_overwrite in right_product.component_overwrites():
# only one overwrite per component is allowed
for co in left_product.component_overwrites():
if co.declaring_component == component_overwrite.declaring_component(
):
raise ValueError(f'overwrite redefinition: {co}')
merged._add_component_overwrite(
component_overwrite=component_overwrite)
return merged
def add_input(self, name, variable_name):
util.not_none(name)
util.not_none(variable_name)
if name in self._inputs_dict:
raise ValueError('input already exists: ' + str(name))
self._inputs_dict[name] = variable_name
def create_deployment(self, namespace: str, deployment: V1Deployment):
'''Create a deployment in a given namespace. Raises an `ApiException` if such a deployment
already exists.'''
not_empty(namespace)
not_none(deployment)
self.apps_api.create_namespaced_deployment(namespace=namespace, body=deployment)
def create_tls_secret(
tls_config: TlsConfig,
tls_secret_name: str,
namespace: str,
):
"""Creates the configured TLS secret for the Concourse web-component in the K8s cluster"""
not_none(tls_config)
not_empty(tls_secret_name)
not_empty(namespace)
ctx = kube_ctx
namespace_helper = ctx.namespace_helper()
namespace_helper.create_if_absent(namespace)
secret_helper = ctx.secret_helper()
if not secret_helper.get_secret(tls_secret_name, namespace):
data = {
'tls.key': tls_config.private_key(),
'tls.crt': tls_config.certificate(),
}
secret_helper.put_secret(
name=tls_secret_name,
data=data,
namespace=namespace,
)
def _push_image(image_reference: str, image_file: str, threads=8):
import util
util.not_none(image_reference)
util.existing_file(image_file)
transport = _mk_transport()
image_reference = normalise_image_reference(image_reference)
image_reference = _parse_image_reference(image_reference)
creds = _mk_credentials(
image_reference=image_reference,
privileges=Privileges.READ_WRITE,
)
with v2_2_image.FromTarball(image_file) as v2_2_img:
try:
with docker_session.Push(
image_reference,
creds,
transport,
threads=threads,
) as session:
session.upload(v2_2_img)
digest = v2_2_img.digest()
logger.info(
f'{image_reference} was uploaded - digest: {digest}')
except Exception as e:
import traceback
traceback.print_exc()
raise e
def greatest_references(references: typing.Iterable[DependencyBase]):
'''
yields the component references from the specified iterable of ComponentReference that
have the greates version (grouped by component name).
Id est: if the sequence contains exactly one version of each contained component name,
the sequence is returned unchanged.
'''
not_none(references)
references = list(references)
for ref in references:
check_type(ref, DependencyBase)
names = [ref.name() for ref in references]
for name in names:
matching_refs = [r for r in references if r.name() == name]
if len(matching_refs) == 1:
# in case reference name was unique, do not bother sorting
# (this also works around issues from non-semver versions)
yield matching_refs[0]
continue
# there might be multiple component versions of the same name
# --> use the greatest version in that case
matching_refs = sorted(
matching_refs,
key=lambda r: semver.parse_version_info(r.version()),
)
# greates version comes last
yield matching_refs[-1]
def create_tls_secret(tls_config: TlsConfig,
tls_secret_name: str,
namespace: str,
basic_auth_cred: BasicAuthCred = None):
""" Creates a secret with the configured TLS certificates in the K8s cluster.
Optionally adds credentials for Basic Authentication"""
not_none(tls_config)
not_empty(tls_secret_name)
not_empty(namespace)
ctx = kube_ctx
namespace_helper = ctx.namespace_helper()
namespace_helper.create_if_absent(namespace)
secret_helper = ctx.secret_helper()
if not secret_helper.get_secret(tls_secret_name, namespace):
data = {
'tls.key': tls_config.private_key(),
'tls.crt': tls_config.certificate(),
}
if basic_auth_cred:
ht = HtpasswdFile()
ht.set_password(basic_auth_cred.user, basic_auth_cred.password)
data['auth'] = ht.to_string().decode('utf-8')
secret_helper.put_secret(
name=tls_secret_name,
data=data,
namespace=namespace,
)
def deploy_secrets_server(secrets_server_config: SecretsServerConfig):
not_none(secrets_server_config)
ctx = kube_ctx
service_helper = ctx.service_helper()
deployment_helper = ctx.deployment_helper()
secrets_helper = ctx.secret_helper()
namespace_helper = ctx.namespace_helper()
namespace = secrets_server_config.namespace()
namespace_helper.create_if_absent(namespace)
secret_name = secrets_server_config.secrets().concourse_secret_name()
# Deploy an empty secret if none exists so that the secrets-server can start.
# However, if there is already a secret we should not purge its contents.
if not secrets_helper.get_secret(secret_name, namespace):
secrets_helper.put_secret(
name=secret_name,
data={},
namespace=namespace,
)
service = generate_secrets_server_service(secrets_server_config)
deployment = generate_secrets_server_deployment(secrets_server_config)
service_helper.replace_or_create_service(namespace, service)
deployment_helper.replace_or_create_deployment(namespace, deployment)
def _github_cfg_for_hostname(self, host_name):
not_none(host_name)
for github_cfg in self.cfg_factory._cfg_elements(
cfg_type_name='github'):
if github_cfg.matches_hostname(host_name=host_name):
return github_cfg
raise RuntimeError('no github_cfg for {h}'.format(host_name))
def from_cfg(protecode_cfg):
not_none(protecode_cfg)
routes = ProtecodeApiRoutes(base_url=protecode_cfg.api_url())
api = ProtecodeApi(api_routes=routes,
basic_credentials=protecode_cfg.credentials(),
tls_verify=protecode_cfg.tls_verify())
return api
def create_service(self, namespace: str, service: V1Service):
'''Create a service in a given namespace. Raises an `ApiException` if such a Service
already exists.
'''
not_empty(namespace)
not_none(service)
self.core_api.create_namespaced_service(namespace=namespace, body=service)
def create(name: str, variant_name: str, args_dict: dict):
if name not in TRAITS:
raise ModelValidationError('no such trait: ' + str(name))
not_none(args_dict)
ctor = TRAITS[name]
return ctor(name=name, variant_name=variant_name, raw_dict=args_dict)
def __init__(self, repo, github_cfg, github_repo_path):
not_none(repo)
if not isinstance(repo, git.Repo):
# assume it's a file path if it's not already a git.Repo
repo = git.Repo(str(repo))
self.repo = repo
self.github_cfg = github_cfg
self.github_repo_path = github_repo_path
def _pull_image(image_reference: str, outfileobj=None):
import util
util.not_none(image_reference)
transport = _mk_transport()
image_reference = normalise_image_reference(image_reference)
image_reference = _parse_image_reference(image_reference)
creds = _mk_credentials(image_reference=image_reference)
# OCI Image Manifest is compatible with Docker Image Manifest Version 2,
# Schema 2. We indicate support for both formats by passing both media types
# as 'Accept' headers.
#
# For reference:
# OCI: https://github.com/opencontainers/image-spec
# Docker: https://docs.docker.com/registry/spec/manifest-v2-2/
accept = docker_http.SUPPORTED_MANIFEST_MIMES
try:
# XXX TODO: use streaming rather than writing to local FS
# if outfile is given, we must use it instead of an ano
outfileobj = outfileobj if outfileobj else tempfile.TemporaryFile()
with tarfile.open(fileobj=outfileobj, mode='w:') as tar:
util.verbose(f'Pulling manifest list from {image_reference}..')
with image_list.FromRegistry(image_reference, creds,
transport) as img_list:
if img_list.exists():
platform = image_list.Platform({
'architecture': _PROCESSOR_ARCHITECTURE,
'os': _OPERATING_SYSTEM,
})
# pytype: disable=wrong-arg-types
with img_list.resolve(platform) as default_child:
save.tarball(_make_tag_if_digest(image_reference),
default_child, tar)
return outfileobj
# pytype: enable=wrong-arg-types
util.info(f'Pulling v2.2 image from {image_reference}..')
with v2_2_image.FromRegistry(image_reference, creds, transport,
accept) as v2_2_img:
if v2_2_img.exists():
save.tarball(_make_tag_if_digest(image_reference),
v2_2_img, tar)
return outfileobj
util.info(f'Pulling v2 image from {image_reference}..')
with v2_image.FromRegistry(image_reference, creds,
transport) as v2_img:
with v2_compat.V22FromV2(v2_img) as v2_2_img:
save.tarball(_make_tag_if_digest(image_reference),
v2_2_img, tar)
return outfileobj
except Exception as e:
outfileobj.close()
util.fail(f'Error pulling and saving image {image_reference}: {e}')
def add_triage(self,
triage: Triage,
scope: TriageScope = None,
product_id=None,
group_id=None):
'''
adds an existing Protecode triage to a specified target. The existing triage is usually
retrieved from an already uploaded product (which is represented by `AnalysisResult`).
This method is offered to support "transporting" existing triages.
Note that - depending on the effective target scope, the `product_id`, `group_id` formal
parameters are either required or forbidden.
@param triage: the triage to "copy"
@param scope: if given, overrides the triage's scope
@param product_id: target product_id. required iff scope in FN, FH, R
@param group_id: target group_id. required iff scope is G(ROUP)
'''
url = self._routes.triage()
# if no scope is set, use the one from passed triage
scope = scope if scope else triage.scope()
# depending on the scope, different arguments are required
if scope == TriageScope.ACCOUNT_WIDE:
none(product_id)
none(group_id)
elif scope in (TriageScope.FILE_NAME, TriageScope.FILE_HASH,
TriageScope.RESULT):
not_none(product_id)
none(group_id)
elif scope == TriageScope.GROUP:
none(product_id)
not_none(group_id)
else:
raise NotImplementedError()
# "copy" data from existing triage
triage_dict = {
'component': triage.component_name(),
'version': triage.component_version(),
'vulns': [triage.vulnerability_id()],
'scope': triage.scope().value,
'reason': triage.reason(),
'description': triage.description(),
}
if product_id:
triage_dict['product_id'] = product_id
if group_id:
triage_dict['group_id'] = group_id
self._put(
url=url,
json=triage_dict,
)
def __init__(
self,
definition_descriptor,
deploy_status,
error_details=None,
):
self.definition_descriptor = not_none(definition_descriptor)
self.deploy_status = not_none(deploy_status)
self.error_details = error_details
def __init__(
self,
definition_descriptor:DefinitionDescriptor,
render_status,
error_details=None,
):
self.definition_descriptor = not_none(definition_descriptor)
self.render_status = not_none(render_status)
self.error_details = error_details
def create_or_update_config_map(self, namespace: str, name: str, data: dict):
not_empty(namespace)
not_empty(name)
not_none(data)
if self.read_config_map(namespace=namespace, name=name):
self.replace_config_map(namespace=namespace, name=name, data=data)
else:
self.create_config_map(namespace=namespace, name=name, data=data)
def __init__(self, api_routes, basic_credentials, tls_verify=False):
self._routes = not_none(api_routes)
self._credentials = not_none(basic_credentials)
self._auth = (basic_credentials.username(), basic_credentials.passwd())
self._tls_verify = tls_verify
self._session_id = None
self._session = requests.Session()
mount_default_adapter(session=self._session, )
self._csrf_token = None
def set_kubecfg(self, kubeconfig_dict: dict):
not_none(kubeconfig_dict)
configuration = kubernetes.client.Configuration()
cfg_loader = KubeConfigLoader(dict(kubeconfig_dict))
cfg_loader.load_and_set(configuration)
# pylint: disable=no-member
kubernetes.client.Configuration.set_default(configuration)
# pylint: enable=no-member
self.kubeconfig = configuration
def __init__(self,
name,
base_definition,
variants,
exception=None,
template='default'):
self.name = not_none(name)
self.base_definition = ensure_dict(base_definition, allow_empty=True)
self.variants = ensure_dict(variants, allow_empty=False)
self.template = not_none(template)
self.exception = exception
def __init__(
self,
githubrepobranch: GitHubRepoBranch,
github_helper: GitHubRepositoryHelper,
release_version: str,
repo_dir: str,
):
self.githubrepobranch = not_none(githubrepobranch)
self.github_helper = not_none(github_helper)
self.release_version = not_empty(release_version)
self.repo_dir = os.path.abspath(not_empty(repo_dir))
def __init__(
self,
type_name,
base_name,
qualifier=None,
logical_name=None,
):
self._type_name = not_none(type_name)
self._base_name = not_none(base_name)
self._qualifier = qualifier if qualifier else ''
self._logical_name = logical_name
def create_config_map(self, namespace: str, name: str, data: dict):
not_empty(namespace)
not_empty(name)
not_none(data)
self.core_api.create_namespaced_config_map(
namespace = namespace,
body = V1ConfigMap(
data=data,
metadata=V1ObjectMeta(name=name, namespace=namespace),
),
)
def __init__(self, api_routes, basic_credentials, tls_verify=False):
self._routes = not_none(api_routes)
self._credentials = not_none(basic_credentials)
self._auth = (basic_credentials.username(), basic_credentials.passwd())
self._request_builder = AuthenticatedRequestBuilder(
basic_auth_username=basic_credentials.username(),
basic_auth_passwd=basic_credentials.passwd(),
verify_ssl=tls_verify)
self._get = partial(requests.get, verify=tls_verify)
self._post = partial(requests.post, verify=tls_verify)
self._put = partial(requests.put, verify=tls_verify)
def __init__(
self,
slack_cfg_name: str,
slack_channel: str,
release_version: str,
release_notes: ReleaseNotes,
githubrepobranch: GitHubRepoBranch,
):
self.slack_cfg_name = not_empty(slack_cfg_name)
self.slack_channel = not_empty(slack_channel)
self.release_version = not_empty(release_version)
self.githubrepobranch = not_none(githubrepobranch)
self.release_notes = not_none(release_notes)
