def validate_component_name(name: str): not_none(name) if len(name) == 0: raise InvalidComponentReferenceError( 'Component name must not be empty') # valid component names are fully qualified github repository URLs without a schema # (e.g. github.com/example_org/example_name) if urllib.parse.urlparse(name).scheme: raise InvalidComponentReferenceError( 'Component name must not contain schema') # prepend dummy schema so that urlparse will parse away the hostname parsed = urllib.parse.urlparse('dummy://' + name) if not parsed.hostname: raise InvalidComponentReferenceError(name) path_parts = parsed.path.strip('/').split('/') if not len(path_parts) == 2: raise InvalidComponentReferenceError( 'Component name must end with github repository path') return name
def _pr_to_upgrade_pull_request(self, pull_request): util.not_none(pull_request) match = self.PR_TITLE_PATTERN.fullmatch(pull_request.title) if match is None: raise ValueError( "PR-title '{t}' did not match title-schema".format( t=pull_request.title)) reference_type_name = match.group(1) if not reference_type_name: # backwards compatibility hack reference_type_name = 'component' reference_type = product.model.reference_type(reference_type_name) ref_name = match.group(2) from_version = match.group(3) to_version = match.group(4) from_ref = reference_type.create(name=ref_name, version=from_version) to_ref = reference_type.create(name=ref_name, version=to_version) return UpgradePullRequest( pull_request=pull_request, from_ref=from_ref, to_ref=to_ref, )
def create_image_pull_secret( credentials: GcrCredentials, image_pull_secret_name: str, namespace: str, ): """Create an image pull secret in the K8s cluster to allow pods to download images from gcr""" not_none(credentials) not_empty(image_pull_secret_name) not_empty(namespace) ctx = kube_ctx namespace_helper = ctx.namespace_helper() namespace_helper.create_if_absent(namespace) secret_helper = ctx.secret_helper() if not secret_helper.get_secret(image_pull_secret_name, namespace): secret_helper.create_gcr_secret( namespace=namespace, name=image_pull_secret_name, password=credentials.passwd(), user_name=credentials.username(), email=credentials.email(), server_url=credentials.host(), ) service_account_helper = ctx.service_account_helper() service_account_helper.patch_image_pull_secret_into_service_account( name="default", namespace=namespace, image_pull_secret_name=image_pull_secret_name )
def create_ingress(self, namespace: str, ingress: V1beta1Ingress): '''Create an ingress in a given namespace. Raises an `ApiException` if such an ingress already exists.''' not_empty(namespace) not_none(ingress) self.extensions_v1beta1_api.create_namespaced_ingress(namespace=namespace, body=ingress)
def merge_products(left_product, right_product): not_none(left_product) not_none(right_product) # start with a copy of left_product merged = ComponentDescriptor.from_dict( deepcopy(dict(left_product.raw.items()))) for component in right_product.components(): existing_component = merged.component(component) if existing_component: # it is acceptable to add an existing component iff it is identical if existing_component.raw == component.raw: continue # skip else: raise ValueError( 'conflicting component definitions: {c1}, {c2}'.format( c1=':'.join((existing_component.name(), existing_component.version())), c2=':'.join((component.name(), component.version())), )) merged.add_component(component) # merge overwrites for component_overwrite in right_product.component_overwrites(): # only one overwrite per component is allowed for co in left_product.component_overwrites(): if co.declaring_component == component_overwrite.declaring_component( ): raise ValueError(f'overwrite redefinition: {co}') merged._add_component_overwrite( component_overwrite=component_overwrite) return merged
def add_input(self, name, variable_name): util.not_none(name) util.not_none(variable_name) if name in self._inputs_dict: raise ValueError('input already exists: ' + str(name)) self._inputs_dict[name] = variable_name
def create_deployment(self, namespace: str, deployment: V1Deployment): '''Create a deployment in a given namespace. Raises an `ApiException` if such a deployment already exists.''' not_empty(namespace) not_none(deployment) self.apps_api.create_namespaced_deployment(namespace=namespace, body=deployment)
def create_tls_secret( tls_config: TlsConfig, tls_secret_name: str, namespace: str, ): """Creates the configured TLS secret for the Concourse web-component in the K8s cluster""" not_none(tls_config) not_empty(tls_secret_name) not_empty(namespace) ctx = kube_ctx namespace_helper = ctx.namespace_helper() namespace_helper.create_if_absent(namespace) secret_helper = ctx.secret_helper() if not secret_helper.get_secret(tls_secret_name, namespace): data = { 'tls.key': tls_config.private_key(), 'tls.crt': tls_config.certificate(), } secret_helper.put_secret( name=tls_secret_name, data=data, namespace=namespace, )
def _push_image(image_reference: str, image_file: str, threads=8): import util util.not_none(image_reference) util.existing_file(image_file) transport = _mk_transport() image_reference = normalise_image_reference(image_reference) image_reference = _parse_image_reference(image_reference) creds = _mk_credentials( image_reference=image_reference, privileges=Privileges.READ_WRITE, ) with v2_2_image.FromTarball(image_file) as v2_2_img: try: with docker_session.Push( image_reference, creds, transport, threads=threads, ) as session: session.upload(v2_2_img) digest = v2_2_img.digest() logger.info( f'{image_reference} was uploaded - digest: {digest}') except Exception as e: import traceback traceback.print_exc() raise e
def greatest_references(references: typing.Iterable[DependencyBase]): ''' yields the component references from the specified iterable of ComponentReference that have the greates version (grouped by component name). Id est: if the sequence contains exactly one version of each contained component name, the sequence is returned unchanged. ''' not_none(references) references = list(references) for ref in references: check_type(ref, DependencyBase) names = [ref.name() for ref in references] for name in names: matching_refs = [r for r in references if r.name() == name] if len(matching_refs) == 1: # in case reference name was unique, do not bother sorting # (this also works around issues from non-semver versions) yield matching_refs[0] continue # there might be multiple component versions of the same name # --> use the greatest version in that case matching_refs = sorted( matching_refs, key=lambda r: semver.parse_version_info(r.version()), ) # greates version comes last yield matching_refs[-1]
def create_tls_secret(tls_config: TlsConfig, tls_secret_name: str, namespace: str, basic_auth_cred: BasicAuthCred = None): """ Creates a secret with the configured TLS certificates in the K8s cluster. Optionally adds credentials for Basic Authentication""" not_none(tls_config) not_empty(tls_secret_name) not_empty(namespace) ctx = kube_ctx namespace_helper = ctx.namespace_helper() namespace_helper.create_if_absent(namespace) secret_helper = ctx.secret_helper() if not secret_helper.get_secret(tls_secret_name, namespace): data = { 'tls.key': tls_config.private_key(), 'tls.crt': tls_config.certificate(), } if basic_auth_cred: ht = HtpasswdFile() ht.set_password(basic_auth_cred.user, basic_auth_cred.password) data['auth'] = ht.to_string().decode('utf-8') secret_helper.put_secret( name=tls_secret_name, data=data, namespace=namespace, )
def deploy_secrets_server(secrets_server_config: SecretsServerConfig): not_none(secrets_server_config) ctx = kube_ctx service_helper = ctx.service_helper() deployment_helper = ctx.deployment_helper() secrets_helper = ctx.secret_helper() namespace_helper = ctx.namespace_helper() namespace = secrets_server_config.namespace() namespace_helper.create_if_absent(namespace) secret_name = secrets_server_config.secrets().concourse_secret_name() # Deploy an empty secret if none exists so that the secrets-server can start. # However, if there is already a secret we should not purge its contents. if not secrets_helper.get_secret(secret_name, namespace): secrets_helper.put_secret( name=secret_name, data={}, namespace=namespace, ) service = generate_secrets_server_service(secrets_server_config) deployment = generate_secrets_server_deployment(secrets_server_config) service_helper.replace_or_create_service(namespace, service) deployment_helper.replace_or_create_deployment(namespace, deployment)
def _github_cfg_for_hostname(self, host_name): not_none(host_name) for github_cfg in self.cfg_factory._cfg_elements( cfg_type_name='github'): if github_cfg.matches_hostname(host_name=host_name): return github_cfg raise RuntimeError('no github_cfg for {h}'.format(host_name))
def from_cfg(protecode_cfg): not_none(protecode_cfg) routes = ProtecodeApiRoutes(base_url=protecode_cfg.api_url()) api = ProtecodeApi(api_routes=routes, basic_credentials=protecode_cfg.credentials(), tls_verify=protecode_cfg.tls_verify()) return api
def create_service(self, namespace: str, service: V1Service): '''Create a service in a given namespace. Raises an `ApiException` if such a Service already exists. ''' not_empty(namespace) not_none(service) self.core_api.create_namespaced_service(namespace=namespace, body=service)
def create(name: str, variant_name: str, args_dict: dict): if name not in TRAITS: raise ModelValidationError('no such trait: ' + str(name)) not_none(args_dict) ctor = TRAITS[name] return ctor(name=name, variant_name=variant_name, raw_dict=args_dict)
def __init__(self, repo, github_cfg, github_repo_path): not_none(repo) if not isinstance(repo, git.Repo): # assume it's a file path if it's not already a git.Repo repo = git.Repo(str(repo)) self.repo = repo self.github_cfg = github_cfg self.github_repo_path = github_repo_path
def _pull_image(image_reference: str, outfileobj=None): import util util.not_none(image_reference) transport = _mk_transport() image_reference = normalise_image_reference(image_reference) image_reference = _parse_image_reference(image_reference) creds = _mk_credentials(image_reference=image_reference) # OCI Image Manifest is compatible with Docker Image Manifest Version 2, # Schema 2. We indicate support for both formats by passing both media types # as 'Accept' headers. # # For reference: # OCI: https://github.com/opencontainers/image-spec # Docker: https://docs.docker.com/registry/spec/manifest-v2-2/ accept = docker_http.SUPPORTED_MANIFEST_MIMES try: # XXX TODO: use streaming rather than writing to local FS # if outfile is given, we must use it instead of an ano outfileobj = outfileobj if outfileobj else tempfile.TemporaryFile() with tarfile.open(fileobj=outfileobj, mode='w:') as tar: util.verbose(f'Pulling manifest list from {image_reference}..') with image_list.FromRegistry(image_reference, creds, transport) as img_list: if img_list.exists(): platform = image_list.Platform({ 'architecture': _PROCESSOR_ARCHITECTURE, 'os': _OPERATING_SYSTEM, }) # pytype: disable=wrong-arg-types with img_list.resolve(platform) as default_child: save.tarball(_make_tag_if_digest(image_reference), default_child, tar) return outfileobj # pytype: enable=wrong-arg-types util.info(f'Pulling v2.2 image from {image_reference}..') with v2_2_image.FromRegistry(image_reference, creds, transport, accept) as v2_2_img: if v2_2_img.exists(): save.tarball(_make_tag_if_digest(image_reference), v2_2_img, tar) return outfileobj util.info(f'Pulling v2 image from {image_reference}..') with v2_image.FromRegistry(image_reference, creds, transport) as v2_img: with v2_compat.V22FromV2(v2_img) as v2_2_img: save.tarball(_make_tag_if_digest(image_reference), v2_2_img, tar) return outfileobj except Exception as e: outfileobj.close() util.fail(f'Error pulling and saving image {image_reference}: {e}')
def add_triage(self, triage: Triage, scope: TriageScope = None, product_id=None, group_id=None): ''' adds an existing Protecode triage to a specified target. The existing triage is usually retrieved from an already uploaded product (which is represented by `AnalysisResult`). This method is offered to support "transporting" existing triages. Note that - depending on the effective target scope, the `product_id`, `group_id` formal parameters are either required or forbidden. @param triage: the triage to "copy" @param scope: if given, overrides the triage's scope @param product_id: target product_id. required iff scope in FN, FH, R @param group_id: target group_id. required iff scope is G(ROUP) ''' url = self._routes.triage() # if no scope is set, use the one from passed triage scope = scope if scope else triage.scope() # depending on the scope, different arguments are required if scope == TriageScope.ACCOUNT_WIDE: none(product_id) none(group_id) elif scope in (TriageScope.FILE_NAME, TriageScope.FILE_HASH, TriageScope.RESULT): not_none(product_id) none(group_id) elif scope == TriageScope.GROUP: none(product_id) not_none(group_id) else: raise NotImplementedError() # "copy" data from existing triage triage_dict = { 'component': triage.component_name(), 'version': triage.component_version(), 'vulns': [triage.vulnerability_id()], 'scope': triage.scope().value, 'reason': triage.reason(), 'description': triage.description(), } if product_id: triage_dict['product_id'] = product_id if group_id: triage_dict['group_id'] = group_id self._put( url=url, json=triage_dict, )
def __init__( self, definition_descriptor, deploy_status, error_details=None, ): self.definition_descriptor = not_none(definition_descriptor) self.deploy_status = not_none(deploy_status) self.error_details = error_details
def __init__( self, definition_descriptor:DefinitionDescriptor, render_status, error_details=None, ): self.definition_descriptor = not_none(definition_descriptor) self.render_status = not_none(render_status) self.error_details = error_details
def create_or_update_config_map(self, namespace: str, name: str, data: dict): not_empty(namespace) not_empty(name) not_none(data) if self.read_config_map(namespace=namespace, name=name): self.replace_config_map(namespace=namespace, name=name, data=data) else: self.create_config_map(namespace=namespace, name=name, data=data)
def __init__(self, api_routes, basic_credentials, tls_verify=False): self._routes = not_none(api_routes) self._credentials = not_none(basic_credentials) self._auth = (basic_credentials.username(), basic_credentials.passwd()) self._tls_verify = tls_verify self._session_id = None self._session = requests.Session() mount_default_adapter(session=self._session, ) self._csrf_token = None
def set_kubecfg(self, kubeconfig_dict: dict): not_none(kubeconfig_dict) configuration = kubernetes.client.Configuration() cfg_loader = KubeConfigLoader(dict(kubeconfig_dict)) cfg_loader.load_and_set(configuration) # pylint: disable=no-member kubernetes.client.Configuration.set_default(configuration) # pylint: enable=no-member self.kubeconfig = configuration
def __init__(self, name, base_definition, variants, exception=None, template='default'): self.name = not_none(name) self.base_definition = ensure_dict(base_definition, allow_empty=True) self.variants = ensure_dict(variants, allow_empty=False) self.template = not_none(template) self.exception = exception
def __init__( self, githubrepobranch: GitHubRepoBranch, github_helper: GitHubRepositoryHelper, release_version: str, repo_dir: str, ): self.githubrepobranch = not_none(githubrepobranch) self.github_helper = not_none(github_helper) self.release_version = not_empty(release_version) self.repo_dir = os.path.abspath(not_empty(repo_dir))
def __init__( self, type_name, base_name, qualifier=None, logical_name=None, ): self._type_name = not_none(type_name) self._base_name = not_none(base_name) self._qualifier = qualifier if qualifier else '' self._logical_name = logical_name
def create_config_map(self, namespace: str, name: str, data: dict): not_empty(namespace) not_empty(name) not_none(data) self.core_api.create_namespaced_config_map( namespace = namespace, body = V1ConfigMap( data=data, metadata=V1ObjectMeta(name=name, namespace=namespace), ), )
def __init__(self, api_routes, basic_credentials, tls_verify=False): self._routes = not_none(api_routes) self._credentials = not_none(basic_credentials) self._auth = (basic_credentials.username(), basic_credentials.passwd()) self._request_builder = AuthenticatedRequestBuilder( basic_auth_username=basic_credentials.username(), basic_auth_passwd=basic_credentials.passwd(), verify_ssl=tls_verify) self._get = partial(requests.get, verify=tls_verify) self._post = partial(requests.post, verify=tls_verify) self._put = partial(requests.put, verify=tls_verify)
def __init__( self, slack_cfg_name: str, slack_channel: str, release_version: str, release_notes: ReleaseNotes, githubrepobranch: GitHubRepoBranch, ): self.slack_cfg_name = not_empty(slack_cfg_name) self.slack_channel = not_empty(slack_channel) self.release_version = not_empty(release_version) self.githubrepobranch = not_none(githubrepobranch) self.release_notes = not_none(release_notes)