def __init__( self, item: Union[V1Deployment, V1StatefulSet], logging=logging.getLogger(__name__), ) -> None: """ This Application wrapper is an interface for creating/deleting k8s deployments and statefulsets soa_config is KubernetesDeploymentConfig. It is not loaded in init because it is not always required. :param item: Kubernetes Object(V1Deployment/V1StatefulSet) that has already been filled up. :param logging: where logs go """ if not item.metadata.namespace: item.metadata.namespace = "paasta" attrs = { attr: item.metadata.labels.get(paasta_prefixed(attr)) for attr in [ "service", "instance", "git_sha", "image_version", "config_sha", ] } replicas = (item.spec.replicas if item.metadata.labels.get( paasta_prefixed("autoscaled"), "false") == "false" else None) self.kube_deployment = KubeDeployment(replicas=replicas, **attrs) self.item = item self.soa_config = None # type: KubernetesDeploymentConfig self.logging = logging
def setup_kube_crd( kube_client: KubeClient, cluster: str, services: Sequence[str], soa_dir: str = DEFAULT_SOA_DIR, ) -> bool: existing_crds = kube_client.apiextensions.list_custom_resource_definition( label_selector=paasta_prefixed("service")) desired_crds = [] for service in services: crd_config = service_configuration_lib.read_extra_service_information( service, f"crd-{cluster}", soa_dir=soa_dir) if not crd_config: log.info("nothing to deploy") continue metadata = crd_config.get("metadata", {}) if "labels" not in metadata: metadata["labels"] = {} metadata["labels"]["yelp.com/paasta_service"] = service metadata["labels"][paasta_prefixed("service")] = service desired_crd = V1beta1CustomResourceDefinition( api_version=crd_config.get("apiVersion"), kind=crd_config.get("kind"), metadata=metadata, spec=crd_config.get("spec"), ) desired_crds.append(desired_crd) return update_crds( kube_client=kube_client, desired_crds=desired_crds, existing_crds=existing_crds, )
def setup_kube_internal_crd( kube_client: KubeClient, ) -> bool: existing_crds = kube_client.apiextensions.list_custom_resource_definition( label_selector=paasta_prefixed("internal") ) return update_crds( kube_client=kube_client, desired_crds=INTERNAL_CRDS, existing_crds=existing_crds )
def get_existing_kubernetes_service_names(kube_client: KubeClient) -> Set[str]: service_objects = kube_client.core.list_namespaced_service( PAASTA_NAMESPACE) return { item.metadata.name for item in service_objects.items if item.metadata.annotations if item.metadata.annotations.get( paasta_prefixed("managed_by")) == "setup_istio_mesh" }
def setup_all_custom_resources( kube_client: KubeClient, soa_dir: str, cluster: str, custom_resource_definitions: Sequence[CustomResourceDefinition], service: str = None, instance: str = None, ) -> bool: cluster_crds = { crd.spec.names.kind for crd in kube_client.apiextensions.list_custom_resource_definition( label_selector=paasta_prefixed("service")).items } log.debug(f"CRDs found: {cluster_crds}") results = [] for crd in custom_resource_definitions: if crd.kube_kind.singular not in cluster_crds: # TODO: kube_kind.singular seems to correspond to `crd.names.kind` # and not `crd.names.singular` log.warning(f"CRD {crd.kube_kind.singular} " f"not found in {cluster}") continue # by convention, entries where key begins with _ are used as templates raw_config_dicts = load_all_configs(cluster=cluster, file_prefix=crd.file_prefix, soa_dir=soa_dir) config_dicts = {} for svc, raw_sdict in raw_config_dicts.items(): sdict = { inst: idict for inst, idict in raw_sdict.items() if inst[0] != "_" } if sdict: config_dicts[svc] = sdict if not config_dicts: continue ensure_namespace(kube_client=kube_client, namespace=f"paasta-{crd.kube_kind.plural}") results.append( setup_custom_resources( kube_client=kube_client, kind=crd.kube_kind, crd=crd, config_dicts=config_dicts, version=crd.version, group=crd.group, cluster=cluster, service=service, instance=instance, )) return any(results) if results else True
def cleanup_all_custom_resources( kube_client: KubeClient, soa_dir: str, cluster: str, custom_resource_definitions: Sequence[CustomResourceDefinition], ) -> bool: cluster_crds = { crd.spec.names.kind for crd in kube_client.apiextensions.list_custom_resource_definition( label_selector=paasta_prefixed("service")).items } log.debug(f"CRDs found: {cluster_crds}") results = [] for crd in custom_resource_definitions: if crd.kube_kind.singular not in cluster_crds: # TODO: kube_kind.singular seems to correspond to `crd.names.kind` # and not `crd.names.singular` log.warning(f"CRD {crd.kube_kind.singular} " f"not found in {cluster}") continue config_dicts = load_all_configs(cluster=cluster, file_prefix=crd.file_prefix, soa_dir=soa_dir) if not config_dicts: continue crs = list_custom_resources( kube_client=kube_client, kind=crd.kube_kind, version=crd.version, group=crd.group, ) for cr in crs: service = config_dicts.get(cr.service) if service is not None: instance = service.get(cr.instance) if instance is not None: continue result = False try: delete_custom_resource( kube_client=kube_client, name=cr.name, namespace=cr.namespace, plural=crd.kube_kind.plural, version=crd.version, group=crd.group, ) result = True except Exception: log.exception("Error while deleting CR {cr.name}") results.append(result) return all(results) if results else True
def is_valid_application(deployment: V1Deployment): is_valid = True missing = [] for attr in ["service", "instance", "git_sha", "config_sha"]: prefixed_attr = paasta_prefixed(attr) if prefixed_attr not in deployment.metadata.labels: is_valid = False missing.append(prefixed_attr) if missing: log.warning(f"deployment/{deployment.metadata.name} in " f"namespace/{deployment.metadata.namespace} " f"is missing following labels: {missing}") return is_valid
def format_custom_resource( instance_config: Mapping[str, Any], service: str, instance: str, cluster: str, kind: str, version: str, group: str, namespace: str, git_sha: str, ) -> Mapping[str, Any]: sanitised_service = sanitise_kubernetes_name(service) sanitised_instance = sanitise_kubernetes_name(instance) resource: Mapping[str, Any] = { "apiVersion": f"{group}/{version}", "kind": kind, "metadata": { "name": f"{sanitised_service}-{sanitised_instance}", "namespace": namespace, "labels": { "yelp.com/paasta_service": service, "yelp.com/paasta_instance": instance, "yelp.com/paasta_cluster": cluster, paasta_prefixed("service"): service, paasta_prefixed("instance"): instance, paasta_prefixed("cluster"): cluster, }, "annotations": {}, }, "spec": instance_config, } url = get_dashboard_url(kind, service, instance, cluster) if url: resource["metadata"]["annotations"]["yelp.com/dashboard_url"] = url resource["metadata"]["annotations"][paasta_prefixed( "dashboard_url")] = url config_hash = get_config_hash(resource) resource["metadata"]["annotations"]["yelp.com/desired_state"] = "running" resource["metadata"]["annotations"][paasta_prefixed( "desired_state")] = "running" resource["metadata"]["labels"]["yelp.com/paasta_config_sha"] = config_hash resource["metadata"]["labels"][paasta_prefixed("config_sha")] = config_hash resource["metadata"]["labels"][paasta_prefixed("git_sha")] = git_sha return resource
def cleanup_kube_crd( kube_client: KubeClient, cluster: str, soa_dir: str = DEFAULT_SOA_DIR, dry_run: bool = False, ) -> bool: service_attr = paasta_prefixed("service") existing_crds = kube_client.apiextensions.list_custom_resource_definition( label_selector=service_attr) success = True for crd in existing_crds.items: service = crd.metadata.labels[service_attr] if not service: log.error( f"CRD {crd.metadata.name} has empty {service_attr} label") continue crd_config = service_configuration_lib.read_extra_service_information( service, f"crd-{cluster}", soa_dir=soa_dir) if crd_config: log.debug( f"CRD {crd.metadata.name} declaration found in {service}") continue log.info(f"CRD {crd.metadata.name} not found in {service} service") if dry_run: log.info("not deleting in dry-run mode") continue try: kube_client.apiextensions.delete_custom_resource_definition( name=crd.metadata.name, body=V1DeleteOptions()) log.info(f"deleted {crd.metadata.name} for {cluster}:{service}") except ApiException as exc: log.error(f"error deploying crd for {cluster}:{service}, " f"status: {exc.status}, reason: {exc.reason}") log.debug(exc.body) success = False return success
def get_attribute(node, attribute): return node.metadata.labels.get(paasta_prefixed(attribute), "unknown")
from paasta_tools.kubernetes_tools import KubeClient from paasta_tools.kubernetes_tools import paasta_prefixed from paasta_tools.kubernetes_tools import update_crds log = logging.getLogger(__name__) INTERNAL_CRDS = [ V1beta1CustomResourceDefinition( api_version="apiextensions.k8s.io/v1beta1", kind="CustomResourceDefinition", metadata={ "name": "deploygroups.paasta.yelp.com", "labels": { paasta_prefixed("internal"): "true", }, }, spec={ "group": "paasta.yelp.com", "versions": [{"name": "v1beta1", "served": True, "storage": True}], "scope": "Namespaced", "names": { "plural": "deploygroups", "singular": "deploygroup", "kind": "DeployGroup", "shortNames": ["dg"], }, "validation": { "openAPIV3Schema": { "type": "object",
from paasta_tools.kubernetes_tools import ensure_namespace from paasta_tools.kubernetes_tools import KubeClient from paasta_tools.kubernetes_tools import limit_size_with_hash from paasta_tools.kubernetes_tools import paasta_prefixed from paasta_tools.kubernetes_tools import registration_label from paasta_tools.kubernetes_tools import sanitise_kubernetes_name from paasta_tools.utils import DEFAULT_SOA_DIR log = logging.getLogger(__name__) UNIFIED_K8S_SVC_NAME = "paasta-routing" UNIFIED_SVC_PORT = 1337 PAASTA_SVC_PORT = 8888 PAASTA_NAMESPACE = "paasta" ANNOTATIONS = {paasta_prefixed("managed_by"): "setup_istio_mesh"} def parse_args() -> argparse.Namespace: parser = argparse.ArgumentParser( description="Creates Kubernetes services.") parser.add_argument( "-v", "--verbose", action="store_true", dest="verbose", default=False, ) parser.add_argument( "--dry-run", action="store_true",
def reconcile_kubernetes_resource( kube_client: KubeClient, service: str, instance_configs: Mapping[str, Any], custom_resources: Sequence[KubeCustomResource], kind: KubeKind, version: str, group: str, crd: CustomResourceDefinition, cluster: str, instance: str = None, ) -> bool: succeeded = True config_handler = LONG_RUNNING_INSTANCE_TYPE_HANDLERS[crd.file_prefix] for inst, config in instance_configs.items(): if instance is not None and instance != inst: continue try: soa_config = config_handler.loader( service=service, instance=inst, cluster=cluster, load_deployments=True, soa_dir=DEFAULT_SOA_DIR, ) git_sha = get_git_sha_from_dockerurl(soa_config.get_docker_url(), long=True) formatted_resource = format_custom_resource( instance_config=config, service=service, instance=inst, cluster=cluster, kind=kind.singular, version=version, group=group, namespace=f"paasta-{kind.plural}", git_sha=git_sha, ) desired_resource = KubeCustomResource( service=service, instance=inst, config_sha=formatted_resource["metadata"]["labels"][ paasta_prefixed("config_sha")], git_sha=formatted_resource["metadata"]["labels"].get( paasta_prefixed("git_sha")), kind=kind.singular, name=formatted_resource["metadata"]["name"], namespace=f"paasta-{kind.plural}", ) if not (service, inst, kind.singular) in [ (c.service, c.instance, c.kind) for c in custom_resources ]: log.info(f"{desired_resource} does not exist so creating") create_custom_resource( kube_client=kube_client, version=version, kind=kind, formatted_resource=formatted_resource, group=group, ) elif desired_resource not in custom_resources: sanitised_service = sanitise_kubernetes_name(service) sanitised_instance = sanitise_kubernetes_name(inst) log.info( f"{desired_resource} exists but config_sha doesn't match") update_custom_resource( kube_client=kube_client, name=f"{sanitised_service}-{sanitised_instance}", version=version, kind=kind, formatted_resource=formatted_resource, group=group, ) else: log.info(f"{desired_resource} is up to date, no action taken") except Exception as e: log.error(str(e)) succeeded = False return succeeded
def setup_kube_crd( kube_client: KubeClient, cluster: str, services: Sequence[str], soa_dir: str = DEFAULT_SOA_DIR, ) -> bool: existing_crds = kube_client.apiextensions.list_custom_resource_definition( label_selector=paasta_prefixed("service")) success = True for service in services: crd_config = service_configuration_lib.read_extra_service_information( service, f"crd-{cluster}", soa_dir=soa_dir) if not crd_config: log.info("nothing to deploy") continue metadata = crd_config.get("metadata", {}) if "labels" not in metadata: metadata["labels"] = {} metadata["labels"]["yelp.com/paasta_service"] = service metadata["labels"][paasta_prefixed("service")] = service desired_crd = V1beta1CustomResourceDefinition( api_version=crd_config.get("apiVersion"), kind=crd_config.get("kind"), metadata=metadata, spec=crd_config.get("spec"), ) existing_crd = None for crd in existing_crds.items: if crd.metadata.name == desired_crd.metadata["name"]: existing_crd = crd break try: if existing_crd: desired_crd.metadata[ "resourceVersion"] = existing_crd.metadata.resource_version kube_client.apiextensions.replace_custom_resource_definition( name=desired_crd.metadata["name"], body=desired_crd) else: try: kube_client.apiextensions.create_custom_resource_definition( body=desired_crd) except ValueError as err: # TODO: kubernetes server will sometimes reply with conditions:null, # figure out how to deal with this correctly, for more details: # https://github.com/kubernetes/kubernetes/pull/64996 if "`conditions`, must not be `None`" in str(err): pass else: raise err log.info( f"deployed {desired_crd.metadata['name']} for {cluster}:{service}" ) except ApiException as exc: log.error(f"error deploying crd for {cluster}:{service}, " f"status: {exc.status}, reason: {exc.reason}") log.debug(exc.body) success = False return success