def __init__(
self,
item: Union[V1Deployment, V1StatefulSet],
logging=logging.getLogger(__name__),
) -> None:
"""
This Application wrapper is an interface for creating/deleting k8s deployments and statefulsets
soa_config is KubernetesDeploymentConfig. It is not loaded in init because it is not always required.
:param item: Kubernetes Object(V1Deployment/V1StatefulSet) that has already been filled up.
:param logging: where logs go
"""
if not item.metadata.namespace:
item.metadata.namespace = "paasta"
attrs = {
attr: item.metadata.labels.get(paasta_prefixed(attr))
for attr in [
"service",
"instance",
"git_sha",
"image_version",
"config_sha",
]
}
replicas = (item.spec.replicas if item.metadata.labels.get(
paasta_prefixed("autoscaled"), "false") == "false" else None)
self.kube_deployment = KubeDeployment(replicas=replicas, **attrs)
self.item = item
self.soa_config = None # type: KubernetesDeploymentConfig
self.logging = logging
def setup_kube_crd(
kube_client: KubeClient,
cluster: str,
services: Sequence[str],
soa_dir: str = DEFAULT_SOA_DIR,
) -> bool:
existing_crds = kube_client.apiextensions.list_custom_resource_definition(
label_selector=paasta_prefixed("service"))
desired_crds = []
for service in services:
crd_config = service_configuration_lib.read_extra_service_information(
service, f"crd-{cluster}", soa_dir=soa_dir)
if not crd_config:
log.info("nothing to deploy")
continue
metadata = crd_config.get("metadata", {})
if "labels" not in metadata:
metadata["labels"] = {}
metadata["labels"]["yelp.com/paasta_service"] = service
metadata["labels"][paasta_prefixed("service")] = service
desired_crd = V1beta1CustomResourceDefinition(
api_version=crd_config.get("apiVersion"),
kind=crd_config.get("kind"),
metadata=metadata,
spec=crd_config.get("spec"),
)
desired_crds.append(desired_crd)
return update_crds(
kube_client=kube_client,
desired_crds=desired_crds,
existing_crds=existing_crds,
)
def setup_kube_internal_crd(
kube_client: KubeClient,
) -> bool:
existing_crds = kube_client.apiextensions.list_custom_resource_definition(
label_selector=paasta_prefixed("internal")
)
return update_crds(
kube_client=kube_client, desired_crds=INTERNAL_CRDS, existing_crds=existing_crds
)
def get_existing_kubernetes_service_names(kube_client: KubeClient) -> Set[str]:
service_objects = kube_client.core.list_namespaced_service(
PAASTA_NAMESPACE)
return {
item.metadata.name
for item in service_objects.items
if item.metadata.annotations if item.metadata.annotations.get(
paasta_prefixed("managed_by")) == "setup_istio_mesh"
}
def setup_all_custom_resources(
kube_client: KubeClient,
soa_dir: str,
cluster: str,
custom_resource_definitions: Sequence[CustomResourceDefinition],
service: str = None,
instance: str = None,
) -> bool:
cluster_crds = {
crd.spec.names.kind
for crd in kube_client.apiextensions.list_custom_resource_definition(
label_selector=paasta_prefixed("service")).items
}
log.debug(f"CRDs found: {cluster_crds}")
results = []
for crd in custom_resource_definitions:
if crd.kube_kind.singular not in cluster_crds:
# TODO: kube_kind.singular seems to correspond to `crd.names.kind`
# and not `crd.names.singular`
log.warning(f"CRD {crd.kube_kind.singular} "
f"not found in {cluster}")
continue
# by convention, entries where key begins with _ are used as templates
raw_config_dicts = load_all_configs(cluster=cluster,
file_prefix=crd.file_prefix,
soa_dir=soa_dir)
config_dicts = {}
for svc, raw_sdict in raw_config_dicts.items():
sdict = {
inst: idict
for inst, idict in raw_sdict.items() if inst[0] != "_"
}
if sdict:
config_dicts[svc] = sdict
if not config_dicts:
continue
ensure_namespace(kube_client=kube_client,
namespace=f"paasta-{crd.kube_kind.plural}")
results.append(
setup_custom_resources(
kube_client=kube_client,
kind=crd.kube_kind,
crd=crd,
config_dicts=config_dicts,
version=crd.version,
group=crd.group,
cluster=cluster,
service=service,
instance=instance,
))
return any(results) if results else True
def cleanup_all_custom_resources(
kube_client: KubeClient,
soa_dir: str,
cluster: str,
custom_resource_definitions: Sequence[CustomResourceDefinition],
) -> bool:
cluster_crds = {
crd.spec.names.kind
for crd in kube_client.apiextensions.list_custom_resource_definition(
label_selector=paasta_prefixed("service")).items
}
log.debug(f"CRDs found: {cluster_crds}")
results = []
for crd in custom_resource_definitions:
if crd.kube_kind.singular not in cluster_crds:
# TODO: kube_kind.singular seems to correspond to `crd.names.kind`
# and not `crd.names.singular`
log.warning(f"CRD {crd.kube_kind.singular} "
f"not found in {cluster}")
continue
config_dicts = load_all_configs(cluster=cluster,
file_prefix=crd.file_prefix,
soa_dir=soa_dir)
if not config_dicts:
continue
crs = list_custom_resources(
kube_client=kube_client,
kind=crd.kube_kind,
version=crd.version,
group=crd.group,
)
for cr in crs:
service = config_dicts.get(cr.service)
if service is not None:
instance = service.get(cr.instance)
if instance is not None:
continue
result = False
try:
delete_custom_resource(
kube_client=kube_client,
name=cr.name,
namespace=cr.namespace,
plural=crd.kube_kind.plural,
version=crd.version,
group=crd.group,
)
result = True
except Exception:
log.exception("Error while deleting CR {cr.name}")
results.append(result)
return all(results) if results else True
def is_valid_application(deployment: V1Deployment):
is_valid = True
missing = []
for attr in ["service", "instance", "git_sha", "config_sha"]:
prefixed_attr = paasta_prefixed(attr)
if prefixed_attr not in deployment.metadata.labels:
is_valid = False
missing.append(prefixed_attr)
if missing:
log.warning(f"deployment/{deployment.metadata.name} in "
f"namespace/{deployment.metadata.namespace} "
f"is missing following labels: {missing}")
return is_valid
def format_custom_resource(
instance_config: Mapping[str, Any],
service: str,
instance: str,
cluster: str,
kind: str,
version: str,
group: str,
namespace: str,
git_sha: str,
) -> Mapping[str, Any]:
sanitised_service = sanitise_kubernetes_name(service)
sanitised_instance = sanitise_kubernetes_name(instance)
resource: Mapping[str, Any] = {
"apiVersion": f"{group}/{version}",
"kind": kind,
"metadata": {
"name": f"{sanitised_service}-{sanitised_instance}",
"namespace": namespace,
"labels": {
"yelp.com/paasta_service": service,
"yelp.com/paasta_instance": instance,
"yelp.com/paasta_cluster": cluster,
paasta_prefixed("service"): service,
paasta_prefixed("instance"): instance,
paasta_prefixed("cluster"): cluster,
},
"annotations": {},
},
"spec": instance_config,
}
url = get_dashboard_url(kind, service, instance, cluster)
if url:
resource["metadata"]["annotations"]["yelp.com/dashboard_url"] = url
resource["metadata"]["annotations"][paasta_prefixed(
"dashboard_url")] = url
config_hash = get_config_hash(resource)
resource["metadata"]["annotations"]["yelp.com/desired_state"] = "running"
resource["metadata"]["annotations"][paasta_prefixed(
"desired_state")] = "running"
resource["metadata"]["labels"]["yelp.com/paasta_config_sha"] = config_hash
resource["metadata"]["labels"][paasta_prefixed("config_sha")] = config_hash
resource["metadata"]["labels"][paasta_prefixed("git_sha")] = git_sha
return resource
def cleanup_kube_crd(
kube_client: KubeClient,
cluster: str,
soa_dir: str = DEFAULT_SOA_DIR,
dry_run: bool = False,
) -> bool:
service_attr = paasta_prefixed("service")
existing_crds = kube_client.apiextensions.list_custom_resource_definition(
label_selector=service_attr)
success = True
for crd in existing_crds.items:
service = crd.metadata.labels[service_attr]
if not service:
log.error(
f"CRD {crd.metadata.name} has empty {service_attr} label")
continue
crd_config = service_configuration_lib.read_extra_service_information(
service, f"crd-{cluster}", soa_dir=soa_dir)
if crd_config:
log.debug(
f"CRD {crd.metadata.name} declaration found in {service}")
continue
log.info(f"CRD {crd.metadata.name} not found in {service} service")
if dry_run:
log.info("not deleting in dry-run mode")
continue
try:
kube_client.apiextensions.delete_custom_resource_definition(
name=crd.metadata.name, body=V1DeleteOptions())
log.info(f"deleted {crd.metadata.name} for {cluster}:{service}")
except ApiException as exc:
log.error(f"error deploying crd for {cluster}:{service}, "
f"status: {exc.status}, reason: {exc.reason}")
log.debug(exc.body)
success = False
return success
def get_attribute(node, attribute):
return node.metadata.labels.get(paasta_prefixed(attribute), "unknown")
from paasta_tools.kubernetes_tools import KubeClient
from paasta_tools.kubernetes_tools import paasta_prefixed
from paasta_tools.kubernetes_tools import update_crds
log = logging.getLogger(__name__)
INTERNAL_CRDS = [
V1beta1CustomResourceDefinition(
api_version="apiextensions.k8s.io/v1beta1",
kind="CustomResourceDefinition",
metadata={
"name": "deploygroups.paasta.yelp.com",
"labels": {
paasta_prefixed("internal"): "true",
},
},
spec={
"group": "paasta.yelp.com",
"versions": [{"name": "v1beta1", "served": True, "storage": True}],
"scope": "Namespaced",
"names": {
"plural": "deploygroups",
"singular": "deploygroup",
"kind": "DeployGroup",
"shortNames": ["dg"],
},
"validation": {
"openAPIV3Schema": {
"type": "object",
from paasta_tools.kubernetes_tools import ensure_namespace
from paasta_tools.kubernetes_tools import KubeClient
from paasta_tools.kubernetes_tools import limit_size_with_hash
from paasta_tools.kubernetes_tools import paasta_prefixed
from paasta_tools.kubernetes_tools import registration_label
from paasta_tools.kubernetes_tools import sanitise_kubernetes_name
from paasta_tools.utils import DEFAULT_SOA_DIR
log = logging.getLogger(__name__)
UNIFIED_K8S_SVC_NAME = "paasta-routing"
UNIFIED_SVC_PORT = 1337
PAASTA_SVC_PORT = 8888
PAASTA_NAMESPACE = "paasta"
ANNOTATIONS = {paasta_prefixed("managed_by"): "setup_istio_mesh"}
def parse_args() -> argparse.Namespace:
parser = argparse.ArgumentParser(
description="Creates Kubernetes services.")
parser.add_argument(
"-v",
"--verbose",
action="store_true",
dest="verbose",
default=False,
)
parser.add_argument(
"--dry-run",
action="store_true",
def reconcile_kubernetes_resource(
kube_client: KubeClient,
service: str,
instance_configs: Mapping[str, Any],
custom_resources: Sequence[KubeCustomResource],
kind: KubeKind,
version: str,
group: str,
crd: CustomResourceDefinition,
cluster: str,
instance: str = None,
) -> bool:
succeeded = True
config_handler = LONG_RUNNING_INSTANCE_TYPE_HANDLERS[crd.file_prefix]
for inst, config in instance_configs.items():
if instance is not None and instance != inst:
continue
try:
soa_config = config_handler.loader(
service=service,
instance=inst,
cluster=cluster,
load_deployments=True,
soa_dir=DEFAULT_SOA_DIR,
)
git_sha = get_git_sha_from_dockerurl(soa_config.get_docker_url(),
long=True)
formatted_resource = format_custom_resource(
instance_config=config,
service=service,
instance=inst,
cluster=cluster,
kind=kind.singular,
version=version,
group=group,
namespace=f"paasta-{kind.plural}",
git_sha=git_sha,
)
desired_resource = KubeCustomResource(
service=service,
instance=inst,
config_sha=formatted_resource["metadata"]["labels"][
paasta_prefixed("config_sha")],
git_sha=formatted_resource["metadata"]["labels"].get(
paasta_prefixed("git_sha")),
kind=kind.singular,
name=formatted_resource["metadata"]["name"],
namespace=f"paasta-{kind.plural}",
)
if not (service, inst, kind.singular) in [
(c.service, c.instance, c.kind) for c in custom_resources
]:
log.info(f"{desired_resource} does not exist so creating")
create_custom_resource(
kube_client=kube_client,
version=version,
kind=kind,
formatted_resource=formatted_resource,
group=group,
)
elif desired_resource not in custom_resources:
sanitised_service = sanitise_kubernetes_name(service)
sanitised_instance = sanitise_kubernetes_name(inst)
log.info(
f"{desired_resource} exists but config_sha doesn't match")
update_custom_resource(
kube_client=kube_client,
name=f"{sanitised_service}-{sanitised_instance}",
version=version,
kind=kind,
formatted_resource=formatted_resource,
group=group,
)
else:
log.info(f"{desired_resource} is up to date, no action taken")
except Exception as e:
log.error(str(e))
succeeded = False
return succeeded
def setup_kube_crd(
kube_client: KubeClient,
cluster: str,
services: Sequence[str],
soa_dir: str = DEFAULT_SOA_DIR,
) -> bool:
existing_crds = kube_client.apiextensions.list_custom_resource_definition(
label_selector=paasta_prefixed("service"))
success = True
for service in services:
crd_config = service_configuration_lib.read_extra_service_information(
service, f"crd-{cluster}", soa_dir=soa_dir)
if not crd_config:
log.info("nothing to deploy")
continue
metadata = crd_config.get("metadata", {})
if "labels" not in metadata:
metadata["labels"] = {}
metadata["labels"]["yelp.com/paasta_service"] = service
metadata["labels"][paasta_prefixed("service")] = service
desired_crd = V1beta1CustomResourceDefinition(
api_version=crd_config.get("apiVersion"),
kind=crd_config.get("kind"),
metadata=metadata,
spec=crd_config.get("spec"),
)
existing_crd = None
for crd in existing_crds.items:
if crd.metadata.name == desired_crd.metadata["name"]:
existing_crd = crd
break
try:
if existing_crd:
desired_crd.metadata[
"resourceVersion"] = existing_crd.metadata.resource_version
kube_client.apiextensions.replace_custom_resource_definition(
name=desired_crd.metadata["name"], body=desired_crd)
else:
try:
kube_client.apiextensions.create_custom_resource_definition(
body=desired_crd)
except ValueError as err:
# TODO: kubernetes server will sometimes reply with conditions:null,
# figure out how to deal with this correctly, for more details:
# https://github.com/kubernetes/kubernetes/pull/64996
if "`conditions`, must not be `None`" in str(err):
pass
else:
raise err
log.info(
f"deployed {desired_crd.metadata['name']} for {cluster}:{service}"
)
except ApiException as exc:
log.error(f"error deploying crd for {cluster}:{service}, "
f"status: {exc.status}, reason: {exc.reason}")
log.debug(exc.body)
success = False
return success
