def _configure_namespace(provider_config):
namespace_field = "namespace"
if namespace_field not in provider_config:
raise ValueError("Must specify namespace in Kubernetes config.")
namespace = provider_config[namespace_field]
field_selector = "metadata.name={}".format(namespace)
try:
namespaces = core_api().list_namespace(
field_selector=field_selector).items
except ApiException:
logger.warning(log_prefix +
not_checking_msg(namespace_field, namespace))
return namespace
if len(namespaces) > 0:
assert len(namespaces) == 1
logger.info(log_prefix +
using_existing_msg(namespace_field, namespace))
return namespace
logger.info(log_prefix + not_found_msg(namespace_field, namespace))
namespace_config = client.V1Namespace(
metadata=client.V1ObjectMeta(name=namespace))
core_api().create_namespace(namespace_config)
logger.info(log_prefix + created_msg(namespace_field, namespace))
return namespace
def _configure_services(namespace, provider_config):
service_field = "services"
if service_field not in provider_config:
logger.info(log_prefix + not_provided_msg(service_field))
return
services = provider_config[service_field]
for service in services:
if "namespace" not in service["metadata"]:
service["metadata"]["namespace"] = namespace
elif service["metadata"]["namespace"] != namespace:
raise InvalidNamespaceError(service_field, namespace)
name = service["metadata"]["name"]
field_selector = "metadata.name={}".format(name)
services = core_api().list_namespaced_service(
namespace, field_selector=field_selector).items
if len(services) > 0:
assert len(services) == 1
existing_service = services[0]
if service == existing_service:
logger.info(log_prefix + using_existing_msg("service", name))
return
else:
logger.info(log_prefix +
updating_existing_msg("service", name))
core_api().patch_namespaced_service(name, namespace, service)
else:
logger.info(log_prefix + not_found_msg("service", name))
core_api().create_namespaced_service(namespace, service)
logger.info(log_prefix + created_msg("service", name))
def create_node(self, node_config, tags, count):
conf = copy.deepcopy(node_config)
pod_spec = conf.get("pod", conf)
service_spec = conf.get("service")
ingress_spec = conf.get("ingress")
node_uuid = str(uuid4())
tags[TAG_RAY_CLUSTER_NAME] = self.cluster_name
tags["ray-node-uuid"] = node_uuid
pod_spec["metadata"]["namespace"] = self.namespace
if "labels" in pod_spec["metadata"]:
pod_spec["metadata"]["labels"].update(tags)
else:
pod_spec["metadata"]["labels"] = tags
# Allow Operator-configured service to access the head node.
if tags[TAG_RAY_NODE_KIND] == NODE_KIND_HEAD:
head_selector = head_service_selector(self.cluster_name)
pod_spec["metadata"]["labels"].update(head_selector)
logger.info(
log_prefix + "calling create_namespaced_pod " "(count={}).".format(count)
)
new_nodes = []
for _ in range(count):
pod = core_api().create_namespaced_pod(self.namespace, pod_spec)
new_nodes.append(pod)
new_svcs = []
if service_spec is not None:
logger.info(
log_prefix + "calling create_namespaced_service "
"(count={}).".format(count)
)
for new_node in new_nodes:
metadata = service_spec.get("metadata", {})
metadata["name"] = new_node.metadata.name
service_spec["metadata"] = metadata
service_spec["spec"]["selector"] = {"ray-node-uuid": node_uuid}
svc = core_api().create_namespaced_service(self.namespace, service_spec)
new_svcs.append(svc)
if ingress_spec is not None:
logger.info(
log_prefix + "calling create_namespaced_ingress "
"(count={}).".format(count)
)
for new_svc in new_svcs:
metadata = ingress_spec.get("metadata", {})
metadata["name"] = new_svc.metadata.name
ingress_spec["metadata"] = metadata
ingress_spec = _add_service_name_to_service_port(
ingress_spec, new_svc.metadata.name
)
networking_api().create_namespaced_ingress(self.namespace, ingress_spec)
def non_terminated_nodes(self, tag_filters):
# Match pods that are in the 'Pending' or 'Running' phase.
# Unfortunately there is no OR operator in field selectors, so we
# have to match on NOT any of the other phases.
field_selector = ",".join(
[
"status.phase!=Failed",
"status.phase!=Unknown",
"status.phase!=Succeeded",
"status.phase!=Terminating",
]
)
tag_filters[TAG_RAY_CLUSTER_NAME] = self.cluster_name
label_selector = to_label_selector(tag_filters)
pod_list = core_api().list_namespaced_pod(
self.namespace, field_selector=field_selector, label_selector=label_selector
)
# Don't return pods marked for deletion,
# i.e. pods with non-null metadata.DeletionTimestamp.
return [
pod.metadata.name
for pod in pod_list.items
if pod.metadata.deletion_timestamp is None
]
def terminate_node(self, node_id):
logger.info(log_prefix + "calling delete_namespaced_pod")
try:
core_api().delete_namespaced_pod(node_id, self.namespace)
except ApiException as e:
if e.status == 404:
logger.warning(log_prefix + f"Tried to delete pod {node_id},"
" but the pod was not found (404).")
else:
raise
try:
core_api().delete_namespaced_service(node_id, self.namespace)
except ApiException:
pass
try:
extensions_beta_api().delete_namespaced_ingress(
node_id,
self.namespace,
)
except ApiException:
pass
def _configure_autoscaler_service_account(namespace, provider_config):
account_field = "autoscaler_service_account"
if account_field not in provider_config:
logger.info(log_prefix + not_provided_msg(account_field))
return
account = provider_config[account_field]
if "namespace" not in account["metadata"]:
account["metadata"]["namespace"] = namespace
elif account["metadata"]["namespace"] != namespace:
raise InvalidNamespaceError(account_field, namespace)
name = account["metadata"]["name"]
field_selector = "metadata.name={}".format(name)
accounts = core_api().list_namespaced_service_account(
namespace, field_selector=field_selector).items
if len(accounts) > 0:
assert len(accounts) == 1
logger.info(log_prefix + using_existing_msg(account_field, name))
return
logger.info(log_prefix + not_found_msg(account_field, name))
core_api().create_namespaced_service_account(namespace, account)
logger.info(log_prefix + created_msg(account_field, name))
def internal_ip(self, node_id):
pod = core_api().read_namespaced_pod(node_id, self.namespace)
return pod.status.pod_ip
def node_tags(self, node_id):
pod = core_api().read_namespaced_pod(node_id, self.namespace)
return pod.metadata.labels
def is_terminated(self, node_id):
pod = core_api().read_namespaced_pod(node_id, self.namespace)
return pod.status.phase not in ["Running", "Pending"]
def is_running(self, node_id):
pod = core_api().read_namespaced_pod(node_id, self.namespace)
return pod.status.phase == "Running"
def _set_node_tags(self, node_id, tags):
pod = core_api().read_namespaced_pod(node_id, self.namespace)
pod.metadata.labels.update(tags)
core_api().patch_namespaced_pod(node_id, self.namespace, pod)