def get_affinity(self):
"""Determine the affinity term for the build pod.
There are a two affinity strategies, which one is used depends on how
the BinderHub is configured.
In the default setup the affinity of each build pod is an "anti-affinity"
which causes the pods to prefer to schedule on separate nodes.
In a setup with docker-in-docker enabled pods for a particular
repository prefer to schedule on the same node in order to reuse the
docker layer cache of previous builds.
"""
resp = self.api.list_namespaced_pod(
self.namespace,
label_selector="component=dind,app=binder",
_request_timeout=KUBE_REQUEST_TIMEOUT,
_preload_content=False,
)
dind_pods = json.loads(resp.read())
if self.sticky_builds and dind_pods:
node_names = [
pod["spec"]["nodeName"] for pod in dind_pods["items"]
]
ranked_nodes = rendezvous_rank(node_names, self.repo_url)
best_node_name = ranked_nodes[0]
affinity = client.V1Affinity(node_affinity=client.V1NodeAffinity(
preferred_during_scheduling_ignored_during_execution=[
client.V1PreferredSchedulingTerm(
weight=100,
preference=client.V1NodeSelectorTerm(
match_expressions=[
client.V1NodeSelectorRequirement(
key="kubernetes.io/hostname",
operator="In",
values=[best_node_name],
)
]),
)
]))
else:
affinity = client.V1Affinity(
pod_anti_affinity=client.V1PodAntiAffinity(
preferred_during_scheduling_ignored_during_execution=[
client.V1WeightedPodAffinityTerm(
weight=100,
pod_affinity_term=client.V1PodAffinityTerm(
topology_key="kubernetes.io/hostname",
label_selector=client.V1LabelSelector(
match_labels=dict(
component=self._component_label)),
),
)
]))
return affinity
def create_affinity_term(item):
return client.V1WeightedPodAffinityTerm(
weight=item['weight'],
pod_affinity_term=client.V1PodAffinityTerm(
topology_key=item['label'],
namespaces=item['namespaces'],
label_selector=client.V1LabelSelector(match_expressions=[
create_affinity_selector(s) for s in item['selectors']
], ),
))
def deploy(self, image, name, ns, port, replicas=1, svc_type="NodePort", traffic_policy="Local", cluster_ip=None, ipv6=False):
"""
Creates a deployment and corresponding service with the given
parameters.
"""
# Use a pod anti-affinity so that the scheduler prefers deploying the
# pods on different nodes. This makes our tests more reliable, since
# some tests expect pods to be scheduled to different nodes.
selector = {'matchLabels': {'app': name}}
terms = [client.V1WeightedPodAffinityTerm(
pod_affinity_term=client.V1PodAffinityTerm(
label_selector=selector,
topology_key="kubernetes.io/hostname"),
weight=100,
)]
anti_aff = client.V1PodAntiAffinity(
preferred_during_scheduling_ignored_during_execution=terms)
# Run a deployment with <replicas> copies of <image>, with the
# pods labelled with "app": <name>.
deployment = client.V1Deployment(
api_version="apps/v1",
kind="Deployment",
metadata=client.V1ObjectMeta(name=name),
spec=client.V1DeploymentSpec(
replicas=replicas,
selector=selector,
template=client.V1PodTemplateSpec(
metadata=client.V1ObjectMeta(labels={"app": name}),
spec=client.V1PodSpec(
affinity=client.V1Affinity(pod_anti_affinity=anti_aff),
containers=[
client.V1Container(name=name,
image=image,
ports=[client.V1ContainerPort(container_port=port)]),
]))))
# Create the deployment.
api_response = client.AppsV1Api().create_namespaced_deployment(
body=deployment,
namespace=ns)
logger.debug("Deployment created. status='%s'" % str(api_response.status))
# Create a service called <name> whose endpoints are the pods
# with "app": <name>; i.e. those just created above.
self.create_service(name, name, ns, port, svc_type, traffic_policy, ipv6=ipv6)
def _generate_affinity(self):
return k8s_client.V1Affinity(
node_affinity=k8s_client.V1NodeAffinity(
preferred_during_scheduling_ignored_during_execution=[
k8s_client.V1PreferredSchedulingTerm(
weight=1,
preference=k8s_client.V1NodeSelectorTerm(
match_expressions=[
k8s_client.V1NodeSelectorRequirement(
key="some_node_label",
operator="In",
values=[
"possible-label-value-1",
"possible-label-value-2",
],
)
]),
)
],
required_during_scheduling_ignored_during_execution=k8s_client.
V1NodeSelector(node_selector_terms=[
k8s_client.V1NodeSelectorTerm(match_expressions=[
k8s_client.V1NodeSelectorRequirement(
key="some_node_label",
operator="In",
values=[
"required-label-value-1",
"required-label-value-2",
],
)
]),
]),
),
pod_affinity=k8s_client.V1PodAffinity(
required_during_scheduling_ignored_during_execution=[
k8s_client.V1PodAffinityTerm(
label_selector=k8s_client.V1LabelSelector(
match_labels={
"some-pod-label-key": "some-pod-label-value"
}),
namespaces=["namespace-a", "namespace-b"],
topology_key="key-1",
)
]),
pod_anti_affinity=k8s_client.V1PodAntiAffinity(
preferred_during_scheduling_ignored_during_execution=[
k8s_client.V1WeightedPodAffinityTerm(
weight=1,
pod_affinity_term=k8s_client.V1PodAffinityTerm(
label_selector=k8s_client.V1LabelSelector(
match_expressions=[
k8s_client.V1LabelSelectorRequirement(
key="some_pod_label",
operator="NotIn",
values=[
"forbidden-label-value-1",
"forbidden-label-value-2",
],
)
]),
namespaces=["namespace-c"],
topology_key="key-2",
),
)
]),
)
def submit(self):
"""Submit a image spec to openshift's s2i and wait for completion """
volume_mounts = [
client.V1VolumeMount(mount_path="/var/run/docker.sock",
name="docker-socket")
]
docker_socket_path = urlparse(self.docker_host).path
volumes = [
client.V1Volume(name="docker-socket",
host_path=client.V1HostPathVolumeSource(
path=docker_socket_path, type='Socket'))
]
if self.push_secret:
volume_mounts.append(
client.V1VolumeMount(mount_path="/root/.docker",
name='docker-push-secret'))
volumes.append(
client.V1Volume(name='docker-push-secret',
secret=client.V1SecretVolumeSource(
secret_name=self.push_secret)))
env = []
if self.git_credentials:
env.append(
client.V1EnvVar(name='GIT_CREDENTIAL_ENV',
value=self.git_credentials))
component_label = "binderhub-build"
self.pod = client.V1Pod(
metadata=client.V1ObjectMeta(
name=self.name,
labels={
"name": self.name,
"component": component_label,
},
annotations={
"binder-repo": self.repo_url,
},
),
spec=client.V1PodSpec(
containers=[
client.V1Container(
image=self.build_image,
name="builder",
args=self.get_cmd(),
volume_mounts=volume_mounts,
resources=client.V1ResourceRequirements(
limits={'memory': self.memory_limit},
requests={'memory': self.memory_limit}),
env=env)
],
tolerations=[
client.V1Toleration(
key='hub.jupyter.org/dedicated',
operator='Equal',
value='user',
effect='NoSchedule',
),
# GKE currently does not permit creating taints on a node pool
# with a `/` in the key field
client.V1Toleration(
key='hub.jupyter.org_dedicated',
operator='Equal',
value='user',
effect='NoSchedule',
),
],
node_selector=self.node_selector,
volumes=volumes,
restart_policy="Never",
affinity=client.V1Affinity(
pod_anti_affinity=client.V1PodAntiAffinity(
preferred_during_scheduling_ignored_during_execution=[
client.V1WeightedPodAffinityTerm(
weight=100,
pod_affinity_term=client.V1PodAffinityTerm(
topology_key="kubernetes.io/hostname",
label_selector=client.V1LabelSelector(
match_labels=dict(
component=component_label))))
]))))
try:
ret = self.api.create_namespaced_pod(self.namespace, self.pod)
except client.rest.ApiException as e:
if e.status == 409:
# Someone else created it!
app_log.info("Build %s already running", self.name)
pass
else:
raise
else:
app_log.info("Started build %s", self.name)
app_log.info("Watching build pod %s", self.name)
while not self.stop_event.is_set():
w = watch.Watch()
try:
for f in w.stream(
self.api.list_namespaced_pod,
self.namespace,
label_selector="name={}".format(self.name),
timeout_seconds=30,
):
if f['type'] == 'DELETED':
self.progress('pod.phasechange', 'Deleted')
return
self.pod = f['object']
if not self.stop_event.is_set():
self.progress('pod.phasechange', self.pod.status.phase)
if self.pod.status.phase == 'Succeeded':
self.cleanup()
elif self.pod.status.phase == 'Failed':
self.cleanup()
except Exception as e:
app_log.exception("Error in watch stream for %s", self.name)
raise
finally:
w.stop()
if self.stop_event.is_set():
app_log.info("Stopping watch of %s", self.name)
return
def update_deploy_v2():
data = json.loads(request.get_data().decode('UTF-8'))
current_app.logger.debug("接受到的数据:{}".format(data))
namespace = handle_input(data.get('namespace'))
deploy_name = handle_input(data.get('deploy_name'))
action = handle_input(data.get('action'))
image = None
replicas = None
toleration = None
pod_anti_affinity = None
pod_affinity = None
node_affinity = None
labels = None
if action == "add_pod_anti_affinity":
print("正在运行{}操作".format(action))
affinity = handle_input(data.get('pod_anti_affinity'))
affinity_type = handle_input(affinity.get('type'))
labelSelector = handle_input(affinity.get('labelSelector'))
key = handle_input(affinity.get('key'))
value = handle_input(affinity.get('value'))
topologyKey = handle_input(affinity.get('topologyKey'))
if affinity_type == "required":
if labelSelector == "matchExpressions":
if not isinstance(value, list):
value = [value]
operator = handle_input(affinity.get('operator'))
if operator != 'In' and operator != 'NotIn':
value = None
print(value)
label_selector = client.V1LabelSelector(match_expressions=[
client.V1LabelSelectorRequirement(
key=key, operator=operator, values=value)
])
elif labelSelector == "matchLabels":
if isinstance(value, list):
return jsonify(
{"error": "{}模式下不支持values设置为数组".format(labelSelector)})
label_selector = client.V1LabelSelector(
match_labels={key: value})
else:
return jsonify(
{"error": "不支持{} labelSelector".format(labelSelector)})
client.V1Affinity
pod_anti_affinity = client.V1PodAntiAffinity(
required_during_scheduling_ignored_during_execution=[
client.V1PodAffinityTerm(label_selector=label_selector,
topology_key=topologyKey)
])
print("添加的互斥调度为:{}".format(pod_anti_affinity))
elif affinity_type == "preferred":
weight = string_to_int(handle_input(affinity.get('weight')))
if weight == None:
return jsonify(
{"error": "{}类型必须设置weight".format(affinity_type)})
if labelSelector == "matchExpressions":
if not isinstance(value, list):
value = [value]
operator = handle_input(affinity.get('operator'))
if operator != 'In' and operator != 'NotIn':
value = None
label_selector = client.V1LabelSelector(match_expressions=[
client.V1LabelSelectorRequirement(
key=key, operator=operator, values=value)
])
elif labelSelector == "matchLabels":
if isinstance(value, list):
return jsonify(
{"error": "{}模式下不支持values设置为数组".format(labelSelector)})
label_selector = client.V1LabelSelector(
match_labels={key: value})
else:
return jsonify(
{"error": "不支持{} labelSelector".format(labelSelector)})
pod_anti_affinity = client.V1PodAntiAffinity(
preferred_during_scheduling_ignored_during_execution=[
client.V1WeightedPodAffinityTerm(
pod_affinity_term=client.V1PodAffinityTerm(
label_selector=label_selector,
topology_key=topologyKey),
weight=weight)
])
print("添加的互斥调度为:{}".format(pod_anti_affinity))
else:
return jsonify({"error": "不支持{}这种调度".format(affinity_type)})
elif action == "delete_pod_anti_affinity":
print("正在运行{}操作".format(action))
pass
elif action == "add_node_affinity":
current_app.logger.debug("正在运行{}操作".format(action))
affinity = handle_input(data.get('node_affinity'))
node_affinity_type = handle_input(affinity.get('type'))
nodeSelector = handle_input(affinity.get('nodeSelector'))
key = handle_input(affinity.get('key'))
value = handle_input(affinity.get('value'))
operator = handle_input(affinity.get('operator'))
values = []
if operator == 'Exists' or operator == 'DoesNotExist':
values == None
else:
if not isinstance(value, list):
values.append(value)
else:
values = value
if node_affinity_type == "preferred":
weight = string_to_int(handle_input(affinity.get('weight')))
if weight == None:
return simple_error_handle(
"{}类型必须设置weight".format(node_affinity_type))
preferred_term = []
if nodeSelector == "matchExpressions":
match_expressions = []
expression = client.V1NodeSelectorRequirement(
key=key,
operator=operator,
values=values,
)
match_expressions.append(expression)
preference = client.V1NodeSelectorTerm(
match_expressions=match_expressions)
# nodeSelector == "matchFields"
else:
match_fields = []
field = client.V1NodeSelectorRequirement(
key=key,
operator=operator,
values=values,
)
match_fields.append(field)
preference = client.V1NodeSelectorTerm(
match_fields=match_fields)
term = client.V1PreferredSchedulingTerm(
weight=weight,
preference=preference,
)
preferred_term.append(term)
node_affinity = client.V1NodeAffinity(
#直接append
preferred_during_scheduling_ignored_during_execution=
preferred_term)
elif node_affinity_type == "required":
current_app.logger.debug(
"node_affinity_type:{}".format(node_affinity_type))
node_selector_terms = []
if nodeSelector == "matchExpressions":
match_expressions = []
expression = client.V1NodeSelectorRequirement(
key=key,
operator=operator,
values=values,
)
match_expressions.append(expression)
term = client.V1NodeSelectorTerm(
match_expressions=match_expressions)
else:
match_fields = []
field = client.V1NodeSelectorRequirement(
key=key,
operator=operator,
values=values,
)
match_fields.append(field)
term = client.V1NodeSelectorTerm(match_fields=match_fields)
node_selector_terms.append(term)
node_affinity = client.V1NodeAffinity(
required_during_scheduling_ignored_during_execution=client.
V1NodeSelector(node_selector_terms=node_selector_terms))
else:
return simple_error_handle("不支持{}这种调度".format(node_affinity_type))
elif action == "delete_node_affinity":
print("正在运行{}操作".format(action))
pass
elif action == "add_toleration":
print("正在运行{}操作".format(action))
t = handle_input(data.get("toleration"))
print(type(toleration), toleration)
effect = t.get('effect')
key = t.get('key')
operator = t.get('operator')
value = t.get('value')
toleration_seconds = handle_toleraion_seconds(
t.get('toleration_seconds'))
print("toleration_seconds:{}".format(toleration_seconds))
toleration = client.V1Toleration(effect=effect,
key=key,
operator=operator,
toleration_seconds=toleration_seconds,
value=value)
print(toleration)
if not toleration:
msg = "{}需要提供toleration(effect,key,operator,value,)".format(action)
return jsonify({"error": msg})
elif action == "delete_toleration":
print("正在运行{}操作".format(action))
t = handle_input(data.get("toleration"))
effect = handle_toleration_item(t.get('effect'))
key = handle_toleration_item(t.get('key'))
operator = handle_toleration_item(t.get('operator'))
value = handle_toleration_item(t.get('value'))
toleration_seconds = handle_toleraion_seconds(
t.get('toleration_seconds'))
print("toleration_seconds:{}".format(toleration_seconds))
# if (effect != None and key != None and operator != None):
toleration = client.V1Toleration(effect=effect,
key=key,
operator=operator,
toleration_seconds=toleration_seconds,
value=value)
if not toleration:
msg = "{}需要提供toleration(effect,key,operator,value,)".format(action)
return jsonify({"error": msg})
elif action == "add_pod_affinity":
pass
elif action == "delete_pod_affinity":
pass
elif action == "update_replicas":
replicas = handle_input(data.get('replicas'))
if not replicas:
msg = "{}需要提供replicas".format(action)
return jsonify({"error": msg})
elif action == "update_image":
project = handle_input(data.get('project'))
env = handle_input(data.get('env'))
imageRepo = handle_input(data.get('imageRepo'))
imageName = handle_input(data.get('imageName'))
imageTag = handle_input(data.get('imageTag'))
if (imageRepo != None and project != None and env != None
and imageName != None and imageTag != None):
image = "{}/{}-{}/{}:{}".format(imageRepo, project, env, imageName,
imageTag)
print("image值{}".format(image))
if not image:
msg = "{}需要提供image".format(action)
return jsonify({"error": msg})
elif action == "add_labels":
pass
elif action == "delete_labels":
pass
else:
msg = "暂时不支持{}操作".format(action)
print(msg)
return jsonify({"error": msg})
return update_deployment_v2(deploy_name=deploy_name, namespace=namespace, action=action, image=image, replicas=replicas,toleration=toleration,node_affinity=node_affinity,\
pod_anti_affinity=pod_anti_affinity,pod_affinity=pod_affinity,labels=labels)
def spawn(self, taskdef: TaskDefinition) -> KubernetesTask:
try:
self.emit_sync('prepare', taskdef=taskdef)
volumes, mounts = create_volumes(taskdef.volumes)
# container definition
container = client.V1Container(
name=taskdef.id,
image=taskdef.image,
env=self.create_env(taskdef),
ports=self.create_ports(taskdef),
image_pull_policy='Always', # taskdef field??
resources=client.V1ResourceRequirements(
requests={
'cpu': str(taskdef.cpu or '0'),
'memory': str(taskdef.memory or '0'),
},
limits={
'cpu': str(taskdef.cpu_limit or '0'),
'memory': str(taskdef.memory_limit or '0'),
},
),
volume_mounts=mounts,
)
labels = {
LABEL_TASK_ID: taskdef.id,
LABEL_PARENT_ID: taskdef.parent,
**taskdef.meta,
}
affinity = None
if (taskdef.affinity is not None) and (taskdef.affinity != {}):
affinity_label = {}
if taskdef.affinity.get("label"):
affinity_label[taskdef.affinity["label"][
"key"]] = taskdef.affinity["label"]["value"]
else:
affinity_label[
"cowait_default_affinity_key"] = "cowait_default_affinity_value"
if taskdef.affinity["type"] == 'spread':
aff_def = client.V1PodAntiAffinity(
preferred_during_scheduling_ignored_during_execution=[
client.V1WeightedPodAffinityTerm(
pod_affinity_term=client.V1PodAffinityTerm(
label_selector=client.
V1LabelSelector(match_expressions=[
client.V1LabelSelectorRequirement(
key=list(affinity_label.keys())[0],
operator="In",
values=[
list(affinity_label.values())
[0]
],
)
]),
topology_key="kubernetes.io/hostname",
),
weight=50)
])
elif taskdef.affinity["type"] == 'group':
aff_def = client.V1PodAffinity(
preferred_during_scheduling_ignored_during_execution=[
client.V1WeightedPodAffinityTerm(
pod_affinity_term=client.V1PodAffinityTerm(
label_selector=client.
V1LabelSelector(match_expressions=[
client.V1LabelSelectorRequirement(
key=list(affinity_label.keys())[0],
operator="In",
values=[
list(affinity_label.values())
[0]
],
)
]),
topology_key="kubernetes.io/hostname",
),
weight=50)
])
else:
aff_def = None
affinity = client.V1Affinity(
pod_anti_affinity=aff_def) if aff_def else None
labels[list(affinity_label.keys())[0]] = list(
affinity_label.values())[0]
pod = self.core.create_namespaced_pod(
namespace=self.namespace,
body=client.V1Pod(
metadata=client.V1ObjectMeta(
name=taskdef.id,
namespace=self.namespace,
labels=labels,
),
spec=client.V1PodSpec(
hostname=taskdef.id,
restart_policy='Never',
image_pull_secrets=self.get_pull_secrets(),
volumes=volumes,
affinity=affinity,
containers=[container],
service_account_name=self.service_account,
),
),
)
# wrap & return task
# print('~~ created kubenetes pod', pod.metadata.name)
task = KubernetesTask(self, taskdef, pod)
self.emit_sync('spawn', task=task)
return task
except urllib3.exceptions.MaxRetryError:
raise ProviderError('Kubernetes engine unavailable')
def create_run_pod(k8s_settings, run_context):
run_id = run_context.id
run_name = run_context.run.to_json()["name"]
labels = {
"run-name": run_name,
"run": run_id,
}
env = get_run_pod_env_vars(run_context)
node_topology_key = "kubernetes.io/hostname"
# NOTE(taylor): preference to run on nodes with other runs
pod_affinities = [
k8s_client.V1WeightedPodAffinityTerm(
weight=50,
pod_affinity_term=k8s_client.V1PodAffinityTerm(
label_selector=k8s_client.V1LabelSelector(match_labels={
"type": "run",
}, ),
topology_key=node_topology_key,
),
),
]
volumes = []
volume_mounts = []
experiment_id = run_context.experiment
if experiment_id:
labels.update({"experiment": experiment_id})
# NOTE(taylor): highest preference to run on nodes with runs in the same experiment
pod_affinities.append(
k8s_client.V1WeightedPodAffinityTerm(
weight=100,
pod_affinity_term=k8s_client.V1PodAffinityTerm(
label_selector=k8s_client.V1LabelSelector(match_labels={
"type":
"run",
"experiment":
experiment_id,
}, ),
topology_key=node_topology_key,
),
))
unacceptable_node_group_types = ["system"]
requests = k8s_settings.resources.get("requests") or {}
limits = k8s_settings.resources.get("limits") or {}
# NOTE(taylor): Preventing GPU-less jobs from running on GPU nodes forces the cluster autoscaler to scale up
# CPU nodes. This prevents a situation where the GPU nodes are not scaled down because they are occupied by
# CPU workloads. The cluster autoscaler does not know that it should create CPU nodes when the GPUs are unused.
# TODO(taylor): This could cause unexpected behavior if the cluster has no CPU nodes. Running CPU jobs on GPU
# nodes could also be an opportunity for more efficient resource utilization, but is avoided for now because the
# workloads cannot be migrated onto CPU nodes by the cluster autoscaler as mentioned above.
# NOTE(taylor): Applying a NoSchedule taint to GPU nodes is another way to achieve this behavior, but does not work as
# well out of the box with clusters that orchestrate doesn't provision. Applying a PreferNoSchedule
# taint to GPU nodes does not resolve the workload migration issue when there are no CPU nodes.
if all(
float(group.get("nvidia.com/gpu", 0)) == 0
for group in (requests, limits)):
unacceptable_node_group_types.append("gpu")
node_affinity = k8s_client.V1NodeAffinity(
required_during_scheduling_ignored_during_execution=k8s_client.
V1NodeSelector(node_selector_terms=[
k8s_client.V1NodeSelectorTerm(match_expressions=[
k8s_client.V1NodeSelectorRequirement(
key="orchestrate.sigopt.com/node-group-type",
operator="NotIn",
values=unacceptable_node_group_types,
)
], )
], ), )
pod_affinity = k8s_client.V1PodAffinity(
preferred_during_scheduling_ignored_during_execution=pod_affinities, )
pod = k8s_client.V1Pod(
metadata=k8s_client.V1ObjectMeta(
owner_references=k8s_settings.owner_references,
labels={
"type": "run",
**labels,
},
name=run_name,
),
spec=k8s_client.V1PodSpec(
affinity=k8s_client.V1Affinity(
node_affinity=node_affinity,
pod_affinity=pod_affinity,
),
containers=[
k8s_client.V1Container(
name="model-runner",
image=k8s_settings.image,
resources=k8s_client.V1ResourceRequirements(
**k8s_settings.resources),
image_pull_policy="Always",
command=[],
args=k8s_settings.args,
env=env,
volume_mounts=volume_mounts,
tty=True,
),
],
volumes=volumes,
restart_policy="Never",
),
)
k8s_settings.api.create_namespaced_pod(k8s_settings.namespace, pod)
return pod
def apply_rekcurd_to_kubernetes(project_id: int,
application_id: str,
service_level: str,
version: str,
insecure_host: str,
insecure_port: int,
replicas_default: int,
replicas_minimum: int,
replicas_maximum: int,
autoscale_cpu_threshold: str,
policy_max_surge: int,
policy_max_unavailable: int,
policy_wait_seconds: int,
container_image: str,
resource_request_cpu: str,
resource_request_memory: str,
resource_limit_cpu: str,
resource_limit_memory: str,
commit_message: str,
service_model_assignment: int,
service_git_url: str = "",
service_git_branch: str = "",
service_boot_script: str = "",
debug_mode: bool = False,
service_id: str = None,
is_creation_mode: bool = False,
display_name: str = None,
description: str = None,
kubernetes_models=None,
**kwargs) -> str:
"""
kubectl apply
:param project_id:
:param application_id:
:param service_level:
:param version:
:param insecure_host:
:param insecure_port:
:param replicas_default:
:param replicas_minimum:
:param replicas_maximum:
:param autoscale_cpu_threshold:
:param policy_max_surge:
:param policy_max_unavailable:
:param policy_wait_seconds:
:param container_image:
:param resource_request_cpu:
:param resource_request_memory:
:param resource_limit_cpu:
:param resource_limit_memory:
:param commit_message:
:param service_model_assignment:
:param service_git_url:
:param service_git_branch:
:param service_boot_script:
:param debug_mode:
:param service_id:
:param is_creation_mode:
:param display_name:
:param description:
:param kubernetes_models:
:param kwargs:
:return:
"""
__num_retry = 5
progress_deadline_seconds = \
int(__num_retry*policy_wait_seconds*replicas_maximum/(policy_max_surge+policy_max_unavailable))
if service_id is None:
is_creation_mode = True
service_id = uuid.uuid4().hex
if kubernetes_models is None:
kubernetes_models = db.session.query(KubernetesModel).filter(
KubernetesModel.project_id == project_id).all()
data_server_model: DataServerModel = db.session.query(
DataServerModel).filter(
DataServerModel.project_id == project_id).first_or_404()
application_model: ApplicationModel = db.session.query(
ApplicationModel).filter(
ApplicationModel.application_id == application_id).first_or_404()
application_name = application_model.application_name
model_model: ModelModel = db.session.query(ModelModel).filter(
ModelModel.model_id == service_model_assignment).first_or_404()
from kubernetes import client
try:
git_secret = load_secret(project_id, application_id, service_level,
GIT_SECRET_PREFIX)
except:
git_secret = None
volume_mounts = dict()
volumes = dict()
if git_secret:
connector_name = "sec-git-name"
secret_name = "sec-{}-{}".format(GIT_SECRET_PREFIX, application_id)
volume_mounts = {
'volume_mounts': [
client.V1VolumeMount(name=connector_name,
mount_path=GIT_SSH_MOUNT_DIR,
read_only=True)
]
}
volumes = {
'volumes': [
client.V1Volume(name=connector_name,
secret=client.V1SecretVolumeSource(
secret_name=secret_name,
items=[
client.V1KeyToPath(key=GIT_ID_RSA,
path=GIT_ID_RSA,
mode=GIT_SSH_MODE),
client.V1KeyToPath(key=GIT_CONFIG,
path=GIT_CONFIG,
mode=GIT_SSH_MODE)
]))
]
}
for kubernetes_model in kubernetes_models:
full_config_path = get_full_config_path(kubernetes_model.config_path)
from kubernetes import config
config.load_kube_config(full_config_path)
pod_env = [
client.V1EnvVar(name="REKCURD_SERVICE_UPDATE_FLAG",
value=commit_message),
client.V1EnvVar(name="REKCURD_KUBERNETES_MODE", value="True"),
client.V1EnvVar(name="REKCURD_DEBUG_MODE", value=str(debug_mode)),
client.V1EnvVar(name="REKCURD_APPLICATION_NAME",
value=application_name),
client.V1EnvVar(name="REKCURD_SERVICE_INSECURE_HOST",
value=insecure_host),
client.V1EnvVar(name="REKCURD_SERVICE_INSECURE_PORT",
value=str(insecure_port)),
client.V1EnvVar(name="REKCURD_SERVICE_ID", value=service_id),
client.V1EnvVar(name="REKCURD_SERVICE_LEVEL", value=service_level),
client.V1EnvVar(name="REKCURD_GRPC_PROTO_VERSION", value=version),
client.V1EnvVar(name="REKCURD_MODEL_MODE",
value=data_server_model.data_server_mode.value),
client.V1EnvVar(name="REKCURD_MODEL_FILE_PATH",
value=model_model.filepath),
client.V1EnvVar(name="REKCURD_CEPH_ACCESS_KEY",
value=str(data_server_model.ceph_access_key
or "xxx")),
client.V1EnvVar(name="REKCURD_CEPH_SECRET_KEY",
value=str(data_server_model.ceph_secret_key
or "xxx")),
client.V1EnvVar(name="REKCURD_CEPH_HOST",
value=str(data_server_model.ceph_host or "xxx")),
client.V1EnvVar(name="REKCURD_CEPH_PORT",
value=str(data_server_model.ceph_port or "1234")),
client.V1EnvVar(name="REKCURD_CEPH_IS_SECURE",
value=str(data_server_model.ceph_is_secure
or "False")),
client.V1EnvVar(name="REKCURD_CEPH_BUCKET_NAME",
value=str(data_server_model.ceph_bucket_name
or "xxx")),
client.V1EnvVar(name="REKCURD_AWS_ACCESS_KEY",
value=str(data_server_model.aws_access_key
or "xxx")),
client.V1EnvVar(name="REKCURD_AWS_SECRET_KEY",
value=str(data_server_model.aws_secret_key
or "xxx")),
client.V1EnvVar(name="REKCURD_AWS_BUCKET_NAME",
value=str(data_server_model.aws_bucket_name
or "xxx")),
client.V1EnvVar(name="REKCURD_GCS_ACCESS_KEY",
value=str(data_server_model.gcs_access_key
or "xxx")),
client.V1EnvVar(name="REKCURD_GCS_SECRET_KEY",
value=str(data_server_model.gcs_secret_key
or "xxx")),
client.V1EnvVar(name="REKCURD_GCS_BUCKET_NAME",
value=str(data_server_model.gcs_bucket_name
or "xxx")),
client.V1EnvVar(name="REKCURD_SERVICE_GIT_URL",
value=service_git_url),
client.V1EnvVar(name="REKCURD_SERVICE_GIT_BRANCH",
value=service_git_branch),
client.V1EnvVar(name="REKCURD_SERVICE_BOOT_SHELL",
value=service_boot_script),
]
"""Namespace registration."""
core_vi_api = client.CoreV1Api()
try:
core_vi_api.read_namespace(name=service_level)
except:
api.logger.info("\"{}\" namespace created".format(service_level))
v1_namespace = client.V1Namespace(
api_version="v1",
kind="Namespace",
metadata=client.V1ObjectMeta(name=service_level))
core_vi_api.create_namespace(body=v1_namespace)
"""Create/patch Deployment."""
v1_deployment = client.V1Deployment(
api_version="apps/v1",
kind="Deployment",
metadata=client.V1ObjectMeta(name="deploy-{0}".format(service_id),
namespace=service_level,
labels={
"rekcurd-worker": "True",
"id": application_id,
"name": application_name,
"sel": service_id
}),
spec=client.V1DeploymentSpec(
min_ready_seconds=policy_wait_seconds,
progress_deadline_seconds=progress_deadline_seconds,
replicas=replicas_default,
revision_history_limit=3,
selector=client.V1LabelSelector(
match_labels={"sel": service_id}),
strategy=client.V1DeploymentStrategy(
type="RollingUpdate",
rolling_update=client.V1RollingUpdateDeployment(
max_surge=policy_max_surge,
max_unavailable=policy_max_unavailable)),
template=client.V1PodTemplateSpec(
metadata=client.V1ObjectMeta(
labels={
"rekcurd-worker": "True",
"id": application_id,
"name": application_name,
"sel": service_id
}),
spec=client.V1PodSpec(affinity=client.V1Affinity(
pod_anti_affinity=client.V1PodAntiAffinity(
preferred_during_scheduling_ignored_during_execution
=[
client.V1WeightedPodAffinityTerm(
pod_affinity_term=client.V1PodAffinityTerm(
label_selector=client.
V1LabelSelector(match_expressions=[
client.V1LabelSelectorRequirement(
key="id",
operator="In",
values=[service_id])
]),
topology_key="kubernetes.io/hostname"),
weight=100)
])),
containers=[
client.V1Container(
env=pod_env,
image=container_image,
image_pull_policy="Always",
name=service_id,
ports=[
client.V1ContainerPort(
container_port=
insecure_port)
],
resources=client.
V1ResourceRequirements(
limits={
"cpu":
str(resource_limit_cpu
),
"memory":
resource_limit_memory
},
requests={
"cpu":
str(resource_request_cpu
),
"memory":
resource_request_memory
}),
security_context=client.
V1SecurityContext(
privileged=True),
**volume_mounts)
],
node_selector={
"host": service_level
},
**volumes))))
apps_v1_api = client.AppsV1Api()
if is_creation_mode:
api.logger.info("Deployment created.")
apps_v1_api.create_namespaced_deployment(body=v1_deployment,
namespace=service_level)
else:
api.logger.info("Deployment patched.")
apps_v1_api.patch_namespaced_deployment(
body=v1_deployment,
name="deploy-{0}".format(service_id),
namespace=service_level)
"""Create/patch Service."""
v1_service = client.V1Service(
api_version="v1",
kind="Service",
metadata=client.V1ObjectMeta(name="svc-{0}".format(service_id),
namespace=service_level,
labels={
"rekcurd-worker": "True",
"id": application_id,
"name": application_name,
"sel": service_id
}),
spec=client.V1ServiceSpec(ports=[
client.V1ServicePort(name="grpc-backend",
port=insecure_port,
protocol="TCP",
target_port=insecure_port)
],
selector={"sel": service_id}))
core_vi_api = client.CoreV1Api()
if is_creation_mode:
api.logger.info("Service created.")
core_vi_api.create_namespaced_service(namespace=service_level,
body=v1_service)
else:
api.logger.info("Service patched.")
core_vi_api.patch_namespaced_service(
namespace=service_level,
name="svc-{0}".format(service_id),
body=v1_service)
"""Create/patch Autoscaler."""
v1_horizontal_pod_autoscaler = client.V1HorizontalPodAutoscaler(
api_version="autoscaling/v1",
kind="HorizontalPodAutoscaler",
metadata=client.V1ObjectMeta(name="hpa-{0}".format(service_id),
namespace=service_level,
labels={
"rekcurd-worker": "True",
"id": application_id,
"name": application_name,
"sel": service_id
}),
spec=client.V1HorizontalPodAutoscalerSpec(
max_replicas=replicas_maximum,
min_replicas=replicas_minimum,
scale_target_ref=client.V1CrossVersionObjectReference(
api_version="apps/v1",
kind="Deployment",
name="deploy-{0}".format(service_id)),
target_cpu_utilization_percentage=autoscale_cpu_threshold))
autoscaling_v1_api = client.AutoscalingV1Api()
if is_creation_mode:
api.logger.info("Autoscaler created.")
autoscaling_v1_api.create_namespaced_horizontal_pod_autoscaler(
namespace=service_level, body=v1_horizontal_pod_autoscaler)
else:
api.logger.info("Autoscaler patched.")
autoscaling_v1_api.patch_namespaced_horizontal_pod_autoscaler(
namespace=service_level,
name="hpa-{0}".format(service_id),
body=v1_horizontal_pod_autoscaler)
"""Create Istio ingress if this is the first application."""
custom_object_api = client.CustomObjectsApi()
try:
custom_object_api.get_namespaced_custom_object(
group="networking.istio.io",
version="v1alpha3",
namespace=service_level,
plural="virtualservices",
name="ing-vs-{0}".format(application_id),
)
except:
ingress_virtual_service_body = {
"apiVersion": "networking.istio.io/v1alpha3",
"kind": "VirtualService",
"metadata": {
"labels": {
"rekcurd-worker": "True",
"id": application_id,
"name": application_name
},
"name": "ing-vs-{0}".format(application_id),
"namespace": service_level
},
"spec": {
"hosts": ["*"],
"gateways": ["rekcurd-ingress-gateway"],
"http": [{
"match": [{
"headers": {
"x-rekcurd-application-name": {
"exact": application_name
},
"x-rekcurd-sevice-level": {
"exact": service_level
},
"x-rekcurd-grpc-version": {
"exact": version
},
}
}],
"route": [{
"destination": {
"port": {
"number": insecure_port
},
"host": "svc-{0}".format(service_id)
},
"weight": 100
}],
"retries": {
"attempts": 25,
"perTryTimeout": "1s"
}
}]
}
}
api.logger.info("Istio created.")
custom_object_api.create_namespaced_custom_object(
group="networking.istio.io",
version="v1alpha3",
namespace=service_level,
plural="virtualservices",
body=ingress_virtual_service_body)
"""Add service model."""
if is_creation_mode:
if display_name is None:
display_name = "{0}-{1}".format(service_level, service_id)
service_model = ServiceModel(service_id=service_id,
application_id=application_id,
display_name=display_name,
description=description,
service_level=service_level,
version=version,
model_id=service_model_assignment,
insecure_host=insecure_host,
insecure_port=insecure_port)
db.session.add(service_model)
db.session.flush()
"""Finish."""
return service_id
def get_statefulset_object(cluster_object):
name = cluster_object['metadata']['name']
namespace = cluster_object['metadata']['namespace']
try:
replicas = cluster_object['spec']['mongodb']['replicas']
except KeyError:
replicas = 3
try:
mongodb_limit_cpu = \
cluster_object['spec']['mongodb']['mongodb_limit_cpu']
except KeyError:
mongodb_limit_cpu = '100m'
try:
mongodb_limit_memory = \
cluster_object['spec']['mongodb']['mongodb_limit_memory']
except KeyError:
mongodb_limit_memory = '64Mi'
try:
hard_pod_anti_affinity = \
cluster_object['spec']['mongodb']['hard_pod_anti_affinity']
except KeyError:
hard_pod_anti_affinity = True
statefulset = client.V1beta1StatefulSet()
# Metadata
statefulset.metadata = client.V1ObjectMeta(
name=name, namespace=namespace, labels=get_default_labels(name=name))
# Spec
statefulset.spec = client.V1beta1StatefulSetSpec(
replicas=replicas,
service_name=name,
template=client.V1PodTemplateSpec())
statefulset.spec.template.metadata = client.V1ObjectMeta(
labels=get_default_labels(name=name))
statefulset.spec.template.spec = client.V1PodSpec(containers=[])
pod_affinity_term = client.V1PodAffinityTerm(
topology_key='kubernetes.io/hostname',
label_selector=client.V1LabelSelector(match_expressions=[
client.V1LabelSelectorRequirement(
key='cluster', operator='In', values=[name])
]))
pod_anti_affinity = client.V1PodAntiAffinity(
required_during_scheduling_ignored_during_execution=[
pod_affinity_term
])
if not hard_pod_anti_affinity:
pod_anti_affinity = client.V1PodAntiAffinity(
preferred_during_scheduling_ignored_during_execution=[
client.V1WeightedPodAffinityTerm(
weight=100, pod_affinity_term=pod_affinity_term)
])
statefulset.spec.template.spec.affinity = client.V1Affinity(
pod_anti_affinity=pod_anti_affinity)
# MongoDB container
mongodb_port = client.V1ContainerPort(name='mongodb',
container_port=27017,
protocol='TCP')
mongodb_tls_volumemount = client.V1VolumeMount(
name='mongo-tls', read_only=True, mount_path='/etc/ssl/mongod')
mongodb_data_volumemount = client.V1VolumeMount(name='mongo-data',
read_only=False,
mount_path='/data/db')
mongodb_resources = client.V1ResourceRequirements(limits={
'cpu':
mongodb_limit_cpu,
'memory':
mongodb_limit_memory
},
requests={
'cpu':
mongodb_limit_cpu,
'memory':
mongodb_limit_memory
})
mongodb_container = client.V1Container(
name='mongod',
env=[
client.V1EnvVar(
name='POD_IP',
value_from=client.V1EnvVarSource(
field_ref=client.V1ObjectFieldSelector(
api_version='v1', field_path='status.podIP')))
],
command=[
'mongod', '--auth', '--replSet', name, '--sslMode', 'requireSSL',
'--clusterAuthMode', 'x509', '--sslPEMKeyFile',
'/etc/ssl/mongod/mongod.pem', '--sslCAFile',
'/etc/ssl/mongod/ca.pem', '--bind_ip', '127.0.0.1,$(POD_IP)'
],
image='mongo:3.6.4',
ports=[mongodb_port],
volume_mounts=[mongodb_tls_volumemount, mongodb_data_volumemount],
resources=mongodb_resources)
# Metrics container
metrics_port = client.V1ContainerPort(name='metrics',
container_port=9001,
protocol='TCP')
metrics_resources = client.V1ResourceRequirements(limits={
'cpu': '50m',
'memory': '16Mi'
},
requests={
'cpu': '50m',
'memory': '16Mi'
})
metrics_secret_name = '{}-monitoring-credentials'.format(name)
metrics_username_env_var = client.V1EnvVar(
name='MONGODB_MONITORING_USERNAME',
value_from=client.V1EnvVarSource(
secret_key_ref=client.V1SecretKeySelector(name=metrics_secret_name,
key='username')))
metrics_password_env_var = client.V1EnvVar(
name='MONGODB_MONITORING_PASSWORD',
value_from=client.V1EnvVarSource(
secret_key_ref=client.V1SecretKeySelector(name=metrics_secret_name,
key='password')))
metrics_container = client.V1Container(
name='prometheus-exporter',
image='quay.io/kubestack/prometheus-mongodb-exporter:latest',
command=[
'/bin/sh', '-c',
'/bin/mongodb_exporter --mongodb.uri mongodb://${MONGODB_MONITORING_USERNAME}:${MONGODB_MONITORING_PASSWORD}@127.0.0.1:27017/admin --mongodb.tls-cert /etc/ssl/mongod/mongod.pem --mongodb.tls-ca /etc/ssl/mongod/ca.pem'
], # flake8: noqa
ports=[metrics_port],
resources=metrics_resources,
volume_mounts=[mongodb_tls_volumemount],
env=[metrics_username_env_var, metrics_password_env_var])
statefulset.spec.template.spec.containers = [
mongodb_container, metrics_container
]
ca_volume = client.V1Volume(name='mongo-ca',
secret=client.V1SecretVolumeSource(
secret_name='{}-ca'.format(name),
items=[
client.V1KeyToPath(key='ca.pem',
path='ca.pem'),
client.V1KeyToPath(key='ca-key.pem',
path='ca-key.pem')
]))
tls_volume = client.V1Volume(name='mongo-tls',
empty_dir=client.V1EmptyDirVolumeSource())
data_volume = client.V1Volume(name='mongo-data',
empty_dir=client.V1EmptyDirVolumeSource())
statefulset.spec.template.spec.volumes = [
ca_volume, tls_volume, data_volume
]
# Init container
tls_init_ca_volumemount = client.V1VolumeMount(
name='mongo-ca', read_only=True, mount_path='/etc/ssl/mongod-ca')
tls_init_mongodb_tls_volumemount = client.V1VolumeMount(
name='mongo-tls', read_only=False, mount_path='/etc/ssl/mongod')
tls_init_container = client.V1Container(
name="cert-init",
image="quay.io/kubestack/mongodb-init:latest",
volume_mounts=[
tls_init_ca_volumemount, tls_init_mongodb_tls_volumemount
],
env=[
client.V1EnvVar(
name='METADATA_NAME',
value_from=client.V1EnvVarSource(
field_ref=client.V1ObjectFieldSelector(
api_version='v1', field_path='metadata.name'))),
client.V1EnvVar(
name='NAMESPACE',
value_from=client.V1EnvVarSource(
field_ref=client.V1ObjectFieldSelector(
api_version='v1', field_path='metadata.namespace')))
],
command=["ansible-playbook", "member-cert.yml"])
statefulset.spec.template.spec.init_containers = [tls_init_container]
return statefulset
