def create_stateful_set_object():
container = client.V1Container(
name="sts-redis",
image="redis",
image_pull_policy="IfNotPresent",
ports=[client.V1ContainerPort(container_port=6379)],
)
# Template
template = client.V1PodTemplateSpec(
metadata=client.V1ObjectMeta(labels={"app": "redis"}),
spec=client.V1PodSpec(containers=[container]))
# Spec
spec = client.V1StatefulSetSpec(
replicas=1,
service_name="redis-test-svc",
selector=client.V1LabelSelector(match_labels={"app": "redis"}),
template=template)
# StatefulSet
statefulset = client.V1StatefulSet(
api_version="apps/v1",
kind="StatefulSet",
metadata=client.V1ObjectMeta(name="statefulset-redis"),
spec=spec)
return statefulset
def create_stateful_set(self, stsName, namespace, replicas, containers,
volumes, volumeClaimTemplates):
api_version = 'apps/v1'
kind = 'StatefulSet'
metadata = client.V1ObjectMeta(name=stsName, namespace=namespace)
# Build spec_selector
spec_selector_match_labels = dict()
spec_selector_match_labels['name'] = stsName
spec_selector_match_labels['namespace'] = namespace
spec_selector = client.V1LabelSelector(
match_labels=spec_selector_match_labels)
# Build spec_template
spec_template_metadata_labels = dict()
spec_template_metadata_labels['name'] = stsName
spec_template_metadata_labels['namespace'] = namespace
spec_template_metadata = client.V1ObjectMeta(
labels=spec_template_metadata_labels)
spec_template_spec = client.V1PodSpec(containers=containers,
volumes=volumes)
spec_template = client.V1PodTemplateSpec(
metadata=spec_template_metadata, spec=spec_template_spec)
# Build spec
spec = client.V1StatefulSetSpec(
service_name=stsName,
replicas=replicas,
selector=spec_selector,
template=spec_template,
volume_claim_templates=volumeClaimTemplates)
# Build body
body = client.V1StatefulSet(api_version=api_version,
kind=kind,
metadata=metadata,
spec=spec)
# Create stateful set
try:
api_response = self.appsApi.create_namespaced_stateful_set(
namespace=namespace, body=body)
print('api_response: ', api_response)
except ApiException as e:
return hiss.hiss(
"Exception when calling AppsV1Api->create_namespaced_stateful_set: %s\n"
% e)
def create_sts_spec(node_name, workload_type):
sts_name = STS_PREFIX + node_name
cmd = get_workload_command(workload_type, sts_name)
container = client.V1Container(
name=sts_name,
image=IMAGE,
command=["/bin/bash"],
args=["-c", cmd],
liveness_probe=client.V1Probe(
_exec=client.V1ExecAction(command=["ls", "/mnt/" + sts_name]),
initial_delay_seconds=5,
period_seconds=5),
volume_mounts=[
client.V1VolumeMount(name=sts_name, mount_path="/mnt/" + sts_name)
])
template = client.V1PodTemplateSpec(
metadata=client.V1ObjectMeta(labels={"app": sts_name}),
spec=client.V1PodSpec(
node_name=node_name,
restart_policy="Always",
termination_grace_period_seconds=10,
containers=[container],
))
spec = client.V1StatefulSetSpec(
replicas=0,
service_name=sts_name,
selector=client.V1LabelSelector(match_labels={"app": sts_name}),
template=template,
volume_claim_templates=[
client.V1PersistentVolumeClaim(
metadata=client.V1ObjectMeta(name=sts_name),
spec=client.V1PersistentVolumeClaimSpec(
access_modes=["ReadWriteOnce"],
storage_class_name="longhorn",
resources=client.V1ResourceRequirements(
requests={"storage": "4Gi"})))
])
statefulset = client.V1StatefulSet(
api_version="apps/v1",
kind="StatefulSet",
metadata=client.V1ObjectMeta(name=sts_name),
spec=spec)
statefulset.spec.replicas
return statefulset
def fake_v1_stateful_set_error():
return client.V1StatefulSet(
api_version='apps/v1',
kind='StatefulSet',
metadata=client.V1ObjectMeta(name='curry-test001',
namespace='curryns'),
spec=client.V1StatefulSetSpec(
replicas=1,
volume_claim_templates=[
client.V1PersistentVolumeClaim(metadata=client.V1ObjectMeta(
name='www'))
],
selector=client.V1LabelSelector(match_labels={'app': 'nginx'}),
template=client.V1PodTemplateSpec(metadata=client.V1ObjectMeta(
name='curry-test001', namespace='curryns')),
service_name='nginx'),
status=client.V1StatefulSetStatus(replicas=2, ready_replicas=1))
def simple_statefulset():
"""Return the Kubernetes config matching the simple-statefulset.yaml manifest."""
return client.V1StatefulSet(
api_version='apps/v1',
kind='StatefulSet',
metadata=client.V1ObjectMeta(name='postgres-statefulset',
labels={'app': 'postgres'}),
spec=client.V1StatefulSetSpec(
replicas=3,
selector=client.V1LabelSelector(match_labels={'app': 'postgres'}),
service_name='simple-service',
template=client.V1PodTemplateSpec(
metadata=client.V1ObjectMeta(labels={'app': 'postgres'}),
spec=client.V1PodSpec(containers=[
client.V1Container(
name='postgres',
image='postgres:9.6',
ports=[client.V1ContainerPort(container_port=5432)])
]))))
def ensure_statefulset_with_containers(api_apps_v1,
name,
namespace,
containers,
volume_paths,
replicas=1,
init_containers=None,
volumes=None):
if volumes is None:
volumes = []
if init_containers is None:
init_containers = []
volume_claim_templates = [
V1PersistentVolumeClaim(metadata=V1ObjectMeta(name=path[0]),
spec=V1PersistentVolumeClaimSpec(
access_modes=['ReadWriteOnce'],
resources=V1ResourceRequirements(
requests={'storage': path[2]}),
storage_class_name=path[3]))
for path in volume_paths
]
ss = client.V1StatefulSet(
api_version="apps/v1",
kind="StatefulSet",
metadata=client.V1ObjectMeta(name=name, labels={'app': name}),
spec=client.V1StatefulSetSpec(
replicas=replicas,
service_name=name,
template=V1PodTemplateSpec(
metadata=V1ObjectMeta(labels={"app": name}),
spec=V1PodSpec(containers=containers,
volumes=volumes,
init_containers=init_containers)),
selector={'matchLabels': {
'app': name
}},
volume_claim_templates=volume_claim_templates))
ensure_statefulset(api_apps_v1,
stateful_set=ss,
namespace=namespace,
name=name)
def create_nifi_ss_object():
container = client.V1Container(
name="apache",
image="apache/nifi:1.9.2",
env=[client.V1EnvVar(name='NIFI_WEB_HTTP_HOST', value='0.0.0.0')],
ports=[client.V1ContainerPort(container_port=8080)])
template = client.V1PodTemplateSpec(
metadata=client.V1ObjectMeta(labels={"app": "nifi"}),
spec=client.V1PodSpec(containers=[container]))
spec = client.V1StatefulSetSpec(replicas=1,
service_name='nifi',
template=template,
selector={'matchLabels': {
'app': 'nifi'
}})
ss = client.V1StatefulSet(api_version="apps/v1",
kind="StatefulSet",
metadata=client.V1ObjectMeta(name='nifi'),
spec=spec)
return ss
def create_deployment(namespace,
image_name,
name,
container_port=None,
environment=None):
api_instance_apps = kubernetes.client.AppsV1Api()
pod_spec = client.V1PodSpec(containers=[
client.V1Container(
name=name,
image=image_name,
ports=[client.V1ContainerPort(container_port=container_port)]
if container_port is not None else None,
# ports=[client.V1ContainerPort(container_port=container_port)],
env=environment)
])
pod_template = client.V1PodTemplateSpec(metadata=client.V1ObjectMeta(
labels={"broker": str(image_name.split("/")[-1])}),
spec=pod_spec)
statefulset_spec = client.V1StatefulSetSpec(
replicas=1,
service_name="svc-%s" % str(image_name.split("/")[-1]),
selector=client.V1LabelSelector(
match_labels={"broker": str(image_name.split("/")[-1])}),
template=pod_template)
statefulset_body = client.V1StatefulSet(
api_version="apps/v1",
kind="StatefulSet",
metadata=client.V1ObjectMeta(name=name),
spec=statefulset_spec)
try:
return api_instance_apps.create_namespaced_deployment(namespace,
statefulset_body,
pretty="true")
except Exception as err:
return err
def stateful_set(self, name, image, command, env, vmounts, vol, vol_claims=None, sec=None, args=None, ports=None, replicas=1, patch=False):
"""Create and upload StatefulSet, patch option also available."""
sts_conf = client.V1StatefulSet()
sts_conf.metadata = client.V1ObjectMeta(name=name)
container = client.V1Container(name=name, image=image, image_pull_policy="IfNotPresent",
volume_mounts=vmounts, command=command, env=env, args=args, security_context=sec)
if ports:
container.ports = list(map(lambda x: client.V1ContainerPort(container_port=x), ports))
template = client.V1PodTemplateSpec(metadata=client.V1ObjectMeta(labels={"app": name}),
spec=client.V1PodSpec(containers=[container], volumes=vol, restart_policy="Always"))
spec = client.V1StatefulSetSpec(replicas=replicas, template=template, selector=client.V1LabelSelector(match_labels={"app": name}),
service_name=name, volume_claim_templates=vol_claims)
sts_conf.spec = spec
try:
api_app.create_namespaced_stateful_set(namespace=self._namespace, body=sts_conf)
LOG.info(f'Service: {name} created.')
except ApiException as e:
if e.status == 409 and patch and not (vol_claims is None):
api_app.patch_namespaced_stateful_set(name=name, namespace=self._namespace, body=sts_conf)
LOG.info(f'Service: {name} patched.')
else:
LOG.error(f'Exception message: {e}')
def simple_statefulset():
"""Return the Kubernetes config matching the simple-statefulset.yaml manifest."""
return client.V1StatefulSet(
api_version="apps/v1",
kind="StatefulSet",
metadata=client.V1ObjectMeta(name="postgres-statefulset",
labels={"app": "postgres"}),
spec=client.V1StatefulSetSpec(
replicas=3,
selector=client.V1LabelSelector(match_labels={"app": "postgres"}),
service_name="simple-service",
template=client.V1PodTemplateSpec(
metadata=client.V1ObjectMeta(labels={"app": "postgres"}),
spec=client.V1PodSpec(containers=[
client.V1Container(
name="postgres",
image="postgres:9.6",
ports=[client.V1ContainerPort(container_port=5432)],
)
]),
),
),
)
def patch_namespaced_statefulset(self, name, image, namespace="default"):
"""Scale deployment using name in namespace to replicas"""
# Configureate Pod template container
container = client.V1Container(name=name)
# Create and configurate a spec section
template = client.V1PodTemplateSpec(
metadata=client.V1ObjectMeta(labels={"app": name}),
spec=client.V1PodSpec(containers=[container]))
# Create the specification of statefulset
spec = client.V1StatefulSetSpec(
template=template,
selector={'matchLabels': {
'app': name
}},
service_name=name)
# Instantiate the statefulset object
statefulset = client.V1StatefulSet(
api_version="apps/v1",
kind="StatefulSet",
metadata=client.V1ObjectMeta(name=name),
spec=spec)
statefulset.spec.template.spec.containers[0].image = image
try:
self.apps_cli.patch_namespaced_stateful_set(
name, namespace, statefulset)
logger.info(
'Image of statefulset {} in namespace {} has been updated to {}'
.format(name, namespace, image))
return
except client.rest.ApiException as e:
if e.status == 404 or not e.status:
logger.info(
'Statefulset {} in namespace {} is not found'.format(
name, namespace))
return
logger.exception(e)
return False
spec=client.V1StatefulSetSpec(
replicas=0,
selector=client.V1LabelSelector(
match_labels={
"app": "mlbench",
"release": os.environ.get("MLBENCH_KUBE_RELEASENAME"),
"set": "",
}),
service_name="",
pod_management_policy="Parallel",
update_strategy=client.V1StatefulSetUpdateStrategy(
type="RollingUpdate"),
template=client.V1PodTemplateSpec(
metadata=client.V1ObjectMeta(
labels={
"app": "mlbench",
"chart": "mlbench-2.0.0",
"component": "worker",
"release": os.environ.get("MLBENCH_KUBE_RELEASENAME"),
"heritage": "Helm",
"set": "",
}),
spec=client.V1PodSpec(
service_account_name="mlbench-worker-sa",
affinity=client.V1Affinity(
pod_anti_affinity=client.V1PodAntiAffinity(
required_during_scheduling_ignored_during_execution=[
client.V1PodAffinityTerm(
label_selector=client.V1LabelSelector(
match_expressions=[
client.V1LabelSelectorRequirement(
key="component",
operator="In",
values=["worker"],
)
]),
topology_key="kubernetes.io/hostname",
)
])),
containers=[
client.V1Container(
name="",
image="",
image_pull_policy="Always",
stdin=True,
tty=True,
ports=[
client.V1ContainerPort(
name="ssh",
container_port=22,
host_port=16166,
protocol="TCP",
)
],
resources=client.V1ResourceRequirements(
limits={
"cpu": "1",
"nvidia.com/gpu": "0"
}),
volume_mounts=[
client.V1VolumeMount(name="mlbench-ssh-key",
mount_path="/ssh-key/root")
],
security_context=client.V1SecurityContext(
privileged=True),
)
],
volumes=[
client.V1Volume(
name="mlbench-ssh-key",
secret=client.V1SecretVolumeSource(
secret_name="{}-ssh-key".format(
os.environ.get("MLBENCH_KUBE_RELEASENAME")),
default_mode=256,
),
)
],
),
),
),
def create_app_deployment(api_client, namespace, server_config, client_config):
replicas = server_config['replicas']
command = server_config['command']
latency = server_config.get('latency', 0)
dropRate = server_config.get('dropRate', 0)
network = shlex.quote(server_config['network'])
client_replicas = client_config.get('replicas', 0)
# Volume
volume = client.V1Volume(name='configvol')
# Env Vars
pod_ns_fs = client.V1ObjectFieldSelector(field_path="metadata.namespace")
pod_ns_src = client.V1EnvVarSource(field_ref=pod_ns_fs)
pod_ns_env = client.V1EnvVar(name="POD_NAMESPACE", value_from=pod_ns_src)
pod_n_fs = client.V1ObjectFieldSelector(field_path="metadata.name")
pod_n_src = client.V1EnvVarSource(field_ref=pod_n_fs)
pod_n_env = client.V1EnvVar(name="POD_NAME", value_from=pod_n_src)
# Init Container
init_container_volume = client.V1VolumeMount(name='configvol',
mount_path="/var/config",
read_only=False)
init_container_spec = client.V1Container(
name="init",
image="init-config:1.0",
command=["python3"],
args=[
"network.py",
str(replicas), "$(POD_NAMESPACE)", "$(POD_NAME)", network
],
env=[pod_ns_env, pod_n_env],
volume_mounts=[init_container_volume])
# wait_service_container_spec = client.V1Container(name="init-wait-service", image="busybox",
# command=['sh'], args=['-c', f'for i in {{1..20}}; do sleep 1; if nslookup app-service.{namespace}.svc.cluster.local; then exit 0; fi; done; exit 1'])
# Container
app_container_volume = client.V1VolumeMount(name='configvol',
mount_path="/var/config",
read_only=False)
app_container_spec = client.V1Container(
name="app",
image="app:1.0",
command=["/bin/sh"],
args=["-c", f"eval 'source ./wrapper.sh; {command}'"],
volume_mounts=[app_container_volume],
env=[pod_ns_env, pod_n_env])
# HAProxy Container
haproxy_container_spec = client.V1Container(name="haproxy",
image="app-haproxy:1.0")
# App Proxy Container
app_proxy_container_spec = client.V1Container(name="goproxy",
image="goproxy:1.0",
args=[
str(replicas),
"$(POD_NAMESPACE)",
str(latency),
str(dropRate),
str(client_replicas)
],
env=[pod_ns_env])
# Pod
pod_spec = client.V1PodSpec(termination_grace_period_seconds=10,
containers=[
app_container_spec, haproxy_container_spec,
app_proxy_container_spec
],
init_containers=[init_container_spec],
volumes=[volume])
pod_metadata = client.V1ObjectMeta(labels={"app": "app-service"})
pod_template_spec = client.V1PodTemplateSpec(metadata=pod_metadata,
spec=pod_spec)
# StatefulSet
statefulset_selector = client.V1LabelSelector(
match_labels={"app": "app-service"})
statefulset_spec = client.V1StatefulSetSpec(
replicas=replicas,
pod_management_policy="Parallel",
service_name="app-service",
selector=statefulset_selector,
template=pod_template_spec)
statefulset_metadata = client.V1ObjectMeta(name="app")
deployment = client.V1StatefulSet(api_version="apps/v1",
metadata=statefulset_metadata,
spec=statefulset_spec)
# Deploy
api_client.create_namespaced_stateful_set(body=deployment,
namespace=namespace)
def create_client_deployment(api_client, namespace, config, server_config):
replicas = config['replicas']
server_replicas = server_config['replicas']
command = config['command']
network = shlex.quote(server_config['network'])
# Volume
volume = client.V1Volume(name='configvol')
# Env Vars
pod_ns_fs = client.V1ObjectFieldSelector(field_path="metadata.namespace")
pod_ns_src = client.V1EnvVarSource(field_ref=pod_ns_fs)
pod_ns_env = client.V1EnvVar(name="POD_NAMESPACE", value_from=pod_ns_src)
pod_n_fs = client.V1ObjectFieldSelector(field_path="metadata.name")
pod_n_src = client.V1EnvVarSource(field_ref=pod_n_fs)
pod_n_env = client.V1EnvVar(name="POD_NAME", value_from=pod_n_src)
# Init Container
init_container_volume = client.V1VolumeMount(name='configvol',
mount_path="/var/config",
read_only=False)
init_container_spec = client.V1Container(
name="init",
image="init-config:1.0",
command=["python3"],
args=[
"network.py",
str(server_replicas), "$(POD_NAMESPACE)", "$(POD_NAME)", network,
str(replicas)
],
env=[pod_ns_env, pod_n_env],
volume_mounts=[init_container_volume])
# Container
app_container_volume = client.V1VolumeMount(name='configvol',
mount_path="/var/config",
read_only=False)
app_container_spec = client.V1Container(
name="client",
image="app:1.0",
command=["/bin/sh"],
args=["-c", f"eval 'source ./wrapper.sh; {command}'"],
volume_mounts=[app_container_volume],
env=[pod_ns_env, pod_n_env])
# Pod
pod_spec = client.V1PodSpec(termination_grace_period_seconds=10,
init_containers=[init_container_spec],
containers=[app_container_spec],
volumes=[volume])
pod_metadata = client.V1ObjectMeta(labels={"client": "client-service"})
pod_template_spec = client.V1PodTemplateSpec(metadata=pod_metadata,
spec=pod_spec)
# StatefulSet
statefulset_selector = client.V1LabelSelector(
match_labels={"client": "client-service"})
statefulset_spec = client.V1StatefulSetSpec(
replicas=replicas,
pod_management_policy="Parallel",
service_name="client-service",
selector=statefulset_selector,
template=pod_template_spec)
statefulset_metadata = client.V1ObjectMeta(name="client")
deployment = client.V1StatefulSet(api_version="apps/v1",
metadata=statefulset_metadata,
spec=statefulset_spec)
# Deploy
api_client.create_namespaced_stateful_set(body=deployment,
namespace=namespace)
def createStatefulSet(username,
replicas,
image,
is_host_network=False,
ssh_port="22"):
statefulset_name = username + "-horovod"
statefulset = client.V1StatefulSet()
#statefulset.api_version="apps/v1beta2"
statefulset.metadata = client.V1ObjectMeta(name=statefulset_name,
labels={
"app": "horovod",
"user": username,
"role": "worker"
})
label_selector = client.V1LabelSelector(match_labels={
"app": "horovod",
"user": username,
"role": "worker"
})
# Pod template 정의
pod_template = client.V1PodTemplateSpec()
pod_template.metadata = client.V1ObjectMeta(labels={
"app": "horovod",
"user": username,
"role": "worker"
})
container = client.V1Container(name="worker")
container.image = image
container.image_pull_policy = "IfNotPresent"
container.env = [
client.V1EnvVar(name="SSHPORT", value=ssh_port),
client.V1EnvVar(name="USESECRETS", value="true"),
# TODO: 바꾸기
client.V1EnvVar(name="ENTRY_POINT", value="train.py")
]
container.ports = [client.V1ContainerPort(container_port=22)]
container.volume_mounts = [
client.V1VolumeMount(name=statefulset_name + "-cm",
mount_path="/horovod/generated"),
client.V1VolumeMount(name=statefulset_name + "-secret",
mount_path="/etc/secret-volume",
read_only=True),
client.V1VolumeMount(name=statefulset_name + "-data",
mount_path="/horovod/data")
]
container.command = ["/horovod/generated/run.sh"]
container.readiness_probe = client.V1Probe(
_exec=client.V1ExecAction(command=["/horovod/generated/check.sh"]),
initial_delay_seconds=1,
period_seconds=2)
pod_spec = client.V1PodSpec(containers=[container])
# Host Network가 설정되있다면
if is_host_network == True:
pod_spec.host_network = True
pod_spec.dns_policy = "ClusterFirstWithHostNet"
pod_spec.volumes = [
client.V1Volume(name=statefulset_name + "-cm",
config_map=client.V1ConfigMapVolumeSource(
name=statefulset_name,
items=[
client.V1KeyToPath(key="hostfile.config",
path="hostfile",
mode=438),
client.V1KeyToPath(key="ssh.readiness",
path="check.sh",
mode=365),
client.V1KeyToPath(key="worker.run",
path="run.sh",
mode=365)
])),
client.V1Volume(name=statefulset_name + "-secret",
secret=client.V1SecretVolumeSource(
secret_name=statefulset_name,
default_mode=448,
items=[
client.V1KeyToPath(key="host-key",
path="id_rsa"),
client.V1KeyToPath(key="host-key-pub",
path="authorized_keys")
])),
client.V1Volume(name=statefulset_name + "-data",
empty_dir=client.V1EmptyDirVolumeSource())
]
pod_spec.subdomain = statefulset_name
pod_spec.hostname = statefulset_name
pod_spec.init_containers = [
client.V1Container(
name="download-data",
image=image,
image_pull_policy="IfNotPresent",
command=["/bin/bash", "-c"],
args=[
"curl http://ywj-horovod.s3.ap-northeast-2.amazonaws.com/horovod/"
+ username + "/train.py > /horovod/data/train.py"
],
volume_mounts=[
client.V1VolumeMount(name=statefulset_name + "-data",
mount_path="/horovod/data")
])
]
pod_template.spec = pod_spec
statefulset.spec = client.V1StatefulSetSpec(
selector=label_selector,
service_name=statefulset_name +
"-worker", # https://kubernetes.io/docs/concepts/workloads/controllers/statefulset/#pod-identity
pod_management_policy="Parallel",
replicas=replicas,
template=pod_template)
return statefulset
def template(context):
pod_spec_volumes = []
pod_spec_volume_mounts = []
stateful_set_spec_volume_claim_templates = []
for pvc in context.get("pvc"):
stateful_set_spec_volume_claim_templates.append(
client.V1PersistentVolumeClaim(
metadata=client.V1ObjectMeta(
name=pvc["name"],
annotations={
"volume.beta.kubernetes.io/storage-class": pvc["class"]
},
),
spec=client.V1PersistentVolumeClaimSpec(
access_modes=["ReadWriteOnce"],
resources=client.V1ResourceRequirements(
requests={"storage": pvc["size"]}
),
),
)
)
pod_spec_volume_mounts.append(
client.V1VolumeMount(name=pvc["name"], mount_path=pvc["mountPath"])
)
if "configmap" in context:
volume_name = "{}-config".format(context["name"])
pod_spec_volumes.append(
client.V1Volume(
name=volume_name,
config_map=client.V1ConfigMapVolumeSource(name=context["name"]),
)
)
pod_spec_volume_mounts.append(
client.V1VolumeMount(name=volume_name, mount_path="/etc/postgresql/")
)
labels = {"app": context["name"]}
pg_isready_exec = client.V1ExecAction(command=["gosu postgres pg_isready"])
return client.V1StatefulSet(
api_version="apps/v1beta1",
kind="StatefulSet",
metadata=client.V1ObjectMeta(name=context["name"]),
spec=client.V1StatefulSetSpec(
service_name=context["name"],
replicas=context["replicas"],
selector=client.V1LabelSelector(match_labels=labels),
template=client.V1PodTemplateSpec(
metadata=client.V1ObjectMeta(labels=labels),
spec=client.V1PodSpec(
containers=[
client.V1Container(
name="postgres",
image=context["image"],
lifecycle=client.V1Lifecycle(
pre_stop=client.V1Handler(
_exec=client.V1ExecAction(
command=[
'gosu postgres pg_ctl -D "$PGDATA" -m fast -w stop'
]
)
)
),
liveness_probe=client.V1Probe(
_exec=pg_isready_exec,
initial_delay_seconds=120,
timeout_seconds=5,
failure_threshold=6,
),
readiness_probe=client.V1Probe(
_exec=pg_isready_exec,
initial_delay_seconds=10,
timeout_seconds=5,
period_seconds=30,
failure_threshold=999,
),
ports=[client.V1ContainerPort(container_port=5432)],
volume_mounts=pod_spec_volume_mounts,
resources=client.V1ResourceRequirements(
**context["resources"]
)
if "resources" in context
else None,
)
],
volumes=pod_spec_volumes,
node_selector=context.get("nodeSelector"),
),
),
volume_claim_templates=stateful_set_spec_volume_claim_templates,
),
)
def template(context):
"""
handle yml env
"""
name = context.get("name")
labels = {"name": name}
image_tag=context["name"].split('-')[1]
image = context["image_namespace"]+image_tag+":"+context["image_branch"]
args = [arg for arg in context["args"]] if context.get("args") else None
limits,requests = context["resources"]["limits"],context["resources"]["requests"]
replicas = context["replicas"][name]
"""
handle cmdb env
"""
filename = "env_" + name.split("-")[1] + ".yml"
env = handle_env("/tmp/{}".format(filename))
workingDir = context["workingDir"]
"""
k8s yaml 组件模块
"""
containers = [
client.V1Container(
name = name,
image = image,
env = env,
args = args,
image_pull_policy = "Always",
# readiness_probe=client.V1Probe(_exec=client.V1ExecAction(command=['cat','/tmp/container_ready']),initial_delay_seconds=10, period_seconds=5),
resources = client.V1ResourceRequirements(limits = limits,requests = requests),
security_context = client.V1SecurityContext(privileged=True),
working_dir = working_dir,
)
]
template = client.V1PodTemplateSpec(
metadata = client.V1ObjectMeta(labels=labels),
spec = client.V1PodSpec(
containers = containers,
dns_policy = "ClusterFirst",
image_pull_secrets = [client.V1LocalObjectReference(name="image-pull-secret")],
restart_policy = "Always"
)
)
spec = client.V1StatefulSetSpec(
service_name = name,
replicas = replicas,
selector = client.V1LabelSelector(match_labels=labels),
template = template
)
return client.V1StatefulSet(
api_version = "apps/v1",
kind = "StatefulSet",
metadata = client.V1ObjectMeta(name = name,labels = labels),
spec = spec
)
def deploy_stateful_set(
self,
component_name,
headless_service,
cpu_count, # deprecated
memory_mb,
image,
replicas,
stateful_set_labels,
pod_labels,
ports=None,
env=None,
cpu_request=1,
cpu_limit=None,
):
if cpu_count is not None:
cpu_request_resources = "%s" % cpu_count
cpu_limit_resources = "%s" % cpu_count
else:
if cpu_limit is None:
cpu_limit = cpu_request
cpu_request_resources = "%s" % cpu_request
cpu_limit_resources = "%s" % cpu_limit
memory_resources = "%sMi" % memory_mb
stateful_set = self.get_stateful_set(stateful_set_labels)
if stateful_set:
changed = False
if stateful_set.spec.replicas != replicas:
self.logger.info("replicas changed from %s to %s" %
(stateful_set.spec.replicas, replicas))
stateful_set.spec.replicas = replicas
changed = True
for container in stateful_set.spec.template.spec.containers:
if container.name == component_name:
if container.image != image:
container.image = image
changed = True
self.logger.info("image changed")
if container.resources.requests is None:
container.resources.requests = {}
if container.resources.limits is None:
container.resources.limits = {}
if "cpu" not in container.resources.requests or container.resources.requests[
"cpu"] != cpu_request_resources:
container.resources.requests[
"cpu"] = cpu_request_resources
changed = True
self.logger.info("cpu requests changed to %s" %
cpu_request_resources)
if "cpu" not in container.resources.limits or container.resources.limits[
"cpu"] != cpu_limit_resources:
container.resources.limits["cpu"] = cpu_limit_resources
changed = True
self.logger.info("cpu limit changed to %s" %
cpu_limit_resources)
if "memory" not in container.resources.requests or container.resources.requests[
"memory"] != memory_resources:
container.resources.requests[
"memory"] = memory_resources
changed = True
self.logger.info("memory request changed to %s" %
memory_resources)
if "memory" not in container.resources.limits or container.resources.limits[
"memory"] != memory_resources:
container.resources.limits["memory"] = memory_resources
changed = True
self.logger.info("memory limit changed to %s" %
memory_resources)
if changed:
self.logger.info("patching stateful_set...")
self.apps_api.patch_namespaced_stateful_set(
name=stateful_set.metadata.name,
namespace=self.environment.namespace,
body=stateful_set,
)
raise deployment_status.StillDeploying(
"Waiting for %s stateful set being patched" %
component_name)
else:
self.logger.info("creating %s stateful_set..." % component_name)
stateful_set = self.apps_api.create_namespaced_stateful_set(
namespace=self.environment.namespace,
body=kubernetes_client.V1StatefulSet(
api_version="apps/v1",
kind="StatefulSet",
metadata=kubernetes_client.V1ObjectMeta(
name=self.generate_object_name(component_name),
namespace=self.environment.namespace,
labels=self.generate_object_labels(
stateful_set_labels)),
spec=kubernetes_client.V1StatefulSetSpec(
service_name=headless_service.metadata.name,
replicas=replicas,
selector=kubernetes_client.V1LabelSelector(
match_labels=self.generate_object_labels(
pod_labels)),
template=kubernetes_client.V1PodTemplateSpec(
metadata=kubernetes_client.V1ObjectMeta(
labels=self.generate_object_labels(
pod_labels)),
spec=kubernetes_client.V1PodSpec(containers=[
kubernetes_client.V1Container(
name=component_name,
image=image,
image_pull_policy=self.environment.
image_pull_policy,
resources=kubernetes_client.
V1ResourceRequirements(
requests={
"cpu": cpu_request_resources,
"memory": memory_resources,
},
limits={
"cpu": cpu_limit_resources,
"memory": memory_resources,
},
),
env=env,
ports=ports,
),
], ),
),
),
),
)
return stateful_set
