def func_to_pod(image, runtime, extra_env, command, args, workdir):
container = client.V1Container(
name="base",
image=image,
env=extra_env + runtime.spec.env,
command=[command],
args=args,
working_dir=workdir,
image_pull_policy=runtime.spec.image_pull_policy,
volume_mounts=runtime.spec.volume_mounts,
resources=runtime.spec.resources,
)
pod_spec = client.V1PodSpec(
containers=[container],
restart_policy="Never",
volumes=runtime.spec.volumes,
service_account=runtime.spec.service_account,
)
if runtime.spec.image_pull_secret:
pod_spec.image_pull_secrets = [
client.V1LocalObjectReference(name=runtime.spec.image_pull_secret)
]
return pod_spec
def create_deployment_object(deployment_name, image_name):
# Configurate Pod template container
container = client.V1Container(
name=image_name,
image="lizhengjie/hcp-re:" + image_name,
image_pull_policy="Always",
ports=[client.V1ContainerPort(container_port=8888)])
image_pull_secret = client.V1LocalObjectReference(name='regcred')
# Create and configurate a spec section
template = client.V1PodTemplateSpec(
metadata=client.V1ObjectMeta(labels={"k8s-app": deployment_name}),
spec=client.V1PodSpec(image_pull_secrets=[image_pull_secret],
containers=[container]))
# Create the specification of deployment
spec = client.V1DeploymentSpec(
replicas=1,
template=template,
selector={'matchLabels': {
'k8s-app': deployment_name
}})
# Instantiate the deployment object
deployment = client.V1Deployment(
api_version="apps/v1",
kind="Deployment",
metadata=client.V1ObjectMeta(name=deployment_name),
spec=spec)
return deployment
def create_job_object(container_image, image_pull_secret=None,service_account_name=None):
pull_secret = client.V1LocalObjectReference(
name=image_pull_secret
)
# Configureate Pod template container
container = client.V1Container(
name="installer",
image=container_image,
command=list(install_command()))
# Create and configurate a spec section
template = client.V1PodTemplateSpec(
metadata=client.V1ObjectMeta(labels={"app": "mcm-installer"}),
spec=client.V1PodSpec(restart_policy="Never", containers=[container],
image_pull_secrets=[pull_secret],service_account_name=service_account_name))
# Create the specification of deployment
spec = client.V1JobSpec(
template=template,
backoff_limit=1)
# Instantiate the job object
job = client.V1Job(
api_version="batch/v1",
kind="Job",
metadata=client.V1ObjectMeta(name=JOB_NAME),
spec=spec)
return job
def add_docker_credentials(kube_manager, pod_spec, namespace):
secret_name = constants.DOCKER_CREDS_SECRET_NAME
if not kube_manager.secret_exists(secret_name, namespace):
raise ValueError(
"Not able to find docker credentials secret: {}".format(
secret_name))
pod_spec.image_pull_secrets = [client.V1LocalObjectReference(secret_name)]
def _get_pod_template(self, image_name, name, container_port, envs, volume_mounts, volumes, service_account_name,
prior_device_name):
env_list = self._get_envs(envs)
volume_mount_list = self._get_volume_mounts(volume_mounts)
volume_list = self._get_volumes(volumes)
image_pull_secret_name = prior_device_name + "-registry"
if container_port != None and container_port != "":
return client.V1PodTemplateSpec(
metadata=client.V1ObjectMeta(labels={"run": name}),
spec=client.V1PodSpec(
service_account_name=service_account_name,
image_pull_secrets=[client.V1LocalObjectReference(
name=image_pull_secret_name
)],
containers=[client.V1Container(
name=name,
image=image_name,
image_pull_policy="Always",
ports=[client.V1ContainerPort(container_port=int(container_port))],
env=env_list,
volume_mounts=volume_mount_list
)],
volumes=volume_list
)
)
return client.V1PodTemplateSpec(
metadata=client.V1ObjectMeta(labels={"run": name}),
spec=client.V1PodSpec(
service_account_name=service_account_name,
image_pull_secrets=[client.V1LocalObjectReference(
name=image_pull_secret_name
)],
containers=[client.V1Container(
name=name,
image=image_name,
image_pull_policy="Always",
env=env_list,
volume_mounts=volume_mount_list
)],
volumes=volume_list
)
)
def create_deployment_object(name=None,namespace=None,image=None,port=None,image_pull_policy=None,\
imagePullSecret=None,labels=None,replicas=None,cpu=None,memory=None,liveness_probe=None,readiness_probe=None):
#configure pod template container
resources = None
volumeMounts = []
volumes = []
if (cpu or memory):
resources = client.V1ResourceRequirements(requests={
"cpu":
str(int(cpu / 2)) + "m",
"memory":
str(int(memory / 2)) + "Mi"
},
limits={
"cpu": str(cpu) + "m",
"memory":
str(memory) + "Mi"
})
vm1 = client.V1VolumeMount(name='log',
mount_path="/opt/microservices/logs")
volumeMounts.append(vm1)
v1 = client.V1Volume(name="log", empty_dir=client.V1EmptyDirVolumeSource())
volumes.append(v1)
image_pull_secret = client.V1LocalObjectReference(name=imagePullSecret)
container = client.V1Container(name=name,
image=image,
image_pull_policy=image_pull_policy,
ports=[
client.V1ContainerPort(
container_port=port,
name="web",
protocol="TCP")
],
resources=resources,
readiness_probe=readiness_probe,
liveness_probe=liveness_probe,
volume_mounts=volumeMounts)
template = client.V1PodTemplateSpec(
metadata=client.V1ObjectMeta(labels=labels),
spec=client.V1PodSpec(containers=[container],
image_pull_secrets=[image_pull_secret],
volumes=volumes))
spec = client.V1DeploymentSpec(replicas=replicas,
template=template,
selector={'matchLabels': labels}
#strategy
)
deployment = client.V1Deployment(api_version="apps/v1",
kind="Deployment",
metadata=client.V1ObjectMeta(
name=name, namespace=namespace),
spec=spec)
return deployment
def _get_pod_template(self):
return client.V1PodTemplateSpec(
metadata=client.V1ObjectMeta(labels={"run": self.name}),
spec=client.V1PodSpec(
service_account_name=self.service_account_name,
image_pull_secrets=[
client.V1LocalObjectReference(name=self.prior_device_name +
"-registry")
],
containers=[self._get_container()],
volumes=self._get_volumes(),
))
def export_deployment(self):
# Configureate Pod template container
container = client.V1Container(
name=self.dm_name,
image=self.image,
ports=[
client.V1ContainerPort(container_port=int(port))
for port in self.container_port
],
image_pull_policy='Always',
env=[
client.V1EnvVar(name='LANG', value='en_US.UTF-8'),
client.V1EnvVar(name='LC_ALL', value='en_US.UTF-8')
],
resources=client.V1ResourceRequirements(limits=self.re_limits,
requests=self.re_requests),
volume_mounts=[
client.V1VolumeMount(mount_path='/opt/logs', name='logs')
],
liveness_probe=client.V1Probe(
initial_delay_seconds=5,
tcp_socket=client.V1TCPSocketAction(
port=int(self.container_port[0]))))
# Create and configurate a spec section
secrets = client.V1LocalObjectReference('registrysecret')
volume = client.V1Volume(
name='logs',
host_path=client.V1HostPathVolumeSource(path='/opt/logs'))
template = client.V1PodTemplateSpec(
metadata=client.V1ObjectMeta(labels={"project": self.dm_name}),
spec=client.V1PodSpec(containers=[container],
image_pull_secrets=[secrets],
volumes=[volume]))
selector = client.V1LabelSelector(
match_labels={"project": self.dm_name})
# Create the specification of deployment
spec = client.ExtensionsV1beta1DeploymentSpec(replicas=int(
self.replicas),
template=template,
selector=selector,
min_ready_seconds=3)
# Instantiate the deployment object
deployment = client.ExtensionsV1beta1Deployment(
api_version="extensions/v1beta1",
kind="Deployment",
metadata=client.V1ObjectMeta(name=self.dm_name),
spec=spec)
return deployment
def create_serviceaccount(self,
api_instance,
namespace,
name,
image_pull_secret_name=None):
body = client.V1ServiceAccount(
api_version='v1',
kind='ServiceAccount',
metadata=client.V1ObjectMeta(name=name, namespace=namespace),
automount_service_account_token=True,
image_pull_secrets=image_pull_secret_name
if image_pull_secret_name is None else
[client.V1LocalObjectReference(name=image_pull_secret_name)])
result = api_instance.create_namespaced_service_account(
namespace, body)
return result
def func_to_pod(image, runtime, extra_env, command, args, workdir):
container = client.V1Container(
name="base",
image=image,
env=extra_env + runtime.spec.env,
command=[command],
args=args,
working_dir=workdir,
image_pull_policy=runtime.spec.image_pull_policy,
volume_mounts=runtime.spec.volume_mounts,
resources=runtime.spec.resources,
)
pod_spec = kube_resource_spec_to_pod_spec(runtime.spec, container)
if runtime.spec.image_pull_secret:
pod_spec.image_pull_secrets = [
client.V1LocalObjectReference(name=runtime.spec.image_pull_secret)
]
return pod_spec
def _create_deployment_object(component_name,
containers,
init_containers,
replicas,
volumes,
labels={},
pull_secrets=[]):
deployment_name = _create_deployment_name(component_name)
# Label the pod with the deployment name, so we can find it easily
labels.update({"k8sdeployment": deployment_name})
# pull_secrets is a list of the names of the k8s secrets containing docker registry credentials
# See https://kubernetes.io/docs/concepts/containers/images/#specifying-imagepullsecrets-on-a-pod
ips = []
for secret in pull_secrets:
ips.append(client.V1LocalObjectReference(name=secret))
# Define pod template
template = client.V1PodTemplateSpec(
metadata=client.V1ObjectMeta(labels=labels),
spec=client.V1PodSpec(hostname=component_name,
containers=containers,
init_containers=init_containers,
volumes=volumes,
image_pull_secrets=ips))
# Define deployment spec
spec = client.ExtensionsV1beta1DeploymentSpec(replicas=replicas,
template=template)
# Create deployment object
deployment = client.ExtensionsV1beta1Deployment(
kind="Deployment",
metadata=client.V1ObjectMeta(name=deployment_name),
spec=spec)
return deployment
def create_deployment_object(self, name, image_tag):
# Configureate Pod template container
container = client.V1Container(
name=name,
image=image_tag,
ports=[
client.V1ContainerPort(
container_port=config.MLFLOW_MODEL_DEFAULT_TARGET_PORT,
name=name)
],
resources=client.V1ResourceRequirements(requests={
"cpu": "100m",
"memory": "200Mi"
}, ))
# Create and configurate a spec section
# !! need create a secret with type docker-registry contains private registry credential
secret = client.V1LocalObjectReference(name='regcred')
template = client.V1PodTemplateSpec(
metadata=client.V1ObjectMeta(labels={"name": name}),
spec=client.V1PodSpec(
containers=[container],
image_pull_secrets=[secret],
),
)
# Create the specification of deployment
spec = client.V1DeploymentSpec(
replicas=1,
template=template,
selector={'matchLabels': {
'name': name
}})
# Instantiate the deployment object
deployment = client.V1Deployment(
api_version="apps/v1",
kind="Deployment",
metadata=client.V1ObjectMeta(name=name),
spec=spec)
return deployment
def _create_kube_podspec(self,
container,
shm_size=None,
scheduler_name=KubernetesConfig.DEFAULT_SCHEDULER,
namespace=KubernetesConfig.K8S_NAMESPACE):
logger.info("Creating pod with scheduler_name = %s, namespace = %s" %
(scheduler_name, namespace))
volumes = []
if shm_size and isinstance(shm_size, int):
dshm_vol = client.V1EmptyDirVolumeSource(medium="Memory",
size_limit=shm_size)
volumes.append(client.V1Volume(name='dshm', empty_dir=dshm_vol))
if not volumes:
volumes = None
spec = client.V1PodSpec(containers=[container],
scheduler_name=scheduler_name,
restart_policy="OnFailure",
volumes=volumes,
image_pull_secrets=[
client.V1LocalObjectReference(
KubernetesConfig.IMAGE_PULL_SECRET)
])
return spec
def export_deployment(self):
# Configureate Pod template container
volume_mounts = []
containers = []
volumes = []
volume_mounts.append(
client.V1VolumeMount(mount_path='/docker/logs', name='logs'))
volumes.append(
client.V1Volume(name='logs',
host_path=client.V1HostPathVolumeSource(
path='/opt/logs', type='DirectoryOrCreate')))
if self.mounts:
for path in self.mounts:
volume_mounts.append(
client.V1VolumeMount(mount_path=path,
name=self.mounts[path]))
volumes.append(
client.V1Volume(name=self.mounts[path],
host_path=client.V1HostPathVolumeSource(
path=path, type='DirectoryOrCreate')))
liveness_probe = client.V1Probe(initial_delay_seconds=15,
tcp_socket=client.V1TCPSocketAction(
port=int(self.container_port[0])))
readiness_probe = client.V1Probe(initial_delay_seconds=15,
tcp_socket=client.V1TCPSocketAction(
port=int(self.container_port[0])))
if self.healthcheck:
liveness_probe = client.V1Probe(initial_delay_seconds=15,
http_get=client.V1HTTPGetAction(
path=self.healthcheck,
port=int(
self.container_port[0])))
readiness_probe = client.V1Probe(initial_delay_seconds=15,
http_get=client.V1HTTPGetAction(
path=self.healthcheck,
port=int(
self.container_port[0])))
Env = [
client.V1EnvVar(name='LANG', value='en_US.UTF-8'),
client.V1EnvVar(name='LC_ALL', value='en_US.UTF-8'),
client.V1EnvVar(name='POD_NAME',
value_from=client.V1EnvVarSource(
field_ref=client.V1ObjectFieldSelector(
field_path='metadata.name'))),
client.V1EnvVar(name='POD_IP',
value_from=client.V1EnvVarSource(
field_ref=client.V1ObjectFieldSelector(
field_path='status.podIP'))),
]
container = client.V1Container(
name=self.dm_name,
image=self.image,
ports=[
client.V1ContainerPort(container_port=int(port))
for port in self.container_port
],
image_pull_policy='Always',
env=Env,
resources=client.V1ResourceRequirements(limits=self.re_limits,
requests=self.re_requests),
volume_mounts=volume_mounts,
liveness_probe=liveness_probe,
readiness_probe=readiness_probe)
containers.append(container)
if self.sidecar:
sidecar_container = client.V1Container(
name='sidecar-%s' % self.dm_name,
image=self.sidecar,
image_pull_policy='Always',
env=Env,
resources=client.V1ResourceRequirements(
limits=self.re_limits, requests=self.re_requests),
volume_mounts=volume_mounts)
containers.append(sidecar_container)
# Create and configurate a spec section
secrets = client.V1LocalObjectReference('registrysecret')
template = client.V1PodTemplateSpec(
metadata=client.V1ObjectMeta(labels={"project": self.dm_name}),
spec=client.V1PodSpec(
containers=containers,
image_pull_secrets=[secrets],
volumes=volumes,
affinity=client.V1Affinity(node_affinity=client.V1NodeAffinity(
preferred_during_scheduling_ignored_during_execution=[
client.V1PreferredSchedulingTerm(
preference=client.V1NodeSelectorTerm(
match_expressions=[
client.V1NodeSelectorRequirement(
key='project',
operator='In',
values=['moji'])
]),
weight=30),
client.V1PreferredSchedulingTerm(
preference=client.V1NodeSelectorTerm(
match_expressions=[
client.V1NodeSelectorRequirement(
key='deploy',
operator='In',
values=[self.dm_name])
]),
weight=70)
]))))
selector = client.V1LabelSelector(
match_labels={"project": self.dm_name})
# Create the specification of deployment
spec = client.ExtensionsV1beta1DeploymentSpec(replicas=int(
self.replicas),
template=template,
selector=selector,
min_ready_seconds=3)
# Instantiate the deployment object
deployment = client.ExtensionsV1beta1Deployment(
api_version="extensions/v1beta1",
kind="Deployment",
metadata=client.V1ObjectMeta(name=self.dm_name),
spec=spec)
return deployment
def create_job_object(kJob, kImage, kVname, kVvalue, kimagepullpolicy,
kimagepullsecret, krestartpolicy, kbackofflimit,
khostpath, kvolname, kvolpath, kvolsubpath, kpvolclaim,
kcommands, kargs):
# This creates a job object dynamically but supports only limited parameters
# If you need any characteristics nt supported here, use a yaml manifest
#
# Configure environment variables
env_list = []
for key in kVname:
value = kVvalue[kVname.index(key)]
v1_envvar = client.V1EnvVar(name=key, value=value)
env_list.append(v1_envvar)
# Configure Volume Devices and Mounts
volnames_list = []
# if kvolname != 'none':
# volname = client.V1VolumeMount(
# name=kvolname,
# mount_path=kvolpath)
# volnames_list.append(volname)
for key in kvolname:
volpath = kvolpath[kvolname.index(key)]
volsubpath = kvolsubpath[kvolname.index(key)]
volname = client.V1VolumeMount(name=key,
mount_path=volpath,
sub_path=volsubpath)
volnames_list.append(volname)
# Configure Volumes list
vol_list = []
if kvolname != 'none':
if kpvolclaim != 'none':
vol = client.V1Volume(
name=kvolname,
persistent_volume_claim=client.
V1PersistentVolumeClaimVolumeSource(claim_name=kpvolclaim))
else:
vol = client.V1Volume(name=kvolname,
host_path=client.V1HostPathVolumeSource(
path=khostpath, type='Directory'))
vol_list.append(vol)
# Configure Pod template container
container = client.V1Container(
name="ctmjob",
image=kImage,
image_pull_policy=kimagepullpolicy,
env=env_list,
command=kcommands if len(kcommands) > 0 else None,
args=kargs if len(kargs) > 0 else None,
volume_mounts=volnames_list)
# Configure Image Pull Secret(s)
imagesecrets = []
isecret = client.V1LocalObjectReference(name=kimagepullsecret)
imagesecrets.append(isecret)
# Create and configure a spec section
template = client.V1PodTemplateSpec(
metadata=client.V1ObjectMeta(name=kJob),
spec=client.V1PodSpec(containers=[container],
image_pull_secrets=imagesecrets,
restart_policy=krestartpolicy,
volumes=vol_list))
# Create the specification of deployment
spec = client.V1JobSpec(template=template, backoff_limit=kbackofflimit)
# Instantiate the job object
job = client.V1Job(api_version="batch/v1",
kind="Job",
metadata=client.V1ObjectMeta(name=kJob),
spec=spec)
return job
def task_to_pod_spec( # pylint: disable=too-many-locals
task: Task,
container_name: Optional[str] = None,
secrets: Optional[List[Secret]] = None,
configmaps: Optional[List[ConfigMap]] = None,
) -> client.V1PodSpec:
"""Convert this job into a POD spec that a k8s scheduler can run
:param task: name for this task
:type task: Task
:param container_name: name for the container if None, it will be the job name
:type container_name: Optional[str]
:param secrets: A list of secrets to inject into the container.
:type secrets: Optional[List[Secret]]
:param configmaps: A list of configmaps to inject.
:type configmaps: Optional[List[ConfigMap]]
:returns: A PodSpec in the k8s client API
:rtype: client.V1PodSpec
"""
limits = {}
if task.num_gpus is not None:
limits['nvidia.com/gpu'] = task.num_gpus
resources = client.V1ResourceRequirements(limits=limits)
volume_mounts = []
if task.mounts is not None:
volume_mounts.extend(
client.V1VolumeMount(mount_path=m.path, name=m.name)
for m in task.mounts)
if secrets is not None:
volume_mounts.extend(
client.V1VolumeMount(
mount_path=s.path, name=s.name, sub_path=s.sub_path)
for s in secrets)
if configmaps is not None:
volume_mounts.extend(
client.V1VolumeMount(
mount_path=c.path, name=c.name, sub_path=c.sub_path)
for c in configmaps)
container = client.V1Container(
args=task.args,
command=listify(task.command),
name=container_name if container_name else task.name,
image=task.image,
volume_mounts=volume_mounts if volume_mounts else None,
image_pull_policy=task.pull_policy,
resources=resources,
env=[
client.V1EnvVar(ODIN_TASK_ENV, task.name),
client.V1EnvVar(ODIN_CRED_ENV,
os.path.join(SECRET_LOC, ODIN_CRED_FILE)),
],
)
volumes = []
if task.mounts is not None:
for mount in task.mounts:
pvc = client.V1PersistentVolumeClaimVolumeSource(
claim_name=mount.claim)
volumes.append(
client.V1Volume(name=mount.name, persistent_volume_claim=pvc))
if secrets is not None:
secrets = {s.name: s for s in secrets}
volumes.extend(
client.V1Volume(name=secret.name,
secret=client.V1SecretVolumeSource(
secret_name=secret.name,
default_mode=secret.mode))
for secret in secrets.values())
if configmaps is not None:
volumes.extend(
client.V1Volume(name=c,
config_map=client.V1ConfigMapVolumeSource(name=c))
for c in set(c.name for c in configmaps))
selector = task.node_selector if task.node_selector else None
regcred = client.V1LocalObjectReference(name='registry')
pod_spec = client.V1PodSpec(
containers=[container],
image_pull_secrets=[regcred],
volumes=volumes if volumes else None,
node_selector=selector,
restart_policy=ResourceHandler.RESTART_NEVER,
)
return pod_spec
def _create_deployment_object(
self,
job_name,
job_image,
deployment_name,
port=80,
replicas=1,
cmd_string=None,
engine_json_file='~/.ipython/profile_default/security/ipcontroller-engine.json',
engine_dir='.'):
""" Create a kubernetes deployment for the job.
Args:
- job_name (string) : Name of the job and deployment
- job_image (string) : Docker image to launch
KWargs:
- port (integer) : Container port
- replicas : Number of replica containers to maintain
Returns:
- True: The deployment object to launch
"""
# sorry, quick hack that doesn't pass this stuff through to test it works.
# TODO it also doesn't only add what is set :(
security_context = None
if self.user_id and self.group_id:
security_context = client.V1SecurityContext(
run_as_group=self.group_id,
run_as_user=self.user_id,
run_as_non_root=self.run_as_non_root)
# Create the enviornment variables and command to initiate IPP
environment_vars = client.V1EnvVar(name="TEST", value="SOME DATA")
launch_args = ["-c", "{0}; /app/deploy.sh;".format(cmd_string)]
print(launch_args)
# Configureate Pod template container
container = None
if security_context:
container = client.V1Container(
name=job_name,
image=job_image,
ports=[client.V1ContainerPort(container_port=port)],
command=['/bin/bash'],
args=launch_args,
env=[environment_vars],
security_context=security_context)
else:
container = client.V1Container(
name=job_name,
image=job_image,
ports=[client.V1ContainerPort(container_port=port)],
command=['/bin/bash'],
args=launch_args,
env=[environment_vars])
# Create a secret to enable pulling images from secure repositories
secret = None
if self.secret:
secret = client.V1LocalObjectReference(name=self.secret)
# Create and configurate a spec section
template = client.V1PodTemplateSpec(
metadata=client.V1ObjectMeta(labels={"app": job_name}),
spec=client.V1PodSpec(containers=[container],
image_pull_secrets=[secret]))
# Create the specification of deployment
spec = client.ExtensionsV1beta1DeploymentSpec(replicas=replicas,
template=template)
# Instantiate the deployment object
deployment = client.ExtensionsV1beta1Deployment(
api_version="extensions/v1beta1",
kind="Deployment",
metadata=client.V1ObjectMeta(name=deployment_name),
spec=spec)
return deployment
def create_job_object(name: str,
container_image: str,
env_list: dict,
command: List[str],
command_args: List[str],
volumes: List[Dict],
init_containers: List[Dict],
output: Output,
namespace: str = "stackl",
container_name: str = "jobcontainer",
api_version: str = "batch/v1",
image_pull_policy: str = "Always",
ttl_seconds_after_finished: int = 3600,
restart_policy: str = "Never",
backoff_limit: int = 0,
active_deadline_seconds: int = 3600,
service_account: str = "stackl-agent-stackl-agent",
image_pull_secrets: List[str] = [],
labels=None) -> client.V1Job:
# pylint: disable=too-many-arguments,too-many-locals,too-many-branches,too-many-statements
"""Creates a Job object using the Kubernetes client
:param name: Job name affix
:type name: str
:param container_image: automation container image
:type container_image: str
:param env_list: Dict with key/values for the environment inside the automation container
:type env_list: dict
:param command: entrypoint command
:type command: List[str]
:param command_args: command arguments
:type command_args: List[str]
:param volumes: volumes and volumemounts
:type volumes: List[Dict]
:param image_pull_secrets: secrets to pull images
:type image_pull_secrets: List[str]
:param init_containers: list with init_containers
:type init_containers: List[Dict]
:param output: output Object
:type output: Output
:param namespace: Kubernetes namespace, defaults to "stackl"
:type namespace: str, optional
:param container_name: name of automation container, defaults to "jobcontainer"
:type container_name: str, optional
:param api_version: Job api version, defaults to "batch/v1"
:type api_version: str, optional
:param image_pull_policy: always pull latest images, defaults to "Always"
:type image_pull_policy: str, optional
:param ttl_seconds_after_finished: Remove jobs after execution with ttl, defaults to 600
:type ttl_seconds_after_finished: int, optional
:param restart_policy: Restart the pod on the same node after failure, defaults to "Never"
:type restart_policy: str, optional
:param backoff_limit: Retries after failure, defaults to 0
:type backoff_limit: int, optional
:param active_deadline_seconds: Timeout on a job, defaults to 3600 seconds
:type active_deadline_seconds: int, optional
:param service_account: Kubernetes service account, defaults to "stackl-agent-stackl-agent"
:type service_account: str, optional
:param labels: metadata labels, defaults to {}
:type labels: dict, optional
:return: automation Job object
:rtype: client.V1Job
"""
id_job = id_generator()
name = name + "-" + id_job
body = client.V1Job(api_version=api_version, kind="Job")
body.metadata = client.V1ObjectMeta(namespace=namespace, name=name)
body.status = client.V1JobStatus()
template = client.V1PodTemplate()
template.template = client.V1PodTemplateSpec()
k8s_volumes = []
cms = []
logging.debug(f"volumes: {volumes}")
# create a k8s volume for each element in volumes
for vol in volumes:
vol_name = name + "-" + vol["name"]
k8s_volume = client.V1Volume(name=vol_name)
if vol["type"] == "config_map":
config_map = client.V1ConfigMapVolumeSource()
config_map.name = vol_name
k8s_volume.config_map = config_map
cms.append(create_cm(vol_name, namespace, vol['data']))
vol['name'] = vol_name
if vol["type"] == "empty_dir":
k8s_volume.empty_dir = client.V1EmptyDirVolumeSource(
medium="Memory")
vol['name'] = vol_name
k8s_volumes.append(k8s_volume)
logging.debug(f"Volumes created for job {name}: {k8s_volumes}")
# create a volume mount for each element in volumes
k8s_volume_mounts = []
for vol in volumes:
if vol["mount_path"]:
volume_mount = client.V1VolumeMount(name=vol["name"],
mount_path=vol["mount_path"])
if "sub_path" in vol:
volume_mount.sub_path = vol["sub_path"]
k8s_volume_mounts.append(volume_mount)
logging.debug(f"Volume mounts created for job {name}: {k8s_volume_mounts}")
# create an environment list
k8s_env_list = []
if env_list:
for key, value in env_list.items():
if isinstance(value, dict):
if 'config_map_key_ref' in value:
k8s_env_from = client.V1EnvVar(
name=key,
value_from=client.V1EnvVarSource(
config_map_key_ref=client.V1ConfigMapKeySelector(
name=value['config_map_key_ref']["name"],
key=value['config_map_key_ref']["key"])))
k8s_env_list.append(k8s_env_from)
elif 'field_ref' in value:
k8s_env_from = client.V1EnvVar(
name=key,
value_from=client.V1EnvVarSource(
field_ref=client.V1ObjectFieldSelector(
field_path=value['field_ref'])))
k8s_env_list.append(k8s_env_from)
else:
k8s_env = client.V1EnvVar(name=key, value=value)
k8s_env_list.append(k8s_env)
k8s_env_from_list = []
# if env_from:
# for env in env_from:
# if 'config_map_ref' in env:
# k8s_env_from = client.V1EnvFromSource(
# config_map_ref=env['config_map_ref'])
# k8s_env_from_list.append(k8s_env_from)
# elif 'secret_ref' in env:
# k8s_env_from = client.V1EnvFromSource(
# secret_ref=env['secret_ref'])
# k8s_env_from_list.append(k8s_env_from)
logging.debug(f"Environment list created for job {name}: {k8s_env_list}")
print(f"Environment list created for job {name}: {k8s_env_list}")
container = client.V1Container(name=container_name,
image=container_image,
env=k8s_env_list,
volume_mounts=k8s_volume_mounts,
image_pull_policy=image_pull_policy,
command=command,
args=command_args,
env_from=k8s_env_from_list)
k8s_init_containers = []
logging.debug(f"Init containers for job {name}: {init_containers}")
for c in init_containers:
k8s_c = client.V1Container(name=c['name'],
image=c['image'],
volume_mounts=k8s_volume_mounts,
env=k8s_env_list)
if 'args' in c:
k8s_c.args = c['args']
k8s_init_containers.append(k8s_c)
k8s_secrets = []
for secret in image_pull_secrets:
k8s_secrets.append(client.V1LocalObjectReference(name=secret))
logging.debug(f"Secret list created for job {name}: {k8s_secrets}")
containers = [container]
if output:
output.volume_mounts = k8s_volume_mounts
output.env = k8s_env_list
output_containers = output.containers
containers = containers + output_containers
template.template.metadata = client.V1ObjectMeta(labels=labels)
template.template.spec = client.V1PodSpec(
containers=containers,
restart_policy=restart_policy,
image_pull_secrets=k8s_secrets,
volumes=k8s_volumes,
init_containers=k8s_init_containers,
service_account_name=service_account)
template.template = client.V1PodTemplateSpec(
metadata=template.template.metadata, spec=template.template.spec)
body.spec = client.V1JobSpec(
ttl_seconds_after_finished=ttl_seconds_after_finished,
template=template.template,
backoff_limit=backoff_limit,
active_deadline_seconds=active_deadline_seconds)
return body, cms
def template(context):
"""
handle yml env
"""
name = context.get("name")
labels = {"name": name}
image_tag=context["name"].split('-')[1]
image = context["image_namespace"]+image_tag+":"+context["image_branch"]
args = [arg for arg in context["args"]] if context.get("args") else None
limits,requests = context["resources"]["limits"],context["resources"]["requests"]
replicas = context["replicas"][name]
"""
handle cmdb env
"""
filename = "env_" + name.split("-")[1] + ".yml"
env = handle_env("/tmp/{}".format(filename))
workingDir = context["workingDir"]
"""
k8s yaml 组件模块
"""
containers = [
client.V1Container(
name = name,
image = image,
env = env,
args = args,
image_pull_policy = "Always",
# readiness_probe=client.V1Probe(_exec=client.V1ExecAction(command=['cat','/tmp/container_ready']),initial_delay_seconds=10, period_seconds=5),
resources = client.V1ResourceRequirements(limits = limits,requests = requests),
security_context = client.V1SecurityContext(privileged=True),
working_dir = working_dir,
)
]
template = client.V1PodTemplateSpec(
metadata = client.V1ObjectMeta(labels=labels),
spec = client.V1PodSpec(
containers = containers,
dns_policy = "ClusterFirst",
image_pull_secrets = [client.V1LocalObjectReference(name="image-pull-secret")],
restart_policy = "Always"
)
)
spec = client.V1StatefulSetSpec(
service_name = name,
replicas = replicas,
selector = client.V1LabelSelector(match_labels=labels),
template = template
)
return client.V1StatefulSet(
api_version = "apps/v1",
kind = "StatefulSet",
metadata = client.V1ObjectMeta(name = name,labels = labels),
spec = spec
)
def post(self, project_id):
"""
"""
resource_registry = {
'db_deployment': False,
'db_service': False,
'image_pull_secret': False,
'app_deployment': False,
'app_service': False,
'ingress_entry': False
}
current_user_id = get_jwt_identity()
current_user_roles = get_jwt_claims()['roles']
app_schema = AppSchema()
app_data = request.get_json()
validated_app_data, errors = app_schema.load(app_data,
partial=("project_id", ))
if errors:
return dict(status='fail', message=errors), 400
existing_app = App.find_first(name=validated_app_data['name'],
project_id=project_id)
if existing_app:
return dict(
status='fail',
message=
f'App with name {validated_app_data["name"]} already exists'
), 409
validated_app_data['port'] = validated_app_data.get('port', 80)
app_name = validated_app_data['name']
app_alias = create_alias(validated_app_data['name'])
app_image = validated_app_data['image']
command = validated_app_data.get('command', None)
# env_vars = validated_app_data['env_vars']
env_vars = validated_app_data.get('env_vars', None)
private_repo = validated_app_data.get('private_image', False)
docker_server = validated_app_data.get('docker_server', None)
docker_username = validated_app_data.get('docker_username', None)
docker_password = validated_app_data.get('docker_password', None)
docker_email = validated_app_data.get('docker_email', None)
replicas = validated_app_data.get('replicas', 1)
app_port = validated_app_data.get('port', None)
image_pull_secret = None
command = command.split() if command else None
project = Project.get_by_id(project_id)
if not project:
return dict(status='fail',
message=f'project {project_id} not found'), 404
if not is_owner_or_admin(project, current_user_id, current_user_roles):
return dict(status='fail', message='Unauthorised'), 403
cluster = project.cluster
namespace = project.alias
if not cluster:
return dict(status='fail', message="Invalid Cluster"), 500
# check if app already exists
app = App.find_first(**{'name': app_name})
if app:
return dict(status='fail',
message=f'App {app_name} already exists'), 409
kube_host = cluster.host
kube_token = cluster.token
service_host = urlsplit(kube_host).hostname
kube_client = create_kube_clients(kube_host, kube_token)
try:
# create the app
new_app = App(name=app_name,
image=app_image,
project_id=project_id,
alias=app_alias,
port=app_port)
if private_repo:
# handle gcr credentials
if 'gcr' in docker_server and docker_username == '_json_key':
docker_password = json.dumps(
json.loads(base64.b64decode(docker_password)))
# create image pull secrets
authstring = base64.b64encode(
f'{docker_username}:{docker_password}'.encode("utf-8"))
secret_dict = dict(
auths={
docker_server: {
"username": docker_username,
"password": docker_password,
"email": docker_email,
"auth": str(authstring, "utf-8")
}
})
secret_b64 = base64.b64encode(
json.dumps(secret_dict).encode("utf-8"))
secret_body = client.V1Secret(
metadata=client.V1ObjectMeta(name=app_alias),
type='kubernetes.io/dockerconfigjson',
data={'.dockerconfigjson': str(secret_b64, "utf-8")})
kube_client.kube.create_namespaced_secret(
namespace=namespace,
body=secret_body,
_preload_content=False)
# update registry
resource_registry['image_pull_secret'] = True
image_pull_secret = client.V1LocalObjectReference(
name=app_alias)
# create app deployment's pvc meta and spec
# pvc_name = f'{app_alias}-pvc'
# pvc_meta = client.V1ObjectMeta(name=pvc_name)
# access_modes = ['ReadWriteOnce']
# storage_class = 'openebs-standard'
# resources = client.V1ResourceRequirements(
# requests=dict(storage='1Gi'))
# pvc_spec = client.V1PersistentVolumeClaimSpec(
# access_modes=access_modes, resources=resources, storage_class_name=storage_class)
# Create a PVC
# pvc = client.V1PersistentVolumeClaim(
# api_version="v1",
# kind="PersistentVolumeClaim",
# metadata=pvc_meta,
# spec=pvc_spec
# )
# kube_client.kube.create_namespaced_persistent_volume_claim(
# namespace=namespace,
# body=pvc
# )
# create deployment
dep_name = f'{app_alias}-deployment'
# # EnvVar
env = []
if env_vars:
for key, value in env_vars.items():
env.append(client.V1EnvVar(name=str(key),
value=str(value)))
# pod template
container = client.V1Container(
name=app_alias,
image=app_image,
ports=[client.V1ContainerPort(container_port=app_port)],
env=env,
command=command
# volume_mounts=[client.V1VolumeMount(mount_path="/data", name=dep_name)]
)
#pod volumes
# volumes = client.V1Volume(
# name=dep_name
# # persistent_volume_claim=client.V1PersistentVolumeClaimVolumeSource(claim_name=pvc_name)
# )
# spec
template = client.V1PodTemplateSpec(
metadata=client.V1ObjectMeta(labels={'app': app_alias}),
spec=client.V1PodSpec(containers=[container],
image_pull_secrets=[image_pull_secret]
# volumes=[volumes]
))
# spec of deployment
spec = client.V1DeploymentSpec(
replicas=replicas,
template=template,
selector={'matchLabels': {
'app': app_alias
}})
# Instantiate the deployment
deployment = client.V1Deployment(
api_version="apps/v1",
kind="Deployment",
metadata=client.V1ObjectMeta(name=dep_name),
spec=spec)
# create deployment in cluster
kube_client.appsv1_api.create_namespaced_deployment(
body=deployment, namespace=namespace, _preload_content=False)
# update registry
resource_registry['app_deployment'] = True
# create service in the cluster
service_name = f'{app_alias}-service'
service_meta = client.V1ObjectMeta(name=service_name,
labels={'app': app_alias})
service_spec = client.V1ServiceSpec(
type='ClusterIP',
ports=[client.V1ServicePort(port=3000, target_port=app_port)],
selector={'app': app_alias})
service = client.V1Service(metadata=service_meta,
spec=service_spec)
kube_client.kube.create_namespaced_service(namespace=namespace,
body=service,
_preload_content=False)
# update resource registry
resource_registry['app_service'] = True
# subdomain for the app
# sub_domain = f'{app_alias}.cranecloud.io'
sub_domain = get_app_subdomain(app_alias)
# create new ingres rule for the application
new_ingress_backend = client.ExtensionsV1beta1IngressBackend(
service_name=service_name, service_port=3000)
new_ingress_rule = client.ExtensionsV1beta1IngressRule(
host=sub_domain,
http=client.ExtensionsV1beta1HTTPIngressRuleValue(paths=[
client.ExtensionsV1beta1HTTPIngressPath(
path="", backend=new_ingress_backend)
]))
ingress_name = f'{project.alias}-ingress'
# Check if there is an ingress resource in the namespace, create if not
ingress_list = kube_client.extension_api.list_namespaced_ingress(
namespace=namespace).items
if not ingress_list:
ingress_meta = client.V1ObjectMeta(name=ingress_name)
ingress_spec = client.ExtensionsV1beta1IngressSpec(
# backend=ingress_backend,
rules=[new_ingress_rule])
ingress_body = client.ExtensionsV1beta1Ingress(
metadata=ingress_meta, spec=ingress_spec)
kube_client.extension_api.create_namespaced_ingress(
namespace=namespace, body=ingress_body)
# update registry
resource_registry['ingress_entry'] = True
else:
# Update ingress with new entry
ingress = ingress_list[0]
ingress.spec.rules.append(new_ingress_rule)
kube_client.extension_api.patch_namespaced_ingress(
name=ingress_name, namespace=namespace, body=ingress)
service_url = f'https://{sub_domain}'
new_app.url = service_url
saved = new_app.save()
if not saved:
return dict(status='fail',
message='Internal Server Error'), 500
new_app_data, _ = app_schema.dump(new_app)
return dict(status='success', data=dict(app=new_app_data)), 201
except client.rest.ApiException as e:
resource_clean_up(resource_registry, app_alias, namespace,
kube_client)
return dict(status='fail', message=json.loads(e.body)), 500
except Exception as e:
resource_clean_up(resource_registry, app_alias, namespace,
kube_client)
return dict(status='fail', message=str(e)), 500
def create_job_object(data):
meta = client.V1ObjectMeta(name=data["name"], namespace=data["namespace"])
labels = None
if "labels" in data:
labels_array = data["labels"].split(',')
labels = dict(s.split('=') for s in labels_array)
meta.labels = labels
annotations = None
if "annotations" in data:
annotations_array = data["annotations"].split(',')
annotations = dict(s.split('=') for s in annotations_array)
meta.annotations = annotations
envs = []
if "environments" in data:
envs_array = data["environments"].splitlines()
tmp_envs = dict(s.split('=', 1) for s in envs_array)
for key in tmp_envs:
envs.append(client.V1EnvVar(name=key, value=tmp_envs[key]))
if "environments_secrets" in data:
envs_array = data["environments_secrets"].splitlines()
tmp_envs = dict(s.split('=', 1) for s in envs_array)
for key in tmp_envs:
if (":" in tmp_envs[key]):
# passing secret env
value = tmp_envs[key]
secrets = value.split(':')
secrect_key = secrets[1]
secrect_name = secrets[0]
envs.append(
client.V1EnvVar(
name=key,
value="",
value_from=client.V1EnvVarSource(
secret_key_ref=client.V1SecretKeySelector(
key=secrect_key, name=secrect_name))))
container = client.V1Container(name=data["container_name"],
image=data["container_image"],
image_pull_policy=data["image_pull_policy"])
if "container_command" in data:
container.command = data["container_command"].split(' ')
if "container_args" in data:
args_array = data["container_args"].splitlines()
container.args = args_array
if "resources_requests" in data:
resources_array = data["resources_requests"].split(",")
tmp = dict(s.split('=', 1) for s in resources_array)
container.resources = client.V1ResourceRequirements(requests=tmp)
if "volume_mounts" in data:
mounts = common.create_volume_mount_yaml(data)
container.volume_mounts = mounts
container.env = envs
if "env_from" in data:
env_froms_data = yaml.full_load(data["env_from"])
env_from = []
for env_from_data in env_froms_data:
if 'configMapRef' in env_from_data:
env_from.append(
client.V1EnvFromSource(
config_map_ref=client.V1ConfigMapEnvSource(
env_from_data['configMapRef']['name'])))
elif 'secretRef' in env_from_data:
env_from.append(
client.V1EnvFromSource(secret_ref=client.V1SecretEnvSource(
env_from_data['secretRef']['name'])))
container.env_from = env_from
template_spec = client.V1PodSpec(containers=[container],
restart_policy=data["job_restart_policy"])
if "volumes" in data:
volumes_data = yaml.safe_load(data["volumes"])
volumes = []
if (isinstance(volumes_data, list)):
for volume_data in volumes_data:
volume = common.create_volume(volume_data)
if volume:
volumes.append(volume)
else:
volume = common.create_volume(volumes_data)
if volume:
volumes.append(volume)
template_spec.volumes = volumes
if "image_pull_secrets" in data:
images_array = data["image_pull_secrets"].split(",")
images = []
for image in images_array:
images.append(client.V1LocalObjectReference(name=image))
template_spec.image_pull_secrets = images
if "tolerations" in data:
tolerations_data = yaml.safe_load(data["tolerations"])
tolerations = []
for toleration_data in tolerations_data:
toleration = common.create_toleration(toleration_data)
if toleration:
tolerations.append(toleration)
template_spec.tolerations = tolerations
template = client.V1PodTemplateSpec(metadata=client.V1ObjectMeta(
name=data["name"],
labels=labels,
annotations=annotations,
),
spec=template_spec)
spec = client.V1JobSpec(template=template)
if "completions" in data:
spec.completions = int(data["completions"])
if "selectors" in data:
selectors_array = data["selectors"].split(',')
selectors = dict(s.split('=') for s in selectors_array)
spec.selector = selectors
if "node_selector" in data:
node_selectors_array = data["node_selector"].split(',')
node_selectors = dict(s.split('=') for s in node_selectors_array)
spec.nodeSelector = node_selectors
if "parallelism" in data:
spec.parallelism = int(data["parallelism"])
if "active_deadline_seconds" in data:
spec.active_deadline_seconds = int(data["active_deadline_seconds"])
if "backoff_limit" in data:
spec.backoff_limit = int(data["backoff_limit"])
job = client.V1Job(api_version=data["api_version"],
kind='Job',
metadata=meta,
spec=spec)
return job
def create_deployment(apps_v1_api, username, token, gpu):
name = 'jlab-{}'.format(username)
try:
init_container = client.V1Container(
name='{}-init'.format(name),
image="ubuntu:18.04",
image_pull_policy="IfNotPresent",
command=["/bin/sh"],
args=["-c", "chown 1001:1001 /persistent_volume"],
volume_mounts=[
client.V1VolumeMount(
name='persistent-volume',
mount_path="/persistent_volume",
sub_path='{}/jupyter'.format(username)
)
]
)
if gpu == True:
limits = {
'nvidia.com/gpu': 1
}
else:
limits = None
container = client.V1Container(
name=name,
image=envvars.DOCKER_IMAGE_JLAB_SERVER,
resources=client.V1ResourceRequirements(
limits=limits
),
image_pull_policy="Always",
ports=[client.V1ContainerPort(container_port=8888)],
env=[
client.V1EnvVar(
name='DES_USER',
value=username
),
client.V1EnvVar(
name='PIP_TARGET',
value='/home/jovyan/work/.pip'
),
client.V1EnvVar(
name='PYTHONPATH',
value='/home/jovyan/work/.pip'
)
],
volume_mounts=[
client.V1VolumeMount(
name='jupyter-config',
mount_path="/home/jovyan/.jupyter/"
),
client.V1VolumeMount(
name='persistent-volume',
mount_path="/home/jovyan/jobs/cutout",
sub_path='{}/cutout'.format(username)
),
client.V1VolumeMount(
name='persistent-volume',
mount_path="/home/jovyan/jobs/query",
sub_path='{}/query'.format(username)
),
client.V1VolumeMount(
name='persistent-volume',
mount_path="/home/jovyan/work",
sub_path='{}/jupyter'.format(username)
)
]
)
volume_config = client.V1Volume(
name='jupyter-config',
config_map=client.V1ConfigMapVolumeSource(
name=name,
items=[client.V1KeyToPath(
key=name,
path="jupyter_notebook_config.py"
)]
)
)
volume_persistent = client.V1Volume(
name='persistent-volume',
persistent_volume_claim=client.V1PersistentVolumeClaimVolumeSource(
claim_name=envvars.PVC_NAME_BASE
)
)
# Template
template = client.V1PodTemplateSpec(
metadata=client.V1ObjectMeta(labels={"app": name}),
spec=client.V1PodSpec(
image_pull_secrets=[
client.V1LocalObjectReference(
name='registry-auth'
)
],
init_containers=[
init_container
],
containers=[
container
],
volumes=[
volume_config,
volume_persistent
],
node_selector = {'gpu': '{}'.format(gpu).lower()}
)
)
# Spec
spec = client.V1DeploymentSpec(
replicas=1,
template=template,
selector=client.V1LabelSelector(
match_labels=dict({'app': name})
)
)
# Deployment
deployment = client.V1Deployment(
api_version="apps/v1",
kind="Deployment",
metadata=client.V1ObjectMeta(name=name),
spec=spec)
# Creation of the Deployment in specified namespace
api_response = apps_v1_api.create_namespaced_deployment(
namespace=namespace, body=deployment
)
# logger.info('Deployment created:\n{}'.format(api_response))
except ApiException as e:
error_msg = str(e).strip()
logger.error(error_msg)
def get_pod_manifest(self):
"""Include volume with the git repository."""
repository = yield self.git_repository()
options = self.user_options
# make sure the pod name is less than 64 characters - if longer, keep
# the last 16 untouched since it is the server hash
if len(self.pod_name) > 63:
self.pod_name = self.pod_name[:47] + self.pod_name[-16:]
# Process the requested server options
server_options = options.get("server_options", {})
self.default_url = server_options.get("defaultUrl")
self.cpu_guarantee = float(server_options.get("cpu_request", 0.1))
# Make the user pods be in Guaranteed QoS class if the user
# had specified a memory request. Otherwise use a sensible default.
self.mem_guarantee = server_options.get("mem_request", "500M")
self.mem_limit = server_options.get("mem_request", "1G")
gpu = server_options.get("gpu_request", {})
if gpu:
self.extra_resource_limits = {"nvidia.com/gpu": str(gpu)}
# Configure the git repository volume
git_volume_name = self.pod_name[:54] + "-git-repo"
# 1. Define a new empty volume.
self.volumes = [
volume for volume in self.volumes
if volume["name"] != git_volume_name
]
volume = {"name": git_volume_name, "emptyDir": {}}
self.volumes.append(volume)
# 2. Define a volume mount for both init and notebook containers.
mount_path = f'/work/{options["project"]}'
volume_mount = {"mountPath": mount_path, "name": git_volume_name}
# 3. Configure the init container
init_container_name = "git-clone"
self.init_containers = [
container for container in self.init_containers
if not container.name.startswith(init_container_name)
]
lfs_auto_fetch = server_options.get("lfs_auto_fetch")
init_container = client.V1Container(
name=init_container_name,
env=[
client.V1EnvVar(name="MOUNT_PATH", value=mount_path),
client.V1EnvVar(name="REPOSITORY", value=repository),
client.V1EnvVar(
name="LFS_AUTO_FETCH",
value="1" if lfs_auto_fetch else "0",
),
client.V1EnvVar(name="COMMIT_SHA",
value=str(options.get("commit_sha"))),
client.V1EnvVar(name="BRANCH",
value=options.get("branch", "master")),
client.V1EnvVar(name="JUPYTERHUB_USER", value=self.user.name),
],
image=options.get("git_clone_image"),
volume_mounts=[volume_mount],
working_dir=mount_path,
security_context=client.V1SecurityContext(run_as_user=0),
)
self.init_containers.append(init_container)
# 4. Configure notebook container git repo volume mount
self.volume_mounts = [
volume_mount for volume_mount in self.volume_mounts
if volume_mount["mountPath"] != mount_path
]
self.volume_mounts.append(volume_mount)
# 5. Configure autosaving script execution hook
self.lifecycle_hooks = {
"preStop": {
"exec": {
"command": [
"/bin/sh",
"-c",
"/usr/local/bin/pre-stop.sh",
"||",
"true",
]
}
}
}
# Finalize the pod configuration
# Set the repository path to the working directory
self.working_dir = mount_path
self.notebook_dir = mount_path
# add git project-specific annotations
self.extra_annotations = {
RENKU_ANNOTATION_PREFIX + "namespace":
options.get("namespace"),
RENKU_ANNOTATION_PREFIX + "projectName":
options.get("project"),
RENKU_ANNOTATION_PREFIX + "projectId":
"{}".format(options.get("project_id")),
RENKU_ANNOTATION_PREFIX + "branch":
options.get("branch"),
RENKU_ANNOTATION_PREFIX + "commit-sha":
options.get("commit_sha"),
}
# add username to labels
safe_username = escapism.escape(self.user.name,
escape_char="-").lower()
self.extra_labels = {
RENKU_ANNOTATION_PREFIX + "username": safe_username
}
self.delete_grace_period = 30
pod = yield super().get_pod_manifest()
# Because repository comes from a coroutine, we can't put it simply in `get_env()`
pod.spec.containers[0].env.append(
client.V1EnvVar("CI_REPOSITORY_URL", repository))
# Add image pull secrets
if options.get("image_pull_secrets"):
secrets = [
client.V1LocalObjectReference(name=name)
for name in options.get("image_pull_secrets")
]
pod.spec.image_pull_secrets = secrets
return pod
def get_pull_secrets(self):
secrets = self.args.get('pull_secrets', ['docker'])
return [client.V1LocalObjectReference(name=s) for s in secrets]
def submit_job(args, command=None):
container_image = args.container
container_name = args.name
body = client.V1Job(api_version="batch/v1", kind="Job", metadata=client.V1ObjectMeta(name=container_name))
body.status = client.V1JobStatus()
template = client.V1PodTemplate()
labels = {
'hugin-job': "1",
'hugin-job-name': f'{container_name}'
}
template.template = client.V1PodTemplateSpec(
metadata=client.V1ObjectMeta(labels=labels)
)
tolerations = []
env = []
if args.environment:
for env_spec in args.environment:
env_name,env_value = env_spec.split("=", 1)
env.append(client.V1EnvVar(name=env_name, value=env_value))
containe_args = dict(
name=f"container-{container_name}",
image=container_image,
env=env,
)
if args.gpu:
tolerations.append(client.V1Toleration(
key='nvidia.com/gpu', operator='Exists', effect='NoSchedule'))
containe_args['resources'] = client.V1ResourceRequirements(limits={"nvidia.com/gpu": 1})
if command or args.command:
containe_args['command'] = command if command else args.command
container = client.V1Container(**containe_args)
pull_secrets = []
if args.pull_secret is not None:
pull_secrets.append(client.V1LocalObjectReference(name=args.pull_secret))
pod_args = dict(containers=[container],
restart_policy='Never',
image_pull_secrets=pull_secrets)
if tolerations:
pod_args['tolerations'] = tolerations
if args.node_selector is not None:
parts = args.node_selector.split("=", 1)
if len(parts) == 2:
affinity = client.V1Affinity(
node_affinity=client.V1NodeAffinity(
required_during_scheduling_ignored_during_execution=client.V1NodeSelector(
node_selector_terms=[client.V1NodeSelectorTerm(
match_expressions=[client.V1NodeSelectorRequirement(
key=parts[0], operator='In', values=[parts[1]])]
)]
)
)
)
pod_args['affinity'] = affinity
template.template.spec = client.V1PodSpec(**pod_args)
body.spec = client.V1JobSpec(ttl_seconds_after_finished=1800, template=template.template)
try:
api_response = batch_v1.create_namespaced_job("default", body, pretty=True)
#print (api_response)
except client.exceptions.ApiException as e:
logging.critical(f"Failed to start job: {e.reason}")
def template(context):
"""
handle yml env
"""
name = context.get("name")
version = context.get("version")
labels = {
"app": name,
"version": "v1"
} if not version else {
"app": name.strip("-v2"),
"version": version
}
image_tag = context["name"].split('-')[1]
image = context["image_namespace"] + image_tag + ":" + context[
"image_branch"]
args = [arg for arg in context["args"]] if context.get("args") else None
limits, requests = context["resources"]["limits"], context["resources"][
"requests"]
replicas = context.get("replicas", 1)
workingDir = context["workingDir"]
if name == "backend-logproxy":
annotations = {"sidecar.istio.io/inject": "false"}
else:
annotations = {"traffic.sidecar.istio.io/excludeOutboundPorts": "6379"}
"""
handle cmdb env
"""
filename = "env_" + name.split("-")[1] + ".yml"
env = handle_env("/tmp/{}".format(filename))
"""
k8s yaml 组件模块
"""
#从svn分支configmap目录中获取相关的目录结构
parentDir, subdir = handle_configmap("configmap")
volumemounts = [
client.V1VolumeMount(mount_path="/{}".format(parentDir),
name="mainfiles")
]
volumes = [
client.V1Volume(
name="mainfiles",
config_map=client.V1ConfigMapVolumeSource(name="mainfiles"))
]
for dir in subdir:
volumemounts.append(
client.V1VolumeMount(mount_path="/{}/{}".format(parentDir, dir),
name=dir))
volumes.append(
client.V1Volume(
name=dir, config_map=client.V1ConfigMapVolumeSource(name=dir)))
if name.startswith("frontend-dispatch"):
containers = [
client.V1Container(
name=name,
image=image,
env=env,
args=args,
volume_mounts=volumemounts,
image_pull_policy="Always",
lifecycle=client.V1Lifecycle(pre_stop=client.V1Handler(
_exec=client.V1ExecAction(
command=["nginx", "-s", "quit"]))),
readiness_probe=client.V1Probe(_exec=client.V1ExecAction(
command=['cat', '/tmp/container_ready']),
initial_delay_seconds=10,
period_seconds=5),
resources=client.V1ResourceRequirements(limits=limits,
requests=requests),
security_context=client.V1SecurityContext(privileged=True),
working_dir=workingDir,
)
]
else:
containers = [
client.V1Container(
name=name,
image=image,
env=env,
args=args,
volume_mounts=volumemounts,
image_pull_policy="Always",
readiness_probe=client.V1Probe(_exec=client.V1ExecAction(
command=['cat', '/tmp/container_ready']),
initial_delay_seconds=10,
period_seconds=5),
resources=client.V1ResourceRequirements(limits=limits,
requests=requests),
security_context=client.V1SecurityContext(privileged=True),
working_dir=workingDir,
)
]
template = client.V1PodTemplateSpec(
metadata=client.V1ObjectMeta(labels=labels, annotations=annotations),
spec=client.V1PodSpec(
containers=containers,
dns_policy="ClusterFirst",
image_pull_secrets=[
client.V1LocalObjectReference(name="image-pull-secret")
],
restart_policy="Always",
volumes=volumes))
spec = client.V1DeploymentSpec(
replicas=replicas,
selector=client.V1LabelSelector(match_labels=labels),
template=template,
strategy=client.ExtensionsV1beta1DeploymentStrategy(
rolling_update=client.ExtensionsV1beta1RollingUpdateDeployment(
max_surge=1, max_unavailable='25%'),
type="RollingUpdate",
),
)
return client.V1Deployment(api_version="apps/v1",
kind="Deployment",
metadata=client.V1ObjectMeta(name=name,
labels=labels),
spec=spec)
def _create_pod(self,
image,
pod_name,
job_name,
port=80,
cmd_string=None,
volumes=[]):
""" Create a kubernetes pod for the job.
Args:
- image (string) : Docker image to launch
- pod_name (string) : Name of the pod
- job_name (string) : App label
KWargs:
- port (integer) : Container port
Returns:
- None
"""
security_context = None
if self.user_id and self.group_id:
security_context = client.V1SecurityContext(
run_as_group=self.group_id,
run_as_user=self.user_id,
run_as_non_root=self.run_as_non_root)
# Create the enviornment variables and command to initiate IPP
environment_vars = client.V1EnvVar(name="TEST", value="SOME DATA")
launch_args = ["-c", "{0};".format(cmd_string)]
volume_mounts = []
# Create mount paths for the volumes
for volume in volumes:
volume_mounts.append(
client.V1VolumeMount(mount_path=volume[1], name=volume[0]))
resources = client.V1ResourceRequirements(limits={
'cpu': str(self.max_cpu),
'memory': self.max_mem
},
requests={
'cpu':
str(self.init_cpu),
'memory': self.init_mem
})
# Configure Pod template container
container = client.V1Container(
name=pod_name,
image=image,
resources=resources,
ports=[client.V1ContainerPort(container_port=port)],
volume_mounts=volume_mounts,
command=['/bin/bash'],
args=launch_args,
env=[environment_vars],
security_context=security_context)
# Create a secret to enable pulling images from secure repositories
secret = None
if self.secret:
secret = client.V1LocalObjectReference(name=self.secret)
# Create list of volumes from (pvc, mount) tuples
volume_defs = []
for volume in volumes:
volume_defs.append(
client.V1Volume(
name=volume[0],
persistent_volume_claim=client.
V1PersistentVolumeClaimVolumeSource(claim_name=volume[0])))
metadata = client.V1ObjectMeta(name=pod_name, labels={"app": job_name})
spec = client.V1PodSpec(containers=[container],
image_pull_secrets=[secret],
volumes=volume_defs)
pod = client.V1Pod(spec=spec, metadata=metadata)
api_response = self.kube_client.create_namespaced_pod(
namespace=self.namespace, body=pod)
logger.debug("Pod created. status='{0}'".format(
str(api_response.status)))
def export_deployment(self):
# Configureate Pod template container
volume_mounts = []
volume_mounts.append(
client.V1VolumeMount(mount_path='/opt/logs', name='logs'))
if self.dm_name == 'launch':
volume_mounts.append(
client.V1VolumeMount(mount_path='/opt/%s/conf' % self.dm_name,
name=self.dm_name))
container = client.V1Container(
name=self.dm_name,
image=self.image,
ports=[
client.V1ContainerPort(container_port=int(port))
for port in self.container_port
],
image_pull_policy='Always',
env=[
client.V1EnvVar(name='LANG', value='en_US.UTF-8'),
client.V1EnvVar(name='LC_ALL', value='en_US.UTF-8')
],
resources=client.V1ResourceRequirements(limits=self.re_limits,
requests=self.re_requests),
volume_mounts=volume_mounts,
liveness_probe=client.V1Probe(
initial_delay_seconds=30,
tcp_socket=client.V1TCPSocketAction(
port=int(self.container_port[0]))),
readiness_probe=client.V1Probe(
initial_delay_seconds=30,
tcp_socket=client.V1TCPSocketAction(
port=int(self.container_port[0]))))
# Create and configurate a spec section
secrets = client.V1LocalObjectReference('registrysecret')
volumes = []
volume = client.V1Volume(
name='logs',
host_path=client.V1HostPathVolumeSource(path='/opt/logs'))
volumes.append(volume)
template = client.V1PodTemplateSpec(
metadata=client.V1ObjectMeta(labels={"project": self.dm_name}),
spec=client.V1PodSpec(
containers=[container],
image_pull_secrets=[secrets],
volumes=volumes,
affinity=client.V1Affinity(node_affinity=client.V1NodeAffinity(
preferred_during_scheduling_ignored_during_execution=[
client.V1PreferredSchedulingTerm(
preference=client.V1NodeSelectorTerm(
match_expressions=[
client.V1NodeSelectorRequirement(
key='project',
operator='In',
values=['moji'])
]),
weight=30),
client.V1PreferredSchedulingTerm(
preference=client.V1NodeSelectorTerm(
match_expressions=[
client.V1NodeSelectorRequirement(
key='deploy',
operator='In',
values=[self.dm_name])
]),
weight=70)
]))))
selector = client.V1LabelSelector(
match_labels={"project": self.dm_name})
# Create the specification of deployment
spec = client.ExtensionsV1beta1DeploymentSpec(replicas=int(
self.replicas),
template=template,
selector=selector,
min_ready_seconds=3)
# Instantiate the deployment object
deployment = client.ExtensionsV1beta1Deployment(
api_version="extensions/v1beta1",
kind="Deployment",
metadata=client.V1ObjectMeta(name=self.dm_name),
spec=spec)
return deployment
def create_pod_template_spec(data):
ports = []
for port in data["ports"].split(','):
portDefinition = client.V1ContainerPort(container_port=int(port))
ports.append(portDefinition)
envs = []
if "environments" in data:
envs_array = data["environments"].splitlines()
tmp_envs = dict(s.split('=', 1) for s in envs_array)
for key in tmp_envs:
envs.append(client.V1EnvVar(name=key, value=tmp_envs[key]))
if "environments_secrets" in data:
envs_array = data["environments_secrets"].splitlines()
tmp_envs = dict(s.split('=', 1) for s in envs_array)
for key in tmp_envs:
if (":" in tmp_envs[key]):
# passing secret env
value = tmp_envs[key]
secrets = value.split(':')
secrect_key = secrets[1]
secrect_name = secrets[0]
envs.append(
client.V1EnvVar(
name=key,
value="",
value_from=client.V1EnvVarSource(
secret_key_ref=client.V1SecretKeySelector(
key=secrect_key, name=secrect_name))))
container = client.V1Container(name=data["container_name"],
image=data["image"],
ports=ports,
env=envs)
if "volume_mounts" in data:
volume_mounts_data = yaml.full_load(data["volume_mounts"])
volume_mounts = []
if (isinstance(volume_mounts_data, list)):
for volume_mount_data in volume_mounts_data:
volume_mount = create_volume_mount(volume_mount_data)
if volume_mount:
volume_mounts.append(volume_mount)
else:
volume_mount = create_volume_mount(volume_mounts_data)
if volume_mount:
volume_mounts.append(volume_mount)
container.volume_mounts = volume_mounts
if "liveness_probe" in data:
container.liveness_probe = load_liveness_readiness_probe(
data["liveness_probe"])
if "readiness_probe" in data:
container.readiness_probe = load_liveness_readiness_probe(
data["readiness_probe"])
if "container_command" in data:
container.command = data["container_command"].split(' ')
if "container_args" in data:
args_array = data["container_args"].splitlines()
container.args = args_array
if "resources_requests" in data:
resources_array = data["resources_requests"].split(",")
tmp_resources = dict(s.split('=', 1) for s in resources_array)
container.resources = client.V1ResourceRequirements(
requests=tmp_resources)
template_spec = client.V1PodSpec(containers=[container])
if "image_pull_secrets" in data:
images_array = data["image_pull_secrets"].split(",")
images = []
for image in images_array:
images.append(client.V1LocalObjectReference(name=image))
template_spec.image_pull_secrets = images
if "volumes" in data:
volumes_data = yaml.full_load(data["volumes"])
volumes = []
if (isinstance(volumes_data, list)):
for volume_data in volumes_data:
volume = create_volume(volume_data)
if volume:
volumes.append(volume)
else:
volume = create_volume(volumes_data)
if volume:
volumes.append(volume)
template_spec.volumes = volumes
return template_spec
def export_deployment(self):
# Configureate Pod template container
volume_mounts = []
containers = []
volumes = []
ports = []
liveness_probe = None
readiness_probe = None
volume_mounts.append(
client.V1VolumeMount(mount_path='/docker/logs', name='logs'))
volumes.append(
client.V1Volume(name='logs',
host_path=client.V1HostPathVolumeSource(
path='/opt/logs', type='DirectoryOrCreate')))
if self.mounts:
for path in self.mounts:
volume_mounts.append(
client.V1VolumeMount(mount_path=path,
name=self.mounts[path]))
volumes.append(
client.V1Volume(name=self.mounts[path],
host_path=client.V1HostPathVolumeSource(
path=path, type='DirectoryOrCreate')))
if self.container_port:
ports = [
client.V1ContainerPort(container_port=int(port))
for port in self.container_port
]
liveness_probe = client.V1Probe(
initial_delay_seconds=15,
tcp_socket=client.V1TCPSocketAction(
port=int(self.container_port[0])))
readiness_probe = client.V1Probe(
initial_delay_seconds=15,
tcp_socket=client.V1TCPSocketAction(
port=int(self.container_port[0])))
if self.healthcheck:
liveness_probe = client.V1Probe(
initial_delay_seconds=15,
http_get=client.V1HTTPGetAction(
path=self.healthcheck,
port=int(self.container_port[0])))
readiness_probe = client.V1Probe(
initial_delay_seconds=15,
http_get=client.V1HTTPGetAction(
path=self.healthcheck,
port=int(self.container_port[0])))
Env = [
client.V1EnvVar(name='LANG', value='en_US.UTF-8'),
client.V1EnvVar(name='LC_ALL', value='en_US.UTF-8'),
client.V1EnvVar(name='POD_NAME',
value_from=client.V1EnvVarSource(
field_ref=client.V1ObjectFieldSelector(
field_path='metadata.name'))),
client.V1EnvVar(name='POD_IP',
value_from=client.V1EnvVarSource(
field_ref=client.V1ObjectFieldSelector(
field_path='status.podIP'))),
]
container = client.V1Container(name=self.dm_name,
image=self.image,
ports=ports,
image_pull_policy='Always',
env=Env,
resources=client.V1ResourceRequirements(
limits=self.re_limits,
requests=self.re_requests),
volume_mounts=volume_mounts)
if liveness_probe and readiness_probe:
container = client.V1Container(
name=self.dm_name,
image=self.image,
ports=ports,
image_pull_policy='Always',
env=Env,
resources=client.V1ResourceRequirements(
limits=self.re_limits, requests=self.re_requests),
volume_mounts=volume_mounts,
liveness_probe=liveness_probe,
readiness_probe=readiness_probe)
containers.append(container)
if self.sidecar:
sidecar_container = client.V1Container(
name='sidecar-%s' % self.dm_name,
image=self.sidecar,
image_pull_policy='Always',
env=Env,
resources=client.V1ResourceRequirements(
limits=self.re_limits, requests=self.re_requests),
volume_mounts=volume_mounts)
containers.append(sidecar_container)
# Create and configurate a spec section
secrets = client.V1LocalObjectReference('registrysecret')
preference_key = self.dm_name
project_values = ['xxxx']
host_aliases = []
db_docker_hosts = db_op.docker_hosts
values = db_docker_hosts.query.with_entities(
db_docker_hosts.ip, db_docker_hosts.hostname).filter(
and_(db_docker_hosts.deployment == self.dm_name,
db_docker_hosts.context == self.context)).all()
db_op.DB.session.remove()
if values:
ips = []
for value in values:
try:
ip, hostname = value
key = "op_docker_hosts_%s" % ip
Redis.lpush(key, hostname)
ips.append(ip)
except Exception as e:
logging.error(e)
for ip in set(ips):
try:
key = "op_docker_hosts_%s" % ip
if Redis.exists(key):
hostnames = Redis.lrange(key, 0, -1)
if hostnames:
host_aliases.append(
client.V1HostAlias(hostnames=hostnames, ip=ip))
Redis.delete(key)
except Exception as e:
logging.error(e)
if self.labels:
if 'deploy' in self.labels:
preference_key = self.labels['deploy']
if 'project' in self.labels:
project_values = [self.labels['project']]
template = client.V1PodTemplateSpec(
metadata=client.V1ObjectMeta(labels={"project": self.dm_name}),
spec=client.V1PodSpec(
containers=containers,
image_pull_secrets=[secrets],
volumes=volumes,
host_aliases=host_aliases,
affinity=client.V1Affinity(node_affinity=client.V1NodeAffinity(
preferred_during_scheduling_ignored_during_execution=[
client.V1PreferredSchedulingTerm(
preference=client.V1NodeSelectorTerm(
match_expressions=[
client.V1NodeSelectorRequirement(
key=preference_key,
operator='In',
values=['mark'])
]),
weight=100)
],
required_during_scheduling_ignored_during_execution=client.
V1NodeSelector(node_selector_terms=[
client.V1NodeSelectorTerm(match_expressions=[
client.V1NodeSelectorRequirement(
key='project',
operator='In',
values=project_values)
])
])))))
selector = client.V1LabelSelector(
match_labels={"project": self.dm_name})
# Create the specification of deployment
spec = client.ExtensionsV1beta1DeploymentSpec(replicas=int(
self.replicas),
template=template,
selector=selector,
min_ready_seconds=3)
# Instantiate the deployment object
deployment = client.ExtensionsV1beta1Deployment(
api_version="extensions/v1beta1",
kind="Deployment",
metadata=client.V1ObjectMeta(name=self.dm_name),
spec=spec)
return deployment
