def attach_output_volume(op):
"""Attaches emptyDir volumes to container operations.
See https://github.com/kubeflow/pipelines/issues/1654
"""
# Handle auto-generated pipeline metadata
op.output_artifact_paths = {
'mlpipeline-ui-metadata': '/output/mlpipeline-ui-metadata.json',
'mlpipeline-metrics': '/output/mlpipeline-metrics.json',
}
# Add somewhere to store regular output
op.add_volume(
k8s_client.V1Volume(name='volume',
empty_dir=k8s_client.V1EmptyDirVolumeSource()))
op.container.add_volume_mount(
k8s_client.V1VolumeMount(name='volume', mount_path='/output'))
# func_to_container_op wants to store outputs under /tmp/outputs
op.add_volume(
k8s_client.V1Volume(name='outputs',
empty_dir=k8s_client.V1EmptyDirVolumeSource()))
op.container.add_volume_mount(
k8s_client.V1VolumeMount(name='outputs', mount_path='/tmp/outputs'))
return op
def dispatch(endpoint, access_key, secret_key, bucket_name, object_name):
job_name = f"local-rebuild-{uuid.uuid1()}"
downloader_env = [
client.V1EnvVar(name="ENDPOINT", value=endpoint),
client.V1EnvVar(name="ACCESS_KEY", value=access_key),
client.V1EnvVar(name="SECRET_KEY", value=secret_key),
client.V1EnvVar(name="BUCKET_NAME", value=bucket_name),
client.V1EnvVar(name="OBJECT_NAME", value=object_name)
]
downloader_container = client.V1Container(
name="downloader",
image=os.getenv("DOWNLOADER_IMAGE"),
env=downloader_env,
volume_mounts=[client.V1VolumeMount(
name="processor-input", mount_path="/output")])
processor_container = client.V1Container(
name="processor",
image=os.getenv("PROCESSOR_IMAGE"),
volume_mounts=[client.V1VolumeMount(name="processor-input", mount_path="/input", read_only=True),
client.V1VolumeMount(name="processor-output", mount_path="/output")])
pod_spec = client.V1PodSpec(
restart_policy="Never",
init_containers=[downloader_container],
containers=[processor_container],
volumes=[
client.V1Volume(name="processor-input",
empty_dir=client.V1EmptyDirVolumeSource()),
client.V1Volume(name="processor-output",
empty_dir=client.V1EmptyDirVolumeSource()),
])
# Create and configure a spec section
template = client.V1PodTemplateSpec(
metadata=client.V1ObjectMeta(name=job_name, labels={
"app": "local-rebuild-processor"}),
spec=pod_spec)
# Create the specification of the job
spec = client.V1JobSpec(
template=template,
backoff_limit=0)
# Instantiate the job object
job = client.V1Job(
api_version="batch/v1",
kind="Job",
metadata=client.V1ObjectMeta(name=job_name, labels={
"app": "local-rebuild-processor"}),
spec=spec)
client.BatchV1Api().create_namespaced_job(
body=job,
namespace="default")
def _mount_v3iod(task):
from kubernetes import client as k8s_client
def add_vol(name, mount_path, host_path):
vol = k8s_client.V1Volume(
name=name,
host_path=k8s_client.V1HostPathVolumeSource(path=host_path, type=''),
)
task.add_volume(vol).add_volume_mount(
k8s_client.V1VolumeMount(mount_path=mount_path, name=name)
)
add_vol(name='shm', mount_path='/dev/shm', host_path='/dev/shm/' + namespace)
add_vol(
name='v3iod-comm',
mount_path='/var/run/iguazio/dayman',
host_path='/var/run/iguazio/dayman/' + namespace,
)
vol = k8s_client.V1Volume(
name='daemon-health', empty_dir=k8s_client.V1EmptyDirVolumeSource()
)
task.add_volume(vol).add_volume_mount(
k8s_client.V1VolumeMount(
mount_path='/var/run/iguazio/daemon_health', name='daemon-health'
)
)
vol = k8s_client.V1Volume(
name='v3io-config',
config_map=k8s_client.V1ConfigMapVolumeSource(
name=v3io_config_configmap, default_mode=420
),
)
task.add_volume(vol).add_volume_mount(
k8s_client.V1VolumeMount(mount_path='/etc/config/v3io', name='v3io-config')
)
# vol = k8s_client.V1Volume(name='v3io-auth',
# secret=k8s_client.V1SecretVolumeSource(secret_name=v3io_auth_secret,
# default_mode=420))
# task.add_volume(vol).add_volume_mount(k8s_client.V1VolumeMount(mount_path='/igz/.igz', name='v3io-auth'))
task.add_env_variable(
k8s_client.V1EnvVar(
name='CURRENT_NODE_IP',
value_from=k8s_client.V1EnvVarSource(
field_ref=k8s_client.V1ObjectFieldSelector(
api_version='v1', field_path='status.hostIP'
)
),
)
)
task.add_env_variable(
k8s_client.V1EnvVar(
name='IGZ_DATA_CONFIG_FILE', value='/igz/java/conf/v3io.conf'
)
)
return task
def with_empty_dir(self, name: str,
mount_path: str) -> Optional['HmlInferenceDeployment']:
"""
Create an empy, writable volume with the given `name` mounted to the
specified `mount_path`
Args:
name (str): The name of the volume to mount
mount_path (str): The path to mount the empty volume
Returns:
A reference to the current `HmlInferenceDeployment` (self)
"""
self.pod_volumes.append(
client.V1Volume(name=name,
empty_dir=client.V1EmptyDirVolumeSource()))
self.k8s_container.add_volume_mount(
client.V1VolumeMount(
name=name,
mount_path=mount_path,
))
return self
def _add_tls_init_container(init_containers, volumes, volume_mounts, tls_info,
tls_config):
# Adds an InitContainer to the pod to set up TLS certificate information. For components that act as a
# server(tls_info["use_tls"] is True), the InitContainer will populate a directory with server and CA certificate
# materials in various formats. For other components (tls_info["use_tls"] is False, or tls_info is not specified),
# the InitContainer will populate a directory with CA certificate materials in PEM and JKS formats.
# In either case, the certificate directory is mounted onto the component container filesystem at the location
# specified by tls_info["component_cert_dir"], if present, otherwise at the configured default mount point
# (tls_config["component_cert_dir"]).
cert_directory = tls_info.get("cert_directory") or tls_config.get(
"component_cert_dir")
env = {}
env["TLS_SERVER"] = "true" if tls_info.get("use_tls") else "false"
# Create the certificate volume and volume mounts
volumes.append(
client.V1Volume(name="tls-info",
empty_dir=client.V1EmptyDirVolumeSource()))
volume_mounts.append(
client.V1VolumeMount(name="tls-info", mount_path=cert_directory))
init_volume_mounts = [
client.V1VolumeMount(name="tls-info",
mount_path=tls_config["cert_path"])
]
# Create the init container
init_containers.append(
_create_container_object("init-tls",
tls_config["image"],
False,
volume_mounts=init_volume_mounts,
env=env))
def recommender_pipeline():
# Load new data
data = dsl.ContainerOp(
name='updatedata',
output_artifact_paths={
'mlpipeline-ui-metadata': '/output/mlpipeline-ui-metadata.json',
'mlpipeline-metrics': '/output/mlpipeline-metrics.json',
},
image='lightbend/kubeflow-datapublisher:0.0.1') \
.add_env_variable(k8s_client.V1EnvVar(name='MINIO_URL',value='http://minio-service.kubeflow.svc.cluster.local:9000')) \
.add_env_variable(k8s_client.V1EnvVar(name='MINIO_KEY', value='minio')) \
.add_env_variable(k8s_client.V1EnvVar(name='MINIO_SECRET', value='minio123')) \
.add_volume(k8s_client.V1Volume(name='outputs', empty_dir=k8s_client.V1EmptyDirVolumeSource())) \
.add_volume_mount(k8s_client.V1VolumeMount(name='outputs', mount_path='/output'))
# Train the model
train = dsl.ContainerOp(
name='trainmodel',
output_artifact_paths={
'mlpipeline-ui-metadata': '/output/mlpipeline-ui-metadata.json',
'mlpipeline-metrics': '/output/mlpipeline-metrics.json',
},
image='lightbend/ml-tf-recommender:0.0.1') \
.add_env_variable(k8s_client.V1EnvVar(name='MINIO_URL',value='minio-service.kubeflow.svc.cluster.local:9000')) \
.add_env_variable(k8s_client.V1EnvVar(name='MINIO_KEY', value='minio')) \
.add_env_variable(k8s_client.V1EnvVar(name='MINIO_SECRET', value='minio123')) \
.add_volume(k8s_client.V1Volume(name='outputs', empty_dir=k8s_client.V1EmptyDirVolumeSource())) \
.add_volume_mount(k8s_client.V1VolumeMount(name='outputs', mount_path='/output'))
train.after(data)
# Publish new model model
publish = dsl.ContainerOp(
name='publishmodel',
output_artifact_paths={
'mlpipeline-ui-metadata': '/output/mlpipeline-ui-metadata.json',
'mlpipeline-metrics': '/output/mlpipeline-metrics.json',
},
image='lightbend/kubeflow-modelpublisher:0.0.1') \
.add_env_variable(k8s_client.V1EnvVar(name='MINIO_URL',value='http://minio-service.kubeflow.svc.cluster.local:9000')) \
.add_env_variable(k8s_client.V1EnvVar(name='MINIO_KEY', value='minio')) \
.add_env_variable(k8s_client.V1EnvVar(name='MINIO_SECRET', value='minio123')) \
.add_env_variable(k8s_client.V1EnvVar(name='KAFKA_BROKERS', value='strimzi-kafka-brokers.boris.svc.cluster.local:9092')) \
.add_env_variable(k8s_client.V1EnvVar(name='DEFAULT_RECOMMENDER_URL', value='http://recommender.boris.svc.cluster.local:8501')) \
.add_env_variable(k8s_client.V1EnvVar(name='ALTERNATIVE_RECOMMENDER_URL', value='http://recommender1.boris.svc.cluster.local:8501')) \
.add_volume(k8s_client.V1Volume(name='outputs', empty_dir=k8s_client.V1EmptyDirVolumeSource())) \
.add_volume_mount(k8s_client.V1VolumeMount(name='outputs', mount_path='/output'))
publish.after(train)
def _add_elk_logging_sidecar(containers, volumes, volume_mounts,
component_name, log_info, filebeat):
if not log_info or not filebeat:
return
log_dir = log_info.get("log_directory")
if not log_dir:
return
sidecar_volume_mounts = []
# Create the volume for component log files and volume mounts for the component and sidecar containers
volumes.append(
client.V1Volume(name="component-log",
empty_dir=client.V1EmptyDirVolumeSource()))
volume_mounts.append(
client.V1VolumeMount(name="component-log", mount_path=log_dir))
sc_path = log_info.get("alternate_fb_path") or "{0}/{1}".format(
filebeat["log_path"], component_name)
sidecar_volume_mounts.append(
client.V1VolumeMount(name="component-log", mount_path=sc_path))
# Create the volume for sidecar data and the volume mount for it
volumes.append(
client.V1Volume(name="filebeat-data",
empty_dir=client.V1EmptyDirVolumeSource()))
sidecar_volume_mounts.append(
client.V1VolumeMount(name="filebeat-data",
mount_path=filebeat["data_path"]))
# Create the volume for the sidecar configuration data and the volume mount for it
# The configuration data is in a k8s ConfigMap that should be created when DCAE is installed.
volumes.append(
client.V1Volume(name="filebeat-conf",
config_map=client.V1ConfigMapVolumeSource(
name=filebeat["config_map"])))
sidecar_volume_mounts.append(
client.V1VolumeMount(name="filebeat-conf",
mount_path=filebeat["config_path"],
sub_path=filebeat["config_subpath"]))
# Finally create the container for the sidecar
containers.append(
_create_container_object("filebeat",
filebeat["image"],
False,
volume_mounts=sidecar_volume_mounts))
def _mount_empty_dir(self, op, name: str, path: str):
# Add a writable volume
op.add_volume(
k8s_client.V1Volume(name=name,
empty_dir=k8s_client.V1EmptyDirVolumeSource()))
op.add_volume_mount(
k8s_client.V1VolumeMount(
name=name,
mount_path=path,
))
def get_obj(self):
"""
:description: Generate volume spec.
"""
if self.config.get("configmap"):
return client.V1Volume(name=self.slug,
config_map=client.V1ConfigMapVolumeSource(
name=self.config.get("configmap")))
return client.V1Volume(name=self.slug,
empty_dir=client.V1EmptyDirVolumeSource())
def get_volume(volume, claim_name=None, volume_mount=None):
if claim_name:
pv_claim = client.V1PersistentVolumeClaimVolumeSource(claim_name=claim_name)
return client.V1Volume(name=volume, persistent_volume_claim=pv_claim)
elif volume_mount:
return client.V1Volume(
name=volume,
host_path=client.V1HostPathVolumeSource(path=volume_mount))
else:
empty_dir = client.V1EmptyDirVolumeSource()
return client.V1Volume(name=volume, empty_dir=empty_dir)
def create_deployment_object(name=None,namespace=None,image=None,port=None,image_pull_policy=None,\
imagePullSecret=None,labels=None,replicas=None,cpu=None,memory=None,liveness_probe=None,readiness_probe=None):
#configure pod template container
resources = None
volumeMounts = []
volumes = []
if (cpu or memory):
resources = client.V1ResourceRequirements(requests={
"cpu":
str(int(cpu / 2)) + "m",
"memory":
str(int(memory / 2)) + "Mi"
},
limits={
"cpu": str(cpu) + "m",
"memory":
str(memory) + "Mi"
})
vm1 = client.V1VolumeMount(name='log',
mount_path="/opt/microservices/logs")
volumeMounts.append(vm1)
v1 = client.V1Volume(name="log", empty_dir=client.V1EmptyDirVolumeSource())
volumes.append(v1)
image_pull_secret = client.V1LocalObjectReference(name=imagePullSecret)
container = client.V1Container(name=name,
image=image,
image_pull_policy=image_pull_policy,
ports=[
client.V1ContainerPort(
container_port=port,
name="web",
protocol="TCP")
],
resources=resources,
readiness_probe=readiness_probe,
liveness_probe=liveness_probe,
volume_mounts=volumeMounts)
template = client.V1PodTemplateSpec(
metadata=client.V1ObjectMeta(labels=labels),
spec=client.V1PodSpec(containers=[container],
image_pull_secrets=[image_pull_secret],
volumes=volumes))
spec = client.V1DeploymentSpec(replicas=replicas,
template=template,
selector={'matchLabels': labels}
#strategy
)
deployment = client.V1Deployment(api_version="apps/v1",
kind="Deployment",
metadata=client.V1ObjectMeta(
name=name, namespace=namespace),
spec=spec)
return deployment
def get_volume(volume, claim_name=None, host_path=None, read_only=None):
if claim_name:
pv_claim = client.V1PersistentVolumeClaimVolumeSource(
claim_name=claim_name, read_only=read_only)
return client.V1Volume(name=volume, persistent_volume_claim=pv_claim)
if host_path:
return client.V1Volume(
name=volume,
host_path=client.V1HostPathVolumeSource(path=host_path))
empty_dir = client.V1EmptyDirVolumeSource()
return client.V1Volume(name=volume, empty_dir=empty_dir)
def inner(*args, **kwargs):
op = fun(*args, **kwargs)
op.output_artifact_paths = {
'mlpipeline-ui-metadata': '/output/mlpipeline-ui-metadata.json',
'mlpipeline-metrics': '/output/mlpipeline-metrics.json',
}
op.add_volume(
k8s_client.V1Volume(name='volume',
empty_dir=k8s_client.V1EmptyDirVolumeSource()))
op.container.add_volume_mount(
k8s_client.V1VolumeMount(name='volume', mount_path='/output'))
return op
def _mount_v3iod(task):
from kubernetes import client as k8s_client
def add_vol(name, mount_path, host_path):
vol = k8s_client.V1Volume(
name=name,
host_path=k8s_client.V1HostPathVolumeSource(path=host_path,
type=""),
)
task.add_volume(vol).add_volume_mount(
k8s_client.V1VolumeMount(mount_path=mount_path, name=name))
add_vol(name="shm",
mount_path="/dev/shm",
host_path="/dev/shm/" + namespace)
add_vol(
name="v3iod-comm",
mount_path="/var/run/iguazio/dayman",
host_path="/var/run/iguazio/dayman/" + namespace,
)
vol = k8s_client.V1Volume(
name="daemon-health",
empty_dir=k8s_client.V1EmptyDirVolumeSource())
task.add_volume(vol).add_volume_mount(
k8s_client.V1VolumeMount(
mount_path="/var/run/iguazio/daemon_health",
name="daemon-health"))
vol = k8s_client.V1Volume(
name="v3io-config",
config_map=k8s_client.V1ConfigMapVolumeSource(
name=v3io_config_configmap, default_mode=420),
)
task.add_volume(vol).add_volume_mount(
k8s_client.V1VolumeMount(mount_path="/etc/config/v3io",
name="v3io-config"))
task.add_env_variable(
k8s_client.V1EnvVar(
name="CURRENT_NODE_IP",
value_from=k8s_client.V1EnvVarSource(
field_ref=k8s_client.V1ObjectFieldSelector(
api_version="v1", field_path="status.hostIP")),
))
task.add_env_variable(
k8s_client.V1EnvVar(name="IGZ_DATA_CONFIG_FILE",
value="/igz/java/conf/v3io.conf"))
return task
def define_empty_dir_volume(self, name, medium="", size_limit=None):
"""
Represents an empty directory for a pod. Empty directory volumes support ownership management and SELinux
relabeling by default, emptyDir volumes are stored on whatever medium is backing the node - that might be disk
or SSD or network storage, depending on your environment. However, you can set the emptyDir.medium field to
"Memory" to tell Kubernetes to mount a tmpfs (RAM-backed filesystem) for you instead. While tmpfs is very fast,
be aware that unlike disks, tmpfs is cleared on node reboot and any files you write will count against your
container’s memory limit.
@param name ,, str name of volume that is created.
@param medium ,,str What type of storage medium should back this directory. The default is "" which means to use the node's default medium. Must be an empty string (default) or Memory.
@param size_limit ,,int Total amount of local storage required for this EmptyDir volume.
"""
empty_dir_vol = client.V1EmptyDirVolumeSource(medium=medium,
size_limit=sizeLimit)
return client.V1Volume(name=name, empty_dir=empty_dir_vol)
def get_shm_volumes():
"""
Mount an tmpfs volume to /dev/shm.
This will set /dev/shm size to half of the RAM of node.
By default, /dev/shm is very small, only 64MB.
Some experiments will fail due to lack of share memory,
such as some experiments running on Pytorch.
"""
volumes, volume_mounts = [], []
shm_volume = client.V1Volume(
name=constants.SHM_VOLUME,
empty_dir=client.V1EmptyDirVolumeSource(medium='Memory'))
volumes.append(shm_volume)
shm_volume_mount = client.V1VolumeMount(name=shm_volume.name,
mount_path='/dev/shm')
volume_mounts.append(shm_volume_mount)
return volumes, volume_mounts
def demo_op(name: str, is_exit_handler=False) -> ContainerOp:
op = ContainerOp(name=name,
image='alpine:latest',
command=['sh', '-c'],
arguments=[
'echo "Running step $0" && echo "$1" > $2',
name,
markdown_metadata(name),
METADATA_FILE_PATH,
],
is_exit_handler=is_exit_handler,
output_artifact_paths=default_artifact_path())
op.add_volume(
k8s.V1Volume(name='volume',
empty_dir=k8s.V1EmptyDirVolumeSource())).add_volume_mount(
k8s.V1VolumeMount(name='volume', mount_path=OUT_DIR))
return op
def with_empty_dir(self, name: str, mount_path: str) -> Optional["HmlContainerOp"]:
"""
Create an empy, writable volume with the given `name` mounted to the
specified `mount_path`
Args:
name (str): The name of the volume to mount
mount_path (str): The path to mount the empty volume
Returns:
A reference to the current `HmlContainerOp` (self)
"""
# Add a writable volume
self.op.add_volume(k8s_client.V1Volume(name=name, empty_dir=k8s_client.V1EmptyDirVolumeSource()))
self.op.add_volume_mount(k8s_client.V1VolumeMount(name=name, mount_path=mount_path))
return self
def _generate_pod_template(self, *args, **kwargs):
containers = kwargs.get("containers", [])
initial_containers = kwargs.get("initial_containers", [])
volumes_json = kwargs.get("volumes", [])
deploy_name = kwargs.get("name")
labels = kwargs.get("labels", {})
labels.update({"app": deploy_name})
restart_policy = kwargs.get("restart_policy", "Always")
volumes = []
for volume in volumes_json:
volume_name = volume.get("name")
host_path = volume.get("host_path", None)
empty_dir = volume.get("empty_dir", None)
parameters = {}
if host_path:
host_path = client.V1HostPathVolumeSource(path=host_path)
parameters.update({"host_path": host_path})
if empty_dir:
empty_dir = client.V1EmptyDirVolumeSource(**empty_dir)
parameters.update({"empty_dir": empty_dir})
persistent_volume_claim = volume.get("pvc", None)
if persistent_volume_claim:
persistent_volume_claim = client.V1PersistentVolumeClaimVolumeSource(
claim_name=persistent_volume_claim)
parameters.update(
{"persistent_volume_claim": persistent_volume_claim})
volumes.append(client.V1Volume(name=volume_name, **parameters))
initial_container_pods = self._generate_container_pods(
initial_containers)
container_pods = self._generate_container_pods(containers)
pod_spec = client.V1PodSpec(
init_containers=initial_container_pods,
containers=container_pods,
volumes=volumes,
restart_policy=restart_policy,
)
spec_metadata = client.V1ObjectMeta(labels=labels)
template_spec = client.V1PodTemplateSpec(metadata=spec_metadata,
spec=pod_spec)
LOG.info("template spec %s", template_spec)
return template_spec
def create_deployment_object(scale, cpu, gpu, instance_name, mem, isSSD):
container = client.V1Container(name = instance_name,
resource = client.V1ResourceRequirements(requests = \
dict(cpu = cpu, memory = mem)))
if isSSD:
volume_medium = 'SSD'
else:
volume_medium = 'memory'
volume = client.V1Volume(name = instance_name,
emptyDir = client.V1EmptyDirVolumeSource(medium=volume_medium))
template = client.V1PodTemplateSpec(spec = client.V1PodSpec(containers=[container], volumes =[volume]))
spec = client.ExtensionsV1beta1DeploymentSpec(replicas=scale,
template=template)
deployment = client.ExtensionsV1beta1Deployment(api_version="extensions/v1beta1",
kind="Deployment",
spec=spec)
return deployment
def _mount_v3iod(task):
from kubernetes import client as k8s_client
def add_vol(name, mount_path, host_path):
vol = k8s_client.V1Volume(
name=name,
host_path=k8s_client.V1HostPathVolumeSource(path=host_path,
type=''))
task.add_volume(vol).add_volume_mount(
k8s_client.V1VolumeMount(mount_path=mount_path, name=name))
add_vol(name='shm',
mount_path='/dev/shm',
host_path='/dev/shm/' + namespace)
add_vol(name='v3iod-comm',
mount_path='/var/run/iguazio/dayman',
host_path='/var/run/iguazio/dayman/' + namespace)
vol = k8s_client.V1Volume(
name='daemon-health',
empty_dir=k8s_client.V1EmptyDirVolumeSource())
task.add_volume(vol).add_volume_mount(
k8s_client.V1VolumeMount(
mount_path='/var/run/iguazio/daemon_health',
name='daemon-health'))
vol = k8s_client.V1Volume(
name='v3io-config',
config_map=k8s_client.V1ConfigMapVolumeSource(
name=v3io_config_configmap, default_mode=420))
task.add_volume(vol).add_volume_mount(
k8s_client.V1VolumeMount(mount_path='/etc/config/v3io',
name='v3io-config'))
vol = k8s_client.V1Volume(name='v3io-auth',
secret=k8s_client.V1SecretVolumeSource(
secret_name=v3io_auth_secret,
default_mode=420))
task.add_volume(vol).add_volume_mount(
k8s_client.V1VolumeMount(mount_path='/igz/.igz', name='v3io-auth'))
return (task)
def kube_body(self):
"""
Build kubernetes deployment
:return: client.V1Deployment
"""
return client.V1Deployment(
metadata=client.V1ObjectMeta(name=self.ref),
spec=client.V1DeploymentSpec(
replicas=1,
selector=client.V1LabelSelector(
match_labels={'mintzone/ref': self.ref}),
template=client.V1PodTemplateSpec(
metadata=client.V1ObjectMeta(
labels={"mintzone/ref": self.ref}),
spec=client.V1PodSpec(
containers=self.containers,
volumes=[
client.V1Volume(
name='data',
empty_dir=client.V1EmptyDirVolumeSource(),
)
]))))
def create_selenium_pod_spec(selenium_type,
container_name,
agent_container_ports,
env_vars,
node_selector=None,
pod_limits=None):
if selenium_type == 'firefox':
agent_image_name = 'pymada/selenium-firefox',
elif selenium_type == 'chrome':
agent_image_name = 'pymada/selenium-chrome'
pod_resource_requirements = None
if pod_limits is not None and type(pod_limits) == dict:
pod_resource_requirements = client.V1ResourceRequirements(
limits=pod_limits)
selenium_ports = [client.V1ContainerPort(container_port=4444)
] + agent_container_ports
selenium_container = client.V1Container(
name=container_name,
image=agent_image_name,
ports=selenium_ports,
env=env_vars,
volume_mounts=[
client.V1VolumeMount(mount_path='/dev/shm', name='dshm')
],
resources=pod_resource_requirements)
pod_spec = client.V1PodSpec(
containers=[selenium_container],
node_selector=node_selector,
volumes=[
client.V1Volume(
name='dshm',
empty_dir=client.V1EmptyDirVolumeSource(medium='Memory'))
])
return pod_spec
def _create_kube_podspec(self,
container,
shm_size=None,
scheduler_name=KubernetesConfig.DEFAULT_SCHEDULER,
namespace=KubernetesConfig.K8S_NAMESPACE):
logger.info("Creating pod with scheduler_name = %s, namespace = %s" %
(scheduler_name, namespace))
volumes = []
if shm_size and isinstance(shm_size, int):
dshm_vol = client.V1EmptyDirVolumeSource(medium="Memory",
size_limit=shm_size)
volumes.append(client.V1Volume(name='dshm', empty_dir=dshm_vol))
if not volumes:
volumes = None
spec = client.V1PodSpec(containers=[container],
scheduler_name=scheduler_name,
restart_policy="OnFailure",
volumes=volumes,
image_pull_secrets=[
client.V1LocalObjectReference(
KubernetesConfig.IMAGE_PULL_SECRET)
])
return spec
def _build_container_op(self, overrides=dict()):
artifacts_volume = k8s_client.V1Volume(
name="artifacts", empty_dir=k8s_client.V1EmptyDirVolumeSource())
op = dsl.ContainerOp(name=f"{self.pipeline_name}-{self.op_name}",
image=self.container_image_url,
command=self.container_command,
arguments=self.container_args,
file_outputs=self.output_files,
output_artifact_paths=self.output_artifact_paths)
# Mount an empty direcotry for us to write output to
self._mount_empty_dir(op, "artifacts", self.kfp_artifact_path)
# Apply the GCP Auth secret
if not self.gcp_auth_secret is None:
op.apply(use_gcp_secret(self.gcp_auth_secret))
# Apply other secrets
for secret_name in self.secrets:
mount_path = self.secrets[secret_name]
op.apply(self._bind_secret(secret_name, mount_path))
logging.info(f"Build Container {self.pipeline_name}.{self.op_name}")
# All my parameters
for name, value in self.data.items():
logging.info(f"\tContainer.ENV: {name}={value}")
op.container.add_env_variable(V1EnvVar(name=name,
value=str(value)))
for name, value in overrides.items():
logging.info(f"\tContainer.ENV: {name}={value} (override)")
op.container.add_env_variable(V1EnvVar(name=name,
value=str(value)))
return op
def create_rc_object(scale, cpu, gpu, instance_name, mem, isSSD):
resources = client.V1ResourceRequirements(
requests=dict(cpu=cpu, memory=mem))
containers = client.V1Container(name=instance_name, resources=resources)
if isSSD:
medium = 'memory'
else:
medium = 'SSD'
empty_dir = client.V1EmptyDirVolumeSource(medium=medium, size_limit=mem)
volumes = client.V1Volume(name=instance_name, empty_dir=empty_dir)
pod_spec = client.V1PodSpec(containers=[containers], volumes=[volumes])
template = client.V1PodTemplateSpec(spec=spec)
rc_spec = client.V1ReplicationControllerSpec(selector=dict(
name, instance_name),
replicas=scale,
template=template)
metadata = client.V1ObjectMeta(labels=dict(name, instance_name),
name=instance_name)
rc = client.V1ReplicationController(api_version='corev1',
kind='ReplicationController',
spec=rc_spec,
metadata=metadata)
return rc
def get_mcrouter_deployment_object(cluster_object):
name = cluster_object['metadata']['name']
namespace = cluster_object['metadata']['namespace']
try:
replicas = cluster_object['spec']['mcrouter']['replicas']
except KeyError:
replicas = 1
try:
mcrouter_limit_cpu = \
cluster_object['spec']['mcrouter']['mcrouter_limit_cpu']
except KeyError:
mcrouter_limit_cpu = '50m'
try:
mcrouter_limit_memory = \
cluster_object['spec']['mcrouter']['mcrouter_limit_memory']
except KeyError:
mcrouter_limit_memory = '32Mi'
deployment = client.AppsV1beta1Deployment()
# Metadata
deployment.metadata = client.V1ObjectMeta(
name="{}-router".format(name),
namespace=namespace,
labels=get_default_labels(name=name))
deployment.metadata.labels['service-type'] = 'mcrouter'
# Spec
deployment.spec = client.AppsV1beta1DeploymentSpec(
replicas=replicas, template=client.V1PodTemplateSpec())
deployment.spec.template = client.V1PodTemplateSpec()
deployment.spec.template.metadata = client.V1ObjectMeta(
labels=deployment.metadata.labels)
deployment.spec.template.spec = client.V1PodSpec(containers=[])
# Mcrouter container
mcrouter_config_volumemount = client.V1VolumeMount(
name='mcrouter-config', read_only=False, mount_path='/etc/mcrouter')
mcrouter_port = client.V1ContainerPort(name='mcrouter',
container_port=11211,
protocol='TCP')
mcrouter_resources = client.V1ResourceRequirements(
limits={
'cpu': mcrouter_limit_cpu,
'memory': mcrouter_limit_memory
},
requests={
'cpu': mcrouter_limit_cpu,
'memory': mcrouter_limit_memory
})
mcrouter_container = client.V1Container(
name='mcrouter',
command=[
'mcrouter', '-p', '11211', '-f', '/etc/mcrouter/mcrouter.conf'
],
image='kubestack/mcrouter:v0.36.0-kbst1',
ports=[mcrouter_port],
volume_mounts=[mcrouter_config_volumemount],
resources=mcrouter_resources)
# Mcrouter config sidecar
sidecar_resources = client.V1ResourceRequirements(limits={
'cpu': '25m',
'memory': '8Mi'
},
requests={
'cpu': '25m',
'memory': '8Mi'
})
sidecar_config_volumemount = client.V1VolumeMount(
name='mcrouter-config', read_only=True, mount_path='/etc/mcrouter')
sidecar_container = client.V1Container(
name='config-sidecar',
args=[
"--debug", "--output=/etc/mcrouter/mcrouter.conf",
"{}-backend.{}.svc.cluster.local".format(name, namespace)
],
image='kubestack/mcrouter_sidecar:v0.1.0',
volume_mounts=[mcrouter_config_volumemount],
resources=sidecar_resources)
# Config Map Volume
mcrouter_config_volume = client.V1Volume(
name='mcrouter-config', empty_dir=client.V1EmptyDirVolumeSource())
deployment.spec.template.spec.volumes = [mcrouter_config_volume]
# Metrics container
metrics_port = client.V1ContainerPort(name='metrics',
container_port=9150,
protocol='TCP')
metrics_resources = client.V1ResourceRequirements(limits={
'cpu': '50m',
'memory': '16Mi'
},
requests={
'cpu': '50m',
'memory': '16Mi'
})
metrics_container = client.V1Container(
name='prometheus-exporter',
image='kubestack/mcrouter_exporter:v0.0.1',
args=[
'-mcrouter.address', 'localhost:11211', '-web.listen-address',
':9150'
],
ports=[metrics_port],
resources=metrics_resources)
deployment.spec.template.spec.containers = [
mcrouter_container, sidecar_container, metrics_container
]
return deployment
""" Creating a Job in a specific namespace"""
from kubernetes import client, config
# Fetching and loading Kubernetes Information
config.load_kube_config()
extension = client.BatchV1Api()
# Volume
volume = client.V1Volume(name="test-volume",
empty_dir=client.V1EmptyDirVolumeSource(medium=""))
# Container
container = client.V1Container(
name="jobtest",
image="nginx:1.7.9",
image_pull_policy="IfNotPresent",
ports=[client.V1ContainerPort(container_port=80)],
volume_mounts=[
client.V1VolumeMount(name=volume.name, mount_path="/kube-example")
],
)
# Init-Container
init_container = client.V1Container(
name="init-container",
image="alpine",
image_pull_policy="IfNotPresent",
command=["touch kube-test.txt"],
volume_mounts=[
def create_job_object(name: str,
container_image: str,
env_list: dict,
command: List[str],
command_args: List[str],
volumes: List[Dict],
init_containers: List[Dict],
output: Output,
namespace: str = "stackl",
container_name: str = "jobcontainer",
api_version: str = "batch/v1",
image_pull_policy: str = "Always",
ttl_seconds_after_finished: int = 3600,
restart_policy: str = "Never",
backoff_limit: int = 0,
active_deadline_seconds: int = 3600,
service_account: str = "stackl-agent-stackl-agent",
image_pull_secrets: List[str] = [],
labels=None) -> client.V1Job:
# pylint: disable=too-many-arguments,too-many-locals,too-many-branches,too-many-statements
"""Creates a Job object using the Kubernetes client
:param name: Job name affix
:type name: str
:param container_image: automation container image
:type container_image: str
:param env_list: Dict with key/values for the environment inside the automation container
:type env_list: dict
:param command: entrypoint command
:type command: List[str]
:param command_args: command arguments
:type command_args: List[str]
:param volumes: volumes and volumemounts
:type volumes: List[Dict]
:param image_pull_secrets: secrets to pull images
:type image_pull_secrets: List[str]
:param init_containers: list with init_containers
:type init_containers: List[Dict]
:param output: output Object
:type output: Output
:param namespace: Kubernetes namespace, defaults to "stackl"
:type namespace: str, optional
:param container_name: name of automation container, defaults to "jobcontainer"
:type container_name: str, optional
:param api_version: Job api version, defaults to "batch/v1"
:type api_version: str, optional
:param image_pull_policy: always pull latest images, defaults to "Always"
:type image_pull_policy: str, optional
:param ttl_seconds_after_finished: Remove jobs after execution with ttl, defaults to 600
:type ttl_seconds_after_finished: int, optional
:param restart_policy: Restart the pod on the same node after failure, defaults to "Never"
:type restart_policy: str, optional
:param backoff_limit: Retries after failure, defaults to 0
:type backoff_limit: int, optional
:param active_deadline_seconds: Timeout on a job, defaults to 3600 seconds
:type active_deadline_seconds: int, optional
:param service_account: Kubernetes service account, defaults to "stackl-agent-stackl-agent"
:type service_account: str, optional
:param labels: metadata labels, defaults to {}
:type labels: dict, optional
:return: automation Job object
:rtype: client.V1Job
"""
id_job = id_generator()
name = name + "-" + id_job
body = client.V1Job(api_version=api_version, kind="Job")
body.metadata = client.V1ObjectMeta(namespace=namespace, name=name)
body.status = client.V1JobStatus()
template = client.V1PodTemplate()
template.template = client.V1PodTemplateSpec()
k8s_volumes = []
cms = []
logging.debug(f"volumes: {volumes}")
# create a k8s volume for each element in volumes
for vol in volumes:
vol_name = name + "-" + vol["name"]
k8s_volume = client.V1Volume(name=vol_name)
if vol["type"] == "config_map":
config_map = client.V1ConfigMapVolumeSource()
config_map.name = vol_name
k8s_volume.config_map = config_map
cms.append(create_cm(vol_name, namespace, vol['data']))
vol['name'] = vol_name
if vol["type"] == "empty_dir":
k8s_volume.empty_dir = client.V1EmptyDirVolumeSource(
medium="Memory")
vol['name'] = vol_name
k8s_volumes.append(k8s_volume)
logging.debug(f"Volumes created for job {name}: {k8s_volumes}")
# create a volume mount for each element in volumes
k8s_volume_mounts = []
for vol in volumes:
if vol["mount_path"]:
volume_mount = client.V1VolumeMount(name=vol["name"],
mount_path=vol["mount_path"])
if "sub_path" in vol:
volume_mount.sub_path = vol["sub_path"]
k8s_volume_mounts.append(volume_mount)
logging.debug(f"Volume mounts created for job {name}: {k8s_volume_mounts}")
# create an environment list
k8s_env_list = []
if env_list:
for key, value in env_list.items():
if isinstance(value, dict):
if 'config_map_key_ref' in value:
k8s_env_from = client.V1EnvVar(
name=key,
value_from=client.V1EnvVarSource(
config_map_key_ref=client.V1ConfigMapKeySelector(
name=value['config_map_key_ref']["name"],
key=value['config_map_key_ref']["key"])))
k8s_env_list.append(k8s_env_from)
elif 'field_ref' in value:
k8s_env_from = client.V1EnvVar(
name=key,
value_from=client.V1EnvVarSource(
field_ref=client.V1ObjectFieldSelector(
field_path=value['field_ref'])))
k8s_env_list.append(k8s_env_from)
else:
k8s_env = client.V1EnvVar(name=key, value=value)
k8s_env_list.append(k8s_env)
k8s_env_from_list = []
# if env_from:
# for env in env_from:
# if 'config_map_ref' in env:
# k8s_env_from = client.V1EnvFromSource(
# config_map_ref=env['config_map_ref'])
# k8s_env_from_list.append(k8s_env_from)
# elif 'secret_ref' in env:
# k8s_env_from = client.V1EnvFromSource(
# secret_ref=env['secret_ref'])
# k8s_env_from_list.append(k8s_env_from)
logging.debug(f"Environment list created for job {name}: {k8s_env_list}")
print(f"Environment list created for job {name}: {k8s_env_list}")
container = client.V1Container(name=container_name,
image=container_image,
env=k8s_env_list,
volume_mounts=k8s_volume_mounts,
image_pull_policy=image_pull_policy,
command=command,
args=command_args,
env_from=k8s_env_from_list)
k8s_init_containers = []
logging.debug(f"Init containers for job {name}: {init_containers}")
for c in init_containers:
k8s_c = client.V1Container(name=c['name'],
image=c['image'],
volume_mounts=k8s_volume_mounts,
env=k8s_env_list)
if 'args' in c:
k8s_c.args = c['args']
k8s_init_containers.append(k8s_c)
k8s_secrets = []
for secret in image_pull_secrets:
k8s_secrets.append(client.V1LocalObjectReference(name=secret))
logging.debug(f"Secret list created for job {name}: {k8s_secrets}")
containers = [container]
if output:
output.volume_mounts = k8s_volume_mounts
output.env = k8s_env_list
output_containers = output.containers
containers = containers + output_containers
template.template.metadata = client.V1ObjectMeta(labels=labels)
template.template.spec = client.V1PodSpec(
containers=containers,
restart_policy=restart_policy,
image_pull_secrets=k8s_secrets,
volumes=k8s_volumes,
init_containers=k8s_init_containers,
service_account_name=service_account)
template.template = client.V1PodTemplateSpec(
metadata=template.template.metadata, spec=template.template.spec)
body.spec = client.V1JobSpec(
ttl_seconds_after_finished=ttl_seconds_after_finished,
template=template.template,
backoff_limit=backoff_limit,
active_deadline_seconds=active_deadline_seconds)
return body, cms
def create_deployment_old(config_file):
"""
Create IBM Spectrum Scale CSI Operator deployment object in operator namespace using
deployment_operator_image_for_crd and deployment_driver_image_for_crd parameters from
config.json file
Args:
param1: config_file - configuration json file
Returns:
None
Raises:
Raises an exception on kubernetes client api failure and asserts
"""
deployment_apps_api_instance = client.AppsV1Api()
deployment_labels = {
"app.kubernetes.io/instance": "ibm-spectrum-scale-csi-operator",
"app.kubernetes.io/managed-by": "ibm-spectrum-scale-csi-operator",
"app.kubernetes.io/name": "ibm-spectrum-scale-csi-operator",
"product": "ibm-spectrum-scale-csi",
"release": "ibm-spectrum-scale-csi-operator"
}
deployment_annotations = {
"productID": "ibm-spectrum-scale-csi-operator",
"productName": "IBM Spectrum Scale CSI Operator",
"productVersion": "2.0.0"
}
deployment_metadata = client.V1ObjectMeta(
name="ibm-spectrum-scale-csi-operator",
labels=deployment_labels,
namespace=namespace_value)
deployment_selector = client.V1LabelSelector(
match_labels={
"app.kubernetes.io/name": "ibm-spectrum-scale-csi-operator"
})
podtemplate_metadata = client.V1ObjectMeta(
labels=deployment_labels, annotations=deployment_annotations)
pod_affinity = client.V1Affinity(node_affinity=client.V1NodeAffinity(
required_during_scheduling_ignored_during_execution=client.
V1NodeSelector(node_selector_terms=[
client.V1NodeSelectorTerm(match_expressions=[
client.V1NodeSelectorRequirement(key="beta.kubernetes.io/arch",
operator="Exists")
])
])))
ansible_pod_container = client.V1Container(
image=config_file["deployment_operator_image_for_crd"],
command=[
"/usr/local/bin/ao-logs", "/tmp/ansible-operator/runner", "stdout"
],
liveness_probe=client.V1Probe(
_exec=client.V1ExecAction(command=["/health_check.sh"]),
initial_delay_seconds=10,
period_seconds=30),
readiness_probe=client.V1Probe(
_exec=client.V1ExecAction(command=["/health_check.sh"]),
initial_delay_seconds=3,
period_seconds=1),
name="ansible",
image_pull_policy="IfNotPresent",
security_context=client.V1SecurityContext(
capabilities=client.V1Capabilities(drop=["ALL"])),
volume_mounts=[
client.V1VolumeMount(mount_path="/tmp/ansible-operator/runner",
name="runner",
read_only=True)
],
env=[
client.V1EnvVar(
name="CSI_DRIVER_IMAGE",
value=config_file["deployment_driver_image_for_crd"])
])
operator_pod_container = client.V1Container(
image=config_file["deployment_operator_image_for_crd"],
name="operator",
image_pull_policy="IfNotPresent",
liveness_probe=client.V1Probe(
_exec=client.V1ExecAction(command=["/health_check.sh"]),
initial_delay_seconds=10,
period_seconds=30),
readiness_probe=client.V1Probe(
_exec=client.V1ExecAction(command=["/health_check.sh"]),
initial_delay_seconds=3,
period_seconds=1),
security_context=client.V1SecurityContext(
capabilities=client.V1Capabilities(drop=["ALL"])),
env=[
client.V1EnvVar(name="WATCH_NAMESPACE",
value_from=client.V1EnvVarSource(
field_ref=client.V1ObjectFieldSelector(
field_path="metadata.namespace"))),
client.V1EnvVar(name="POD_NAME",
value_from=client.V1EnvVarSource(
field_ref=client.V1ObjectFieldSelector(
field_path="metadata.name"))),
client.V1EnvVar(name="OPERATOR_NAME",
value="ibm-spectrum-scale-csi-operator"),
client.V1EnvVar(
name="CSI_DRIVER_IMAGE",
value=config_file["deployment_driver_image_for_crd"])
],
volume_mounts=[
client.V1VolumeMount(mount_path="/tmp/ansible-operator/runner",
name="runner")
])
pod_spec = client.V1PodSpec(
affinity=pod_affinity,
containers=[ansible_pod_container, operator_pod_container],
service_account_name="ibm-spectrum-scale-csi-operator",
volumes=[
client.V1Volume(
empty_dir=client.V1EmptyDirVolumeSource(medium="Memory"),
name="runner")
])
podtemplate_spec = client.V1PodTemplateSpec(metadata=podtemplate_metadata,
spec=pod_spec)
deployment_spec = client.V1DeploymentSpec(replicas=1,
selector=deployment_selector,
template=podtemplate_spec)
body_dep = client.V1Deployment(kind='Deployment',
api_version='apps/v1',
metadata=deployment_metadata,
spec=deployment_spec)
try:
LOGGER.info("creating deployment for operator")
deployment_apps_api_response = deployment_apps_api_instance.create_namespaced_deployment(
namespace=namespace_value, body=body_dep)
LOGGER.debug(str(deployment_apps_api_response))
except ApiException as e:
LOGGER.error(
f"Exception when calling RbacAuthorizationV1Api->create_namespaced_deployment: {e}"
)
assert False
