def get_volume_mounts(
self,
docker_volumes: Sequence[DockerVolume],
aws_ebs_volumes: Sequence[AwsEbsVolume],
persistent_volumes: Sequence[PersistentVolume],
) -> Sequence[V1VolumeMount]:
return [
V1VolumeMount(
mount_path=docker_volume['containerPath'],
name=self.get_docker_volume_name(docker_volume),
read_only=self.read_only_mode(docker_volume),
) for docker_volume in docker_volumes
] + [
V1VolumeMount(
mount_path=aws_ebs_volume['container_path'],
name=self.get_aws_ebs_volume_name(aws_ebs_volume),
read_only=self.read_only_mode(aws_ebs_volume),
) for aws_ebs_volume in aws_ebs_volumes
] + [
V1VolumeMount(
mount_path=volume['container_path'],
name=self.get_persistent_volume_name(volume),
read_only=self.read_only_mode(volume),
) for volume in persistent_volumes
]
def test_get_volume_mounts(self):
with mock.patch(
'paasta_tools.kubernetes_tools.KubernetesDeploymentConfig.get_sanitised_volume_name',
autospec=True,
return_value='some-volume',
):
mock_volumes = [
{
'hostPath': '/nail/blah',
'containerPath': '/nail/foo'
},
{
'hostPath': '/nail/thing',
'containerPath': '/nail/bar',
'mode': 'RW'
},
]
expected_volumes = [
V1VolumeMount(
mount_path='/nail/foo',
name='some-volume',
read_only=True,
),
V1VolumeMount(
mount_path='/nail/bar',
name='some-volume',
read_only=False,
),
]
assert self.deployment.get_volume_mounts(
mock_volumes) == expected_volumes
def __init__(self) -> None:
metadata = V1ObjectMeta(name="postgres", labels={"app": "postgres"})
label_selector = V1LabelSelector(match_labels={"app": "postgres"})
env = [V1EnvVar(name="POSTGRES_HOST_AUTH_METHOD", value="trust")]
ports = [V1ContainerPort(container_port=5432, name="sql")]
volume_mounts = [
V1VolumeMount(name="data", mount_path="/data"),
V1VolumeMount(
name="postgres-init", mount_path="/docker-entrypoint-initdb.d"
),
]
volume_config = V1ConfigMapVolumeSource(
name="postgres-init",
)
volumes = [V1Volume(name="postgres-init", config_map=volume_config)]
container = V1Container(
name="postgres",
image="postgres:14.3",
env=env,
ports=ports,
volume_mounts=volume_mounts,
)
pod_spec = V1PodSpec(containers=[container], volumes=volumes)
template_spec = V1PodTemplateSpec(metadata=metadata, spec=pod_spec)
claim_templates = [
V1PersistentVolumeClaim(
metadata=V1ObjectMeta(name="data"),
spec=V1PersistentVolumeClaimSpec(
access_modes=["ReadWriteOnce"],
resources=V1ResourceRequirements(requests={"storage": "1Gi"}),
),
)
]
self.stateful_set = V1StatefulSet(
api_version="apps/v1",
kind="StatefulSet",
metadata=metadata,
spec=V1StatefulSetSpec(
service_name="postgres",
replicas=1,
selector=label_selector,
template=template_spec,
volume_claim_templates=claim_templates,
),
)
def setUp(self):
super().setUp()
self.cluster_dict = getExampleClusterDefinition()
self.cluster_object = V1MongoClusterConfiguration(**self.cluster_dict)
self.name = self.cluster_object.metadata.name
self.namespace = self.cluster_object.metadata.namespace
self.stateful_set = V1beta1StatefulSet(
metadata=self._createMeta(self.name),
spec=V1beta1StatefulSetSpec(
replicas=3,
service_name=self.name,
template=V1PodTemplateSpec(
metadata=V1ObjectMeta(labels=KubernetesResources.
createDefaultLabels(self.name)),
spec=V1PodSpec(containers=[
V1Container(
name="mongodb",
env=[
V1EnvVar(name="POD_IP",
value_from=V1EnvVarSource(
field_ref=V1ObjectFieldSelector(
api_version="v1",
field_path="status.podIP")))
],
command=[
"mongod", "--replSet", self.name, "--bind_ip",
"0.0.0.0", "--smallfiles", "--noprealloc"
],
image="mongo:3.6.4",
ports=[
V1ContainerPort(name="mongodb",
container_port=27017,
protocol="TCP")
],
volume_mounts=[
V1VolumeMount(name="mongo-storage",
read_only=False,
mount_path="/data/db")
],
resources=V1ResourceRequirements(limits={
"cpu": "100m",
"memory": "64Mi"
},
requests={
"cpu": "100m",
"memory":
"64Mi"
}))
])),
volume_claim_templates=[
V1PersistentVolumeClaim(
metadata=V1ObjectMeta(name="mongo-storage"),
spec=V1PersistentVolumeClaimSpec(
access_modes=["ReadWriteOnce"],
resources=V1ResourceRequirements(
requests={"storage": "30Gi"})))
],
),
)
def define_configmap(self, data):
"""This returns a k8s configmap using the data from the new-workflow
POST.
"""
with start_action(action_type="define_configmap"):
ni_cmd = data["command"]
idkey = list_digest(ni_cmd)
cm_name = "command.{}.json".format(idkey)
k8s_vol = V1Volume(
name="noninteractive-command",
config_map=V1ConfigMapVolumeSource(name=cm_name),
)
k8s_mt = V1VolumeMount(
name="noninteractive-command",
mount_path=("/opt/lsst/software/jupyterlab/" +
"noninteractive/command/"),
read_only=True,
)
self.cmd_vol = k8s_vol
self.cmd_mt = k8s_mt
# Now the configmap
cm_data = {}
cm_data.update(data)
del cm_data["image"]
del cm_data["size"]
jd = json.dumps(data, sort_keys=True, indent=4)
k8s_configmap = V1ConfigMap(
metadata=V1ObjectMeta(name=cm_name),
data={"command.json": json.dumps(data)},
)
self.log.debug("Created configmap '{}': {}".format(cm_name, jd))
self.cfg_map = k8s_configmap
def __init__(self,
name,
mount,
fs_type,
image,
monitors,
pool,
secret_name,
sub_path,
user="******",
read_only=False):
self.mount = V1VolumeMount(name=name,
mount_path=mount,
read_only=read_only,
sub_path=sub_path)
self.volume = V1Volume(
name=name,
rbd=V1RBDVolumeSource(
fs_type=fs_type,
image=image,
monitors=monitors.split(","),
pool=pool,
secret_ref=V1LocalObjectReference(secret_name),
read_only=read_only,
user=user))
def create_volumes(task_volumes) -> Tuple[List[V1Volume], List[V1VolumeMount]]:
index = 0
mounts = []
volumes = []
for target, volume in task_volumes.items():
index += 1
name = volume.get('name', f'volume{index}')
for source_type, VolumeSource in VOLUME_SOURCES.items():
if source_type not in volume:
continue
volume_config = volume[source_type]
volumes.append(
V1Volume(**{
'name': name,
source_type: VolumeSource(**volume_config),
}))
mounts.append(
V1VolumeMount(
name=name,
read_only=volume.get('read_only', False),
mount_path=target,
))
return volumes, mounts
def __init__(self, name, mount, claim_name, read_only=False):
self.mount = V1VolumeMount(name=name,
mount_path=mount,
read_only=read_only)
self.volume = V1Volume(
name=name,
persistent_volume_claim=V1PersistentVolumeClaimVolumeSource(
claim_name=claim_name))
def get_volume_mounts(self, volumes: Sequence[DockerVolume]) -> Sequence[V1VolumeMount]:
volume_mounts = []
for volume in volumes:
volume_mounts.append(
V1VolumeMount(
mount_path=volume['containerPath'],
name=self.get_sanitised_volume_name(volume['containerPath']),
read_only=True if volume.get('mode', 'RO') == 'RO' else False,
),
)
return volume_mounts
def _create_flush_job(
batch_api: BatchV1Api,
command: List[str],
env: List[V1EnvVar],
image: str,
name: str,
namespace: str,
service_account_name: str,
) -> V1Job:
logger.info(f"creating job: {name}")
try:
return batch_api.create_namespaced_job(
namespace=namespace,
body=V1Job(
api_version="batch/v1",
kind="Job",
metadata=V1ObjectMeta(name=name, namespace=namespace),
spec=V1JobSpec(
template=V1PodTemplateSpec(
spec=V1PodSpec(
containers=[
V1Container(
image=image,
command=command,
name="flush",
volume_mounts=[
V1VolumeMount(mount_path="/data", name="queue")
],
env=env,
)
],
restart_policy="OnFailure",
volumes=[
V1Volume(
name="queue",
persistent_volume_claim=(
V1PersistentVolumeClaimVolumeSource(
claim_name=name
)
),
)
],
service_account_name=service_account_name,
)
)
),
),
)
except ApiException as e:
if e.reason == CONFLICT and json.loads(e.body)["reason"] == ALREADY_EXISTS:
logger.info(f"using existing job: {name}")
return batch_api.read_namespaced_job(name, namespace)
raise
def get_kubernetes_volume_mounts(
volumes: PVector['DockerVolume']) -> List[V1VolumeMount]:
"""
Given a list of volume mounts, return a list corresponding to the Kubernetes objects
representing these mounts.
"""
return [
V1VolumeMount(
mount_path=volume["container_path"],
name=get_sanitised_volume_name(f"host--{volume['host_path']}",
length_limit=63),
read_only=volume.get("mode", "RO") == "RO",
) for volume in volumes
]
def volume_pipeline():
op1 = op1_op()
op1.add_volume(
V1Volume(name='gcp-credentials',
secret=V1SecretVolumeSource(secret_name='user-gcp-sa')))
op1.container.add_volume_mount(
V1VolumeMount(mount_path='/secret/gcp-credentials',
name='gcp-credentials'))
op1.container.add_env_variable(
V1EnvVar(name='GOOGLE_APPLICATION_CREDENTIALS',
value='/secret/gcp-credentials/user-gcp-sa.json'))
op1.container.add_env_variable(V1EnvVar(name='Foo', value='bar'))
op2 = op2_op(op1.output)
def core_config_mount(self, name, config_map, key, target_path, read_only=True):
if name not in self.core_config_volumes:
self.core_config_volumes[name] = V1Volume(
name=name,
config_map=V1ConfigMapVolumeSource(
name=config_map,
optional=False
)
)
self.core_config_mounts[target_path] = V1VolumeMount(
name=name,
mount_path=target_path,
sub_path=key,
read_only=read_only
)
def _createStatefulSet(self) -> V1beta1StatefulSet:
return V1beta1StatefulSet(
metadata=self._createMeta(self.name),
spec=V1beta1StatefulSetSpec(
replicas=3,
service_name=self.name,
template=V1PodTemplateSpec(
metadata=V1ObjectMeta(labels=KubernetesResources.
createDefaultLabels(self.name)),
spec=V1PodSpec(containers=[
V1Container(
name="mongodb",
env=[
V1EnvVar(name="POD_IP",
value_from=V1EnvVarSource(
field_ref=V1ObjectFieldSelector(
api_version="v1",
field_path="status.podIP")))
],
command=[
"mongod", "--wiredTigerCacheSizeGB", "0.25",
"--replSet", self.name, "--bind_ip", "0.0.0.0",
"--smallfiles", "--noprealloc"
],
image="mongo:3.6.4",
ports=[
V1ContainerPort(name="mongodb",
container_port=27017,
protocol="TCP")
],
volume_mounts=[
V1VolumeMount(name="mongo-storage",
read_only=False,
mount_path="/data/db")
],
resources=self._createResourceLimits())
])),
volume_claim_templates=[
V1PersistentVolumeClaim(
metadata=V1ObjectMeta(name="mongo-storage"),
spec=V1PersistentVolumeClaimSpec(
access_modes=["ReadWriteOnce"],
resources=V1ResourceRequirements(
requests={"storage": "30Gi"})))
],
),
)
def volume_pipeline():
op1 = dsl.ContainerOp(name='download',
image='google/cloud-sdk',
command=['sh', '-c'],
arguments=['ls | tee /tmp/results.txt'],
file_outputs={'downloaded': '/tmp/results.txt'}) \
.add_volume(V1Volume(name='gcp-credentials',
secret=V1SecretVolumeSource(secret_name='user-gcp-sa'))) \
.add_volume_mount(V1VolumeMount(mount_path='/secret/gcp-credentials',
name='gcp-credentials')) \
.add_env_variable(V1EnvVar(name='GOOGLE_APPLICATION_CREDENTIALS',
value='/secret/gcp-credentials/user-gcp-sa.json')) \
.add_env_variable(V1EnvVar(name='Foo', value='bar'))
op2 = dsl.ContainerOp(name='echo',
image='library/bash',
command=['sh', '-c'],
arguments=['echo %s' % op1.output])
def start_stateful_container(self, service_name, container_name, spec, labels):
# Setup PVC
deployment_name = service_name + '-' + container_name
mounts, volumes = [], []
for volume_name, volume_spec in spec.volumes.items():
mount_name = deployment_name + volume_name
# Check if the PVC exists, create if not
self._ensure_pvc(mount_name, volume_spec.storage_class, volume_spec.capacity)
# Create the volume info
volumes.append(V1Volume(
name=mount_name,
persistent_volume_claim=V1PersistentVolumeClaimVolumeSource(mount_name)
))
mounts.append(V1VolumeMount(mount_path=volume_spec.mount_path, name=mount_name))
self._create_deployment(service_name, deployment_name, spec.container,
30, 1, labels, volumes=volumes, mounts=mounts)
def create_cluster(self, spec, cluster_management, namespace_name,
volume_claim_name):
count = int(spec[FRAMEWORK_RESOURCES][FRAMEWORK_DPU_COUNT])
version = str(spec[FRAMEWORK_VERSION])
image = "tensorflow/tensorflow:" + version + "-gpu"
###
v1_api = cluster_management.kube_api
api_response_list = []
for i in range(count):
body = kubernetes.client.V1Pod()
body.api_version = "v1"
body.kind = "Pod"
meta = V1ObjectMeta()
meta.generate_name = "tensorflow-"
body.metadata = meta
uuid = str(uuid4())
container = V1Container(name=uuid, image=image)
pod_spec = V1PodSpec(containers=[container])
container_mounts = V1VolumeMount(
mount_path=GLUSTER_DEFAULT_MOUNT_PATH,
name=CONTAINER_VOLUME_PREFIX)
container.volume_mounts = [container_mounts]
compute_resource = V1ResourceRequirements()
compute_resource.limits = {"nvidia.com/gpu": 1}
compute_resource.requests = {"nvidia.com/gpu": 1}
container.resources = compute_resource
claim = V1PersistentVolumeClaimVolumeSource(
claim_name=volume_claim_name)
volume_claim = V1Volume(name=CONTAINER_VOLUME_PREFIX,
persistent_volume_claim=claim)
volume_claim.persistent_volume_claim = claim
pod_spec.volumes = [volume_claim]
body.spec = pod_spec
try:
api_response = v1_api.create_namespaced_pod(
namespace_name, body)
except ApiException as e:
raise Exception(
"Exception when calling CoreV1Api->create_namespaced_pod: %s\n"
% e)
api_response_list.append(api_response)
return api_response_list
def __init__(self,
name,
mount,
monitors,
secret_name,
fs_path,
sub_path,
user="******",
read_only=False):
self.mount = V1VolumeMount(name=name,
mount_path=mount,
read_only=read_only,
sub_path=sub_path)
self.volume = V1Volume(
name=name,
cephfs=V1CephFSVolumeSource(
monitors=monitors.split(","),
path=fs_path,
secret_ref=V1LocalObjectReference(secret_name),
read_only=read_only,
user=user))
def _create_volumes(self, service_name):
volumes, mounts = [], []
# Attach the mount that provides the config file
volumes.extend(self.config_volumes.values())
mounts.extend(self.config_mounts.values())
# Attach the mount that provides the update
volumes.append(V1Volume(
name='update-directory',
persistent_volume_claim=V1PersistentVolumeClaimVolumeSource(
claim_name=FILE_UPDATE_VOLUME,
read_only=True
),
))
mounts.append(V1VolumeMount(
name='update-directory',
mount_path=CONTAINER_UPDATE_DIRECTORY,
sub_path=service_name,
read_only=True,
))
return volumes, mounts
def perform_cloud_ops():
# set GOOGLE_APPLICATION_CREDENTIALS env to credentials file
# set GOOGLE_CLOUD_PROJECT env to project id
domain = os.getenv('DOMAIN')
assert domain
logger.info(f'using domain: {domain}')
static_ip = os.getenv('STATIC_IP')
assert static_ip
logger.info(f'using static IP: {static_ip}')
admin_email = os.getenv('ADMIN_EMAIL')
assert admin_email
logger.info(f'using ACME admin email: {admin_email}')
oauth_client_id = os.getenv('OAUTH_CLIENT_ID')
assert oauth_client_id
logger.info(f'using oauth client id: {oauth_client_id}')
oauth_client_secret = os.getenv('OAUTH_CLIENT_SECRET')
assert oauth_client_secret
logger.info(f'using oauth client secret: {oauth_client_secret}')
oauth_secret = os.getenv('OAUTH_SECRET')
assert oauth_secret
logger.info(f'using oauth secret: {oauth_secret}')
oauth_domain = os.getenv('OAUTH_DOMAIN')
assert oauth_domain
logger.info(f'using domain: {oauth_domain}')
django_secret_key = os.getenv('DJANGO_SECRET_KEY')
assert django_secret_key
logger.info(f'using DJANGO_SECRET_KEY: {django_secret_key}')
credentials, project = google.auth.default()
gcloud_client = container_v1.ClusterManagerClient(credentials=credentials)
scan_clusters(gcloud_client, project)
# FIXME add the k8s cert to a trust store
urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
auth_gcloud_k8s(credentials)
api_core_v1 = client.CoreV1Api()
api_apps_v1 = client.AppsV1Api()
api_storage_v1 = client.StorageV1Api()
api_custom = client.CustomObjectsApi()
api_extensions_v1_beta1 = client.ExtensionsV1beta1Api()
api_ext_v1_beta1 = client.ApiextensionsV1beta1Api()
api_rbac_auth_v1_b1 = client.RbacAuthorizationV1beta1Api()
ensure_traefik(api_core_v1, api_ext_v1_beta1, api_apps_v1, api_custom,
api_rbac_auth_v1_b1, admin_email, domain, static_ip,
oauth_client_id, oauth_client_secret, oauth_domain,
oauth_secret)
with open(os.getenv('GOOGLE_APPLICATION_CREDENTIALS'), 'rb') as f:
gcloud_credentials_b64 = b64encode(f.read()).decode('UTF-8')
ensure_secret(api=api_core_v1,
name='webui-credentials',
namespace='default',
secret=V1Secret(
metadata=client.V1ObjectMeta(name='webui-credentials'),
data={'gcloud-credentials': gcloud_credentials_b64}))
webui_volume_paths = [
('data', '/opt/nipyapi/data', '20Gi', 'standard'),
]
webui_volume_mounts = [
V1VolumeMount(name=path[0], mount_path=path[1])
for path in webui_volume_paths
]
webui_volume_mounts.append(
V1VolumeMount(name='webui-credentials',
mount_path='/root/webui',
read_only=True))
dind_volume_paths = [
('docker', '/var/lib/docker', '200Gi', 'standard'),
]
dind_volume_mounts = [
V1VolumeMount(name=path[0], mount_path=path[1])
for path in dind_volume_paths
]
shared_volume_mounts = [
V1VolumeMount(name='dind-socket', mount_path='/var/run-shared')
]
ensure_statefulset_with_containers(
api_apps_v1=api_apps_v1,
name='admin',
namespace='default',
replicas=1,
containers=[
V1Container(
name='webui',
image='aichrist/nipyapi-ds:latest',
env=[
# FIXME use k8s secrets for these values
V1EnvVar(name='DOMAIN', value=domain),
V1EnvVar(name='STATIC_IP', value=static_ip),
V1EnvVar(name='ADMIN_EMAIL', value=admin_email),
V1EnvVar(name='OAUTH_CLIENT_ID', value=oauth_client_id),
V1EnvVar(name='OAUTH_CLIENT_SECRET',
value=oauth_client_secret),
V1EnvVar(name='OAUTH_SECRET', value=oauth_secret),
V1EnvVar(name='OAUTH_DOMAIN', value=oauth_domain),
V1EnvVar(name='DJANGO_SECRET_KEY',
value=django_secret_key),
V1EnvVar(name='GOOGLE_APPLICATION_CREDENTIALS',
value='/root/webui/gcloud_credentials.json'),
V1EnvVar(name='CLOUDSDK_AUTH_CREDENTIAL_FILE_OVERRIDE',
value='/root/webui/gcloud_credentials.json'),
V1EnvVar(name='GOOGLE_CLOUD_PROJECT',
value=os.getenv('GOOGLE_CLOUD_PROJECT')),
V1EnvVar(name='DOCKER_HOST',
value='unix:///var/run-shared/docker.sock'),
],
ports=[V1ContainerPort(container_port=8000)],
volume_mounts=webui_volume_mounts + shared_volume_mounts),
V1Container(
name='dind',
image='docker:19-dind',
security_context=V1SecurityContext(privileged=True),
command=[
'dockerd', '-H', 'unix:///var/run-shared/docker.sock'
],
volume_mounts=dind_volume_mounts + shared_volume_mounts)
],
volumes=[
V1Volume(name='dind-socket', empty_dir={}),
V1Volume(name='webui-credentials',
projected=V1ProjectedVolumeSource(sources=[
V1VolumeProjection(secret=V1SecretProjection(
name='webui-credentials',
items=[
V1KeyToPath(key='gcloud-credentials',
path='gcloud_credentials.json')
]))
]))
],
volume_paths=webui_volume_paths + dind_volume_paths)
ensure_ingress_routed_svc(api_core_v1=api_core_v1,
api_custom=api_custom,
domain=domain,
hostname='admin',
name='admin',
target_name='admin',
namespace='default',
port_name='web',
svc_port=80,
target_port=8000)
reg_volume_paths = [
('database', '/opt/nifi-registry/nifi-registry-current/database',
'10Gi', 'standard'),
('flow-storage',
'/opt/nifi-registry/nifi-registry-current/flow_storage', '20Gi',
'standard'),
]
reg_volume_mounts = [
V1VolumeMount(name=path[0], mount_path=path[1])
for path in reg_volume_paths
]
ensure_statefulset_with_containers(
api_apps_v1=api_apps_v1,
name='registry',
namespace='default',
replicas=1,
containers=[
V1Container(name='registry',
image='apache/nifi-registry:latest',
env=[
V1EnvVar(name='NIFI_REGISTRY_WEB_HTTP_PORT',
value='19090'),
],
ports=[V1ContainerPort(container_port=19090)],
volume_mounts=reg_volume_mounts),
],
init_containers=[
V1Container(
name='init-permissions',
image='busybox',
command=[
'sh', '-c',
'chown -R 1000:1000 /opt/nifi-registry/nifi-registry-current'
],
volume_mounts=[
V1VolumeMount(name=path[0], mount_path=path[1])
for path in reg_volume_paths
])
],
volumes=[],
volume_paths=reg_volume_paths)
ensure_ingress_routed_svc(api_core_v1=api_core_v1,
api_custom=api_custom,
domain=domain,
hostname='registry',
name='registry',
target_name='registry',
namespace='default',
port_name='web',
svc_port=80,
target_port=19090)
perform_nifi_ops(api_apps_v1, api_core_v1, api_custom, domain)
perform_build_ops_bg()
perform_mirror_ops_bg()
def create_nifi_instances(api_apps_v1, api_core_v1, api_custom, domain):
for instance in NifiInstance.objects.filter(state='PENDING_CREATE'):
instance.state = 'CREATING'
instance.save()
port_name = 'web'
instance.state = 'CREATE_FAILED'
try:
namespace = 'default'
if instance.namespace is not None and instance.namespace != 'default':
namespace = instance.namespace
ensure_namespace(api_core_v1, namespace)
else:
instance.namespace = 'default'
# deploy nifi
nifi_volume_paths = [
('db-repo', '/opt/nifi/nifi-current/database_repository',
'20Gi', 'standard'),
('flowfile-repo', '/opt/nifi/nifi-current/flowfile_repository',
'20Gi', 'standard'),
('provenance-repo',
'/opt/nifi/nifi-current/provenance_repository', '20Gi',
'standard'),
('content-repo', '/opt/nifi/nifi-current/content_repository',
'20Gi', 'standard'),
]
ensure_statefulset_with_containers(
api_apps_v1=api_apps_v1,
name=instance.hostname,
namespace=namespace,
replicas=1,
containers=[
V1Container(name='nifi',
image=instance.image,
env=[
V1EnvVar(name='NIFI_WEB_HTTP_HOST',
value='0.0.0.0')
],
ports=[V1ContainerPort(container_port=8080)],
volume_mounts=[
V1VolumeMount(name=path[0],
mount_path=path[1])
for path in nifi_volume_paths
])
],
init_containers=[
V1Container(name='init-permissions',
image='busybox',
command=[
'sh', '-c',
'chown -R 1000:1000 /opt/nifi/nifi-current'
],
volume_mounts=[
V1VolumeMount(name=path[0],
mount_path=path[1])
for path in nifi_volume_paths
])
],
volume_paths=nifi_volume_paths)
ensure_ingress_routed_svc(api_core_v1=api_core_v1,
api_custom=api_custom,
domain=domain,
hostname=instance.hostname,
name=instance.hostname,
target_name=instance.hostname,
namespace=namespace,
port_name=port_name,
svc_port=80,
target_port=8080)
# deploy mongo
if instance.deploy_mongo:
mongo_volume_paths = [
('db', '/data/db', '20Gi', 'standard'),
]
ensure_statefulset_with_containers(
api_apps_v1=api_apps_v1,
name='mongo',
namespace=namespace,
replicas=1,
containers=[
V1Container(
name='mongo',
image='mongo',
env=[
V1EnvVar(name='MONGO_INITDB_ROOT_USERNAME',
value='admin'),
V1EnvVar(name='MONGO_INITDB_ROOT_PASSWORD',
value='admin')
],
ports=[
V1ContainerPort(name='mongo',
container_port=27017)
],
volume_mounts=[
V1VolumeMount(name=path[0], mount_path=path[1])
for path in mongo_volume_paths
])
],
volume_paths=mongo_volume_paths)
ensure_service(api=api_core_v1,
service=V1Service(
api_version="v1",
metadata=V1ObjectMeta(name='mongo'),
spec=V1ServiceSpec(
type='ClusterIP',
ports=[
V1ServicePort(protocol='TCP',
port=27017,
name='mongo',
target_port=27017),
],
selector={'app': 'mongo'})),
name='mongo',
namespace=namespace)
ensure_single_container_deployment(
api_apps_v1,
V1Container(
name='mongo-express',
image='mongo-express',
env=[
V1EnvVar(name='ME_CONFIG_MONGODB_ADMINUSERNAME',
value='admin'),
V1EnvVar(name='ME_CONFIG_MONGODB_ADMINPASSWORD',
value='admin')
],
ports=[V1ContainerPort(container_port=8000)]),
'mongo-express', instance.namespace)
ensure_ingress_routed_svc(api_core_v1=api_core_v1,
api_custom=api_custom,
domain=domain,
hostname="mongo-" +
instance.hostname,
name="mongo-" + instance.hostname,
target_name="mongo-express",
namespace=namespace,
port_name=port_name,
svc_port=80,
target_port=8081)
if instance.deploy_kafka:
# deploy zookeeper
ensure_single_container_deployment(
api_apps_v1,
V1Container(name='zookeeper',
image='wurstmeister/zookeeper',
env=[],
ports=[V1ContainerPort(container_port=2181)]),
'zookeeper', instance.namespace)
ensure_service(api=api_core_v1,
service=V1Service(
api_version="v1",
metadata=V1ObjectMeta(name='zookeeper'),
spec=V1ServiceSpec(
type='ClusterIP',
ports=[
V1ServicePort(protocol='TCP',
port=2181,
name='zookeeper',
target_port=2181),
],
selector={'app': 'zookeeper'})),
name='zookeeper',
namespace=namespace)
# deploy kafka
ensure_single_container_deployment(
api_apps_v1,
V1Container(name='kafka',
image='wurstmeister/kafka',
env=[
V1EnvVar(name='KAFKA_ADVERTISED_HOST_NAME',
value='kafka'),
V1EnvVar(name='KAFKA_ZOOKEEPER_CONNECT',
value='zookeeper:2181'),
V1EnvVar(name='KAFKA_PORT', value='9092')
],
ports=[V1ContainerPort(container_port=9092)]),
'kafka', instance.namespace)
ensure_service(api=api_core_v1,
service=V1Service(
api_version="v1",
metadata=V1ObjectMeta(name='kafka'),
spec=V1ServiceSpec(
type='ClusterIP',
ports=[
V1ServicePort(protocol='TCP',
port=9092,
name='kafka',
target_port=9092),
],
selector={'app': 'kafka'})),
name='kafka',
namespace=namespace)
if instance.deploy_prometheus:
# deploy prometheus
ensure_single_container_deployment(
api_apps_v1,
V1Container(name='prometheus',
image='prom/prometheus',
env=[],
ports=[V1ContainerPort(container_port=9090)]),
'prometheus', instance.namespace)
ensure_ingress_routed_svc(api_core_v1=api_core_v1,
api_custom=api_custom,
domain=domain,
hostname="prometheus-" +
instance.hostname,
name="prometheus",
target_name="prometheus",
namespace=namespace,
port_name=port_name,
svc_port=9090,
target_port=9090)
if instance.deploy_jupyter:
# deploy jupyter
instance.jupyter_token = str(uuid.uuid1())
ensure_single_container_deployment(
api_apps_v1,
V1Container(
name='jupyter',
image='jupyter/datascience-notebook',
command=[
'start-notebook.sh',
'--NotebookApp.token=' + instance.jupyter_token
],
env=[],
ports=[V1ContainerPort(container_port=8888)]),
'jupyter', instance.namespace)
ensure_ingress_routed_svc(api_core_v1=api_core_v1,
api_custom=api_custom,
domain=domain,
hostname="jupyter-" +
instance.hostname,
name="jupyter",
target_name="jupyter",
namespace=namespace,
port_name=port_name,
svc_port=8888,
target_port=8888)
# deploy custom instance types
custom_instances = Instance.objects.filter(parent=instance)
for ci in custom_instances:
inst_type: InstanceType = ci.instance_type
env_vars = [
V1EnvVar(name=e.name, value=e.default_value)
for e in InstanceTypeEnvVar.objects.filter(
instance_type=inst_type)
]
ports = [
V1ContainerPort(container_port=p.internal)
for p in InstanceTypePort.objects.filter(
instance_type=inst_type)
]
ensure_single_container_deployment(
api_apps_v1,
V1Container(name=inst_type.container_name,
image=inst_type.image,
env=env_vars,
ports=ports), inst_type.container_name,
instance.namespace)
for svc in InstanceTypeIngressRoutedService.objects.filter(
instance_type=inst_type):
ensure_ingress_routed_svc(
api_core_v1=api_core_v1,
api_custom=api_custom,
domain=domain,
hostname=svc.service_name + '-' + instance.hostname,
name=svc.service_name + '-' + instance.hostname,
target_name=inst_type.container_name,
namespace=namespace,
port_name=port_name,
svc_port=svc.svc_port,
target_port=svc.target_port)
instance.state = 'RUNNING'
finally:
instance.save()
def __init__(self, name, mount, path, read_only=True):
self.mount = V1VolumeMount(name=name,
mount_path=mount,
read_only=read_only)
self.volume = V1Volume(name=name,
host_path=V1HostPathVolumeSource(path=path))
def __init__(self, name, mount, read_only=False):
self.mount = V1VolumeMount(name=name,
mount_path=mount,
read_only=read_only)
self.volume = V1Volume(name=name, empty_dir=V1EmptyDirVolumeSource())
def createsvc(self, deploy, port, imagename, namespace, envvar, nameapp,
service, pvc, volumename, datadir):
bservice = client.V1Service()
smeta = V1ObjectMeta()
dcmeta = V1ObjectMeta()
pmt = V1ObjectMeta()
sspec = client.V1ServiceSpec()
bdc = openshift.client.V1DeploymentConfig()
dcspec = openshift.client.V1DeploymentConfigSpec()
strategy = openshift.client.V1DeploymentStrategy()
rollingparams = openshift.client.V1RollingDeploymentStrategyParams()
podtemp = client.V1PodTemplateSpec()
podspec = client.V1PodSpec()
container = client.V1Container()
idname = nameapp + "-" + deploy
smeta.name = idname # !!!
smeta.namespace = namespace
smeta.labels = {"label": idname, "bundle": service + "-" + nameapp}
sspec.selector = {"label": idname}
sspec.ports = []
for l in range(0, len(port)):
p = client.V1ServicePort()
p.name = "{port}-{tcp}".format(**port[l])
p.protocol = "TCP"
p.port = port[l]['tcp']
p.target_port = "{port}-{tcp}".format(**port[l])
sspec.ports.append(p)
if port[l]['route'] == 'yes':
self.createroute(p.target_port, idname, namespace, service,
nameapp, l)
bservice.api_version = 'v1'
bservice.kind = 'Service'
bservice.metadata = smeta
bservice.spec = sspec
bservice.api_version = 'v1'
# DeploymentConfig
dcmeta.labels = {"label": idname, "bundle": service + "-" + nameapp}
dcmeta.name = idname
dcmeta.namespace = namespace
rollingparams.interval_seconds = 1
strategy.labels = {"label": idname, "bundle": service + "-" + nameapp}
strategy.type = 'Rolling'
strategy.rolling_params = rollingparams
container.image = imagename
container.name = idname
container.env = []
if pvc:
vm = V1VolumeMount()
vm.mount_path = datadir
vm.name = volumename
container.volume_mounts = [vm]
else:
pass
for key in envvar:
v = client.V1EnvVar()
v.name = key
v.value = envvar[key]
container.env.append(v)
container.ports = []
for o in range(0, len(port)):
p = client.V1ContainerPort()
p.name = ("{port}-{tcp}".format(**port[o]))
p.protocol = "TCP"
p.container_port = port[o]['tcp']
container.ports.append(p)
pmt.labels = {"label": idname, "bundle": service + "-" + nameapp}
pmt.name = idname
podspec.containers = [container]
if pvc:
vol = V1Volume()
pvcname = V1PersistentVolumeClaimVolumeSource()
volgfs = V1GlusterfsVolumeSource()
pvcname.claim_name = volumename
volgfs.endpoints = volumename
volgfs.path = datadir
#vol.glusterfs = volgfs
vol.name = volumename
vol.persistent_volume_claim = pvcname
podspec.volumes = [vol]
else:
pass
podtemp.metadata = pmt
podtemp.spec = podspec
dcspec.replicas = 1
dcspec.selector = {"label": idname}
dcspec.template = podtemp
dcspec.strategy = strategy
bdc.api_version = 'v1'
bdc.spec = dcspec
bdc.metadata = dcmeta
bdc.kind = 'DeploymentConfig'
try:
self.k1.create_namespaced_service(namespace=namespace,
body=bservice,
pretty='true')
except ApiException as e:
print("Exception when calling OapiApi->create_service: %s\n" % e)
try:
self.o1.create_namespaced_deployment_config(namespace=namespace,
body=bdc,
pretty='true')
except ApiException as e:
print("Exception when calling OapiApi->create_dc: %s\n" % e)
def apply_pod_profile(self,
username,
pod,
profile,
gpu_types,
default_mount_path,
gpu_mode=None,
selected_gpu_type="ALL"):
api_client = kubernetes.client.ApiClient()
pod.metadata.labels['jupyterhub.opendatahub.io/user'] = escape(
username)
profile_volumes = profile.get('volumes')
if profile_volumes:
for volume in profile_volumes:
volume_name = re.sub('[^a-zA-Z0-9\.]', '-',
volume['name']).lower()
read_only = volume['persistentVolumeClaim'].get('readOnly')
pvc = V1PersistentVolumeClaimVolumeSource(
volume['persistentVolumeClaim']['claimName'],
read_only=read_only)
mount_path = self.generate_volume_path(volume.get('mountPath'),
default_mount_path,
volume_name)
pod.spec.volumes.append(
V1Volume(name=volume_name, persistent_volume_claim=pvc))
pod.spec.containers[0].volume_mounts.append(
V1VolumeMount(name=volume_name, mount_path=mount_path))
profile_environment = profile.get('env')
if profile_environment:
# Kept for backwards compatibility with simplified env var definitions
if isinstance(profile_environment, dict):
for k, v in profile['env'].items():
update = False
for e in pod.spec.containers[0].env:
if e.name == k:
e.value = v
update = True
break
if not update:
pod.spec.containers[0].env.append(V1EnvVar(k, v))
elif isinstance(profile_environment, list):
for i in profile_environment:
r = type("Response", (), {})
r.data = json.dumps(i)
env_var = api_client.deserialize(r, V1EnvVar)
pod.spec.containers[0].env.append(env_var)
resource_var = None
resource_json = type("Response", (), {})
resource_json.data = json.dumps(profile.get('resources'))
resource_var = api_client.deserialize(resource_json,
V1ResourceRequirements)
if resource_var:
pod.spec.containers[0].resources = resource_var
mem_limit = resource_var.limits.get('memory', '')
if mem_limit:
pod.spec.containers[0].env.append(
V1EnvVar(name='MEM_LIMIT',
value=self.get_mem_limit(mem_limit)))
for c in pod.spec.containers:
update = False
if type(c) is dict:
env = c['env']
else:
env = c.env
for e in env:
if type(e) is dict:
if e['name'] == _JUPYTERHUB_USER_NAME_ENV:
e['value'] = username
update = True
break
else:
if e.name == _JUPYTERHUB_USER_NAME_ENV:
e.value = username
update = True
break
if not update:
env.append(V1EnvVar(_JUPYTERHUB_USER_NAME_ENV, username))
self.apply_gpu_config(gpu_mode, profile, gpu_types, pod,
selected_gpu_type)
node_tolerations = profile.get('node_tolerations', [])
node_affinity = profile.get('node_affinity', {})
self.apply_pod_schedulers(node_tolerations, node_affinity, pod)
return pod
def start_stateful_container(self, service_name: str, container_name: str,
spec, labels: dict[str, str], change_key: str):
# Setup PVC
deployment_name = self._dependency_name(service_name, container_name)
mounts, volumes = [], []
for volume_name, volume_spec in spec.volumes.items():
mount_name = f'{deployment_name}-{volume_name}'
# Check if the PVC exists, create if not
self._ensure_pvc(mount_name, volume_spec.storage_class, volume_spec.capacity, deployment_name)
# Create the volume info
volumes.append(V1Volume(
name=mount_name,
persistent_volume_claim=V1PersistentVolumeClaimVolumeSource(mount_name)
))
mounts.append(V1VolumeMount(mount_path=volume_spec.mount_path, name=mount_name))
# Read the key being used for the deployment instance or generate a new one
instance_key = uuid.uuid4().hex
try:
old_deployment = self.apps_api.read_namespaced_deployment(deployment_name, self.namespace)
for container in old_deployment.spec.template.spec.containers:
for env in container.env:
if env.name == 'AL_INSTANCE_KEY':
instance_key = env.value
break
except ApiException as error:
if error.status != 404:
raise
# Setup the deployment itself
labels['container'] = container_name
spec.container.environment.append({'name': 'AL_INSTANCE_KEY', 'value': instance_key})
self._create_deployment(service_name, deployment_name, spec.container,
30, 1, labels, volumes=volumes, mounts=mounts,
core_mounts=spec.run_as_core, change_key=change_key)
# Setup a service to direct to the deployment
try:
service = self.api.read_namespaced_service(deployment_name, self.namespace)
service.metadata.labels = labels
service.spec.selector = labels
service.spec.ports = [V1ServicePort(port=int(_p)) for _p in spec.container.ports]
self.api.patch_namespaced_service(deployment_name, self.namespace, service)
except ApiException as error:
if error.status != 404:
raise
service = V1Service(
metadata=V1ObjectMeta(name=deployment_name, labels=labels),
spec=V1ServiceSpec(
cluster_ip='None',
selector=labels,
ports=[V1ServicePort(port=int(_p)) for _p in spec.container.ports]
)
)
self.api.create_namespaced_service(self.namespace, service)
# Add entries to the environment variable list to point to this container
self._service_limited_env[service_name][f'{container_name}_host'] = deployment_name
self._service_limited_env[service_name][f'{container_name}_key'] = instance_key
if spec.container.ports:
self._service_limited_env[service_name][f'{container_name}_port'] = spec.container.ports[0]
("host--hello_world", 0, "host--hello--world"),
("host--hello_world" + ("a" * 60), 63,
"host--hello--worldaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa--1090"),
("a/b/c/d/e/f/g/h/i/j/k/l", 63,
"aslash-bslash-cslash-dslash-eslash-fslash-gslash-hslash-i--646b"),
))
def test_get_sanitised_volume_name(name, length_limit, expected):
assert get_sanitised_volume_name(name, length_limit) == expected
@pytest.mark.parametrize("volumes,expected", (
(v({
"container_path": "/a",
"host_path": "/b",
"mode": "RO"
}), [V1VolumeMount(mount_path="/a", name="host--slash-b", read_only=True)
]),
(v(
{
"container_path": "/a",
"host_path": "/b",
"mode": "RO"
},
{
"container_path": "/b",
"host_path": "/a/b/cd/e/f/g/h/u/j/k/l",
"mode": "RW"
},
), [
V1VolumeMount(mount_path="/a", name="host--slash-b", read_only=True),
V1VolumeMount(
def run(provider, provider_kwargs, cluster=None, job=None, storage=None):
# TODO, temp fix
s3 = storage["s3"]
_validate_fields(
provider=provider_kwargs, cluster=cluster, job=job, storage=storage, s3=s3
)
_required_run_arguments(provider_kwargs, cluster, job, storage, s3)
response = {"job": {}}
if "name" not in job["meta"] or not job["meta"]["name"]:
since_epoch = int(time.time())
job["meta"]["name"] = "{}-{}".format(JOB_DEFAULT_NAME, since_epoch)
if "bucket_name" not in s3 or not s3["bucket_name"]:
s3["bucket_name"] = job["meta"]["name"]
container_engine_client = new_client(
ContainerEngineClient,
composite_class=ContainerEngineClientCompositeOperations,
name=provider_kwargs["profile"]["name"],
)
compute_cluster = get_cluster_by_name(
container_engine_client,
provider_kwargs["profile"]["compartment_id"],
name=cluster["name"],
)
if not compute_cluster:
response["msg"] = "Failed to find a cluster with name: {}".format(
cluster["name"]
)
return False, response
refreshed = refresh_kube_config(
compute_cluster.id, name=provider_kwargs["profile"]["name"]
)
if not refreshed:
response["msg"] = "Failed to refresh the kubernetes config"
return False, response
node_manager = NodeManager()
if not node_manager.discover():
response["msg"] = "Failed to discover any nodes to schedule jobs on"
return False, response
node = node_manager.select()
if not node:
response["msg"] = "Failed to select a node to schedule on"
return False, response
# Ensure we have the newest config
scheduler = KubenetesScheduler()
jobio_args = [
"jobio",
"run",
]
jobio_args.extend(job["commands"])
jobio_args.extend(["--job-meta-name", job["meta"]["name"]])
if "output_path" in job:
jobio_args.extend(
["--job-output-path", job["output_path"],]
)
if "capture" in job and job["capture"]:
jobio_args.append("--job-capture")
if "debug" in job["meta"]:
jobio_args.append("--job-meta-debug")
if "env_override" in job["meta"]:
jobio_args.append("--job-meta-env-override")
# Maintained by the pod
volumes = []
# Maintained by the container
volume_mounts = []
# Environment to pass to the container
envs = []
# Prepare config for the scheduler
scheduler_config = {}
if storage and storage["enable"]:
validate_dict_values(storage, required_storage_fields, throw=True)
jobio_args.append("--storage-enable")
# Means that results should be exported to the specified storage
# Create kubernetes secrets
core_api = client.CoreV1Api()
# storage_api = client.StorageV1Api()
# Storage endpoint credentials secret (Tied to a profile and job)
secret_profile_name = "{}-{}-{}".format(
STORAGE_CREDENTIALS_NAME, s3["name"], job["meta"]["name"]
)
try:
storage_credentials_secret = core_api.read_namespaced_secret(
secret_profile_name, KUBERNETES_NAMESPACE
)
except ApiException:
storage_credentials_secret = None
# volumes
secret_volume_source = V1SecretVolumeSource(secret_name=secret_profile_name)
secret_volume = V1Volume(name=secret_profile_name, secret=secret_volume_source)
volumes.append(secret_volume)
# Where the storage credentials should be mounted
# in the compute unit
secret_mount = V1VolumeMount(
name=secret_profile_name,
mount_path=storage["credentials_path"],
read_only=True,
)
volume_mounts.append(secret_mount)
if s3:
validate_dict_values(s3, required_staging_values, verbose=True, throw=True)
jobio_args.append("--storage-s3")
# S3 storage
# Look for s3 credentials and config files
s3_config = load_aws_config(
s3["config_file"], s3["credentials_file"], profile_name=s3["name"],
)
s3_config["endpoint_url"] = storage["endpoint"]
if not storage_credentials_secret:
secret_data = dict(
aws_access_key_id=s3_config["aws_access_key_id"],
aws_secret_access_key=s3_config["aws_secret_access_key"],
)
secret_metadata = V1ObjectMeta(name=secret_profile_name)
secrets_config = dict(metadata=secret_metadata, string_data=secret_data)
scheduler_config.update(dict(secret_kwargs=secrets_config))
# If `access_key`
# TODO, unify argument endpoint, with s3 config endpoint'
s3_resource = boto3.resource("s3", **s3_config)
bucket = bucket_exists(s3_resource.meta.client, s3["bucket_name"])
if not bucket:
bucket = s3_resource.create_bucket(
Bucket=s3["bucket_name"],
CreateBucketConfiguration={
"LocationConstraint": s3_config["region_name"]
},
)
if "upload_path" in storage and storage["upload_path"]:
# Upload local path to the bucket as designated input for the job
uploaded = None
if os.path.exists(storage["upload_path"]):
if os.path.isdir(storage["upload_path"]):
uploaded = upload_directory_to_s3(
s3_resource.meta.client,
storage["upload_path"],
s3["bucket_name"],
s3_prefix=s3["bucket_input_prefix"],
)
elif os.path.isfile(storage["upload_path"]):
s3_path = os.path.basename(storage["upload_path"])
if s3["bucket_input_prefix"]:
s3_path = os.path.join(s3["bucket_input_prefix"], s3_path)
# Upload
uploaded = upload_to_s3(
s3_resource.meta.client,
storage["upload_path"],
s3_path,
s3["bucket_name"],
)
if not uploaded:
response[
"msg"
] = "Failed to local path: {} in the upload folder to s3".format(
storage["upload_path"]
)
return False, response
jobio_args.extend(
[
"--s3-region-name",
s3_config["region_name"],
"--storage-secrets-dir",
storage["credentials_path"],
"--storage-endpoint",
storage["endpoint"],
"--storage-input-path",
storage["input_path"],
"--storage-output-path",
storage["output_path"],
"--bucket-name",
s3["bucket_name"],
"--bucket-input-prefix",
s3["bucket_input_prefix"],
"--bucket-output-prefix",
s3["bucket_output_prefix"],
]
)
# Provide a way to allow pod specific output prefixes
field_ref = client.V1ObjectFieldSelector(field_path="metadata.name")
env_var_source = client.V1EnvVarSource(field_ref=field_ref)
# HACK, Set the output prefix in the bucket to the name of the pod
env_output_prefix = client.V1EnvVar(
name="JOBIO_BUCKET_OUTPUT_PREFIX", value_from=env_var_source
)
envs.append(env_output_prefix)
if scheduler_config:
prepared = scheduler.prepare(**scheduler_config)
if not prepared:
response["msg"] = "Failed to prepare the scheduler"
return False, response
container_spec = dict(
name=job["meta"]["name"],
image=cluster["image"],
env=envs,
args=jobio_args,
volume_mounts=volume_mounts,
)
# If the working directory does not exist inside the container
# It will set permissions where it will be unable to expand the
# s3 bucket if the user doesn't have root permissions
if "working_dir" in job:
container_spec.update({"working_dir": job["working_dir"]})
# If the container requires a specific set of resources
resources = {}
if "min_cores" in job:
resources["requests"] = {"cpu": job["min_cores"]}
if "max_cores" in job:
resources["limits"] = {"cpu": job["max_cores"]}
if "min_memory" in job:
resources["requests"].update({"memory": job["min_memory"]})
if "max_memory" in job:
resources["limits"].update({"memory": job["max_memory"]})
if resources:
resource_req = client.V1ResourceRequirements(**resources)
container_spec.update({"resources": resource_req})
# args=jobio_args,
pod_spec = dict(node_name=node.metadata.name, volumes=volumes, dns_policy="Default")
job_spec = dict(
backoff_limit=2,
parallelism=job["meta"]["num_parallel"],
completions=job["meta"]["num_jobs"],
)
task = dict(
container_kwargs=container_spec,
pod_spec_kwargs=pod_spec,
job_spec_kwargs=job_spec,
)
job = scheduler.submit(**task)
if not job:
response["msg"] = "Failed to submit the job"
return False, response
response["job"] = job
response["msg"] = "Job submitted"
return True, response
def generate_secrets_server_deployment(
secrets_server_config: SecretsServerConfig, ):
service_name = secrets_server_config.service_name()
secret_name = secrets_server_config.secrets().concourse_secret_name()
# We need to ensure that the labels and selectors match for both the deployment and the service,
# therefore we base them on the configured service name.
labels = {'app': service_name}
return V1Deployment(
kind='Deployment',
metadata=V1ObjectMeta(name=service_name, labels=labels),
spec=V1DeploymentSpec(
replicas=1,
selector=V1LabelSelector(match_labels=labels),
template=V1PodTemplateSpec(
metadata=V1ObjectMeta(labels=labels),
spec=V1PodSpec(containers=[
V1Container(
image='eu.gcr.io/gardener-project/cc/job-image:latest',
image_pull_policy='IfNotPresent',
name='secrets-server',
resources=V1ResourceRequirements(
requests={
'cpu': '50m',
'memory': '50Mi'
},
limits={
'cpu': '50m',
'memory': '50Mi'
},
),
command=['bash'],
args=[
'-c', '''
# chdir to secrets dir; create if absent
mkdir -p /secrets && cd /secrets
# make Kubernetes serviceaccount secrets available by default
cp -r /var/run/secrets/kubernetes.io/serviceaccount serviceaccount
# store Kubernetes service endpoint env as file for consumer
env | grep KUBERNETES_SERVICE > serviceaccount/env
# launch secrets server serving secrets dir contents on all IFs
python3 -m http.server 8080
'''
],
ports=[
V1ContainerPort(container_port=8080),
],
liveness_probe=V1Probe(
tcp_socket=V1TCPSocketAction(port=8080),
initial_delay_seconds=10,
period_seconds=10,
),
volume_mounts=[
V1VolumeMount(
name=secret_name,
mount_path='/secrets/concourse-secrets',
read_only=True,
),
],
),
],
node_selector={
"worker.garden.sapcloud.io/group":
"cc-control"
},
volumes=[
V1Volume(name=secret_name,
secret=V1SecretVolumeSource(
secret_name=secret_name, ))
]))))
def launch(self, name, docker_config: DockerConfig, mounts, env, blocking: bool = True):
name = (self.prefix + 'update-' + name.lower()).replace('_', '-')
# If we have been given a username or password for the registry, we have to
# update it, if we haven't been, make sure its been cleaned up in the system
# so we don't leave passwords lying around
pull_secret_name = f'{name}-job-pull-secret'
use_pull_secret = False
try:
# Check if there is already a username/password defined for this job
current_pull_secret = self.api.read_namespaced_secret(pull_secret_name, self.namespace,
_request_timeout=API_TIMEOUT)
except ApiException as error:
if error.status != 404:
raise
current_pull_secret = None
if docker_config.registry_username or docker_config.registry_password:
use_pull_secret = True
# Build the secret we want to make
new_pull_secret = V1Secret(
metadata=V1ObjectMeta(name=pull_secret_name, namespace=self.namespace),
type='kubernetes.io/dockerconfigjson',
string_data={
'.dockerconfigjson': create_docker_auth_config(
image=docker_config.image,
username=docker_config.registry_username,
password=docker_config.registry_password,
)
}
)
# Send it to the server
if current_pull_secret:
self.api.replace_namespaced_secret(pull_secret_name, namespace=self.namespace,
body=new_pull_secret, _request_timeout=API_TIMEOUT)
else:
self.api.create_namespaced_secret(namespace=self.namespace, body=new_pull_secret,
_request_timeout=API_TIMEOUT)
elif current_pull_secret:
# If there is a password set in kubernetes, but not in our configuration clear it out
self.api.delete_namespaced_secret(pull_secret_name, self.namespace, _request_timeout=API_TIMEOUT)
try:
self.batch_api.delete_namespaced_job(name=name, namespace=self.namespace,
propagation_policy='Background', _request_timeout=API_TIMEOUT)
while True:
self.batch_api.read_namespaced_job(namespace=self.namespace, name=name,
_request_timeout=API_TIMEOUT)
time.sleep(1)
except ApiException:
pass
volumes = []
volume_mounts = []
for index, mnt in enumerate(mounts):
volumes.append(V1Volume(
name=f'mount-{index}',
persistent_volume_claim=V1PersistentVolumeClaimVolumeSource(
claim_name=mnt['volume'],
read_only=False
),
))
volume_mounts.append(V1VolumeMount(
name=f'mount-{index}',
mount_path=mnt['dest_path'],
sub_path=mnt['source_path'],
read_only=False,
))
if CONFIGURATION_CONFIGMAP:
volumes.append(V1Volume(
name='mount-configuration',
config_map=V1ConfigMapVolumeSource(
name=CONFIGURATION_CONFIGMAP
),
))
volume_mounts.append(V1VolumeMount(
name='mount-configuration',
mount_path='/etc/assemblyline/config.yml',
sub_path="config",
read_only=True,
))
section = 'service'
labels = {
'app': 'assemblyline',
'section': section,
'privilege': 'core',
'component': 'update-script',
}
labels.update(self.extra_labels)
metadata = V1ObjectMeta(
name=name,
labels=labels
)
environment_variables = [V1EnvVar(name=_e.name, value=_e.value) for _e in docker_config.environment]
environment_variables.extend([V1EnvVar(name=k, value=v) for k, v in env.items()])
environment_variables.extend([V1EnvVar(name=k, value=os.environ[k])
for k in INHERITED_VARIABLES if k in os.environ])
environment_variables.append(V1EnvVar(name="LOG_LEVEL", value=self.log_level))
cores = docker_config.cpu_cores
memory = docker_config.ram_mb
memory_min = min(docker_config.ram_mb_min, memory)
container = V1Container(
name=name,
image=docker_config.image,
command=docker_config.command,
env=environment_variables,
image_pull_policy='Always',
volume_mounts=volume_mounts,
resources=V1ResourceRequirements(
limits={'cpu': cores, 'memory': f'{memory}Mi'},
requests={'cpu': cores / 4, 'memory': f'{memory_min}Mi'},
)
)
pod = V1PodSpec(
volumes=volumes,
restart_policy='Never',
containers=[container],
priority_class_name=self.priority_class,
)
if use_pull_secret:
pod.image_pull_secrets = [V1LocalObjectReference(name=pull_secret_name)]
job = V1Job(
metadata=metadata,
spec=V1JobSpec(
backoff_limit=1,
completions=1,
template=V1PodTemplateSpec(
metadata=metadata,
spec=pod
)
)
)
status = self.batch_api.create_namespaced_job(namespace=self.namespace, body=job,
_request_timeout=API_TIMEOUT).status
if blocking:
try:
while not (status.failed or status.succeeded):
time.sleep(3)
status = self.batch_api.read_namespaced_job(namespace=self.namespace, name=name,
_request_timeout=API_TIMEOUT).status
self.batch_api.delete_namespaced_job(name=name, namespace=self.namespace,
propagation_policy='Background', _request_timeout=API_TIMEOUT)
except ApiException as error:
if error.status != 404:
raise