def create_workspace_pvc(body):
# If the type is New, then create a new PVC, else use an existing one
# It always deletes the current one and creates a new
annotations = {
"rok/creds-secret-name": rok_secret_name(),
"jupyter-workspace": body["ws_name"],
}
if body["ws_type"] == "Existing":
annotations['rok/origin'] = body["ws_rok_url"]
labels = {"component": "singleuser-storage"}
# Create a PVC for the New/Existing Type
pvc = client.V1PersistentVolumeClaim(
metadata=client.V1ObjectMeta(
name=body['ws_name'],
namespace=body['ns'],
annotations=annotations,
labels=labels,
),
spec=client.V1PersistentVolumeClaimSpec(
access_modes=['ReadWriteOnce'],
resources=client.V1ResourceRequirements(
requests={'storage': body['ws_size'] + 'Gi'})))
delete_existing_pvc(pvc.metadata.name, pvc.metadata.namespace)
provision_new_pvc(pvc)
def create_k8s_pvc(namespace: str, name: str, access_modes: tuple,
storage_class_name: str, size: str, user):
"""
Create a Kubernetes persistent volume claim in a namespace in the cluster.
An existing pvc with this name in this namespace leads to a no-op.
Returns the new pvc.
"""
core_v1 = get_user_core_v1(user)
try:
k8s_spec = client.V1PersistentVolumeClaimSpec(
access_modes=access_modes,
storage_class_name=storage_class_name,
resources=client.V1ResourceRequirements(
requests={'storage': size}))
k8s_pvc = client.V1PersistentVolumeClaim(
api_version="v1",
kind="PersistentVolumeClaim",
metadata=client.V1ObjectMeta(name=name),
spec=k8s_spec)
core_v1.create_namespaced_persistent_volume_claim(namespace=namespace,
body=k8s_pvc)
logger.info(f"Created Kubernetes pvc '{namespace}:{name}'")
except client.rest.ApiException as e:
if e.status == 409:
logger.warning(
f"Tried to create already existing Kubernetes pvc '{namespace}:{name}'. Skipping the creation and using the existing one."
)
else:
raise e
return core_v1.read_namespaced_persistent_volume_claim(namespace=namespace,
name=name)
def create_clone_pvc(pvc_values, sc_name, pvc_name, from_pvc_name,
created_objects):
api_instance = client.CoreV1Api()
pvc_metadata = client.V1ObjectMeta(name=pvc_name)
pvc_resources = client.V1ResourceRequirements(
requests={"storage": pvc_values["storage"]})
pvc_data_source = client.V1TypedLocalObjectReference(
kind="PersistentVolumeClaim", name=from_pvc_name)
pvc_spec = client.V1PersistentVolumeClaimSpec(
access_modes=[pvc_values["access_modes"]],
resources=pvc_resources,
storage_class_name=sc_name,
data_source=pvc_data_source)
pvc_body = client.V1PersistentVolumeClaim(api_version="v1",
kind="PersistentVolumeClaim",
metadata=pvc_metadata,
spec=pvc_spec)
try:
LOGGER.info(
f'PVC Create from Clone : Creating pvc {pvc_name} with parameters {str(pvc_values)} and storageclass {str(sc_name)} from PVC {from_pvc_name}'
)
api_response = api_instance.create_namespaced_persistent_volume_claim(
namespace=namespace_value, body=pvc_body, pretty=True)
LOGGER.debug(str(api_response))
created_objects["clone_pvc"].append(pvc_name)
except ApiException as e:
LOGGER.info(f'PVC {pvc_name} creation operation has been failed')
LOGGER.error(
f"Exception when calling CoreV1Api->create_namespaced_persistent_volume_claim: {e}"
)
cleanup.clean_with_created_objects(created_objects)
assert False
def fxt_pvc(test_namespace):
pvc_name = "pvc-test"
api = client.CoreV1Api()
pvc_body = client.V1PersistentVolumeClaim(
api_version="v1",
kind="PersistentVolumeClaim",
metadata=client.V1ObjectMeta(name=pvc_name, ),
spec=client.V1PersistentVolumeClaimSpec(
storage_class_name="nfss1",
access_modes=["ReadWriteMany"],
resources=client.V1ResourceRequirements(
requests={"storage": "1Gi"}),
),
)
try:
response = api.create_namespaced_persistent_volume_claim(
namespace=test_namespace, body=pvc_body)
except ApiException as e:
logging.error("That didn't work: %s" % e)
pvcs = api.list_namespaced_persistent_volume_claim(
namespace=test_namespace)
logging.info(
f"PVC {pvcs.items[0].metadata.name} currently {pvcs.items[0].status.phase}"
)
assert len(pvcs.items) == 1
yield pvc_name
logging.info("Destroying PersistentVolumeClaim")
delete_pvc = api.delete_namespaced_persistent_volume_claim(
name="pvc-test", namespace=test_namespace)
assert delete_pvc, "Unable to delete PVC"
def _create_pvc_for_pv(self, vol):
name = vol.metadata.name
namespace = self.get_user_namespace()
pvcname = name
pvd = client.V1PersistentVolumeClaim(
spec=client.V1PersistentVolumeClaimSpec(
volume_name=name,
access_modes=vol.spec.access_modes,
resources=client.V1ResourceRequirements(
requests=vol.spec.capacity),
selector=client.V1LabelSelector(match_labels={"name": name}),
storage_class_name=vol.spec.storage_class_name))
md = client.V1ObjectMeta(name=pvcname, labels={"name": pvcname})
pvd.metadata = md
self.log.info("Creating PVC '%s' in namespace '%s'" %
(pvcname, namespace))
try:
self.api.create_namespaced_persistent_volume_claim(namespace, pvd)
except ApiException as e:
if e.status != 409:
self.log.exception("Create PVC '%s' " % pvcname +
"in namespace '%s' " % namespace +
"failed: %s" % str(e))
raise
else:
self.log.info("PVC '%s' " % pvcname +
"in namespace '%s' " % namespace +
"already exists.")
def create_datavol_pvc(body, i):
# body: Dict (request body)
pvc_nm = body["vol_name" + i]
# Create a PVC if its a new Data Volume
if body["vol_type" + i] == "New":
size = body["vol_size" + i] + "Gi"
mode = body["vol_access_modes" + i]
pvc = client.V1PersistentVolumeClaim(
metadata=client.V1ObjectMeta(
name=pvc_nm,
namespace=body["ns"]
),
spec=client.V1PersistentVolumeClaimSpec(
access_modes=[mode],
resources=client.V1ResourceRequirements(
requests={
"storage": size
}
)
)
)
create_pvc(pvc)
return
def deploy_volume_claim(
self,
labels={},
name_suffix="",
ports=None,
env=None,
):
volume_claim = self.get_volume_claim(labels)
if not volume_claim:
self.logger.info("creating volume claim (%s) ..." % name_suffix)
storage_class_name = None
if self.environment.storage_class:
storage_class_name = self.environment.storage_class
volume_claim = self.core_api.create_namespaced_persistent_volume_claim(
namespace=self.environment.namespace,
body=kubernetes_client.V1PersistentVolumeClaim(
api_version="v1",
kind="PersistentVolumeClaim",
metadata=kubernetes_client.V1ObjectMeta(
name=self.generate_object_name(name_suffix),
namespace=self.environment.namespace,
labels=self.generate_object_labels(labels),
),
spec=kubernetes_client.V1PersistentVolumeClaimSpec(
access_modes=["ReadWriteOnce"],
resources=kubernetes_client.V1ResourceRequirements(
requests={
"storage": "1Gi",
}, ),
storage_class_name=storage_class_name,
),
),
)
return volume_claim
def create_datavol_pvc(body, i):
# It always deletes the current one and creates a new
pvc_nm = body['vol_name' + i]
size = body['vol_size' + i] + 'Gi'
annotations = {
"rok/creds-secret-name": rok_secret_name(),
"jupyter-dataset": pvc_nm,
}
if body["ws_type"] == "Existing":
annotations['rok/origin'] = body["vol_rok_url" + i]
labels = {"component": "singleuser-storage"}
# Create a PVC for the New/Existing Type
pvc = client.V1PersistentVolumeClaim(
metadata=client.V1ObjectMeta(
name=pvc_nm,
namespace=body['ns'],
annotations=annotations,
labels=labels,
),
spec=client.V1PersistentVolumeClaimSpec(
access_modes=['ReadWriteOnce'],
resources=client.V1ResourceRequirements(
requests={'storage': size})))
delete_existing_pvc(pvc.metadata.name, pvc.metadata.namespace)
provision_new_pvc(pvc)
def create_volume_claim_template():
return client.V1PersistentVolumeClaim(
metadata=client.V1ObjectMeta(name="data"),
spec=client.V1PersistentVolumeClaimSpec(
access_modes=["ReadWriteOnce"],
resources=client.V1ResourceRequirements(
requests={"storage": "1Gi"})))
def create(self, vc_spec=None):
"""Creates a volume claim on the cluster.
Args:
vc_spec: A dictionary of keyword arguments that will be passed to V1PersistentVolumeClaimSpec()
Returns: A dictionary containing the results of creating the volume claim on the cluster.
"""
if vc_spec is None:
vc_spec = self.vc_spec
# Convert claim name to lower case
claim_name = str(self.name).lower()
vc_body = client.V1PersistentVolumeClaim()
vc_body.metadata = client.V1ObjectMeta(namespace=self.namespace,
name=claim_name)
resources = client.V1ResourceRequirements(
requests={'storage': str(self.disk_space) + 'Gi'})
vc_body.spec = client.V1PersistentVolumeClaimSpec(resources=resources,
**vc_spec)
self.creation_response = api_request(
api.create_namespaced_persistent_volume_claim, self.namespace,
vc_body)
return self.creation_response
def create_k8s_nfs_resources(self) -> bool:
"""
Create NFS resources such as PV and PVC in Kubernetes.
"""
from kubernetes import client as k8sclient
pv_name = "nfs-ckpt-pv-{}".format(uuid.uuid4())
persistent_volume = k8sclient.V1PersistentVolume(
api_version="v1",
kind="PersistentVolume",
metadata=k8sclient.V1ObjectMeta(
name=pv_name,
labels={'app': pv_name}
),
spec=k8sclient.V1PersistentVolumeSpec(
access_modes=["ReadWriteMany"],
nfs=k8sclient.V1NFSVolumeSource(
path=self.params.path,
server=self.params.server
),
capacity={'storage': '10Gi'},
storage_class_name=""
)
)
k8s_api_client = k8sclient.CoreV1Api()
try:
k8s_api_client.create_persistent_volume(persistent_volume)
self.params.pv_name = pv_name
except k8sclient.rest.ApiException as e:
print("Got exception: %s\n while creating the NFS PV", e)
return False
pvc_name = "nfs-ckpt-pvc-{}".format(uuid.uuid4())
persistent_volume_claim = k8sclient.V1PersistentVolumeClaim(
api_version="v1",
kind="PersistentVolumeClaim",
metadata=k8sclient.V1ObjectMeta(
name=pvc_name
),
spec=k8sclient.V1PersistentVolumeClaimSpec(
access_modes=["ReadWriteMany"],
resources=k8sclient.V1ResourceRequirements(
requests={'storage': '10Gi'}
),
selector=k8sclient.V1LabelSelector(
match_labels={'app': self.params.pv_name}
),
storage_class_name=""
)
)
try:
k8s_api_client.create_namespaced_persistent_volume_claim(self.params.namespace, persistent_volume_claim)
self.params.pvc_name = pvc_name
except k8sclient.rest.ApiException as e:
print("Got exception: %s\n while creating the NFS PVC", e)
return False
return True
def __init__(self, name, size, access_mode):
resource_req = client.V1ResourceRequirements(
requests={'storage': size + 'i'})
pvc_spec = client.V1PersistentVolumeClaimSpec(
access_modes=[access_mode], resources=resource_req)
super(StandardPVC,
self).__init__(metadata=client.V1ObjectMeta(name=name),
spec=pvc_spec)
def k8s_object(self):
return client.V1PersistentVolumeClaim(
spec=client.V1PersistentVolumeClaimSpec(
access_modes=["ReadWriteOnce"],
resources=client.V1ResourceRequirements(
requests={"storage": self.storage})),
metadata=client.V1ObjectMeta(name=self.name,
namespace=self.namespace))
def get_persistent_volume_claim_spec(volume, access_modes='ReadWriteOnce', ):
access_modes = to_list(access_modes)
resources = client.V1ResourceRequirements(requests={'storage': STORAGE_BY_VOLUME[volume]})
selector = client.V1LabelSelector(match_labels=get_labels(volume))
return client.V1PersistentVolumeClaimSpec(
access_modes=access_modes,
resources=resources,
selector=selector)
def __init__(self, name):
resource_req = client.V1ResourceRequirements(
requests={'storage': '1Mi'})
pvc_spec = client.V1PersistentVolumeClaimSpec(
access_modes=['ReadWriteMany'],
resources=resource_req,
storage_class_name=NFSPv.STORAGE_CLASS)
super(NFSPvc, self).__init__(metadata=client.V1ObjectMeta(name=name),
spec=pvc_spec)
def simple_persistentvolumeclaim():
"""Return the Kubernetes config matching the simple-persistentvolumeclaim.yaml manifest."""
return client.V1PersistentVolumeClaim(
api_version='v1',
kind='PersistentVolumeClaim',
metadata=client.V1ObjectMeta(name='my-pvc'),
spec=client.V1PersistentVolumeClaimSpec(
access_modes=['ReadWriteMany'],
resources=client.V1ResourceRequirements(
requests={'storage': '16Mi'})))
def post_pvc(namespace):
body = request.get_json()
pvc = client.V1PersistentVolumeClaim(
metadata=client.V1ObjectMeta(name=body["name"], namespace=namespace),
spec=client.V1PersistentVolumeClaimSpec(
access_modes=[body["mode"]],
resources=client.V1ResourceRequirements(
requests={"storage": body["size"] + "Gi"})))
return jsonify(api.create_pvc(pvc, namespace=namespace))
def create_persistent_volume_claim(self, name, storage_size_in_g,
access_modes=["ReadWriteMany"],
storage_class_name="glusterfs-storage"):
pvc = client.V1PersistentVolumeClaim()
pvc.metadata = client.V1ObjectMeta(name=name)
storage_size = "{}Gi".format(storage_size_in_g)
resources = client.V1ResourceRequirements(requests={"storage": storage_size})
pvc.spec = client.V1PersistentVolumeClaimSpec(access_modes=access_modes,
resources=resources,
storage_class_name=storage_class_name)
return self.core.create_namespaced_persistent_volume_claim(self.namespace, pvc)
def post_pvc(namespace):
body = request.get_json()
pvc = client.V1PersistentVolumeClaim(
metadata=client.V1ObjectMeta(name=body['name'], namespace=namespace),
spec=client.V1PersistentVolumeClaimSpec(
access_modes=[body['mode']],
resources=client.V1ResourceRequirements(
requests={'storage': body['size'] + 'Gi'})))
return jsonify(api.post_pvc(pvc))
def create_pvc(pvc_values, sc_name, pvc_name, config_value=None, pv_name=None):
"""
creates persistent volume claim
Args:
param1: pvc_values - values required for creation of pvc
param2: sc_name - name of storage class , pvc associated with
if "notusingsc" no storage class
param3: pvc_name - name of pvc to be created
param4: config_value - configuration file
param5: pv_name - name of pv , pvc associated with
if None , no pv is associated
Returns:
None
Raises:
Raises an exception on kubernetes client api failure and asserts
"""
api_instance = client.CoreV1Api()
pvc_metadata = client.V1ObjectMeta(name=pvc_name)
pvc_resources = client.V1ResourceRequirements(
requests={"storage": pvc_values["storage"]})
if sc_name == "":
global test
test = config_value
pvc_spec = client.V1PersistentVolumeClaimSpec(
access_modes=[pvc_values["access_modes"]],
resources=pvc_resources,
storage_class_name=sc_name,
volume_name=pv_name)
pvc_body = client.V1PersistentVolumeClaim(api_version="v1",
kind="PersistentVolumeClaim",
metadata=pvc_metadata,
spec=pvc_spec)
try:
LOGGER.info(
f'Creating pvc {pvc_name} with parameters {str(pvc_values)} and storageclass {str(sc_name)}'
)
api_response = api_instance.create_namespaced_persistent_volume_claim(
namespace=namespace_value, body=pvc_body, pretty=True)
LOGGER.debug(str(api_response))
LOGGER.info(f'PVC {pvc_name} has been created successfully')
except ApiException as e:
LOGGER.info(f'PVC {pvc_name} creation operation has been failed')
LOGGER.error(
f"Exception when calling CoreV1Api->create_namespaced_persistent_volume_claim: {e}"
)
assert False
def create_pvc_from_snapshot(pvc_values, sc_name, pvc_name, snap_name,
created_objects):
"""
creates persistent volume claim from snapshot
Args:
param1: pvc_values - values required for creation of pvc
param2: sc_name - name of storage class , pvc associated with
param3: pvc_name - name of pvc to be created
param4: snap_name - name of snapshot to recover data from
Returns:
None
Raises:
Raises an exception on kubernetes client api failure and asserts
"""
api_instance = client.CoreV1Api()
pvc_metadata = client.V1ObjectMeta(name=pvc_name)
pvc_resources = client.V1ResourceRequirements(
requests={"storage": pvc_values["storage"]})
pvc_data_source = client.V1TypedLocalObjectReference(
api_group="snapshot.storage.k8s.io",
kind="VolumeSnapshot",
name=snap_name)
pvc_spec = client.V1PersistentVolumeClaimSpec(
access_modes=[pvc_values["access_modes"]],
resources=pvc_resources,
storage_class_name=sc_name,
data_source=pvc_data_source)
pvc_body = client.V1PersistentVolumeClaim(api_version="v1",
kind="PersistentVolumeClaim",
metadata=pvc_metadata,
spec=pvc_spec)
try:
LOGGER.info(
f'PVC Create from snapshot : Creating pvc {pvc_name} with parameters {str(pvc_values)} and storageclass {str(sc_name)}'
)
api_response = api_instance.create_namespaced_persistent_volume_claim(
namespace=namespace_value, body=pvc_body, pretty=True)
LOGGER.debug(str(api_response))
created_objects["restore_pvc"].append(pvc_name)
except ApiException as e:
LOGGER.info(f'PVC {pvc_name} creation operation has been failed')
LOGGER.error(
f"Exception when calling CoreV1Api->create_namespaced_persistent_volume_claim: {e}"
)
cleanup.clean_with_created_objects(created_objects)
assert False
def persistent_volume_claim(self, name, volume_name, storage, accessModes=["ReadWriteOnce"]):
"""Create a volume claim."""
claim_vol = client.V1PersistentVolumeClaim(kind="PersistentVolumeClaim", api_version="v1")
claim_vol.metadata = client.V1ObjectMeta(name=name)
spec = client.V1PersistentVolumeClaimSpec(volume_name=volume_name, access_modes=accessModes, storage_class_name=volume_name)
spec.resources = client.V1ResourceRequirements(requests={"storage": storage})
claim_vol.spec = spec
try:
api_core.create_namespaced_persistent_volume_claim(namespace=self._namespace, body=claim_vol)
LOG.info(f'Volume claim: {name} created.')
except ApiException as e:
LOG.error(f'Exception message: {e}')
def get(self):
return client.V1PersistentVolumeClaim(
kind='PersistentVolumeClaim',
api_version='v1',
metadata=client.V1ObjectMeta(name='postgres-pvc',
labels={'pvc': 'postgres'}),
spec=client.V1PersistentVolumeClaimSpec(
storage_class_name='manual',
access_modes=['ReadWriteOnce'],
resources=client.V1ResourceRequirements(
requests={'storage': '2Gi'}),
volume_name='postgres-pv'))
def simple_persistentvolumeclaim():
"""Return the Kubernetes config matching the simple-persistentvolumeclaim.yaml manifest."""
return client.V1PersistentVolumeClaim(
api_version="v1",
kind="PersistentVolumeClaim",
metadata=client.V1ObjectMeta(name="my-pvc"),
spec=client.V1PersistentVolumeClaimSpec(
access_modes=["ReadWriteMany"],
resources=client.V1ResourceRequirements(
requests={"storage": "16Mi"}),
),
)
def create_pvc_object(user_id: str, storage: str = '2Gi'):
pvc = client.V1PersistentVolumeClaim(
metadata={'name': 'claim-' + user_id},
spec=client.V1PersistentVolumeClaimSpec(
storage_class_name="standard",
access_modes=['ReadWriteOnce', ],
resources=client.V1ResourceRequirements(
requests={'storage': storage}
)
)
)
return pvc
def __init__(self, name, pv_name):
super(TritonVolume,
self).__init__(k8s_resource=k8s_client.V1PersistentVolumeClaim(
api_version="v1",
kind="PersistentVolumeClaim",
metadata=k8s_client.V1ObjectMeta(name=pv_name),
spec=k8s_client.V1PersistentVolumeClaimSpec(
access_modes=['ReadWriteMany'],
resources=k8s_client.V1ResourceRequirements(
requests={'storage': '2000Gi'}),
storage_class_name="nfs-client")),
action='apply',
name=name)
name = name
def pvc_from_dict(vol, namespace):
if vol is None:
return None
return client.V1PersistentVolumeClaim(
metadata=client.V1ObjectMeta(name=vol["name"], namespace=namespace,),
spec=client.V1PersistentVolumeClaimSpec(
access_modes=[vol["mode"]],
storage_class_name=handle_storage_class(vol),
resources=client.V1ResourceRequirements(
requests={"storage": vol["size"]}
),
),
)
def create_workspace_pvc(body):
# body: Dict (request body)
"""If the type is New, then create a new PVC, else use an existing one"""
if body["ws_type"] == "New":
pvc = client.V1PersistentVolumeClaim(
metadata=client.V1ObjectMeta(name=body["ws_name"],
namespace=body["ns"]),
spec=client.V1PersistentVolumeClaimSpec(
access_modes=[body["ws_access_modes"]],
resources=client.V1ResourceRequirements(
requests={"storage": body["ws_size"] + "Gi"})))
create_pvc(pvc)
return
def create_sts_spec(node_name, workload_type):
sts_name = STS_PREFIX + node_name
cmd = get_workload_command(workload_type, sts_name)
container = client.V1Container(
name=sts_name,
image=IMAGE,
command=["/bin/bash"],
args=["-c", cmd],
liveness_probe=client.V1Probe(
_exec=client.V1ExecAction(command=["ls", "/mnt/" + sts_name]),
initial_delay_seconds=5,
period_seconds=5),
volume_mounts=[
client.V1VolumeMount(name=sts_name, mount_path="/mnt/" + sts_name)
])
template = client.V1PodTemplateSpec(
metadata=client.V1ObjectMeta(labels={"app": sts_name}),
spec=client.V1PodSpec(
node_name=node_name,
restart_policy="Always",
termination_grace_period_seconds=10,
containers=[container],
))
spec = client.V1StatefulSetSpec(
replicas=0,
service_name=sts_name,
selector=client.V1LabelSelector(match_labels={"app": sts_name}),
template=template,
volume_claim_templates=[
client.V1PersistentVolumeClaim(
metadata=client.V1ObjectMeta(name=sts_name),
spec=client.V1PersistentVolumeClaimSpec(
access_modes=["ReadWriteOnce"],
storage_class_name="longhorn",
resources=client.V1ResourceRequirements(
requests={"storage": "4Gi"})))
])
statefulset = client.V1StatefulSet(
api_version="apps/v1",
kind="StatefulSet",
metadata=client.V1ObjectMeta(name=sts_name),
spec=spec)
statefulset.spec.replicas
return statefulset
def pvc_from_dict(body, namespace):
"""
body: json object (frontend json data)
Convert the PVC json object that is sent from the backend to a python
client PVC instance.
"""
return client.V1PersistentVolumeClaim(
metadata=client.V1ObjectMeta(name=body["name"], namespace=namespace),
spec=client.V1PersistentVolumeClaimSpec(
access_modes=[body["mode"]],
storage_class_name=handle_storage_class(body),
resources=client.V1ResourceRequirements(
requests={"storage": body["size"]}),
),
)
