def _get_project_oidc_secret(self):
try:
secret = KubeClient().secrets.get(
f"{self.project.namespace}-projman-secrets",
namespace=self.project.namespace)
return base64.b64decode(
secret.get('data').get('oidc-client-secret')).decode('utf-8')
except Exception:
return None
def _get_project_oidc_secret(self):
try:
secret = KubeClient().secrets.get(
f"keycloak-client-secret-projman-{self.project.namespace}",
namespace=self.project.namespace)
return base64.b64decode(
secret.get('data').get('CLIENT_SECRET')).decode('utf-8')
except Exception:
return None
def configure(self, app_config, provider_config):
playbook_vars = {
'kube_cloud_provider':
self._cb_provider_id_to_kube_provider_id(
provider_config.get('cloud_provider').PROVIDER_ID),
'cluster_hostname':
app_config.get('config_kube_rke', {}).get('rke_cluster_id'),
'rke_registration_server':
app_config.get('config_kube_rke',
{}).get('rke_registration_server'),
'rke_registration_token':
app_config.get('config_kube_rke', {}).get('rke_registration_token')
}
result = super().configure(app_config,
provider_config,
playbook_vars=playbook_vars)
if self.has_reached_desired_state(provider_config):
kube_client = KubeClient()
node_ip = provider_config.get('host_config', {}).get('private_ip')
k8s_node = kube_client.nodes.find(address=node_ip)[0]
labels = {
'usegalaxy.org/cm_node_name':
app_config.get('deployment_config', {}).get('name', '')
}
autoscaling_group = app_config.get('config_cloudman',
{}).get('autoscaling_group', '')
if autoscaling_group:
labels[
'usegalaxy.org/cm_autoscaling_group'] = autoscaling_group
kube_client.nodes.set_label(k8s_node, labels)
return result
else:
raise NodeNotRegistered(f"Node has not been added to the cluster")
def delete(self, provider, deployment):
"""
Delete resource(s) associated with the supplied deployment.
This is a blocking call that will wait until the instance is marked
as deleted or disappears from the provider.
*Note* that this method will delete resource(s) associated with
the deployment - this is an un-recoverable action.
"""
app_config = deployment.get('app_config')
rancher_cfg = app_config.get('config_rancher_kube')
rancher_client = self._create_rancher_client(rancher_cfg)
node_ip = deployment.get('launch_result', {}).get('cloudLaunch',
{}).get('publicIP')
try:
rancher_node_id = rancher_client.find_node(ip=node_ip)
if rancher_node_id:
try:
kube_client = KubeClient()
k8s_node = kube_client.nodes.find(node_ip)[0]
# stop new jobs being scheduled on this node
kube_client.nodes.cordon(k8s_node)
# let existing jobs finish
kube_client.nodes.wait_till_jobs_complete(k8s_node)
# drain remaining pods
kube_client.nodes.drain(k8s_node, timeout=120)
finally:
# remove node from rancher
rancher_client.delete_node(rancher_node_id)
finally:
# delete the VM
return super().delete(provider, deployment)
def delete(self, namespace):
self.check_permissions('helmsman.delete_namespace')
client = KubeClient()
existing = self.get(namespace)
if not existing:
raise NamespaceNotFoundException(
f"Namespace {namespace} cannot be found.")
else:
client.namespaces.delete(namespace)
def create(self, namespace):
self.check_permissions('helmsman.add_namespace')
client = KubeClient()
existing = self.get(namespace)
if existing:
raise NamespaceExistsException(
f"Namespace '{namespace}' already exists.")
else:
client.namespaces.create(namespace)
return self.get(namespace)
def check_node_no_longer_exists(self, node_name):
# Newly added node should now be registered with the cluster
kube_client = KubeClient()
k8s_node = kube_client.nodes.find(
labels={'usegalaxy.org/cm_node_name': node_name})
if not k8s_node:
return True
else:
raise NodeNotDeleted(
f"Deleted node with name: {node_name} still attached to the cluster."
)
def has_reached_desired_state(self, provider_config):
# Newly added node should now be registered with the cluster
kube_client = KubeClient()
node_ip = provider_config.get('host_config', {}).get('private_ip')
k8s_node = kube_client.nodes.find(address=node_ip)
if k8s_node and k8s_node[0]:
return True
else:
raise NodeNotRegistered(
f"New node with ip: {node_ip} has still not registered with k8s cluster"
)
def find_matching_node(self, labels=None):
labels = labels.copy() if labels else {}
kube_client = KubeClient()
# find the k8s node which matches these labels
k8s_matches = kube_client.nodes.find(labels=labels)
if k8s_matches:
k8s_node = k8s_matches[0]
node_name = k8s_node.get('metadata', {}).get(
'labels', {}).get('usegalaxy.org/cm_node_name')
# find the corresponding cloudman node
for node in self.cluster.nodes.list():
if node.name == node_name:
return node
return None
def delete(self, provider, deployment):
"""
Delete resource(s) associated with the supplied deployment.
This is a blocking call that will wait until the instance is marked
as deleted or disappears from the provider.
*Note* that this method will delete resource(s) associated with
the deployment - this is an un-recoverable action.
"""
deployment_name = deployment.get('name')
try:
kube_client = KubeClient()
k8s_node = kube_client.nodes.find(
labels={'usegalaxy.org/cm_node_name': deployment_name})
if k8s_node:
k8s_node = k8s_node[0]
try:
# stop new jobs being scheduled on this node
print(f"Cordoning node: {deployment_name}")
kube_client.nodes.cordon(k8s_node)
# let existing jobs finish
print(
f"Waiting for jobs to finish on node: {deployment_name}"
)
kube_client.nodes.wait_till_jobs_complete(k8s_node)
# drain remaining pods
print(f"Draining node: {deployment_name}")
kube_client.nodes.drain(k8s_node, timeout=120)
finally:
# delete the k8s node
print(f"Deleting k8s node: {deployment_name}")
kube_client.nodes.delete(k8s_node)
finally:
# delete the VM
result = super().delete(provider, deployment)
if self.check_node_no_longer_exists(deployment_name):
return result
else:
raise NodeNotDeleted(f"Node has not been removed from the cluster")
def list(self):
return [
KubeNamespace(self, **namespace)
for namespace in KubeClient().namespaces.list()
if self.has_permissions('helmsman.view_namespace', namespace)
]
def node_not_present(node):
kube_client = KubeClient()
print(f"Checking for presence of node: {node.name}")
k8s_node = kube_client.nodes.find(
labels={'usegalaxy.org/cm_node_name': node.name})
return not k8s_node
