def main():
group = "argoproj.io"
version = "v1alpha1"
plural = "hyperparamworkflows"
config.load_incluster_config()
namespace = 'default'
api_client = client.ApiClient()
custom_api = client.CustomObjectsApi(api_client)
watch = kwatch.Watch(return_type=object)
print("Starting loop")
for event in watch.stream(custom_api.list_namespaced_custom_object, group,
version, namespace, plural):
if event['type'] == 'ADDED':
hparams = event['raw_object']['spec']['hyperparams']
if event['raw_object']['spec']['algorithm'] == 'grid':
experiments = grid_search(hparams)
wf = generate_workflow(event['raw_object'], experiments)
try:
resp = custom_api.create_namespaced_custom_object(
group,
version,
namespace,
"workflows",
wf,
pretty=True)
except client.rest.ApiException:
continue
print(yaml.dump(wf))
if event['type'] == 'DELETED':
# TODO: This would be better managed with resource owners
name = event['raw_object']['metadata']['name']
custom_api.delete_namespaced_custom_object(
group,
version,
namespace,
"workflows",
name=name,
body=client.V1DeleteOptions())
def serve(self) -> None:
# For deployed clusters, we should always be running inside
# a Rook cluster. For development convenience, also support
# running outside (reading ~/.kube config)
if self._rook_env.has_namespace():
config.load_incluster_config()
else:
self.log.warning("DEVELOPMENT ONLY: Reading kube config from ~")
config.load_kube_config()
# So that I can do port forwarding from my workstation - jcsp
from kubernetes.client import configuration
configuration.verify_ssl = False
self._k8s_CoreV1_api = client.CoreV1Api()
self._k8s_BatchV1_api = client.BatchV1Api()
self._k8s_CustomObjects_api = client.CustomObjectsApi()
self._k8s_StorageV1_api = client.StorageV1Api()
try:
# XXX mystery hack -- I need to do an API call from
# this context, or subsequent API usage from handle_command
# fails with SSLError('bad handshake'). Suspect some kind of
# thread context setup in SSL lib?
self._k8s_CoreV1_api.list_namespaced_pod(self._rook_env.namespace)
except ApiException:
# Ignore here to make self.available() fail with a proper error message
pass
assert isinstance(self.storage_class, str)
self._rook_cluster = RookCluster(self._k8s_CoreV1_api,
self._k8s_BatchV1_api,
self._k8s_CustomObjects_api,
self._k8s_StorageV1_api,
self._rook_env, self.storage_class)
self._initialized.set()
while not self._shutdown.is_set():
self._shutdown.wait(5)
def poll(self):
if not self._crd:
self._create_custom_resource_definitions()
api = client.CustomObjectsApi()
mapping = dict()
group = self._crd.spec['group']
plural = self._crd.spec['names']['plural']
version = self._crd.spec['version']
kwargs = {}
if self.resource_version:
kwargs['resource_version'] = self.resource_version
try:
resp = api.list_cluster_custom_object(group, version, plural,
**kwargs)
except (client.rest.ApiException,
urllib3.exceptions.MaxRetryError) as e:
raise CustomResourceDefinitionLoadingFailed(e)
# Doesn't work
# self.resource_version = resp['metadata']['resourceVersion']
for item in resp['items']:
try:
version = item['metadata']['resourceVersion']
name = item['metadata']['name']
# This, however, does seem to work
# self.resource_version = max(
# version,
# self.resource_version)
namespace = item['metadata']['namespace']
if 'options' in item:
options = self._read_options_v2(item)
else:
options = self._read_options_v1(item)
scope = 'vcenter_' + options['scope']
jinja2_options = options.get('jinja2_options', {})
template = item['template']
path = '/'.join([scope, namespace, name]) + '.yaml.j2'
mapping[path] = (version, template, jinja2_options)
except KeyError as e:
LOG.error("Failed for %s/%s due to missing key %s", namespace,
name, e)
self.mapping = mapping
def terminate_run(self, project_id, deployment_id, run_id):
"""
Terminates a run in Kubeflow Pipelines.
Parameters
----------
project_id : str
deployment_id : str
run_id : str
Returns
-------
projects.schemas.message.Message
Raises
------
NotFound
When any of project_id, deployment_id, or run_id does not exist.
"""
load_kube_config()
api = client.CustomObjectsApi()
custom_objects = api.list_namespaced_custom_object(
"machinelearning.seldon.io",
"v1alpha2",
KF_PIPELINES_NAMESPACE,
"seldondeployments"
)
deployments_objects = custom_objects["items"]
if deployments_objects:
for deployment in deployments_objects:
if deployment["metadata"]["name"] == deployment_id:
undeploy_pipeline(deployment)
deployment_run = get_deployment_runs(deployment_id)
if not deployment_run:
raise NotFound("Deployment run does not exist.")
kfp_client().runs.delete_run(deployment_run["runId"])
return schemas.Message(message="Deployment deleted")
def down(splunk, stack_id, force=False):
stacks.update_config(splunk, stack_id, {
"status": stacks.DELETING,
"deleted_time": time.time(),
})
stack_config = stacks.get_stack_config(splunk, stack_id)
cluster_name = stack_config["cluster"]
cluster_config = clusters.get_cluster(splunk, cluster_name)
api_client = clusters.create_client(splunk, cluster_name)
core_api = kuberneteslib.CoreV1Api(api_client)
custom_objects_api = kuberneteslib.CustomObjectsApi(api_client)
try:
services.delete_all_load_balancers(core_api, stack_id, stack_config["namespace"])
stack_deployment.delete_objects(api_client, stack_id, stack_config, cluster_config)
except:
if not force:
raise
stacks.update_config(splunk, stack_id, {
"status": stacks.DELETED,
})
def check_vs_class(vs_class_name):
"""
checks volume snapshot class vs_class_name exists or not
return True , if vs_class_name exists
else return False
"""
api_instance = client.CustomObjectsApi()
try:
api_response = api_instance.get_cluster_custom_object(
group="snapshot.storage.k8s.io",
version="v1",
plural="volumesnapshotclasses",
name=vs_class_name
)
LOGGER.debug(api_response)
LOGGER.info(f"Volume Snapshot Class Check : {vs_class_name} exists")
return True
except ApiException:
LOGGER.info(f"volume snapshot class {vs_class_name} does not exists")
return False
def list_custom_objects(group: str, version: str, plural: str,
ns: str = "default",
secrets: Secrets = None) -> List[Dict[str, Any]]:
"""
List custom objects in the given namespace.
Read more about custom resources here:
https://kubernetes.io/docs/concepts/extend-kubernetes/api-extension/custom-resources/
""" # noqa: E501
api = client.CustomObjectsApi(create_k8s_api_client(secrets))
try:
r = api.list_namespaced_custom_object(
group, version, ns, plural, _preload_content=False
)
return json.loads(r.data)
except ApiException as x:
raise ActivityFailed(
"Failed to create custom resource object: '{}' {}".format(
x.reason, x.body))
def get_instance_list(self):
"""
Gets the current list of instances.
:returns: the list of instances or None
"""
# connect to Custom Object Api
api_instance = client.CustomObjectsApi()
try:
instances = api_instance.list_cluster_custom_object(
group=self.group,
version=self.version,
plural=self.plural_instance,
pretty='pretty')
logger.debug(
f"Instances list retrieved successfully ({len(instances.get('items'))} items)"
)
except ApiException as e:
logger.error(f"Failed retrieving the list of instances: '{e}'")
return None
return instances
def vmHandler(dom, operation):
try:
jsondict = client.CustomObjectsApi().get_namespaced_custom_object(
group=GROUP,
version=VERSION,
namespace='default',
plural=PLURAL,
name=dom.name())
# print(jsondict)
if operation == "Delete":
logger.debug('Callback domain deletion to virtlet')
deleteVM(dom.name(), V1DeleteOptions())
else:
logger.debug('Callback domain changes to virtlet')
vm_xml = get_xml(dom.name())
vm_json = toKubeJson(xmlToJson(vm_xml))
body = updateXmlStructureInJson(jsondict, vm_json)
modifyVM(dom.name(), body)
except:
logger.error('Oops! ', exc_info=1)
def list(cls, **kwargs):
items = []
crd_api = client.CustomObjectsApi()
args, sig_kwargs = cls.list_sig()
raw_list = crd_api.list_cluster_custom_object(
*args, **{
**sig_kwargs,
**kwargs
})
pipe = cache_client.pipeline()
for item in raw_list['items']:
o = cls(item)
items.append(o)
pipe.set(cls.name_plural() + "_" + o.name,
json.dumps(item),
ex=cache_client.default_cache_time)
pipe.execute()
return items
def __init__(self, config_file=None, context=None, # pylint: disable=too-many-arguments
client_configuration=None, persist_config=True):
"""
KFServing client constructor
:param config_file: kubeconfig file, defaults to ~/.kube/config
:param context: kubernetes context
:param client_configuration: kubernetes configuration object
:param persist_config:
"""
if config_file or not utils.is_running_in_k8s():
config.load_kube_config(
config_file=config_file,
context=context,
client_configuration=client_configuration,
persist_config=persist_config)
else:
config.load_incluster_config()
self.core_api = client.CoreV1Api()
self.app_api = client.AppsV1Api()
self.api_instance = client.CustomObjectsApi()
def kube_apis(cli_arguments) -> KubeApis:
"""
Set up kubernets-client to operate in cluster.
:param cli_arguments: a set of command-line arguments
:return: KubeApis
"""
context_name = cli_arguments['context']
kubeconfig = cli_arguments['kubeconfig']
config.load_kube_config(config_file=kubeconfig,
context=context_name,
persist_config=False)
v1 = client.CoreV1Api()
extensions_v1_beta1 = client.ExtensionsV1beta1Api()
apps_v1_api = client.AppsV1Api()
rbac_v1_beta1 = client.RbacAuthorizationV1beta1Api()
api_extensions_v1_beta1 = client.ApiextensionsV1beta1Api()
custom_objects = client.CustomObjectsApi()
return KubeApis(v1, extensions_v1_beta1, apps_v1_api, rbac_v1_beta1,
api_extensions_v1_beta1, custom_objects)
def clean_nuts(self, valid_id_rotation):
crds = client.CustomObjectsApi()
nuts = crds.list_cluster_custom_object(self.squirrel.domain_api,
self.squirrel.api_version,
'nuts')["items"]
for n in nuts:
if n["data"].get("id_rotation", 10000000000) != valid_id_rotation:
try:
api_response = crds.delete_namespaced_custom_object(\
self.squirrel.domain_api, \
self.squirrel.api_version, \
n["metadata"]["namespace"], \
'nuts', \
n["metadata"]["name"],
client.V1DeleteOptions())
print(api_response)
except ApiException as e:
print(
"Exception when calling CustomObjectsApi->delete_namespaced_custom_object: %s\n"
% e)
def delete_vip_crd(self, service_obj):
crds = client.CustomObjectsApi()
name = service_obj.metadata.name
namespace = service_obj.metadata.namespace
options = client.V1DeleteOptions()
print("delete_vip_crd: Deleting VIP CRD %s/%s" % (namespace, name))
try:
# TODO figure out a good value for grace period
# TODO figure out if we want this done via garbage collection
crds.delete_namespaced_custom_object(GROUP,
VERSION,
namespace,
PLURAL,
name,
options,
grace_period_seconds=30)
except ApiException as e:
print(
"delete_vip_crd: Exception while deleting VIP %s/%s, exception=%s"
% (namespace, name, e))
def watch_cr(queue):
group = "examplecnf.openshift.io"
version = "v1"
namespace = "example-cnf"
plural = "testpmdmacs"
config.load_incluster_config()
custom_api = client.CustomObjectsApi()
w = watch.Watch()
now = datetime.utcnow().replace(tzinfo=tzutc())
for event in w.stream(custom_api.list_cluster_custom_object,
group=group,
version=version,
plural=plural):
if event['type'] == 'ADDED':
meta = event['object']['metadata']
spec = event['object']['spec']
created = parser.parse(meta['creationTimestamp'])
if created > now:
queue.put(event['object'])
def myVmVolEventHandler(event, pool, vol):
try:
jsondict = client.CustomObjectsApi().get_namespaced_custom_object(
group=GROUP_VM_DISK,
version=VERSION_VM_DISK,
namespace='default',
plural=PLURAL_VM_DISK,
name=vol)
# print(jsondict)
if event == "Delete":
logger.debug('Callback volume deletion to virtlet')
deleteVol(vol, V1DeleteOptions())
else:
logger.debug('Callback volume changes to virtlet')
vol_xml = get_volume_xml(pool, vol)
vol_json = toKubeJson(xmlToJson(vol_xml))
body = updateXmlStructureInJson(jsondict, vol_json)
modifyVol(vol, body)
except:
logger.error('Oops! ', exc_info=1)
def check_vs_class_deleted(vs_class_name, created_objects):
"""
if volume snapshot class vs_class_name exists , assert
"""
if keep_objects:
return
api_instance = client.CustomObjectsApi()
try:
api_response = api_instance.get_cluster_custom_object(
group="snapshot.storage.k8s.io",
version="v1",
plural="volumesnapshotclasses",
name=vs_class_name
)
LOGGER.debug(api_response)
LOGGER.error(f"Volume Snapshot Class Delete : {vs_class_name} is not deleted , asserting")
clean_with_created_objects(created_objects)
assert False
except ApiException:
LOGGER.info(f"Volume Snapshot Class Delete : {vs_class_name} deletion confirmed")
def create(self, body):
"""create or apply a custom resource in k8s"""
pretty = 'true'
config.load_kube_config()
api_instance = client.CustomObjectsApi()
try:
api_response = api_instance.create_namespaced_custom_object(
self.group,
self.version,
self.namespace,
plural=self.plural,
body=body,
pretty=pretty)
log.debug(f"create custom resource response: {api_response}")
except ApiException as e:
log.error(
"Exception when calling CustomObjectsApi->create_namespaced_custom_object: %s\n"
% e)
raise Exception(str(e))
return api_response
def find_decision(self, name, namespace='default'):
# get the resource and print out data
resource = ''
api = client.CustomObjectsApi(self.api_client)
try:
resource = api.get_namespaced_custom_object(
group="mizar.com",
version="v1",
name=name,
namespace=namespace,
plural="decisions",
)
except ApiException as e:
pass
if resource == '':
ifFind = 0 # No decision
else:
ifFind = 1 # Find the existing decision
# print(ifFind)
return ifFind
def wait_ready_spec(self, model_spec: ModelSpec, timeout_secs=None) -> bool:
create_k8s_client()
ready = False
t0 = time.time()
while not ready:
api_instance = client.CustomObjectsApi()
existing = api_instance.get_namespaced_custom_object(
"machinelearning.seldon.io",
"v1",
model_spec.runtime_options.k8s_options.namespace,
"seldondeployments",
model_spec.model_details.name,
)
if "status" in existing and "state" in existing["status"]:
ready = existing["status"]["state"] == "Available"
if timeout_secs is not None:
t1 = time.time()
if t1 - t0 > timeout_secs:
return ready
return ready
def _change_certificate_status(self, name: str, namespace: str, body: dict):
"""
:param name: k8s certificate name
:param namespace: namespace
:param body: k8s certificate body
:return:
"""
try:
# update customObject status
client.CustomObjectsApi().replace_namespaced_custom_object_status(
group=self._group,
version=self._version,
namespace=namespace,
plural=self._plural,
name=name,
body=body
)
except ApiException as e:
LOG.error(f"Exception when calling CustomObjectsApi->patch_cluster_custom_object, error: {e}")
def delete_cluster_custom_object(group: str, version: str, plural: str,
name: str,
secrets: Secrets = None) -> Dict[str, Any]:
"""
Delete a custom object cluster wide.
Read more about custom resources here:
https://kubernetes.io/docs/concepts/extend-kubernetes/api-extension/custom-resources/
""" # noqa: E501
api = client.CustomObjectsApi(create_k8s_api_client(secrets))
try:
r = api.delete_cluster_custom_object(
group, version, plural, name, _preload_content=False
)
return json.loads(r.data)
except ApiException as x:
raise ActivityFailed(
"Failed to delete custom resource object: '{}' {}".format(
x.reason, x.body))
def myVmSnapshotEventHandler(event, vm, snap):
try:
jsondict = client.CustomObjectsApi().get_namespaced_custom_object(
group=GROUP_VM_SNAPSHOT,
version=VERSION_VM_SNAPSHOT,
namespace='default',
plural=PLURAL_VM_SNAPSHOT,
name=snap)
# print(jsondict)
if event == "Delete":
logger.debug('Callback snapshot deletion to virtlet')
deleteSnapshot(snap, V1DeleteOptions())
else:
logger.debug('Callback snapshot changes to virtlet')
snap_xml = get_snapshot_xml(vm, snap)
snap_json = toKubeJson(xmlToJson(snap_xml))
body = updateXmlStructureInJson(jsondict, snap_json)
modifySnapshot(snap, body)
except:
logger.error('Oops! ', exc_info=1)
def wait_for_job(client,
namespace,
name,
timeout=datetime.timedelta(minutes=10),
polling_interval=datetime.timedelta(seconds=30),
status_callback=None):
"""Wait for the specified job to finish.
Args:
client: K8s api client.
namespace: namespace for the job.
name: Name of the job.
timeout: How long to wait for the job.
polling_interval: How often to poll for the status of the job.
status_callback: (Optional): Callable. If supplied this callable is
invoked after we poll the job. Callable takes a single argument which
is the job.
"""
crd_api = k8s_client.CustomObjectsApi(client)
end_time = datetime.datetime.now() + timeout
while True:
results = crd_api.get_namespaced_custom_object(
TF_JOB_GROUP, TF_JOB_VERSION, namespace, TF_JOB_PLURAL, name)
if status_callback:
status_callback(results)
# If we poll the CRD quick enough status won't have been set yet.
if results.get("status", {}).get("phase", {}) == "Done":
return results
if datetime.datetime.now() + polling_interval > end_time:
raise util.TimeoutError(
"Timeout waiting for job {0} in namespace {1} to finish.".format(
name, namespace))
time.sleep(polling_interval.seconds)
# Linter complains if we don't have a return statement even though
# this code is unreachable.
return None
def watch(name=None, namespace=None, timeout_seconds=600):
"""Watch the created or patched InferenceService in the specified namespace"""
if namespace is None:
namespace = utils.get_default_target_namespace()
tbl = TableLogger(columns='NAME,READY,DEFAULT_TRAFFIC,CANARY_TRAFFIC,URL',
colwidth={
'NAME': 20,
'READY': 10,
'DEFAULT_TRAFFIC': 15,
'CANARY_TRAFFIC': 15,
'URL': 50
},
border=False)
stream = k8s_watch.Watch().stream(
client.CustomObjectsApi().list_namespaced_custom_object,
constants.KFSERVING_GROUP,
constants.KFSERVING_VERSION,
namespace,
constants.KFSERVING_PLURAL,
timeout_seconds=timeout_seconds)
for event in stream:
isvc = event['object']
isvc_name = isvc['metadata']['name']
if name and name != isvc_name:
continue
else:
url = isvc['status'].get('url', '')
default_traffic = isvc['status'].get('traffic', '')
canary_traffic = isvc['status'].get('canaryTraffic', '')
status = 'Unknown'
for condition in isvc['status'].get('conditions', {}):
if condition.get('type', '') == 'Ready':
status = condition.get('status', 'Unknown')
tbl(isvc_name, status, default_traffic, canary_traffic, url)
if name == isvc_name and status == 'True':
break
def work(self) -> None:
while not self.queue.is_empty():
item = self.queue.get()
event = item['event'].value
obj = item['object']
metadata = obj['metadata']
name = metadata['name']
ray = obj['status']['ray']
if ray['generation'] != metadata['generation']:
co_api = client.CustomObjectsApi()
print(f"Working {event} event of {name}")
# do things
template = obj['spec']['template']
try:
template_obj = co_api.get_namespaced_custom_object(
self.group,
self.version,
self.namespace,
self.template_plural,
template,
)
except ApiException as e:
ray['created'] = False
ray['message'] = f"Template {template}: {e.reason}"
else:
ray['created'] = True
ray['message'] = "Actor created"
# update status
ray['generation'] = metadata['generation']
co_api.replace_namespaced_custom_object_status(
self.group, self.version, self.namespace,
self.instance_plural, name, obj)
time.sleep(2)
if self.running:
self.work()
def create_vs_content(vs_content_name, vs_name, body_params, created_objects):
"""
create volume snapshot content with vs_content_name
body_params contains configurable parameters
"""
content_body = {
"apiVersion": "snapshot.storage.k8s.io/v1",
"kind": "VolumeSnapshotContent",
"metadata": {
"name": vs_content_name
},
"spec": {
"deletionPolicy": body_params["deletionPolicy"],
"driver": "spectrumscale.csi.ibm.com",
"source": {
"snapshotHandle": body_params["snapshotHandle"]
},
"volumeSnapshotRef": {
"name": vs_name,
"namespace": namespace_value
}
}
}
custom_object_api_instance = client.CustomObjectsApi()
try:
custom_object_api_response = custom_object_api_instance.create_cluster_custom_object(
group="snapshot.storage.k8s.io",
version="v1",
plural="volumesnapshotcontents",
body=content_body,
pretty=True
)
LOGGER.debug(custom_object_api_response)
created_objects["vscontent"].append(vs_content_name)
LOGGER.info(f"Volume Snapshot Content Create : {vs_content_name} is created with {body_params}")
except ApiException as e:
LOGGER.error(
f"Exception when calling CustomObjectsApi->create_namespaced_custom_object: {e}")
clean_with_created_objects(created_objects)
assert False
def get_url(self, model_spec: ModelSpec):
if self.inside_cluster is None:
ingress = create_ingress(model_spec)
ingress_host_url = ingress.get_external_host_url(model_spec)
return (
f"{ingress_host_url}" +
f"/seldon/{model_spec.runtime_options.k8s_options.namespace}/"
+ f"{model_spec.model_details.name}" +
model_spec.protocol.get_predict_path(model_spec.model_details))
else:
# TODO check why needed this here
config.load_incluster_config()
api_instance = client.CustomObjectsApi()
api_response = api_instance.get_namespaced_custom_object_status(
"machinelearning.seldon.io",
"v1",
model_spec.runtime_options.k8s_options.namespace,
"seldondeployments",
model_spec.model_details.name,
)
return api_response["status"]["address"]["url"]
def create_tf_job(self, namespace, job):
"""Create the provided TFJob in the specified namespace.
The TFJob version is defined in TF_JOB_VERSION in fairing.constants.
The version TFJob need to be installed before creating the TFJob.
:param namespace: The custom resource
:param job: The JSON schema of the Resource to create
:returns: object: Created TFJob.
"""
api_instance = client.CustomObjectsApi()
try:
return api_instance.create_namespaced_custom_object(
constants.TF_JOB_GROUP, constants.TF_JOB_VERSION, namespace,
constants.TF_JOB_PLURAL, job)
except client.rest.ApiException:
raise RuntimeError(
"Failed to create TFJob. Perhaps the CRD TFJob version "
"{} in not installed(If you use different version you can pass it "
"as ENV variable called "
"`TF_JOB_VERSION`)?".format(constants.TF_JOB_VERSION))
def __init__(self):
check_microk8s_kube_config_file()
load_kubernetes_config()
self.api_client = client.ApiClient()
self.custom_def_cli = client.CustomObjectsApi()
self.core_cli = client.CoreV1Api()
self.apps_cli = client.AppsV1Api()
self.jobs_cli = client.BatchV1Api()
self.cronjobs_cli = client.BatchV1beta1Api()
self.rbac_cli = client.RbacAuthorizationV1Api()
self.network_cli = client.NetworkingV1beta1Api()
self.network_policy_cli = client.NetworkingV1Api()
self.extenstion_cli = client.ExtensionsV1beta1Api()
self.crd_cli = client.ApiextensionsV1beta1Api()
self.storage_cli = client.StorageV1Api()
self.admission_cli = client.AdmissionregistrationV1beta1Api()
self.delete_options = client.V1DeleteOptions()
self.delete_options.grace_period_seconds = 2
self.delete_options.propagation_policy = 'Foreground'
self.core_cli.api_client.configuration.assert_hostname = False
self.apps_cli.api_client.configuration.assert_hostname = False
