def _ensure_crds() -> None:
"""
ensure_crds makes sure that all the required CRDs have been created
"""
crdv1 = client.ApiextensionsV1beta1Api()
crd = _load_mongodb_crd()
k8s_conditions.ignore_if_doesnt_exist(
lambda: crdv1.delete_custom_resource_definition("mongodb.mongodb.com"))
# Make sure that the CRD has being deleted before trying to create it again
if not k8s_conditions.wait(
lambda: crdv1.list_custom_resource_definition(
field_selector="metadata.name==mongodb.mongodb.com"),
lambda crd_list: len(crd_list.items) == 0,
timeout=5,
sleep_time=0.5,
):
raise Exception(
"Execution timed out while waiting for the CRD to be deleted")
# TODO: fix this, when calling create_custom_resource_definition, we get the error
# ValueError("Invalid value for `conditions`, must not be `None`")
# but the crd is still successfully created
try:
crdv1.create_custom_resource_definition(body=crd)
except ValueError as e:
pass
print("Ensured CRDs")
def get_clients():
"""
Get clusters, clients for each cluster, and the active cluster
:return: ((Dict) {cluster : context}
(Dict) {cluster : {client_type : client}}
(str) name of active cluster)
"""
clusters = k8s_config.update_available_clusters()
contexts, active_context = config.list_kube_config_contexts(
) # assuming current active cluster is the hub
active_cluster = active_context["context"]["cluster"]
clients = {}
for cluster in clusters:
context = clusters[cluster]
# building all clients for all clusters
clients[cluster] = {}
clients[cluster]["apps_client"] = client.AppsV1Api(
api_client=config.new_client_from_config(context=context))
clients[cluster]["batch_client"] = client.BatchV1Api(
api_client=config.new_client_from_config(context=context))
clients[cluster]["core_client"] = client.CoreV1Api(
api_client=config.new_client_from_config(context=context))
clients[cluster]["ext_client"] = client.ExtensionsV1beta1Api(
api_client=config.new_client_from_config(context=context))
clients[cluster]["api_client"] = client.ApiextensionsV1beta1Api(
api_client=config.new_client_from_config(context=context))
clients[cluster]["customs_client"] = client.CustomObjectsApi(
api_client=config.new_client_from_config(context=context))
return clusters, clients, active_cluster
def check_crd_deleted():
"""
Function for checking CRD (Custom Resource Defination) is deleted or not
If CRD is not deleted in 60 seconds,function asserts
Args:
None
Returns:
None
Raises:
Raises an exception on kubernetes client api failure and asserts
"""
var = True
count = 12
list_crd_api_instance = client.ApiextensionsV1beta1Api()
while (var and count > 0):
try:
list_crd_api_response = list_crd_api_instance.read_custom_resource_definition(
pretty=True, name="ibm-spectrum-scale-csi")
LOGGER.debug(list_crd_api_response)
LOGGER.info("still deleting crd")
count -= 1
time.sleep(5)
except ApiException:
LOGGER.info("crd deleted")
var = False
if count <= 0:
LOGGER.error("crd is not deleted")
assert False
def create_crd():
"""
Create IBM Spectrum Scale CSI Operator CRD (Custom Resource Defination) Object
Args:
None
Returns:
None
Raises:
Raises an ValueError exception but it is expected. hence we pass.
"""
filepath = "../../operator/deploy/crds/csiscaleoperators.csi.ibm.com.crd.yaml"
try:
with open(filepath, "r") as f:
loadcrd_yaml = yaml.full_load(f.read())
except yaml.YAMLError as exc:
print("Error in configuration file:", exc)
assert False
crd_api_instance = client.ApiextensionsV1beta1Api()
try:
LOGGER.info(
"Creating IBM SpectrumScale CRD object using csiscaleoperators.csi.ibm.com.crd.yaml file"
)
crd_api_response = crd_api_instance.create_custom_resource_definition(
loadcrd_yaml, pretty=True)
LOGGER.debug(str(crd_api_response))
except ValueError:
LOGGER.info(
"while there is valuerror expection,but CRD created successfully")
def check_crd_exists():
"""
Checks custom resource defination exists or not
Args:
None
Returns:
return True , if crd exists
return False , if crd does not exists
Raises:
None
"""
read_crd_api_instance = client.ApiextensionsV1beta1Api()
try:
read_crd_api_response = read_crd_api_instance.read_custom_resource_definition(
pretty=True, name="csiscaleoperators.csi.ibm.com")
LOGGER.debug(str(read_crd_api_response))
LOGGER.info("crd exists")
return True
except ApiException:
LOGGER.info("crd does not exist")
return False
def __init__(self, bearer_token=None):
'''
Initialize connection to Kubernetes
'''
self.bearer_token = bearer_token
api_client = None
try:
config.load_incluster_config()
except config.config_exception.ConfigException:
config.load_kube_config()
if self.bearer_token:
# Configure API key authorization: Bearer Token
configuration = client.Configuration()
configuration.api_key_prefix['authorization'] = 'Bearer'
configuration.api_key['authorization'] = self.bearer_token
api_client = client.ApiClient(configuration)
self.client = client.CoreV1Api(api_client)
self.batch_api = client.BatchV1Api(api_client)
self.batch_v1beta1_api = client.BatchV1beta1Api(api_client)
self.custom_objects = client.CustomObjectsApi(api_client)
self.api_extensions = client.ApiextensionsV1beta1Api(api_client)
self.extension_api = client.ExtensionsV1beta1Api(api_client)
self.apps_v1_api = client.AppsV1Api(api_client)
def get_crds() -> Optional[Dict]:
crdv1 = client.ApiextensionsV1beta1Api()
try:
crd = crdv1.list_custom_resource_definition(pretty="true")
except ApiException as e:
print("Exception when calling list_custom_resource_definition: %s\n" % e)
return None
return crd.to_dict()
def validate_cluster(splunk, record):
from kubernetes import client
try:
connection_stanza = splunklib.client.Stanza(splunk,
"",
skip_refresh=True)
connection_stanza.refresh(
state=splunklib.data.record({"content": record}))
config = create_client_configuration(connection_stanza)
api_client = client.ApiClient(config)
version_api = client.VersionApi(api_client)
version_api.get_code()
except errors.ApplicationError as e:
raise Exception("Could not connect to Kubernetes.\n\n%s" % e)
except Exception:
raise Exception(traceback.format_exc())
try:
extensions_api = client.ApiextensionsV1beta1Api(api_client)
crd = extensions_api.read_custom_resource_definition(
"standalones.enterprise.splunk.com")
if crd.spec.version != "v1alpha2":
raise errors.ApplicationError(
"Unexpected Splunk Operator version: %s" % crd.spec.version)
except client.rest.ApiException as e:
if e.status == 404:
raise errors.ApplicationError("Could not find Splunk Operator.")
raise
except errors.ApplicationError:
raise
except Exception:
raise Exception(traceback.format_exc())
try:
indexer_server_count = 0
for server in record.indexer_server.split(","):
components = server.split(":")
if len(components) != 2:
raise errors.ApplicationError(
"Expect format \"<server>:<port>,...\" for indexer server. Got \"%s\""
% (server))
hostname = components[0].strip()
port = int(components[1].strip())
import socket
s = socket.socket()
try:
s.connect((hostname, port))
except Exception as e:
raise errors.ApplicationError(
"Could not connect to indexer server \"%s\": %s" %
(server, e))
finally:
s.close()
indexer_server_count += 1
if indexer_server_count == 0:
raise errors.ApplicationError("Invalid or misssing indexer server")
except errors.ApplicationError:
raise
except Exception:
raise Exception(traceback.format_exc())
def __init__(self,
group: str,
version: str,
plural: str,
namespace: str,
template_cr: Dict,
labels: Dict) -> None:
"""Constructor."""
if 'KUBERNETES_PORT' in os.environ:
_LOGGER.info('%s/%s: Handler starting "incluster_config" mode', group, plural)
config.load_incluster_config()
else:
_LOGGER.info('%s/%s: Handler starting "kube_config" mode', group, plural)
config.load_kube_config()
# Instantiate required K8s APIs
self.core_api = client.CoreV1Api()
self.crd_api = client.ApiextensionsV1beta1Api()
self.co_api = client.CustomObjectsApi()
# K8s stream watcher
self.watcher = watch.Watch()
# Configs set to specify which CRs to monitor/control
self.group = group
self.version = version
self.plural = plural
self.namespace = namespace
self.label_selector = ''
for k, val in labels.items():
self.label_selector += k + '=' + val + ','
self.label_selector = self.label_selector[:-1]
# Latest resource version processed by watcher
self.resv_watcher = ''
# Callback stack for watch on cr
self.callbacks = {
'ADDED': OrderedDict(),
'MODIFIED': OrderedDict(),
'DELETED': OrderedDict(),
'REPROCESS': OrderedDict()
}
# JSON template used while creating custom resources
self.raw_cr = template_cr
# Lock objects to synchronize processing of CRs
self.cr_locks = defaultdict(threading.RLock)
# Dict to save thread exceptions
self.thread_exceptions = {}
# Init threads
self.watcher_thread = threading.Thread(target=self._watch_on_crs_loop, daemon=True)
self.reprocess_thread = threading.Thread(target=self._reprocess_crs_loop)
# Control flag for thread
self.thread_run = True
self.number_executor_threads = 1
def confirm_custom_resource_definition(context):
if context is None:
raise SystemExit(
"invalid empty context for CustomResourceDefinition given")
load_kube(context)
api = client.ApiextensionsV1beta1Api()
return general_confirm("CustomResourceDefinition",
lambda: api.list_custom_resource_definition(),
lambda i: i.metadata.name)
def crds_applied():
print("Checking for CRDs")
v1_ext = client.ApiextensionsV1beta1Api()
for x in range(5):
if x != 0:
time.sleep(SLEEP_INTERVAL)
current_crds = [x["spec"]["names"]["kind"].lower() for x in v1_ext.list_custom_resource_definition().to_dict()['items']]
if "configuration" in current_crds and "instance" in current_crds:
return True
return False
def _create_custom_resource_definitions(self):
if self._crd:
return
api = client.ApiextensionsV1beta1Api()
self._crd = \
CustomResourceDefinitionLoader._custom_resource_definition()
try:
api.create_custom_resource_definition(self._crd)
except client.rest.ApiException:
pass
def wait_for_custom_resource_definition_is_away(context, name):
if name is None:
raise SystemExit(
"invalid empty name for CustomResourceDefinition given")
if context is None:
raise SystemExit("invalid empty name context given")
load_kube(context)
print("check removal of", "CustomResourceDefinition", name)
api = client.ApiextensionsV1beta1Api()
return general_away_check(
None, "CustomResourceDefinition", name,
lambda: api.read_custom_resource_definition_with_http_info(name))
def Read_Custom_Resource_Definition_Status(self, name):
configuration = client.Configuration()
api_instance = client.ApiextensionsV1beta1Api(
client.ApiClient(configuration))
try:
api_response = api_instance.read_custom_resource_definition_status(
name, pretty='true')
pprint(api_response)
except ApiException as e:
print(
"Exception when calling ApiextensionsV1beta1Api->read_custom_resource_definition_status: %s\n"
% e)
def remove_custom_resource_definition(context, name):
if context is None:
raise SystemExit(
"invalid empty context for CustomResourceDefinition given")
if name is None:
raise SystemExit(
"invalid empty name for CustomResourceDefinition given")
load_kube(context)
api = client.ApiextensionsV1beta1Api()
ret, status, _ = api.delete_custom_resource_definition_with_http_info(name)
handle_status(ret, status, "CustomResourceDefinition", None, name)
def __init__(self):
# Create Kubernetes config.
load_incluster_config()
config = Configuration()
config.debug = Settings.KUBERNETES_SERVICE_DEBUG
self.api_client = client.ApiClient(config)
# Re-usable API client instances.
self.core_api = client.CoreV1Api(self.api_client)
self.custom_objects_api = client.CustomObjectsApi(self.api_client)
self.extensions_api = client.ApiextensionsV1beta1Api(self.api_client)
self.apps_api = client.AppsV1beta1Api(self.api_client)
def verify_can_run(self):
v1 = client.ApiextensionsV1beta1Api(self.api_client)
current_crds = [
x['spec']['names']['kind'].lower()
for x in v1.list_custom_resource_definition().to_dict()['items']
]
if 'nfsbucket' not in current_crds:
print(
"You need to create the CRD with kubectl apply -f nfs-crd.yaml"
)
raise RuntimeError(
"CRD does not exist in kubernetes cluster, controller can not start!"
)
def __init__(self) -> None:
kube_config.load_kube_config(config_file=os.environ.get(
'KUBECONFIG', KUBE_CONFIG_PATH), )
models.V1beta1PodDisruptionBudgetStatus.disrupted_pods = property(
fget=lambda *args, **kwargs: models.
V1beta1PodDisruptionBudgetStatus.disrupted_pods(*args, **kwargs),
fset=_set_disrupted_pods,
)
self.deployments = kube_client.AppsV1Api()
self.core = kube_client.CoreV1Api()
self.policy = kube_client.PolicyV1beta1Api()
self.apiextensions = kube_client.ApiextensionsV1beta1Api()
self.custom = kube_client.CustomObjectsApi()
def deploy_operator(namespace, operator_yaml):
# We use `kubectl` here because for the kubernetes client we would
# need to handle the document kind individually.
kubectl(["apply", "-f", operator_yaml])
operator_deployed = False
def is_operator_deployed(event):
if (event["type"] == "ADDED"
and event["object"].metadata.name == "mssql-operator"):
return True, event["object"]
else:
return False, event["object"]
apps_api = kclient.AppsV1Api()
operator_success, operator_deployed = kube_watch_event(
is_operator_deployed,
apps_api.list_namespaced_deployment,
namespace=namespace,
timeout_seconds=10)
log(LogLevel.ALL, "operator deployed:", operator_success)
def is_crd_deployed(event):
expected = "sqlservers.mssql.microsoft.com"
if (event["type"] == "ADDED"
and event["object"].metadata.name == expected):
return True, event["object"]
else:
return False, event["object"]
extensions_api = kclient.ApiextensionsV1beta1Api()
# We need to sleep here.
# Kubernetes client API will raise the following exception if the operator
# is not fully deployed when trying to list_custom_resource_definition:
# File "kubernetes/client/models/v1beta1_custom_resource_definition_status.py", line 105, in conditions # noqa: E501
# raise ValueError("Invalid value for `conditions`, must not be `None`")
log(LogLevel.ALL, "Waiting 10 seconds...")
time.sleep(10)
# Wait for crd to be deployed.
log(LogLevel.ALL,
"Waiting for Custom Resource Definition to be initialized")
log(LogLevel.INFO,
"This can be slow during the first deployment because we need to",
"pull docker images")
kube_watch_event(is_crd_deployed,
extensions_api.list_custom_resource_definition)
return operator_success
def __init__(self,
context=None,
usecloning=False,
host='127.0.0.1',
port=22,
user='******',
debug=False):
self.host = host
self.port = port
self.user = user
self.usecloning = usecloning
self.conn = 'OK'
contexts, current = config.list_kube_config_contexts()
if context is not None:
contexts = [
entry for entry in contexts if entry['name'] == context
]
if contexts:
context = contexts[0]
contextname = context['name']
else:
self.conn = None
else:
context = current
contextname = current['name']
config.load_kube_config(context=contextname)
if 'namespace' in context['context']:
self.namespace = context['context']['namespace']
else:
self.namespace = 'default'
self.crds = client.CustomObjectsApi()
extensions = client.ApiextensionsV1beta1Api()
current_crds = [
x for x in extensions.list_custom_resource_definition().to_dict()
['items'] if x['spec']['names']['kind'].lower() == 'virtualmachine'
]
if not current_crds:
common.pprint("Kubevirt not installed", color='red')
self.conn = None
self.host = context
return
self.core = client.CoreV1Api()
self.debug = debug
if host == '127.0.0.1' and len(contextname.split('/')) == 3 and len(
contextname.split('/')[1].split(':')) == 2:
self.host = contextname.split('/')[1].split(':')[0].replace(
'-', '.')
return
def create_crd(cls):
api_extensions_api = client.ApiextensionsV1beta1Api()
crd = {
"apiVersion": "apiextensions.k8s.io/v1beta1",
"kind": "CustomResourceDefinition",
"metadata": {
"name": cls.name_plural() + "." + GROUP
},
"spec": {
"group": GROUP,
"version": cls.version(),
"scope": "Cluster"
if issubclass(cls, SystemResourceModel) else "Namespaced",
"names": {
"plural": cls.name_plural(),
"singular": cls.name_singular(),
"kind": cls.kind(),
"listKind": cls.list_kind(),
}
}
}
# Patching the client to get around this issue:
# https://github.com/kubernetes-incubator/client-python/issues/415
@property
def accepted_names(self):
return self._accepted_names
@accepted_names.setter
def accepted_names(self, accepted_names):
self._accepted_names = accepted_names
@property
def conditions(self):
return self._conditions
@conditions.setter
def conditions(self, conditions):
self._conditions = conditions
V1beta1CustomResourceDefinitionStatus.accepted_names = accepted_names
V1beta1CustomResourceDefinitionStatus.conditions = conditions
try:
api_extensions_api.create_custom_resource_definition(crd)
except ApiException as e:
# If it already exists don't error
if e.status != 409:
raise
def kube_apis(cli_arguments) -> KubeApis:
"""
Set up kubernets-client to operate in cluster.
:param cli_arguments: a set of command-line arguments
:return: KubeApis
"""
context_name = cli_arguments['context']
kubeconfig = cli_arguments['kubeconfig']
config.load_kube_config(config_file=kubeconfig, context=context_name, persist_config=False)
v1 = client.CoreV1Api()
extensions_v1_beta1 = client.ExtensionsV1beta1Api()
rbac_v1_beta1 = client.RbacAuthorizationV1beta1Api()
api_extensions_v1_beta1 = client.ApiextensionsV1beta1Api()
custom_objects = client.CustomObjectsApi()
return KubeApis(v1, extensions_v1_beta1, rbac_v1_beta1, api_extensions_v1_beta1, custom_objects)
def __init__(self):
check_microk8s_kube_config_file()
load_kubernetes_config()
self.api_client = client.ApiClient()
self.custom_def_cli = client.CustomObjectsApi()
self.core_cli = client.CoreV1Api()
self.apps_cli = client.AppsV1Api()
self.rbac_cli = client.RbacAuthorizationV1Api()
self.extenstion_cli = client.ExtensionsV1beta1Api()
self.crd_cli = client.ApiextensionsV1beta1Api()
self.storage_cli = client.StorageV1Api()
self.admission_cli = client.AdmissionregistrationV1beta1Api()
self.delete_options = client.V1DeleteOptions()
self.delete_options.grace_period_seconds = 2
self.delete_options.propagation_policy = 'Foreground'
self.core_cli.api_client.configuration.assert_hostname = False
self.apps_cli.api_client.configuration.assert_hostname = False
def create_crds(self):
# API Extensions V1 beta1 API client.
crd_api = client.ApiextensionsV1beta1Api()
crd_dir = os.path.join(os.path.dirname(__file__), '../resources/crds')
crd_paths = [os.path.abspath(os.path.join(crd_dir, name))
for name in os.listdir(crd_dir)]
for path in crd_paths:
with open(path) as crd_file:
crd_json = json.dumps(
yaml.load(crd_file.read()), sort_keys=True, indent=2)
crd = deserialize_object(
crd_json, 'V1beta1CustomResourceDefinition')
try:
crd_api.create_custom_resource_definition(crd)
except Exception:
pass
def __init__(self, squirrel):
self.squirrel = squirrel
config.load_incluster_config()
self.configuration = client.Configuration()
self.configuration.assert_hostname = False
self.api_client = client.api_client.ApiClient(
configuration=self.configuration)
self.v1 = client.ApiextensionsV1beta1Api(self.api_client)
print("Creating definition")
for d in ["nuts.yml", "nutcrackers.yml"]:
self.definition = '%s/%s' % (squirrel.squirrel_controller, d)
with open(self.definition) as data:
body = yaml.load(data)
try:
self.v1.create_custom_resource_definition(body)
except Exception as e:
print("Ignoring error: %s" % e)
self.crds = client.CustomObjectsApi(self.api_client)
def __init__(self):
atexit.register(self.clean_up)
if 'KUBERNETES_PORT' in os.environ:
config.load_incluster_config()
else:
config.load_kube_config()
self.configuration = client.Configuration()
self.configuration.assert_hostname = False
self.api_client = client.api_client.ApiClient(
configuration=self.configuration)
self.v1_client = client.ApiextensionsV1beta1Api(self.api_client)
self.crd_client = client.CustomObjectsApi(self.api_client)
self.jobs = {}
# launch default clean up process
self.clean_up()
def wait_for_crd(cls):
name = cls.name_plural() + "." + GROUP
api_extensions_api = client.ApiextensionsV1beta1Api()
while True:
try:
crd = api_extensions_api.read_custom_resource_definition(name)
for cond in crd.status.conditions:
if cond.type == "Established":
return
if cond.type == "NamesAccepted":
if cond.status == "False":
raise ValueError(
"Failed to create CRD {0}: name conflict: {1}".
format(name, cond.message))
except ApiException as e:
# If it doesn't exist just wait, k8s may be slow
if e.status != 404:
raise
time.sleep(1)
def __init__(self, *args, **kwargs):
"""
Constructor
Args:
args -- posistional arguments
kwargs -- Named arguments
Returns:
K8s object
"""
super(K8s, self).__init__(*args, **kwargs)
lib.log.debug("K8s init", extra=dict(**kwargs))
kube_config = kwargs["kube_config"] if "kube_config" in kwargs else ""
if kube_config and len(kube_config) > 0:
lib.log.info("using kube_config=%s", kube_config)
config.load_kube_config(config_file=kube_config)
else:
lib.log.info("no kube_config, running in-cluster")
config.load_incluster_config()
self.v1 = client.CoreV1Api()
self.v1App = client.AppsV1Api()
self.v1ext = client.ExtensionsV1beta1Api()
self.v1beta1 = client.ApiextensionsV1beta1Api()
self.custom = client.CustomObjectsApi()
self.auto_scaler = client.AutoscalingV1Api()
self.rbac = client.RbacAuthorizationV1Api()
if 'cluster_name' in kwargs:
lib.log.info("using explicit cluster_set= %s, cluster_name=%s",
kwargs.get('cluster_set'), kwargs.get('cluster_name'))
self.cluster_info = {
"cluster.name": kwargs.get('cluster_name'),
"cluster.set": kwargs.get('cluster_set')
}
else:
lib.log.info("getting cluster info")
self.cluster_info = self.get_cluster_info()
lib.log.debug("kube-system/cluster-data ConfigMap: {}".format(
self.cluster_info))
def _ensure_crds():
"""
ensure_crds makes sure that all the required CRDs have been created
"""
crdv1 = client.ApiextensionsV1beta1Api()
crd = _load_mongodb_crd()
ignore_if_doesnt_exist(
lambda: crdv1.delete_custom_resource_definition("mongodb.mongodb.com")
)
# TODO: fix this, when calling create_custom_resource_definition, we get the error
# ValueError("Invalid value for `conditions`, must not be `None`")
# but the crd is still successfully created
try:
crdv1.create_custom_resource_definition(body=crd)
except ValueError as e:
pass
print("Ensured CRDs")
def create_scaling_decision_crd(self):
definition = '../../example/scaling_decision_crd.yml'
v1 = client.ApiextensionsV1beta1Api()
current_crds = [
x['spec']['names']['kind'].lower()
for x in v1.list_custom_resource_definition().to_dict()['items']
]
# print(current_crds)
if 'decision' not in current_crds:
print("Creating Decision definition")
with open(definition) as data:
body = yaml.load(data, Loader=yaml.Loader)
# print(yaml.dump(body))
try:
v1.create_custom_resource_definition(body)
except ValueError as exception: # Check: https://github.com/kubernetes-client/python/issues/1098
if str(
exception
) == 'Invalid value for `conditions`, must not be `None`':
print("Skipping invalid \'conditions\' value...")
else:
raise exception
