def _get_api(cluster):
""" Get custom objects api associated with a cluster specifier.
"""
if cluster is None:
load_incluster_config()
api = CustomObjectsApi()
elif cluster == 'remote_transcode':
host = os.getenv('REMOTE_TRANSCODE_HOST')
port = os.getenv('REMOTE_TRANSCODE_PORT')
token = os.getenv('REMOTE_TRANSCODE_TOKEN')
cert = os.getenv('REMOTE_TRANSCODE_CERT')
conf = Configuration()
conf.api_key['authorization'] = token
conf.host = f'https://{host}:{port}'
conf.verify_ssl = True
conf.ssl_ca_cert = cert
api_client = ApiClient(conf)
api = CustomObjectsApi(api_client)
else:
cluster_obj = JobCluster.objects.get(pk=cluster)
host = cluster_obj.host
port = cluster_obj.port
token = cluster_obj.token
fd, cert = tempfile.mkstemp(text=True)
with open(fd, 'w') as f:
f.write(cluster_obj.cert)
conf = Configuration()
conf.api_key['authorization'] = token
conf.host = f'https://{host}:{port}'
conf.verify_ssl = True
conf.ssl_ca_cert = cert
api_client = ApiClient(conf)
api = CustomObjectsApi(api_client)
return api
def __init__(self):
""" Intializes the connection. If environment variables for
remote transcode are defined, connect to that cluster.
"""
host = os.getenv('REMOTE_TRANSCODE_HOST')
port = os.getenv('REMOTE_TRANSCODE_PORT')
token = os.getenv('REMOTE_TRANSCODE_TOKEN')
cert = os.getenv('REMOTE_TRANSCODE_CERT')
self.remote = host is not None
if self.remote:
conf = Configuration()
conf.api_key['authorization'] = token
conf.host = f'https://{host}:{port}'
conf.verify_ssl = True
conf.ssl_ca_cert = cert
api_client = ApiClient(conf)
self.corev1 = CoreV1Api(api_client)
self.custom = CustomObjectsApi(api_client)
else:
load_incluster_config()
self.corev1 = CoreV1Api()
self.custom = CustomObjectsApi()
self.setup_common_steps()
def __init__(self, alg):
""" Intializes the connection. If algorithm object includes
a remote cluster, use that. Otherwise, use this cluster.
"""
if alg.cluster:
host = alg.cluster.host
port = alg.cluster.port
token = alg.cluster.token
fd, cert = tempfile.mkstemp(text=True)
with open(fd, 'w') as f:
f.write(alg.cluster.cert)
conf = Configuration()
conf.api_key['authorization'] = token
conf.host = f'{PROTO}{host}:{port}'
conf.verify_ssl = True
conf.ssl_ca_cert = cert
api_client = ApiClient(conf)
self.corev1 = CoreV1Api(api_client)
self.custom = CustomObjectsApi(api_client)
else:
load_incluster_config()
self.corev1 = CoreV1Api()
self.custom = CustomObjectsApi()
# Read in the manifest.
if alg.manifest:
self.manifest = yaml.safe_load(alg.manifest.open(mode='r'))
# Save off the algorithm.
self.alg = alg
def _cleanup(resource):
k8s_v1beta1_client = ApiextensionsV1beta1Api(admin_mc.k8s_client)
k8s_client = CustomObjectsApi(admin_mc.k8s_client)
def clean():
kind = resource["kind"]
metadata = resource["metadata"]
api_version = resource["apiVersion"]
api_version_parts = api_version.split("/")
if len(api_version_parts) != 2:
raise ValueError("Error parsing ApiVersion [" + api_version +
"]." + "Expected form \"group/version\"")
group = api_version_parts[0]
version = api_version_parts[1]
crd_list = k8s_v1beta1_client.\
list_custom_resource_definition().items
crd = list(
filter(
lambda x: x.spec.names.kind == kind and x.spec.group ==
group and x.spec.version == version, crd_list))[0]
try:
k8s_client.delete_namespaced_custom_object(
group, version, metadata["namespace"],
crd.spec.names.plural, metadata["name"], {})
except ApiException as e:
body = json.loads(e.body)
if body["code"] not in WAIT_HTTP_ERROR_CODES:
raise e
request.addfinalizer(clean)
def __init__(self,
kube_api_client: ApiClient,
name: str,
namespace: str = env_variables['namespace']):
BaseCustomResource.__init__(self, name, namespace)
self.crd_api = CustomObjectsApi(kube_api_client)
self._assigned_secret = None
def _patch_and_delete_stubborn_custom_resources( # type: ignore
group: str,
version: str,
plural: str,
namespace: str,
status_element: str,
logger: kopf.Logger,
use_async=True,
**_: Any,
):
logger.info(f"_patch_and_delete_stubborn_custom_resources for {plural}.{group} in namespace {namespace}")
co = CustomObjectsApi()
resp = co.list_namespaced_custom_object(group=group, version=version, plural=plural, namespace=namespace)
failed_res = [
item.get("metadata").get("name")
for item in resp["items"]
if item.get("status", {}).get(status_element) in ["Failed", "Completed", "InProgress"]
]
for item in failed_res:
try:
logger.info(f"Patching item {item} in {plural}.{group}")
patch = json.loads("""{"metadata":{"finalizers":[]}}""")
co.patch_namespaced_custom_object(
group=group, version=version, plural=plural, namespace=namespace, name=item, body=patch
)
logger.info(f"Deleting item {item} in {plural}.{group}")
co.delete_namespaced_custom_object(
group=group,
version=version,
plural=plural,
namespace=namespace,
name=item,
)
except ApiException as e:
logger.warn("Trying to patch and delete failed: %s\n" % e)
def test_cluster_node_count(admin_mc, remove_resource,
raw_remove_custom_resource):
"""Test that the cluster node count gets updated as nodes are added"""
client = admin_mc.client
cluster = client.create_cluster(
name=random_str(), rancherKubernetesEngineConfig={"accessKey": "junk"})
remove_resource(cluster)
def _check_node_count(cluster, nodes):
c = client.reload(cluster)
return c.nodeCount == nodes
def _node_count_fail(cluster, nodes):
c = client.reload(cluster)
s = "cluster {} failed to have proper node count, expected: {} has: {}"
return s.format(c.id, nodes, c.nodeCount)
node_count = 0
wait_for(lambda: _check_node_count(cluster, node_count),
fail_handler=lambda: _node_count_fail(cluster, node_count))
# Nodes have to be created manually through k8s client to attach to a
# pending cluster
k8s_dynamic_client = CustomObjectsApi(admin_mc.k8s_client)
body = {
"metadata": {
"name": random_str(),
"namespace": cluster.id,
},
"kind": "Node",
"apiVersion": "management.cattle.io/v3",
}
dynamic_nt = k8s_dynamic_client.create_namespaced_custom_object(
"management.cattle.io", "v3", cluster.id, 'nodes', body)
raw_remove_custom_resource(dynamic_nt)
node_count = 1
wait_for(lambda: _check_node_count(cluster, node_count),
fail_handler=lambda: _node_count_fail(cluster, node_count))
# Create node number 2
body['metadata']['name'] = random_str()
dynamic_nt1 = k8s_dynamic_client.create_namespaced_custom_object(
"management.cattle.io", "v3", cluster.id, 'nodes', body)
raw_remove_custom_resource(dynamic_nt1)
node_count = 2
wait_for(lambda: _check_node_count(cluster, node_count),
fail_handler=lambda: _node_count_fail(cluster, node_count))
# Delete a node
k8s_dynamic_client.delete_namespaced_custom_object(
"management.cattle.io", "v3", cluster.id, 'nodes',
dynamic_nt1['metadata']['name'], {})
node_count = 1
wait_for(lambda: _check_node_count(cluster, node_count),
fail_handler=lambda: _node_count_fail(cluster, node_count))
def set_cluster_psp(admin_mc, value):
"""Enable or Disable the pod security policy at the local cluster"""
k8s_dynamic_client = CustomObjectsApi(admin_mc.k8s_client)
# these create a mock pspts... not valid for real psp's
def update_cluster():
try:
local_cluster = k8s_dynamic_client.get_cluster_custom_object(
"management.cattle.io", "v3", "clusters", "local")
local_cluster["metadata"]["annotations"][
"capabilities/pspEnabled"] = value
k8s_dynamic_client.replace_cluster_custom_object(
"management.cattle.io", "v3", "clusters", "local",
local_cluster)
except ApiException as e:
assert e.status == 409
return False
return True
wait_for(update_cluster)
def check_psp():
cluster_obj = admin_mc.client.by_id_cluster(id="local")
return str(cluster_obj.capabilities.pspEnabled).lower() == value
wait_for(check_psp)
def top_node(self, node):
custom = CustomObjectsApi(self.api_client)
data = custom.get_cluster_custom_object("metrics.k8s.io", "v1beta1",
"nodes", node)
node = data['metadata']['name']
cpu = parse_resource(data['usage']['cpu'])
memory = parse_resource(data['usage']['memory'])
return NodeMetric(node=node, cpu=cpu, memory=memory / ONE_GIBI)
def __init__(
self,
kube_api_client: ApiClient,
name: str,
namespace: str = consts.DEFAULT_NAMESPACE,
):
super().__init__(name, namespace)
self.crd_api = CustomObjectsApi(kube_api_client)
def __init__(
self,
kube_api_client: ApiClient,
name: str,
namespace: str = env_variables["namespace"],
):
super().__init__(name, namespace)
self.crd_api = CustomObjectsApi(kube_api_client)
def setUp(self):
self.namespace = "default"
self.yml_file = "{}/files/spark.yml".format(os.path.abspath('.'))
self.group = 'sparkoperator.k8s.io'
self.version = 'v1beta1'
self.plural = 'sparkapplications'
self.job_name = "{}-{}".format("spark-test-job", int(time.time()))
config = kube_config.load_kube_config()
self.api_instance = CustomObjectsApi(ApiClient(config))
def test_auth_label(admin_mc, user_factory):
user = user_factory()
k8s_client = CustomObjectsApi(admin_mc.k8s_client)
user_token = wait_for(
lambda: user_token_creation(k8s_client, user.user.id),
timeout=30,
fail_handler=lambda: "failed to find token for factory user login")
label_name = "authn.management.cattle.io/kind"
assert user_token["metadata"]["labels"][label_name] == "session"
def get_control_plane(self):
crd_api = CustomObjectsApi(self.management_kube_api_client)
return crd_api.get_namespaced_custom_object(
group=HyperShift.HYPERSHIFT_API_GROUP,
version=HyperShift.HYPERSHIFT_API_VERSION,
plural=HyperShift.HOSTED_CONTROL_PLANE_PLOURAL,
name=self.name,
namespace=self.namespace,
)
def get_node_stats(self, node_name):
cust = CustomObjectsApi()
if node_name:
return cust.get_cluster_custom_object('metrics.k8s.io', 'v1beta1',
'nodes', node_name)
else:
return cust.list_cluster_custom_object('metrics.k8s.io', 'v1beta1',
'nodes')
def test_legacy_template_migrate_and_delete(admin_mc, admin_cc,
remove_resource, user_mc,
raw_remove_custom_resource):
"""Asserts that any node template not in cattle-global-nt namespace is
duplicated into cattle-global-nt, then deleted"""
admin_client = admin_mc.client
admin_cc_client = admin_cc.client
user_client = user_mc.client
k8s_dynamic_client = CustomObjectsApi(admin_mc.k8s_client)
ns = admin_cc_client.create_namespace(name="ns-" + random_str(),
clusterId=admin_cc.cluster.id)
remove_resource(ns)
node_template_name = "nt-" + random_str()
body = {
"metadata": {
"name": node_template_name,
"annotations": {
"field.cattle.io/creatorId": user_mc.user.id
}
},
"kind": "NodeTemplate",
"apiVersion": "management.cattle.io/v3",
"azureConfig": {
"customData": "asdfsadfsd"
}
}
dynamic_nt = k8s_dynamic_client.create_namespaced_custom_object(
"management.cattle.io", "v3", ns.name, 'nodetemplates', body)
raw_remove_custom_resource(dynamic_nt)
def migrated_template_exists(id):
try:
nt = user_client.by_id_node_template(id=id)
remove_resource(nt)
return nt
except ApiError as e:
assert e.error.status == 403
return False
id = "cattle-global-nt:nt-" + ns.id + "-" + dynamic_nt["metadata"]["name"]
nt = wait_for(
lambda: migrated_template_exists(id),
fail_handler=lambda: "failed waiting for node template to migrate")
# assert that config has not been removed from node template
assert nt.azureConfig["customData"] ==\
dynamic_nt["azureConfig"]["customData"]
wait_for(
lambda: admin_client.by_id_node_template(id=ns.name + ":" +
node_template_name) is None,
fail_handler=lambda: "failed waiting for old node template to delete")
def __init__(
self,
kube_api_client: ApiClient,
name: str,
namespace: str = consts.DEFAULT_NAMESPACE,
):
super().__init__(name, namespace)
self.crd_api = CustomObjectsApi(kube_api_client)
self.ref.kind = self._kind
self.ref.group = self._api_group
self.ref.version = self._api_version
def __init__(
self,
kube_api_client: ApiClient,
name: str,
namespace: str = env_variables["namespace"],
):
super().__init__(name, namespace)
self.crd_api = CustomObjectsApi(kube_api_client)
self.ref.kind = self._kind
self.ref.group = self._api_group
self.ref.version = self._api_version
def get_pod_stats(self, tenant, vnf_uuid):
cust = CustomObjectsApi()
data = ['metrics.k8s.io', 'v1beta1', tenant, 'pods']
try:
if vnf_uuid:
data.append(self.vnfs[vnf_uuid].name)
return cust.get_namespaced_custom_object(*data)
else:
return cust.list_namespaced_custom_object(*data)
except ApiException as ex:
raise KeyError(ex)
def __init__(self, alg):
""" Intializes the connection. If algorithm object includes
a remote cluster, use that. Otherwise, use this cluster.
"""
if alg.cluster:
host = alg.cluster.host
port = alg.cluster.port
token = alg.cluster.token
fd, cert = tempfile.mkstemp(text=True)
with open(fd, 'w') as f:
f.write(alg.cluster.cert)
conf = Configuration()
conf.api_key['authorization'] = token
conf.host = f'{PROTO}{host}:{port}'
conf.verify_ssl = True
conf.ssl_ca_cert = cert
api_client = ApiClient(conf)
self.corev1 = CoreV1Api(api_client)
self.custom = CustomObjectsApi(api_client)
else:
load_incluster_config()
self.corev1 = CoreV1Api()
self.custom = CustomObjectsApi()
# Read in the manifest.
if alg.manifest:
self.manifest = yaml.safe_load(alg.manifest.open(mode='r'))
if 'volumeClaimTemplates' in self.manifest['spec']:
for claim in self.manifest['spec']['volumeClaimTemplates']:
storage_class_name = claim['spec'].get(
'storageClassName', None)
if storage_class_name is None:
claim['storageClassName'] = os.getenv(
'WORKFLOW_STORAGE_CLASS')
logger.warning(
f"Implicitly sc to pvc of Algo:{alg.pk}")
# Save off the algorithm.
self.alg = alg
def test_roletemplate_finalizer_cleanup(admin_mc, remove_resource):
""" This ensures that roletemplates cleanup for clusters < v2.2.8
is performed correctly"""
client = admin_mc.client
rt = client.create_roleTemplate(name="rt-" + random_str())
remove_resource(rt)
assert rt.annotations["field.cattle.io/rtUpgrade"] == "true"
# create rt without rancher api with a bad finalizer
api = CustomObjectsApi(admin_mc.k8s_client)
json = {
"apiVersion": "management.cattle.io/v3",
"kind": "RoleTemplate",
"metadata": {
"finalizers": [
"clusterscoped.controller.cattle.io/" +
"cluster-roletemplate-sync_fake", "fake-finalizer"
],
"name":
"test-" + random_str(),
}
}
rt_k8s = api.create_cluster_custom_object(
group="management.cattle.io",
version="v3",
plural="roletemplates",
body=json,
)
rt_name = rt_k8s["metadata"]["name"]
rt_k8s = client.by_id_roleTemplate(id=rt_name)
remove_resource(rt_k8s)
def check_annotation():
rt1 = client.by_id_roleTemplate(rt_k8s.id)
try:
if rt1.annotations["field.cattle.io/rtUpgrade"] == "true":
return True
else:
return False
except (AttributeError, KeyError):
return False
wait_for(check_annotation, fail_handler=lambda: "annotation was not added")
rt1 = api.get_cluster_custom_object(
group="management.cattle.io",
version="v3",
plural="roletemplates",
name=rt_k8s.id,
)
if "finalizers" in rt1["metadata"]:
assert "clusterscoped.controller.cattle.io/grb-sync_fake" \
not in rt1["metadata"]["finalizers"]
def kube_api_logs(args):
client = ClientFactory.create_kube_api_client(args.kubeconfig_path)
for item in ClusterDeployment.list_all_namespaces(
CustomObjectsApi(client)).get("items", []):
if item["spec"]["clusterName"]:
download_logs_kube_api(
client,
item["spec"]["clusterName"],
item["metadata"]["namespace"],
args.dest,
args.must_gather,
args.kubeconfig_path,
)
def set_nodepool_node_count(self, kube_api_client: ApiClient, node_count: int) -> None:
log.info(f"Setting HyperShift cluster {self.name} node count to: {node_count}")
crd_api = CustomObjectsApi(kube_api_client)
node_count = node_count
body = {"spec": {"nodeCount": node_count}}
crd_api.patch_namespaced_custom_object(
group=HyperShift.HYPERSHIFT_API_GROUP,
version=HyperShift.HYPERSHIFT_API_VERSION,
plural=HyperShift.NODEPOOL_PLOURAL,
name=self.name,
namespace=HyperShift.NODEPOOL_NAMESPACE,
body=body,
)
def create_kubeconfig(request, dind_cc, client):
# request cluster scoped kubeconfig, permissions may not be synced yet
def generateKubeconfig(max_attempts=5):
for attempt in range(1, max_attempts+1):
try:
# get cluster for client
cluster = client.by_id_cluster(dind_cc.cluster.id)
return cluster.generateKubeconfig()['config']
except ApiError as err:
if attempt == max_attempts:
raise err
time.sleep(1)
cluster_kubeconfig = generateKubeconfig()
# write cluster scoped kubeconfig
cluster_kubeconfig_file = "kubeconfig-" + random_str() + ".yml"
f = open(cluster_kubeconfig_file, "w")
f.write(cluster_kubeconfig)
f.close()
# cleanup file when done
request.addfinalizer(lambda: os.remove(cluster_kubeconfig_file))
# extract token name
config = yaml.safe_load(cluster_kubeconfig)
token_name = config['users'][0]['user']['token'].split(':')[0]
# wait for token to sync
crd_client = CustomObjectsApi(
kubernetes_api_client(
dind_cc.admin_mc.client,
dind_cc.cluster.id
)
)
def cluster_token_available():
try:
return crd_client.get_namespaced_custom_object(
'cluster.cattle.io',
'v3',
'cattle-system',
'clusterauthtokens',
token_name
)
except ApiException:
return None
wait_for(cluster_token_available)
return cluster_kubeconfig_file
def test_globalrolebinding_finalizer_cleanup(admin_mc, remove_resource):
"""This ensures that globalrolebinding cleanup of clusters < v2.2.8
is performed correctly"""
client = admin_mc.client
grb = client.create_globalRoleBinding(globalRoleId="admin",
userId="u-" + random_str())
remove_resource(grb)
assert grb.annotations["field.cattle.io/grbUpgrade"] == "true"
# create a grb without the rancher api with a bad finalizer
api = CustomObjectsApi(admin_mc.k8s_client)
json = {
"apiVersion": "management.cattle.io/v3",
"globalRoleName": "admin",
"kind": "GlobalRoleBinding",
"metadata": {
"finalizers": ["clusterscoped.controller.cattle.io/grb-sync_fake"],
"generation": 1,
"name": "grb-" + random_str(),
},
"userName": "******" + random_str(),
}
grb_k8s = api.create_cluster_custom_object(
group="management.cattle.io",
version="v3",
plural="globalrolebindings",
body=json,
)
grb_name = grb_k8s["metadata"]["name"]
grb_k8s = client.by_id_globalRoleBinding(id=grb_name)
remove_resource(grb_k8s)
def check_annotation():
grb1 = client.by_id_globalRoleBinding(grb_k8s.id)
try:
if grb1.annotations["field.cattle.io/grbUpgrade"] == "true":
return True
else:
return False
except (AttributeError, KeyError):
return False
wait_for(check_annotation, fail_handler=lambda: "annotation was not added")
grb1 = api.get_cluster_custom_object(
group="management.cattle.io",
version="v3",
plural="globalrolebindings",
name=grb_k8s.id,
)
assert ("clusterscoped.controller.cattle.io/grb-sync_fake"
not in grb1["metadata"]["finalizers"])
def set_nodepool_replicas(self, node_count: int) -> None:
log.info(
f"Setting HyperShift cluster {self.name} replicas to: {node_count}"
)
crd_api = CustomObjectsApi(self.management_kube_api_client)
body = {"spec": {"replicas": node_count}}
crd_api.patch_namespaced_custom_object(
group=HyperShift.HYPERSHIFT_API_GROUP,
version=HyperShift.HYPERSHIFT_API_VERSION,
plural=HyperShift.NODEPOOL_PLOURAL,
name=self.name,
namespace=HyperShift.NODEPOOL_NAMESPACE,
body=body,
)
def k8s_api_client(client, cluster_name, kube_path=None):
kube_path = None
if kube_path is not None:
kube_file = open(kube_path, "r")
kube_config = kube_file.read()
kube_file.close()
else:
cluster = client.rancher_api_client().by_id_cluster(cluster_name)
kube_config = cluster.generateKubeconfig().config
loader = KubeConfigLoader(config_dict=yaml.full_load(kube_config))
client_configuration = type.__call__(Configuration)
loader.load_and_set(client_configuration)
client_configuration.api_key = {}
client_configuration.verify_ssl = False
k8s_client = ApiClient(configuration=client_configuration)
return CustomObjectsApi(api_client=k8s_client).api_client
def execute(self, context):
# initialize config
try:
config = kube_config.load_incluster_config()
except:
config = kube_config.load_kube_config()
# create an instance of the API class
api_instance = CustomObjectsApi(ApiClient(config))
# params to create custom object
params = [self.group, self.version, self.namespace, self.plural]
crd_created = self.create_custom_definition(api_instance, *params)
if crd_created:
w = Watch()
for event in w.stream(api_instance.list_namespaced_custom_object, *params, timeout_seconds=self.timeout):
job_name = event.get('object', {}).get('metadata', {}).get('name')
job_state = event.get('object', {}).get('status', {}).get('applicationState', {}).get('state')
if job_name == self.job_name and job_state == "COMPLETED":
break
def top_pods(self):
custom = CustomObjectsApi(self.api_client)
data = custom.list_cluster_custom_object("metrics.k8s.io", "v1beta1",
"pods")
usage_by_pod = collections.defaultdict(list)
for pod_data in data['items']:
pod_name = pod_data['metadata']['name']
for container_data in pod_data['containers']:
usage_by_pod[pod_name].append({
'pod':
container_data['name'],
'cpu':
parse_resource(container_data['usage']['cpu']),
'memory':
parse_resource(container_data['usage']['memory']) /
ONE_MEBI,
})
return usage_by_pod
def _create_custom_object_with_plural(custom_object: Manifest, plural: str):
logging.info("Creating %s %r ", custom_object.body["kind"],
custom_object.name)
try:
group, version = custom_object.body.get("apiVersion").rsplit("/", 1)
return CustomObjectsApi().create_namespaced_custom_object(
namespace=custom_object.namespace,
body=custom_object.body,
group=group,
version=version,
plural=plural,
)
except ApiException as err:
logging.error(
"Failed to create %s %r: %s",
custom_object.body["kind"],
custom_object.name,
err.reason,
)
raise
