def run_deployment(pod_spec,
replicas,
deploy_name,
template_label,
config_path=None):
template = client.V1PodTemplateSpec(
metadata=client.V1ObjectMeta(labels=template_label), spec=pod_spec)
spec = client.V1DeploymentSpec(replicas=replicas,
template=template,
selector={'matchLabels': template_label})
deployment_metadata = client.V1ObjectMeta(name=deploy_name,
labels=template_label)
deployment = client.V1Deployment(api_version="apps/v1",
kind="Deployment",
metadata=deployment_metadata,
spec=spec)
if config_path is None:
base_dir = os.getcwd()
config_path = os.path.join(base_dir, 'k3s_config.yaml')
kube_config.load_kube_config(config_file=config_path)
appsv1_client = client.AppsV1Api()
appsv1_client.create_namespaced_deployment(body=deployment,
namespace="default")
def get_pod_list(config_path=None):
if config_path is None:
base_dir = os.getcwd()
config_path = os.path.join(base_dir, 'k3s_config.yaml')
kube_config.load_kube_config(config_file=config_path)
k_client = client.CoreV1Api()
output = []
pod_info = k_client.list_namespaced_pod('default')
for pod in pod_info.items:
pod_age = None
if pod.status.start_time is not None:
pod_age = datetime.datetime.now(tz=tzutc()) - pod.status.start_time
restart_count = None
if pod.status.container_statuses is not None:
restart_count = pod.status.container_statuses[0].restart_count
output.append({
'name': pod.metadata.name,
'node_name': pod.spec.node_name,
'status': pod.status.phase,
'age': pod_age,
'restart_count': restart_count,
'deletion_timestamp': pod.metadata.deletion_timestamp
})
return output
def __configure_by_params(self):
""" Return API client from configuration file """
auth_args = AUTH_ARG_SPEC.keys()
core_configuration = Configuration()
for key, value in iteritems(self.params):
if key in auth_args and value is not None:
if key == 'api_key':
setattr(
sdk.configuration,
key, {'authorization': "Bearer {0}".format(value)})
setattr(
core_configuration,
key, {'authorization': "Bearer {0}".format(value)})
else:
setattr(sdk.configuration, key, value)
setattr(core_configuration, key, value)
if not self.params.get('verify_ssl'):
sdk.configuration.verify_ssl = False
core_configuration.verify_ssl = False
kube_config.load_kube_config(client_configuration=sdk.configuration)
Configuration.set_default(core_configuration)
return sdk.DefaultApi(), core_client.CoreV1Api()
def get_kube_configuration():
config = Configuration()
config.host = None
if os.path.exists(os.path.expanduser(kube_config.KUBE_CONFIG_DEFAULT_LOCATION)):
kube_config.load_kube_config(client_configuration=config)
else:
print(
"Unable to load config from %s" % kube_config.KUBE_CONFIG_DEFAULT_LOCATION
)
for url in [
"https://%s:8443" % DEFAULT_E2E_HOST,
"http://%s:8080" % DEFAULT_E2E_HOST,
]:
try:
urllib3.PoolManager().request("GET", url)
config.host = url
config.verify_ssl = False
urllib3.disable_warnings()
break
except urllib3.exceptions.HTTPError:
pass
if config.host is None:
raise unittest.SkipTest("Unable to find a running Kubernetes instance")
config.assert_hostname = False
return config
def get_e2e_configuration():
config = Configuration()
config.host = '172.16.50.14'
if os.path.exists(
os.path.expanduser(kube_config.KUBE_CONFIG_DEFAULT_LOCATION)):
kube_config.load_kube_config(client_configuration=config)
else:
print('Unable to load config from %s' %
kube_config.KUBE_CONFIG_DEFAULT_LOCATION)
for url in [
'https://%s:8443' % DEFAULT_E2E_HOST,
'http://%s:8080' % DEFAULT_E2E_HOST
]:
try:
urllib3.PoolManager().request('GET', url)
config.host = url
config.verify_ssl = False
urllib3.disable_warnings()
break
except urllib3.exceptions.HTTPError:
pass
if config.host is None:
raise unittest.SkipTest('Unable to find a running Kubernetes instance')
print('Running test against : %s' % config.host)
config.assert_hostname = False
return config
def build(self, url=None):
config = Configuration()
config.host = None
if os.path.exists(
os.path.expanduser(kube_config.KUBE_CONFIG_DEFAULT_LOCATION)):
kube_config.load_kube_config(client_configuration=config)
else:
print('Unable to load config from %s' %
kube_config.KUBE_CONFIG_DEFAULT_LOCATION)
if url is None:
url = 'http://%s:8085' % Config.DEFAULT_HOST
try:
urllib3.PoolManager().request('GET', url)
config.host = url
config.verify_ssl = False
urllib3.disable_warnings()
except urllib3.exceptions.HTTPError:
raise urllib3.exceptions.HTTPError(
'Unable to find a running Kubernetes instance')
self.config = config
#Resetting old Client & Related API's
self.clnt = None
# api_manager.reset()
print('Running test against : %s' % config.host)
return config
def __create_kubevirt_client(self, config_file, verify_ssl, context):
sdk.configuration.verify_ssl = verify_ssl
kube_config.load_kube_config(
config_file=config_file,
context=context,
client_configuration=sdk.configuration
)
return sdk.DefaultApi()
def get_pod_logs(pod_name, config_path=None):
if config_path is None:
base_dir = os.getcwd()
config_path = os.path.join(base_dir, 'k3s_config.yaml')
kube_config.load_kube_config(config_file=config_path)
k_client = client.CoreV1Api()
return k_client.read_namespaced_pod_log(pod_name, 'default')
def __init__(self) -> None:
if os.path.exists(SERVICE_TOKEN_FILENAME):
load_incluster_config()
else:
load_kube_config()
self.api = ApiClient()
version_api = VersionApi(self.api)
self._is_openshift = "eks" not in version_api.get_code().git_version
def run_master_deployment(deploy_name='pymada-master-deployment',
template_label={'app': 'pymada-master'},
container_port=8000,
container_name='pymada-master-container',
config_path=None,
auth_token=None,
max_task_duration=None,
max_task_retries=None):
env_vars = []
if auth_token is not None:
env_vars.append(client.V1EnvVar("PYMADA_TOKEN_AUTH", auth_token))
if max_task_duration is not None:
env_vars.append(
client.V1EnvVar("PYMADA_MAX_TASK_DURATION_SECONDS",
str(max_task_duration)))
if max_task_retries is not None:
env_vars.append(
client.V1EnvVar("PYMADA_MAX_TASK_RETRIES", str(max_task_retries)))
container_ports = [client.V1ContainerPort(container_port=container_port)]
container = client.V1Container(name=container_name,
image='pymada/master',
ports=container_ports,
env=env_vars)
pod_node_selector = {'pymada-role': 'master'}
pod_spec = client.V1PodSpec(containers=[container],
node_selector=pod_node_selector)
run_deployment(pod_spec,
1,
deploy_name,
template_label,
config_path=config_path)
base_dir = os.path.dirname(os.path.realpath(__file__))
if config_path is None:
cwd = os.getcwd()
config_path = os.path.join(cwd, 'k3s_config.yaml')
kube_config.load_kube_config(config_file=config_path)
kube_client = client.ApiClient()
if len(get_service_status('service=pymada-master-service').items) == 0:
utils.create_from_yaml(
kube_client,
os.path.join(base_dir, 'kube_yaml', 'pymada_master_service.yaml'))
if len(get_service_status('service=pymada-master-nodeport').items) == 0:
utils.create_from_yaml(
kube_client,
os.path.join(base_dir, 'kube_yaml', 'pymada_master_nodeport.yaml'))
def test_deploy(record_xml_attribute, deploy_name, namespace, model_dir,
export_dir):
util.set_pytest_junit(record_xml_attribute, "test_deploy")
util.maybe_activate_service_account()
app_dir = os.path.join(os.path.dirname(__file__), "../serving/GCS")
app_dir = os.path.abspath(app_dir)
logging.info("--app_dir not set defaulting to: %s", app_dir)
# TODO (@jinchihe) Using kustomize 2.0.3 to work around below issue:
# https://github.com/kubernetes-sigs/kustomize/issues/1295
kusUrl = 'https://github.com/kubernetes-sigs/kustomize/' \
'releases/download/v2.0.3/kustomize_2.0.3_linux_amd64'
util.run(['wget', '-q', '-O', '/usr/local/bin/kustomize', kusUrl],
cwd=app_dir)
util.run(['chmod', 'a+x', '/usr/local/bin/kustomize'], cwd=app_dir)
# TODO (@jinchihe): The kubectl need to be upgraded to 1.14.0 due to below issue.
# Invalid object doesn't have additional properties ...
kusUrl = 'https://storage.googleapis.com/kubernetes-release/' \
'release/v1.14.0/bin/linux/amd64/kubectl'
util.run(['wget', '-q', '-O', '/usr/local/bin/kubectl', kusUrl],
cwd=app_dir)
util.run(['chmod', 'a+x', '/usr/local/bin/kubectl'], cwd=app_dir)
# Configure custom parameters using kustomize
configmap = 'mnist-map-serving'
util.run(['kustomize', 'edit', 'set', 'namespace', namespace], cwd=app_dir)
util.run([
'kustomize', 'edit', 'add', 'configmap', configmap,
'--from-literal=name' + '=' + deploy_name
],
cwd=app_dir)
util.run([
'kustomize', 'edit', 'add', 'configmap', configmap,
'--from-literal=modelBasePath=' + model_dir
],
cwd=app_dir)
util.run([
'kustomize', 'edit', 'add', 'configmap', configmap,
'--from-literal=exportDir=' + export_dir
],
cwd=app_dir)
# Apply the components
util.run(['kustomize', 'build', app_dir, '-o', 'generated.yaml'],
cwd=app_dir)
util.run(['kubectl', 'apply', '-f', 'generated.yaml'], cwd=app_dir)
kube_config.load_kube_config()
api_client = k8s_client.ApiClient()
util.wait_for_deployment(api_client,
namespace,
deploy_name,
timeout_minutes=4)
def get_service_status(label_selector='', config_path=None):
if config_path is None:
base_dir = os.getcwd()
config_path = os.path.join(base_dir, 'k3s_config.yaml')
kube_config.load_kube_config(config_file=config_path)
k_client = client.CoreV1Api()
return k_client.list_namespaced_service('default',
label_selector=label_selector)
def test_jupyter(record_xml_attribute, env, namespace):
"""Test the jupyter notebook.
Args:
record_xml_attribute: Test fixture provided by pytest.
env: ksonnet environment.
namespace: namespace to run in.
"""
util.set_pytest_junit(record_xml_attribute, "jupyter_test")
app_credentials = os.getenv("GOOGLE_APPLICATION_CREDENTIALS")
if app_credentials:
logging.info("Activate service account")
util.run([
"gcloud", "auth", "activate-service-account",
"--key-file=" + app_credentials
])
# util.load_kube_config appears to hang on python3
kube_config.load_kube_config()
api_client = k8s_client.ApiClient()
host = api_client.configuration.host
logging.info("Kubernetes master: %s", host)
master = host.rsplit("/", 1)[-1]
this_dir = os.path.dirname(__file__)
app_dir = os.path.join(this_dir, "test_app")
ks_cmd = ks_util.get_ksonnet_cmd(app_dir)
name = "jupyter-test"
service = "jupyter-test"
component = "jupyter"
params = ""
ks_util.setup_ks_app(app_dir, env, namespace, component, params)
util.run([ks_cmd, "apply", env, "-c", component], cwd=app_dir)
conditions = ["Running"]
results = util.wait_for_cr_condition(api_client, GROUP, PLURAL, VERSION,
namespace, name, conditions)
logging.info("Result of CRD:\n%s", results)
# We proxy the request through the APIServer so that we can connect
# from outside the cluster.
url = ("https://{master}/api/v1/namespaces/{namespace}/services/{service}:80"
"/proxy/default/jupyter/lab?").format(
master=master, namespace=namespace, service=service)
logging.info("Request: %s", url)
r = send_request(url, verify=False)
if r.status_code != requests.codes.OK:
msg = "Request to {0} exited with status code: {1} and content: {2}".format(
url, r.status_code, r.content)
logging.error(msg)
raise RuntimeError(msg)
def __create_core_client(self, config_file, verify_ssl, context):
configuration = Configuration()
configuration.verify_ssl = verify_ssl
Configuration.set_default(configuration)
kube_config.load_kube_config(config_file=config_file)
if context:
return core_client.CoreV1Api(
api_client=config.new_client_from_config(context=context))
return core_client.CoreV1Api()
def delete_deployment(deployment_name, config_path=None):
if config_path is None:
base_dir = os.getcwd()
config_path = os.path.join(base_dir, 'k3s_config.yaml')
kube_config.load_kube_config(config_file=config_path)
appsv1_client = client.AppsV1Api()
try:
appsv1_client.delete_namespaced_deployment(
deployment_name, 'default', propagation_policy='Foreground')
except client.rest.ApiException:
# ignore 404s
pass
def get_deployment_status(label_selector=None, config_path=None):
if config_path is None:
base_dir = os.getcwd()
config_path = os.path.join(base_dir, 'k3s_config.yaml')
kube_config.load_kube_config(config_file=config_path)
appsv1_client = client.AppsV1Api()
api_response = appsv1_client.list_namespaced_deployment(
'default', label_selector=label_selector)
return api_response.to_dict()
def load_vim_config(self, vim_uuid):
"""
Returns a kubernetes APIclient object for the initial configuration of the wrapper
"""
vim_config = None
if not os.path.isfile("/tmp/{}".format(vim_uuid)):
with open("/tmp/{}".format(vim_uuid), 'w') as f:
vim_config = KubernetesWrapperEngine.get_vim_config(self, vim_uuid)
if vim_config is not None:
f.write(vim_config)
if vim_config is not None:
kube_config.load_kube_config(config_file="/tmp/{}".format(vim_uuid))
else:
kube_config.load_kube_config(config_file="/tmp/{}".format(vim_uuid))
def get_oc_api_client(token=None, server=None, verify_ssl=False):
config = OCConfiguration()
load_kube_config(config_file=os.environ.get('KUBECONFIG'),
client_configuration=config)
if token:
config.token = token
if server:
config.server = server
config.verify_ssl = verify_ssl
return OCApiClient(config)
def test_profiles():
app_credentials = os.getenv("GOOGLE_APPLICATION_CREDENTIALS")
if app_credentials:
logging.info("Activate service account")
util.run([
"gcloud", "auth", "activate-service-account",
"--key-file=" + app_credentials
])
# util.load_kube_config appears to hang on python3
kube_config.load_kube_config()
api_client = k8s_client.ApiClient()
this_dir = os.path.dirname(__file__)
util.run(["kubectl", "apply", "-f", "sample_profile.yaml"], cwd=this_dir)
# TODO: check CR status/condition instead of sleep
# conditions = ["Ready"]
# namespace = "kubeflow"
# name = "john"
# results = util.wait_for_cr_condition(api_client, GROUP, PLURAL, VERSION,
# namespace, name, conditions)
# logging.info("Result of CRD:\n%s", results)
time.sleep(10)
# Verifies the namespace is created.
name = "john" # The name of the profile, also the new namespace's name.
coreV1 = k8s_client.CoreV1Api(api_client)
retry_read_namespace = retry(
wait_exponential_multiplier=
1000, # wait 2^i * 1000 ms, on the i-th retry
wait_exponential_max=60000, # 60 sec max
)(coreV1.read_namespace)
resp = retry_read_namespace(name)
logging.info("found namespace: %s", resp)
rbacV1 = k8s_client.RbacAuthorizationV1Api(api_client)
resp = rbacV1.read_namespaced_role("edit", name)
logging.info("role: %s", resp)
resp = rbacV1.read_namespaced_role_binding("default", name)
logging.info("role binding: %s", resp)
# delete the profile and make sure namespace is deleted
util.run(["kubectl", "delete", "-f", "sample_profile.yaml"], cwd=this_dir)
time.sleep(15)
with pytest.raises(ApiException) as e:
resp = coreV1.read_namespace(name)
logging.info("exception info: %s", e)
def test_predict(master, namespace, service):
app_credentials = os.getenv("GOOGLE_APPLICATION_CREDENTIALS")
if app_credentials:
print("Activate service account")
util.run([
"gcloud", "auth", "activate-service-account",
"--key-file=" + app_credentials
])
if not master:
print("--master set; using kubeconfig")
# util.load_kube_config appears to hang on python3
kube_config.load_kube_config()
api_client = k8s_client.ApiClient()
host = api_client.configuration.host
print("host={0}".format(host))
master = host.rsplit("/", 1)[-1]
this_dir = os.path.dirname(__file__)
test_data = os.path.join(this_dir, "test_data", "instances.json")
with open(test_data) as hf:
instances = json.load(hf)
# We proxy the request through the APIServer so that we can connect
# from outside the cluster.
url = (
"https://{master}/api/v1/namespaces/{namespace}/services/{service}:8500"
"/proxy/v1/models/mnist:predict").format(master=master,
namespace=namespace,
service=service)
logging.info("Request: %s", url)
r = send_request(url, json=instances, verify=False)
if r.status_code != requests.codes.OK:
msg = "Request to {0} exited with status code: {1} and content: {2}".format(
url, r.status_code, r.content)
logging.error(msg)
raise RuntimeError(msg)
content = r.content
if six.PY3 and hasattr(content, "decode"):
content = content.decode()
result = json.loads(content)
assert len(result["predictions"]) == 1
predictions = result["predictions"][0]
assert "classes" in predictions
assert "predictions" in predictions
assert len(predictions["predictions"]) == 10
logging.info("URL %s returned; %s", url, content)
def __init__(self, config, manifest_path=None):
"""
A class to configure k8s after boot
Args:
config (str): File path for the kubernetes configuration file
manfiest_path (str): Path for kubernetes manifests to be applied
"""
self.config = config
if not manifest_path:
manifest_path = MANIFESTSPATH
self.manifest_path = manifest_path
kube_config.load_kube_config(config_file=config)
config = Configuration()
self.api = k8sclient.CoreV1Api()
self.client = api_client.ApiClient(configuration=config)
def __configure_by_file(self):
""" Return API client from configuration file """
if not self.params.get('kubeconfig'):
config_file = os.path.expanduser(
kube_config.KUBE_CONFIG_DEFAULT_LOCATION)
else:
config_file = self.params.get('kubeconfig')
try:
kube_config.load_kube_config(config_file=config_file,
context=self.params.get('context'))
config = kubernetes.client.Configuration()
if not self.params.get('verify_ssl'):
setattr(config, 'verify_ssl', False)
return kubernetes.client.ApiClient(configuration=config)
except (IOError, ConfigException):
raise
def __configure_by_file(self):
""" Return API client from configuration file """
if not self.params.get('kubeconfig'):
config_file = os.path.expanduser(
kube_config.KUBE_CONFIG_DEFAULT_LOCATION)
else:
config_file = self.params.get('kubeconfig')
try:
if not self.params.get('verify_ssl'):
sdk.configuration.verify_ssl = False
kube_config.load_kube_config(
config_file=config_file,
context=self.params.get('context'),
client_configuration=sdk.configuration)
return sdk.DefaultApi()
except (IOError, ConfigException):
raise
def create_spec_from_file(cls):
from kubernetes.config import kube_config
if not os.path.exists(
os.path.expanduser(kube_config.KUBE_CONFIG_DEFAULT_LOCATION)):
return {}
kube_config.load_kube_config()
k8s_cfg = k8s_config.Configuration()
return {
"host": k8s_cfg.host,
"certificate-authority": k8s_cfg.ssl_ca_cert,
"api_key": k8s_cfg.api_key,
"api_key_prefix": k8s_cfg.api_key_prefix,
"client-certificate": k8s_cfg.cert_file,
"client-key": k8s_cfg.key_file,
"tls_insecure": k8s_cfg.verify_ssl
}
def __configure_by_params(self):
""" Return API client from configuration file """
kube_config.load_kube_config()
config = kubernetes.client.Configuration()
auth_args = AUTH_ARG_SPEC.keys()
for key, value in iteritems(self.params):
if key in auth_args and value is not None:
if key == 'api_key':
setattr(config, key,
{'authorization': "Bearer {0}".format(value)})
else:
setattr(config, key, value)
if not self.params.get('verify_ssl'):
setattr(config, 'verify_ssl', False)
return kubernetes.client.ApiClient(configuration=config)
def load_initial_config(self):
"""
Returns a kubernetes APIclient object for the initial configuration of the wrapper
"""
vim_config = None
vim_list = None
while vim_list is not None:
vim_list = self.get_vim_list()
if vim_list is not None:
if len(vim_list) > 0:
vim_list = str(list(vim_list[0])[0])
with open("/tmp/{}".format(vim_list), 'w') as f:
vim_config = KubernetesWrapperEngine.get_vim_config(self, vim_list)
if vim_config is not None:
f.write(vim_config)
if vim_config is not None:
kube_config.load_kube_config(config_file="/tmp/{}".format(vim_list))
else:
LOG.info("K8S Cluster is not configured")
time.sleep(10)
def test_profiles(record_xml_attribute,
profileFile="profile_v1beta1_profile.yaml"):
util.set_pytest_junit(record_xml_attribute, "test_profile_e2e")
app_credentials = os.getenv("GOOGLE_APPLICATION_CREDENTIALS")
util.maybe_activate_service_account()
# util.load_kube_config appears to hang on python3
kube_config.load_kube_config()
api_client = k8s_client.ApiClient()
profileYamlFile = profileFile
#Profile Creation
group, version, name = createProfile(api_client, profileYamlFile)
verifyProfileCreation(api_client, group, version, name)
verifyNamespaceCreation(api_client, name)
verifyServiceAccounts(api_client, name)
verifyRolebindings(api_client, name)
#Profile deletion
deleteProfile(api_client, group, version, name)
verifyProfileDeletion(api_client, group, version, name)
def test_jupyter(master, namespace, service):
app_credentials = os.getenv("GOOGLE_APPLICATION_CREDENTIALS")
if app_credentials:
print("Activate service account")
util.run([
"gcloud", "auth", "activate-service-account",
"--key-file=" + app_credentials
])
if not master:
print("--master set; using kubeconfig")
# util.load_kube_config appears to hang on python3
kube_config.load_kube_config()
api_client = k8s_client.ApiClient()
host = api_client.configuration.host
print("host={0}".format(host))
master = host.rsplit("/", 1)[-1]
this_dir = os.path.dirname(__file__)
test_data = os.path.join(this_dir, "test_data", "instances.json")
def get_node_list(config_path=None):
if config_path is None:
base_dir = os.getcwd()
config_path = os.path.join(base_dir, 'k3s_config.yaml')
kube_config.load_kube_config(config_file=config_path)
k_client = client.CoreV1Api()
output = []
node_data = k_client.list_node()
for node in node_data.items:
node_images = []
if node.status.images is not None:
for image in node.status.images:
if len(image.names) == 2 and 'pymada' in image.names[1]:
node_images.append(image.names[1])
output.append({'name': node.metadata.name, 'images': node_images})
return output
def load_initial_config(self):
"""
Returns a kubernetes APIclient object for the initial configuration of the wrapper
"""
vim_config = None
vim_list = None
if os.getenv("KUBECONFIG") is True:
kube_config.load_kube_config()
else:
vim_list = self.get_vim_list()
if vim_list is not None:
if len(vim_list) > 0:
vim_list = str(list(vim_list[0])[0])
with open("/tmp/{}".format(vim_list), 'w') as f:
vim_config = self.get_vim_config(vim_list)
if vim_config is not None:
f.write(vim_config)
if vim_config is not None:
kube_config.load_kube_config(
config_file="/tmp/{}".format(vim_list))
else:
LOG.info("K8S Cluster is not configured")
def get_e2e_configuration():
config = Configuration()
config.host = None
if os.path.exists(
os.path.expanduser(kube_config.KUBE_CONFIG_DEFAULT_LOCATION)):
kube_config.load_kube_config(client_configuration=config)
else:
print('Unable to load config from %s' %
kube_config.KUBE_CONFIG_DEFAULT_LOCATION)
for url in ['https://%s:8443' % DEFAULT_E2E_HOST,
'http://%s:8080' % DEFAULT_E2E_HOST]:
try:
urllib3.PoolManager().request('GET', url)
config.host = url
config.verify_ssl = False
urllib3.disable_warnings()
break
except urllib3.exceptions.HTTPError:
pass
if config.host is None:
raise unittest.SkipTest('Unable to find a running Kubernetes instance')
print('Running test against : %s' % config.host)
config.assert_hostname = False
return config
