def test_kubernetes(clipper_conn, num_apps, num_models):
time.sleep(10)
print(clipper_conn.cm.get_query_addr())
print(clipper_conn.inspect_instance())
try:
logger.info("Running integration test with %d apps and %d models" %
(num_apps, num_models))
for a in range(num_apps):
create_and_test_app(clipper_conn, "testapp%s" % a, num_models)
if not os.path.exists(CLIPPER_TEMP_DIR):
os.makedirs(CLIPPER_TEMP_DIR)
tmp_log_dir = tempfile.mkdtemp(dir=CLIPPER_TEMP_DIR)
logger.info(clipper_conn.get_clipper_logs(tmp_log_dir))
# Remove temp files
shutil.rmtree(tmp_log_dir)
log_clipper_state(clipper_conn)
logger.info("SUCCESS")
except BenchmarkException:
log_clipper_state(clipper_conn)
logger.exception("BenchmarkException")
create_kubernetes_connection(cleanup=True,
start_clipper=False,
connect=False)
sys.exit(1)
except ClipperException:
log_clipper_state(clipper_conn)
logger.exception("ClipperException")
create_kubernetes_connection(cleanup=True,
start_clipper=False,
connect=False)
sys.exit(1)
def test():
conn_1 = create_kubernetes_connection(
cleanup=False, start_clipper=True, namespace='ns-1')
conn_2 = create_kubernetes_connection(
cleanup=False, start_clipper=True, namespace='ns-2')
deploy_(conn_1)
deploy_(conn_2)
time.sleep(10)
res_1 = predict_(conn_1.get_query_addr(), [.1, .2, .3])
res_2 = predict_(conn_2.get_query_addr(), [.1, .2, .3])
assert not res_1['default']
assert not res_2['default']
conn_1.stop_all()
conn_2.stop_all()
def test_service_types_of_kubernetes_container_manager(self):
cluster_name = "k8s-cluster-{}".format(random.randint(0, 5000))
logger.info("Test that service_types is 'dict' type or not")
service_types = ['redis', 'management', 'query', 'query-rpc', 'metric']
with self.assertRaises(cl.ClipperException) as c:
create_kubernetes_connection(cleanup=True,
new_name=cluster_name,
service_types=service_types)
self.assertTrue(
"service_types must be 'dict' type" in str(c.exception))
logger.info("Test that service_types has unknown keys or not")
service_types = {
'redis': 'NodePort',
'UNKNOWN_KEY': 'NodePort', # for test
'query': 'NodePort',
'query-rpc': 'NodePort',
'metric': 'NodePort'
}
with self.assertRaises(cl.ClipperException) as c:
create_kubernetes_connection(cleanup=True,
new_name=cluster_name,
service_types=service_types)
self.assertTrue("service_types has unknown keys" in str(c.exception))
logger.info("Test that service_types has unknown values or not")
service_types = {
'redis': 'NodePort',
'management': 'NodePort',
'query': 'NodePort',
'query-rpc': 'UNKNOWN_VALUE', # for test
'metric': 'NodePort'
}
with self.assertRaises(cl.ClipperException) as c:
create_kubernetes_connection(cleanup=True,
new_name=cluster_name,
service_types=service_types)
self.assertTrue("service_types has unknown values" in str(c.exception))
logger.info("Test that service_types has 'ExternalName' value or not")
service_types = {
'redis': 'NodePort',
'management': 'NodePort',
'query': 'NodePort',
'query-rpc': 'ExternalName', # for test
'metric': 'NodePort'
}
with self.assertRaises(cl.ClipperException) as c:
create_kubernetes_connection(cleanup=True,
new_name=cluster_name,
service_types=service_types)
self.assertTrue("Clipper does not support" in str(c.exception))
def create(name, use_kubernetes=False):
if use_kubernetes:
conn = create_kubernetes_connection(cleanup=False,
start_clipper=True,
new_name=name)
else:
conn = create_docker_connection(cleanup=False,
start_clipper=True,
new_name=name)
return conn
if len(sys.argv) > 1:
num_apps = int(sys.argv[1])
if len(sys.argv) > 2:
num_models = int(sys.argv[2])
except IndexError:
# it's okay to pass here, just use the default values
# for num_apps and num_models
pass
try:
# Test without proxy first
import random
cluster_name = "k8-{}".format(random.randint(0, 5000))
clipper_conn = create_kubernetes_connection(cleanup=False,
start_clipper=True,
with_proxy=False,
new_name=cluster_name)
test_kubernetes(clipper_conn, num_apps, num_models)
clipper_conn.stop_all()
try:
import subprocess32 as subprocess
except:
import subprocess
import shlex
proc = subprocess.Popen(shlex.split('kubectl proxy -p 8080'))
# Test with proxy. Assumes proxy is running at 127.0.0.1:8080
proxy_name = "k8s-proxy-test-cluster-{}".format(random.randint(
0, 5000))
clipper_conn = create_kubernetes_connection(cleanup=True,
if __name__ == "__main__":
num_apps = 3
num_models = 3
try:
if len(sys.argv) > 1:
num_apps = int(sys.argv[1])
if len(sys.argv) > 2:
num_models = int(sys.argv[2])
except IndexError:
# it's okay to pass here, just use the default values
# for num_apps and num_models
pass
try:
# Test without proxy first
clipper_conn = create_kubernetes_connection(cleanup=True,
start_clipper=True,
with_proxy=False)
test_kubernetes(clipper_conn, num_apps, num_models)
clipper_conn.stop_all()
# Test with proxy. Assumes proxy is running at 127.0.0.1:8080
clipper_conn = create_kubernetes_connection(cleanup=True,
start_clipper=True,
with_proxy=True)
test_kubernetes(clipper_conn, 1, 1)
clipper_conn.stop_all()
except Exception as e:
logger.exception("Exception: {}".format(e))
sys.exit(1)
addr = clipper_conn.get_query_addr()
response = requests.post("http://%s/%s/predict" % (addr, app_name),
headers=headers,
data=json.dumps({'input': [0.1, 0.2]}))
response.json()
if response.status_code != requests.codes.ok:
logger.error("Error: %s" % response.text)
raise BenchmarkException("Error creating app %s" % app_name)
deploy_model(clipper_conn, name, link=True)
if __name__ == "__main__":
try:
clipper_conn = create_kubernetes_connection(cleanup=True,
start_clipper=True,
num_frontend_replicas=2)
time.sleep(10)
print(clipper_conn.cm.get_query_addr())
try:
create_and_test_app(clipper_conn, "testapp0")
logger.info("Begin Kubernetes Multiple Frontend Test")
k8s_beta = clipper_conn.cm._k8s_beta
if (k8s_beta.read_namespaced_deployment(
'query-frontend-0', namespace='default').to_dict()
['status']['available_replicas'] != 1):
raise BenchmarkException(
"Wrong number of replicas of query-frontend-0."
"Expected {}, found {}".format(
query = metric_addr + '/api/v1/targets'
logger.info("Querying: {}".format(query))
res = requests.get(query).json()
logger.info(res)
assert res['status'] == 'success'
active_targets = res['data']['activeTargets']
assert len(active_targets) == 3, 'Wrong number of targets'
for target in active_targets:
assert target['health'] == 'up', "Target {} is not up!".format(target)
if __name__ == "__main__":
try:
clipper_conn = create_kubernetes_connection(cleanup=True,
start_clipper=True)
time.sleep(60)
logger.info(clipper_conn.cm.get_query_addr())
try:
create_and_test_app(clipper_conn, "kube-metric")
# Start Metric Check
metric_api_addr = clipper_conn.cm.get_metric_addr()
# Account for InvalidSchema: No connection adapters were found
if not metric_api_addr.startswith('http://'):
metric_api_addr = 'http://' + metric_api_addr
logger.info("Test 1: Checking status of 3 node exporter")
# Sleep & retry is need to for kubelet to setup networking
logger.info(res)
assert res['status'] == 'success'
active_targets = res['data']['activeTargets']
assert len(active_targets) == 3, 'Wrong number of targets'
for target in active_targets:
assert target['health'] == 'up', "Target {} is not up!".format(target)
if __name__ == "__main__":
import random
cluster_name = 'metric-k8s-{}'.format(random.randint(0, 5000))
try:
clipper_conn = create_kubernetes_connection(new_name=cluster_name,
cleanup=False,
start_clipper=True)
time.sleep(60)
logger.info(clipper_conn.cm.get_query_addr())
try:
create_and_test_app(clipper_conn, "kube-metric")
# Start Metric Check
metric_api_addr = clipper_conn.cm.get_metric_addr()
# Account for InvalidSchema: No connection adapters were found
if not metric_api_addr.startswith('http://'):
metric_api_addr = 'http://' + metric_api_addr
logger.info("Test 1: Checking status of 3 node exporter")
data=json.dumps({'input': [0.1, 0.2]}))
response.json()
if response.status_code != requests.codes.ok:
logger.error("Error: %s" % response.text)
raise BenchmarkException("Error creating app %s" % app_name)
deploy_model(clipper_conn, name, link=True)
if __name__ == "__main__":
import random
cluster_name = "k8-frontx-{}".format(random.randint(0, 5000))
try:
clipper_conn = create_kubernetes_connection(cleanup=False,
start_clipper=True,
num_frontend_replicas=2,
new_name=cluster_name)
time.sleep(10)
print(clipper_conn.cm.get_query_addr())
try:
create_and_test_app(clipper_conn, "testapp0")
logger.info("Begin Kubernetes Multiple Frontend Test")
k8s_beta = clipper_conn.cm._k8s_beta
if (k8s_beta.read_namespaced_deployment(
'query-frontend-0-at-{}'.format(cluster_name),
namespace='default').to_dict()['status']
['available_replicas'] != 1):
raise BenchmarkException(
"Wrong number of replicas of query-frontend-0."