def test_model(clipper_conn, app, version):
time.sleep(25)
num_preds = 25
num_defaults = 0
addr = clipper_conn.get_query_addr()
for i in range(num_preds):
response = requests.post(
"http://%s/%s/predict" % (addr, app),
headers=headers,
data=json.dumps({
'input': get_test_point()
}))
result = response.json()
if response.status_code == requests.codes.ok and result["default"]:
num_defaults += 1
elif response.status_code != requests.codes.ok:
print(result)
raise BenchmarkException(response.text)
if num_defaults > 0:
print("Error: %d/%d predictions were default" % (num_defaults,
num_preds))
if num_defaults > num_preds / 2:
raise BenchmarkException("Error querying APP %s, MODEL %s:%d" %
(app, model_name, version))
def send_requests(clipper_conn):
success = False
num_tries = 0
while not success and num_tries < 5:
time.sleep(30)
num_preds = 25
num_success = 0
addr = clipper_conn.get_query_addr()
logger.info("ADDR: {}".format(addr))
for i in range(num_preds):
response = requests.post(
"http://%s/%s/predict" % (addr, APP_NAME),
headers=headers,
data=json.dumps({'input': list(np.random.random(30))}))
result = response.json()
if response.status_code == requests.codes.ok and not result[
"default"]:
num_success += 1
else:
logger.warning(result)
if num_success < num_preds:
logger.error(
"Error: %d/%d predictions were default or unsuccessful" %
(num_preds - num_success, num_preds))
if num_success > num_preds / 2.0:
success = True
num_tries += 1
if not success:
raise BenchmarkException("Error querying R model")
def deploy_model(clipper_conn, name, version, link=False):
app_name = "%s-app" % name
model_name = "%s-model" % name
clipper_conn.build_and_deploy_model(
model_name,
version,
"doubles",
fake_model_data,
"clipper/noop-container:{}".format(clipper_version),
num_replicas=1,
container_registry=
"568959175238.dkr.ecr.us-west-1.amazonaws.com/clipper")
time.sleep(10)
if link:
clipper_conn.link_model_to_app(app_name, model_name)
success = False
num_tries = 0
while not success and num_tries < 5:
time.sleep(30)
num_preds = 25
num_defaults = 0
addr = clipper_conn.get_query_addr()
for i in range(num_preds):
try:
response = requests.post(
"http://%s/%s/predict" % (addr, app_name),
headers=headers,
data=json.dumps({'input': list([1.2, 1.3])}))
result = response.json()
if response.status_code == requests.codes.ok and result[
"default"]:
num_defaults += 1
except requests.RequestException:
num_defaults += 1
if num_defaults > 0:
logger.error("Error: %d/%d predictions were default" %
(num_defaults, num_preds))
if num_defaults < num_preds / 2:
success = True
num_tries += 1
if not success:
raise BenchmarkException("Error querying APP %s, MODEL %s:%d" %
(app_name, model_name, version))
def create_and_test_app(clipper_conn, name):
app_name = "{}-app".format(name)
clipper_conn.register_application(app_name, "doubles", "default_pred",
100000)
time.sleep(1)
addr = clipper_conn.get_query_addr()
response = requests.post("http://%s/%s/predict" % (addr, app_name),
headers=headers,
data=json.dumps({'input': [0.1, 0.2]}))
response.json()
if response.status_code != requests.codes.ok:
logger.error("Error: %s" % response.text)
raise BenchmarkException("Error creating app %s" % app_name)
deploy_model(clipper_conn, name, link=True)
def deploy_model(clipper_conn, name, link=False):
app_name = "{}-app".format(name)
model_name = "{}-model".format(name)
clipper_conn.build_and_deploy_model(
model_name,
1,
"doubles",
fake_model_data,
"{}/noop-container:{}".format(clipper_registry, clipper_version),
num_replicas=2, # We set it to 2 for metric purpose.
container_registry=CLIPPER_CONTAINER_REGISTRY)
time.sleep(10)
if link:
clipper_conn.link_model_to_app(app_name, model_name)
success = False
num_tries = 0
while not success and num_tries < MAX_RETRY:
time.sleep(30)
num_preds = 25
num_defaults = 0
addr = clipper_conn.get_query_addr()
for i in range(num_preds):
response = requests.post("http://%s/%s/predict" % (addr, app_name),
headers=headers,
data=json.dumps(
{'input': list([1.2, 1.3])}))
result = response.json()
if response.status_code == requests.codes.ok and result["default"]:
num_defaults += 1
if num_defaults > 0:
logger.error("Error: %d/%d predictions were default" %
(num_defaults, num_preds))
if num_defaults < num_preds / 2:
success = True
num_tries += 1
if not success:
raise BenchmarkException("Error querying APP %s, MODEL %s:%d" %
(app_name, model_name, 1))
def create_and_test_app(clipper_conn, name, num_models):
app_name = "%s-app" % name
clipper_conn.register_application(app_name, "doubles", "default_pred",
100000)
time.sleep(1)
addr = clipper_conn.get_query_addr()
response = requests.post("http://%s/%s/predict" % (addr, app_name),
headers=headers,
data=json.dumps({'input': list([1.2, 1.3])}))
response.json()
if response.status_code != requests.codes.ok:
logger.error("Error: %s" % response.text)
raise BenchmarkException("Error creating app %s" % app_name)
for i in range(num_models):
if i == 0:
deploy_model(clipper_conn, name, i, link=True)
else:
deploy_model(clipper_conn, name, i)
time.sleep(1)
new_name=cluster_name)
try:
clipper_conn.register_application(app_name, "integers",
"default_pred", 100000)
time.sleep(1)
addr = clipper_conn.get_query_addr()
response = requests.post("http://%s/%s/predict" % (addr, app_name),
headers=headers,
data=json.dumps(
{'input': get_test_point()}))
result = response.json()
if response.status_code != requests.codes.ok:
print("Error: %s" % response.text)
raise BenchmarkException("Error creating app %s" % app_name)
version = 1
dtrain = xgb.DMatrix(get_test_point(), label=[0])
param = {
'max_depth': 2,
'eta': 1,
'silent': 1,
'objective': 'binary:logistic'
}
watchlist = [(dtrain, 'train')]
num_round = 2
bst = xgb.train(param, dtrain, num_round, watchlist)
def predict(xs):
return [str(bst.predict(xgb.DMatrix(xs)))]
num_frontend_replicas=2)
time.sleep(10)
print(clipper_conn.cm.get_query_addr())
try:
create_and_test_app(clipper_conn, "testapp0")
logger.info("Begin Kubernetes Multiple Frontend Test")
k8s_beta = clipper_conn.cm._k8s_beta
if (k8s_beta.read_namespaced_deployment(
'query-frontend-0', namespace='default').to_dict()
['status']['available_replicas'] != 1):
raise BenchmarkException(
"Wrong number of replicas of query-frontend-0."
"Expected {}, found {}".format(
1,
k8s_beta.read_namespaced_deployment(
'query-frontend-0', namespace='default').to_dict()
['status']['available_replicas']))
if (k8s_beta.read_namespaced_deployment(
'query-frontend-1', namespace='default').to_dict()
['status']['available_replicas'] != 1):
raise BenchmarkException(
"Wrong number of replicas of query-frontend-1."
"Expected {}, found {}".format(
1,
k8s_beta.read_namespaced_deployment(
'query-frontend-1', namespace='default').to_dict()
['status']['available_replicas']))
logger.info("Ok: we have 2 query frontend depolyments")
def run_test():
spark = SparkSession\
.builder\
.appName("clipper-pyspark")\
.getOrCreate()
training = spark.createDataFrame([(0, "a b c d e spark", 1.0),
(1, "b d", 0.0), (2, "spark f g h", 1.0),
(3, "hadoop mapreduce", 0.0)],
columns + ["label"])
# Configure an ML pipeline, which consists of three stages: tokenizer, hashingTF, and lr.
tokenizer = Tokenizer(inputCol="text", outputCol="words")
hashingTF = HashingTF(inputCol=tokenizer.getOutputCol(),
outputCol="features")
lr = LogisticRegression(maxIter=10, regParam=0.001)
pipeline = Pipeline(stages=[tokenizer, hashingTF, lr])
# Fit the pipeline to training documents.
model = pipeline.fit(training)
# Prepare test documents, which are unlabeled (id, text) tuples.
test = spark.createDataFrame([(4, "spark i j k"), (5, "l m n"),
(6, "spark hadoop spark"),
(7, "apache hadoop")], columns)
# Make predictions on test documents and print columns of interest.
prediction = model.transform(test)
selected = prediction.select("id", "text", "probability", "prediction")
for row in selected.collect():
rid, text, prob, prediction = row
print("(%d, %s) --> prob=%s, prediction=%f" %
(rid, text, str(prob), prediction))
# test predict function
print(
predict(spark, model,
[json.dumps((np.random.randint(1000), "spark abcd"))]))
try:
clipper_conn = create_docker_connection(cleanup=True,
start_clipper=True)
try:
clipper_conn.register_application(app_name, "strings",
"default_pred", 10000000)
time.sleep(1)
addr = clipper_conn.get_query_addr()
response = requests.post("http://%s/%s/predict" % (addr, app_name),
headers=headers,
data=json.dumps({
'input':
json.dumps((np.random.randint(1000),
"spark abcd"))
}))
result = response.json()
if response.status_code != requests.codes.ok:
print("Error: %s" % response.text)
raise BenchmarkException("Error creating app %s" % app_name)
version = 1
deploy_pyspark_model(clipper_conn, model_name, version, "strings",
predict, model, spark.sparkContext)
clipper_conn.link_model_to_app(app_name, model_name)
time.sleep(30)
num_preds = 25
num_defaults = 0
addr = clipper_conn.get_query_addr()
for i in range(num_preds):
response = requests.post(
"http://%s/%s/predict" % (addr, app_name),
headers=headers,
data=json.dumps({
'input':
json.dumps((np.random.randint(1000), "spark abcd"))
}))
result = response.json()
if response.status_code == requests.codes.ok and result[
"default"]:
num_defaults += 1
if num_defaults > 0:
print("Error: %d/%d predictions were default" %
(num_defaults, num_preds))
if num_defaults > num_preds / 2:
raise BenchmarkException("Error querying APP %s, MODEL %s:%d" %
(app_name, model_name, version))
version += 1
deploy_pyspark_model(clipper_conn, model_name, version, "strings",
predict, model, spark.sparkContext)
time.sleep(30)
num_preds = 25
num_defaults = 0
addr = clipper_conn.get_query_addr()
for i in range(num_preds):
response = requests.post(
"http://%s/%s/predict" % (addr, app_name),
headers=headers,
data=json.dumps({
'input':
json.dumps((np.random.randint(1000), "spark abcd"))
}))
result = response.json()
if response.status_code == requests.codes.ok and result[
"default"]:
num_defaults += 1
if num_defaults > 0:
print("Error: %d/%d predictions were default" %
(num_defaults, num_preds))
if num_defaults > num_preds / 2:
raise BenchmarkException("Error querying APP %s, MODEL %s:%d" %
(app_name, model_name, version))
except BenchmarkException as e:
log_docker(clipper_conn)
log_clipper_state(clipper_conn)
logger.exception("BenchmarkException")
clipper_conn = create_docker_connection(cleanup=True,
start_clipper=False)
sys.exit(1)
else:
spark.stop()
clipper_conn = create_docker_connection(cleanup=True,
start_clipper=False)
logger.info("ALL TESTS PASSED")
except Exception as e:
log_docker(clipper_conn)
logger.exception("Exception")
clipper_conn = create_docker_connection(cleanup=True,
start_clipper=False)
sys.exit(1)