def create_docker_connection(cleanup=True, start_clipper=True):
logger.info("Creating DockerContainerManager")
cm = DockerContainerManager(clipper_query_port=find_unbound_port(),
clipper_management_port=find_unbound_port(),
clipper_rpc_port=find_unbound_port(),
redis_port=find_unbound_port())
cl = ClipperConnection(cm)
if cleanup:
cl.stop_all()
docker_client = get_docker_client()
docker_client.containers.prune(filters={"label": CLIPPER_DOCKER_LABEL})
if start_clipper:
# Try to start Clipper in a retry loop here to address flaky tests
# as described in https://github.com/ucbrise/clipper/issues/352
while True:
try:
logger.info("Starting Clipper")
cl.start_clipper()
time.sleep(1)
break
except docker.errors.APIError as e:
logger.info(
"Problem starting Clipper: {}\nTrying again.".format(e))
cl.stop_all()
cm = DockerContainerManager(
clipper_query_port=find_unbound_port(),
clipper_management_port=find_unbound_port(),
clipper_rpc_port=find_unbound_port(),
redis_port=find_unbound_port())
cl = ClipperConnection(cm)
else:
cl.connect()
return cl
def create_kubernetes_connection(cleanup=False,
start_clipper=False,
connect=False,
with_proxy=False,
num_frontend_replicas=1,
cleanup_name='default-cluster',
new_name='default-cluster',
connect_name='default-cluster',
service_types=None,
namespace='default'):
logger.info("Creating KubernetesContainerManager")
cl = None
assert cleanup or start_clipper or connect, "You must set at least one of {cleanup, start_clipper, connect} to be true."
if with_proxy:
kubernetes_proxy_addr = "127.0.0.1:8080"
else:
kubernetes_proxy_addr = None
if cleanup:
logger.info("Cleaning up Kubernetes Cluster {}".format(cleanup_name))
cm = KubernetesContainerManager(
cluster_name=cleanup_name,
useInternalIP=USE_MINIKUBE,
service_types=service_types,
kubernetes_proxy_addr=kubernetes_proxy_addr)
cl = ClipperConnection(cm)
cl.stop_all()
logger.info("Done cleaning up clipper")
if start_clipper:
logger.info("Starting up Kubernetes Cluster {}".format(new_name))
cm = KubernetesContainerManager(
cluster_name=new_name,
kubernetes_proxy_addr=kubernetes_proxy_addr,
namespace=namespace,
useInternalIP=USE_MINIKUBE,
service_types=service_types,
create_namespace_if_not_exists=True)
cl = ClipperConnection(cm)
cl.start_clipper(num_frontend_replicas=num_frontend_replicas)
if connect:
try:
cm = KubernetesContainerManager(
cluster_name=connect_name,
useInternalIP=USE_MINIKUBE,
service_types=service_types,
kubernetes_proxy_addr=kubernetes_proxy_addr)
cl = ClipperConnection(cm)
cl.connect()
except Exception:
pass
return cl
def create_docker_connection(cleanup=False,
start_clipper=False,
cleanup_name='default-cluster',
new_name='default-cluster',
use_centralized_log=False):
logger.info("Creating DockerContainerManager")
cl = None
assert cleanup or start_clipper, "You must set at least one of {cleanup, start_clipper} to be true."
if cleanup:
logger.info("Cleaning up Docker cluster {}".format(cleanup_name))
cm = DockerContainerManager(
cluster_name=cleanup_name,
clipper_query_port=find_unbound_port(),
clipper_management_port=find_unbound_port(),
clipper_rpc_port=find_unbound_port(),
fluentd_port=find_unbound_port(),
redis_port=find_unbound_port(),
prometheus_port=find_unbound_port(),
)
cl = ClipperConnection(cm)
cl.stop_all(graceful=False)
if start_clipper:
# Try to start Clipper in a retry loop here to address flaky tests
# as described in https://github.com/ucbrise/clipper/issues/352
logger.info("Starting up Docker cluster {}".format(new_name))
while True:
cm = DockerContainerManager(
cluster_name=new_name,
clipper_query_port=find_unbound_port(),
clipper_management_port=find_unbound_port(),
clipper_rpc_port=find_unbound_port(),
fluentd_port=find_unbound_port(),
redis_port=find_unbound_port(),
prometheus_port=find_unbound_port(),
use_centralized_log=use_centralized_log)
cl = ClipperConnection(cm)
try:
logger.info("Starting Clipper")
cl.start_clipper()
time.sleep(1)
break
except docker.errors.APIError as e:
logger.info(
"Problem starting Clipper: {}\nTrying again.".format(e))
cl.stop_all()
return cl
def setup_clipper():
app_name = 'inceptionv3-app'
model_name = 'inceptionv3-model'
clipper_conn = ClipperConnection(DockerContainerManager())
clipper_conn.connect()
pytorch_deployer.deploy_pytorch_model(
clipper_conn=clipper_conn,
name=model_name,
version='1',
input_type='bytes',
func=incept_predict,
pytorch_model=incept,
num_replicas=10,
batch_size=1,
pkgs_to_install=['pillow', 'pyarrow', 'torch', 'torchvision'])
clipper_conn.register_application(name=app_name,
input_type="bytes",
default_output="-1.0",
slo_micros=10000000) # 10s
clipper_conn.link_model_to_app(app_name=app_name, model_name=model_name)
print(
"url: ", "http://{addr}/{app_name}/predict".format(addr="",
app_name=app_name))
def signal_handler(signal, frame):
print("Stopping Clipper...")
clipper_conn = ClipperConnection(
KubernetesContainerManager(useInternalIP=True))
# clipper_conn = ClipperConnection(DockerContainerManager())
clipper_conn.stop_all()
sys.exit(0)
def main():
# Setup container manager.
# k8 = KubernetesContainerManager(kubernetes_proxy_addr="127.0.0.1:8080",
# useInternalIP=True)
# clipper_conn = ClipperConnection(k8)
swarm = DockerContainerManager()
clipper_conn = ClipperConnection(swarm)
clipper_conn.stop_all()
clipper_conn.start_clipper()
# Register application.
clipper_conn.register_application(name="sum-app",
input_type="doubles",
default_output="-1.0",
slo_micros=10000000)
# Model deployement.
python_deployer.deploy_python_closure(clipper_conn,
name="sum-model",
version=1,
input_type="doubles",
func=sum)
# Link application to model.
clipper_conn.link_model_to_app(app_name="sum-app",
model_name="sum-model")
# Test
headers = {"Content-type": "application/json"}
response = requests.post("http://localhost:1337/sum-app/predict",
headers=headers,
data=json.dumps({"input": list(np.random.random(10))})).json()
print(response)
def create_kubernetes_connection(cleanup=True,
start_clipper=True,
connect=True,
with_proxy=False,
num_frontend_replicas=1):
logger.info("Creating KubernetesContainerManager")
if with_proxy:
cm = KubernetesContainerManager(kubernetes_proxy_addr="127.0.0.1:8080")
else:
cm = KubernetesContainerManager()
cl = ClipperConnection(cm)
if cleanup:
cl.stop_all()
# Give kubernetes some time to clean up
time.sleep(20)
logger.info("Done cleaning up clipper")
if start_clipper:
logger.info("Starting Clipper")
cl.start_clipper(
query_frontend_image=
"568959175238.dkr.ecr.us-west-1.amazonaws.com/clipper/query_frontend:{}".
format(clipper_version),
mgmt_frontend_image=
"568959175238.dkr.ecr.us-west-1.amazonaws.com/clipper/management_frontend:{}".
format(clipper_version),
num_frontend_replicas=num_frontend_replicas)
time.sleep(1)
if connect:
try:
cl.connect()
except Exception:
pass
except ClipperException:
pass
return cl
def main(version, label):
from clipper_admin import ClipperConnection, KubernetesContainerManager
clipper_conn = ClipperConnection(
KubernetesContainerManager(useInternalIP=True))
clipper_conn.connect()
from clipper_admin.deployers import python as python_deployer
registry = 'localhost:5000'
python_deployer.deploy_python_closure(clipper_conn,
name="sum-model",
version=version,
input_type="doubles",
func=feature_sum,
labels=[label],
registry=registry)
def __init__(self, sleep_time):
from clipper_admin import ClipperConnection, DockerContainerManager
#from clipper_admin.deployers import python as python_deployer
from clipper_admin.deployers import pytorch as pt_deployer
self.clipper_conn = ClipperConnection(DockerContainerManager())
try:
self.clipper_conn.connect()
self.clipper_conn.stop_all()
except Exception:
pass
self.clipper_conn.start_clipper()
self.clipper_conn.register_application(name="hello-world",
input_type="strings",
default_output="-1.0",
slo_micros=10**8)
model = Model()
def policy(ptmodel, x):
batch = (len(x))
arr = []
for j in x:
xs, masks = j.split("###")
res = np.frombuffer(base64.decodestring(xs), dtype=np.float32)
res = res.reshape((64, 19, 19, 3))
res = np.frombuffer(base64.decodestring(masks),
dtype=np.float32)
res = res.reshape((64, 362))
for i in x:
time.sleep(sleep_time)
return [
np.random.rand(64).astype(np.float32) for i in range(batch)
]
pt_deployer.deploy_pytorch_model(self.clipper_conn,
name="policy",
version=1,
input_type="strings",
func=policy,
pytorch_model=model)
self.clipper_conn.link_model_to_app(app_name="hello-world",
model_name="policy")
def __init__(self, shape, model_name):
from clipper_admin import ClipperConnection, DockerContainerManager
from clipper_admin.deployers import python as python_deployer
from clipper_admin.deployers import pytorch as pytorch_deployer
self.clipper_conn = ClipperConnection(DockerContainerManager())
try:
self.clipper_conn.connect()
self.clipper_conn.stop_all()
except Exception:
pass
self.clipper_conn.start_clipper()
self.clipper_conn.register_application(name="hello-world",
input_type="strings",
default_output="-1.0",
slo_micros=10**8)
ptmodel = get_model(model_name)
def policy(model, x):
print(len(x))
batch = (len(x))
arr = []
for j in x:
print(type(j), len(j))
res = np.frombuffer(base64.decodestring(j), dtype=np.float32)
print(res.shape)
arr += [res]
x = np.array(arr)
x = x.reshape((-1, ) + shape[1:])
print("new shape", x.shape)
return evaluate_model(model, x).reshape((batch, shape[0]))
pytorch_deployer.deploy_pytorch_model(self.clipper_conn,
name="policy",
version=1,
input_type="strings",
func=policy,
pytorch_model=ptmodel)
self.clipper_conn.link_model_to_app(app_name="hello-world",
model_name="policy")
def deployModelToClipper():
"""Deploy model to clipper and replace its entry."""
global app_name, model_name, model_version
print('Deploying model to clipper, model_name={}, model_version={}'.format(
model_name, model_version))
# Setup clipper and deploy model
clipper_conn = ClipperConnection(DockerContainerManager(redis_port=6380))
try:
clipper_conn.start_clipper()
except:
clipper_conn.connect()
try:
# input_type must be bytes as inputs will be serialized into bytes with pickle
clipper_conn.register_application(name=app_name,
input_type="bytes",
default_output="-1.0",
slo_micros=1000000)
except Exception as e:
print(e)
try:
deploy_python_closure(clipper_conn,
name=model_name,
version=model_version,
input_type="bytes",
batch_size=1,
func=predict,
base_image='hysia-clipper-base-container-gpu')
except Exception as e:
print(e)
try:
clipper_conn.link_model_to_app(app_name=app_name,
model_name=model_name)
except Exception as e:
print(e)
replaceDefaultEntry()
print('{} deployed to clipper!'.format(model_name))
def setup_clipper():
app_name = 'predict-app'
model_name = "predict-model"
clipper_conn = ClipperConnection(DockerContainerManager())
clipper_conn.connect()
deploy_python_closure(clipper_conn,
name="predict-model",
version='1',
input_type="bytes",
func=join_predict)
clipper_conn.register_application(name=app_name,
input_type="bytes",
default_output="-1.0",
slo_micros=10000000) # 10s
clipper_conn.link_model_to_app(app_name=app_name, model_name=model_name)
print(
"url: ", "http://{addr}/{app_name}/predict".format(addr="",
app_name=app_name))
def setup_clipper():
app_name = 'resnet101-app'
model_name = 'resnet101-model'
clipper_conn = ClipperConnection(DockerContainerManager())
clipper_conn.connect()
pytorch_deployer.deploy_pytorch_model(clipper_conn=clipper_conn,
name=model_name,
version='1',
input_type='bytes',
func=resnet_predict,
pytorch_model=resnet101,
pkgs_to_install=['pillow', 'torch', 'torchvision'])
clipper_conn.register_application(name=app_name,
input_type="bytes",
default_output="-1.0",
slo_micros=10000000) # 10s
clipper_conn.link_model_to_app(app_name=app_name, model_name=model_name)
print("query_adress: ", clipper_conn.get_query_addr())
print("app_name: ", )
print("model_name: ", )
print("url: ", "http://{addr}/{app_name}/predict".format(addr=clipper_conn.get_query_addr(),app_name=app_name))
def register(model_name, sess, func):
"""
Register a tf session with its function
Input:
- model_name: name of the model, string
- sess: TF session
- func: the function that runs the TF session
Return:
- clipper connection
"""
clipper_conn = ClipperConnection(DockerContainerManager())
clipper_conn.connect()
deploy_tensorflow_model(
clipper_conn=clipper_conn,
name=model_name,
version='1.0',
input_type='strings',
func=func,
tf_sess_or_saved_model_path=sess,
)
print(model_name, "registered")
return clipper_conn
# turn into predictions
with open('submission.csv') as fh:
lines = fh.readlines()[1:] # ignore first line
preds = [line.strip().split(',')[1] for line in lines]
return preds
# pickle function and write to appropriate location
s = six.StringIO()
c = CloudPickler(s, 2)
c.dump(libffm)
serialized_prediction_function = s.getvalue()
filepath = 'docker/lib/func.pkl'
with open(filepath, 'w') as fh:
fh.write(serialized_prediction_function)
# refresh creds
os.system('gcloud container clusters get-credentials redis-cluster')
os.system('kubectl cluster-info')
clipper_conn = ClipperConnection(KubernetesContainerManager(clipper_ip, useInternalIP=True))
clipper_conn.connect()
# Build model and deploy to clipper
version = int(time.time())
clipper_conn.build_and_deploy_model('ffm', version, 'strings', 'docker/lib', 'clipper/python-closure-container:develop', container_registry='ryanhoque')
# Uncomment the following if first time
#clipper_conn.link_model_to_app(app_name="testbed", model_name="ffm")
# finally deploy new version of model to clipper (set version as timestamp)
print('Successfully deployed model ffm version ' + str(version) + ' to Clipper.')
version='1.0',
input_type='strings',
func=func,
tf_sess_or_saved_model_path=sess,
)
print(model_name, "registered")
return clipper_conn
# Stop Clipper on Ctrl-C
def signal_handler(signal, frame):
print("Stopping Clipper...")
clipper_conn = ClipperConnection(DockerContainerManager())
clipper_conn.stop_all()
sys.exit(0)
if __name__ == '__main__':
signal.signal(signal.SIGINT, signal_handler)
clipper_conn = ClipperConnection(
DockerContainerManager(use_centralized_log=False))
clipper_conn.stop_all()
clipper_conn.start_clipper()
print('Clipper Started')
try:
while True:
time.sleep(2)
except Exception as e:
clipper_conn.stop_all()
print("done")
help=
"The Docker container registry to which to push the freshly built model image"
)
args = parser.parse_args()
arg_errs = []
if not args.model_name:
arg_errs.append(
"The name of the model being deployed must be specified!")
if not args.model_version:
arg_errs.append(
"The version of the model being deployed must be specified!")
if not args.model_data_path:
arg_errs.append(
"The path to the serialized R model data must be specified!")
if len(arg_errs) > 0:
for err in arg_errs:
print(err)
raise ClipperException()
# Note: This container manager is only necessary for
# creating a connection object that can be used to build the model
cm = DockerContainerManager()
conn = ClipperConnection(cm)
conn.build_model(args.model_name, args.model_version, args.model_data_path,
CLIPPER_R_CONTAINER_BASE_IMAGE, args.registry)
def signal_handler(signal, frame):
print("Stopping Clipper...")
clipper_conn = ClipperConnection(DockerContainerManager())
clipper_conn.stop_all()
sys.exit(0)
accuracy_op = graph.get_tensor_by_name('Mean_1:0')
oX = graph.get_tensor_by_name('Placeholder:0')
oY = graph.get_tensor_by_name('Placeholder_1:0')
def predict(X):
print("inputs {}".format(X))
result = sess.run(load_infer_op, feed_dict={oX: X})
ret = [str(i) for i in result]
print("return is {}".format(ret))
return ret
manager = KubernetesContainerManager(kubernetes_proxy_addr=K8S_ADDR,
namespace=K8S_NS)
clipper_conn = ClipperConnection(manager)
clipper_conn.connect()
# clipper_conn.delete_application(APP_NAME)
# clipper_conn.register_application(
# name = APP_NAME, input_type = 'doubles', default_output = '0', slo_micros = 100000000)
deploy_tensorflow_model(clipper_conn,
name=PREDICT_NAME,
version=VERSION,
input_type="doubles",
func=predict,
tf_sess_or_saved_model_path=sess,
registry=REGISTRY,
pkgs_to_install=['tensorflow'])
model = nn.Linear(3, 1)
def predict(model, inputs):
inputs = shift(inputs)
inputs = torch.tensor(inputs).float()
pred = model(inputs)
pred = pred.data.numpy()
return [str(x) for x in pred]
APP_NAME = "test-app"
MODEL_NAME = "test-pytorch-model"
# Setup clipper and deploy pytorch model
clipper_conn = ClipperConnection(DockerContainerManager(redis_port=6380))
try:
clipper_conn.start_clipper()
clipper_conn.register_application(name=APP_NAME,
input_type="doubles",
default_output="-1.0",
slo_micros=1000000)
deploy_pytorch_model(clipper_conn,
name=MODEL_NAME,
version="1",
input_type="doubles",
func=predict,
pytorch_model=model,
pkgs_to_install=pip_deps)
clipper_conn.link_model_to_app(app_name=APP_NAME, model_name=MODEL_NAME)
except:
def get_new_connection_instance(cluster_name, use_centralized_log):
return ClipperConnection(
DockerContainerManager(cluster_name=cluster_name,
use_centralized_log=use_centralized_log))
from clipper_admin import ClipperConnection, DockerContainerManager
clipper_conn = ClipperConnection(DockerContainerManager(extra_container_kwargs={'cpuset_cpus':'0-35'}))
clipper_conn.start_clipper(cache_size=0)
from clipper_admin import ClipperConnection, KubernetesContainerManager
from clipper_admin.deployers import python as python_deployer
import requests, json, numpy as np
import time
clipper_conn = ClipperConnection(
KubernetesContainerManager("https://35.197.66.133", useInternalIP=True))
clipper_conn.connect()
addr = clipper_conn.get_query_addr()
headers = {"Content-type": "application/json"}
# sample from 5 arbitrary valid feature vectors
possible_data = [
"27,17,45,28,2,28,27,29,28,1,1,,23,68fd1e64,960c983b,9fbfbfd5,38c11726,25c83c98,7e0ccccf,fe06fd10,062b5529,a73ee510,ca53fc84,67360210,895d8bbb,4f8e2224,f862f261,b4cc2435,4c0041e5,e5ba7672,b4abdd09,21ddcdc9,5840adea,36a7ab86,,32c7478e,85e4d73f,010f6491,ee63dd9b",
"1,1,19,7,1,3,1,7,7,1,1,,2,09ca0b81,8947f767,a87e61f7,c4ba2a67,25c83c98,7e0ccccf,ce6020cc,062b5529,a73ee510,b04d3cfe,70dcd184,899eb56b,aca22cf9,b28479f6,a473257f,88f592e4,d4bb7bd8,bd17c3da,1d04f4a4,a458ea53,82bdc0bb,,32c7478e,5bdcd9c4,010f6491,cca57dcc",
"8,11,38,9,316,25,8,11,10,1,1,,9,05db9164,09e68b86,aa8c1539,85dd697c,25c83c98,7e0ccccf,bc252bd0,5b392875,a73ee510,ef5c0d3c,0bd0c3b3,d8c29807,c0e6befc,8ceecbc8,d2f03b75,c64d548f,e5ba7672,63cdbb21,cf99e5de,5840adea,5f957280,,55dd3565,1793a828,e8b83407,b7d9c3bc",
",4,13,20,17700,,0,20,1,,0,,20,68fd1e64,08d6d899,9143c832,f56b7dd5,0942e0a7,7e0ccccf,e88f1cec,0b153874,a73ee510,3b08e48b,8f410860,ae1bb660,b8eec0b1,b28479f6,bffbd637,bad5ee18,776ce399,bbf70d82,,,0429f84b,,be7c41b4,c0d61a5c,,",
"16,18,5203,8,0,0,4,49,10,0,1,,0,05db9164,9f7e1d07,0253bbf5,d6420627,4cf72387,,0db090eb,0b153874,a73ee510,3b08e48b,10e6a64f,31adfaee,38b5339a,07d13a8f,3e25e5f5,1621c7f4,e5ba7672,6a58e423,21ddcdc9,5840adea,bcc7a461,,32c7478e,3214afd4,ea9a246c,e7ecb821"
]
fh = open('predictions.log', 'a')
while (True):
# poisson process with rate 0.1
interarrival_time = np.random.exponential(scale=10)
time.sleep(interarrival_time)
num_requests = int(np.random.random() * 10) + 1 # uniform between 1 and 10
input = list()
for _ in range(num_requests):
input.append(possible_data[int(np.random.random() * 5)])
def frontend(args):
print(args)
batch_size = args.batch_size
num_models = args.num_models
func = predict
red_func = predict
assert args.redundancy_mode in ["none", "equal"]
redundancy_mode = 0
if args.queue_mode == "single_queue":
queue_mode = 0
elif args.queue_mode == "rr":
queue_mode = 1
else:
assert False, "Unrecognized queue mode '{}'".format(args.queue_mode)
model_instance_ip_port = []
red_model_instance_ip_port = []
cur_port = base_port
if num_models < len(args.f_ips):
# Round up to highest int so as not to launch more models than needed.
num_between = int(len(args.f_ips) / num_models + 0.5)
chosen_indices = list(range(0, len(args.f_ips), num_between))
print("Range is", chosen_indices)
# Shift our chosen indices so that they are evenly distributed
# throughout the clients.
delta = len(args.f_ips) - chosen_indices[-1]
shift = delta // 2
if len(args.f_ips) == 15:
shift += 1
chosen_indices = [i + shift for i in chosen_indices]
print("Shifted range is", chosen_indices)
for i in chosen_indices:
model_instance_ip_port.append((args.f_ips[i], cur_port))
else:
for i in range(num_models):
model_instance_ip_port.append(
(args.f_ips[i % len(args.f_ips)], cur_port))
# Wrap around to the next port number if we will ned to repeat workers.
if i % len(args.f_ips) == len(args.f_ips) - 1:
cur_port += 1
print("Model instance ip, port:", model_instance_ip_port)
client_ip_port = []
if len(args.f_client_ports) != len(args.f_client_ips):
assert len(args.f_client_ports) == 1
args.f_client_ports *= len(args.f_client_ips)
client_ip_port = [(ip, int(port)) for ip, port in zip(args.f_client_ips, args.f_client_ports)]
cm = DistributedParmDockerContainerManager(model_instance_ip_port=model_instance_ip_port,
red_model_instance_ip_port=red_model_instance_ip_port,
client_ip_port=client_ip_port)
clipper_conn = ClipperConnection(cm, distributed=True)
frontend_args = {
"redundancy_mode": redundancy_mode,
"queue_mode": queue_mode,
"num_models": num_models,
"num_redundant_models": 0,
"batch_size": batch_size,
"mode": args.f_mode,
}
clipper_conn.start_clipper(frontend_args=frontend_args)
red_input_type = "bytes"
pytorch_deployer.create_endpoint(
clipper_conn=clipper_conn,
name="bg",
input_type="bytes",
func=func,
pytorch_model=model,
pkgs_to_install=['pillow'],
num_replicas=num_models,
batch_size=batch_size,
num_red_replicas=0,
red_func=red_func,
red_input_type=red_input_type,
red_pytorch_model=red_model,
prefer_original=False,
slo_micros=10000000 * 10)
sleep_time = 5
print("Sleeping for", sleep_time, "seconds to let things start up")
time.sleep(sleep_time)
cm.run_clients(wait=False)
# Listen to a ud socket to determine when we should quit.
sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
sock.bind('/home/ubuntu/bg_sock')
sock.listen(5)
(clientsocket, address) = sock.accept()
print("Stopping all clients")
cm.stop_all_clients()
print("Sending response")
clientsocket.sendall('1'.encode())
clientsocket.close()
sock.close()
print("Stopping all")
clipper_conn.stop_all()
X = ckd.iloc[:300, ]
X = X.loc[:, X.columns != 'classification']
X.fillna(0, inplace=True)
y = ckd.iloc[:300, -1]
# train a classifier
model = GradientBoostingClassifier(random_state=2019)
model.fit(X, y)
# First we need to import Clipper
from clipper_admin import ClipperConnection, KubernetesContainerManager
from clipper_admin.deployers.python import deploy_python_closure
# Create a Clipper connection
clipper_conn = ClipperConnection(
KubernetesContainerManager(useInternalIP=True,
kubernetes_proxy_addr="127.0.0.1:8080"))
# Start a Clipper cluster or connect to a running one
clipper_conn.start_clipper()
# Register an app called 'kddtutorial'. This would create a REST endpoint
clipper_conn.register_application(name="kddtutorial",
input_type="doubles",
default_output="-1.0",
slo_micros=10000000)
# Access the trained model via closure capture
def predict(inputs):
global model
parser = argparse.ArgumentParser()
# parser.add_argument('-n', '--num_nodes', type=int, default=3)
parser.add_argument('node_id', type=int)
args = parser.parse_args()
# num_nodes = args.num_nodes
node_id = args.node_id
clipper_conn = ClipperConnection(
DockerContainerManager(
cluster_name='clipper_cluster_{}'.format(node_id),
docker_ip_address='localhost',
clipper_query_port=1337 + node_id,
clipper_management_port=2337 + node_id,
clipper_rpc_port=7000 + node_id,
redis_ip=None,
redis_port=6379 + node_id,
prometheus_port=9090 + node_id,
# WARING: DO NOT CHANGE THE RULE OF NETWORK NAMES
docker_network='clipper_network_{}'.format(node_id),
# SINCE THIS IS USED BY reset.sh TO IDENTIFY CLIPPER CONTAINERS
extra_container_kwargs={})
) # for node_id in range(args.num_nodes)]
try:
clipper_conn.start_clipper()
clipper_conn.register_application(name="default",
input_type="string",
default_output="",
slo_micros=100000)
def __del__(self): clipper_conn = ClipperConnection(DockerContainerManager()) clipper_conn.stop_all()
from clipper_admin import ClipperConnection, DockerContainerManager
from clipper_admin.deployers import python as py_deployer
import random
cl = ClipperConnection(DockerContainerManager())
clipper_url = "http://localhost:1337/pong/predict"
headers = {'Content-Type': 'application/json'}
start = datetime.now()
clipper_response = requests.post(clipper_url,
headers=headers,
data=json.dumps(req_json))
def frontend(args):
print(args)
batch_size = args.batch_size
num_models = args.num_models
num_redundant_models = args.num_redundant_models
func = predict
red_func = predict
if args.redundancy_mode == "none" or args.redundancy_mode == "equal":
redundancy_mode = 0
elif args.redundancy_mode == "coded":
redundancy_mode = 2
elif args.redundancy_mode == "cheap":
redundancy_mode = 3
if args.queue_mode == "single_queue":
queue_mode = 0
single_queue = True
else:
# Round robin
queue_mode = 1
single_queue = False
assert len(args.f_ips) == num_models + num_redundant_models
if len(args.f_ports) != len(args.f_ips):
assert len(args.f_ports) == 1
args.f_ports *= len(args.f_ips)
model_instance_ip_port = []
red_model_instance_ip_port = []
for i in range(len(args.f_ips)):
if i < num_models:
model_instance_ip_port.append(
(args.f_ips[i], int(args.f_ports[i])))
else:
red_model_instance_ip_port.append(
(args.f_ips[i], int(args.f_ports[i])))
client_ip_port = []
if len(args.f_client_ports) != len(args.f_client_ips):
assert len(args.f_client_ports) == 1
args.f_client_ports *= len(args.f_client_ips)
client_ip_port = [
(ip, int(port))
for ip, port in zip(args.f_client_ips, args.f_client_ports)
]
cm = DistributedParmDockerContainerManager(
model_instance_ip_port=model_instance_ip_port,
red_model_instance_ip_port=red_model_instance_ip_port,
client_ip_port=client_ip_port)
clipper_conn = ClipperConnection(cm, distributed=True)
frontend_args = {
"redundancy_mode": redundancy_mode,
"queue_mode": queue_mode,
"num_models": num_models,
"num_redundant_models": num_redundant_models,
"batch_size": batch_size,
"mode": args.f_mode
}
clipper_conn.start_clipper(frontend_args=frontend_args)
if args.redundancy_mode == "coded":
red_input_type = "floats"
else:
red_input_type = "bytes"
pytorch_deployer.create_endpoint(clipper_conn=clipper_conn,
name="example",
input_type="bytes",
func=func,
pytorch_model=model,
pkgs_to_install=['pillow'],
num_replicas=num_models,
batch_size=batch_size,
num_red_replicas=num_redundant_models,
red_func=red_func,
red_input_type=red_input_type,
red_pytorch_model=red_model,
prefer_original=False,
slo_micros=10000000 * 10)
sleep_time = 5
print("Sleeping for", sleep_time, "seconds to let things start up")
time.sleep(sleep_time)
total_time = cm.run_clients()
print(total_time)
with open(args.f_outfile, 'w') as outfile:
outfile.write("{:.4f}".format(total_time))
clipper_conn.stop_all()
def signal_handler(signal, frame):
logging.info("\nGracefully stopping server...")
conn = ClipperConnection(DockerContainerManager())
conn.stop_all()
sys.exit(0)
