def main():
# Setup container manager.
# k8 = KubernetesContainerManager(kubernetes_proxy_addr="127.0.0.1:8080",
# useInternalIP=True)
# clipper_conn = ClipperConnection(k8)
swarm = DockerContainerManager()
clipper_conn = ClipperConnection(swarm)
clipper_conn.stop_all()
clipper_conn.start_clipper()
# Register application.
clipper_conn.register_application(name="sum-app",
input_type="doubles",
default_output="-1.0",
slo_micros=10000000)
# Model deployement.
python_deployer.deploy_python_closure(clipper_conn,
name="sum-model",
version=1,
input_type="doubles",
func=sum)
# Link application to model.
clipper_conn.link_model_to_app(app_name="sum-app",
model_name="sum-model")
# Test
headers = {"Content-type": "application/json"}
response = requests.post("http://localhost:1337/sum-app/predict",
headers=headers,
data=json.dumps({"input": list(np.random.random(10))})).json()
print(response)
def create_docker_connection(cleanup=True, start_clipper=True):
logger.info("Creating DockerContainerManager")
cm = DockerContainerManager(clipper_query_port=find_unbound_port(),
clipper_management_port=find_unbound_port(),
clipper_rpc_port=find_unbound_port(),
redis_port=find_unbound_port())
cl = ClipperConnection(cm)
if cleanup:
cl.stop_all()
docker_client = get_docker_client()
docker_client.containers.prune(filters={"label": CLIPPER_DOCKER_LABEL})
if start_clipper:
# Try to start Clipper in a retry loop here to address flaky tests
# as described in https://github.com/ucbrise/clipper/issues/352
while True:
try:
logger.info("Starting Clipper")
cl.start_clipper()
time.sleep(1)
break
except docker.errors.APIError as e:
logger.info(
"Problem starting Clipper: {}\nTrying again.".format(e))
cl.stop_all()
cm = DockerContainerManager(
clipper_query_port=find_unbound_port(),
clipper_management_port=find_unbound_port(),
clipper_rpc_port=find_unbound_port(),
redis_port=find_unbound_port())
cl = ClipperConnection(cm)
else:
cl.connect()
return cl
def create_kubernetes_connection(cleanup=True,
start_clipper=True,
connect=True,
with_proxy=False,
num_frontend_replicas=1):
logger.info("Creating KubernetesContainerManager")
if with_proxy:
cm = KubernetesContainerManager(kubernetes_proxy_addr="127.0.0.1:8080")
else:
cm = KubernetesContainerManager()
cl = ClipperConnection(cm)
if cleanup:
cl.stop_all()
# Give kubernetes some time to clean up
time.sleep(20)
logger.info("Done cleaning up clipper")
if start_clipper:
logger.info("Starting Clipper")
cl.start_clipper(
query_frontend_image=
"568959175238.dkr.ecr.us-west-1.amazonaws.com/clipper/query_frontend:{}".
format(clipper_version),
mgmt_frontend_image=
"568959175238.dkr.ecr.us-west-1.amazonaws.com/clipper/management_frontend:{}".
format(clipper_version),
num_frontend_replicas=num_frontend_replicas)
time.sleep(1)
if connect:
try:
cl.connect()
except Exception:
pass
except ClipperException:
pass
return cl
def create_kubernetes_connection(cleanup=False,
start_clipper=False,
connect=False,
with_proxy=False,
num_frontend_replicas=1,
cleanup_name='default-cluster',
new_name='default-cluster',
connect_name='default-cluster',
service_types=None,
namespace='default'):
logger.info("Creating KubernetesContainerManager")
cl = None
assert cleanup or start_clipper or connect, "You must set at least one of {cleanup, start_clipper, connect} to be true."
if with_proxy:
kubernetes_proxy_addr = "127.0.0.1:8080"
else:
kubernetes_proxy_addr = None
if cleanup:
logger.info("Cleaning up Kubernetes Cluster {}".format(cleanup_name))
cm = KubernetesContainerManager(
cluster_name=cleanup_name,
useInternalIP=USE_MINIKUBE,
service_types=service_types,
kubernetes_proxy_addr=kubernetes_proxy_addr)
cl = ClipperConnection(cm)
cl.stop_all()
logger.info("Done cleaning up clipper")
if start_clipper:
logger.info("Starting up Kubernetes Cluster {}".format(new_name))
cm = KubernetesContainerManager(
cluster_name=new_name,
kubernetes_proxy_addr=kubernetes_proxy_addr,
namespace=namespace,
useInternalIP=USE_MINIKUBE,
service_types=service_types,
create_namespace_if_not_exists=True)
cl = ClipperConnection(cm)
cl.start_clipper(num_frontend_replicas=num_frontend_replicas)
if connect:
try:
cm = KubernetesContainerManager(
cluster_name=connect_name,
useInternalIP=USE_MINIKUBE,
service_types=service_types,
kubernetes_proxy_addr=kubernetes_proxy_addr)
cl = ClipperConnection(cm)
cl.connect()
except Exception:
pass
return cl
def create_docker_connection(cleanup=False,
start_clipper=False,
cleanup_name='default-cluster',
new_name='default-cluster',
use_centralized_log=False):
logger.info("Creating DockerContainerManager")
cl = None
assert cleanup or start_clipper, "You must set at least one of {cleanup, start_clipper} to be true."
if cleanup:
logger.info("Cleaning up Docker cluster {}".format(cleanup_name))
cm = DockerContainerManager(
cluster_name=cleanup_name,
clipper_query_port=find_unbound_port(),
clipper_management_port=find_unbound_port(),
clipper_rpc_port=find_unbound_port(),
fluentd_port=find_unbound_port(),
redis_port=find_unbound_port(),
prometheus_port=find_unbound_port(),
)
cl = ClipperConnection(cm)
cl.stop_all(graceful=False)
if start_clipper:
# Try to start Clipper in a retry loop here to address flaky tests
# as described in https://github.com/ucbrise/clipper/issues/352
logger.info("Starting up Docker cluster {}".format(new_name))
while True:
cm = DockerContainerManager(
cluster_name=new_name,
clipper_query_port=find_unbound_port(),
clipper_management_port=find_unbound_port(),
clipper_rpc_port=find_unbound_port(),
fluentd_port=find_unbound_port(),
redis_port=find_unbound_port(),
prometheus_port=find_unbound_port(),
use_centralized_log=use_centralized_log)
cl = ClipperConnection(cm)
try:
logger.info("Starting Clipper")
cl.start_clipper()
time.sleep(1)
break
except docker.errors.APIError as e:
logger.info(
"Problem starting Clipper: {}\nTrying again.".format(e))
cl.stop_all()
return cl
class Clip(object):
def __init__(self, sleep_time):
from clipper_admin import ClipperConnection, DockerContainerManager
#from clipper_admin.deployers import python as python_deployer
from clipper_admin.deployers import pytorch as pt_deployer
self.clipper_conn = ClipperConnection(DockerContainerManager())
try:
self.clipper_conn.connect()
self.clipper_conn.stop_all()
except Exception:
pass
self.clipper_conn.start_clipper()
self.clipper_conn.register_application(name="hello-world",
input_type="strings",
default_output="-1.0",
slo_micros=10**8)
model = Model()
def policy(ptmodel, x):
batch = (len(x))
arr = []
for j in x:
xs, masks = j.split("###")
res = np.frombuffer(base64.decodestring(xs), dtype=np.float32)
res = res.reshape((64, 19, 19, 3))
res = np.frombuffer(base64.decodestring(masks),
dtype=np.float32)
res = res.reshape((64, 362))
for i in x:
time.sleep(sleep_time)
return [
np.random.rand(64).astype(np.float32) for i in range(batch)
]
pt_deployer.deploy_pytorch_model(self.clipper_conn,
name="policy",
version=1,
input_type="strings",
func=policy,
pytorch_model=model)
self.clipper_conn.link_model_to_app(app_name="hello-world",
model_name="policy")
class Clip(object):
def __init__(self, shape, model_name):
from clipper_admin import ClipperConnection, DockerContainerManager
from clipper_admin.deployers import python as python_deployer
from clipper_admin.deployers import pytorch as pytorch_deployer
self.clipper_conn = ClipperConnection(DockerContainerManager())
try:
self.clipper_conn.connect()
self.clipper_conn.stop_all()
except Exception:
pass
self.clipper_conn.start_clipper()
self.clipper_conn.register_application(name="hello-world",
input_type="strings",
default_output="-1.0",
slo_micros=10**8)
ptmodel = get_model(model_name)
def policy(model, x):
print(len(x))
batch = (len(x))
arr = []
for j in x:
print(type(j), len(j))
res = np.frombuffer(base64.decodestring(j), dtype=np.float32)
print(res.shape)
arr += [res]
x = np.array(arr)
x = x.reshape((-1, ) + shape[1:])
print("new shape", x.shape)
return evaluate_model(model, x).reshape((batch, shape[0]))
pytorch_deployer.deploy_pytorch_model(self.clipper_conn,
name="policy",
version=1,
input_type="strings",
func=policy,
pytorch_model=ptmodel)
self.clipper_conn.link_model_to_app(app_name="hello-world",
model_name="policy")
def deployModelToClipper():
"""Deploy model to clipper and replace its entry."""
global app_name, model_name, model_version
print('Deploying model to clipper, model_name={}, model_version={}'.format(
model_name, model_version))
# Setup clipper and deploy model
clipper_conn = ClipperConnection(DockerContainerManager(redis_port=6380))
try:
clipper_conn.start_clipper()
except:
clipper_conn.connect()
try:
# input_type must be bytes as inputs will be serialized into bytes with pickle
clipper_conn.register_application(name=app_name,
input_type="bytes",
default_output="-1.0",
slo_micros=1000000)
except Exception as e:
print(e)
try:
deploy_python_closure(clipper_conn,
name=model_name,
version=model_version,
input_type="bytes",
batch_size=1,
func=predict,
base_image='hysia-clipper-base-container-gpu')
except Exception as e:
print(e)
try:
clipper_conn.link_model_to_app(app_name=app_name,
model_name=model_name)
except Exception as e:
print(e)
replaceDefaultEntry()
print('{} deployed to clipper!'.format(model_name))
def frontend(args):
print(args)
batch_size = args.batch_size
num_models = args.num_models
num_redundant_models = args.num_redundant_models
func = predict
red_func = predict
if args.redundancy_mode == "none" or args.redundancy_mode == "equal":
redundancy_mode = 0
elif args.redundancy_mode == "coded":
redundancy_mode = 2
elif args.redundancy_mode == "cheap":
redundancy_mode = 3
if args.queue_mode == "single_queue":
queue_mode = 0
single_queue = True
else:
# Round robin
queue_mode = 1
single_queue = False
assert len(args.f_ips) == num_models + num_redundant_models
if len(args.f_ports) != len(args.f_ips):
assert len(args.f_ports) == 1
args.f_ports *= len(args.f_ips)
model_instance_ip_port = []
red_model_instance_ip_port = []
for i in range(len(args.f_ips)):
if i < num_models:
model_instance_ip_port.append(
(args.f_ips[i], int(args.f_ports[i])))
else:
red_model_instance_ip_port.append(
(args.f_ips[i], int(args.f_ports[i])))
client_ip_port = []
if len(args.f_client_ports) != len(args.f_client_ips):
assert len(args.f_client_ports) == 1
args.f_client_ports *= len(args.f_client_ips)
client_ip_port = [
(ip, int(port))
for ip, port in zip(args.f_client_ips, args.f_client_ports)
]
cm = DistributedParmDockerContainerManager(
model_instance_ip_port=model_instance_ip_port,
red_model_instance_ip_port=red_model_instance_ip_port,
client_ip_port=client_ip_port)
clipper_conn = ClipperConnection(cm, distributed=True)
frontend_args = {
"redundancy_mode": redundancy_mode,
"queue_mode": queue_mode,
"num_models": num_models,
"num_redundant_models": num_redundant_models,
"batch_size": batch_size,
"mode": args.f_mode
}
clipper_conn.start_clipper(frontend_args=frontend_args)
if args.redundancy_mode == "coded":
red_input_type = "floats"
else:
red_input_type = "bytes"
pytorch_deployer.create_endpoint(clipper_conn=clipper_conn,
name="example",
input_type="bytes",
func=func,
pytorch_model=model,
pkgs_to_install=['pillow'],
num_replicas=num_models,
batch_size=batch_size,
num_red_replicas=num_redundant_models,
red_func=red_func,
red_input_type=red_input_type,
red_pytorch_model=red_model,
prefer_original=False,
slo_micros=10000000 * 10)
sleep_time = 5
print("Sleeping for", sleep_time, "seconds to let things start up")
time.sleep(sleep_time)
total_time = cm.run_clients()
print(total_time)
with open(args.f_outfile, 'w') as outfile:
outfile.write("{:.4f}".format(total_time))
clipper_conn.stop_all()
from clipper_admin import ClipperConnection, DockerContainerManager
from clipper_admin.deployers import python as py_deployer
import random
cl = ClipperConnection(DockerContainerManager())
cl.stop_all()
cl.start_clipper()
# cl.register_application(name="pong", input_type="doubles", slo_micros=1000000, default_output="1")
def random_predict(xs):
action = random.randint(0, 2)
return [str(action) for _ in xs]
# cl.register_application(name="pong", input_type="doubles", default_output="-1.0", slo_micros=100000)
# py_deployer.deploy_python_closure(cl, name="rand-model", version=1, input_type="doubles", func=random_predict, registry="hsubbaraj")
# cl.link_model_to_app(app_name="pong", model_name="rand-model")
py_deployer.create_endpoint(cl,
name="pong",
input_type="doubles",
func=random_predict,
default_output="0",
slo_micros=100000)
print("CLIPPER ADDRESS: " + cl.get_query_addr())
# clipper_start
from clipper_admin import ClipperConnection, DockerContainerManager
clipper_conn = ClipperConnection(DockerContainerManager())
clipper_conn.start_clipper()
clipper_conn.connect()
clipper_conn.register_application(
name="digit",
input_type="doubles",
default_output="-1.0",
slo_micros=10000000) # 10,000,000 micros == 10 sec
clipper_conn.get_all_apps()
#################################################
######### Define Own Prediction Function ########
#################################################
import sklearn
import numpy as np
from sklearn.neural_network import MLPClassifier
from sklearn.externals import joblib
from clipper_admin.deployers import python as python_deployer
for version_postfix in ["10x1k", "10x2k", "20x1k", "15x2k"]:
model_path = "../../models/sklearn/"
model_name = "dig_nn_model_" + version_postfix + ".sav"
clf = joblib.load(model_path + model_name)
response = apply_forecast_impl(parameters[0].decode("utf-8").split(','))
code = '202'
except Exception as e:
response = str(e)
code = '500'
return [str(code+', '+response) for _ in parameters]
if __name__ == '__main__':
# setup logging format
format = "%(asctime)-15s %(message)s"
logging.basicConfig(
filename='./timeseries/log.log', level=logging.DEBUG, format=format)
# set up logging to console
console = logging.StreamHandler(sys.stdout)
console.setLevel(logging.ERROR)
logging.getLogger().addHandler(console)
signal.signal(signal.SIGINT, signal_handler)
conn = ClipperConnection(DockerContainerManager())
conn.start_clipper()
try:
conn.register_application(name="forecast",input_type="strings",default_output="500, Error executing call.",slo_micros=100000000)
python_deployer.deploy_python_closure(conn, name="do-forecast", version=1, input_type="strings", func=do_forecast, base_image='wamsiv/timeseries:latest')
conn.link_model_to_app(app_name="forecast", model_name='do-forecast')
print(subprocess.getoutput(["docker update --restart=on-failure $(docker ps -a | grep 'clipper/query_frontend:0.3.0' | awk '{ print $1 }')"]))
input("Server started. Press ctrl+c to stop server.\n")
except Exception as e:
logging.error("Encountered {}. Stopping server...".format(e))
conn.stop_all()
conn.stop_all()
from clipper_admin import ClipperConnection, DockerContainerManager
clipper_conn = ClipperConnection(DockerContainerManager(extra_container_kwargs={'cpuset_cpus':'0-35'}))
clipper_conn.start_clipper(cache_size=0)
def frontend(args):
print(args)
batch_size = args.batch_size
num_models = args.num_models
func = predict
red_func = predict
assert args.redundancy_mode in ["none", "equal"]
redundancy_mode = 0
if args.queue_mode == "single_queue":
queue_mode = 0
elif args.queue_mode == "rr":
queue_mode = 1
else:
assert False, "Unrecognized queue mode '{}'".format(args.queue_mode)
model_instance_ip_port = []
red_model_instance_ip_port = []
cur_port = base_port
if num_models < len(args.f_ips):
# Round up to highest int so as not to launch more models than needed.
num_between = int(len(args.f_ips) / num_models + 0.5)
chosen_indices = list(range(0, len(args.f_ips), num_between))
print("Range is", chosen_indices)
# Shift our chosen indices so that they are evenly distributed
# throughout the clients.
delta = len(args.f_ips) - chosen_indices[-1]
shift = delta // 2
if len(args.f_ips) == 15:
shift += 1
chosen_indices = [i + shift for i in chosen_indices]
print("Shifted range is", chosen_indices)
for i in chosen_indices:
model_instance_ip_port.append((args.f_ips[i], cur_port))
else:
for i in range(num_models):
model_instance_ip_port.append(
(args.f_ips[i % len(args.f_ips)], cur_port))
# Wrap around to the next port number if we will ned to repeat workers.
if i % len(args.f_ips) == len(args.f_ips) - 1:
cur_port += 1
print("Model instance ip, port:", model_instance_ip_port)
client_ip_port = []
if len(args.f_client_ports) != len(args.f_client_ips):
assert len(args.f_client_ports) == 1
args.f_client_ports *= len(args.f_client_ips)
client_ip_port = [(ip, int(port)) for ip, port in zip(args.f_client_ips, args.f_client_ports)]
cm = DistributedParmDockerContainerManager(model_instance_ip_port=model_instance_ip_port,
red_model_instance_ip_port=red_model_instance_ip_port,
client_ip_port=client_ip_port)
clipper_conn = ClipperConnection(cm, distributed=True)
frontend_args = {
"redundancy_mode": redundancy_mode,
"queue_mode": queue_mode,
"num_models": num_models,
"num_redundant_models": 0,
"batch_size": batch_size,
"mode": args.f_mode,
}
clipper_conn.start_clipper(frontend_args=frontend_args)
red_input_type = "bytes"
pytorch_deployer.create_endpoint(
clipper_conn=clipper_conn,
name="bg",
input_type="bytes",
func=func,
pytorch_model=model,
pkgs_to_install=['pillow'],
num_replicas=num_models,
batch_size=batch_size,
num_red_replicas=0,
red_func=red_func,
red_input_type=red_input_type,
red_pytorch_model=red_model,
prefer_original=False,
slo_micros=10000000 * 10)
sleep_time = 5
print("Sleeping for", sleep_time, "seconds to let things start up")
time.sleep(sleep_time)
cm.run_clients(wait=False)
# Listen to a ud socket to determine when we should quit.
sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
sock.bind('/home/ubuntu/bg_sock')
sock.listen(5)
(clientsocket, address) = sock.accept()
print("Stopping all clients")
cm.stop_all_clients()
print("Sending response")
clientsocket.sendall('1'.encode())
clientsocket.close()
sock.close()
print("Stopping all")
clipper_conn.stop_all()
def create_clipper_conn():
conn = ClipperConnection(DockerContainerManager())
conn.start_clipper()
return conn
