def setup_clipper():
app_name = 'inceptionv3-app'
model_name = 'inceptionv3-model'
clipper_conn = ClipperConnection(DockerContainerManager())
clipper_conn.connect()
pytorch_deployer.deploy_pytorch_model(
clipper_conn=clipper_conn,
name=model_name,
version='1',
input_type='bytes',
func=incept_predict,
pytorch_model=incept,
num_replicas=10,
batch_size=1,
pkgs_to_install=['pillow', 'pyarrow', 'torch', 'torchvision'])
clipper_conn.register_application(name=app_name,
input_type="bytes",
default_output="-1.0",
slo_micros=10000000) # 10s
clipper_conn.link_model_to_app(app_name=app_name, model_name=model_name)
print(
"url: ", "http://{addr}/{app_name}/predict".format(addr="",
app_name=app_name))
def deploy_and_test_model(clipper_conn,
model,
version,
link_model=False,
predict_fn=predict):
deploy_pytorch_model(clipper_conn, model_name, version, "integers",
predict_fn, model)
time.sleep(5)
if link_model:
clipper_conn.link_model_to_app(app_name, model_name)
time.sleep(5)
test_model(clipper_conn, app_name, version)
def __init__(self): self.latent_size = 20 self.img_size = 128 model = loadModel() deploy_pytorch_model( clipper_conn, name="example", version = 1, input_type="doubles", func=predict, pytorch_model=model ) clipper_conn.link_model_to_app(app_name="Shoe", model_name="example")
def __init__(self, sleep_time):
from clipper_admin import ClipperConnection, DockerContainerManager
#from clipper_admin.deployers import python as python_deployer
from clipper_admin.deployers import pytorch as pt_deployer
self.clipper_conn = ClipperConnection(DockerContainerManager())
try:
self.clipper_conn.connect()
self.clipper_conn.stop_all()
except Exception:
pass
self.clipper_conn.start_clipper()
self.clipper_conn.register_application(name="hello-world",
input_type="strings",
default_output="-1.0",
slo_micros=10**8)
model = Model()
def policy(ptmodel, x):
batch = (len(x))
arr = []
for j in x:
xs, masks = j.split("###")
res = np.frombuffer(base64.decodestring(xs), dtype=np.float32)
res = res.reshape((64, 19, 19, 3))
res = np.frombuffer(base64.decodestring(masks),
dtype=np.float32)
res = res.reshape((64, 362))
for i in x:
time.sleep(sleep_time)
return [
np.random.rand(64).astype(np.float32) for i in range(batch)
]
pt_deployer.deploy_pytorch_model(self.clipper_conn,
name="policy",
version=1,
input_type="strings",
func=policy,
pytorch_model=model)
self.clipper_conn.link_model_to_app(app_name="hello-world",
model_name="policy")
def __init__(self, shape, model_name):
from clipper_admin import ClipperConnection, DockerContainerManager
from clipper_admin.deployers import python as python_deployer
from clipper_admin.deployers import pytorch as pytorch_deployer
self.clipper_conn = ClipperConnection(DockerContainerManager())
try:
self.clipper_conn.connect()
self.clipper_conn.stop_all()
except Exception:
pass
self.clipper_conn.start_clipper()
self.clipper_conn.register_application(name="hello-world",
input_type="strings",
default_output="-1.0",
slo_micros=10**8)
ptmodel = get_model(model_name)
def policy(model, x):
print(len(x))
batch = (len(x))
arr = []
for j in x:
print(type(j), len(j))
res = np.frombuffer(base64.decodestring(j), dtype=np.float32)
print(res.shape)
arr += [res]
x = np.array(arr)
x = x.reshape((-1, ) + shape[1:])
print("new shape", x.shape)
return evaluate_model(model, x).reshape((batch, shape[0]))
pytorch_deployer.deploy_pytorch_model(self.clipper_conn,
name="policy",
version=1,
input_type="strings",
func=policy,
pytorch_model=ptmodel)
self.clipper_conn.link_model_to_app(app_name="hello-world",
model_name="policy")
def setup_clipper():
app_name = 'resnet101-app'
model_name = 'resnet101-model'
clipper_conn = ClipperConnection(DockerContainerManager())
clipper_conn.connect()
pytorch_deployer.deploy_pytorch_model(clipper_conn=clipper_conn,
name=model_name,
version='1',
input_type='bytes',
func=resnet_predict,
pytorch_model=resnet101,
pkgs_to_install=['pillow', 'torch', 'torchvision'])
clipper_conn.register_application(name=app_name,
input_type="bytes",
default_output="-1.0",
slo_micros=10000000) # 10s
clipper_conn.link_model_to_app(app_name=app_name, model_name=model_name)
print("query_adress: ", clipper_conn.get_query_addr())
print("app_name: ", )
print("model_name: ", )
print("url: ", "http://{addr}/{app_name}/predict".format(addr=clipper_conn.get_query_addr(),app_name=app_name))
clipper_conn.start_clipper()
clipper_conn.register_application(name="pytorch-example-2", input_type="doubles", default_output="-1.0", slo_micros=100000)
# model = nn.Linear(1, 1)
# Define a shift function to normalize prediction inputs
def pred(model, inputs):
preds = []
for i in inputs:
preds.append(np.random.rand(1,10))
return preds
# return [model(torch.FloatTensor(np.reshape(i, (-1, 3, 32, 32))).data.numpy()) for i in inputs]
# return [str(model(torch.FloatTensor(np.reshape(i, (1, 3, 32, 32)))).data.numpy()) for i in inputs]
deploy_pytorch_model(
clipper_conn,
name="pytorch-nn",
version=1,
input_type="doubles",
func=pred,
pytorch_model=net,
registry="hsubbaraj"
)
clipper_conn.link_model_to_app(app_name="pytorch-example-2", model_name="pytorch-nn")
time.sleep(2)
print("deployed model")
APP_NAME = "test-app"
MODEL_NAME = "test-pytorch-model"
# Setup clipper and deploy pytorch model
clipper_conn = ClipperConnection(DockerContainerManager(redis_port=6380))
try:
clipper_conn.start_clipper()
clipper_conn.register_application(name=APP_NAME,
input_type="doubles",
default_output="-1.0",
slo_micros=1000000)
deploy_pytorch_model(clipper_conn,
name=MODEL_NAME,
version="1",
input_type="doubles",
func=predict,
pytorch_model=model,
pkgs_to_install=pip_deps)
clipper_conn.link_model_to_app(app_name=APP_NAME, model_name=MODEL_NAME)
except:
clipper_conn.connect()
# Check all apps
print(clipper_conn.get_all_apps())
# Test inference
# inputs = np.array([[1., 2., 3.], [2., 3., 4.], [3., 4., 5.]])
# print(predict(model, inputs))
inputs = np.array([1., 2., 3.]).tolist(
) # Inputs can only be one-dimensional or there will be json serialization error
def predict(model, inputs):
trans = shift(inputs)
pred = model(trans)
# print(pred)
pred = pred.data.numpy()
# print(pred)
# print(pred.shape)
# pred = pred.flatten()
# print(pred)
# print(pred.shape)
#Fix bug, append str(pred.shape) tuple as string, and append dtype
return pred.astype(np.str).tolist()
# return [np.array2string(x, precision=8,separator=",") for x in pred]
# res = pred.tostring().encode('base64')
# print(res)
# return [res]
# return [np.array_str(pred)]
deploy_pytorch_model(clipper_conn,
name="example",
version=1,
input_type="doubles",
func=predict,
pytorch_model=model)
clipper_conn.link_model_to_app(app_name="Shoe", model_name="example")
# deploy_pytorch_model(
# clipper_conn,
# name="superresolution-model",
# version=1,
# input_type="bytes",
# func=image_enhance,
# pytorch_model=model,
# pkgs_to_install=['opencv-python','numpy','six', 'Pillow', 'wheel', 'certifi']
# )
deploy_pytorch_model(
clipper_conn,
name="superresolution-model",
version=1,
input_type="bytes",
func=image_enhance,
pytorch_model=model,
base_image='custom-model-image',
pkgs_to_install=['opencv-python','numpy','six', 'Pillow','wheel',]
)
print("linking model to app...")
clipper_conn.link_model_to_app(app_name="superresolution", model_name="superresolution-model")
def query(addr, filename):
url = "http://%s/superresolution/predict" % addr
req_json = json.dumps({
"input":
base64.b64encode(open(filename, "rb").read()).decode() # bytes to unicode
})
data_transforms = transforms.Compose([
transforms.Resize((224, 224)),
transforms.ToTensor(),
transforms.Normalize(train_mean, train_std)])
image = get_image(data_transforms, image_url)
pred = np.argmax(model(image).detach().numpy())
return CLASSES[pred]
model = get_model()
clipper_conn.stop_models(model_names='logo-detector')
deploy_pytorch_model(
clipper_conn,
name="logo-detector",
version=1,
input_type="strings",
func=predict,
pytorch_model=model,
pkgs_to_install=['Pillow', 'torchvision', 'torch', 'numpy', 'requests']
)
clipper_conn.get_clipper_logs()
clipper_conn.link_model_to_app(app_name="logo-detector", model_name="logo-detector")
clipper_conn.get_linked_models(app_name="logo-detector")
clipper_conn.cm.get_num_replicas(name="logo-detector", version="1")
clipper_conn.get_clipper_logs()