def setup_clipper(): app_name = 'inceptionv3-app' model_name = 'inceptionv3-model' clipper_conn = ClipperConnection(DockerContainerManager()) clipper_conn.connect() pytorch_deployer.deploy_pytorch_model( clipper_conn=clipper_conn, name=model_name, version='1', input_type='bytes', func=incept_predict, pytorch_model=incept, num_replicas=10, batch_size=1, pkgs_to_install=['pillow', 'pyarrow', 'torch', 'torchvision']) clipper_conn.register_application(name=app_name, input_type="bytes", default_output="-1.0", slo_micros=10000000) # 10s clipper_conn.link_model_to_app(app_name=app_name, model_name=model_name) print( "url: ", "http://{addr}/{app_name}/predict".format(addr="", app_name=app_name))
def deploy_and_test_model(clipper_conn, model, version, link_model=False, predict_fn=predict): deploy_pytorch_model(clipper_conn, model_name, version, "integers", predict_fn, model) time.sleep(5) if link_model: clipper_conn.link_model_to_app(app_name, model_name) time.sleep(5) test_model(clipper_conn, app_name, version)
def __init__(self): self.latent_size = 20 self.img_size = 128 model = loadModel() deploy_pytorch_model( clipper_conn, name="example", version = 1, input_type="doubles", func=predict, pytorch_model=model ) clipper_conn.link_model_to_app(app_name="Shoe", model_name="example")
def __init__(self, sleep_time): from clipper_admin import ClipperConnection, DockerContainerManager #from clipper_admin.deployers import python as python_deployer from clipper_admin.deployers import pytorch as pt_deployer self.clipper_conn = ClipperConnection(DockerContainerManager()) try: self.clipper_conn.connect() self.clipper_conn.stop_all() except Exception: pass self.clipper_conn.start_clipper() self.clipper_conn.register_application(name="hello-world", input_type="strings", default_output="-1.0", slo_micros=10**8) model = Model() def policy(ptmodel, x): batch = (len(x)) arr = [] for j in x: xs, masks = j.split("###") res = np.frombuffer(base64.decodestring(xs), dtype=np.float32) res = res.reshape((64, 19, 19, 3)) res = np.frombuffer(base64.decodestring(masks), dtype=np.float32) res = res.reshape((64, 362)) for i in x: time.sleep(sleep_time) return [ np.random.rand(64).astype(np.float32) for i in range(batch) ] pt_deployer.deploy_pytorch_model(self.clipper_conn, name="policy", version=1, input_type="strings", func=policy, pytorch_model=model) self.clipper_conn.link_model_to_app(app_name="hello-world", model_name="policy")
def __init__(self, shape, model_name): from clipper_admin import ClipperConnection, DockerContainerManager from clipper_admin.deployers import python as python_deployer from clipper_admin.deployers import pytorch as pytorch_deployer self.clipper_conn = ClipperConnection(DockerContainerManager()) try: self.clipper_conn.connect() self.clipper_conn.stop_all() except Exception: pass self.clipper_conn.start_clipper() self.clipper_conn.register_application(name="hello-world", input_type="strings", default_output="-1.0", slo_micros=10**8) ptmodel = get_model(model_name) def policy(model, x): print(len(x)) batch = (len(x)) arr = [] for j in x: print(type(j), len(j)) res = np.frombuffer(base64.decodestring(j), dtype=np.float32) print(res.shape) arr += [res] x = np.array(arr) x = x.reshape((-1, ) + shape[1:]) print("new shape", x.shape) return evaluate_model(model, x).reshape((batch, shape[0])) pytorch_deployer.deploy_pytorch_model(self.clipper_conn, name="policy", version=1, input_type="strings", func=policy, pytorch_model=ptmodel) self.clipper_conn.link_model_to_app(app_name="hello-world", model_name="policy")
def setup_clipper(): app_name = 'resnet101-app' model_name = 'resnet101-model' clipper_conn = ClipperConnection(DockerContainerManager()) clipper_conn.connect() pytorch_deployer.deploy_pytorch_model(clipper_conn=clipper_conn, name=model_name, version='1', input_type='bytes', func=resnet_predict, pytorch_model=resnet101, pkgs_to_install=['pillow', 'torch', 'torchvision']) clipper_conn.register_application(name=app_name, input_type="bytes", default_output="-1.0", slo_micros=10000000) # 10s clipper_conn.link_model_to_app(app_name=app_name, model_name=model_name) print("query_adress: ", clipper_conn.get_query_addr()) print("app_name: ", ) print("model_name: ", ) print("url: ", "http://{addr}/{app_name}/predict".format(addr=clipper_conn.get_query_addr(),app_name=app_name))
clipper_conn.start_clipper() clipper_conn.register_application(name="pytorch-example-2", input_type="doubles", default_output="-1.0", slo_micros=100000) # model = nn.Linear(1, 1) # Define a shift function to normalize prediction inputs def pred(model, inputs): preds = [] for i in inputs: preds.append(np.random.rand(1,10)) return preds # return [model(torch.FloatTensor(np.reshape(i, (-1, 3, 32, 32))).data.numpy()) for i in inputs] # return [str(model(torch.FloatTensor(np.reshape(i, (1, 3, 32, 32)))).data.numpy()) for i in inputs] deploy_pytorch_model( clipper_conn, name="pytorch-nn", version=1, input_type="doubles", func=pred, pytorch_model=net, registry="hsubbaraj" ) clipper_conn.link_model_to_app(app_name="pytorch-example-2", model_name="pytorch-nn") time.sleep(2) print("deployed model")
APP_NAME = "test-app" MODEL_NAME = "test-pytorch-model" # Setup clipper and deploy pytorch model clipper_conn = ClipperConnection(DockerContainerManager(redis_port=6380)) try: clipper_conn.start_clipper() clipper_conn.register_application(name=APP_NAME, input_type="doubles", default_output="-1.0", slo_micros=1000000) deploy_pytorch_model(clipper_conn, name=MODEL_NAME, version="1", input_type="doubles", func=predict, pytorch_model=model, pkgs_to_install=pip_deps) clipper_conn.link_model_to_app(app_name=APP_NAME, model_name=MODEL_NAME) except: clipper_conn.connect() # Check all apps print(clipper_conn.get_all_apps()) # Test inference # inputs = np.array([[1., 2., 3.], [2., 3., 4.], [3., 4., 5.]]) # print(predict(model, inputs)) inputs = np.array([1., 2., 3.]).tolist( ) # Inputs can only be one-dimensional or there will be json serialization error
def predict(model, inputs): trans = shift(inputs) pred = model(trans) # print(pred) pred = pred.data.numpy() # print(pred) # print(pred.shape) # pred = pred.flatten() # print(pred) # print(pred.shape) #Fix bug, append str(pred.shape) tuple as string, and append dtype return pred.astype(np.str).tolist() # return [np.array2string(x, precision=8,separator=",") for x in pred] # res = pred.tostring().encode('base64') # print(res) # return [res] # return [np.array_str(pred)] deploy_pytorch_model(clipper_conn, name="example", version=1, input_type="doubles", func=predict, pytorch_model=model) clipper_conn.link_model_to_app(app_name="Shoe", model_name="example")
# deploy_pytorch_model( # clipper_conn, # name="superresolution-model", # version=1, # input_type="bytes", # func=image_enhance, # pytorch_model=model, # pkgs_to_install=['opencv-python','numpy','six', 'Pillow', 'wheel', 'certifi'] # ) deploy_pytorch_model( clipper_conn, name="superresolution-model", version=1, input_type="bytes", func=image_enhance, pytorch_model=model, base_image='custom-model-image', pkgs_to_install=['opencv-python','numpy','six', 'Pillow','wheel',] ) print("linking model to app...") clipper_conn.link_model_to_app(app_name="superresolution", model_name="superresolution-model") def query(addr, filename): url = "http://%s/superresolution/predict" % addr req_json = json.dumps({ "input": base64.b64encode(open(filename, "rb").read()).decode() # bytes to unicode })
data_transforms = transforms.Compose([ transforms.Resize((224, 224)), transforms.ToTensor(), transforms.Normalize(train_mean, train_std)]) image = get_image(data_transforms, image_url) pred = np.argmax(model(image).detach().numpy()) return CLASSES[pred] model = get_model() clipper_conn.stop_models(model_names='logo-detector') deploy_pytorch_model( clipper_conn, name="logo-detector", version=1, input_type="strings", func=predict, pytorch_model=model, pkgs_to_install=['Pillow', 'torchvision', 'torch', 'numpy', 'requests'] ) clipper_conn.get_clipper_logs() clipper_conn.link_model_to_app(app_name="logo-detector", model_name="logo-detector") clipper_conn.get_linked_models(app_name="logo-detector") clipper_conn.cm.get_num_replicas(name="logo-detector", version="1") clipper_conn.get_clipper_logs()