def main(): # Setup container manager. # k8 = KubernetesContainerManager(kubernetes_proxy_addr="127.0.0.1:8080", # useInternalIP=True) # clipper_conn = ClipperConnection(k8) swarm = DockerContainerManager() clipper_conn = ClipperConnection(swarm) clipper_conn.stop_all() clipper_conn.start_clipper() # Register application. clipper_conn.register_application(name="sum-app", input_type="doubles", default_output="-1.0", slo_micros=10000000) # Model deployement. python_deployer.deploy_python_closure(clipper_conn, name="sum-model", version=1, input_type="doubles", func=sum) # Link application to model. clipper_conn.link_model_to_app(app_name="sum-app", model_name="sum-model") # Test headers = {"Content-type": "application/json"} response = requests.post("http://localhost:1337/sum-app/predict", headers=headers, data=json.dumps({"input": list(np.random.random(10))})).json() print(response)
def setup_clipper(): app_name = 'inceptionv3-app' model_name = 'inceptionv3-model' clipper_conn = ClipperConnection(DockerContainerManager()) clipper_conn.connect() pytorch_deployer.deploy_pytorch_model( clipper_conn=clipper_conn, name=model_name, version='1', input_type='bytes', func=incept_predict, pytorch_model=incept, num_replicas=10, batch_size=1, pkgs_to_install=['pillow', 'pyarrow', 'torch', 'torchvision']) clipper_conn.register_application(name=app_name, input_type="bytes", default_output="-1.0", slo_micros=10000000) # 10s clipper_conn.link_model_to_app(app_name=app_name, model_name=model_name) print( "url: ", "http://{addr}/{app_name}/predict".format(addr="", app_name=app_name))
class Clip(object): def __init__(self, sleep_time): from clipper_admin import ClipperConnection, DockerContainerManager #from clipper_admin.deployers import python as python_deployer from clipper_admin.deployers import pytorch as pt_deployer self.clipper_conn = ClipperConnection(DockerContainerManager()) try: self.clipper_conn.connect() self.clipper_conn.stop_all() except Exception: pass self.clipper_conn.start_clipper() self.clipper_conn.register_application(name="hello-world", input_type="strings", default_output="-1.0", slo_micros=10**8) model = Model() def policy(ptmodel, x): batch = (len(x)) arr = [] for j in x: xs, masks = j.split("###") res = np.frombuffer(base64.decodestring(xs), dtype=np.float32) res = res.reshape((64, 19, 19, 3)) res = np.frombuffer(base64.decodestring(masks), dtype=np.float32) res = res.reshape((64, 362)) for i in x: time.sleep(sleep_time) return [ np.random.rand(64).astype(np.float32) for i in range(batch) ] pt_deployer.deploy_pytorch_model(self.clipper_conn, name="policy", version=1, input_type="strings", func=policy, pytorch_model=model) self.clipper_conn.link_model_to_app(app_name="hello-world", model_name="policy")
class Clip(object): def __init__(self, shape, model_name): from clipper_admin import ClipperConnection, DockerContainerManager from clipper_admin.deployers import python as python_deployer from clipper_admin.deployers import pytorch as pytorch_deployer self.clipper_conn = ClipperConnection(DockerContainerManager()) try: self.clipper_conn.connect() self.clipper_conn.stop_all() except Exception: pass self.clipper_conn.start_clipper() self.clipper_conn.register_application(name="hello-world", input_type="strings", default_output="-1.0", slo_micros=10**8) ptmodel = get_model(model_name) def policy(model, x): print(len(x)) batch = (len(x)) arr = [] for j in x: print(type(j), len(j)) res = np.frombuffer(base64.decodestring(j), dtype=np.float32) print(res.shape) arr += [res] x = np.array(arr) x = x.reshape((-1, ) + shape[1:]) print("new shape", x.shape) return evaluate_model(model, x).reshape((batch, shape[0])) pytorch_deployer.deploy_pytorch_model(self.clipper_conn, name="policy", version=1, input_type="strings", func=policy, pytorch_model=ptmodel) self.clipper_conn.link_model_to_app(app_name="hello-world", model_name="policy")
def deployModelToClipper(): """Deploy model to clipper and replace its entry.""" global app_name, model_name, model_version print('Deploying model to clipper, model_name={}, model_version={}'.format( model_name, model_version)) # Setup clipper and deploy model clipper_conn = ClipperConnection(DockerContainerManager(redis_port=6380)) try: clipper_conn.start_clipper() except: clipper_conn.connect() try: # input_type must be bytes as inputs will be serialized into bytes with pickle clipper_conn.register_application(name=app_name, input_type="bytes", default_output="-1.0", slo_micros=1000000) except Exception as e: print(e) try: deploy_python_closure(clipper_conn, name=model_name, version=model_version, input_type="bytes", batch_size=1, func=predict, base_image='hysia-clipper-base-container-gpu') except Exception as e: print(e) try: clipper_conn.link_model_to_app(app_name=app_name, model_name=model_name) except Exception as e: print(e) replaceDefaultEntry() print('{} deployed to clipper!'.format(model_name))
def setup_clipper(): app_name = 'predict-app' model_name = "predict-model" clipper_conn = ClipperConnection(DockerContainerManager()) clipper_conn.connect() deploy_python_closure(clipper_conn, name="predict-model", version='1', input_type="bytes", func=join_predict) clipper_conn.register_application(name=app_name, input_type="bytes", default_output="-1.0", slo_micros=10000000) # 10s clipper_conn.link_model_to_app(app_name=app_name, model_name=model_name) print( "url: ", "http://{addr}/{app_name}/predict".format(addr="", app_name=app_name))
def setup_clipper(): app_name = 'resnet101-app' model_name = 'resnet101-model' clipper_conn = ClipperConnection(DockerContainerManager()) clipper_conn.connect() pytorch_deployer.deploy_pytorch_model(clipper_conn=clipper_conn, name=model_name, version='1', input_type='bytes', func=resnet_predict, pytorch_model=resnet101, pkgs_to_install=['pillow', 'torch', 'torchvision']) clipper_conn.register_application(name=app_name, input_type="bytes", default_output="-1.0", slo_micros=10000000) # 10s clipper_conn.link_model_to_app(app_name=app_name, model_name=model_name) print("query_adress: ", clipper_conn.get_query_addr()) print("app_name: ", ) print("model_name: ", ) print("url: ", "http://{addr}/{app_name}/predict".format(addr=clipper_conn.get_query_addr(),app_name=app_name))
clipper_conn = ClipperConnection(DockerContainerManager()) clipper_conn.start_clipper() clipper_addr = clipper_conn.get_query_addr() def preprocess(inputs): inputArr = (inputs[0]).split(",") floats = inputArr[:-1] rounded = [round(float(i),1) for i in floats] rounded.append(inputArr[-1]) output = [(str(rounded))[1:-1]] return output python_deployer.deploy_python_closure( clipper_conn, name="process-iris", # The name of the model in Clipper version=1, # A unique identifier to assign to this model. input_type="string", # The type of data the model function expects as input func=preprocess # The model function to deploy ) clipper_conn.register_application( name="process-app", input_type="strings", default_output="-1", slo_micros=9000000) #will return default value in 9 seconds clipper_conn.link_model_to_app(app_name="process-app", model_name="process-iris")
import json from datetime import datetime from clipper_admin import ClipperConnection, DockerContainerManager clipper_conn = ClipperConnection(DockerContainerManager()) from thermal_model import get_model_per_zone, normal_schedule, dr_schedule, execute_schedule try: clipper_conn.start_clipper() default_output = json.dumps([-1] * 24) clipper_conn.register_application(name="ciee_thermal", input_type="string", default_output=default_output, slo_micros=1000000000) print 'apps', clipper_conn.get_all_apps() models = get_model_per_zone("2018-01-30 00:00:00 PST") # model parameters: # zone: string # date: string # schedule: [(hsp, csp), ... x 24 ...] def execute_thermal_model(params): """ Accepts list of JSON string as argument """ ret = [] for param in params: args = json.loads(param) zone = args['zone'] date = str(args['date']) schedule = args['schedule'] temps, actions = execute_schedule(date, schedule, models[zone], 65)
from clipper_admin import ClipperConnection, DockerContainerManager import p1_agg_clipper clipper_conn = ClipperConnection(DockerContainerManager()) clipper_conn.start_clipper() clipper_conn.register_application(name="p1app", input_type="strings", default_output="-1.0", slo_micros=100000) clipper_conn.get_all_apps() model_r = p1_agg_clipper.main() def feature_sum(xs): forecast_df = p1_agg_clipper.main() return [str(x) for x in forecast_df.yhat] #return [str(sum(x)) for x in xs] def predict(delay): #delay = eval(delay) future_r = model_r.make_future_dataframe(periods=30, freq='D') forecast_r = model_r.predict(future_r) #forecast_r.index = forecast_r['ds'] #forecast #pred_r = pd.DataFrame(forecast_r['yhat'][len(forecast_r)-delay:len(forecast_r)]) #pred_r=pred_r.reset_index()
inputs = shift(inputs) inputs = torch.tensor(inputs).float() pred = model(inputs) pred = pred.data.numpy() return [str(x) for x in pred] APP_NAME = "test-app" MODEL_NAME = "test-pytorch-model" # Setup clipper and deploy pytorch model clipper_conn = ClipperConnection(DockerContainerManager(redis_port=6380)) try: clipper_conn.start_clipper() clipper_conn.register_application(name=APP_NAME, input_type="doubles", default_output="-1.0", slo_micros=1000000) deploy_pytorch_model(clipper_conn, name=MODEL_NAME, version="1", input_type="doubles", func=predict, pytorch_model=model, pkgs_to_install=pip_deps) clipper_conn.link_model_to_app(app_name=APP_NAME, model_name=MODEL_NAME) except: clipper_conn.connect() # Check all apps print(clipper_conn.get_all_apps())
response = apply_forecast_impl(parameters[0].decode("utf-8").split(',')) code = '202' except Exception as e: response = str(e) code = '500' return [str(code+', '+response) for _ in parameters] if __name__ == '__main__': # setup logging format format = "%(asctime)-15s %(message)s" logging.basicConfig( filename='./timeseries/log.log', level=logging.DEBUG, format=format) # set up logging to console console = logging.StreamHandler(sys.stdout) console.setLevel(logging.ERROR) logging.getLogger().addHandler(console) signal.signal(signal.SIGINT, signal_handler) conn = ClipperConnection(DockerContainerManager()) conn.start_clipper() try: conn.register_application(name="forecast",input_type="strings",default_output="500, Error executing call.",slo_micros=100000000) python_deployer.deploy_python_closure(conn, name="do-forecast", version=1, input_type="strings", func=do_forecast, base_image='wamsiv/timeseries:latest') conn.link_model_to_app(app_name="forecast", model_name='do-forecast') print(subprocess.getoutput(["docker update --restart=on-failure $(docker ps -a | grep 'clipper/query_frontend:0.3.0' | awk '{ print $1 }')"])) input("Server started. Press ctrl+c to stop server.\n") except Exception as e: logging.error("Encountered {}. Stopping server...".format(e)) conn.stop_all() conn.stop_all()
import logging, xgboost as xgb, numpy as np from sklearn.metrics import mean_absolute_error import joblib import pandas as pd from datetime import datetime import pickle import time import matplotlib.pyplot as plt plt.show(block=True) from clipper_admin import ClipperConnection, DockerContainerManager clipper_conn = ClipperConnection(DockerContainerManager()) print("Start Clipper...") clipper_conn.start_clipper() print("Register Clipper application...") clipper_conn.register_application('xgboost-airlines', 'doubles', 'default_pred', 100000) # In[17]: training_examples = pd.read_pickle( "../data/processed/airlines_training_examples.pkl") f1 = open("../data/processed/airlines_training_targets.pkl", 'rb') training_targets = pickle.load(f1) f1.close() test_examples = pd.read_pickle("../data/processed/airlines_test_examples.pkl") def get_train_points(): return training_examples.values.tolist()
def load_from_ckp(): pass def load_from_frozen(): pass if __name__ == "__main__": clipper_conn = ClipperConnection(DockerContainerManager()) clipper_conn.stop_all() clipper_conn.start_clipper() clipper_conn.register_application(name=app_name, input_type="integers", default_output="rabbit", slo_micros=100000) if len(sys.argv) != 3: print("invalid usage") print( "usage: python deploy.py --mode checkpoint|frozen|sess-checkpoint|sess-frozen" ) exit(1) if sys.argv[2] == "sess-checkpoint": sess = load_from_ckp() deploy_tensorflow_model(clipper_conn, model_name, version=1, input_type="integers", func=predict, tf_sess_or_saved_model_path=sess)
model_dir = '.' print( ' app_name="%s"\n model_name="%s"\n input_type="%s"\n slo_micros=%d' % (app_name, model_name, input_type, slo_micros)) clipper_conn = ClipperConnection(DockerContainerManager()) clipper_conn.connect() #----------------------------------------------------------------------- # spark = SparkSession.builder.appName("clipper-pyspark").getOrCreate() # spark.sparkContext.setLogLevel("DEBUG") #----------------------------------------------------------------------- info = clipper_conn.get_app_info(app_name) if (info is None): print(' Registering app %s' % app_name) clipper_conn.register_application(name=app_name, input_type=input_type, default_output="None", slo_micros=slo_micros) version = '1' new_app = True else: if len(info['linked_models']) > 0: model_name = info['linked_models'][0] version = str( int(clipper_conn.get_current_model_version(model_name)) + 1) else: version = '1' new_app = False print(' version: %s' % (version)) #----------------------------------------------------------------------- if args.deploy == 'python':
# In[26]: try: clipper_conn.start_clipper() except: print("Clipper already running") clipper_conn.connect() # In[28]: # We will register it to deploy a simple model. try: clipper_conn.register_application(name='simple_model', input_type='floats', default_output="-1.0", slo_micros=100000) except ClipperException as e: print(str(e)) # In[29]: with open('model.pickle', 'rb') as handle: model = pickle.load(handle) # In[42]: def predict(xs): print("xs: ", xs) res = []
from clipper_admin.exceptions import ClipperException from docker.errors import APIError import json import requests # Set up connection clipper_conn = ClipperConnection(DockerContainerManager()) try: clipper_conn.start_clipper() except (APIError, ClipperException): clipper_conn.connect() # Deploy Sum function clipper_conn.register_application(name="Sum", input_type="doubles", default_output="-1.0", slo_micros=100000) # Define model func def feature_sum(xs): return [str(sum(x)) for x in xs] # Deploy python model python_deployer.deploy_python_closure(clipper_conn, name="sum-model", version=1, input_type="doubles", func=feature_sum)
optimizer.step() # print statistics running_loss += loss.item() if i % 2000 == 1999: # print every 2000 mini-batches print('[%d, %5d] loss: %.3f' % (epoch + 1, i + 1, running_loss / 2000)) running_loss = 0.0 print('Finished Training') # try: clipper_conn = ClipperConnection(KubernetesContainerManager(useInternalIP=True)) # clipper_conn = ClipperConnection(DockerContainerManager()) clipper_conn.start_clipper() clipper_conn.register_application(name="pytorch-example-2", input_type="doubles", default_output="-1.0", slo_micros=100000) # model = nn.Linear(1, 1) # Define a shift function to normalize prediction inputs def pred(model, inputs): preds = [] for i in inputs: preds.append(np.random.rand(1,10)) return preds # return [model(torch.FloatTensor(np.reshape(i, (-1, 3, 32, 32))).data.numpy()) for i in inputs] # return [str(model(torch.FloatTensor(np.reshape(i, (1, 3, 32, 32)))).data.numpy()) for i in inputs] deploy_pytorch_model( clipper_conn, name="pytorch-nn",
clipper_query_port=1337 + node_id, clipper_management_port=2337 + node_id, clipper_rpc_port=7000 + node_id, redis_ip=None, redis_port=6379 + node_id, prometheus_port=9090 + node_id, # WARING: DO NOT CHANGE THE RULE OF NETWORK NAMES docker_network='clipper_network_{}'.format(node_id), # SINCE THIS IS USED BY reset.sh TO IDENTIFY CLIPPER CONTAINERS extra_container_kwargs={}) ) # for node_id in range(args.num_nodes)] try: clipper_conn.start_clipper() clipper_conn.register_application(name="default", input_type="string", default_output="", slo_micros=100000) python_deployer.deploy_python_closure(clipper_conn, name="echo-model", version=1, input_type="string", func=echo_model) clipper_conn.link_model_to_app(app_name="default", model_name="echo-model") except: exit(1) exit(0)
def lin_model_5(inputs): print(inputs) l = [3.1 * x[0] + 1.3 * x[1] - 10.8 for x in inputs] print(l) return l clipper_conn = ClipperConnection( KubernetesContainerManager(useInternalIP=True)) # clipper_conn = ClipperConnection(DockerContainerManager()) clipper_conn.start_clipper() #Deploy lin_model_1 clipper_conn.register_application(name="linear1", input_type="doubles", default_output="-1.0", slo_micros=100000) deploy_python_closure(clipper_conn, name="lin-model-1", version=1, input_type="doubles", func=lin_model_1, registry="hsubbaraj") clipper_conn.link_model_to_app(app_name="linear1", model_name="lin-model-1") print("deployed model 1") #Deploy lin_model_2 clipper_conn.register_application(name="linear2", input_type="doubles", default_output="-1.0", slo_micros=100000)
from clipper_admin import ClipperConnection, KubernetesContainerManager clipper_conn = ClipperConnection( KubernetesContainerManager(useInternalIP=True)) clipper_conn.connect() clipper_conn.register_application(name="hello-world", input_type="doubles", default_output="-1.0", slo_micros=100000) clipper_conn.get_all_apps() def feature_sum(xs): return [str(sum(x)) for x in xs] from clipper_admin.deployers import python as python_deployer registry = 'localhost:5000' python_deployer.deploy_python_closure(clipper_conn, name="sum-model", version=1, input_type="doubles", func=feature_sum, registry=registry)
os.unlink(tmpout.name) return upscaled from clipper_admin import ClipperConnection, DockerContainerManager from clipper_admin.deployers.pytorch import deploy_pytorch_model from torch import nn clipper_conn = ClipperConnection(DockerContainerManager()) clipper_conn.stop_all() # clipper_conn.connect() clipper_conn.start_clipper() clipper_conn.register_application(name="superresolution", input_type="bytes", default_output="undefined", slo_micros=100000) print("going to deploy...") # deploy_pytorch_model( # clipper_conn, # name="superresolution-model", # version=1, # input_type="bytes", # func=image_enhance, # pytorch_model=model, # pkgs_to_install=['opencv-python','numpy','six', 'Pillow', 'wheel', 'certifi'] # ) deploy_pytorch_model(
# In[ ]: try: clipper_conn.start_clipper() except: print("Clipper already running") clipper_conn.connect() # In[ ]: # We will register it to deploy an xgboost model. clipper_conn.register_application('xgboost-test', 'integers', 'default_pred', 100000) # In[ ]: def get_test_point(): return [np.random.randint(255) for _ in range(784)] # In[ ]: # Create a training matrix. dtrain = xgb.DMatrix(get_test_point(), label=[0]) # We then create parameters, watchlist, and specify the number of rounds
# First we need to import Clipper from clipper_admin import ClipperConnection, KubernetesContainerManager from clipper_admin.deployers.python import deploy_python_closure # Create a Clipper connection clipper_conn = ClipperConnection( KubernetesContainerManager(useInternalIP=True, kubernetes_proxy_addr="127.0.0.1:8080")) # Start a Clipper cluster or connect to a running one clipper_conn.start_clipper() # Register an app called 'kddtutorial'. This would create a REST endpoint clipper_conn.register_application(name="kddtutorial", input_type="doubles", default_output="-1.0", slo_micros=10000000) # Access the trained model via closure capture def predict(inputs): global model pred = model.predict(inputs) return [str(p) for p in pred] # Point to the gradient boosting model model = model # Deploy the 'predict' function as a model deploy_python_closure(
# clipper_start from clipper_admin import ClipperConnection, DockerContainerManager clipper_conn = ClipperConnection(DockerContainerManager()) clipper_conn.start_clipper() clipper_conn.connect() clipper_conn.register_application( name="digit", input_type="doubles", default_output="-1.0", slo_micros=10000000) # 10,000,000 micros == 10 sec clipper_conn.get_all_apps() ################################################# ######### Define Own Prediction Function ######## ################################################# import sklearn import numpy as np from sklearn.neural_network import MLPClassifier from sklearn.externals import joblib from clipper_admin.deployers import python as python_deployer for version_postfix in ["10x1k", "10x2k", "20x1k", "15x2k"]: model_path = "../../models/sklearn/" model_name = "dig_nn_model_" + version_postfix + ".sav" clf = joblib.load(model_path + model_name)
from clipper_admin import ClipperConnection, DockerContainerManager from clipper_admin.deployers.pytorch import deploy_pytorch_model clipper_conn = ClipperConnection(DockerContainerManager()) try: clipper_conn.connect() except: clipper_conn.start_clipper() clipper_conn.register_application(name="logo-detector", input_type="strings", default_output="no logo", slo_micros=10000000) clipper_conn.get_all_apps() from pathlib import Path from io import BytesIO import numpy as np from PIL import Image import requests import torch from torch.utils.data import DataLoader import torch.nn as nn import torchvision from torchvision.transforms import transforms DEVICE = torch.device('cuda' if torch.cuda.is_available() else 'cpu') # pylint: disable=maybe-no-member CLASSES = [ 'ADAC', 'FCB', 'HP', 'adidas', 'aldi', 'apple', 'becks', 'bmw', 'carlsberg', 'chimay', 'cocacola', 'corona', 'dhl', 'erdinger', 'esso',