def app(self): import kfserving model = DummyModel("TestModelSeldon") model.load() server = kfserving.KFServer(kfserving.server.SELDON_HTTP_PROTOCOL) server.register_model(model) return server.createApplication()
def app(self): import kfserving model = DummyModel("TestModelSeldon") model.load() server = kfserving.KFServer(Protocol.seldon_http) server.register_model(model) return server.createApplication()
def app(self): # pylint: disable=no-self-use model = CustomTransformer("TestModel", predictor_host='localhost', protocol=Protocol.tensorflow_http) model.load() server = kfserving.KFServer(Protocol.tensorflow_http) server.register_model(model) return server.create_application()
def serve(): model_service = ModelService( name=MODEL_NAME, model_artifact=MODEL_ARTIFACT, ) model_service.load() kfserver = kfserving.KFServer() kfserver.start(models=[model_service])
def main(args): model, device, confidence_estimators, estimator_filenames, bootleg_annotator = init( args) model_server = KFModelServer(args.inference_name, args, model.numericalizer, model, device, confidence_estimators, estimator_filenames, bootleg_annotator) model_server.load() kfserving.KFServer(workers=1).start([model_server])
def main(): args, extra = parse_args(sys.argv[1:]) # Pretrained Alibi explainer alibi_model = None if args.storage_uri is not None: alibi_model = os.path.join( kfserving.Storage.download(args.storage_uri), EXPLAINER_FILENAME) with open(alibi_model, "rb") as f: logging.info("Loading Alibi model") alibi_model = dill.load(f) explainer = AlibiExplainer( args.model_name, args.predictor_host, ExplainerMethod(args.command), extra, alibi_model, ) explainer.load() kfserving.KFServer().start(models=[explainer])
def main(): parser = configure_arg_parser() args, _ = parser.parse_known_args() size = os.environ['MODEL_INPUT_SIZE'] size_h, size_w = size.split(',') size_h = int(size_h) size_w = int(size_w) keras_model = os.environ['KERAS_MODEL_NAME'] labels = os.environ['MODEL_LABELS'].split(',') preprocessor = create_preprocessor(keras_model, target_size=(size_w, size_w)) transformer = ImageTransformer(args.model_name, predictor_host=args.predictor_host, preprocessor=preprocessor, labels=labels) kfserver = kfserving.KFServer() kfserver.start(models=[transformer])
def app(self): # pylint: disable=no-self-use model = DummyModel("TestModelSeldon") model.load() server = kfserving.KFServer(Protocol.seldon_http) server.register_model(model) return server.createApplication()
DEFAULT_LOCAL_MODEL_DIR = "/tmp/model" DEFAULT_NTHREAD = 1 parser = argparse.ArgumentParser(parents=[kfserving.kfserver.parser]) # pylint:disable=c-extension-no-member parser.add_argument('--model_dir', required=True, help='A URI pointer to the model directory') parser.add_argument('--model_name', default=DEFAULT_MODEL_NAME, help='The name that the model is served under.') parser.add_argument('--nthread', default=DEFAULT_NTHREAD, help='Number of threads to use by LightGBM.') args, _ = parser.parse_known_args() if __name__ == "__main__": model = LightGBMModel(args.model_name, args.model_dir, args.nthread) try: model.load() except Exception as e: ex_type, ex_value = sys.exc_info()[:2] logging.error( f"fail to load model {args.model_name} from dir {args.model_dir}. " f"exception type {ex_type}, exception msg: {ex_value}") model_repository = LightGBMModelRepository(args.model_dir, args.nthread) # LightGBM doesn't support multi-process, so the number of http server workers should be 1. kfserver = kfserving.KFServer(workers=1, registered_models=model_repository) # pylint:disable=c-extension-no-member kfserver.start([model] if model.ready else [])
# Assign key value pair to dict # strip() removes white space from the ends of strings keys[name.strip()] = value.strip() keys["model_snapshot"] = json.loads(keys["model_snapshot"]) models = keys["model_snapshot"]["models"] model_names = [] # Get all the model_names for model, value in models.items(): model_names.append(model) if not model_names: model_names = [DEFAULT_MODEL_NAME] print(f"Wrapper : Model names {model_names}") return model_names if __name__ == "__main__": model_names = parse_config() models = [] for model_name in model_names: transformer = ImageTransformer(model_name, predictor_host=args.predictor_host) models.append(transformer) kfserving.KFServer( registered_models=TransformerModelRepository(args.predictor_host), http_port=8080, ).start(models=models)
# limitations under the License. import kfserving import argparse from tfserver import TFModel DEFAULT_MODEL_NAME = "model" DEFAULT_LOCAL_MODEL_DIR = "/tmp/model" parser = argparse.ArgumentParser(parents=[kfserving.kfserver.parser]) parser.add_argument('--model_dir', required=True, help='The path of the model directory') parser.add_argument('--model_name', default=DEFAULT_MODEL_NAME, help='The name that the model is served under.') parser.add_argument('--input_name', required=True, help='The model of input layer name.') parser.add_argument('--output_name', required=True, help='The name of output layer name.') args, _ = parser.parse_known_args() if __name__ == "__main__": model = TFModel(args.model_name, args.model_dir, args.input_name, args.output_name) model.load() kfserving.KFServer().start([model])
# # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import kfserving import argparse from xgbserver import XGBoostModel DEFAULT_MODEL_NAME = "default" DEFAULT_LOCAL_MODEL_DIR = "/tmp/model" parser = argparse.ArgumentParser(parents=[kfserving.server.parser]) #pylint:disable=c-extension-no-member parser.add_argument('--model_dir', required=True, help='A URI pointer to the model directory') parser.add_argument('--model_name', default=DEFAULT_MODEL_NAME, help='The name that the model is served under.') args, _ = parser.parse_known_args() if __name__ == "__main__": model = XGBoostModel(args.model_name, args.model_dir) model.load() kfserving.KFServer().start([model]) #pylint:disable=c-extension-no-member
self.service_name = service_name self.ready = False def load(self) -> None: pass def predict(self, request: Dict) -> Dict: response = request response["echo"] = "Hello from {}".format(self.service_name) return response if __name__ == "__main__": service_name = "kf-serving-default" host_name = os.environ.get('HOSTNAME') if host_name is not None: x = re.compile(r'(kfserving-\d+)').search(host_name) if x is not None: service_name = x[0] dummy_mirror_service = DummyMirrorService(service_name) dummy_mirror_service.load() kfserving.KFServer(workers=1).start([dummy_mirror_service])
answer_start = tf.argmax(answer_start_scores, axis=1).numpy( )[0] # Get the most likely beginning of answer with the argmax of the score answer_end = (tf.argmax(answer_end_scores, axis=1) + 1).numpy( )[0] # Get the most likely end of answer with the argmax of the score answer = self.tokenizer.convert_tokens_to_string( self.tokenizer.convert_ids_to_tokens( input_ids[answer_start:answer_end])) print(f"Question: {question}") print(f"Answer: {answer}\n") results[question] = answer #output = self.model(input_batch) #scores = torch.nn.functional.softmax(output, dim=1)[0] #_, top_5 = torch.topk(output, 5) #for idx in top_5[0]: # results[self.classes[idx]] = scores[idx].item() return {"predictions": results} if __name__ == "__main__": model = KFServing_BERT_QA_Model("kfserving-custom-model") model.load() kfserving.KFServer(workers=1).start([model])
"instances": model_inputs.tolist(), "token": inputs["token"] } logging.info("token =======> %s", str(inputs["token"])) return payload def postprocess(self, predictions: List) -> List: output = torch.tensor(predictions['predictions']) samples = (output).permute( 1, 0, 2, 3) * 10.0 #We found that this factor can speed up training samples = samples * stdTorch + meanTorch means = torch.mean(samples, dim=0, keepdim=True)[0, ...] # Sum up over all samples means = means.cpu().detach().numpy() means.shape = (output.shape[2], output.shape[3]) plt.imsave('/tmp/out.png', means) with open('/tmp/out.png', 'rb') as open_file: byte_content = open_file.read() base64_bytes = base64.b64encode(byte_content) base64_string = base64_bytes.decode('utf-8') logging.info("prep =======> %s", str(means.shape)) return {'image': base64_string, 'p_class': ""} if __name__ == "__main__": transformer = ImageTransformer(args.model_name, predictor_host=args.predictor_host) kfserver = kfserving.KFServer() kfserver.start(models=[transformer])
def run(): model = KFServer(Serve.SERVICE_NAME) model.load() kfserving.KFServer(workers=1).start([model])
import kfserving import argparse from transformer import Transformer parser = argparse.ArgumentParser(parents=[kfserving.kfserver.parser]) parser.add_argument('--model_name', default="model", help='The name that the model is served under.') parser.add_argument('--predictor_host', help='The URL for the model predict function', required=True) args, _ = parser.parse_known_args() if __name__ == "__main__": transformer = Transformer(args.model_name, predictor_host=args.predictor_host) kfserver = kfserving.KFServer(workers=4) kfserver.start(models=[transformer])
def main(args): model_server = KFModelServer(args) kfserving.KFServer(workers=1).start([model_server])
import logging import sys import kfserving from sklearnserver import SKLearnModel, SKLearnModelRepository DEFAULT_MODEL_NAME = "model" DEFAULT_LOCAL_MODEL_DIR = "/tmp/model" parser = argparse.ArgumentParser(parents=[kfserving.kfserver.parser]) parser.add_argument('--model_dir', required=True, help='A URI pointer to the model binary') parser.add_argument('--model_name', default=DEFAULT_MODEL_NAME, help='The name that the model is served under.') args, _ = parser.parse_known_args() if __name__ == "__main__": model = SKLearnModel(args.model_name, args.model_dir) try: model.load() except Exception: ex_type, ex_value, _ = sys.exc_info() logging.error( f"fail to load model {args.model_name} from dir {args.model_dir}. " f"exception type {ex_type}, exception msg: {ex_value}") model.ready = False kfserving.KFServer(registered_models=SKLearnModelRepository( args.model_dir)).start([model] if model.ready else [])
def load_classes_names(file_name): names = {} with open(file_name) as f: for id, name in enumerate(f): names[id] = name return names start_time = time.time() boxes, classes, scores = handle_predictions(request["predictions"][0],confidence=0.3,iou_threshold=0.5) class_names = load_classes_names(os.path.join(self.base_path,"metadata", FLAGS.classes_file)) classs=[] for key in classes: classs.append(class_names[key].strip()) stop_time = time.time() logging.info('post process time: {:.3f}s'.format((stop_time - start_time))) return {"predictions": [boxes.tolist(), classs, scores.tolist()]} if __name__ == "__main__": parser = argparse.ArgumentParser() parser.add_argument('--http_port', default=8080, type=int, help='The HTTP Port listened to by the model server.') parser.add_argument('--out_dir', default="model", help='out dir') parser.add_argument('--model-name', type=str, help='model name') parser.add_argument('--classes_file', default="voc.names", type=str, help='name of the class file') FLAGS, _ = parser.parse_known_args() model = KFServing(FLAGS.model_name) model.load() kfserving.KFServer(http_port=FLAGS.http_port).start([model])
DEFAULT_MODEL_PATH = './model' parser = argparse.ArgumentParser(parents=[kfserving.kfserver.parser]) parser.add_argument('--model_name', default=DEFAULT_MODEL_NAME, help='The name that the model is served under.') parser.add_argument('--influx_host', default=DEFAULT_INFLUX_HOST, help='InfluxDB Host') parser.add_argument('--influx_port', default=DEFAULT_INFLUX_PORT, help='InfluxDB Port') parser.add_argument('--influx_database', default=DEFAULT_INFLUX_DATABASE, help='InfluxDB Database') parser.add_argument('--model_path', default=DEFAULT_MODEL_PATH, help='Concept Drift Model Path') args, _ = parser.parse_known_args() if __name__ == "__main__": model = ConceptDriftModel(args.model_name, influx_host=args.influx_host, influx_port=args.influx_port, database=args.influx_database, model_path=args.model_path) model.load() server = kfserving.KFServer( http_port=8080, max_buffer_size=9223372036854775807, workers=1) logging.info("BUFFER SIZE %s", server.max_buffer_size) server.start(models=[model])
DEFAULT_MODEL_NAME = "default" DEFAULT_LOCAL_MODEL_DIR = "/tmp/model" DEFAULT_NTHREAD = 1 parser = argparse.ArgumentParser(parents=[kfserving.kfserver.parser]) # pylint:disable=c-extension-no-member parser.add_argument('--model_dir', required=True, help='A URI pointer to the model directory') parser.add_argument('--model_name', default=DEFAULT_MODEL_NAME, help='The name that the model is served under.') parser.add_argument('--nthread', default=DEFAULT_NTHREAD, help='Number of threads to use by XGBoost.') args, _ = parser.parse_known_args() if __name__ == "__main__": model = XGBoostModel(args.model_name, args.model_dir, args.nthread) try: model.load() except Exception as e: ex_type, ex_value, _ = sys.exc_info() logging.error( f"fail to load model {args.model_name} from dir {args.model_dir}. " f"exception type {ex_type}, exception msg: {ex_value}") model.ready = False kfserving.KFServer(registered_models=XGBoostModelRepository( args.model_dir, args.nthread)).start([model] if model.ready else []) # pylint:disable=c-extension-no-member
help="The URL for the model predict function", required=True) parser.add_argument('--model_name', default=DEFAULT_MODEL_NAME, help='The name that the model is served under.') parser.add_argument('--influx_host', default=DEFAULT_INFLUX_HOST, help='InfluxDB Host') parser.add_argument('--influx_port', default=DEFAULT_INFLUX_PORT, help='InfluxDB Port') parser.add_argument('--influx_database', default=DEFAULT_INFLUX_DATABASE, help='InfluxDB Database') args, _ = parser.parse_known_args() if __name__ == "__main__": transformer = RecommenderTransformer(args.model_name, predictor_host=args.predictor_host, influx_host=args.influx_host, influx_port=args.influx_port, database=args.influx_database) server = kfserving.KFServer(registered_models=TransformerModelRepository( args.predictor_host), http_port=8080, max_buffer_size=9223372036854775807, workers=1) logging.info("BUFFER SIZE %s", server.max_buffer_size) server.start(models=[transformer])
if np.ndim(X) != 2: for i in range(len(X)): input[feature_cols[i]].append(X[i]) else: for i in range(len(X)): for j in range(len(X[i])): input[feature_cols[j]].append(X[i][j]) # Open a Session to predict with tf.Session() as sess: tf.saved_model.loader.load(sess, [tf.saved_model.tag_constants.SERVING], exported_path) predictor= tf.contrib.predictor.from_saved_model(exported_path,signature_def_key='predict') output_dict= predictor(input) sess.close() class_ids=[] for id in output_dict["class_ids"]: class_ids.append(id[0]) #return {"predictions":np.array(class_ids).tolist()} return {"predictions":output_dict["probabilities"].tolist()} if __name__ == "__main__": parser = argparse.ArgumentParser() parser.add_argument('--http_port', default=8080, type=int, help='The HTTP Port listened to by the model server.') parser.add_argument('--storage_uri', help='storage uri for your model') parser.add_argument('--out_dir', help='out dir') args, _ = parser.parse_known_args() model = KFServing("blerssi-model") model.load() kfserving.KFServer(http_port=args.http_port).start([model])
def app(self): # pylint: disable=no-self-use model = DummyModel("TestModel") model.load() server = kfserving.KFServer() server.register_model(model) return server.create_application()
if not model_names: model_names = [DEFAULT_MODEL_NAME] if not management_address: management_address = DEFAULT_MANAGEMENT_ADDRESS if not model_store: model_store = DEFAULT_MODEL_STORE logging.info( "Wrapper : Model names %s, inference address %s, management address %s, model store %s", model_names, inference_address, management_address, model_store) return model_names, inference_address, management_address, model_store if __name__ == "__main__": model_names, inference_address, management_address, model_dir = parse_config( ) models = [] for model_name in model_names: model = TorchserveModel(model_name, inference_address, management_address, model_dir) models.append(model) kfserving.KFServer( registered_models=TSModelRepository(inference_address, management_address, model_dir), http_port=8080, ).start(models)
# Input follows the Tensorflow V1 HTTP API for binary values # https://www.tensorflow.org/tfx/serving/api_rest#encoding_binary_values data = inputs[0]["image"]["b64"] raw_img_data = base64.b64decode(data) input_image = Image.open(io.BytesIO(raw_img_data)) preprocess = transforms.Compose([ transforms.Resize(256), transforms.CenterCrop(224), transforms.ToTensor(), transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]), ]) input_tensor = preprocess(input_image) input_batch = input_tensor.unsqueeze(0) output = self.model(input_batch) torch.nn.functional.softmax(output, dim=1)[0] values, top_5 = torch.topk(output, 5) return {"predictions": values.tolist()} if __name__ == "__main__": kfserving.KFServer(workers=1).start({"custom-model": AlexNetModel})
DEFAULT_MAX_ITER = "1000" DEFAULT_NB_CLASSES = "10" parser = argparse.ArgumentParser(parents=[kfserving.kfserver.parser]) parser.add_argument('--model_name', default=DEFAULT_MODEL_NAME, help='The name that the model is served under.') parser.add_argument('--adversary_type', default=DEFAULT_ADVERSARY_TYPE, help='What type of adversarial tool to use.') parser.add_argument('--max_iter', default=DEFAULT_MAX_ITER, help='The max number of iterations to run.') parser.add_argument('--nb_classes', default=DEFAULT_NB_CLASSES, help='The number of different classification types.') parser.add_argument('--predictor_host', help='The host for the predictor', required=True) args, _ = parser.parse_known_args() if __name__ == "__main__": model = ARTModel(args.model_name, args.predictor_host, adversary_type=args.adversary_type, nb_classes=args.nb_classes, max_iter=args.max_iter) model.load() kfserving.KFServer().start([model], nest_asyncio=True)
help='Explainer method', required=True) parser.add_argument('--explainerUri', help='The URL of a pretrained explainer', default=os.environ.get(ENV_STORAGE_URI)) parser.add_argument('--config', default=os.environ.get(CONFIG_ENV), help='Custom configuration parameters') args, _ = parser.parse_known_args() if __name__ == "__main__": # Pretrained Alibi explainer alibi_model = None if args.explainerUri is not None: alibi_model = os.path.join( kfserving.Storage.download(args.explainerUri), EXPLAINER_FILENAME) with open(alibi_model, 'rb') as f: alibi_model = dill.load(f) # Custom configuration if args.config is None: config = {} else: config = json.loads(args.config) explainer = AlibiExplainer(args.explainer_name, args.predict_url, args.protocol, ExplainerMethod(args.type), config, alibi_model) explainer.load() kfserving.KFServer().start(models=[explainer])
import argparse from alibiexplainer import AlibiExplainer from alibiexplainer.explainer import ExplainerMethod #pylint:disable=no-name-in-module DEFAULT_MODEL_NAME = "model" parser = argparse.ArgumentParser(parents=[kfserving.server.parser]) #pylint:disable=c-extension-no-member parser.add_argument('--model_name', default=DEFAULT_MODEL_NAME, help='The name that the model is served under.') parser.add_argument('--predict_url', help='The URL for the model predict function', required=True) parser.add_argument('--method', type=ExplainerMethod, choices=list(ExplainerMethod), default="anchor_tabular", help='Explainer method') parser.add_argument('--training_data', help='The URL for the training data') args, _ = parser.parse_known_args() if __name__ == "__main__": explainer = AlibiExplainer(args.model_name, args.predict_url, args.protocol, ExplainerMethod(args.method), training_data_url=args.training_data) explainer.load() kfserving.KFServer().start(models=[explainer]) #pylint:disable=c-extension-no-member