def app(self):
import kfserving
model = DummyModel("TestModelSeldon")
model.load()
server = kfserving.KFServer(kfserving.server.SELDON_HTTP_PROTOCOL)
server.register_model(model)
return server.createApplication()
def app(self):
import kfserving
model = DummyModel("TestModelSeldon")
model.load()
server = kfserving.KFServer(Protocol.seldon_http)
server.register_model(model)
return server.createApplication()
def app(self): # pylint: disable=no-self-use
model = CustomTransformer("TestModel",
predictor_host='localhost',
protocol=Protocol.tensorflow_http)
model.load()
server = kfserving.KFServer(Protocol.tensorflow_http)
server.register_model(model)
return server.create_application()
def serve():
model_service = ModelService(
name=MODEL_NAME,
model_artifact=MODEL_ARTIFACT,
)
model_service.load()
kfserver = kfserving.KFServer()
kfserver.start(models=[model_service])
def main(args):
model, device, confidence_estimators, estimator_filenames, bootleg_annotator = init(
args)
model_server = KFModelServer(args.inference_name, args,
model.numericalizer, model, device,
confidence_estimators, estimator_filenames,
bootleg_annotator)
model_server.load()
kfserving.KFServer(workers=1).start([model_server])
def main():
args, extra = parse_args(sys.argv[1:])
# Pretrained Alibi explainer
alibi_model = None
if args.storage_uri is not None:
alibi_model = os.path.join(
kfserving.Storage.download(args.storage_uri), EXPLAINER_FILENAME)
with open(alibi_model, "rb") as f:
logging.info("Loading Alibi model")
alibi_model = dill.load(f)
explainer = AlibiExplainer(
args.model_name,
args.predictor_host,
ExplainerMethod(args.command),
extra,
alibi_model,
)
explainer.load()
kfserving.KFServer().start(models=[explainer])
def main():
parser = configure_arg_parser()
args, _ = parser.parse_known_args()
size = os.environ['MODEL_INPUT_SIZE']
size_h, size_w = size.split(',')
size_h = int(size_h)
size_w = int(size_w)
keras_model = os.environ['KERAS_MODEL_NAME']
labels = os.environ['MODEL_LABELS'].split(',')
preprocessor = create_preprocessor(keras_model,
target_size=(size_w, size_w))
transformer = ImageTransformer(args.model_name,
predictor_host=args.predictor_host,
preprocessor=preprocessor,
labels=labels)
kfserver = kfserving.KFServer()
kfserver.start(models=[transformer])
def app(self): # pylint: disable=no-self-use
model = DummyModel("TestModelSeldon")
model.load()
server = kfserving.KFServer(Protocol.seldon_http)
server.register_model(model)
return server.createApplication()
DEFAULT_LOCAL_MODEL_DIR = "/tmp/model"
DEFAULT_NTHREAD = 1
parser = argparse.ArgumentParser(parents=[kfserving.kfserver.parser]) # pylint:disable=c-extension-no-member
parser.add_argument('--model_dir',
required=True,
help='A URI pointer to the model directory')
parser.add_argument('--model_name',
default=DEFAULT_MODEL_NAME,
help='The name that the model is served under.')
parser.add_argument('--nthread',
default=DEFAULT_NTHREAD,
help='Number of threads to use by LightGBM.')
args, _ = parser.parse_known_args()
if __name__ == "__main__":
model = LightGBMModel(args.model_name, args.model_dir, args.nthread)
try:
model.load()
except Exception as e:
ex_type, ex_value = sys.exc_info()[:2]
logging.error(
f"fail to load model {args.model_name} from dir {args.model_dir}. "
f"exception type {ex_type}, exception msg: {ex_value}")
model_repository = LightGBMModelRepository(args.model_dir, args.nthread)
# LightGBM doesn't support multi-process, so the number of http server workers should be 1.
kfserver = kfserving.KFServer(workers=1,
registered_models=model_repository) # pylint:disable=c-extension-no-member
kfserver.start([model] if model.ready else [])
# Assign key value pair to dict
# strip() removes white space from the ends of strings
keys[name.strip()] = value.strip()
keys["model_snapshot"] = json.loads(keys["model_snapshot"])
models = keys["model_snapshot"]["models"]
model_names = []
# Get all the model_names
for model, value in models.items():
model_names.append(model)
if not model_names:
model_names = [DEFAULT_MODEL_NAME]
print(f"Wrapper : Model names {model_names}")
return model_names
if __name__ == "__main__":
model_names = parse_config()
models = []
for model_name in model_names:
transformer = ImageTransformer(model_name,
predictor_host=args.predictor_host)
models.append(transformer)
kfserving.KFServer(
registered_models=TransformerModelRepository(args.predictor_host),
http_port=8080,
).start(models=models)
# limitations under the License.
import kfserving
import argparse
from tfserver import TFModel
DEFAULT_MODEL_NAME = "model"
DEFAULT_LOCAL_MODEL_DIR = "/tmp/model"
parser = argparse.ArgumentParser(parents=[kfserving.kfserver.parser])
parser.add_argument('--model_dir',
required=True,
help='The path of the model directory')
parser.add_argument('--model_name',
default=DEFAULT_MODEL_NAME,
help='The name that the model is served under.')
parser.add_argument('--input_name',
required=True,
help='The model of input layer name.')
parser.add_argument('--output_name',
required=True,
help='The name of output layer name.')
args, _ = parser.parse_known_args()
if __name__ == "__main__":
model = TFModel(args.model_name, args.model_dir, args.input_name,
args.output_name)
model.load()
kfserving.KFServer().start([model])
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import kfserving
import argparse
from xgbserver import XGBoostModel
DEFAULT_MODEL_NAME = "default"
DEFAULT_LOCAL_MODEL_DIR = "/tmp/model"
parser = argparse.ArgumentParser(parents=[kfserving.server.parser]) #pylint:disable=c-extension-no-member
parser.add_argument('--model_dir',
required=True,
help='A URI pointer to the model directory')
parser.add_argument('--model_name',
default=DEFAULT_MODEL_NAME,
help='The name that the model is served under.')
args, _ = parser.parse_known_args()
if __name__ == "__main__":
model = XGBoostModel(args.model_name, args.model_dir)
model.load()
kfserving.KFServer().start([model]) #pylint:disable=c-extension-no-member
self.service_name = service_name
self.ready = False
def load(self) -> None:
pass
def predict(self, request: Dict) -> Dict:
response = request
response["echo"] = "Hello from {}".format(self.service_name)
return response
if __name__ == "__main__":
service_name = "kf-serving-default"
host_name = os.environ.get('HOSTNAME')
if host_name is not None:
x = re.compile(r'(kfserving-\d+)').search(host_name)
if x is not None:
service_name = x[0]
dummy_mirror_service = DummyMirrorService(service_name)
dummy_mirror_service.load()
kfserving.KFServer(workers=1).start([dummy_mirror_service])
answer_start = tf.argmax(answer_start_scores, axis=1).numpy(
)[0] # Get the most likely beginning of answer with the argmax of the score
answer_end = (tf.argmax(answer_end_scores, axis=1) + 1).numpy(
)[0] # Get the most likely end of answer with the argmax of the score
answer = self.tokenizer.convert_tokens_to_string(
self.tokenizer.convert_ids_to_tokens(
input_ids[answer_start:answer_end]))
print(f"Question: {question}")
print(f"Answer: {answer}\n")
results[question] = answer
#output = self.model(input_batch)
#scores = torch.nn.functional.softmax(output, dim=1)[0]
#_, top_5 = torch.topk(output, 5)
#for idx in top_5[0]:
# results[self.classes[idx]] = scores[idx].item()
return {"predictions": results}
if __name__ == "__main__":
model = KFServing_BERT_QA_Model("kfserving-custom-model")
model.load()
kfserving.KFServer(workers=1).start([model])
"instances": model_inputs.tolist(),
"token": inputs["token"]
}
logging.info("token =======> %s", str(inputs["token"]))
return payload
def postprocess(self, predictions: List) -> List:
output = torch.tensor(predictions['predictions'])
samples = (output).permute(
1, 0, 2,
3) * 10.0 #We found that this factor can speed up training
samples = samples * stdTorch + meanTorch
means = torch.mean(samples, dim=0,
keepdim=True)[0, ...] # Sum up over all samples
means = means.cpu().detach().numpy()
means.shape = (output.shape[2], output.shape[3])
plt.imsave('/tmp/out.png', means)
with open('/tmp/out.png', 'rb') as open_file:
byte_content = open_file.read()
base64_bytes = base64.b64encode(byte_content)
base64_string = base64_bytes.decode('utf-8')
logging.info("prep =======> %s", str(means.shape))
return {'image': base64_string, 'p_class': ""}
if __name__ == "__main__":
transformer = ImageTransformer(args.model_name,
predictor_host=args.predictor_host)
kfserver = kfserving.KFServer()
kfserver.start(models=[transformer])
def run():
model = KFServer(Serve.SERVICE_NAME)
model.load()
kfserving.KFServer(workers=1).start([model])
import kfserving
import argparse
from transformer import Transformer
parser = argparse.ArgumentParser(parents=[kfserving.kfserver.parser])
parser.add_argument('--model_name',
default="model",
help='The name that the model is served under.')
parser.add_argument('--predictor_host',
help='The URL for the model predict function',
required=True)
args, _ = parser.parse_known_args()
if __name__ == "__main__":
transformer = Transformer(args.model_name,
predictor_host=args.predictor_host)
kfserver = kfserving.KFServer(workers=4)
kfserver.start(models=[transformer])
def main(args):
model_server = KFModelServer(args)
kfserving.KFServer(workers=1).start([model_server])
import logging
import sys
import kfserving
from sklearnserver import SKLearnModel, SKLearnModelRepository
DEFAULT_MODEL_NAME = "model"
DEFAULT_LOCAL_MODEL_DIR = "/tmp/model"
parser = argparse.ArgumentParser(parents=[kfserving.kfserver.parser])
parser.add_argument('--model_dir',
required=True,
help='A URI pointer to the model binary')
parser.add_argument('--model_name',
default=DEFAULT_MODEL_NAME,
help='The name that the model is served under.')
args, _ = parser.parse_known_args()
if __name__ == "__main__":
model = SKLearnModel(args.model_name, args.model_dir)
try:
model.load()
except Exception:
ex_type, ex_value, _ = sys.exc_info()
logging.error(
f"fail to load model {args.model_name} from dir {args.model_dir}. "
f"exception type {ex_type}, exception msg: {ex_value}")
model.ready = False
kfserving.KFServer(registered_models=SKLearnModelRepository(
args.model_dir)).start([model] if model.ready else [])
def load_classes_names(file_name):
names = {}
with open(file_name) as f:
for id, name in enumerate(f):
names[id] = name
return names
start_time = time.time()
boxes, classes, scores = handle_predictions(request["predictions"][0],confidence=0.3,iou_threshold=0.5)
class_names = load_classes_names(os.path.join(self.base_path,"metadata", FLAGS.classes_file))
classs=[]
for key in classes:
classs.append(class_names[key].strip())
stop_time = time.time()
logging.info('post process time: {:.3f}s'.format((stop_time - start_time)))
return {"predictions": [boxes.tolist(), classs, scores.tolist()]}
if __name__ == "__main__":
parser = argparse.ArgumentParser()
parser.add_argument('--http_port', default=8080, type=int,
help='The HTTP Port listened to by the model server.')
parser.add_argument('--out_dir', default="model", help='out dir')
parser.add_argument('--model-name', type=str, help='model name')
parser.add_argument('--classes_file', default="voc.names", type=str, help='name of the class file')
FLAGS, _ = parser.parse_known_args()
model = KFServing(FLAGS.model_name)
model.load()
kfserving.KFServer(http_port=FLAGS.http_port).start([model])
DEFAULT_MODEL_PATH = './model'
parser = argparse.ArgumentParser(parents=[kfserving.kfserver.parser])
parser.add_argument('--model_name', default=DEFAULT_MODEL_NAME,
help='The name that the model is served under.')
parser.add_argument('--influx_host', default=DEFAULT_INFLUX_HOST,
help='InfluxDB Host')
parser.add_argument('--influx_port', default=DEFAULT_INFLUX_PORT,
help='InfluxDB Port')
parser.add_argument('--influx_database', default=DEFAULT_INFLUX_DATABASE,
help='InfluxDB Database')
parser.add_argument('--model_path', default=DEFAULT_MODEL_PATH,
help='Concept Drift Model Path')
args, _ = parser.parse_known_args()
if __name__ == "__main__":
model = ConceptDriftModel(args.model_name, influx_host=args.influx_host,
influx_port=args.influx_port,
database=args.influx_database,
model_path=args.model_path)
model.load()
server = kfserving.KFServer(
http_port=8080,
max_buffer_size=9223372036854775807,
workers=1)
logging.info("BUFFER SIZE %s", server.max_buffer_size)
server.start(models=[model])
DEFAULT_MODEL_NAME = "default"
DEFAULT_LOCAL_MODEL_DIR = "/tmp/model"
DEFAULT_NTHREAD = 1
parser = argparse.ArgumentParser(parents=[kfserving.kfserver.parser]) # pylint:disable=c-extension-no-member
parser.add_argument('--model_dir',
required=True,
help='A URI pointer to the model directory')
parser.add_argument('--model_name',
default=DEFAULT_MODEL_NAME,
help='The name that the model is served under.')
parser.add_argument('--nthread',
default=DEFAULT_NTHREAD,
help='Number of threads to use by XGBoost.')
args, _ = parser.parse_known_args()
if __name__ == "__main__":
model = XGBoostModel(args.model_name, args.model_dir, args.nthread)
try:
model.load()
except Exception as e:
ex_type, ex_value, _ = sys.exc_info()
logging.error(
f"fail to load model {args.model_name} from dir {args.model_dir}. "
f"exception type {ex_type}, exception msg: {ex_value}")
model.ready = False
kfserving.KFServer(registered_models=XGBoostModelRepository(
args.model_dir, args.nthread)).start([model] if model.ready else []) # pylint:disable=c-extension-no-member
help="The URL for the model predict function",
required=True)
parser.add_argument('--model_name',
default=DEFAULT_MODEL_NAME,
help='The name that the model is served under.')
parser.add_argument('--influx_host',
default=DEFAULT_INFLUX_HOST,
help='InfluxDB Host')
parser.add_argument('--influx_port',
default=DEFAULT_INFLUX_PORT,
help='InfluxDB Port')
parser.add_argument('--influx_database',
default=DEFAULT_INFLUX_DATABASE,
help='InfluxDB Database')
args, _ = parser.parse_known_args()
if __name__ == "__main__":
transformer = RecommenderTransformer(args.model_name,
predictor_host=args.predictor_host,
influx_host=args.influx_host,
influx_port=args.influx_port,
database=args.influx_database)
server = kfserving.KFServer(registered_models=TransformerModelRepository(
args.predictor_host),
http_port=8080,
max_buffer_size=9223372036854775807,
workers=1)
logging.info("BUFFER SIZE %s", server.max_buffer_size)
server.start(models=[transformer])
if np.ndim(X) != 2:
for i in range(len(X)):
input[feature_cols[i]].append(X[i])
else:
for i in range(len(X)):
for j in range(len(X[i])):
input[feature_cols[j]].append(X[i][j])
# Open a Session to predict
with tf.Session() as sess:
tf.saved_model.loader.load(sess, [tf.saved_model.tag_constants.SERVING], exported_path)
predictor= tf.contrib.predictor.from_saved_model(exported_path,signature_def_key='predict')
output_dict= predictor(input)
sess.close()
class_ids=[]
for id in output_dict["class_ids"]:
class_ids.append(id[0])
#return {"predictions":np.array(class_ids).tolist()}
return {"predictions":output_dict["probabilities"].tolist()}
if __name__ == "__main__":
parser = argparse.ArgumentParser()
parser.add_argument('--http_port', default=8080, type=int,
help='The HTTP Port listened to by the model server.')
parser.add_argument('--storage_uri', help='storage uri for your model')
parser.add_argument('--out_dir', help='out dir')
args, _ = parser.parse_known_args()
model = KFServing("blerssi-model")
model.load()
kfserving.KFServer(http_port=args.http_port).start([model])
def app(self): # pylint: disable=no-self-use
model = DummyModel("TestModel")
model.load()
server = kfserving.KFServer()
server.register_model(model)
return server.create_application()
if not model_names:
model_names = [DEFAULT_MODEL_NAME]
if not management_address:
management_address = DEFAULT_MANAGEMENT_ADDRESS
if not model_store:
model_store = DEFAULT_MODEL_STORE
logging.info(
"Wrapper : Model names %s, inference address %s, management address %s, model store %s",
model_names, inference_address, management_address, model_store)
return model_names, inference_address, management_address, model_store
if __name__ == "__main__":
model_names, inference_address, management_address, model_dir = parse_config(
)
models = []
for model_name in model_names:
model = TorchserveModel(model_name, inference_address,
management_address, model_dir)
models.append(model)
kfserving.KFServer(
registered_models=TSModelRepository(inference_address,
management_address, model_dir),
http_port=8080,
).start(models)
# Input follows the Tensorflow V1 HTTP API for binary values
# https://www.tensorflow.org/tfx/serving/api_rest#encoding_binary_values
data = inputs[0]["image"]["b64"]
raw_img_data = base64.b64decode(data)
input_image = Image.open(io.BytesIO(raw_img_data))
preprocess = transforms.Compose([
transforms.Resize(256),
transforms.CenterCrop(224),
transforms.ToTensor(),
transforms.Normalize(mean=[0.485, 0.456, 0.406],
std=[0.229, 0.224, 0.225]),
])
input_tensor = preprocess(input_image)
input_batch = input_tensor.unsqueeze(0)
output = self.model(input_batch)
torch.nn.functional.softmax(output, dim=1)[0]
values, top_5 = torch.topk(output, 5)
return {"predictions": values.tolist()}
if __name__ == "__main__":
kfserving.KFServer(workers=1).start({"custom-model": AlexNetModel})
DEFAULT_MAX_ITER = "1000"
DEFAULT_NB_CLASSES = "10"
parser = argparse.ArgumentParser(parents=[kfserving.kfserver.parser])
parser.add_argument('--model_name',
default=DEFAULT_MODEL_NAME,
help='The name that the model is served under.')
parser.add_argument('--adversary_type',
default=DEFAULT_ADVERSARY_TYPE,
help='What type of adversarial tool to use.')
parser.add_argument('--max_iter',
default=DEFAULT_MAX_ITER,
help='The max number of iterations to run.')
parser.add_argument('--nb_classes',
default=DEFAULT_NB_CLASSES,
help='The number of different classification types.')
parser.add_argument('--predictor_host',
help='The host for the predictor',
required=True)
args, _ = parser.parse_known_args()
if __name__ == "__main__":
model = ARTModel(args.model_name,
args.predictor_host,
adversary_type=args.adversary_type,
nb_classes=args.nb_classes,
max_iter=args.max_iter)
model.load()
kfserving.KFServer().start([model], nest_asyncio=True)
help='Explainer method',
required=True)
parser.add_argument('--explainerUri',
help='The URL of a pretrained explainer',
default=os.environ.get(ENV_STORAGE_URI))
parser.add_argument('--config',
default=os.environ.get(CONFIG_ENV),
help='Custom configuration parameters')
args, _ = parser.parse_known_args()
if __name__ == "__main__":
# Pretrained Alibi explainer
alibi_model = None
if args.explainerUri is not None:
alibi_model = os.path.join(
kfserving.Storage.download(args.explainerUri), EXPLAINER_FILENAME)
with open(alibi_model, 'rb') as f:
alibi_model = dill.load(f)
# Custom configuration
if args.config is None:
config = {}
else:
config = json.loads(args.config)
explainer = AlibiExplainer(args.explainer_name,
args.predict_url, args.protocol,
ExplainerMethod(args.type), config, alibi_model)
explainer.load()
kfserving.KFServer().start(models=[explainer])
import argparse
from alibiexplainer import AlibiExplainer
from alibiexplainer.explainer import ExplainerMethod #pylint:disable=no-name-in-module
DEFAULT_MODEL_NAME = "model"
parser = argparse.ArgumentParser(parents=[kfserving.server.parser]) #pylint:disable=c-extension-no-member
parser.add_argument('--model_name',
default=DEFAULT_MODEL_NAME,
help='The name that the model is served under.')
parser.add_argument('--predict_url',
help='The URL for the model predict function',
required=True)
parser.add_argument('--method',
type=ExplainerMethod,
choices=list(ExplainerMethod),
default="anchor_tabular",
help='Explainer method')
parser.add_argument('--training_data', help='The URL for the training data')
args, _ = parser.parse_known_args()
if __name__ == "__main__":
explainer = AlibiExplainer(args.model_name,
args.predict_url,
args.protocol,
ExplainerMethod(args.method),
training_data_url=args.training_data)
explainer.load()
kfserving.KFServer().start(models=[explainer]) #pylint:disable=c-extension-no-member
