model_name = model_config["name"] model_base_path = model_config["base_path"] model_platform = model_config.get("platform", "tensorflow") custom_op_paths = model_config.get("custom_op_paths", "") if model_platform == "tensorflow": inference_service = TensorFlowInferenceService( model_name, model_base_path, custom_op_paths, args.verbose) elif model_platform == "mxnet": inference_service = MxnetInferenceService( model_name, model_base_path, args.verbose) elif model_platform == "onnx": inference_service = OnnxInferenceService( model_name, model_base_path, args.verbose) elif model_platform == "h2o": inference_service = H2oInferenceService( model_name, model_base_path, args.verbose) elif model_platform == "scikitlearn": inference_service = ScikitlearnInferenceService( model_name, model_base_path, arg.verbose) elif model_platform == "xgboost": inference_service = XgboostInferenceService( model_name, model_base_path, arg.verbose) model_name_service_map[model_name] = inference_service else: # Read from command-line parameter if args.model_platform == "tensorflow": inference_service = TensorFlowInferenceService(args.model_name, args.model_base_path, args.custom_op_paths, args.verbose)
def init(self): if self.args.model_config_file != "": # Read from configuration file with open(self.args.model_config_file) as data_file: model_config_file_dict = json.load(data_file) # Example: [{u'platform': u'tensorflow', u'name': u'tensorflow_template_application', u'base_path': u'/Users/tobe/code/simple_tensorflow_serving/models/tensorflow_template_application_model/'}, {u'platform': u'tensorflow', u'name': u'deep_image_model', u'base_path': u'/Users/tobe/code/simple_tensorflow_serving/models/deep_image_model/'}] model_config_list = model_config_file_dict["model_config_list"] for model_config in model_config_list: # Example: {"name": "tensorflow_template_application", "base_path": "/", "platform": "tensorflow"} model_name = model_config["name"] model_base_path = model_config["base_path"] model_platform = model_config.get("platform", "tensorflow") custom_op_paths = model_config.get("custom_op_paths", "") session_config = model_config.get("session_config", {}) if model_platform == "tensorflow": inference_service = TensorFlowInferenceService( model_name, model_base_path, custom_op_paths, session_config) elif model_platform == "mxnet": inference_service = MxnetInferenceService(model_name, model_base_path) elif model_platform == "onnx": inference_service = OnnxInferenceService(model_name, model_base_path) elif model_platform == "pytorch_onnx": inference_service = PytorchOnnxInferenceService( model_name, model_base_path) elif model_platform == "h2o": inference_service = H2oInferenceService(model_name, model_base_path) elif model_platform == "scikitlearn": inference_service = ScikitlearnInferenceService( model_name, model_base_path) elif model_platform == "xgboost": inference_service = XgboostInferenceService( model_name, model_base_path) elif model_platform == "pmml": inference_service = PmmlInferenceService(model_name, model_base_path) elif model_platform == "spark": inference_service = SparkInferenceService(model_name, model_base_path) self.model_name_service_map[model_name] = inference_service else: # Read from command-line parameter if self.args.model_platform == "tensorflow": session_config = json.loads(self.args.session_config) inference_service = TensorFlowInferenceService( self.args.model_name, self.args.model_base_path, self.args.custom_op_paths, session_config) elif self.args.model_platform == "mxnet": inference_service = MxnetInferenceService(self.args.model_name, self.args.model_base_path) elif self.args.model_platform == "h2o": inference_service = H2oInferenceService(self.args.model_name, self.args.model_base_path) elif self.args.model_platform == "onnx": inference_service = OnnxInferenceService(self.args.model_name, self.args.model_base_path) elif self.args.model_platform == "pytorch_onnx": inference_service = PytorchOnnxInferenceService( self.args.model_name, self.args.model_base_path) elif self.args.model_platform == "scikitlearn": inference_service = ScikitlearnInferenceService( self.args.model_name, self.args.model_base_path) elif self.args.model_platform == "xgboost": inference_service = XgboostInferenceService(self.args.model_name, self.args.model_base_path) elif self.args.model_platform == "pmml": inference_service = PmmlInferenceService(self.args.model_name, self.args.model_base_path) elif self.args.model_platform == "spark": inference_service = SparkInferenceService(self.args.model_name, self.args.model_base_path) self.model_name_service_map[self.args.model_name] = inference_service # Start thread to periodically reload models or not if self.args.reload_models == "True" or self.args.reload_models == "true": for model_name, inference_service in self.model_name_service_map.items(): if inference_service.platform == "tensorflow": inference_service.dynamically_reload_models()