def class_impl(self, project_dir):
if self.type in [TensorFlowPredictorType, TensorFlowNeuronPredictorType]:
target_class_name = "TensorFlowPredictor"
validations = TENSORFLOW_CLASS_VALIDATION
elif self.type == ONNXPredictorType:
target_class_name = "ONNXPredictor"
validations = ONNX_CLASS_VALIDATION
elif self.type == PythonPredictorType:
target_class_name = "PythonPredictor"
validations = PYTHON_CLASS_VALIDATION
try:
predictor_class = self._get_class_impl(
"cortex_predictor", os.path.join(project_dir, self.path), target_class_name
)
except CortexException as e:
e.wrap("error in " + self.path)
raise
finally:
refresh_logger()
try:
_validate_impl(predictor_class, validations, self.api_spec)
if self.type == PythonPredictorType:
_validate_python_predictor_with_models(predictor_class, self.api_spec)
except CortexException as e:
e.wrap("error in " + self.path)
raise
return predictor_class
def get_request_handler_impl(self, api_name, project_dir):
api = self.apis[api_name]
model_type = api.get("tensorflow")
if model_type is None:
model_type = api.get("onnx")
if model_type is None: # unexpected
raise CortexException(api_name, 'missing "tensorflow" or "onnx" key')
request_handler_path = model_type.get("request_handler")
try:
impl = self.load_module(
"request_handler", api["name"], os.path.join(project_dir, request_handler_path)
)
except CortexException as e:
e.wrap("api " + api_name, "error in " + request_handler_path)
raise
finally:
refresh_logger()
try:
_validate_impl(impl, REQUEST_HANDLER_IMPL_VALIDATION)
except CortexException as e:
e.wrap("api " + api_name, request_handler_path)
raise
return impl
def initialize_impl(self, project_dir, client=None, api_spec=None, job_spec=None):
class_impl = self.class_impl(project_dir)
constructor_args = inspect.getfullargspec(class_impl.__init__).args
args = {}
config = deepcopy(api_spec["predictor"]["config"])
if job_spec is not None and job_spec.get("config") is not None:
util.merge_dicts_in_place_overwrite(config, job_spec["config"])
if "config" in constructor_args:
args["config"] = config
if "job_spec" in constructor_args:
args["job_spec"] = job_spec
try:
if self.type == "onnx":
args["onnx_client"] = client
return class_impl(**args)
elif self.type == "tensorflow":
args["tensorflow_client"] = client
return class_impl(**args)
else:
return class_impl(**args)
except Exception as e:
raise UserRuntimeException(self.path, "__init__", str(e)) from e
finally:
refresh_logger()
def class_impl(self, project_dir):
target_class_name, validations = self.get_target_and_validations()
try:
impl = self._load_module("cortex_predictor", os.path.join(project_dir, self.path))
except CortexException as e:
e.wrap("error in " + self.path)
raise
finally:
refresh_logger()
try:
classes = inspect.getmembers(impl, inspect.isclass)
predictor_class = None
for class_df in classes:
if class_df[0] == target_class_name:
if predictor_class is not None:
raise UserException(
"multiple definitions for {} class found; please check your imports and class definitions and ensure that there is only one Predictor class definition".format(
target_class_name
)
)
predictor_class = class_df[1]
if predictor_class is None:
raise UserException("{} class is not defined".format(target_class_name))
_validate_impl(predictor_class, validations)
except CortexException as e:
e.wrap("error in " + self.path)
raise
return predictor_class
def initialize_impl(self, project_dir, client=None):
class_impl = self.class_impl(project_dir)
try:
if self.type == "python":
return class_impl(self.config)
else:
return class_impl(client, self.config)
except Exception as e:
raise UserRuntimeException(self.path, "__init__", str(e)) from e
finally:
refresh_logger()
def start(args):
api = None
try:
ctx = Context(s3_path=args.context, cache_dir=args.cache_dir, workload_id=args.workload_id)
api = ctx.apis_id_map[args.api]
local_cache["api"] = api
local_cache["ctx"] = ctx
if api["predictor"]["type"] != "onnx":
raise CortexException(api["name"], "predictor type is not onnx")
cx_logger().info("loading the predictor from {}".format(api["predictor"]["path"]))
_, prefix = ctx.storage.deconstruct_s3_path(api["predictor"]["model"])
model_path = os.path.join(args.model_dir, os.path.basename(prefix))
local_cache["client"] = ONNXClient(model_path)
predictor_class = ctx.get_predictor_class(api["name"], args.project_dir)
try:
local_cache["predictor"] = predictor_class(
local_cache["client"], api["predictor"]["config"]
)
except Exception as e:
raise UserRuntimeException(api["predictor"]["path"], "__init__", str(e)) from e
finally:
refresh_logger()
except Exception as e:
cx_logger().exception("failed to start api")
sys.exit(1)
if api.get("tracker") is not None and api["tracker"].get("model_type") == "classification":
try:
local_cache["class_set"] = api_utils.get_classes(ctx, api["name"])
except Exception as e:
cx_logger().warn("an error occurred while attempting to load classes", exc_info=True)
cx_logger().info("ONNX model signature: {}".format(local_cache["client"].input_signature))
waitress_kwargs = {}
if api["predictor"].get("config") is not None:
for key, value in api["predictor"]["config"].items():
if key.startswith("waitress_"):
waitress_kwargs[key[len("waitress_") :]] = value
if len(waitress_kwargs) > 0:
cx_logger().info("waitress parameters: {}".format(waitress_kwargs))
waitress_kwargs["listen"] = "*:{}".format(args.port)
cx_logger().info("{} api is live".format(api["name"]))
open("/health_check.txt", "a").close()
serve(app, **waitress_kwargs)
def initialize_impl(self, project_dir, client=None):
class_impl = self.class_impl(project_dir)
try:
if self.type == "onnx":
return class_impl(onnx_client=client, config=self.config)
elif self.type == "tensorflow":
return class_impl(tensorflow_client=client, config=self.config)
else:
return class_impl(config=self.config)
except Exception as e:
raise UserRuntimeException(self.path, "__init__", str(e)) from e
finally:
refresh_logger()
def start(args):
api = None
try:
ctx = Context(s3_path=args.context,
cache_dir=args.cache_dir,
workload_id=args.workload_id)
api = ctx.apis_id_map[args.api]
local_cache["api"] = api
local_cache["ctx"] = ctx
if api.get("predictor") is None:
raise CortexException(api["name"], "predictor key not configured")
cx_logger().info("loading the predictor from {}".format(
api["predictor"]["path"]))
local_cache["predictor"] = ctx.get_predictor_impl(
api["name"], args.project_dir)
if util.has_function(local_cache["predictor"], "init"):
try:
model_path = None
if api["predictor"].get("model") is not None:
_, prefix = ctx.storage.deconstruct_s3_path(
api["predictor"]["model"])
model_path = os.path.join(
args.model_dir,
os.path.basename(os.path.normpath(prefix)))
cx_logger().info("calling the predictor's init() function")
local_cache["predictor"].init(model_path,
api["predictor"]["metadata"])
except Exception as e:
raise UserRuntimeException(api["predictor"]["path"], "init",
str(e)) from e
finally:
refresh_logger()
except:
cx_logger().exception("failed to start api")
sys.exit(1)
if api.get("tracker") is not None and api["tracker"].get(
"model_type") == "classification":
try:
local_cache["class_set"] = api_utils.get_classes(ctx, api["name"])
except Exception as e:
cx_logger().warn(
"an error occurred while attempting to load classes",
exc_info=True)
cx_logger().info("{} api is live".format(api["name"]))
serve(app, listen="*:{}".format(args.port))
def get_predictor_impl(self, api_name, project_dir):
api = self.apis[api_name]
try:
impl = self.load_module(
"predictor", api["name"], os.path.join(project_dir, api["predictor"]["path"])
)
except CortexException as e:
e.wrap("api " + api_name, "error in " + api["predictor"]["path"])
raise
finally:
refresh_logger()
try:
_validate_impl(impl, PREDICTOR_IMPL_VALIDATION)
except CortexException as e:
e.wrap("api " + api_name, api["predictor"]["path"])
raise
return impl
def class_impl(self, project_dir):
if self.type in [
TensorFlowPredictorType, TensorFlowNeuronPredictorType
]:
target_class_name = "TensorFlowPredictor"
validations = TENSORFLOW_CLASS_VALIDATION
elif self.type == ONNXPredictorType:
target_class_name = "ONNXPredictor"
validations = ONNX_CLASS_VALIDATION
elif self.type == PythonPredictorType:
target_class_name = "PythonPredictor"
validations = PYTHON_CLASS_VALIDATION
try:
with FileLock("/run/init_stagger.lock"):
impl = self._load_module("cortex_predictor",
os.path.join(project_dir, self.path))
except CortexException as e:
e.wrap("error in " + self.path)
raise
finally:
refresh_logger()
try:
classes = inspect.getmembers(impl, inspect.isclass)
predictor_class = None
for class_df in classes:
if class_df[0] == target_class_name:
if predictor_class is not None:
raise UserException(
f"multiple definitions for {target_class_name} class found; please check your imports and class definitions and ensure that there is only one Predictor class definition"
)
predictor_class = class_df[1]
if predictor_class is None:
raise UserException(
f"{target_class_name} class is not defined")
_validate_impl(predictor_class, validations, self.api_spec)
if self.type == PythonPredictorType:
_validate_python_predictor_with_models(predictor_class,
self.api_spec)
except CortexException as e:
e.wrap("error in " + self.path)
raise
return predictor_class
def get_predictor_class(self, api_name, project_dir):
api = self.apis[api_name]
if api["predictor"]["type"] == "tensorflow":
target_class_name = "TensorFlowPredictor"
validations = TENSORFLOW_CLASS_VALIDATION
elif api["predictor"]["type"] == "onnx":
target_class_name = "ONNXPredictor"
validations = ONNX_CLASS_VALIDATION
elif api["predictor"]["type"] == "python":
target_class_name = "PythonPredictor"
validations = PYTHON_CLASS_VALIDATION
try:
impl = self.load_module(
"predictor", api["name"], os.path.join(project_dir, api["predictor"]["path"])
)
except CortexException as e:
e.wrap("api " + api_name, "error in " + api["predictor"]["path"])
raise
finally:
refresh_logger()
try:
classes = inspect.getmembers(impl, inspect.isclass)
predictor_class = None
for class_df in classes:
if class_df[0] == target_class_name:
if predictor_class is not None:
raise UserException(
"multiple definitions for {} class found; please check your imports and class definitions and ensure that there is only one Predictor class definition".format(
target_class_name
)
)
predictor_class = class_df[1]
if predictor_class is None:
raise UserException("{} class is not defined".format(target_class_name))
_validate_impl(predictor_class, validations)
except CortexException as e:
e.wrap("api " + api_name, "error in " + api["predictor"]["path"])
raise
return predictor_class
def initialize_impl(
self,
project_dir: str,
client: Union[PythonClient, TensorFlowClient, ONNXClient],
job_spec: Optional[dict] = None,
):
"""
Initialize predictor class as provided by the user.
job_spec is a dictionary when the "kind" of the API is set to "BatchAPI". Otherwise, it's None.
"""
# build args
class_impl = self.class_impl(project_dir)
constructor_args = inspect.getfullargspec(class_impl.__init__).args
config = deepcopy(self.config)
args = {}
if job_spec is not None and job_spec.get("config") is not None:
util.merge_dicts_in_place_overwrite(config, job_spec["config"])
if "config" in constructor_args:
args["config"] = config
if "job_spec" in constructor_args:
args["job_spec"] = job_spec
# initialize predictor class
try:
if self.type == PythonPredictorType:
if _are_models_specified(self.api_spec):
args["python_client"] = client
initialized_impl = class_impl(**args)
client.set_load_method(initialized_impl.load_model)
else:
initialized_impl = class_impl(**args)
if self.type in [TensorFlowPredictorType, TensorFlowNeuronPredictorType]:
args["tensorflow_client"] = client
initialized_impl = class_impl(**args)
if self.type == ONNXPredictorType:
args["onnx_client"] = client
initialized_impl = class_impl(**args)
except Exception as e:
raise UserRuntimeException(self.path, "__init__", str(e)) from e
finally:
refresh_logger()
# initialize the crons if models have been specified and if the API kind is RealtimeAPI
if _are_models_specified(self.api_spec) and self.api_spec["kind"] == "RealtimeAPI":
if not self.multiple_processes and self.caching_enabled:
self.crons += [
ModelTreeUpdater(
interval=10,
api_spec=self.api_spec,
tree=self.models_tree,
ondisk_models_dir=self.model_dir,
),
ModelsGC(
interval=10,
api_spec=self.api_spec,
models=self.models,
tree=self.models_tree,
),
]
if not self.caching_enabled and self.type in [PythonPredictorType, ONNXPredictorType]:
self.crons += [
FileBasedModelsGC(interval=10, models=self.models, download_dir=self.model_dir)
]
for cron in self.crons:
cron.start()
return initialized_impl
