def __init__(
self,
api_spec: dict,
models: ModelsHolder,
model_dir: str,
models_tree: Optional[ModelsTree],
lock_dir: Optional[str] = "/run/cron",
load_model_fn: Optional[Callable[[str], Any]] = None,
):
"""
Setup Python model client.
Args:
api_spec: API configuration.
models: Holding all models into memory.
model_dir: Where the models are saved on disk.
models_tree: A tree of the available models from upstream.
lock_dir: Where the resource locks are found. Only when processes_per_replica > 0 and caching disabled.
load_model_fn: Function to load model into memory.
"""
self._api_spec = api_spec
self._models = models
self._models_tree = models_tree
self._model_dir = model_dir
self._lock_dir = lock_dir
self._spec_models = get_models_from_api_spec(api_spec)
if (self._api_spec["predictor"]["multi_model_reloading"] and
self._api_spec["predictor"]["multi_model_reloading"]["dir"]):
self._models_dir = True
else:
self._models_dir = False
self._spec_model_names = self._spec_models.get_field("name")
# for when local models are used
self._spec_local_model_names = self._spec_models.get_local_model_names(
)
self._local_model_ts = int(
datetime.datetime.now(datetime.timezone.utc).timestamp())
self._multiple_processes = self._api_spec["predictor"][
"processes_per_replica"] > 1
self._caching_enabled = self._is_model_caching_enabled()
if callable(load_model_fn):
self._models.set_callback("load", load_model_fn)
def __init__(
self,
api_spec: dict,
models: ModelsHolder,
model_dir: str,
models_tree: Optional[ModelsTree],
lock_dir: Optional[str] = "/run/cron",
):
"""
Setup ONNX runtime.
Args:
api_spec: API configuration.
models: Holding all models into memory.
model_dir: Where the models are saved on disk.
models_tree: A tree of the available models from upstream.
lock_dir: Where the resource locks are found. Only when processes_per_replica > 0 and caching disabled.
"""
if not onnx_dependencies_installed:
raise NameError("onnx dependencies not installed")
self._api_spec = api_spec
self._models = models
self._models_tree = models_tree
self._model_dir = model_dir
self._lock_dir = lock_dir
self._spec_models = get_models_from_api_spec(api_spec)
if (self._api_spec["predictor"]["models"]
and self._api_spec["predictor"]["models"]["dir"] is not None):
self._models_dir = True
else:
self._models_dir = False
self._spec_model_names = self._spec_models.get_field("name")
# for when local models are used
self._spec_local_model_names = self._spec_models.get_local_model_names(
)
self._local_model_ts = int(
datetime.datetime.now(datetime.timezone.utc).timestamp())
self._multiple_processes = self._api_spec["predictor"][
"processes_per_replica"] > 1
self._caching_enabled = self._is_model_caching_enabled()
self._models.set_callback("load", self._load_model)
def __init__(
self,
tf_serving_url,
api_spec: dict,
models: Optional[ModelsHolder],
model_dir: Optional[str],
models_tree: Optional[ModelsTree],
):
"""
Setup gRPC connection to TensorFlow Serving container.
Args:
tf_serving_url: Localhost URL to TF Serving container (i.e. "localhost:9000")
api_spec: API configuration.
models: Holding all models into memory. Only when processes_per_replica = 1 and caching enabled.
model_dir: Where the models are saved on disk. Only when processes_per_replica = 1 and caching enabled.
models_tree: A tree of the available models from upstream. Only when processes_per_replica = 1 and caching enabled.
"""
self.tf_serving_url = tf_serving_url
self._api_spec = api_spec
self._models = models
self._models_tree = models_tree
self._model_dir = model_dir
self._spec_models = get_models_from_api_spec(api_spec)
if (self._api_spec["predictor"]["models"]
and self._api_spec["predictor"]["models"]["dir"] is not None):
self._models_dir = True
else:
self._models_dir = False
self._spec_model_names = self._spec_models.get_field("name")
self._multiple_processes = self._api_spec["predictor"][
"processes_per_replica"] > 1
self._caching_enabled = self._is_model_caching_enabled()
if self._models:
self._models.set_callback("load", self._load_model)
self._models.set_callback("remove", self._remove_models)
self._client = TensorFlowServingAPI(tf_serving_url)
