def __init__( self, api_spec: dict, models: ModelsHolder, model_dir: str, models_tree: Optional[ModelsTree], lock_dir: Optional[str] = "/run/cron", load_model_fn: Optional[Callable[[str], Any]] = None, ): """ Setup Python model client. Args: api_spec: API configuration. models: Holding all models into memory. model_dir: Where the models are saved on disk. models_tree: A tree of the available models from upstream. lock_dir: Where the resource locks are found. Only when processes_per_replica > 0 and caching disabled. load_model_fn: Function to load model into memory. """ self._api_spec = api_spec self._models = models self._models_tree = models_tree self._model_dir = model_dir self._lock_dir = lock_dir self._spec_models = get_models_from_api_spec(api_spec) if (self._api_spec["predictor"]["multi_model_reloading"] and self._api_spec["predictor"]["multi_model_reloading"]["dir"]): self._models_dir = True else: self._models_dir = False self._spec_model_names = self._spec_models.get_field("name") # for when local models are used self._spec_local_model_names = self._spec_models.get_local_model_names( ) self._local_model_ts = int( datetime.datetime.now(datetime.timezone.utc).timestamp()) self._multiple_processes = self._api_spec["predictor"][ "processes_per_replica"] > 1 self._caching_enabled = self._is_model_caching_enabled() if callable(load_model_fn): self._models.set_callback("load", load_model_fn)
def __init__( self, api_spec: dict, models: ModelsHolder, model_dir: str, models_tree: Optional[ModelsTree], lock_dir: Optional[str] = "/run/cron", ): """ Setup ONNX runtime. Args: api_spec: API configuration. models: Holding all models into memory. model_dir: Where the models are saved on disk. models_tree: A tree of the available models from upstream. lock_dir: Where the resource locks are found. Only when processes_per_replica > 0 and caching disabled. """ if not onnx_dependencies_installed: raise NameError("onnx dependencies not installed") self._api_spec = api_spec self._models = models self._models_tree = models_tree self._model_dir = model_dir self._lock_dir = lock_dir self._spec_models = get_models_from_api_spec(api_spec) if (self._api_spec["predictor"]["models"] and self._api_spec["predictor"]["models"]["dir"] is not None): self._models_dir = True else: self._models_dir = False self._spec_model_names = self._spec_models.get_field("name") # for when local models are used self._spec_local_model_names = self._spec_models.get_local_model_names( ) self._local_model_ts = int( datetime.datetime.now(datetime.timezone.utc).timestamp()) self._multiple_processes = self._api_spec["predictor"][ "processes_per_replica"] > 1 self._caching_enabled = self._is_model_caching_enabled() self._models.set_callback("load", self._load_model)
def __init__( self, tf_serving_url, api_spec: dict, models: Optional[ModelsHolder], model_dir: Optional[str], models_tree: Optional[ModelsTree], ): """ Setup gRPC connection to TensorFlow Serving container. Args: tf_serving_url: Localhost URL to TF Serving container (i.e. "localhost:9000") api_spec: API configuration. models: Holding all models into memory. Only when processes_per_replica = 1 and caching enabled. model_dir: Where the models are saved on disk. Only when processes_per_replica = 1 and caching enabled. models_tree: A tree of the available models from upstream. Only when processes_per_replica = 1 and caching enabled. """ self.tf_serving_url = tf_serving_url self._api_spec = api_spec self._models = models self._models_tree = models_tree self._model_dir = model_dir self._spec_models = get_models_from_api_spec(api_spec) if (self._api_spec["predictor"]["models"] and self._api_spec["predictor"]["models"]["dir"] is not None): self._models_dir = True else: self._models_dir = False self._spec_model_names = self._spec_models.get_field("name") self._multiple_processes = self._api_spec["predictor"][ "processes_per_replica"] > 1 self._caching_enabled = self._is_model_caching_enabled() if self._models: self._models.set_callback("load", self._load_model) self._models.set_callback("remove", self._remove_models) self._client = TensorFlowServingAPI(tf_serving_url)