def class_impl(self, project_dir): if self.type in [TensorFlowPredictorType, TensorFlowNeuronPredictorType]: target_class_name = "TensorFlowPredictor" validations = TENSORFLOW_CLASS_VALIDATION elif self.type == ONNXPredictorType: target_class_name = "ONNXPredictor" validations = ONNX_CLASS_VALIDATION elif self.type == PythonPredictorType: target_class_name = "PythonPredictor" validations = PYTHON_CLASS_VALIDATION try: predictor_class = self._get_class_impl( "cortex_predictor", os.path.join(project_dir, self.path), target_class_name ) except CortexException as e: e.wrap("error in " + self.path) raise finally: refresh_logger() try: _validate_impl(predictor_class, validations, self.api_spec) if self.type == PythonPredictorType: _validate_python_predictor_with_models(predictor_class, self.api_spec) except CortexException as e: e.wrap("error in " + self.path) raise return predictor_class
def get_request_handler_impl(self, api_name, project_dir): api = self.apis[api_name] model_type = api.get("tensorflow") if model_type is None: model_type = api.get("onnx") if model_type is None: # unexpected raise CortexException(api_name, 'missing "tensorflow" or "onnx" key') request_handler_path = model_type.get("request_handler") try: impl = self.load_module( "request_handler", api["name"], os.path.join(project_dir, request_handler_path) ) except CortexException as e: e.wrap("api " + api_name, "error in " + request_handler_path) raise finally: refresh_logger() try: _validate_impl(impl, REQUEST_HANDLER_IMPL_VALIDATION) except CortexException as e: e.wrap("api " + api_name, request_handler_path) raise return impl
def initialize_impl(self, project_dir, client=None, api_spec=None, job_spec=None): class_impl = self.class_impl(project_dir) constructor_args = inspect.getfullargspec(class_impl.__init__).args args = {} config = deepcopy(api_spec["predictor"]["config"]) if job_spec is not None and job_spec.get("config") is not None: util.merge_dicts_in_place_overwrite(config, job_spec["config"]) if "config" in constructor_args: args["config"] = config if "job_spec" in constructor_args: args["job_spec"] = job_spec try: if self.type == "onnx": args["onnx_client"] = client return class_impl(**args) elif self.type == "tensorflow": args["tensorflow_client"] = client return class_impl(**args) else: return class_impl(**args) except Exception as e: raise UserRuntimeException(self.path, "__init__", str(e)) from e finally: refresh_logger()
def class_impl(self, project_dir): target_class_name, validations = self.get_target_and_validations() try: impl = self._load_module("cortex_predictor", os.path.join(project_dir, self.path)) except CortexException as e: e.wrap("error in " + self.path) raise finally: refresh_logger() try: classes = inspect.getmembers(impl, inspect.isclass) predictor_class = None for class_df in classes: if class_df[0] == target_class_name: if predictor_class is not None: raise UserException( "multiple definitions for {} class found; please check your imports and class definitions and ensure that there is only one Predictor class definition".format( target_class_name ) ) predictor_class = class_df[1] if predictor_class is None: raise UserException("{} class is not defined".format(target_class_name)) _validate_impl(predictor_class, validations) except CortexException as e: e.wrap("error in " + self.path) raise return predictor_class
def initialize_impl(self, project_dir, client=None): class_impl = self.class_impl(project_dir) try: if self.type == "python": return class_impl(self.config) else: return class_impl(client, self.config) except Exception as e: raise UserRuntimeException(self.path, "__init__", str(e)) from e finally: refresh_logger()
def start(args): api = None try: ctx = Context(s3_path=args.context, cache_dir=args.cache_dir, workload_id=args.workload_id) api = ctx.apis_id_map[args.api] local_cache["api"] = api local_cache["ctx"] = ctx if api["predictor"]["type"] != "onnx": raise CortexException(api["name"], "predictor type is not onnx") cx_logger().info("loading the predictor from {}".format(api["predictor"]["path"])) _, prefix = ctx.storage.deconstruct_s3_path(api["predictor"]["model"]) model_path = os.path.join(args.model_dir, os.path.basename(prefix)) local_cache["client"] = ONNXClient(model_path) predictor_class = ctx.get_predictor_class(api["name"], args.project_dir) try: local_cache["predictor"] = predictor_class( local_cache["client"], api["predictor"]["config"] ) except Exception as e: raise UserRuntimeException(api["predictor"]["path"], "__init__", str(e)) from e finally: refresh_logger() except Exception as e: cx_logger().exception("failed to start api") sys.exit(1) if api.get("tracker") is not None and api["tracker"].get("model_type") == "classification": try: local_cache["class_set"] = api_utils.get_classes(ctx, api["name"]) except Exception as e: cx_logger().warn("an error occurred while attempting to load classes", exc_info=True) cx_logger().info("ONNX model signature: {}".format(local_cache["client"].input_signature)) waitress_kwargs = {} if api["predictor"].get("config") is not None: for key, value in api["predictor"]["config"].items(): if key.startswith("waitress_"): waitress_kwargs[key[len("waitress_") :]] = value if len(waitress_kwargs) > 0: cx_logger().info("waitress parameters: {}".format(waitress_kwargs)) waitress_kwargs["listen"] = "*:{}".format(args.port) cx_logger().info("{} api is live".format(api["name"])) open("/health_check.txt", "a").close() serve(app, **waitress_kwargs)
def initialize_impl(self, project_dir, client=None): class_impl = self.class_impl(project_dir) try: if self.type == "onnx": return class_impl(onnx_client=client, config=self.config) elif self.type == "tensorflow": return class_impl(tensorflow_client=client, config=self.config) else: return class_impl(config=self.config) except Exception as e: raise UserRuntimeException(self.path, "__init__", str(e)) from e finally: refresh_logger()
def start(args): api = None try: ctx = Context(s3_path=args.context, cache_dir=args.cache_dir, workload_id=args.workload_id) api = ctx.apis_id_map[args.api] local_cache["api"] = api local_cache["ctx"] = ctx if api.get("predictor") is None: raise CortexException(api["name"], "predictor key not configured") cx_logger().info("loading the predictor from {}".format( api["predictor"]["path"])) local_cache["predictor"] = ctx.get_predictor_impl( api["name"], args.project_dir) if util.has_function(local_cache["predictor"], "init"): try: model_path = None if api["predictor"].get("model") is not None: _, prefix = ctx.storage.deconstruct_s3_path( api["predictor"]["model"]) model_path = os.path.join( args.model_dir, os.path.basename(os.path.normpath(prefix))) cx_logger().info("calling the predictor's init() function") local_cache["predictor"].init(model_path, api["predictor"]["metadata"]) except Exception as e: raise UserRuntimeException(api["predictor"]["path"], "init", str(e)) from e finally: refresh_logger() except: cx_logger().exception("failed to start api") sys.exit(1) if api.get("tracker") is not None and api["tracker"].get( "model_type") == "classification": try: local_cache["class_set"] = api_utils.get_classes(ctx, api["name"]) except Exception as e: cx_logger().warn( "an error occurred while attempting to load classes", exc_info=True) cx_logger().info("{} api is live".format(api["name"])) serve(app, listen="*:{}".format(args.port))
def get_predictor_impl(self, api_name, project_dir): api = self.apis[api_name] try: impl = self.load_module( "predictor", api["name"], os.path.join(project_dir, api["predictor"]["path"]) ) except CortexException as e: e.wrap("api " + api_name, "error in " + api["predictor"]["path"]) raise finally: refresh_logger() try: _validate_impl(impl, PREDICTOR_IMPL_VALIDATION) except CortexException as e: e.wrap("api " + api_name, api["predictor"]["path"]) raise return impl
def class_impl(self, project_dir): if self.type in [ TensorFlowPredictorType, TensorFlowNeuronPredictorType ]: target_class_name = "TensorFlowPredictor" validations = TENSORFLOW_CLASS_VALIDATION elif self.type == ONNXPredictorType: target_class_name = "ONNXPredictor" validations = ONNX_CLASS_VALIDATION elif self.type == PythonPredictorType: target_class_name = "PythonPredictor" validations = PYTHON_CLASS_VALIDATION try: with FileLock("/run/init_stagger.lock"): impl = self._load_module("cortex_predictor", os.path.join(project_dir, self.path)) except CortexException as e: e.wrap("error in " + self.path) raise finally: refresh_logger() try: classes = inspect.getmembers(impl, inspect.isclass) predictor_class = None for class_df in classes: if class_df[0] == target_class_name: if predictor_class is not None: raise UserException( f"multiple definitions for {target_class_name} class found; please check your imports and class definitions and ensure that there is only one Predictor class definition" ) predictor_class = class_df[1] if predictor_class is None: raise UserException( f"{target_class_name} class is not defined") _validate_impl(predictor_class, validations, self.api_spec) if self.type == PythonPredictorType: _validate_python_predictor_with_models(predictor_class, self.api_spec) except CortexException as e: e.wrap("error in " + self.path) raise return predictor_class
def get_predictor_class(self, api_name, project_dir): api = self.apis[api_name] if api["predictor"]["type"] == "tensorflow": target_class_name = "TensorFlowPredictor" validations = TENSORFLOW_CLASS_VALIDATION elif api["predictor"]["type"] == "onnx": target_class_name = "ONNXPredictor" validations = ONNX_CLASS_VALIDATION elif api["predictor"]["type"] == "python": target_class_name = "PythonPredictor" validations = PYTHON_CLASS_VALIDATION try: impl = self.load_module( "predictor", api["name"], os.path.join(project_dir, api["predictor"]["path"]) ) except CortexException as e: e.wrap("api " + api_name, "error in " + api["predictor"]["path"]) raise finally: refresh_logger() try: classes = inspect.getmembers(impl, inspect.isclass) predictor_class = None for class_df in classes: if class_df[0] == target_class_name: if predictor_class is not None: raise UserException( "multiple definitions for {} class found; please check your imports and class definitions and ensure that there is only one Predictor class definition".format( target_class_name ) ) predictor_class = class_df[1] if predictor_class is None: raise UserException("{} class is not defined".format(target_class_name)) _validate_impl(predictor_class, validations) except CortexException as e: e.wrap("api " + api_name, "error in " + api["predictor"]["path"]) raise return predictor_class
def initialize_impl( self, project_dir: str, client: Union[PythonClient, TensorFlowClient, ONNXClient], job_spec: Optional[dict] = None, ): """ Initialize predictor class as provided by the user. job_spec is a dictionary when the "kind" of the API is set to "BatchAPI". Otherwise, it's None. """ # build args class_impl = self.class_impl(project_dir) constructor_args = inspect.getfullargspec(class_impl.__init__).args config = deepcopy(self.config) args = {} if job_spec is not None and job_spec.get("config") is not None: util.merge_dicts_in_place_overwrite(config, job_spec["config"]) if "config" in constructor_args: args["config"] = config if "job_spec" in constructor_args: args["job_spec"] = job_spec # initialize predictor class try: if self.type == PythonPredictorType: if _are_models_specified(self.api_spec): args["python_client"] = client initialized_impl = class_impl(**args) client.set_load_method(initialized_impl.load_model) else: initialized_impl = class_impl(**args) if self.type in [TensorFlowPredictorType, TensorFlowNeuronPredictorType]: args["tensorflow_client"] = client initialized_impl = class_impl(**args) if self.type == ONNXPredictorType: args["onnx_client"] = client initialized_impl = class_impl(**args) except Exception as e: raise UserRuntimeException(self.path, "__init__", str(e)) from e finally: refresh_logger() # initialize the crons if models have been specified and if the API kind is RealtimeAPI if _are_models_specified(self.api_spec) and self.api_spec["kind"] == "RealtimeAPI": if not self.multiple_processes and self.caching_enabled: self.crons += [ ModelTreeUpdater( interval=10, api_spec=self.api_spec, tree=self.models_tree, ondisk_models_dir=self.model_dir, ), ModelsGC( interval=10, api_spec=self.api_spec, models=self.models, tree=self.models_tree, ), ] if not self.caching_enabled and self.type in [PythonPredictorType, ONNXPredictorType]: self.crons += [ FileBasedModelsGC(interval=10, models=self.models, download_dir=self.model_dir) ] for cron in self.crons: cron.start() return initialized_impl