def _process_query(self, endpoint_name, start): self.logger.log(logging.DEBUG, f"Processing query {endpoint_name}...") try: self._add_CORS_header() if not self.request.body: self.request.body = {} # extract request data explicitly for caching purpose request_json = self.request.body.decode("utf-8") # Sanitize input data data = self._sanitize_request_data(json.loads(request_json)) except Exception as e: self.logger.log(logging.ERROR, str(e)) err_msg = format_exception(e, "Invalid Input Data") self.error_out(400, err_msg) return try: (po_name, _) = self._get_actual_model(endpoint_name) # po_name is None if self.python_service.ps.query_objects.get( # endpoint_name) is None if not po_name: self.error_out( 404, "UnknownURI", info=f'Endpoint "{endpoint_name}" does not exist') return po_obj = self.python_service.ps.query_objects.get(po_name) if not po_obj: self.error_out(404, "UnknownURI", info=f'Endpoint "{po_name}" does not exist') return if po_name != endpoint_name: self.logger.log(logging.INFO, f"Querying actual model: po_name={po_name}") uid = _get_uuid() # record query w/ request ID in query log qry = Query(po_name, request_json) gls_time = 0 # send a query to PythonService and return (gls_time, _) = self._handle_result(po_name, data, qry, uid) # if error occurred, GLS time is None. if not gls_time: return except Exception as e: self.logger.log(logging.ERROR, str(e)) err_msg = format_exception(e, "process query") self.error_out(500, "Error processing query", info=err_msg) return
def query(self, object_uri, params, uid): """Execute a QueryObject query""" logger.debug(f'Querying Python service {object_uri}...') try: if not isinstance(params, dict) and not isinstance(params, list): return QueryFailed( uri=object_uri, error=('Query parameter needs to be a dictionary or a list' f'. Given value is of type {type(params)}')) obj_info = self.query_objects.get(object_uri) logger.debug(f'Found object {obj_info}') if obj_info: pred_obj = obj_info['endpoint_obj'] version = obj_info['version'] if not pred_obj: return QueryFailed( uri=object_uri, error=("There is no query object associated to the " f'endpoint: {object_uri}')) logger.debug(f'Querying endpoint with params ({params})...') if isinstance(params, dict): result = pred_obj.query(**params) else: result = pred_obj.query(*params) return QuerySuccessful(object_uri, version, result) else: return UnknownURI(object_uri) except Exception as e: err_msg = format_exception(e, '/query') logger.error(err_msg) return QueryFailed(uri=object_uri, error=err_msg)
def on_state_change(settings, tabpy_state, python_service, logger=logging.getLogger(__name__)): try: logger.log(logging.INFO, "Loading state from state file") config = util._get_state_from_file( settings[SettingsParameters.StateFilePath], logger=logger) new_ps_state = TabPyState(config=config, settings=settings) (has_changes, changes) = _get_latest_service_state(settings, tabpy_state, new_ps_state, python_service) if not has_changes: logger.info("Nothing changed, return.") return new_endpoints = new_ps_state.get_endpoints() for object_name in changes['endpoints']: (object_type, object_version, object_path) = changes['endpoints'][object_name] if not object_path and not object_version: # removal logger.info(f'Removing object: URI={object_name}') python_service.manage_request(DeleteObjects([object_name])) cleanup_endpoint_files(object_name, settings[SettingsParameters.UploadDir], logger=logger) else: endpoint_info = new_endpoints[object_name] is_update = object_version > 1 if object_type == 'alias': msg = LoadObject(object_name, endpoint_info['target'], object_version, is_update, 'alias') else: local_path = object_path msg = LoadObject(object_name, local_path, object_version, is_update, object_type) python_service.manage_request(msg) wait_for_endpoint_loaded(python_service, object_name) # cleanup old version of endpoint files if object_version > 2: cleanup_endpoint_files( object_name, settings[SettingsParameters.UploadDir], logger=logger, retain_versions=[object_version, object_version - 1]) except Exception as e: err_msg = format_exception(e, 'on_state_change') logger.log(logging.ERROR, f'Error submitting update model request: error={err_msg}')
def put(self, name): if self.should_fail_with_not_authorized(): self.fail_with_not_authorized() return self.logger.log(logging.DEBUG, f'Processing PUT for /endpoints/{name}') try: if not self.request.body: self.error_out(400, "Input body cannot be empty") self.finish() return try: request_data = json.loads( self.request.body.decode('utf-8')) except BaseException as ex: self.error_out( 400, log_message="Failed to decode input body", info=str(ex)) self.finish() return # check if endpoint exists endpoints = self.tabpy_state.get_endpoints(name) if len(endpoints) == 0: self.error_out(404, f'endpoint {name} does not exist.') self.finish() return new_version = int(endpoints[name]['version']) + 1 self.logger.log( logging.INFO, f'Endpoint info: {request_data}') err_msg = yield self._add_or_update_endpoint( 'update', name, new_version, request_data) if err_msg: self.error_out(400, err_msg) self.finish() else: self.write(self.tabpy_state.get_endpoints(name)) self.finish() except Exception as e: err_msg = format_exception(e, 'update_endpoint') self.error_out(500, err_msg) self.finish()
def post(self): if self.should_fail_with_not_authorized(): self.fail_with_not_authorized() return try: if not self.request.body: self.error_out(400, "Input body cannot be empty") self.finish() return try: request_data = json.loads(self.request.body.decode("utf-8")) except Exception as ex: self.error_out(400, "Failed to decode input body", str(ex)) self.finish() return if "name" not in request_data: self.error_out(400, "name is required to add an endpoint.") self.finish() return name = request_data["name"] # check if endpoint already exist if name in self.tabpy_state.get_endpoints(): self.error_out(400, f"endpoint {name} already exists.") self.finish() return self.logger.log(logging.DEBUG, f'Adding endpoint "{name}"') err_msg = yield self._add_or_update_endpoint( "add", name, 1, request_data) if err_msg: self.error_out(400, err_msg) else: self.logger.log(logging.DEBUG, f"Endpoint {name} successfully added") self.set_status(201) self.write(self.tabpy_state.get_endpoints(name)) self.finish() return except Exception as e: err_msg = format_exception(e, "/add_endpoint") self.error_out(500, "error adding endpoint", err_msg) self.finish() return
def delete(self, name): if self.should_fail_with_not_authorized(): self.fail_with_not_authorized() return self.logger.log( logging.DEBUG, f'Processing DELETE for /endpoints/{name}') try: endpoints = self.tabpy_state.get_endpoints(name) if len(endpoints) == 0: self.error_out(404, f'endpoint {name} does not exist.') self.finish() return # update state try: endpoint_info = self.tabpy_state.delete_endpoint(name) except Exception as e: self.error_out(400, f'Error when removing endpoint: {e.message}') self.finish() return # delete files if endpoint_info['type'] != 'alias': delete_path = get_query_object_path( self.settings['state_file_path'], name, None) try: yield self._delete_po_future(delete_path) except Exception as e: self.error_out(400, f'Error while deleting: {e}') self.finish() return self.set_status(204) self.finish() except Exception as e: err_msg = format_exception(e, 'delete endpoint') self.error_out(500, err_msg) self.finish() on_state_change(self.settings, self.tabpy_state, self.python_service, self.logger)
def delete(self, name): if self.should_fail_with_auth_error() != AuthErrorStates.NONE: self.fail_with_auth_error() return self.logger.log(logging.DEBUG, f"Processing DELETE for /endpoints/{name}") try: endpoints = self.tabpy_state.get_endpoints(name) if len(endpoints) == 0: self.error_out(404, f"endpoint {name} does not exist.") self.finish() return # update state try: endpoint_info = self.tabpy_state.delete_endpoint(name) except Exception as e: self.error_out(400, f"Error when removing endpoint: {e.message}") self.finish() return # delete files if endpoint_info["type"] != "alias": delete_path = get_query_object_path( self.settings["state_file_path"], name, None) try: yield self._delete_po_future(delete_path) except Exception as e: self.error_out(400, f"Error while deleting: {e}") self.finish() return self.set_status(204) self.finish() except Exception as e: err_msg = format_exception(e, "delete endpoint") self.error_out(500, err_msg) self.finish() on_state_change(self.settings, self.tabpy_state, self.python_service, self.logger)
def put(self, name): if self.should_fail_with_auth_error() != AuthErrorStates.NONE: self.fail_with_auth_error() return self.logger.log(logging.DEBUG, f"Processing PUT for /endpoints/{name}") try: if not self.request.body: self.error_out(400, "Input body cannot be empty") self.finish() return try: request_data = json.loads(self.request.body.decode("utf-8")) except BaseException as ex: self.error_out(400, log_message="Failed to decode input body", info=str(ex)) self.finish() return # check if endpoint exists endpoints = self.tabpy_state.get_endpoints(name) if len(endpoints) == 0: self.error_out(404, f"endpoint {name} does not exist.") self.finish() return new_version = int(endpoints[name]["version"]) + 1 self.logger.log(logging.INFO, f"Endpoint info: {request_data}") err_msg = yield self._add_or_update_endpoint( "update", name, new_version, request_data) if err_msg: self.error_out(400, err_msg) self.finish() else: self.write(self.tabpy_state.get_endpoints(name)) self.finish() except Exception as e: err_msg = format_exception(e, "update_endpoint") self.error_out(500, err_msg) self.finish()
def post(self): if self.should_fail_with_auth_error() != AuthErrorStates.NONE: self.fail_with_auth_error() return self._add_CORS_header() try: yield self._post_impl() except Exception as e: err_msg = f"{e.__class__.__name__} : {str(e)}" if err_msg != "KeyError : 'response'": err_msg = format_exception(e, "POST /evaluate") self.error_out(500, "Error processing script", info=err_msg) else: self.error_out( 404, "Error processing script", info="The endpoint you're " "trying to query did not respond. Please make sure the " "endpoint exists and the correct set of arguments are " "provided.", )
def post(self): if self.should_fail_with_not_authorized(): self.fail_with_not_authorized() return self._add_CORS_header() try: body = json.loads(self.request.body.decode("utf-8")) if "script" not in body: self.error_out(400, "Script is empty.") return # Transforming user script into a proper function. user_code = body["script"] arguments = None arguments_str = "" if "data" in body: arguments = body["data"] if arguments is not None: if not isinstance(arguments, dict): self.error_out( 400, "Script parameters need to be provided as a dictionary." ) return else: arguments_expected = [] for i in range(1, len(arguments.keys()) + 1): arguments_expected.append("_arg" + str(i)) if sorted(arguments_expected) == sorted(arguments.keys()): arguments_str = ", " + ", ".join(arguments.keys()) else: self.error_out( 400, "Variables names should follow " "the format _arg1, _arg2, _argN", ) return function_to_evaluate = f"def _user_script(tabpy{arguments_str}):\n" for u in user_code.splitlines(): function_to_evaluate += " " + u + "\n" self.logger.log(logging.INFO, f"function to evaluate={function_to_evaluate}") try: result = yield self._call_subprocess(function_to_evaluate, arguments) except ( gen.TimeoutError, requests.exceptions.ConnectTimeout, requests.exceptions.ReadTimeout, ): self.logger.log(logging.ERROR, self._error_message_timeout) self.error_out(408, self._error_message_timeout) return if result is None: self.error_out(400, "Error running script. No return value") else: self.write(json.dumps(result)) self.finish() except Exception as e: err_msg = f"{e.__class__.__name__} : {str(e)}" if err_msg != "KeyError : 'response'": err_msg = format_exception(e, "POST /evaluate") self.error_out(500, "Error processing script", info=err_msg) else: self.error_out( 404, "Error processing script", info="The endpoint you're " "trying to query did not respond. Please make sure the " "endpoint exists and the correct set of arguments are " "provided.", )