class RestClient(IOpenable, IConfigurable, IReferenceable): """ Abstract client that calls remove endpoints using HTTP/REST protocol. ### Configuration parameters ### - base_route: base route for remote URI - connection(s): - discovery_key: (optional) a key to retrieve the connection from IDiscovery - protocol: connection protocol: http or https - host: host name or IP address - port: port number - uri: resource URI or connection string with all parameters in it - options: - retries: number of retries (default: 3) - connect_timeout: connection timeout in milliseconds (default: 10 sec) - timeout: invocation timeout in milliseconds (default: 10 sec) ### References ### - *:logger:*:*:1.0 (optional) ILogger components to pass log messages - *:counters:*:*:1.0 (optional) ICounters components to pass collected measurements - *:discovery:*:*:1.0 (optional) IDiscovery services to resolve connection Example: class MyRestClient(RestClient, IMyClient): def get_data(self, correlation_id, id): timing = self.instrument(correlationId, 'myclient.get_data') result = self._controller.get_data(correlationId, id) timing.end_timing() return result ... client = MyRestClient() client.configure(ConfigParams.fromTuples("connection.protocol", "http", "connection.host", "localhost", "connection.port", 8080)) data = client.getData("123", "1") ... """ _default_config = None _client = None _uri = None _timeout = 1000 _connection_resolver = None _logger = None _counters = None _options = None _base_route = None _retries = 1 _headers = None _connect_timeout = 1000 def __init__(self): """ Creates a new instance of the client. """ self._connection_resolver = HttpConnectionResolver() self._default_config = ConfigParams.from_tuples( "connection.protocol", "http", "connection.host", "0.0.0.0", "connection.port", 3000, "options.timeout", 10000, "options.request_max_size", 1024 * 1024, "options.connect_timeout", 10000, "options.retries", 3, "options.debug", True) self._logger = CompositeLogger() self._counters = CompositeCounters() self._options = ConfigParams() self._headers = {} def set_references(self, references): """ Sets references to dependent components. :param references: references to locate the component dependencies. """ self._logger.set_references(references) self._counters.set_references(references) self._connection_resolver.set_references(references) def configure(self, config): """ Configures component by passing configuration parameters. :param config: configuration parameters to be set. """ config = config.set_defaults(self._default_config) self._connection_resolver.configure(config) self._options.override(config.get_section("options")) self._retries = config.get_as_integer_with_default( "options.retries", self._retries) self._connect_timeout = config.get_as_integer_with_default( "options.connect_timeout", self._connect_timeout) self._timeout = config.get_as_integer_with_default( "options.timeout", self._timeout) self._base_route = config.get_as_string_with_default( "base_route", self._base_route) def _instrument(self, correlation_id, name): """ Adds instrumentation to log calls and measure call time. It returns a Timing object that is used to end the time measurement. :param correlation_id: (optional) transaction id to trace execution through call chain. :param name: a method name. :return: Timing object to end the time measurement. """ TYPE_NAME = self.__class__.__name__ or 'unknown-target' self._logger.trace(correlation_id, f"Calling {name} method {TYPE_NAME}") self._counters.increment_one(f"{TYPE_NAME}.{name}.call_count") return self._counters.begin_timing(f"{TYPE_NAME}.{name}.call_count") def _instrument_error(self, correlation_id, name, err, result=None, callback=None): """ Adds instrumentation to error handling. :param correlation_id: (optional) transaction id to trace execution through call chain. :param name: a method name. :param err: an occured error :param result: (optional) an execution result :param callback: (optional) an execution callback """ if err is not None: TYPE_NAME = self.__class__.__name__ or 'unknown-target' self._logger.error(correlation_id, err, f"Failed to call {name} method of {TYPE_NAME}") self._counters.increment_one(f"{name}.call_errors") if callback: callback(err, result) def is_opened(self): """ Checks if the component is opened. :return: true if the component has been opened and false otherwise. """ return self._client is not None def open(self, correlation_id): """ Opens the component. :param correlation_id: (optional) transaction id to trace execution through call chain. """ if self.is_opened(): return connection = self._connection_resolver.resolve(correlation_id) self._uri = connection.get_uri() self._client = requests self._logger.debug(correlation_id, "Connected via REST to " + self._uri) def close(self, correlation_id): """ Closes component and frees used resources. :param correlation_id: (optional) transaction id to trace execution through call chain. """ if self._client is not None: self._logger.debug(correlation_id, "Disconnected from " + self._uri) self._client = None self._uri = None def _to_json(self, obj): if obj is None: return None if isinstance(obj, set): obj = list(obj) if isinstance(obj, list): result = [] for item in obj: item = self._to_json(item) result.append(item) return result if isinstance(obj, dict): result = {} for (k, v) in obj.items(): v = self._to_json(v) result[k] = v return result if hasattr(obj, 'to_json'): return obj.to_json() if hasattr(obj, '__dict__'): return self._to_json(obj.__dict__) return obj def fix_route(self, route) -> str: if route is not None and len(route) > 0: if route[0] != '/': route = f'/{route}' return route return '' def createRequestRoute(self, route): builder = '' if self._uri is not None and len(self._uri) > 0: builder = self._uri builder += self.fix_route(self._base_route) if route[0] != '/': builder += '/' builder += route return builder def add_correlation_id(self, correlation_id=None, params=None): params = params or {} if not (correlation_id is None): params['correlation_id'] = correlation_id return params def add_filter_params(self, params=None, filters=None): params = params or {} if not (filters is None): params.update(filters) return params def add_paging_params(self, params=None, paging=None): params = params or {} if not (paging is None): if not (paging['total'] is None): params['total'] = paging['total'] if not (paging['skip'] is None): params['skip'] = paging['skip'] if not (paging['take'] is None): params['take'] = paging['take'] # params.update(paging) return params def call(self, method, route, correlation_id=None, params=None, data=None): """ Calls a remote method via HTTP/REST protocol. :param method: HTTP method: "get", "head", "post", "put", "delete" :param route: a command route. Base route will be added to this route :param correlation_id: (optional) transaction id to trace execution through call chain. :param params: (optional) query parameters. :param data: (optional) body object. :return: result object """ method = method.upper() route = self.createRequestRoute(route) params = self.add_correlation_id(correlation_id=correlation_id, params=params) response = None result = None try: # Call the service data = self._to_json(data) response = requests.request(method, route, params=params, json=data, timeout=self._timeout) except Exception as ex: error = InvocationException(correlation_id, 'REST_ERROR', 'REST operation failed: ' + str(ex)).wrap(ex) raise error if response.status_code == 404 or response.status_code == 204: return None try: # Retrieve JSON data result = response.json() except: # Data is not in JSON if response.status_code < 400: raise UnknownException(correlation_id, 'FORMAT_ERROR', 'Failed to deserialize JSON data: ' + response.text) \ .with_details('response', response.text) else: raise UnknownException(correlation_id, 'UNKNOWN', 'Unknown error occured: ' + response.text) \ .with_details('response', response.text) # Return result if response.status_code < 400: return result # Raise error # Todo: We need to implement proper from_value method error = ErrorDescription.from_json(result) error.status = response.status_code raise ApplicationExceptionFactory.create(error)
class HttpEndpoint(IOpenable, IConfigurable, IReferenceable): """ Used for creating HTTP endpoints. An endpoint is a URL, at which a given service can be accessed by a client. ### Configuration parameters ### Parameters to pass to the [[configure]] method for component configuration: - connection(s) - the connection resolver's connections; - "connection.discovery_key" - the key to use for connection resolving in a discovery service; - "connection.protocol" - the connection's protocol; - "connection.host" - the target host; - "connection.port" - the target port; - "connection.uri" - the target URI. ### References ### A logger, counters, and a connection resolver can be referenced by passing the following references to the object's [[setReferences]] method: - *:logger:*:*:1.0 (optional) ILogger components to pass log messages - *:counters:*:*:1.0 (optional) ICounters components to pass collected measurements - *:discovery:*:*:1.0 (optional) IDiscovery services to resolve connection Example: def my_method(_config, _references): endpoint = HttpEndpoint() if (_config) endpoint.configure(_config) if (_references) endpoint.setReferences(_references) ... endpoint.open(correlationId) ... """ _default_config = None _connection_resolver = None _logger = None _counters = None _registrations = None _service = None _server = None _debug = False _uri = None _file_max_size = 200 * 1024 * 1024 _maintenance_enabled = False _protocol_upgrade_enabled = False def __init__(self): """ Creates HttpEndpoint """ self._default_config = ConfigParams.from_tuples( "connection.protocol", "http", "connection.host", "0.0.0.0", "connection.port", 3000, "credential.ssl_key_file", None, "credential.ssl_crt_file", None, "credential.ssl_ca_file", None, "options.maintenance_enabled", False, "options.request_max_size", 1024 * 1024, "options.file_max_size", 200 * 1024 * 1024, "connection.connect_timeout", 60000, "connection.debug", True) self._connection_resolver = HttpConnectionResolver() self._logger = CompositeLogger() self._counters = CompositeCounters() self._registrations = [] def configure(self, config): """ Configures this HttpEndpoint using the given configuration parameters. - connection(s) - the connection resolver's connections; - "connection.discovery_key" - the key to use for connection resolving in a discovery service; - "connection.protocol" - the connection's protocol; - "connection.host" - the target host; - "connection.port" - the target port; - "connection.uri" - the target URI. :param config: configuration parameters, containing a "connection(s)" section. """ config = config.set_defaults(self._default_config) self._connection_resolver.configure(config) self._file_max_size = config.get_as_boolean_with_default( 'options.file_max_size', self._file_max_size) self._maintenance_enabled = config.get_as_long_with_default( 'options.maintenance_enabled', self._maintenance_enabled) self._protocol_upgrade_enabled = config.get_as_boolean_with_default( 'options.protocol_upgrade_enabled', self._protocol_upgrade_enabled) self._debug = config.get_as_boolean_with_default( 'connection.debug', self._debug) def set_references(self, references): """ Sets references to this endpoint's logger, counters, and connection resolver. - *:logger:*:*:1.0 (optional) ILogger components to pass log messages - *:counters:*:*:1.0 (optional) ICounters components to pass collected measurements - *:discovery:*:*:1.0 (optional) IDiscovery services to resolve connection :param references: an IReferences object, containing references to a logger, counters, and a connection resolver. """ self._logger.set_references(references) self._counters.set_references(references) self._connection_resolver.set_references(references) def is_opened(self): """ Checks if the component is opened. :return: whether or not this endpoint is open with an actively listening REST server. """ return not (self._server is None) def open(self, correlation_id): """ Opens a connection using the parameters resolved by the referenced connection resolver and creates a REST server (service) using the set options and parameters. :param correlation_id: (optional) transaction id to trace execution through call chain. """ if self.is_opened(): return connection = self._connection_resolver.resolve(correlation_id) if connection is None: raise ConfigException(correlation_id, "NO_CONNECTION", "Connection for REST client is not defined") self._uri = connection.get_uri() # verify https with bottle certfile = None keyfile = None if connection.get_protocol('http') == 'https': certfile = connection.get_as_nullable_string('ssl_crt_file') keyfile = connection.get_as_nullable_string('ssl_key_file') # Create instance of bottle application self._service = SessionMiddleware( bottle.Bottle(catchall=True, autojson=True)).app self._service.config['catchall'] = True self._service.config['autojson'] = True # Enable CORS requests self._service.add_hook('after_request', self._enable_cors) self._service.route('/', 'OPTIONS', self._options_handler) self._service.route('/<path:path>', 'OPTIONS', self._options_handler) self._service.add_hook('after_request', self._do_maintance) self._service.add_hook('after_request', self._no_cache) self._service.add_hook('before_request', self._add_compatibility) # Register routes # self.perform_registrations() def start_server(): self._service.run(server=self._server, debug=self._debug) # self.perform_registrations() host = connection.get_host() port = connection.get_port() # Starting service try: self._server = SSLCherryPyServer(host=host, port=port, certfile=certfile, keyfile=keyfile) # Start server in thread Thread(target=start_server).start() # Give 2 sec for initialization self._connection_resolver.register(correlation_id) self._logger.debug( correlation_id, f"Opened REST service at {self._uri}", ) self.perform_registrations() except Exception as ex: self._server = None raise ConnectionException(correlation_id, 'CANNOT_CONNECT', 'Opening REST service failed') \ .wrap(ex).with_details('url', self._uri) def close(self, correlation_id): """ Closes this endpoint and the REST server (service) that was opened earlier. :param correlation_id: (optional) transaction id to trace execution through call chain. """ try: if not (self._server is None): self._server.shutdown() self._service.close() self._logger.debug(correlation_id, "Closed REST service at %s", self._uri) self._server = None self._service = None self._uri = None except Exception as ex: self._logger.warn(correlation_id, "Failed while closing REST service: " + str(ex)) def register(self, registration): """ Registers a registerable object for dynamic endpoint discovery. :param registration: the registration to add. """ self._registrations.append(registration) def unregister(self, registration): """ Unregisters a registerable object, so that it is no longer used in dynamic endpoint discovery. :param registration: the registration to remove. """ self._registrations.remove(registration) def perform_registrations(self): for registration in self._registrations: registration.register() def fix_route(self, route) -> str: if route is not None and len(route) > 0: if route[0] != '/': route = f'/{route}' return route return '' def register_route(self, method, route, schema, handler): """ Registers an action in this objects REST server (service) by the given method and route. :param method: the HTTP method of the route. :param route: the route to register in this object's REST server (service). :param schema: the schema to use for parameter validation. :param handler: the action to perform at the given route. """ method = method.upper() # if method == 'DELETE': # method = 'DEL' route = self.fix_route(route) def wrapper(*args, **kwargs): try: if isinstance(schema, Schema): params = self.get_data() correlation_id = params[ 'correlation_id'] if 'correlation_id' in params else None error = schema.validate_and_throw_exception( correlation_id, params, False) return handler(*args, **kwargs) except Exception as ex: return HttpResponseSender.send_error(ex) self._service.route(route, method, wrapper) def get_data(self): if request.json: return request.json else: return None def _enable_cors(self): response.headers['Access-Control-Max-Age'] = '5' response.headers['Access-Control-Allow-Origin'] = '*' response.headers[ 'Access-Control-Allow-Methods'] = 'PUT, GET, POST, DELETE, OPTIONS' response.headers[ 'Access-Control-Allow-Headers'] = 'Authorization, Origin, Accept, Content-Type, X-Requested-With' def _do_maintance(self): """ :return: maintenance error code """ # Make this more sophisticated if self._maintenance_enabled: response.headers['Retry-After'] = 3600 response.status = 503 def _no_cache(self): """ Prevents IE from caching REST requests """ response.headers[ 'Cache-Control'] = 'no-cache, no-store, must-revalidate' response.headers['Pragma'] = 'no-cache' response.headers['Expires'] = 0 def _add_compatibility(self): def inner(name): if request.query: param = request.query[name] if param: return param if request.body: param = request.json[name] if param: return param if request.params: param = request.params[name] if param: return param return None request['param'] = inner request['route'] = {'params': request.params} def _options_handler(self, ath=None): return def get_param(self, param, default=None): return request.params.get(param, default) def get_correlation_id(self): return request.query.get('correlation_id') def register_route_with_auth(self, method, route, schema, authorize, action): """ Registers an action with authorization in this objects REST server (service) by the given method and route. :param method: the HTTP method of the route. :param route: the route to register in this object's REST server (service). :param schema: the schema to use for parameter validation. :param authorize: the authorization interceptor :param action: the action to perform at the given route. """ if authorize: next_action = action action = lambda req, res: authorize( request, response, next_action(response, response)) self.register_route(method, route, schema, action) def register_interceptor(self, route, action): """ Registers a middleware action for the given route. :param route: the route to register in this object's REST server (service). :param action: the middleware action to perform at the given route. """ route = self.fix_route(route) self._service.add_hook( 'before_request', lambda: action(request, response) if not (route is not None and route != '' and request.url. startswith(route)) else None)