class DummyController(IReferenceable, IReconfigurable, IOpenable, INotifiable):
__timer: FixedRateTimer = None
__logger: CompositeLogger = None
__message: str = None
__counter: int = None
def __init__(self):
self.__message = "Hello World!"
self.__logger = CompositeLogger()
self.__timer = FixedRateTimer(self, 1000, 1000)
self.__counter = 0
@property
def message(self) -> str:
return self.__message
@message.setter
def message(self, value: str):
self.__message = value
@property
def counter(self) -> int:
return self.__counter
@counter.setter
def counter(self, value: int):
self.__counter = value
def configure(self, config: ConfigParams):
self.__message = config.get_as_string_with_default("message", self.__message)
def set_references(self, references: IReferences):
self.__logger.set_references(references)
def is_open(self) -> bool:
return self.__timer.is_started()
def open(self, correlation_id: Optional[str]):
self.__timer.start()
self.__logger.trace(correlation_id, "Dummy controller opened")
def close(self, correlation_id: Optional[str]):
self.__timer.stop()
self.__logger.trace(correlation_id, "Dummy controller closed")
def notify(self, correlation_id: Optional[str], args: Parameters):
self.counter += 1
self.__logger.info(correlation_id, "%d - %s", self.counter, self.message)
class RestService(IOpenable, IConfigurable, IReferenceable, IUnreferenceable,
IRegisterable):
"""
Abstract service that receives remove calls via HTTP/REST protocol.
### Configuration parameters ###
- base_route: base route for remote URI
- dependencies:
- endpoint: override for HTTP Endpoint dependency
- controller: override for Controller dependency
- connection(s):
- discovery_key: (optional) a key to retrieve the connection from IDiscovery
- protocol: connection protocol: http or https
- host: host name or IP address
- port: port number
- uri: resource URI or connection string with all parameters in it
### References ###
- *:logger:*:*:1.0 (optional) ILogger components to pass log messages
- *:counters:*:*:1.0 (optional) ICounters components to pass collected measurements
- *:discovery:*:*:1.0 (optional) IDiscovery services to resolve connection
- *:endpoint:http:*:1.0 (optional) HttpEndpoint reference
Example:
class MyRestService(RestService):
_controller = None
...
def __init__(self):
super(MyRestService, self).__init__()
self._dependencyResolver.put("controller", Descriptor("mygroup","controller","*","*","1.0"))
def set_references(self, references):
super(MyRestService, self).set_references(references)
self._controller = self._dependencyResolver.get_required("controller")
def register():
...
service = MyRestService()
service.configure(ConfigParams.from_tuples("connection.protocol", "http",
"connection.host", "localhost",
"connection.port", 8080))
service.set_references(References.from_tuples(Descriptor("mygroup","controller","default","default","1.0"), controller))
service.open("123")
"""
_default_config = None
_debug = False
_dependency_resolver = None
_logger = None
_counters = None
# _registered = None
_local_endpoint = None
_config = None
_references = None
_base_route = None
_endpoint = None
_opened = None
def __init__(self):
self._default_config = ConfigParams.from_tuples(
"base_route", None, "dependencies.endpoint",
"*:endpoint:http:*:1.0")
# self._registered = False
self._dependency_resolver = DependencyResolver()
self._logger = CompositeLogger()
self._counters = CompositeCounters()
def _instrument(self, correlation_id, name):
"""
Adds instrumentation to log calls and measure call time. It returns a Timing object that is used to end the time measurement.
:param correlation_id: (optional) transaction id to trace execution through call chain.
:param name: a method name.
"""
self._logger.trace(correlation_id, "Executing " + name + " method")
return self._counters.begin_timing(name + ".exec_time")
def set_references(self, references):
"""
Sets references to dependent components.
:param references: references to locate the component dependencies.
"""
self._references = references
self._logger.set_references(references)
self._counters.set_references(references)
self._dependency_resolver.set_references(references)
self._endpoint = self._dependency_resolver.get_one_optional('endpoint')
if self._endpoint is None:
self._endpoint = self.create_endpoint()
self._local_endpoint = True
else:
self._local_endpoint = False
self._endpoint.register(self)
def configure(self, config):
"""
Configures component by passing configuration parameters.
:param config: configuration parameters to be set.
"""
config = config.set_defaults(self._default_config)
self._config = config
self._dependency_resolver.configure(config)
self._base_route = config.get_as_string_with_default(
"base_route", self._base_route)
def unset_references(self):
"""
Unsets (clears) previously set references to dependent components.
"""
if not (self._endpoint is None):
self._endpoint.unregister(self)
self._endpoint = None
def create_endpoint(self):
endpoint = HttpEndpoint()
if not (self._config is None):
endpoint.configure(self._config)
if not (self._references is None):
endpoint.set_references(self._references)
return endpoint
def _instrument(self, correlation_id, name):
self._logger.trace(correlation_id, f"Executing {name} method")
self._counters.increment_one(f"{name}.exec_count")
return self._counters.begin_timing(f"{name}.exec_time")
def _instrument_error(self, correlation_id, name, error, result, callback):
if not (error is None):
self._logger.error(correlation_id, error,
f"Failed to execute {name} method")
self._counters.increment_one(f"{name}.exec_error")
if not (callback is None):
callback(error, result)
def is_opened(self):
"""
Checks if the component is opened.
:return: true if the component has been opened and false otherwise.
"""
return self._opened
def open(self, correlation_id):
"""
Opens the component.
:param correlation_id: (optional) transaction id to trace execution through call chain.
"""
if self.is_opened():
return
if self._endpoint is None:
self._endpoint = self.create_endpoint()
self._endpoint.register(self)
self._local_endpoint = True
if self._local_endpoint:
self._endpoint.open(correlation_id)
self._opened = True
# # register route
# if self._registered != True:
# self.add_route()
# self._registered = True
def close(self, correlation_id):
"""
Closes component and frees used resources.
:param correlation_id: (optional) transaction id to trace execution through call chain.
"""
if not self._opened:
return
if self._endpoint is None:
raise InvalidStateException(correlation_id, "NO_ENDPOINT",
"HTTP endpoint is missing")
if self._local_endpoint:
self._endpoint.close(correlation_id)
self._opened = False
def send_result(self, result):
"""
Creates a callback function that sends result as JSON object. That callack function call be called directly or passed as a parameter to business logic components.
If object is not null it returns 200 status code. For null results it returns
204 status code. If error occur it sends ErrorDescription with approproate status code.
:param result: a body object to result.
:return: execution result.
"""
return HttpResponseSender.send_result(result)
def send_created_result(self, result):
"""
Creates a callback function that sends newly created object as JSON. That callack function call be called directly or passed as a parameter to business logic components.
If object is not null it returns 201 status code. For null results it returns
204 status code. If error occur it sends ErrorDescription with approproate status code.
:param result: a body object to result.
:return: execution result.
"""
return HttpResponseSender.send_created_result(result)
def send_deleted_result(self):
"""
Creates a callback function that sends newly created object as JSON. That callack function call be called directly or passed as a parameter to business logic components.
If object is not null it returns 200 status code. For null results it returns
204 status code. If error occur it sends ErrorDescription with approproate status code.
:return: execution result.
"""
return HttpResponseSender.send_deleted_result()
def send_error(self, error):
"""
Sends error serialized as ErrorDescription object and appropriate HTTP status code. If status code is not defined, it uses 500 status code.
:param error: an error object to be sent.
"""
return HttpResponseSender.send_error(error)
def fix_route(self, route) -> str:
if (route is not None and len(route) > 0):
if (route[0] != '/'):
route = f'/{route}'
return route
return ''
def register_route(self, method, route, schema, handler):
"""
Registers an action in this objects REST server (service) by the given method and route.
:param method: the HTTP method of the route.
:param route: the route to register in this object's REST server (service).
:param schema: the schema to use for parameter validation.
:param handler: the action to perform at the given route.
"""
if self._endpoint is None:
return
route = f"{self.fix_route(self._base_route)}{self.fix_route(route)}"
# if not (self._base_route is None) and len(self._base_route) > 0:
# base_route = self._base_route
# if base_route[0] != '/':
# base_route = '/' + base_route
# if route[0] != '/':
# base_route = base_route + '/'
# route = base_route + route
self._endpoint.register_route(method, route, schema, handler)
def register(self):
pass
def get_data(self):
data = bottle.request.json
if isinstance(data, str):
return json.loads(bottle.request.json)
elif bottle.request.json:
return bottle.request.json
else:
return None
def get_request_param(self):
if (not (bottle is None)) and (not (bottle.request is None)) and (
not (bottle.request.params is None)):
return bottle.request.params
else:
return {}
def _append_base_route(self, route):
route = route or ''
if self._base_route is not None and len(self._base_route) > 0:
base_route = self._base_route
if base_route[0] != '/':
base_route = '/' + base_route
route = base_route + route
return route
def register_route_with_auth(self, method, route, schema, authorize,
action):
"""
Registers a route with authorization in HTTP endpoint.
:param method: HTTP method: "get", "head", "post", "put", "delete"
:param route: a command route. Base route will be added to this route
:param schema: a validation schema to validate received parameters.
:param authorize: an authorization interceptor
:param action: an action function that is called when operation is invoked.
"""
if self._endpoint is None:
return
route = self._append_base_route(self.fix_route(route))
self._endpoint.register_route_with_auth(method, route, schema,
authorize, method)
def register_interceptor(self, route, action):
"""
Registers a middleware for a given route in HTTP endpoint.
:param route: a command route. Base route will be added to this route
:param action: an action function that is called when middleware is invoked.
"""
if self._endpoint is None:
return
route = self._append_base_route(self.fix_route(route))
self._endpoint.register_interceptor(route, action)
class MessageQueue(IConfigurable, IReferenceable, IMessageQueue):
"""
Abstract message queue.
Abstract message queue that is used as a basis for specific message queue implementations.
### Configuration parameters ###
- name: name of the message queue
- connection(s):
- discovery_key: key to retrieve parameters from discovery service
- protocol: connection protocol like http, https, tcp, udp
- host: host name or IP address
- port: port number
- uri: resource URI or connection string with all parameters in it
- credential(s):
- store_key: key to retrieve parameters from credential store
- username: user name
- password: user password
- access_id: application access id
- access_key: application secret key
### References ###
- *:logger:*:*:1.0 (optional) ILogger components to pass log messages
- *:counters:*:*:1.0 (optional) ICounters components to pass collected measurements
- *:discovery:*:*:1.0 (optional) IDiscovery components to discover connection(s)
- *:credential-store:*:*:1.0 (optional) ICredentialStore componetns to lookup credential(s)
"""
_name = None
_capabilities = None
_lock = None
_logger = None
_counters = None
_credential_resolver = None
_connection_resolver = None
def __init__(self, name=None):
"""
Creates a new instance of the message queue.
:param name: (optional) a queue name
"""
self._lock = threading.Lock()
self._logger = CompositeLogger()
self._counters = CompositeCounters()
self._connection_resolver = ConnectionResolver()
self._credential_resolver = CredentialResolver()
self._name = name
def configure(self, config):
"""
Configures component by passing configuration parameters.
:param config: configuration parameters to be set.
"""
self._name = NameResolver.resolve(config)
self._logger.configure(config)
self._credential_resolver.configure(config)
self._connection_resolver.configure(config)
def set_references(self, references):
"""
Sets references to dependent components.
:param references: references to locate the component dependencies.
"""
self._logger.set_references(references)
self._counters.set_references(references)
self._credential_resolver.set_references(references)
self._connection_resolver.set_references(references)
def open(self, correlation_id):
"""
Opens the component.
:param correlation_id: (optional) transaction id to trace execution through call chain.
"""
connection = self._connection_resolver.resolve(correlation_id)
credential = self._credential_resolver.lookup(correlation_id)
self._open_with_params(correlation_id, connection, credential)
def _open_with_params(self, correlation_id, connection, credential):
"""
Opens the component with given connection and credential parameters.
:param correlation_id: (optional) transaction id to trace execution through call chain.
:param connection: connection parameters
:param credential: credential parameters
"""
raise NotImplementedError('Abstract method that shall be overriden')
def get_name(self):
"""
Gets the queue name
:return: the queue name.
"""
return self._name if self._name != None else "undefined"
def get_capabilities(self):
"""
Gets the queue capabilities
:return: the queue's capabilities object.
"""
return self._capabilities
def send_as_object(self, correlation_id, message_type, message):
"""
Sends an object into the queue.
Before sending the object is converted into JSON string and wrapped in a [[MessageEnvelop]].
:param correlation_id: (optional) transaction id to trace execution through call chain.
:param message_type: a message type
:param message: an object value to be sent
"""
envelop = MessageEnvelop(correlation_id, message_type, message)
self.send(correlation_id, envelop)
def begin_listen(self, correlation_id, receiver):
"""
Listens for incoming messages without blocking the current thread.
:param correlation_id: (optional) transaction id to trace execution through call chain.
:param receiver: a receiver to receive incoming messages.
"""
# Start listening on a parallel tread
thread = threading.Thread(target=self.listen,
args=(correlation_id, receiver))
thread.daemon = True
thread.start()
def __str__(self):
"""
Gets a string representation of the object.
:return: a string representation of the object.
"""
return "[" + self.get_name() + "]"
class RestClient(IOpenable, IConfigurable, IReferenceable):
"""
Abstract client that calls remove endpoints using HTTP/REST protocol.
### Configuration parameters ###
- base_route: base route for remote URI
- connection(s):
- discovery_key: (optional) a key to retrieve the connection from IDiscovery
- protocol: connection protocol: http or https
- host: host name or IP address
- port: port number
- uri: resource URI or connection string with all parameters in it
- options:
- retries: number of retries (default: 3)
- connect_timeout: connection timeout in milliseconds (default: 10 sec)
- timeout: invocation timeout in milliseconds (default: 10 sec)
### References ###
- *:logger:*:*:1.0 (optional) ILogger components to pass log messages
- *:counters:*:*:1.0 (optional) ICounters components to pass collected measurements
- *:discovery:*:*:1.0 (optional) IDiscovery services to resolve connection
Example:
class MyRestClient(RestClient, IMyClient):
def get_data(self, correlation_id, id):
timing = self.instrument(correlationId, 'myclient.get_data')
result = self._controller.get_data(correlationId, id)
timing.end_timing()
return result
...
client = MyRestClient()
client.configure(ConfigParams.fromTuples("connection.protocol", "http",
"connection.host", "localhost",
"connection.port", 8080))
data = client.getData("123", "1")
...
"""
_default_config = None
_client = None
_uri = None
_timeout = 1000
_connection_resolver = None
_logger = None
_counters = None
_options = None
_base_route = None
_retries = 1
_headers = None
_connect_timeout = 1000
def __init__(self):
"""
Creates a new instance of the client.
"""
self._connection_resolver = HttpConnectionResolver()
self._default_config = ConfigParams.from_tuples(
"connection.protocol", "http", "connection.host", "0.0.0.0",
"connection.port", 3000, "options.timeout", 10000,
"options.request_max_size", 1024 * 1024, "options.connect_timeout",
10000, "options.retries", 3, "options.debug", True)
self._logger = CompositeLogger()
self._counters = CompositeCounters()
self._options = ConfigParams()
self._headers = {}
def set_references(self, references):
"""
Sets references to dependent components.
:param references: references to locate the component dependencies.
"""
self._logger.set_references(references)
self._counters.set_references(references)
self._connection_resolver.set_references(references)
def configure(self, config):
"""
Configures component by passing configuration parameters.
:param config: configuration parameters to be set.
"""
config = config.set_defaults(self._default_config)
self._connection_resolver.configure(config)
self._options.override(config.get_section("options"))
self._retries = config.get_as_integer_with_default(
"options.retries", self._retries)
self._connect_timeout = config.get_as_integer_with_default(
"options.connect_timeout", self._connect_timeout)
self._timeout = config.get_as_integer_with_default(
"options.timeout", self._timeout)
self._base_route = config.get_as_string_with_default(
"base_route", self._base_route)
def _instrument(self, correlation_id, name):
"""
Adds instrumentation to log calls and measure call time. It returns a Timing object that is used to end the time measurement.
:param correlation_id: (optional) transaction id to trace execution through call chain.
:param name: a method name.
:return: Timing object to end the time measurement.
"""
TYPE_NAME = self.__class__.__name__ or 'unknown-target'
self._logger.trace(correlation_id,
f"Calling {name} method {TYPE_NAME}")
self._counters.increment_one(f"{TYPE_NAME}.{name}.call_count")
return self._counters.begin_timing(f"{TYPE_NAME}.{name}.call_count")
def _instrument_error(self,
correlation_id,
name,
err,
result=None,
callback=None):
"""
Adds instrumentation to error handling.
:param correlation_id: (optional) transaction id to trace execution through call chain.
:param name: a method name.
:param err: an occured error
:param result: (optional) an execution result
:param callback: (optional) an execution callback
"""
if err is not None:
TYPE_NAME = self.__class__.__name__ or 'unknown-target'
self._logger.error(correlation_id, err,
f"Failed to call {name} method of {TYPE_NAME}")
self._counters.increment_one(f"{name}.call_errors")
if callback:
callback(err, result)
def is_opened(self):
"""
Checks if the component is opened.
:return: true if the component has been opened and false otherwise.
"""
return self._client is not None
def open(self, correlation_id):
"""
Opens the component.
:param correlation_id: (optional) transaction id to trace execution through call chain.
"""
if self.is_opened():
return
connection = self._connection_resolver.resolve(correlation_id)
self._uri = connection.get_uri()
self._client = requests
self._logger.debug(correlation_id,
"Connected via REST to " + self._uri)
def close(self, correlation_id):
"""
Closes component and frees used resources.
:param correlation_id: (optional) transaction id to trace execution through call chain.
"""
if self._client is not None:
self._logger.debug(correlation_id,
"Disconnected from " + self._uri)
self._client = None
self._uri = None
def _to_json(self, obj):
if obj is None:
return None
if isinstance(obj, set):
obj = list(obj)
if isinstance(obj, list):
result = []
for item in obj:
item = self._to_json(item)
result.append(item)
return result
if isinstance(obj, dict):
result = {}
for (k, v) in obj.items():
v = self._to_json(v)
result[k] = v
return result
if hasattr(obj, 'to_json'):
return obj.to_json()
if hasattr(obj, '__dict__'):
return self._to_json(obj.__dict__)
return obj
def fix_route(self, route) -> str:
if route is not None and len(route) > 0:
if route[0] != '/':
route = f'/{route}'
return route
return ''
def createRequestRoute(self, route):
builder = ''
if self._uri is not None and len(self._uri) > 0:
builder = self._uri
builder += self.fix_route(self._base_route)
if route[0] != '/':
builder += '/'
builder += route
return builder
def add_correlation_id(self, correlation_id=None, params=None):
params = params or {}
if not (correlation_id is None):
params['correlation_id'] = correlation_id
return params
def add_filter_params(self, params=None, filters=None):
params = params or {}
if not (filters is None):
params.update(filters)
return params
def add_paging_params(self, params=None, paging=None):
params = params or {}
if not (paging is None):
if not (paging['total'] is None):
params['total'] = paging['total']
if not (paging['skip'] is None):
params['skip'] = paging['skip']
if not (paging['take'] is None):
params['take'] = paging['take']
# params.update(paging)
return params
def call(self, method, route, correlation_id=None, params=None, data=None):
"""
Calls a remote method via HTTP/REST protocol.
:param method: HTTP method: "get", "head", "post", "put", "delete"
:param route: a command route. Base route will be added to this route
:param correlation_id: (optional) transaction id to trace execution through call chain.
:param params: (optional) query parameters.
:param data: (optional) body object.
:return: result object
"""
method = method.upper()
route = self.createRequestRoute(route)
params = self.add_correlation_id(correlation_id=correlation_id,
params=params)
response = None
result = None
try:
# Call the service
data = self._to_json(data)
response = requests.request(method,
route,
params=params,
json=data,
timeout=self._timeout)
except Exception as ex:
error = InvocationException(correlation_id, 'REST_ERROR',
'REST operation failed: ' +
str(ex)).wrap(ex)
raise error
if response.status_code == 404 or response.status_code == 204:
return None
try:
# Retrieve JSON data
result = response.json()
except:
# Data is not in JSON
if response.status_code < 400:
raise UnknownException(correlation_id, 'FORMAT_ERROR',
'Failed to deserialize JSON data: ' + response.text) \
.with_details('response', response.text)
else:
raise UnknownException(correlation_id, 'UNKNOWN', 'Unknown error occured: ' + response.text) \
.with_details('response', response.text)
# Return result
if response.status_code < 400:
return result
# Raise error
# Todo: We need to implement proper from_value method
error = ErrorDescription.from_json(result)
error.status = response.status_code
raise ApplicationExceptionFactory.create(error)
class DirectClient(IConfigurable, IReferenceable, IOpenable):
"""
Abstract client that calls controller directly in the same memory space. It is used when multiple microservices are deployed in a single container (monolyth) and communication between them can be done by direct calls rather then through the network.
### Configuration parameters ###
- dependencies:
- controller: override controller descriptor
### References ###
- *:logger:*:*:1.0 (optional) ILogger components to pass log messages
- *:counters:*:*:1.0 (optional) ICounters components to pass collected measurements
- *:controller:*:*:1.0 controller to call business methods
Example:
class MyDirectClient(DirectClient, IMyClient):
def __init__(self):
super(MyDirectClient, self).__init__()
self._dependencyResolver.put('controller', Descriptor("mygroup", "controller", "*", "*", "*"))
...
def get_data(self, correlation_id, id):
timing = self.instrument(correlationId, 'myclient.get_data')
result = self._controller.get_data(correlationId, id)
timing.end_timing()
return result
client = MyDirectClient()
client.set_references(References.from_tuples(Descriptor("mygroup","controller","default","default","1.0"), controller))
data = client.get_data("123", "1")
...
"""
_controller = None
_opened = True
_logger = None
_counters = None
_dependency_resolver = None
def __init__(self):
"""
Creates a new instance of the client.
"""
self._logger = CompositeLogger()
self._counters = CompositeCounters()
self._dependency_resolver = DependencyResolver()
self._dependency_resolver.put('controller', 'none')
def configure(self, config):
"""
Configures component by passing configuration parameters.
:param config: configuration parameters to be set.
"""
self._dependency_resolver.configure(config)
def set_references(self, references):
"""
Sets references to dependent components.
:param references: references to locate the component dependencies.
"""
self._logger.set_references(references)
self._counters.set_references(references)
self._dependency_resolver.set_references(references)
self._controller = self._dependency_resolver.get_one_required('controller')
def _instrument(self, correlation_id, name):
"""
Adds instrumentation to log calls and measure call time. It returns a Timing object that is used to end the time measurement.
:param correlation_id: (optional) transaction id to trace execution through call chain.
:param name: a method name.
:return: Timing object to end the time measurement.
"""
self._logger.trace(correlation_id, f"Executing {name} method")
return self._counters.begin_timing(f"{name} .call_time")
def _instrument_error(self, correlation_id, name, err, result, callback):
"""
Adds instrumentation to error handling.
:param correlation_id: (optional) transaction id to trace execution through call chain.
:param name: a method name.
:param err: an occured error
:param result: (optional) an execution result
:param callback: (optional) an execution callback
"""
if err is not None:
self._logger.error(correlation_id, err, f'Failed to call {name} method')
self._counters.increment_one(f"{name}.call_errors")
if callback:
callback(err, result)
def is_opened(self):
"""
Checks if the component is opened.
:return: true if the component has been opened and false otherwise.
"""
return self._opened
def open(self, correlation_id):
"""
Opens the component.
:param correlation_id: (optional) transaction id to trace execution through call chain.
"""
if self._opened:
return
if self._controller is None:
raise ConnectionException(correlation_id, 'NO_CONTROLLER', 'Controller references is missing')
self._opened = True
self._logger.info(correlation_id, 'Opened direct client')
def close(self, correlation_id):
"""
Closes component and frees used resources.
:param correlation_id: (optional) transaction id to trace execution through call chain.
"""
if self._opened:
self._logger.info(correlation_id, 'Closed direct client')
self._opened = False
class MemoryPersistence(IReferenceable, IOpenable, ICleanable):
"""
Abstract persistence component that stores data in memory.
This is the most basic persistence component that is only
able to store data items of any type. Specific CRUD operations
over the data items must be implemented in child classes by
accessing <code>this._items</code> property and calling
[[save]] method.
The component supports loading and saving items from another
data source. That allows to use it as a base class for file
and other types of persistence components that cache all data
in memory.
### References ###
- *:logger:*:*:1.0 (optional) ILogger components to pass log messages
Example:
class MyMemoryPersistence(MemoryPersistence):
def get_by_name(self, correlationId, name):
item = self.find(name)
...
return item
persistence = MyMemoryPersistence()
persistence.set("123", MyData("ABC"))
print str(persistence.get_by_name("123", "ABC")))
"""
_logger = None
_items = None
_loader = None
_saver = None
_lock = None
_opened = False
def __init__(self, loader = None, saver = None):
"""
Creates a new instance of the persistence.
:param loader: (optional) a loader to load items from external datasource.
:param saver: (optional) a saver to save items to external datasource.
"""
self._lock = threading.Lock()
self._logger = CompositeLogger()
self._items = []
self._loader = loader
self._saver = saver
def set_references(self, references):
"""
Sets references to dependent components.
:param references: references to locate the component dependencies.
"""
self._logger.set_references(references)
def is_opened(self):
"""
Checks if the component is opened.
:return: true if the component has been opened and false otherwise.
"""
return self._opened
def open(self, correlation_id):
"""
Opens the component.
:param correlation_id: (optional) transaction id to trace execution through call chain.
"""
self.load(correlation_id)
self._opened = True
def close(self, correlation_id):
"""
Closes component and frees used resources.
:param correlation_id: (optional) transaction id to trace execution through call chain.
"""
self.save(correlation_id)
self._opened = False
def _convert_to_public(self, value):
return value
def _convert_from_public(self, value):
return value
def load(self, correlation_id):
if self._loader == None: return
self._lock.acquire()
try:
items = self._loader.load(correlation_id)
self._items = []
for item in items:
item = self._convert_to_public(item)
self._items.append(item)
finally:
self._lock.release()
self._logger.trace(correlation_id, "Loaded " + str(len(self._items)) + " items")
def save(self, correlation_id):
"""
Saves items to external data source using configured saver component.
:param correlation_id: (optional) transaction id to trace execution through call chain.
"""
if self._saver == None: return
self._lock.acquire()
try:
items = []
for item in self._items:
item = self._convert_from_public(item)
items.append(item)
self._saver.save(correlation_id, items)
finally:
self._lock.release()
self._logger.trace(correlation_id, "Saved " + str(len(self._items)) + " items")
def clear(self, correlation_id):
"""
Clears component state.
:param correlation_id: (optional) transaction id to trace execution through call chain.
"""
self._lock.acquire()
try:
del self._items[:]
finally:
self._lock.release()
self._logger.trace(correlation_id, "Cleared items")
# Outside of lock to avoid reentry
self.save(correlation_id)
class HttpEndpoint(IOpenable, IConfigurable, IReferenceable):
"""
Used for creating HTTP endpoints. An endpoint is a URL, at which a given service can be accessed by a client.
### Configuration parameters ###
Parameters to pass to the [[configure]] method for component configuration:
- connection(s) - the connection resolver's connections;
- "connection.discovery_key" - the key to use for connection resolving in a discovery service;
- "connection.protocol" - the connection's protocol;
- "connection.host" - the target host;
- "connection.port" - the target port;
- "connection.uri" - the target URI.
### References ###
A logger, counters, and a connection resolver can be referenced by passing the following references to the object's [[setReferences]] method:
- *:logger:*:*:1.0 (optional) ILogger components to pass log messages
- *:counters:*:*:1.0 (optional) ICounters components to pass collected measurements
- *:discovery:*:*:1.0 (optional) IDiscovery services to resolve connection
Example:
def my_method(_config, _references):
endpoint = HttpEndpoint()
if (_config)
endpoint.configure(_config)
if (_references)
endpoint.setReferences(_references)
...
endpoint.open(correlationId)
...
"""
_default_config = None
_connection_resolver = None
_logger = None
_counters = None
_registrations = None
_service = None
_server = None
_debug = False
_uri = None
_file_max_size = 200 * 1024 * 1024
_maintenance_enabled = False
_protocol_upgrade_enabled = False
def __init__(self):
"""
Creates HttpEndpoint
"""
self._default_config = ConfigParams.from_tuples(
"connection.protocol", "http", "connection.host", "0.0.0.0",
"connection.port", 3000, "credential.ssl_key_file", None,
"credential.ssl_crt_file", None, "credential.ssl_ca_file", None,
"options.maintenance_enabled", False, "options.request_max_size",
1024 * 1024, "options.file_max_size", 200 * 1024 * 1024,
"connection.connect_timeout", 60000, "connection.debug", True)
self._connection_resolver = HttpConnectionResolver()
self._logger = CompositeLogger()
self._counters = CompositeCounters()
self._registrations = []
def configure(self, config):
"""
Configures this HttpEndpoint using the given configuration parameters.
- connection(s) - the connection resolver's connections;
- "connection.discovery_key" - the key to use for connection resolving in a discovery service;
- "connection.protocol" - the connection's protocol;
- "connection.host" - the target host;
- "connection.port" - the target port;
- "connection.uri" - the target URI.
:param config: configuration parameters, containing a "connection(s)" section.
"""
config = config.set_defaults(self._default_config)
self._connection_resolver.configure(config)
self._file_max_size = config.get_as_boolean_with_default(
'options.file_max_size', self._file_max_size)
self._maintenance_enabled = config.get_as_long_with_default(
'options.maintenance_enabled', self._maintenance_enabled)
self._protocol_upgrade_enabled = config.get_as_boolean_with_default(
'options.protocol_upgrade_enabled', self._protocol_upgrade_enabled)
self._debug = config.get_as_boolean_with_default(
'connection.debug', self._debug)
def set_references(self, references):
"""
Sets references to this endpoint's logger, counters, and connection resolver.
- *:logger:*:*:1.0 (optional) ILogger components to pass log messages
- *:counters:*:*:1.0 (optional) ICounters components to pass collected measurements
- *:discovery:*:*:1.0 (optional) IDiscovery services to resolve connection
:param references: an IReferences object, containing references to a logger, counters, and a connection resolver.
"""
self._logger.set_references(references)
self._counters.set_references(references)
self._connection_resolver.set_references(references)
def is_opened(self):
"""
Checks if the component is opened.
:return: whether or not this endpoint is open with an actively listening REST server.
"""
return not (self._server is None)
def open(self, correlation_id):
"""
Opens a connection using the parameters resolved by the referenced connection resolver and creates a REST server (service) using the set options and parameters.
:param correlation_id: (optional) transaction id to trace execution through call chain.
"""
if self.is_opened():
return
connection = self._connection_resolver.resolve(correlation_id)
if connection is None:
raise ConfigException(correlation_id, "NO_CONNECTION",
"Connection for REST client is not defined")
self._uri = connection.get_uri()
# verify https with bottle
certfile = None
keyfile = None
if connection.get_protocol('http') == 'https':
certfile = connection.get_as_nullable_string('ssl_crt_file')
keyfile = connection.get_as_nullable_string('ssl_key_file')
# Create instance of bottle application
self._service = SessionMiddleware(
bottle.Bottle(catchall=True, autojson=True)).app
self._service.config['catchall'] = True
self._service.config['autojson'] = True
# Enable CORS requests
self._service.add_hook('after_request', self._enable_cors)
self._service.route('/', 'OPTIONS', self._options_handler)
self._service.route('/<path:path>', 'OPTIONS', self._options_handler)
self._service.add_hook('after_request', self._do_maintance)
self._service.add_hook('after_request', self._no_cache)
self._service.add_hook('before_request', self._add_compatibility)
# Register routes
# self.perform_registrations()
def start_server():
self._service.run(server=self._server, debug=self._debug)
# self.perform_registrations()
host = connection.get_host()
port = connection.get_port()
# Starting service
try:
self._server = SSLCherryPyServer(host=host,
port=port,
certfile=certfile,
keyfile=keyfile)
# Start server in thread
Thread(target=start_server).start()
# Give 2 sec for initialization
self._connection_resolver.register(correlation_id)
self._logger.debug(
correlation_id,
f"Opened REST service at {self._uri}",
)
self.perform_registrations()
except Exception as ex:
self._server = None
raise ConnectionException(correlation_id, 'CANNOT_CONNECT', 'Opening REST service failed') \
.wrap(ex).with_details('url', self._uri)
def close(self, correlation_id):
"""
Closes this endpoint and the REST server (service) that was opened earlier.
:param correlation_id: (optional) transaction id to trace execution through call chain.
"""
try:
if not (self._server is None):
self._server.shutdown()
self._service.close()
self._logger.debug(correlation_id, "Closed REST service at %s",
self._uri)
self._server = None
self._service = None
self._uri = None
except Exception as ex:
self._logger.warn(correlation_id,
"Failed while closing REST service: " + str(ex))
def register(self, registration):
"""
Registers a registerable object for dynamic endpoint discovery.
:param registration: the registration to add.
"""
self._registrations.append(registration)
def unregister(self, registration):
"""
Unregisters a registerable object, so that it is no longer used in dynamic endpoint discovery.
:param registration: the registration to remove.
"""
self._registrations.remove(registration)
def perform_registrations(self):
for registration in self._registrations:
registration.register()
def fix_route(self, route) -> str:
if route is not None and len(route) > 0:
if route[0] != '/':
route = f'/{route}'
return route
return ''
def register_route(self, method, route, schema, handler):
"""
Registers an action in this objects REST server (service) by the given method and route.
:param method: the HTTP method of the route.
:param route: the route to register in this object's REST server (service).
:param schema: the schema to use for parameter validation.
:param handler: the action to perform at the given route.
"""
method = method.upper()
# if method == 'DELETE':
# method = 'DEL'
route = self.fix_route(route)
def wrapper(*args, **kwargs):
try:
if isinstance(schema, Schema):
params = self.get_data()
correlation_id = params[
'correlation_id'] if 'correlation_id' in params else None
error = schema.validate_and_throw_exception(
correlation_id, params, False)
return handler(*args, **kwargs)
except Exception as ex:
return HttpResponseSender.send_error(ex)
self._service.route(route, method, wrapper)
def get_data(self):
if request.json:
return request.json
else:
return None
def _enable_cors(self):
response.headers['Access-Control-Max-Age'] = '5'
response.headers['Access-Control-Allow-Origin'] = '*'
response.headers[
'Access-Control-Allow-Methods'] = 'PUT, GET, POST, DELETE, OPTIONS'
response.headers[
'Access-Control-Allow-Headers'] = 'Authorization, Origin, Accept, Content-Type, X-Requested-With'
def _do_maintance(self):
"""
:return: maintenance error code
"""
# Make this more sophisticated
if self._maintenance_enabled:
response.headers['Retry-After'] = 3600
response.status = 503
def _no_cache(self):
"""
Prevents IE from caching REST requests
"""
response.headers[
'Cache-Control'] = 'no-cache, no-store, must-revalidate'
response.headers['Pragma'] = 'no-cache'
response.headers['Expires'] = 0
def _add_compatibility(self):
def inner(name):
if request.query:
param = request.query[name]
if param:
return param
if request.body:
param = request.json[name]
if param:
return param
if request.params:
param = request.params[name]
if param:
return param
return None
request['param'] = inner
request['route'] = {'params': request.params}
def _options_handler(self, ath=None):
return
def get_param(self, param, default=None):
return request.params.get(param, default)
def get_correlation_id(self):
return request.query.get('correlation_id')
def register_route_with_auth(self, method, route, schema, authorize,
action):
"""
Registers an action with authorization in this objects REST server (service)
by the given method and route.
:param method: the HTTP method of the route.
:param route: the route to register in this object's REST server (service).
:param schema: the schema to use for parameter validation.
:param authorize: the authorization interceptor
:param action: the action to perform at the given route.
"""
if authorize:
next_action = action
action = lambda req, res: authorize(
request, response, next_action(response, response))
self.register_route(method, route, schema, action)
def register_interceptor(self, route, action):
"""
Registers a middleware action for the given route.
:param route: the route to register in this object's REST server (service).
:param action: the middleware action to perform at the given route.
"""
route = self.fix_route(route)
self._service.add_hook(
'before_request', lambda: action(request, response)
if not (route is not None and route != '' and request.url.
startswith(route)) else None)
class PrometheusCounters(CachedCounters, IReferenceable, IOpenable):
"""
Performance counters that send their metrics to Prometheus service.
The component is normally used in passive mode conjunction with :class:`PrometheusMetricsService <pip_services3_prometheus.services.PrometheusMetricsService.PrometheusMetricsService>`.
Alternatively when connection parameters are set it can push metrics to Prometheus PushGateway.
### Configuration parameters ###
- connection(s):
- discovery_key: (optional) a key to retrieve the connection from :class:`IDiscovery <pip_services3_components.connect.IDiscovery.IDiscovery>`
- protocol: connection protocol: http or https
- host: host name or IP address
- port: port number
- uri: resource URI or connection string with all parameters in it
- options:
- retries: number of retries (default: 3)
- connect_timeout: connection timeout in milliseconds (default: 10 sec)
- timeout: invocation timeout in milliseconds (default: 10 sec)
### References ###
- `*:logger:*:*:1.0` (optional) :class:`ILogger <pip_services3_components.log.ILogger.ILogger>` components to pass log messages
- `*:counters:*:*:1.0` (optional) :class:`ICounters <pip_services3_components.count.ICounters.ICounters>` components to pass collected measurements
- `*:discovery:*:*:1.0` (optional) :class:`IDiscovery <pip_services3_components.connect.IDiscovery.IDiscovery>` services to resolve connection
See :class:`RestService <pip_services3_rpc.services.RestService.RestService>`, :class:`CommandableHttpService <pip_services3_rpc.services.CommandableHttpService.CommandableHttpService>`,
Example:
.. code-block:: python
counters = PrometheusCounters()
counters.configure(ConfigParams.from_tuples(
"connection.protocol", "http",
"connection.host", "localhost",
"connection.port", 8080
))
counters.open("123")
counters.increment("mycomponent.mymethod.calls")
timing = counters.begin_timing("mycomponent.mymethod.exec_time")
try:
...
finally:
timing.end_timing()
counters.dump()
"""
def __init__(self):
"""
Creates a new instance of the performance counters.
"""
super(PrometheusCounters, self).__init__()
self.__logger = CompositeLogger()
self.__connection_resolver = HttpConnectionResolver()
self.__opened = False
self.__source: str = None
self.__instance: str = None
self.__push_enabled: bool = None
self.__client: Any = None
self.__request_route: str = None
def configure(self, config: ConfigParams):
"""
Configures component by passing configuration parameters.
:param config: configuration parameters to be set.
"""
super().configure(config)
self.__connection_resolver.configure(config)
self.__source = config.get_as_float_with_default(
'source', self.__source)
self.__instance = config.get_as_float_with_default(
'instance', self.__instance)
self.__push_enabled = config.get_as_float_with_default(
'push_enabled', True)
def set_references(self, references: IReferences):
"""
Sets references to dependent components.
:param references: references to locate the component dependencies.
"""
self.__logger.set_references(references)
self.__connection_resolver.set_references(references)
context_info = references.get_one_optional(
Descriptor("pip-services", "context-info", "default", "*", "1.0"))
if context_info is not None and self.__source is None:
self.__source = context_info.name
if context_info is not None and self.__instance is None:
self.__instance = context_info.context_id
def is_open(self) -> bool:
"""
Checks if the component is opened.
:return: true if the component has been opened and false otherwise.
"""
return self.__opened
def open(self, correlation_id: Optional[str]):
"""
Opens the component.
:param correlation_id: (optional) transaction id to trace execution through call chain.
"""
if self.__opened or not self.__push_enabled:
return
self.__opened = True
try:
connection = self.__connection_resolver.resolve(correlation_id)
job = self.__source or 'unknown'
instance = self.__instance or socket.gethostname()
self.__request_route = "/metrics/job/" + job + "/instance/" + instance
uri = connection.get_as_string('uri').split('://')[-1]
if connection.get_as_string('protocol') == 'https':
self.__client = HTTPSConnectionPool(uri)
else:
self.__client = HTTPConnectionPool(uri)
except Exception as err:
self.__client = None
self.__logger.warn(
correlation_id,
"Connection to Prometheus server is not configured: " +
str(err))
def close(self, correlation_id: Optional[str]):
"""
Closes component and frees used resources.
:param correlation_id: (optional) transaction id to trace execution through call chain.
"""
self.__opened = False
self.__request_route = None
try:
if self.__client:
self.__client.close()
finally:
self.__client = None
def _save(self, counters: List[Counter]):
"""
Saves the current counters measurements.
:param counters: current counters measurements to be saves.
"""
if self.__client is None or not self.__push_enabled: return
body = PrometheusCounterConverter.to_string(counters, None, None)
err = None
response = None
try:
response = self.__client.request('PUT',
self.__request_route,
body=body)
except Exception as ex:
err = ex
finally:
if err or response.status >= 400:
self.__logger.error("prometheus-counters", err,
"Failed to push metrics to prometheus")