def __init__(self, name=None):
"""
Creates a new instance of the message queue.
:param name: (optional) a queue name
"""
self._lock = threading.Lock()
self._logger = CompositeLogger()
self._counters = CompositeCounters()
self._connection_resolver = ConnectionResolver()
self._credential_resolver = CredentialResolver()
self._name = name
def __init__(self):
super().__init__()
self.__connection_resolver: ConnectionResolver = ConnectionResolver()
self.__credential_resolver: CredentialResolver = CredentialResolver()
self.__lock: str = IdGenerator.next_long()
self.__timeout: int = 30000
self.__retries: int = 3
self.__client: redis.Redis = None
def __init__(self):
"""
Creates a new instance of this cache
"""
self.__connection_resolver: ConnectionResolver = ConnectionResolver()
self.__credential_resolver: CredentialResolver = CredentialResolver()
self.__timeout: int = 30000
self.__retries: int = 3
self.__client: redis.Redis = None
def __init__(self):
"""
Creates a new instance of this cache.
"""
self.__connection_resolver: ConnectionResolver = ConnectionResolver()
# self.__max_key_size: int = 250
# self.__max_expiration: int = 2592000
# self.__max_value: int = 1048576
self.__pool_size: int = 5
self.__reconnect: int = 10000
self.__timeout: int = 5000
self.__retries: int = 5
self.__failures: int = 5
self.__retry: int = 30000
# self.__remove: bool = False
self.__idle: int = 5000
self.__client: pymemcache.HashClient = None
def __init__(self,
name: str = None,
capabilities: MessagingCapabilities = None):
"""
Creates a new instance of the message queue.
:param name: (optional) a queue name
:param capabilities: (optional) a capabilities of this message queue
"""
self._lock: threading.Lock = threading.Lock()
self._event = threading.Event()
self._capabilities: MessagingCapabilities = None
self._logger: CompositeLogger = CompositeLogger()
self._counters: CompositeCounters = CompositeCounters()
self._connection_resolver: ConnectionResolver = ConnectionResolver()
self._credential_resolver: CredentialResolver = CredentialResolver()
self._name: str = name
self._capabilities = capabilities or \
MessagingCapabilities(False, False, False, False, False, False, False, False, False)
def test_resolve(self):
connection_resolver = ConnectionResolver(RestConfig)
connection_params = connection_resolver.resolve("correlation_id")
assert "http" == connection_params.get("protocol")
assert "localhost" == connection_params.get("host")
assert "3000" == connection_params.get("port")
RestConfigDiscovery = ConfigParams.from_tuples(
"connection.protocol", "http", "connection.host", "localhost",
"connection.port", 3000, "connection.discovery_key",
"Discovery key value")
references = References()
connection_resolver = ConnectionResolver(RestConfigDiscovery,
references)
try:
connection_params = connection_resolver.resolve("correlation_id")
except Exception as ex:
assert "Discovery wasn't found to make resolution" == ex.message
def __init__(self):
# The connections resolver.
self._connection_resolver: ConnectionResolver = ConnectionResolver()
# The credentials resolver.
self._credential_resolver: CredentialResolver = CredentialResolver()
def __init__(self):
self._connection_resolver: ConnectionResolver = ConnectionResolver()
self._credential_resolver: CredentialResolver = CredentialResolver()
class MongoDbConnectionResolver(IReferenceable, IConfigurable):
"""
MongoDbConnectionResolver implementation.
Helper class that resolves MongoDB connection
and credential parameters, validates them and generates a connection URI.
It is able to process multiple connections to MongoDB cluster nodes.
### Configuration parameters ###
- connection(s):
- discovery_key: (optional) a key to retrieve the connection from IDiscovery
- host: host name or IP address
- port: port number (default: 27017)
- database: database name
- uri: resource URI or connection string with all parameters in it
- credential(s):
- store_key: (optional) a key to retrieve the credentials from ICredentialStore
- username: user name
- password: user password
### References ###
- *:discovery:*:*:1.0 (optional) IDiscovery services
- *:credential-store:*:*:1.0 (optional) Credential stores to resolve credentials
"""
_connection_resolver = ConnectionResolver()
_credential_resolver = CredentialResolver()
def configure(self, config):
"""
Configures component by passing configuration parameters.
:param config: configuration parameters to be set.
"""
self._connection_resolver.configure(config)
self._credential_resolver.configure(config)
def set_references(self, references):
"""
Sets references to dependent components.
:param references: references to locate the component dependencies.
"""
self._connection_resolver.set_references(references)
self._credential_resolver.set_references(references)
def validate_connection(self, correlation_id, connection):
uri = connection.get_uri()
if uri == None:
return None
host = connection.get_host()
if host == None:
return ConfigException(correlation_id, "NO_HOST",
"Connection host is not set")
port = connection.get_port()
if port == 0:
return ConfigException(correlation_id, "NO_PORT",
"Connection port is not set")
database = connection.get_as_nullable_string("database")
if database == None:
return ConfigException(correlation_id, "NO_DATABASE",
"Connection database is not set")
def validate_connections(self, correlation_id, connections):
if connections == None or len(connections) == 0:
return ConfigException(correlation_id, "NO_CONNECTION",
"Database connection is not set")
for connection in connections:
error = self.validate_connection(correlation_id, connection)
def compose_uri(self, connections, credential):
for connection in connections:
uri = connection.get_uri()
if uri != None:
return uri
hosts = ''
for connection in connections:
host = connection.get_host()
port = connection.get_port()
if len(hosts) > 0:
hosts = hosts + ','
hosts = hosts + host + (':' + str(port) if port != None else '')
database = ''
for connection in connections:
database = connection.get_as_nullable_string("database") \
if connection.get_as_nullable_string("database") != None \
else database
if len(database) > 0:
database = '/' + database
auth = ''
if credential != None:
username = credential.get_username()
if username != None:
password = credential.get_password()
if password != None:
auth = username + ':' + password + '@'
else:
auth = username + '@'
options = ConfigParams()
for connection in connections:
options = options.override(connection)
if credential != None:
options = options.override(credential)
options.remove("uri")
options.remove("host")
options.remove("port")
options.remove("database")
# options.remove("username")
# options.remove("password")
parameters = ''
keys = options.get_key_names()
for key in keys:
if len(parameters) > 0:
parameters += '&'
parameters += key
value = options.get_as_string(key)
if value != None:
parameters += '=' + value
if len(parameters) > 0:
parameters = '?' + parameters
uri = "mongodb://" + auth + hosts + database + parameters
return uri
def resolve(self, correlation_id):
"""
Resolves MongoDB connection URI from connection and credential parameters.
:param correlation_id: (optional) transaction id to trace execution through call chain.
:return: a resolved URI
"""
connections = self._connection_resolver.resolve_all(correlation_id)
credential = self._credential_resolver.lookup(correlation_id)
self.validate_connections(correlation_id, connections)
return self.compose_uri(connections, credential)
def __init__(self):
# # Create connection resolver.
self._connection_resolver: ConnectionResolver = ConnectionResolver()
# # The base credential resolver.
self._credential_resolver: CredentialResolver = CredentialResolver()
class MessageQueue(IConfigurable, IReferenceable, IMessageQueue):
"""
Abstract message queue.
Abstract message queue that is used as a basis for specific message queue implementations.
### Configuration parameters ###
- name: name of the message queue
- connection(s):
- discovery_key: key to retrieve parameters from discovery service
- protocol: connection protocol like http, https, tcp, udp
- host: host name or IP address
- port: port number
- uri: resource URI or connection string with all parameters in it
- credential(s):
- store_key: key to retrieve parameters from credential store
- username: user name
- password: user password
- access_id: application access id
- access_key: application secret key
### References ###
- *:logger:*:*:1.0 (optional) ILogger components to pass log messages
- *:counters:*:*:1.0 (optional) ICounters components to pass collected measurements
- *:discovery:*:*:1.0 (optional) IDiscovery components to discover connection(s)
- *:credential-store:*:*:1.0 (optional) ICredentialStore componetns to lookup credential(s)
"""
_name = None
_capabilities = None
_lock = None
_logger = None
_counters = None
_credential_resolver = None
_connection_resolver = None
def __init__(self, name=None):
"""
Creates a new instance of the message queue.
:param name: (optional) a queue name
"""
self._lock = threading.Lock()
self._logger = CompositeLogger()
self._counters = CompositeCounters()
self._connection_resolver = ConnectionResolver()
self._credential_resolver = CredentialResolver()
self._name = name
def configure(self, config):
"""
Configures component by passing configuration parameters.
:param config: configuration parameters to be set.
"""
self._name = NameResolver.resolve(config)
self._logger.configure(config)
self._credential_resolver.configure(config)
self._connection_resolver.configure(config)
def set_references(self, references):
"""
Sets references to dependent components.
:param references: references to locate the component dependencies.
"""
self._logger.set_references(references)
self._counters.set_references(references)
self._credential_resolver.set_references(references)
self._connection_resolver.set_references(references)
def open(self, correlation_id):
"""
Opens the component.
:param correlation_id: (optional) transaction id to trace execution through call chain.
"""
connection = self._connection_resolver.resolve(correlation_id)
credential = self._credential_resolver.lookup(correlation_id)
self._open_with_params(correlation_id, connection, credential)
def _open_with_params(self, correlation_id, connection, credential):
"""
Opens the component with given connection and credential parameters.
:param correlation_id: (optional) transaction id to trace execution through call chain.
:param connection: connection parameters
:param credential: credential parameters
"""
raise NotImplementedError('Abstract method that shall be overriden')
def get_name(self):
"""
Gets the queue name
:return: the queue name.
"""
return self._name if self._name != None else "undefined"
def get_capabilities(self):
"""
Gets the queue capabilities
:return: the queue's capabilities object.
"""
return self._capabilities
def send_as_object(self, correlation_id, message_type, message):
"""
Sends an object into the queue.
Before sending the object is converted into JSON string and wrapped in a [[MessageEnvelop]].
:param correlation_id: (optional) transaction id to trace execution through call chain.
:param message_type: a message type
:param message: an object value to be sent
"""
envelop = MessageEnvelop(correlation_id, message_type, message)
self.send(correlation_id, envelop)
def begin_listen(self, correlation_id, receiver):
"""
Listens for incoming messages without blocking the current thread.
:param correlation_id: (optional) transaction id to trace execution through call chain.
:param receiver: a receiver to receive incoming messages.
"""
# Start listening on a parallel tread
thread = threading.Thread(target=self.listen,
args=(correlation_id, receiver))
thread.daemon = True
thread.start()
def __str__(self):
"""
Gets a string representation of the object.
:return: a string representation of the object.
"""
return "[" + self.get_name() + "]"
def test_register(self):
connection_params = ConnectionParams()
connection_resolver = ConnectionResolver(RestConfig)
connection_resolver.register("correlation_id", connection_params)
config_list = connection_resolver.get_all()
assert 1 == len(config_list)
connection_params.set_discovery_key("Discovery key value")
connection_resolver.register("correlation_id", connection_params)
config_list = connection_resolver.get_all()
assert 1 == len(config_list)
references = References()
connection_resolver.set_references(references)
connection_resolver.register("correlation_id", connection_params)
config_list = connection_resolver.get_all()
assert 2 == len(config_list)
assert "http" == config_list[0]["protocol"]
assert "localhost" == config_list[0]["host"]
assert "3000" == config_list[0]["port"]
assert "Discovery key value" == config_list[1]["discovery_key"]
def test_configure(self):
connection_resolver = ConnectionResolver(RestConfig)
config_list = connection_resolver.get_all()
assert "http" == config_list[0]["protocol"]
assert "localhost" == config_list[0]["host"]
assert "3000" == config_list[0]["port"]
class MainFacadeService(FacadeService, IOpenable):
_default_config = ConfigParams.from_tuples(
'root_path', '', 'connection.protocol', 'http', 'connection.hostname',
'0.0.0.0', 'connection.port', 8080, 'credential.ssl_key_file', None,
'credential.ssl_crt_file', None, 'credential.ssl_ca_file', None,
'options.debug', True, 'options.maintenance_enabled', False,
'options.max_sockets', 50, 'options.max_req_size', '1mb')
__server = None
__service = None
__http = None
__connection_resolver = ConnectionResolver()
__credential_resolver = CredentialResolver()
__debug = True
__maintance_enabled = False
__max_sockets = 50
__max_req_size = '1mb'
def __init__(self):
super(MainFacadeService, self).__init__()
self._root_path = ''
# bottle app
self.__service = super()._partition
def is_maintance_enabled(self) -> bool:
return self.__maintance_enabled
def set_maintance_enabled(self, value: bool):
self.__maintance_enabled = True
def configure(self, config):
config = config.set_defaults(MainFacadeService._default_config)
self.__connection_resolver.configure(config)
self.__credential_resolver.configure(config)
self._root_path = config.get_as_string_with_default(
'root_path', self._root_path)
if len(self._root_path) > 0 and not (self._root_path.startswith('/')):
self._root_path = '/' + self._root_path
self.__debug = config.get_as_boolean_with_default(
'options.debug', self.__debug)
self.__maintance_enabled = config.get_as_boolean_with_default(
'options.maintenance_enabled', self.__maintance_enabled)
self.__max_sockets = config.get_as_integer_with_default(
'options.max_sockets', self.__max_sockets)
self.__max_req_size = config.get_as_string_with_default(
'options.max_req_size', self.__max_req_size)
def set_references(self, references):
super().set_references(references)
self.__connection_resolver.set_references(references)
self.__credential_resolver.set_references(references)
def is_open(self):
return self.__http is not None
def open(self, correlation_id):
if self.__http is not None:
return
connection = self._get_connetcion(correlation_id)
credential = self._get_credential(correlation_id, connection)
self.__server = self.__create_server(connection, credential)
self.__configure_service()
host = connection.get_host()
host_name = socket.gethostname()
port = connection.get_port()
self.__server.host = host
self.__server.port = port
def start_server():
try:
self.__service.run(server=self.__server, debug=self.__debug)
except Exception as ex:
self._logger.error(correlation_id, ex,
'Failed to start HTTP server at {}:{}',
host_name, port)
# Start server in thread
Thread(target=start_server, daemon=True).start()
# Time for start server
time.sleep(0.01)
self._logger.info(correlation_id, 'Started HTTP server {}:{}',
host_name, port)
def close(self, correlation_id):
try:
if self.__server is not None:
self.__server.shutdown()
self.__service.close()
self._logger.debug(correlation_id, "Closed HTTP server")
self.__server = None
self.__service = None
except Exception as ex:
self._logger.warn(correlation_id,
"Failed while closing HTTP server: " + str(ex))
def _get_connetcion(self, correlation_id):
connection = self.__connection_resolver.resolve(correlation_id)
# Check for connection
if connection is None:
raise ConfigException(correlation_id, "NO_CONNECTION",
"Connection for REST client is not defined")
else:
# Check for type
protocol = connection.get_protocol('http')
if 'http' != protocol and 'https' != protocol:
raise ConfigException(
correlation_id, "WRONG_PROTOCOL",
"Protocol is not supported by REST connection"
).with_details("protocol", protocol)
# Check for host
elif connection.get_host() is None:
raise ConfigException(
correlation_id, "NO_HOST",
"No host is configured in REST connection")
# Check for port
elif connection.get_port() == 0:
raise ConfigException(
correlation_id, "NO_PORT",
"No port is configured in REST connection")
return connection
def _get_credential(self, correlation_id, connection):
# Credentials are not required unless HTTPS is used
if connection.get_protocol('http') != 'https':
return
credential = self.__credential_resolver.lookup(correlation_id)
# Check for connection
if credential is None:
raise ConfigException(
correlation_id, "NO_CREDENTIAL",
"SSL certificates are not configured for HTTPS protocol")
else:
if credential.get_as_nullable_string('ssl_key_file') is None:
raise ConfigException(
correlation_id, "NO_SSL_KEY_FILE",
"SSL key file is not configured in credentials")
elif credential.get_as_nullable_string('ssl_crt_file') is None:
raise ConfigException(
correlation_id, "NO_SSL_CRT_FILE",
"SSL crt file is not configured in credentials")
return credential
def __create_server(self, connection, credential):
if connection.get_protocol('http') == 'https':
if connection.get_protocol('http') == 'https':
ssl_key_file = credential.get_as_nullable_string(
'ssl_key_file')
with open(ssl_key_file, 'rb') as file:
private_key = file.read()
ssl_crt_file = credential.get_as_nullable_string(
'ssl_crt_file')
with open(ssl_crt_file, 'rb') as file:
certfile = file.read()
# ca = []
#
# ssl_ca_file = credential.get_as_nullable_string('ssl_ca_file')
# if ssl_ca_file is not None:
# with open(ssl_ca_file, 'rb') as file:
# ca_text = file.read()
# while ca_text is not None and len(ca_text.strip()) > 0:
# crt_index = ca_text.rindex(b'-----BEGIN CERTIFICATE-----')
# if crt_index > -1:
# ca.append(ca_text[crt_index:])
# ca_text = ca_text[0:crt_index]
return SSLCherryPyServer(
certfile=certfile,
keyfile=private_key,
request_queue_size=self.__max_sockets,
max_request_body_size=self.__max_req_size)
return SSLCherryPyServer(request_queue_size=self.__max_sockets,
max_request_body_size=self.__max_req_size)
def __configure_service(self):
self.__service.config['catchall'] = True
self.__service.config['autojson'] = True
# Enable CORS requests
self.__service.add_hook('after_request', self.__enable_cors)
self.__service.add_hook('after_request', self.__do_maintance)
self.__service.add_hook('after_request', self.__no_cache)
def __enable_cors(self):
response.headers['Access-Control-Max-Age'] = '5'
response.headers['Access-Control-Allow-Origin'] = '*'
response.headers[
'Access-Control-Allow-Methods'] = 'PUT, GET, POST, DELETE, OPTIONS'
response.headers[
'Access-Control-Allow-Headers'] = 'Authorization, Origin, Accept, Content-Type, X-Requested-With'
def __do_maintance(self):
"""
:return: maintenance error code
"""
# Make this more sophisticated
if self.__maintance_enabled:
response.headers['Retry-After'] = 3600
response.status = 503
def __no_cache(self):
"""
Prevents IE from caching REST requests
"""
response.headers[
'Cache-Control'] = 'no-cache, no-store, must-revalidate'
response.headers['Pragma'] = 'no-cache'
response.headers['Expires'] = 0