def __init__(self, name=None):
"""
Creates a new instance of the message queue.
:param name: (optional) a queue name
"""
self._lock = threading.Lock()
self._logger = CompositeLogger()
self._counters = CompositeCounters()
self._connection_resolver = ConnectionResolver()
self._credential_resolver = CredentialResolver()
self._name = name
def __init__(self):
super().__init__()
self.__connection_resolver: ConnectionResolver = ConnectionResolver()
self.__credential_resolver: CredentialResolver = CredentialResolver()
self.__lock: str = IdGenerator.next_long()
self.__timeout: int = 30000
self.__retries: int = 3
self.__client: redis.Redis = None
def __init__(self):
"""
Creates a new instance of this cache
"""
self.__connection_resolver: ConnectionResolver = ConnectionResolver()
self.__credential_resolver: CredentialResolver = CredentialResolver()
self.__timeout: int = 30000
self.__retries: int = 3
self.__client: redis.Redis = None
def __init__(self):
super(CompositeConnectionResolver, self).__init__()
# The connection options
self._options: ConfigParams = None
# The connections resolver.
self._connection_resolver = ConnectionResolver()
# The credentials resolver.
self._credential_resolver = CredentialResolver()
# The cluster support (multiple connections)
self._cluster_supported = True
# The default protocol
self._default_protocol: str = None
# The default port number
self._default_port: int = 0
# The list of supported protocols
self._supported_protocols: List[str] = []
def __init__(self,
name: str = None,
capabilities: MessagingCapabilities = None):
"""
Creates a new instance of the message queue.
:param name: (optional) a queue name
:param capabilities: (optional) a capabilities of this message queue
"""
self._lock: threading.Lock = threading.Lock()
self._event = threading.Event()
self._capabilities: MessagingCapabilities = None
self._logger: CompositeLogger = CompositeLogger()
self._counters: CompositeCounters = CompositeCounters()
self._connection_resolver: ConnectionResolver = ConnectionResolver()
self._credential_resolver: CredentialResolver = CredentialResolver()
self._name: str = name
self._capabilities = capabilities or \
MessagingCapabilities(False, False, False, False, False, False, False, False, False)
def __init__(self):
# The connections resolver.
self._connection_resolver: ConnectionResolver = ConnectionResolver()
# The credentials resolver.
self._credential_resolver: CredentialResolver = CredentialResolver()
def __init__(self):
self._connection_resolver: ConnectionResolver = ConnectionResolver()
self._credential_resolver: CredentialResolver = CredentialResolver()
class MongoDbConnectionResolver(IReferenceable, IConfigurable):
"""
MongoDbConnectionResolver implementation.
Helper class that resolves MongoDB connection
and credential parameters, validates them and generates a connection URI.
It is able to process multiple connections to MongoDB cluster nodes.
### Configuration parameters ###
- connection(s):
- discovery_key: (optional) a key to retrieve the connection from IDiscovery
- host: host name or IP address
- port: port number (default: 27017)
- database: database name
- uri: resource URI or connection string with all parameters in it
- credential(s):
- store_key: (optional) a key to retrieve the credentials from ICredentialStore
- username: user name
- password: user password
### References ###
- *:discovery:*:*:1.0 (optional) IDiscovery services
- *:credential-store:*:*:1.0 (optional) Credential stores to resolve credentials
"""
_connection_resolver = ConnectionResolver()
_credential_resolver = CredentialResolver()
def configure(self, config):
"""
Configures component by passing configuration parameters.
:param config: configuration parameters to be set.
"""
self._connection_resolver.configure(config)
self._credential_resolver.configure(config)
def set_references(self, references):
"""
Sets references to dependent components.
:param references: references to locate the component dependencies.
"""
self._connection_resolver.set_references(references)
self._credential_resolver.set_references(references)
def validate_connection(self, correlation_id, connection):
uri = connection.get_uri()
if uri == None:
return None
host = connection.get_host()
if host == None:
return ConfigException(correlation_id, "NO_HOST",
"Connection host is not set")
port = connection.get_port()
if port == 0:
return ConfigException(correlation_id, "NO_PORT",
"Connection port is not set")
database = connection.get_as_nullable_string("database")
if database == None:
return ConfigException(correlation_id, "NO_DATABASE",
"Connection database is not set")
def validate_connections(self, correlation_id, connections):
if connections == None or len(connections) == 0:
return ConfigException(correlation_id, "NO_CONNECTION",
"Database connection is not set")
for connection in connections:
error = self.validate_connection(correlation_id, connection)
def compose_uri(self, connections, credential):
for connection in connections:
uri = connection.get_uri()
if uri != None:
return uri
hosts = ''
for connection in connections:
host = connection.get_host()
port = connection.get_port()
if len(hosts) > 0:
hosts = hosts + ','
hosts = hosts + host + (':' + str(port) if port != None else '')
database = ''
for connection in connections:
database = connection.get_as_nullable_string("database") \
if connection.get_as_nullable_string("database") != None \
else database
if len(database) > 0:
database = '/' + database
auth = ''
if credential != None:
username = credential.get_username()
if username != None:
password = credential.get_password()
if password != None:
auth = username + ':' + password + '@'
else:
auth = username + '@'
options = ConfigParams()
for connection in connections:
options = options.override(connection)
if credential != None:
options = options.override(credential)
options.remove("uri")
options.remove("host")
options.remove("port")
options.remove("database")
# options.remove("username")
# options.remove("password")
parameters = ''
keys = options.get_key_names()
for key in keys:
if len(parameters) > 0:
parameters += '&'
parameters += key
value = options.get_as_string(key)
if value != None:
parameters += '=' + value
if len(parameters) > 0:
parameters = '?' + parameters
uri = "mongodb://" + auth + hosts + database + parameters
return uri
def resolve(self, correlation_id):
"""
Resolves MongoDB connection URI from connection and credential parameters.
:param correlation_id: (optional) transaction id to trace execution through call chain.
:return: a resolved URI
"""
connections = self._connection_resolver.resolve_all(correlation_id)
credential = self._credential_resolver.lookup(correlation_id)
self.validate_connections(correlation_id, connections)
return self.compose_uri(connections, credential)
class MessageQueue(IConfigurable, IReferenceable, IMessageQueue):
"""
Abstract message queue.
Abstract message queue that is used as a basis for specific message queue implementations.
### Configuration parameters ###
- name: name of the message queue
- connection(s):
- discovery_key: key to retrieve parameters from discovery service
- protocol: connection protocol like http, https, tcp, udp
- host: host name or IP address
- port: port number
- uri: resource URI or connection string with all parameters in it
- credential(s):
- store_key: key to retrieve parameters from credential store
- username: user name
- password: user password
- access_id: application access id
- access_key: application secret key
### References ###
- *:logger:*:*:1.0 (optional) ILogger components to pass log messages
- *:counters:*:*:1.0 (optional) ICounters components to pass collected measurements
- *:discovery:*:*:1.0 (optional) IDiscovery components to discover connection(s)
- *:credential-store:*:*:1.0 (optional) ICredentialStore componetns to lookup credential(s)
"""
_name = None
_capabilities = None
_lock = None
_logger = None
_counters = None
_credential_resolver = None
_connection_resolver = None
def __init__(self, name=None):
"""
Creates a new instance of the message queue.
:param name: (optional) a queue name
"""
self._lock = threading.Lock()
self._logger = CompositeLogger()
self._counters = CompositeCounters()
self._connection_resolver = ConnectionResolver()
self._credential_resolver = CredentialResolver()
self._name = name
def configure(self, config):
"""
Configures component by passing configuration parameters.
:param config: configuration parameters to be set.
"""
self._name = NameResolver.resolve(config)
self._logger.configure(config)
self._credential_resolver.configure(config)
self._connection_resolver.configure(config)
def set_references(self, references):
"""
Sets references to dependent components.
:param references: references to locate the component dependencies.
"""
self._logger.set_references(references)
self._counters.set_references(references)
self._credential_resolver.set_references(references)
self._connection_resolver.set_references(references)
def open(self, correlation_id):
"""
Opens the component.
:param correlation_id: (optional) transaction id to trace execution through call chain.
"""
connection = self._connection_resolver.resolve(correlation_id)
credential = self._credential_resolver.lookup(correlation_id)
self._open_with_params(correlation_id, connection, credential)
def _open_with_params(self, correlation_id, connection, credential):
"""
Opens the component with given connection and credential parameters.
:param correlation_id: (optional) transaction id to trace execution through call chain.
:param connection: connection parameters
:param credential: credential parameters
"""
raise NotImplementedError('Abstract method that shall be overriden')
def get_name(self):
"""
Gets the queue name
:return: the queue name.
"""
return self._name if self._name != None else "undefined"
def get_capabilities(self):
"""
Gets the queue capabilities
:return: the queue's capabilities object.
"""
return self._capabilities
def send_as_object(self, correlation_id, message_type, message):
"""
Sends an object into the queue.
Before sending the object is converted into JSON string and wrapped in a [[MessageEnvelop]].
:param correlation_id: (optional) transaction id to trace execution through call chain.
:param message_type: a message type
:param message: an object value to be sent
"""
envelop = MessageEnvelop(correlation_id, message_type, message)
self.send(correlation_id, envelop)
def begin_listen(self, correlation_id, receiver):
"""
Listens for incoming messages without blocking the current thread.
:param correlation_id: (optional) transaction id to trace execution through call chain.
:param receiver: a receiver to receive incoming messages.
"""
# Start listening on a parallel tread
thread = threading.Thread(target=self.listen,
args=(correlation_id, receiver))
thread.daemon = True
thread.start()
def __str__(self):
"""
Gets a string representation of the object.
:return: a string representation of the object.
"""
return "[" + self.get_name() + "]"
class CompositeConnectionResolver(IReferenceable, IConfigurable):
"""
Helper class that resolves connection and credential parameters,
validates them and generates connection options.
### Configuration parameters ###
- connection(s):
- discovery_key: (optional) a key to retrieve the connection from :class:`IDiscovery <pip_services3_components.connect.IDiscovery.IDiscovery>`
- protocol: communication protocol
- host: host name or IP address
- port: port number
- uri: resource URI or connection string with all parameters in it
- credential(s):
- store_key: (optional) a key to retrieve the credentials from :class:`ICredentialStore <pip_services3_components.auth.ICredentialStore.ICredentialStore>`
- username: user name
- password: user password
### References ###
- `\*:discovery:\*:\*:1.0` (optional) :class:`IDiscovery <pip_services3_components.connect.IDiscovery.IDiscovery>` services to resolve connections
- `\*:credential-store:\*:\*:1.0` (optional) Credential stores to resolve credentials
"""
def __init__(self):
super(CompositeConnectionResolver, self).__init__()
# The connection options
self._options: ConfigParams = None
# The connections resolver.
self._connection_resolver = ConnectionResolver()
# The credentials resolver.
self._credential_resolver = CredentialResolver()
# The cluster support (multiple connections)
self._cluster_supported = True
# The default protocol
self._default_protocol: str = None
# The default port number
self._default_port: int = 0
# The list of supported protocols
self._supported_protocols: List[str] = []
def configure(self, config: ConfigParams):
"""
Configures component by passing configuration parameters.
:param config: configuration parameters to be set.
"""
self._connection_resolver.configure(config)
self._credential_resolver.configure(config)
self._options = config.get_section('options')
def set_references(self, references: IReferences):
"""
Sets references to dependent components.
:param references: references to locate the component dependencies.
"""
self._connection_resolver.set_references(references)
self._credential_resolver.set_references(references)
def resolve(self, correlation_id: Optional[str]) -> ConfigParams:
"""
Resolves connection options from connection and credential parameters.
:param correlation_id: (optional) transaction id to trace execution through call chain.
:return: resolved options or error
"""
connections = self._connection_resolver.resolve_all(correlation_id)
if len(connections) > 0 and not self._cluster_supported:
raise ConfigException(
correlation_id, "MULTIPLE_CONNECTIONS_NOT_SUPPORTED",
"Multiple (cluster) connections are not supported")
# Validate connections
for connection in connections:
self._validate_connection(correlation_id, connection)
credential = self._credential_resolver.lookup(correlation_id)
# Validate credential
self._validate_credential(correlation_id, credential)
options = self._compose_options(connections, credential, self._options)
return options
def compose(self, correlation_id: Optional[str],
connections: List[ConnectionParams],
credential: CredentialParams,
parameters: ConfigParams) -> ConfigParams:
"""
Composes Composite connection options from connection and credential parameters.
:param correlation_id: (optional) transaction id to trace execution through call chain.
:param connections: connection parameters
:param credential: credential parameters
:param parameters: optional parameters
:return: resolved options or error.
"""
# Validate connection parameters
for connection in connections:
self._validate_connection(correlation_id, connection)
# Validate credential parameters
self._validate_credential(correlation_id, credential)
options = self._compose_options(connections, credential, parameters)
return options
def _validate_connection(self, correlation_id: Optional[str],
connection: ConnectionParams):
"""
Validates connection parameters.
This method can be overriden in child classes.
:param correlation_id: (optional) transaction id to trace execution through call chain.
:param connection: connection parameters to be validated
:return: error or `None` if validation was successful
"""
if connection is None:
raise ConfigException(
correlation_id, "NO_CONNECTION",
"Connection parameters are not set is not set")
# URI usually contains all information
uri = connection.get_uri()
if uri is not None: return None
protocol = connection.get_protocol_with_default(self._default_protocol)
if protocol is None:
raise ConfigException(correlation_id, "NO_PROTOCOL",
"Connection protocol is not set")
if self._supported_protocols is not None and protocol in self._supported_protocols:
raise ConfigException(
correlation_id, "UNSUPPORTED_PROTOCOL",
"The protocol " + protocol + " is not supported")
host = connection.get_host()
if host is None:
raise ConfigException(correlation_id, "NO_HOST",
"Connection host is not set")
port = connection.get_port_with_default(self._default_port)
if port == 0:
raise ConfigException(correlation_id, "NO_PORT",
"Connection port is not set")
return None
def _validate_credential(self, correlation_id: Optional[str],
credential: CredentialParams):
"""
Validates credential parameters.
This method can be overriden in child classes.
:param correlation_id: (optional) transaction id to trace execution through call chain.
:param credential: credential parameters to be validated
:return: error or `None` if validation was successful
"""
return None
def _compose_options(self, connections: List[ConnectionParams],
credential: CredentialParams,
parameters: ConfigParams) -> ConfigParams:
# Connection options
options = ConfigParams()
# Merge connection parameters
for connection in connections:
options = self._merge_connection(options, connection)
# Merge credential parameters
options = self._merge_credential(options, credential)
# Merge optional parameters
options = self._merge_optional(options, parameters)
# Perform final processing
options = self._finalize_options(options)
return options
def _merge_connection(self, options: ConfigParams,
connection: ConnectionParams) -> ConfigParams:
"""
Merges connection options with connection parameters
This method can be overriden in child classes.
:param options: connection options
:param connection: connection parameters to be merged
:return: merged connection options.
"""
merged_options = options.set_defaults(connection)
return merged_options
def _merge_credential(self, options: ConfigParams,
credential: CredentialParams) -> ConfigParams:
"""
Merges connection options with credential parameters
This method can be overriden in child classes.
:param options: connection options
:param credential: credential parameters to be merged
:return: merged connection options.
"""
merged_options = options.override(credential)
return merged_options
def _merge_optional(self, options: ConfigParams,
parameters: ConfigParams) -> ConfigParams:
"""
Merges connection options with optional parameters
This method can be overriden in child classes.
:param options: connection options
:param parameters: optional parameters to be merged
:return: merged connection options.
"""
merged_options = options.override(parameters)
return merged_options
def _finalize_options(self, options: ConfigParams) -> ConfigParams:
"""
Finalize merged options
This method can be overriden in child classes.
:param options: connection options
:return: finalized connection options
"""
return options
class MainFacadeService(FacadeService, IOpenable):
_default_config = ConfigParams.from_tuples(
'root_path', '', 'connection.protocol', 'http', 'connection.hostname',
'0.0.0.0', 'connection.port', 8080, 'credential.ssl_key_file', None,
'credential.ssl_crt_file', None, 'credential.ssl_ca_file', None,
'options.debug', True, 'options.maintenance_enabled', False,
'options.max_sockets', 50, 'options.max_req_size', '1mb')
__server = None
__service = None
__http = None
__connection_resolver = ConnectionResolver()
__credential_resolver = CredentialResolver()
__debug = True
__maintance_enabled = False
__max_sockets = 50
__max_req_size = '1mb'
def __init__(self):
super(MainFacadeService, self).__init__()
self._root_path = ''
# bottle app
self.__service = super()._partition
def is_maintance_enabled(self) -> bool:
return self.__maintance_enabled
def set_maintance_enabled(self, value: bool):
self.__maintance_enabled = True
def configure(self, config):
config = config.set_defaults(MainFacadeService._default_config)
self.__connection_resolver.configure(config)
self.__credential_resolver.configure(config)
self._root_path = config.get_as_string_with_default(
'root_path', self._root_path)
if len(self._root_path) > 0 and not (self._root_path.startswith('/')):
self._root_path = '/' + self._root_path
self.__debug = config.get_as_boolean_with_default(
'options.debug', self.__debug)
self.__maintance_enabled = config.get_as_boolean_with_default(
'options.maintenance_enabled', self.__maintance_enabled)
self.__max_sockets = config.get_as_integer_with_default(
'options.max_sockets', self.__max_sockets)
self.__max_req_size = config.get_as_string_with_default(
'options.max_req_size', self.__max_req_size)
def set_references(self, references):
super().set_references(references)
self.__connection_resolver.set_references(references)
self.__credential_resolver.set_references(references)
def is_open(self):
return self.__http is not None
def open(self, correlation_id):
if self.__http is not None:
return
connection = self._get_connetcion(correlation_id)
credential = self._get_credential(correlation_id, connection)
self.__server = self.__create_server(connection, credential)
self.__configure_service()
host = connection.get_host()
host_name = socket.gethostname()
port = connection.get_port()
self.__server.host = host
self.__server.port = port
def start_server():
try:
self.__service.run(server=self.__server, debug=self.__debug)
except Exception as ex:
self._logger.error(correlation_id, ex,
'Failed to start HTTP server at {}:{}',
host_name, port)
# Start server in thread
Thread(target=start_server, daemon=True).start()
# Time for start server
time.sleep(0.01)
self._logger.info(correlation_id, 'Started HTTP server {}:{}',
host_name, port)
def close(self, correlation_id):
try:
if self.__server is not None:
self.__server.shutdown()
self.__service.close()
self._logger.debug(correlation_id, "Closed HTTP server")
self.__server = None
self.__service = None
except Exception as ex:
self._logger.warn(correlation_id,
"Failed while closing HTTP server: " + str(ex))
def _get_connetcion(self, correlation_id):
connection = self.__connection_resolver.resolve(correlation_id)
# Check for connection
if connection is None:
raise ConfigException(correlation_id, "NO_CONNECTION",
"Connection for REST client is not defined")
else:
# Check for type
protocol = connection.get_protocol('http')
if 'http' != protocol and 'https' != protocol:
raise ConfigException(
correlation_id, "WRONG_PROTOCOL",
"Protocol is not supported by REST connection"
).with_details("protocol", protocol)
# Check for host
elif connection.get_host() is None:
raise ConfigException(
correlation_id, "NO_HOST",
"No host is configured in REST connection")
# Check for port
elif connection.get_port() == 0:
raise ConfigException(
correlation_id, "NO_PORT",
"No port is configured in REST connection")
return connection
def _get_credential(self, correlation_id, connection):
# Credentials are not required unless HTTPS is used
if connection.get_protocol('http') != 'https':
return
credential = self.__credential_resolver.lookup(correlation_id)
# Check for connection
if credential is None:
raise ConfigException(
correlation_id, "NO_CREDENTIAL",
"SSL certificates are not configured for HTTPS protocol")
else:
if credential.get_as_nullable_string('ssl_key_file') is None:
raise ConfigException(
correlation_id, "NO_SSL_KEY_FILE",
"SSL key file is not configured in credentials")
elif credential.get_as_nullable_string('ssl_crt_file') is None:
raise ConfigException(
correlation_id, "NO_SSL_CRT_FILE",
"SSL crt file is not configured in credentials")
return credential
def __create_server(self, connection, credential):
if connection.get_protocol('http') == 'https':
if connection.get_protocol('http') == 'https':
ssl_key_file = credential.get_as_nullable_string(
'ssl_key_file')
with open(ssl_key_file, 'rb') as file:
private_key = file.read()
ssl_crt_file = credential.get_as_nullable_string(
'ssl_crt_file')
with open(ssl_crt_file, 'rb') as file:
certfile = file.read()
# ca = []
#
# ssl_ca_file = credential.get_as_nullable_string('ssl_ca_file')
# if ssl_ca_file is not None:
# with open(ssl_ca_file, 'rb') as file:
# ca_text = file.read()
# while ca_text is not None and len(ca_text.strip()) > 0:
# crt_index = ca_text.rindex(b'-----BEGIN CERTIFICATE-----')
# if crt_index > -1:
# ca.append(ca_text[crt_index:])
# ca_text = ca_text[0:crt_index]
return SSLCherryPyServer(
certfile=certfile,
keyfile=private_key,
request_queue_size=self.__max_sockets,
max_request_body_size=self.__max_req_size)
return SSLCherryPyServer(request_queue_size=self.__max_sockets,
max_request_body_size=self.__max_req_size)
def __configure_service(self):
self.__service.config['catchall'] = True
self.__service.config['autojson'] = True
# Enable CORS requests
self.__service.add_hook('after_request', self.__enable_cors)
self.__service.add_hook('after_request', self.__do_maintance)
self.__service.add_hook('after_request', self.__no_cache)
def __enable_cors(self):
response.headers['Access-Control-Max-Age'] = '5'
response.headers['Access-Control-Allow-Origin'] = '*'
response.headers[
'Access-Control-Allow-Methods'] = 'PUT, GET, POST, DELETE, OPTIONS'
response.headers[
'Access-Control-Allow-Headers'] = 'Authorization, Origin, Accept, Content-Type, X-Requested-With'
def __do_maintance(self):
"""
:return: maintenance error code
"""
# Make this more sophisticated
if self.__maintance_enabled:
response.headers['Retry-After'] = 3600
response.status = 503
def __no_cache(self):
"""
Prevents IE from caching REST requests
"""
response.headers[
'Cache-Control'] = 'no-cache, no-store, must-revalidate'
response.headers['Pragma'] = 'no-cache'
response.headers['Expires'] = 0