class GrpcEndpoint(IOpenable, IConfigurable, IReferenceable):
"""
Used for creating GRPC endpoints. An endpoint is a URL, at which a given service can be accessed by a client.
### Configuration parameters ###
Parameters to pass to the :func:`configure` method for component configuration:
- connection(s) - the connection resolver's connections:
- "connection.discovery_key" - the key to use for connection resolving in a discovery service;
- "connection.protocol" - the connection's protocol;
- "connection.host" - the target host;
- "connection.port" - the target port;
- "connection.uri" - the target URI.
- credential - the HTTPS credentials:
- "credential.ssl_key_file" - the SSL private key in PEM
- "credential.ssl_crt_file" - the SSL certificate in PEM
- "credential.ssl_ca_file" - the certificate authorities (root cerfiticates) in PEM
### References ###
A logger, counters, and a connection resolver can be referenced by passing the
following references to the object's :func:`set_references` method:
- logger: **\*:logger:\*:\*:1.0"**;
- counters: **"\*:counters:\*:\*:1.0"**;
- discovery: **"\*:discovery:\*:\*:1.0"** (for the connection resolver).
.. code-block:: python
def my_method(self, _config, _references):
endpoint = GrpcEndpoint()
if self._config:
endpoint.configure(self._config)
if self._references:
endpoint.set_references(self._references)
...
self._endpoint.open(correlation_id)
...
"""
__defaultConfig = ConfigParams.from_tuples(
"connection.protocol", "http",
"connection.host", "0.0.0.0",
"connection.port", 3000,
"credential.ssl_key_file", None,
"credential.ssl_crt_file", None,
"credential.ssl_ca_file", None,
"options.maintenance_enabled", None,
"options.request_max_size", 1024 * 1024,
"options.file_max_size", 200 * 1024 * 1024,
"options.connect_timeout", 60000,
"options.debug", None
)
def __init__(self):
self.__server: Any = None
self.__connection_resolver = HttpConnectionResolver()
self.__logger = CompositeLogger()
self.__counters = CompositeCounters()
self.__maintenance_enabled = False
self.__file_max_size = 200 * 1024 * 1024
self.__uri: str = None
self.__registrations: List[IRegisterable] = []
self.__commandable_methods: Any = None
self.__commandable_schemas: Any = None
self.__commandable_service: Any = None
self.__interceptors = []
def configure(self, config: ConfigParams):
"""
Configures this HttpEndpoint using the given configuration parameters.
### Configuration parameters ###
- connection(s) - the connection resolver's connections;
- "connection.discovery_key" - the key to use for connection resolving in a dis
- "connection.protocol" - the connection's protocol;
- "connection.host" - the target host;
- "connection.port" - the target port;
- "connection.uri" - the target URI.
- "credential.ssl_key_file" - SSL private key in PEM
- "credential.ssl_crt_file" - SSL certificate in PEM
- "credential.ssl_ca_file" - Certificate authority (root certificate) in PEM
:param config: configuration parameters, containing a "connection(s)" section.
"""
config = config.set_defaults(GrpcEndpoint.__defaultConfig)
self.__connection_resolver.configure(config)
self.__maintenance_enabled = config.get_as_boolean_with_default('options.maintenance_enabled',
self.__maintenance_enabled)
self.__file_max_size = ConfigParams().get_as_long_with_default(key='options.file_max_size',
default_value=self.__file_max_size)
def set_references(self, references: IReferences):
"""
Sets references to this endpoint's logger, counters, and connection resolver.
__References:__
- logger: **"\*:logger:\*:\*:1.0"**
- counters: **"\*:counters:\*:\*:1.0"**
- discovery: **"\*:discovery:\*:\*:1.0"** (for the connection resolver)
:param references: an IReferences object, containing references to a logger, counters, and a connection resolver.
"""
self.__logger.set_references(references)
self.__counters.set_references(references)
self.__connection_resolver.set_references(references)
def is_open(self) -> bool:
"""
:return: whether or not this endpoint is open with an actively listening GRPC server.
"""
return self.__server is not None
def open(self, correlation_id: Optional[str], max_workers: int = 15):
"""
Opens a connection using the parameters resolved by the referenced connection
resolver and creates a GRPC server (service) using the set options and parameters.
:param correlation_id: (optional) transaction id to trace execution through call chain.
:param max_workers: max count of workers in thread pool.
"""
if self.is_open():
return
connection = self.__connection_resolver.resolve(correlation_id)
self.__uri = connection.get_as_string('uri')
try:
self.__connection_resolver.register(correlation_id)
credentials = None
if connection.get_as_string_with_default('protocol', 'http') == 'https':
ssl_key_file = connection.get_as_nullable_string('ssl_key_file')
ssl_crt_file = connection.get_as_nullable_string('ssl_crt_file')
with open(ssl_key_file, 'rb') as file:
private_key = file.read()
with open(ssl_crt_file, 'rb') as file:
certificate = file.read()
credentials = grpc.ssl_server_credentials(((private_key, certificate),))
# Create instance of express application
if len(self.__interceptors) > 0:
self.__server = grpc.server(futures.ThreadPoolExecutor(max_workers=max_workers),
interceptors=self.__interceptors)
else:
self.__server = grpc.server(futures.ThreadPoolExecutor(max_workers=max_workers))
if credentials:
self.__server.add_secure_port(str(connection.get_as_string('host')) + ':' +
str(connection.get_as_string('port')), credentials)
else:
self.__server.add_insecure_port(
str(connection.get_as_string('host')) + ':' + str(connection.get_as_string('port')))
# Start operations
self.__server.start()
self.__connection_resolver.register(correlation_id)
self.__perform_registrations()
self.__logger.debug(correlation_id, 'Opened GRPC service at {}'.format(self.__uri))
except Exception as ex:
self.__server = None
err = ConnectionException(correlation_id, 'CANNOT_CONNECT', 'Opening GRPC service failed').wrap(
ex).with_details('url', self.__uri)
raise err
def close(self, correlation_id: Optional[str]):
"""
Closes this endpoint and the GRPC server (service) that was opened earlier.
:param correlation_id: (optional) transaction id to trace execution through call chain.
"""
if self.__server is not None:
self.__uri = None
self.__commandable_methods = None
self.__commandable_schemas = None
self.__commandable_service = None
# Eat exceptions
try:
self.__server.stop(None)
self.__logger.debug(correlation_id, 'Closed GRPC service at {}'.format(self.__uri))
self.__server = None
GrpcEndpoint.__interceptors = []
GrpcEndpoint.__registrations = []
GrpcEndpoint.__connection_resolver = HttpConnectionResolver()
except Exception as ex:
self.__logger.warn(correlation_id, 'Failed while closing GRPC service: '.format(ex))
def register(self, registration: IRegisterable):
"""
Registers a registerable object for dynamic endpoint discovery.
:param registration: the registration to add.
"""
if registration is not None:
self.__registrations.append(registration)
def unregister(self, registration: IRegisterable):
"""
Unregisters a registerable object, so that it is no longer used in dynamic
endpoint discovery.
:param registration: the registration to remove.
"""
self.__registrations = list(filter(lambda r: r == registration, self.__registrations))
def __perform_registrations(self):
for registration in self.__registrations:
registration.register()
self.__register_commandable_service()
def __register_commandable_service(self):
if self.__commandable_methods is None:
return
self.__commandable_service = _CommandableMediator()
self.__commandable_service.invoke_func(self.__invoke_commandable_method)
self.register_service(self.__commandable_service)
def __invoke_commandable_method(self, request: Any, context: Any):
method = request.method
action = self.__commandable_methods[method] if self.__commandable_methods else None
correlation_id = request.correlation_id
response = commandable_pb2.InvokeReply()
# Handle method not found
if action is None:
err = InvocationException(correlation_id, 'METHOD_NOT_FOUND',
'Method ' + method + ' was not found').with_details('method', method)
resp_err = ErrorDescriptionFactory.create(err)
response.error = commandable_pb2.ErrorDescription()
response.error.category = resp_err.category
response.error.code = resp_err.code
response.error.correlation_id = resp_err.correlation_id
response.error.status = resp_err.status
response.error.message = resp_err.message
response.error.cause = resp_err.cause
response.error.stack_trace = resp_err.stack_trace
response.error.details.addAll(resp_err.details)
response.resultEmpty = True
response.resultJson = ''
return response
try:
# Convert arguments
args_empty = request.args_empty
args_json = request.args_json
args = Parameters.from_json(args_json) if not args_empty and args_json else Parameters()
try:
result = action(correlation_id, args, method)
response.result_empty = result is None
if type(result) is DataPage:
response.result_json = json.dumps(result.to_json())
else:
response.result_json = json.dumps(result) if result is not None else ''
# TODO: Validate schema
schema = self.__commandable_schemas[method]
if schema:
pass
except Exception as ex:
# Process result and generate response
resp_err = ErrorDescriptionFactory.create(ex)
response.error.category = resp_err.category
response.error.code = resp_err.code
response.error.correlation_id = resp_err.correlation_id
response.error.status = resp_err.status
response.error.message = resp_err.message
response.error.cause = resp_err.cause
response.error.stack_trace = resp_err.stack_trace
response.error.details.update(resp_err.details)
response.result_empty = True
response.result_json = ''
except Exception as ex:
# Handle unexpected exception
err = InvocationException(correlation_id, 'METHOD_FAILED', 'Method ' + method + ' failed').wrap(
ex).with_details('method', method)
resp_err = ErrorDescriptionFactory.create(err)
response.error.category = resp_err.category
response.error.code = resp_err.code
response.error.correlation_id = resp_err.correlation_id
response.error.status = resp_err.status
response.error.message = resp_err.message
response.error.cause = resp_err.cause
response.error.stack_trace = resp_err.stack_trace
response.error.details.update(resp_err.details)
response.result_empty = True
response.result_json = ''
return response
def _register_method(self, name: str, schema: Schema, action: Callable[[Optional[str], Optional[str], Parameters], None]):
"""
Registers a method in GRPC service.
:param name: a method name
:param schema: a validation schema to validate received parameters.
:param action: an action function that is called when operation is invoked.
"""
# TODO
pass
def register_service(self, service):
"""
Registers a service with related implementation
:param service: a GRPC service object.
"""
service.add_servicer_to_server(self.__server)
def _register_commandable_method(self, method: str, schema: Schema,
action: Callable[[Optional[str], Optional[str], Parameters], None]):
"""
Registers a commandable method in this objects GRPC server (service) by the given name.
:param method: the GRPC method name.
:param schema: the schema to use for parameter validation.
:param action: the action to perform at the given route.
"""
self.__commandable_methods = self.__commandable_methods or {}
self.__commandable_methods[method] = action
self.__commandable_schemas = self.__commandable_schemas or {}
self.__commandable_schemas[method] = schema
def _register_interceptor(self, interceptor: Callable):
"""
Registers a middleware for methods in GRPC endpoint.
:param interceptor: the middleware action to perform at the given route.
"""
self.__interceptors.append(interceptor)
class MongoDbConnection(IReferenceable, IReferences, IOpenable):
"""
MongoDB connection using plain driver.
By defining a connection and sharing it through multiple persistence components
you can reduce number of used database connections.
### Configuration parameters ###
- connection(s):
- discovery_key: (optional) a key to retrieve the connection from [[https://rawgit.com/pip-services-python/pip-services3-components-python/master/doc/api/interfaces/connect.idiscovery.html IDiscovery]]
- host: host name or IP address
- port: port number (default: 27017)
- uri: resource URI or connection string with all parameters in it
- credential(s):
- store_key: (optional) a key to retrieve the credentials from [[https://rawgit.com/pip-services-python/pip-services3-components-python/master/doc/api/interfaces/auth.icredentialstore.html ICredentialStore]]
- username: (optional) user name
- password: (optional) user password
- options:
- max_pool_size: (optional) maximum connection pool size (default: 2)
- keep_alive: (optional) enable connection keep alive (default: true)
- connect_timeout: (optional) connection timeout in milliseconds (default: 5000)
- socket_timeout: (optional) socket timeout in milliseconds (default: 360000)
- auto_reconnect: (optional) enable auto reconnection (default: true)
- reconnect_interval: (optional) reconnection interval in milliseconds (default: 1000)
- max_page_size: (optional) maximum page size (default: 100)
- replica_set: (optional) name of replica set
- ssl: (optional) enable SSL connection (default: false)
- auth_source: (optional) authentication source
- debug: (optional) enable debug output (default: false).
### References ###
- <code>\*:logger:\*:\*:1.0</code> (optional) [[https://rawgit.com/pip-services-python/pip-services3-components-python/master/doc/api/interfaces/log.ilogger.html ILogger]] components to pass log messages
- <code>\*:discovery:\*:\*:1.0</code> (optional) [[https://rawgit.com/pip-services-python/pip-services3-components-python/master/doc/api/interfaces/connect.idiscovery.html IDiscovery]] services
- <code>\*:credential-store:\*:\*:1.0</code> (optional) Credential stores to resolve credentials
"""
# The logger
_logger = None
# The connection resolver
_connection_resolver = None
# The configuration options.
_options = None
# The MongoDB connection object.
_connection = None
# The MongoDB database name.
_database_name = None
# The MongoDb database object.
_db = None
def __init__(self):
"""
Creates a new instance of the connection component.
"""
self.__default_config = ConfigParams.from_tuples(
'options.max_pool_size', 2, 'options.keep_alive', 1,
'options.connect_timeout', 5000, 'options.auto_reconnect', True,
'options.max_page_size', 100, 'options.debug', True)
self._logger = CompositeLogger()
self._connection_resolver = MongoDbConnectionResolver()
self._options = ConfigParams()
def configure(self, config):
"""
Configures component by passing configuration parameters.
:param config: configuration parameters to be set.
"""
config = config.set_defaults(self.__default_config)
self._connection_resolver.configure(config)
self._options = self._options.override(config.get_section('options'))
def set_references(self, references):
"""
Sets references to dependent components.
:param references: references to locate the component dependencies.
"""
self._logger.set_references(references)
self._connection_resolver.set_references(references)
def is_opened(self):
"""
Checks if the component is opened.
:return: true if the component has been opened and false otherwise.
"""
return self._connection is not None
def __compose_settings(self):
max_pool_size = self._options.get_as_nullable_string('max_pool_size')
keep_alive = self._options.get_as_boolean('keep_alive')
connection_timeout_ms = self._options.get_as_nullable_integer(
'connect_timeout')
socket_timeout_ms = self._options.get_as_nullable_integer(
'socket_timeout')
auto_reconnect = self._options.get_as_nullable_boolean(
'auto_reconnect')
reconnect_interval = self._options.get_as_nullable_integer(
'reconnect_interval')
debug = self._options.get_as_nullable_boolean('debug')
ssl = self._options.get_as_nullable_boolean('ssl')
replica_set = self._options.get_as_nullable_string('replica_set')
auth_source = self._options.get_as_nullable_string('auth_source')
auth_user = self._options.get_as_nullable_string('auth_user')
auth_password = self._options.get_as_nullable_string('auth_password')
settings = {
'maxPoolSize': max_pool_size,
'socketKeepAlive': keep_alive,
'connectTimeoutMS': connection_timeout_ms,
'socketTimeoutMS': socket_timeout_ms,
}
if ssl is not None:
settings['ssl'] = ssl
if replica_set is not None:
settings['replica_set'] = replica_set
if auth_source is not None:
settings['auth_source'] = auth_source
if auth_user is not None:
settings['auth.user'] = auth_user
if auth_password is not None:
settings['auth.password'] = auth_password
return settings
def open(self, correlation_id):
"""
Opens the component.
:param correlation_id: (optional) transaction id to trace execution through call chain.
:return: callback function that receives error or null no errors occured.
"""
self._logger.debug(correlation_id, 'Connecting to mongodb')
try:
settings = self.__compose_settings()
# settings['use_new_url_parser'] = True
# settings['use_undefined_topology'] = True
settings['appname'] = correlation_id
uri = self._connection_resolver.resolve(correlation_id)
settings = self.__del_none_objects(settings)
client = pymongo.MongoClient(uri, **settings)
self._connection = client
self._db = client.get_database()
self._database_name = self._db.name
except Exception as ex:
raise ConnectionException(
correlation_id, 'CONNECT_FAILED',
'Connection to mongodb failed').with_cause(ex)
def __del_none_objects(self, settings):
new_settings = {}
for k in settings.keys():
if settings[k] is not None:
new_settings[k] = settings[k]
return new_settings
def close(self, correlation_id):
"""
Closes component and frees used resources.
:param correlation_id: (optional) transaction id to trace execution through call chain.
:return: callback function that receives error or null no errors occured.
"""
if self._connection is None:
return
try:
self._connection.close()
self._connection = None
self._db = None
self._database_name = None
self._logger.debug(
correlation_id, 'Disconnected from mongodb database {}'.format(
self._database_name))
except Exception as ex:
raise ConnectionException(
correlation_id, 'DISCONNECT_FAILED',
'Disconnect from mongodb failed: ').with_cause(ex)
def get_connection(self):
return self._connection
def get_database(self):
return self._db
def get_database_name(self):
return self._database_name
class GrpcClient(IOpenable, IReferenceable, IConfigurable):
"""
Abstract client that calls remove endpoints using GRPC protocol.
### Configuration parameters ###
- connection(s):
- discovery_key: (optional) a key to retrieve the connection from :func:`link`
- protocol: connection protocol: http or https
- host: host name or IP address
- port: port number
- uri: resource URI or connection string with all parameters in it
- options:
- retries: number of retries (default: 3)
- connect_timeout: connection timeout in milliseconds (default: 10 sec)
- timeout: invocation timeout in milliseconds (default: 10 sec)
.. code-block:: python
class MyGrpcClient(GrpcClient, IMyClient):
...
def get_data(self, correlation_id, id ):
timing = self.instrument(correlation_id, 'myclient.get_data')
result = self.call("get_data", correlation_id, { id: id })
timing.end_timing()
return result
...
client = MyGrpcClient()
client.configure(ConfigParams.from_tuples(
"connection.protocol", "http",
"connection.host", "localhost",
"connection.port", 8080
))
result = client.get_data("123", "1")
"""
_default_config = ConfigParams.from_tuples(
"connection.protocol", "http", "connection.host", "0.0.0.0",
"connection.port", 3000, "options.request_max_size", 1024 * 1024,
"options.connect_timeout", 10000, "options.timeout", 10000,
"options.retries", 3, "options.debug", True)
def __init__(self, client_name: str):
"""
Creates a new instance of the client.
:param client_name: a client name.
"""
self.__client = None
self.__client_name = None
# The GRPC client channel
self._channel = None
# The connection resolver.
self._connection_resolver = HttpConnectionResolver()
# The logger.
self._logger = CompositeLogger()
# The performance counters.
self._counters = CompositeCounters()
# The configuration options.
self._options = ConfigParams()
# The connection timeout in milliseconds.
self._connection_timeout = 100000
# The invocation timeout in milliseconds.
self._timeout = 100000
# The remote service uri which is calculated on open.
self._uri: str = None
self.__client_name = client_name
def configure(self, config: ConfigParams):
"""
Configures component by passing configuration parameters.
:param config: configuration parameters to be set.
"""
config = config.set_defaults(GrpcClient._default_config)
self._connection_resolver.configure(config)
self._options = self._options.override(config.get_section('options'))
self._connection_timeout = config.get_as_integer_with_default(
'options.connect_timeout', self._connection_timeout)
self._timeout = config.get_as_integer_with_default(
'options.timeout', self._timeout)
def set_references(self, references: IReferences):
"""
Sets references to dependent components.
:param references: references to locate the component dependencies.
"""
self._logger.set_references(references)
self._counters.set_references(references)
self._connection_resolver.set_references(references)
def _instrument(self, correlation_id: Optional[str],
name: str) -> CounterTiming:
"""
Adds instrumentation to log calls and measure call time.
It returns a CounterTiming object that is used to end the time measurement.
:param correlation_id: (optional) transaction id to trace execution through call chain.
:param name: a method name.
:return: CounterTiming object to end the time measurement.
"""
self._logger.trace(correlation_id, 'Executing {} method'.format(name))
self._counters.increment_one(name + '.call_count')
return self._counters.begin_timing(name + '.call_time')
def _instrument_error(self,
correlation_id: Optional[str],
name: str,
err: Exception,
reerror=False):
"""
Adds instrumentation to error handling.
:param correlation_id: (optional) transaction id to trace execution through call chain.
:param name: a method name.
:param err: an occured error
:param reerror: if true - throw error
"""
if err is not None:
self._logger.error(correlation_id, err,
'Failed to call {} method'.format(name))
self._counters.increment_one(name + '.call_errors')
if reerror is not None and reerror is True:
raise err
def is_open(self) -> bool:
"""
Checks if the component is opened.
:return: Returns true if the component has been opened and false otherwise.
"""
return self._channel is not None
def open(self, correlation_id: Optional[str]):
"""
Opens the component.
:param correlation_id: (optional) transaction id to trace execution through call chain.
"""
if self.is_open():
return None
try:
connection = self._connection_resolver.resolve(correlation_id)
self._uri = connection.get_as_string('uri')
options = [('grpc.max_connection_idle_ms',
self._connection_timeout),
('grpc.client_idle_timeout_ms', self._timeout)]
if connection.get_as_string_with_default('protocol',
'http') == 'https':
ssl_ca_file = connection.get_as_nullable_string('ssl_ca_file')
with open(ssl_ca_file, 'rb') as file:
trusted_root = file.read()
credentials = grpc.ssl_channel_credentials(trusted_root)
channel = grpc.secure_channel(
str(connection.get_as_string('host')) + ':' +
str(connection.get_as_string('port')),
credentials=credentials,
options=options)
else:
channel = grpc.insecure_channel(
str(connection.get_as_string('host')) + ':' +
str(connection.get_as_string('port')),
options=options)
self._channel = channel
except Exception as ex:
raise ConnectionException(
correlation_id, 'CANNOT_CONNECT',
'Opening GRPC client failed').wrap(ex).with_details(
'url', self._uri)
def close(self, correlation_id: Optional[str]):
"""
Closes component and frees used resources.
:param correlation_id: (optional) transaction id to trace execution through call chain.
"""
if self._channel is not None:
# Eat exceptions
try:
self._logger.debug(
correlation_id,
'Closed GRPC service at {}'.format(self._uri))
except Exception as ex:
self._logger.warn(
correlation_id,
'Failed while closing GRPC service: {}'.format(ex))
# if self.__client is not None:
# self.__client = None
self._channel.close()
self._channel = None
self._uri = None
GrpcClient._connection_resolver = HttpConnectionResolver()
def call(self, method: str, client: Any, request: Any) -> Future:
"""
Calls a remote method via GRPC protocol.
:param method: name of the calling method
:param client: current client
:param request: (optional) request object.
:return: (optional) future that receives result object or error.
"""
client = client(self._channel)
executor = futures.ThreadPoolExecutor(max_workers=1)
response = executor.submit(client.__dict__[method], request)
return response
def _add_filter_params(self, params: Any, filter: Any) -> Any:
"""
AddFilterParams method are adds filter parameters (with the same name as they defined)
to invocation parameter map.
:param params: invocation parameters.
:param filter: (optional) filter parameters
:return: invocation parameters with added filter parameters.
"""
params = StringValueMap() if params is None else params
if filter is not None:
for k in filter.keys():
params.put(k, filter[k])
return params
def _add_paging_params(self, params: Any, paging: Any) -> Any:
"""
AddPagingParams method are adds paging parameters (skip, take, total) to invocation parameter map.
:param params: invocation parameters.
:param paging: (optional) paging parameters
:return: invocation parameters with added paging parameters.
"""
params = StringValueMap() if params is None else params
if paging is not None:
params.put('total', paging.total)
if paging.skip is not None:
params.put('skip', paging.skip)
if paging.take is not None:
params.put('take', paging.take)
return params