def __init__(self):
self._default_config = ConfigParams.from_tuples(
"base_route", None, "dependencies.endpoint",
"*:endpoint:http:*:1.0")
# self._registered = False
self._dependency_resolver = DependencyResolver()
self._logger = CompositeLogger()
self._counters = CompositeCounters()
def __init__(self):
"""
Creates a new instance of the client.
"""
self._logger = CompositeLogger()
self._counters = CompositeCounters()
self._dependency_resolver = DependencyResolver()
self._dependency_resolver.put('controller', 'none')
def __init__(self, table_name: str = None, schema_name: str = None):
"""
Creates a new instance of the persistence component.
:param table_name: (optional) a table name.
:param schema_name: (optional) a schema name.
"""
self.__config: ConfigParams = None
self.__references: IReferences = None
self.__opened: bool = None
self.__local_connection: bool = None
self.__schema_statements = []
# The dependency resolver.
self._dependency_resolver: DependencyResolver = DependencyResolver(self._default_config)
# The logger.
self._logger: CompositeLogger = CompositeLogger()
# The MySQL connection component.
self._connection: MySqlConnection = None
# The MySQL connection pool object.
self._client: Any = None
# The MySQL database name.
self._database_name: str = None
# The MySQL table object.
self._table_name: str = None
# Max number of objects in data pages
self._max_page_size: int = 100
self._table_name = table_name
self._schema_name = schema_name
def __init__(self):
# The dependency resolver.
self._dependency_resolver: DependencyResolver = DependencyResolver(
self._default_config)
# The logger.
self._logger: CompositeLogger = CompositeLogger()
# The performance counters.
self._counters: CompositeCounters = CompositeCounters()
self._debug = False
# The base route.
self._base_route: str = None
# The HTTP endpoint that exposes this service.
self._endpoint: HttpEndpoint = None
# The tracer.
self._tracer: CompositeTracer = CompositeTracer()
self._config: ConfigParams = None
self._swagger_service: ISwaggerService = None
self._swagger_enabled = False
self._swagger_route = 'swagger'
self.__local_endpoint: bool = None
self.__references: IReferences = None
self.__opened: bool = None
class Component(IConfigurable, IReferenceable):
"""
Component class implementation.
Abstract component that supportes configurable dependencies,
logging and performance counters.
### Configuration parameters ###
- dependencies:
- [dependency name 1]: Dependency 1 locator (descriptor)
- ...
- [dependency name N]: Dependency N locator (descriptor)
### References ###
- *:counters:*:*:1.0 (optional) [[ICounters]] components to pass collected measurements
- *:logger:*:*:1.0 (optional) [[ILogger]] components to pass log messages
- ... References must match configured dependencies.
"""
_logger = None
_counters = None
_dependency_resolver = None
def __init__(self):
self._logger = CompositeLogger()
self._counters = CompositeCounters()
self._dependency_resolver = DependencyResolver()
def configure(self, config):
"""
Configures component by passing configuration parameters.
:param config: configuration parameters to be set.
"""
self._dependency_resolver.configure(config)
self._logger.configure(config)
def set_references(self, references):
"""
Sets references to dependent components.
:param references: references to locate the component dependencies.
"""
self._dependency_resolver.set_references(references)
self._logger.set_references(references)
self._counters.set_references(references)
def __init__(self, collection: str = None):
"""
Creates a new instance of the persistence component.
:param collection: (optional) a collection name.
"""
self._lock: threading.Lock = threading.Lock()
self._connection_resolver: MongoDbConnectionResolver = MongoDbConnectionResolver(
)
self._options: ConfigParams = ConfigParams()
# The logger.
self._logger: CompositeLogger = CompositeLogger()
# The dependency resolver.
self._dependency_resolver = DependencyResolver(self.__default_config)
# The MongoDB database name.
self._database_name: str = None
# The MongoDb database object.
self._db: Any = None
# The MongoDb collection object.
self._collection: Collection = None
# The MongoDB connection object.
self._client: Any = None
# The MongoDB connection component.
self._connection: MongoDbConnection = None
self._max_page_size = 100
# The MongoDB colleciton object.
self._collection_name: str = collection
self.__config: ConfigParams = None
self.__references: IReferences = None
self.__opened = False
self.__local_connection = False
self.__indexes: List[MongoDbIndex] = []
def test_resolve_depedencies(self):
ref1 = "AAA"
ref2 = "BBB"
refs = References.from_tuples(
"Reference1", ref1,
Descriptor("pip-services-commons", "reference", "object", "ref2",
"1.0"), ref2)
resolver = DependencyResolver.from_tuples(
"ref1", "Reference1", "ref2",
Descriptor("pip-services-commons", "reference", "*", "*", "*"))
resolver.set_references(refs)
assert ref1 == resolver.get_one_required("ref1")
assert ref2 == resolver.get_one_required("ref2")
assert None == resolver.get_one_optional("ref3")
def __init__(self):
"""
Creates a new instance of the client.
"""
# The controller reference.
self._controller: Any = None
# The open flag.
self._opened: bool = True
# The logger.
self._logger: CompositeLogger = CompositeLogger()
# The tracer.
self._tracer: CompositeTracer = CompositeTracer()
# The performance counters
self._counters: CompositeCounters = CompositeCounters()
# The dependency resolver to get controller reference.
self._dependency_resolver: DependencyResolver = DependencyResolver()
self._dependency_resolver.put('controller', 'none')
class FacadeRoutes(ABC, IConfigurable, IReferenceable):
_logger = CompositeLogger()
_counters = CompositeCounters()
_dependencyResolver = DependencyResolver()
_service: IFacadeService
def __init__(self):
self._dependencyResolver.put(
"service",
Descriptor("pip-services-facade", "service", "*", "*", "*"))
def configure(self, config):
self._dependencyResolver.configure(config)
def set_references(self, references):
self._logger.set_references(references)
self._counters.set_references(references)
self._dependencyResolver.set_references(references)
self._service = self._dependencyResolver.get_one_required('service')
self._register()
def instrument(self, correlation_id, method, route):
self._logger.debug(correlation_id, "Calling {} {}", method, route)
self._counters.increment_one(route + "." + method + ".calls")
def get_correlation_id(self, req):
return req.query.get("correlation_id")
def register_route(self, method, route, action):
def action_curl(req=None, res=None):
correlation_id = self.get_correlation_id(req)
self.instrument(correlation_id, method, route)
action(req, res)
self._service.register_route(method, route, action_curl)
@abstractmethod
def _register(self):
pass
class FacadeService(IConfigurable, IReferenceable, IFacadeService):
_root_path = ''
_partition = SessionMiddleware(bottle.Bottle(catchall=True, autojson=True)).app
_dependency_resolver = DependencyResolver()
_logger = CompositeLogger()
def configure(self, config):
self._root_path = config.get_as_string_with_default('root_path', self._root_path)
if len(self._root_path) > 0 and not (self._root_path.startswith('/')):
self._root_path = '/' + self._root_path
def set_references(self, references):
self._dependency_resolver.set_references(references)
self._logger.set_references(references)
def get_root_path(self):
return self._root_path
def register_middleware(self, action):
self._partition.add_hook('before_request', action)
def register_middleware_for_path(self, path, action):
self._partition.add_hook('before_request', lambda: action() if not (
path is not None and path != '' and bottle.request.url.startswith(path)) else None)
def register_route(self, method, route, action):
if method == 'del':
method = 'delete'
self._logger.debug(None, 'Registering route {} {}', method, self.get_root_path() + route)
self._partition.route(route, method, action)
def register_route_with_auth(self, method, route, action, authorize):
if method == 'del':
method = 'delete'
self._logger.debug(None, 'Registering route {} {}', method, self.get_root_path() + route)
self._partition.route(route, method, action, authorize=authorize)
def test_configure_depedencies(self):
ref1 = "AAA"
ref2 = "BBB"
refs = References.from_tuples(
"Reference1", ref1,
Descriptor("pip-services-commons", "reference", "object", "ref2",
"1.0"), ref2)
config = ConfigParams.from_tuples(
"dependencies.ref1", "Reference1", "dependencies.ref2",
"pip-services-commons:reference:*:*:*", "dependencies.ref3", None)
resolver = DependencyResolver(config)
resolver.set_references(refs)
assert ref1 == resolver.get_one_required("ref1")
assert ref2 == resolver.get_one_required("ref2")
assert None == resolver.get_one_optional("ref3")
def __init__(self):
self._logger: CompositeLogger = CompositeLogger()
self._counters: CompositeCounters = CompositeCounters()
self._dependency_resolver: DependencyResolver = DependencyResolver()
self._tracer: CompositeTracer = CompositeTracer()
class FacadeOperations(ABC, IConfigurable, IReferenceable):
_logger = CompositeLogger()
_counters = CompositeCounters()
_dependencyResolver = DependencyResolver()
def configure(self, config):
self._dependencyResolver.configure(config)
def set_references(self, references):
self._logger.set_references(references)
self._counters.set_references(references)
self._dependencyResolver.set_references(references)
def _get_correlation_id(self, req):
return req.query.get("correlation_id")
def _get_filter_params(self, req):
key_value_req = {}
for key, value in req.query.items:
if key not in ['skip', 'take', 'total']:
key_value_req[key] = value
filter = FilterParams.from_value(key_value_req)
return filter
def _get_paging_params(self, req):
key_value_req = {}
for key, value in req.query.items:
if key in ['skip', 'take', 'total']:
key_value_req[key] = value
filter = FilterParams.from_value(key_value_req)
return filter
def _send_result(self, res):
return HttpResponseSender.send_result(res)
def _send_empty_result(self, res):
return HttpResponseSender.send_empty_result(res)
def _send_created_result(self, res):
return HttpResponseSender.send_created_result(res)
def _send_deleted_result(self, res):
return HttpResponseSender.send_deleted_result(res)
def _send_error(self, err):
HttpResponseSender.send_error(err)
def _send_bad_request(self, req, err, message):
correlation_id = self._get_correlation_id(req)
error = BadRequestException(correlation_id, 'BAD_REQUEST', message)
self._send_error(error)
def _send_unauthorized(self, req, message):
correlation_id = self._get_correlation_id(req)
error = UnauthorizedException(correlation_id, 'UNAUTHORIZED', message)
self._send_error(error)
def _send_not_found(self, req, message):
correlation_id = self._get_correlation_id(req)
error = NotFoundException(correlation_id, 'NOT_FOUND', message)
self._send_error(error)
def _send_conflict(self, req, message):
correlation_id = self._get_correlation_id(req)
error = ConflictException(correlation_id, 'CONFLICT', message)
self._send_error(error)
def _send_session_expired(self, req, message):
correlation_id = self._get_correlation_id(req)
error = UnknownException(correlation_id, 'SESSION_EXPIRED', message)
error.status = 440
self._send_error(error)
def _send_internal_error(self, req, message):
correlation_id = self._get_correlation_id(req)
error = UnknownException(correlation_id, 'INTERNAL', message)
self._send_error(error)
def _send_server_unavailable(self, req, message):
correlation_id = self._get_correlation_id(req)
error = ConflictException(correlation_id, 'SERVER_UNAVAILABLE',
message)
error.status = 503
self._send_error(error)
class RestService(IOpenable, IConfigurable, IReferenceable, IUnreferenceable,
IRegisterable):
"""
Abstract service that receives remove calls via HTTP/REST protocol.
### Configuration parameters ###
- base_route: base route for remote URI
- dependencies:
- endpoint: override for HTTP Endpoint dependency
- controller: override for Controller dependency
- connection(s):
- discovery_key: (optional) a key to retrieve the connection from IDiscovery
- protocol: connection protocol: http or https
- host: host name or IP address
- port: port number
- uri: resource URI or connection string with all parameters in it
### References ###
- *:logger:*:*:1.0 (optional) ILogger components to pass log messages
- *:counters:*:*:1.0 (optional) ICounters components to pass collected measurements
- *:discovery:*:*:1.0 (optional) IDiscovery services to resolve connection
- *:endpoint:http:*:1.0 (optional) HttpEndpoint reference
Example:
class MyRestService(RestService):
_controller = None
...
def __init__(self):
super(MyRestService, self).__init__()
self._dependencyResolver.put("controller", Descriptor("mygroup","controller","*","*","1.0"))
def set_references(self, references):
super(MyRestService, self).set_references(references)
self._controller = self._dependencyResolver.get_required("controller")
def register():
...
service = MyRestService()
service.configure(ConfigParams.from_tuples("connection.protocol", "http",
"connection.host", "localhost",
"connection.port", 8080))
service.set_references(References.from_tuples(Descriptor("mygroup","controller","default","default","1.0"), controller))
service.open("123")
"""
_default_config = None
_debug = False
_dependency_resolver = None
_logger = None
_counters = None
# _registered = None
_local_endpoint = None
_config = None
_references = None
_base_route = None
_endpoint = None
_opened = None
def __init__(self):
self._default_config = ConfigParams.from_tuples(
"base_route", None, "dependencies.endpoint",
"*:endpoint:http:*:1.0")
# self._registered = False
self._dependency_resolver = DependencyResolver()
self._logger = CompositeLogger()
self._counters = CompositeCounters()
def _instrument(self, correlation_id, name):
"""
Adds instrumentation to log calls and measure call time. It returns a Timing object that is used to end the time measurement.
:param correlation_id: (optional) transaction id to trace execution through call chain.
:param name: a method name.
"""
self._logger.trace(correlation_id, "Executing " + name + " method")
return self._counters.begin_timing(name + ".exec_time")
def set_references(self, references):
"""
Sets references to dependent components.
:param references: references to locate the component dependencies.
"""
self._references = references
self._logger.set_references(references)
self._counters.set_references(references)
self._dependency_resolver.set_references(references)
self._endpoint = self._dependency_resolver.get_one_optional('endpoint')
if self._endpoint is None:
self._endpoint = self.create_endpoint()
self._local_endpoint = True
else:
self._local_endpoint = False
self._endpoint.register(self)
def configure(self, config):
"""
Configures component by passing configuration parameters.
:param config: configuration parameters to be set.
"""
config = config.set_defaults(self._default_config)
self._config = config
self._dependency_resolver.configure(config)
self._base_route = config.get_as_string_with_default(
"base_route", self._base_route)
def unset_references(self):
"""
Unsets (clears) previously set references to dependent components.
"""
if not (self._endpoint is None):
self._endpoint.unregister(self)
self._endpoint = None
def create_endpoint(self):
endpoint = HttpEndpoint()
if not (self._config is None):
endpoint.configure(self._config)
if not (self._references is None):
endpoint.set_references(self._references)
return endpoint
def _instrument(self, correlation_id, name):
self._logger.trace(correlation_id, f"Executing {name} method")
self._counters.increment_one(f"{name}.exec_count")
return self._counters.begin_timing(f"{name}.exec_time")
def _instrument_error(self, correlation_id, name, error, result, callback):
if not (error is None):
self._logger.error(correlation_id, error,
f"Failed to execute {name} method")
self._counters.increment_one(f"{name}.exec_error")
if not (callback is None):
callback(error, result)
def is_opened(self):
"""
Checks if the component is opened.
:return: true if the component has been opened and false otherwise.
"""
return self._opened
def open(self, correlation_id):
"""
Opens the component.
:param correlation_id: (optional) transaction id to trace execution through call chain.
"""
if self.is_opened():
return
if self._endpoint is None:
self._endpoint = self.create_endpoint()
self._endpoint.register(self)
self._local_endpoint = True
if self._local_endpoint:
self._endpoint.open(correlation_id)
self._opened = True
# # register route
# if self._registered != True:
# self.add_route()
# self._registered = True
def close(self, correlation_id):
"""
Closes component and frees used resources.
:param correlation_id: (optional) transaction id to trace execution through call chain.
"""
if not self._opened:
return
if self._endpoint is None:
raise InvalidStateException(correlation_id, "NO_ENDPOINT",
"HTTP endpoint is missing")
if self._local_endpoint:
self._endpoint.close(correlation_id)
self._opened = False
def send_result(self, result):
"""
Creates a callback function that sends result as JSON object. That callack function call be called directly or passed as a parameter to business logic components.
If object is not null it returns 200 status code. For null results it returns
204 status code. If error occur it sends ErrorDescription with approproate status code.
:param result: a body object to result.
:return: execution result.
"""
return HttpResponseSender.send_result(result)
def send_created_result(self, result):
"""
Creates a callback function that sends newly created object as JSON. That callack function call be called directly or passed as a parameter to business logic components.
If object is not null it returns 201 status code. For null results it returns
204 status code. If error occur it sends ErrorDescription with approproate status code.
:param result: a body object to result.
:return: execution result.
"""
return HttpResponseSender.send_created_result(result)
def send_deleted_result(self):
"""
Creates a callback function that sends newly created object as JSON. That callack function call be called directly or passed as a parameter to business logic components.
If object is not null it returns 200 status code. For null results it returns
204 status code. If error occur it sends ErrorDescription with approproate status code.
:return: execution result.
"""
return HttpResponseSender.send_deleted_result()
def send_error(self, error):
"""
Sends error serialized as ErrorDescription object and appropriate HTTP status code. If status code is not defined, it uses 500 status code.
:param error: an error object to be sent.
"""
return HttpResponseSender.send_error(error)
def fix_route(self, route) -> str:
if (route is not None and len(route) > 0):
if (route[0] != '/'):
route = f'/{route}'
return route
return ''
def register_route(self, method, route, schema, handler):
"""
Registers an action in this objects REST server (service) by the given method and route.
:param method: the HTTP method of the route.
:param route: the route to register in this object's REST server (service).
:param schema: the schema to use for parameter validation.
:param handler: the action to perform at the given route.
"""
if self._endpoint is None:
return
route = f"{self.fix_route(self._base_route)}{self.fix_route(route)}"
# if not (self._base_route is None) and len(self._base_route) > 0:
# base_route = self._base_route
# if base_route[0] != '/':
# base_route = '/' + base_route
# if route[0] != '/':
# base_route = base_route + '/'
# route = base_route + route
self._endpoint.register_route(method, route, schema, handler)
def register(self):
pass
def get_data(self):
data = bottle.request.json
if isinstance(data, str):
return json.loads(bottle.request.json)
elif bottle.request.json:
return bottle.request.json
else:
return None
def get_request_param(self):
if (not (bottle is None)) and (not (bottle.request is None)) and (
not (bottle.request.params is None)):
return bottle.request.params
else:
return {}
def _append_base_route(self, route):
route = route or ''
if self._base_route is not None and len(self._base_route) > 0:
base_route = self._base_route
if base_route[0] != '/':
base_route = '/' + base_route
route = base_route + route
return route
def register_route_with_auth(self, method, route, schema, authorize,
action):
"""
Registers a route with authorization in HTTP endpoint.
:param method: HTTP method: "get", "head", "post", "put", "delete"
:param route: a command route. Base route will be added to this route
:param schema: a validation schema to validate received parameters.
:param authorize: an authorization interceptor
:param action: an action function that is called when operation is invoked.
"""
if self._endpoint is None:
return
route = self._append_base_route(self.fix_route(route))
self._endpoint.register_route_with_auth(method, route, schema,
authorize, method)
def register_interceptor(self, route, action):
"""
Registers a middleware for a given route in HTTP endpoint.
:param route: a command route. Base route will be added to this route
:param action: an action function that is called when middleware is invoked.
"""
if self._endpoint is None:
return
route = self._append_base_route(self.fix_route(route))
self._endpoint.register_interceptor(route, action)
class MongoDbPersistence(IReferenceable, IUnreferenceable, IConfigurable,
IOpenable, ICleanable):
"""
Abstract persistence component that stores data in MongoDB
using the official MongoDB driver.
This is the most basic persistence component that is only
able to store data items of any type. Specific CRUD operations
over the data items must be implemented in child classes by
accessing **self.__collection** or **self.__model** properties.
### Configuration parameters ###
- collection: (optional) MongoDB collection name
- connection(s):
- discovery_key: (optional) a key to retrieve the connection from :class:`IDiscovery <pip_services3_components.connect.IDiscovery.IDiscovery>`
- host: host name or IP address
- port: port number (default: 27017)
- uri: resource URI or connection string with all parameters in it
- credential(s):
- store_key: (optional) a key to retrieve the credentials from :class:`ICredentialStore <pip_services3_components.auth.ICredentialStore.ICredentialStore>`
- username: (optional) user name
- password: (optional) user password
- options:
- max_pool_size: (optional) maximum connection pool size (default: 2)
- keep_alive: (optional) enable connection keep alive (default: true)
- connect_timeout: (optional) connection timeout in milliseconds (default: 5000)
- socket_timeout: (optional) socket timeout in milliseconds (default: 360000)
- auto_reconnect: (optional) enable auto reconnection (default: true)
- reconnect_interval: (optional) reconnection interval in milliseconds (default: 1000)
- max_page_size: (optional) maximum page size (default: 100)
- replica_set: (optional) name of replica set
- ssl: (optional) enable SSL connection (default: false)
- auth_source: (optional) authentication source
- debug: (optional) enable debug output (default: false).
### References ###
- `*:logger:*:*:1.0` (optional) :class:`ILogger <pip_services3_components.log.ILogger.ILogger>` components to pass log messages
- `*:discovery:*:*:1.0` (optional) :class:`IDiscovery <pip_services3_components.connect.IDiscovery.IDiscovery>` services
- `*:credential-store:*:*:1.0` (optional) :class:`ICredentialStore <pip_services3_components.auth.ICredentialStore.ICredentialStore>` stores to resolve credentials
Example:
.. code-block:: python
class MyMongoDbPersistence(MongoDbPersistence):
def __init__(self):
super(MyMongoDbPersistence, self).__init__("mydata", MyData)
def get_by_name(self, correlationId, name):
item = self._collection.find_one({ 'name': name })
return item
def set(self, correlationId, item):
item = self._collection.find_one_and_update(
{ '_id': item.id }, { '$set': item },
return_document = pymongo.ReturnDocument.AFTER,
upsert = True
)
persistence = MyMongoDbPersistence()
persistence.configure(ConfigParams.from_tuples("host", "localhost", "port", 27017))
persitence.open("123")
persistence.set("123", { name: "ABC" })
item = persistence.get_by_name("123", "ABC")
print (item)
"""
__default_config = ConfigParams.from_tuples(
"collection",
None,
"dependencies.connection",
"*:connection:mongodb:*:1.0",
# "connect.type", "mongodb",
# "connect.database", "test",
# "connect.host", "localhost",
# "connect.port", 27017,
"options.max_pool_size",
2,
"options.keep_alive",
1,
"options.connect_timeout",
5000,
"options.auto_reconnect",
True,
"options.max_page_size",
100,
"options.debug",
True)
def __init__(self, collection: str = None):
"""
Creates a new instance of the persistence component.
:param collection: (optional) a collection name.
"""
self._lock: threading.Lock = threading.Lock()
self._connection_resolver: MongoDbConnectionResolver = MongoDbConnectionResolver(
)
self._options: ConfigParams = ConfigParams()
# The logger.
self._logger: CompositeLogger = CompositeLogger()
# The dependency resolver.
self._dependency_resolver = DependencyResolver(self.__default_config)
# The MongoDB database name.
self._database_name: str = None
# The MongoDb database object.
self._db: Any = None
# The MongoDb collection object.
self._collection: Collection = None
# The MongoDB connection object.
self._client: Any = None
# The MongoDB connection component.
self._connection: MongoDbConnection = None
self._max_page_size = 100
# The MongoDB colleciton object.
self._collection_name: str = collection
self.__config: ConfigParams = None
self.__references: IReferences = None
self.__opened = False
self.__local_connection = False
self.__indexes: List[MongoDbIndex] = []
def configure(self, config: ConfigParams):
"""
Configures component by passing configuration parameters.
:param config: configuration parameters to be set.
"""
config = config.set_defaults(self.__default_config)
self.__config = config
self._logger.configure(config)
self._connection_resolver.configure(config)
self._dependency_resolver.configure(config)
self._max_page_size = config.get_as_integer_with_default(
"options.max_page_size", self._max_page_size)
self._collection_name = config.get_as_string_with_default(
'collection', self._collection_name)
def set_references(self, references: IReferences):
"""
Sets references to dependent components.
:param references: references to locate the component dependencies.
"""
self.__references = references
self._logger.set_references(references)
self._connection_resolver.set_references(references)
# Get connection
self._dependency_resolver.set_references(references)
self._connection = self._dependency_resolver.get_one_optional(
'connection')
# Or create a local one
if self._connection is None:
self._connection = self.__create_connection()
self.__local_connection = True
else:
self.__local_connection = False
def unset_references(self):
"""
Unsets (clears) previously set references to dependent components.
"""
self._connection = None
def __create_connection(self) -> MongoDbConnection:
connection = MongoDbConnection()
if self.__config:
connection.configure(self.__config)
if self.__references:
connection.set_references(self.__references)
return connection
def _ensure_index(self, keys: Any, options: Any = None):
"""
Adds index definition to create it on opening
:param keys: index keys (fields)
:param options: index options
"""
if not keys:
return
self.__indexes.append(MongoDbIndex(keys, options))
def _clear_schema(self):
"""
Clears all auto-created objects
"""
self.__indexes = []
def _define_schema(self):
# TODO: override in child class
pass
def _convert_to_public(self, value: Any) -> Any:
"""
Converts object value from internal to public format.
:param value: an object in internal format to convert.
:return: converted object in public format.
"""
if value is None: return None
if value.get('_id'):
value['id'] = value['_id']
value.pop('_id', None)
return type('object', (object, ), value)
def _convert_from_public(self, value: Any) -> Any:
"""
Convert object value from public to internal format.
:param value: an object in public format to convert.
:return: converted object in internal format.
"""
if isinstance(value, dict):
return deepcopy(value)
value = PropertyReflector.get_properties(value)
if value.get('id'):
value['_id'] = value.get('id') or value.get('_id')
value.pop('id', None)
return value
def is_open(self) -> bool:
"""
Checks if the component is opened.
:return: true if the component has been opened and false otherwise.
"""
return self.__opened
def open(self, correlation_id: Optional[str]):
"""
Opens the component.
:param correlation_id: (optional) transaction id to trace execution through call chain.
"""
if self.__opened:
return
if self._connection is None:
self._connection = self.__create_connection()
self.__local_connection = True
if self.__local_connection:
self._connection.open(correlation_id)
if self._connection is None:
raise InvalidStateException(correlation_id, 'NO_CONNECTION',
'MongoDB connection is missing')
if not self._connection.is_open():
raise ConnectionException(correlation_id, "CONNECT_FAILED",
"MongoDB connection is not opened")
self.__opened = False
self._client = self._connection.get_connection()
self._db = self._connection.get_database()
self._database_name = self._connection.get_database_name()
try:
self._collection = self._db.get_collection(self._collection_name)
# Define database schema
self._define_schema()
# Recreate indexes
for index in self.__indexes:
keys = [(k, pymongo.ASCENDING) if v > 0 else
(k, pymongo.DESCENDING) for k, v in index.keys.items()]
index.options = index.options or {}
self._collection.create_index(keys, **(index.options or {}))
index_name = index.options.get('name') or ','.join(
deepcopy(index.keys))
self._logger.debug(correlation_id,
"Created index %s for collection %s",
index_name, self._collection_name)
self.__opened = True
self._logger.debug(
correlation_id,
"Connected to mongodb database %s, collection %s",
self._database_name, self._collection_name)
except Exception as ex:
raise ConnectionException(
correlation_id, "CONNECT_FAILED",
"Connection to mongodb failed").with_cause(ex)
def __del_none_objects(self, settings):
new_settings = {}
for k in settings.keys():
if settings[k] is not None:
new_settings[k] = settings[k]
return new_settings
def close(self, correlation_id: Optional[str]):
"""
Closes component and frees used resources.
:param correlation_id: (optional) transaction id to trace execution through call chain.
"""
if not self.__opened:
return
try:
if self._client is not None:
self._client.close()
if self._connection is None:
raise InvalidStateException(correlation_id, 'NO_CONNECTION',
'MongoDb connection is missing')
if self.__local_connection:
self._connection.close(correlation_id)
self._collection = None
self._db = None
self._client = None
self.__opened = False
self._logger.debug(
correlation_id, "Disconnected from mongodb database " +
str(self._database_name))
except Exception as ex:
raise ConnectionException(None, 'DISCONNECT_FAILED', 'Disconnect from mongodb failed: ' + str(ex)) \
.with_cause(ex)
def clear(self, correlation_id: Optional[str]):
"""
Clears component state.
:param correlation_id: (optional) transaction id to trace execution through call chain.
"""
if self._collection_name is None:
raise Exception("Collection name is not defined")
self._collection.delete_many({})
def create(self, correlation_id: Optional[str], item: T) -> T:
"""
Creates a data item.
:param correlation_id: (optional) transaction id to trace execution through call chain.
:param item: an item to be created.
:return: a created item
"""
if item is None:
return
new_item = self._convert_from_public(item)
result = self._collection.insert_one(new_item)
item = self._collection.find_one({'_id': result.inserted_id})
item = self._convert_to_public(item)
return item
def delete_by_filter(self, correlation_id: Optional[str], filter: Any):
"""
Deletes data items that match to a given filter.
This method shall be called by a public :func:`delete_by_filter` method from child class that
receives :class:`FilterParams <pip_services3_commons.data.FilterParams.FilterParams>` and converts them into a filter function.
:param correlation_id: (optional) transaction id to trace execution through call chain.
:param filter: (optional) a filter function to filter items.
"""
result = self._collection.delete_many(filter)
count = 0 if result is None else result.deleted_count
self._logger.trace(correlation_id, "Deleted %d items from %s", count,
self._collection_name)
def get_one_random(self, correlation_id: Optional[str],
filter: Any) -> Optional[T]:
"""
Gets a random item from items that match to a given filter.
This method shall be called by a public get_one_random method from child class
that receives FilterParams and converts them into a filter function.
:param correlation_id: (optional) transaction id to trace execution through call chain.
:return: a random item.
"""
count = self._collection.count_documents(filter)
pos = random.randint(0, count)
statement = self._collection.find(filter).skip(
pos if pos > 0 else 0).limit(1)
for item in statement:
if item is None:
self._logger.trace(correlation_id,
"Random item wasn't found from %s",
self._collection_name)
else:
self._logger.trace(correlation_id,
"Retrieved random item from %s",
self._collection_name)
item = self._convert_to_public(item)
return item
return None
def get_page_by_filter(self,
correlation_id: Optional[str],
filter: Any,
paging: PagingParams,
sort: Any = None,
select: Any = None) -> DataPage:
"""
Gets a page of data items retrieved by a given filter and sorted according to sort parameters.
This method shall be called by a public get_page_by_filter method from child class that
receives FilterParams and converts them into a filter function.
:param correlation_id: (optional) transaction id to trace execution through call chain.
:param filter: (optional) a filter JSON object
:param paging: (optional) paging parameters
:param sort: (optional) sorting JSON object
:param select: (optional) projection JSON object
:return: a data page of result by filter
"""
# Adjust max item count based on configuration
paging = paging if paging is not None else PagingParams()
skip = paging.get_skip(-1)
take = paging.get_take(self._max_page_size)
paging_enabled = paging.total
# Configure statement
statement = self._collection.find(filter, projection=select or {})
if skip >= 0:
statement = statement.skip(skip)
statement = statement.limit(take)
if sort is not None:
statement = statement.sort(sort)
# Retrive page items
items = []
for item in statement:
item = self._convert_to_public(item)
items.append(item)
if items:
self._logger.trace(correlation_id, "Retrieved %d from %s",
len(items), self._collection_name)
# Calculate total if needed
total = None
if paging_enabled:
total = self._collection.count_documents(filter)
return DataPage(items, total)
def get_list_by_filter(self,
correlation_id: Optional[str],
filter: Any,
sort: Any = None,
select: Any = None) -> List[T]:
"""
Gets a list of data items retrieved by a given filter and sorted according to sort parameters.
This method shall be called by a public get_list_by_filter method from child class that
receives FilterParams and converts them into a filter function.
:param correlation_id: (optional) transaction id to trace execution through call chain.
:param filter: (optional) a filter function to filter items
:param sort: (optional) sorting parameters
:param select: (optional) projection parameters (not used yet)
:return: a data list of results by filter.
"""
# Configure statement
statement = self._collection.find(filter, projection=select or {})
if sort is not None:
statement = statement.sort(sort)
# Retrive page items
items = []
for item in statement:
item = self._convert_to_public(item)
items.append(item)
if items:
self._logger.trace(correlation_id, "Retrieved %d from %s",
len(items), self._collection_name)
return items
def get_count_by_filter(self, correlation_id: Optional[str],
filter: Any) -> int:
"""
Gets a number of data items retrieved by a given filter.
This method shall be called by a public get_count_by_filter method from child class that
receives FilterParams and converts them into a filter function.
:param correlation_id: (optional) transaction id to trace execution through call chain.
:param filter: (optional) a filter JSON object
:return: a number of filtered items.
"""
count = self._collection.count_documents(filter)
if count is not None:
self._logger.trace(correlation_id, "Counted %d items in %s", count,
self._collection_name)
return count
class GrpcService(IOpenable, IConfigurable, IRegisterable, IUnreferenceable):
"""
Abstract service that receives remove calls via GRPC protocol.
### Configuration parameters ###
- dependencies:
- endpoint: override for GRPC Endpoint dependency
- controller: override for Controller dependency
- connection(s):
- discovery_key: (optional) a key to retrieve the connection from [[https://rawgit.com/pip-services-node/pip-services3-components-node/master/doc/api/interfaces/connect.idiscovery.html IDiscovery]]
- protocol: connection protocol: http or https
- host: host name or IP address
- port: port number
- uri: resource URI or connection string with all parameters in it
- credential - the HTTPS credentials:
- ssl_key_file: the SSL private key in PEM
- ssl_crt_file: the SSL certificate in PEM
- ssl_ca_file: the certificate authorities (root cerfiticates) in PEM
### Example ###
TODO!!!
"""
__default_config = ConfigParams.from_tuples("dependencies.endpoint",
"*:endpoint:grpc:*:1.0")
__service_name = None
__config = None
__references = None
__local_endpoint = None
__registrable = None
__implementation = {}
__interceptors = []
__opened = None
# The GRPC endpoint that exposes this service.
_endpoint = None
# The dependency resolver.
_dependency_resolver = DependencyResolver(__default_config)
# The logger.
_logger = CompositeLogger()
# The performance counters.
_counters = CompositeCounters()
def __init__(self, service_name=None):
self.__service_name = service_name
self.__registrable = lambda: self._register_service()
def configure(self, config):
"""
Configures component by passing configuration parameters.
:param config: configuration parameters to be set.
"""
config = config.set_defaults(GrpcService.__default_config)
self.__config = config
self._dependency_resolver.configure(config)
def set_references(self, references):
"""
Sets references to this endpoint's logger, counters, and connection resolver.
__References:__
- logger: <code>"\*:logger:\*:\*:1.0"</code>
- counters: <code>"\*:counters:\*:\*:1.0"</code>
- discovery: <code>"\*:discovery:\*:\*:1.0"</code> (for the connection resolver)
:param references: an IReferences object, containing references to a logger, counters,
and a connection resolver.
"""
self._logger.set_references(references)
self._counters.set_references(references)
self._dependency_resolver.set_references(references)
# Get endpoint
self._endpoint = self._dependency_resolver.get_one_optional('endpoint')
# Or create a local one
if self._endpoint is None:
self._endpoint = self.__create_endpoint()
self.__local_endpoint = True
else:
self.__local_endpoint = False
# Add registration callback to the endpoint
self._endpoint.register(self)
def unset_references(self):
"""
Unsets (clears) previously set references to dependent components.
"""
# Remove registration callback from endpoint
if self._endpoint is not None:
self._endpoint.unregister(self.__registrable)
self._endpoint = None
def __create_endpoint(self):
endpoint = GrpcEndpoint()
if self.__config:
endpoint.configure(self.__config)
if self.__references:
endpoint.set_references(self.__references)
return endpoint
def _instrument(self, correlation_id, name):
"""
Adds instrumentation to log calls and measure call time.
It returns a Timing object that is used to end the time measurement.
:param correlation_id: (optional) transaction id to trace execution through call chain.
:param name: a method name.
:return: Timing object to end the time measurement.
"""
self._logger.trace(correlation_id, 'Executing {} method'.format(name))
self._counters.increment_one(name + '.exec_time')
return self._counters.begin_timing(name + '.exec_time')
def _instrument_error(self, correlation_id, name, err, reerror=False):
if err is not None:
self._logger.error(correlation_id, err,
'Failed to execute {} method'.format(name))
self._counters.increment_one(name + '.exec_errors')
if reerror:
raise reerror
def is_open(self):
"""
Checks if the component is opened.
:return: true if the component has been opened and false otherwise.
"""
return self.__opened
def open(self, correlation_id):
"""
Opens the component.
:param correlation_id: (optional) transaction id to trace execution through call chain.
:param callback: callback function that receives error or null no errors occured.
"""
# TODO maybe need add async
if self.__opened:
return None
if self._endpoint is None:
self._endpoint = self.__create_endpoint()
self._endpoint.register(self)
self.__local_endpoint = True
if self.__local_endpoint:
try:
self._endpoint.open(correlation_id)
self.__opened = True
except Exception as ex:
self.__opened = False
raise ex
else:
self.__opened = True
def close(self, correlation_id):
"""
Closes component and frees used resources.
:param correlation_id: (optional) transaction id to trace execution through call chain.
:param callback: callback function that receives error or null no errors occured.
"""
if not self.__opened:
return None
if self._endpoint is None:
raise InvalidStateException(correlation_id, 'NO_ENDPOINT',
'HTTP endpoint is missing')
if self.__local_endpoint:
self._endpoint.close(correlation_id)
self.__opened = False
def register_commadable_method(self, method, schema, action):
self._endpoint.register_commandable_method(method, schema, action)
def _register_interceptor(self, action):
if self._endpoint is not None:
self._endpoint.register_interceptor(action)
def _register_service(self, implementation):
# self.register()
if self._endpoint is not None:
self._endpoint.register_service(implementation)
def register(self):
"""
class DirectClient(IConfigurable, IReferenceable, IOpenable):
"""
Abstract client that calls controller directly in the same memory space. It is used when multiple microservices are deployed in a single container (monolyth) and communication between them can be done by direct calls rather then through the network.
### Configuration parameters ###
- dependencies:
- controller: override controller descriptor
### References ###
- *:logger:*:*:1.0 (optional) ILogger components to pass log messages
- *:counters:*:*:1.0 (optional) ICounters components to pass collected measurements
- *:controller:*:*:1.0 controller to call business methods
Example:
class MyDirectClient(DirectClient, IMyClient):
def __init__(self):
super(MyDirectClient, self).__init__()
self._dependencyResolver.put('controller', Descriptor("mygroup", "controller", "*", "*", "*"))
...
def get_data(self, correlation_id, id):
timing = self.instrument(correlationId, 'myclient.get_data')
result = self._controller.get_data(correlationId, id)
timing.end_timing()
return result
client = MyDirectClient()
client.set_references(References.from_tuples(Descriptor("mygroup","controller","default","default","1.0"), controller))
data = client.get_data("123", "1")
...
"""
_controller = None
_opened = True
_logger = None
_counters = None
_dependency_resolver = None
def __init__(self):
"""
Creates a new instance of the client.
"""
self._logger = CompositeLogger()
self._counters = CompositeCounters()
self._dependency_resolver = DependencyResolver()
self._dependency_resolver.put('controller', 'none')
def configure(self, config):
"""
Configures component by passing configuration parameters.
:param config: configuration parameters to be set.
"""
self._dependency_resolver.configure(config)
def set_references(self, references):
"""
Sets references to dependent components.
:param references: references to locate the component dependencies.
"""
self._logger.set_references(references)
self._counters.set_references(references)
self._dependency_resolver.set_references(references)
self._controller = self._dependency_resolver.get_one_required('controller')
def _instrument(self, correlation_id, name):
"""
Adds instrumentation to log calls and measure call time. It returns a Timing object that is used to end the time measurement.
:param correlation_id: (optional) transaction id to trace execution through call chain.
:param name: a method name.
:return: Timing object to end the time measurement.
"""
self._logger.trace(correlation_id, f"Executing {name} method")
return self._counters.begin_timing(f"{name} .call_time")
def _instrument_error(self, correlation_id, name, err, result, callback):
"""
Adds instrumentation to error handling.
:param correlation_id: (optional) transaction id to trace execution through call chain.
:param name: a method name.
:param err: an occured error
:param result: (optional) an execution result
:param callback: (optional) an execution callback
"""
if err is not None:
self._logger.error(correlation_id, err, f'Failed to call {name} method')
self._counters.increment_one(f"{name}.call_errors")
if callback:
callback(err, result)
def is_opened(self):
"""
Checks if the component is opened.
:return: true if the component has been opened and false otherwise.
"""
return self._opened
def open(self, correlation_id):
"""
Opens the component.
:param correlation_id: (optional) transaction id to trace execution through call chain.
"""
if self._opened:
return
if self._controller is None:
raise ConnectionException(correlation_id, 'NO_CONTROLLER', 'Controller references is missing')
self._opened = True
self._logger.info(correlation_id, 'Opened direct client')
def close(self, correlation_id):
"""
Closes component and frees used resources.
:param correlation_id: (optional) transaction id to trace execution through call chain.
"""
if self._opened:
self._logger.info(correlation_id, 'Closed direct client')
self._opened = False