Ejemplo n.º 1
0
class GrpcEndpoint(IOpenable, IConfigurable, IReferenceable):
    """
    Used for creating GRPC endpoints. An endpoint is a URL, at which a given service can be accessed by a client.

    ### Configuration parameters ###

    Parameters to pass to the :func:`configure` method for component configuration:

        - connection(s) - the connection resolver's connections:
            - "connection.discovery_key" - the key to use for connection resolving in a discovery service;
            - "connection.protocol" - the connection's protocol;
            - "connection.host" - the target host;
            - "connection.port" - the target port;
            - "connection.uri" - the target URI.
        - credential - the HTTPS credentials:
            - "credential.ssl_key_file" - the SSL private key in PEM
            - "credential.ssl_crt_file" - the SSL certificate in PEM
            - "credential.ssl_ca_file" - the certificate authorities (root cerfiticates) in PEM

    ### References ###
    
    A logger, counters, and a connection resolver can be referenced by passing the
    following references to the object's :func:`set_references` method:

        - logger: **\*:logger:\*:\*:1.0"**;
        - counters: **"\*:counters:\*:\*:1.0"**;
        - discovery: **"\*:discovery:\*:\*:1.0"** (for the connection resolver).

    .. code-block:: python

        def my_method(self, _config, _references):
            endpoint = GrpcEndpoint()
            if self._config:
                endpoint.configure(self._config)
            if self._references:
                endpoint.set_references(self._references)
            ...

            self._endpoint.open(correlation_id)
            ...


    """

    __defaultConfig = ConfigParams.from_tuples(
        "connection.protocol", "http",
        "connection.host", "0.0.0.0",
        "connection.port", 3000,

        "credential.ssl_key_file", None,
        "credential.ssl_crt_file", None,
        "credential.ssl_ca_file", None,

        "options.maintenance_enabled", None,
        "options.request_max_size", 1024 * 1024,
        "options.file_max_size", 200 * 1024 * 1024,
        "options.connect_timeout", 60000,
        "options.debug", None
    )

    def __init__(self):
        self.__server: Any = None
        self.__connection_resolver = HttpConnectionResolver()
        self.__logger = CompositeLogger()
        self.__counters = CompositeCounters()
        self.__maintenance_enabled = False
        self.__file_max_size = 200 * 1024 * 1024
        self.__uri: str = None
        self.__registrations: List[IRegisterable] = []
        self.__commandable_methods: Any = None
        self.__commandable_schemas: Any = None
        self.__commandable_service: Any = None
        self.__interceptors = []

    def configure(self, config: ConfigParams):
        """
        Configures this HttpEndpoint using the given configuration parameters.
        
        ### Configuration parameters ###
            - connection(s) - the connection resolver's connections;
            - "connection.discovery_key" - the key to use for connection resolving in a dis
            - "connection.protocol" - the connection's protocol;
            - "connection.host" - the target host;
            - "connection.port" - the target port;
            - "connection.uri" - the target URI.
            - "credential.ssl_key_file" - SSL private key in PEM
            - "credential.ssl_crt_file" - SSL certificate in PEM
            - "credential.ssl_ca_file" - Certificate authority (root certificate) in PEM

        :param config: configuration parameters, containing a "connection(s)" section.
        """
        config = config.set_defaults(GrpcEndpoint.__defaultConfig)
        self.__connection_resolver.configure(config)

        self.__maintenance_enabled = config.get_as_boolean_with_default('options.maintenance_enabled',
                                                                        self.__maintenance_enabled)
        self.__file_max_size = ConfigParams().get_as_long_with_default(key='options.file_max_size',
                                                                       default_value=self.__file_max_size)

    def set_references(self, references: IReferences):
        """
        Sets references to this endpoint's logger, counters, and connection resolver.

        __References:__
        - logger: **"\*:logger:\*:\*:1.0"**
        - counters: **"\*:counters:\*:\*:1.0"**
        - discovery: **"\*:discovery:\*:\*:1.0"** (for the connection resolver)

        :param references: an IReferences object, containing references to a logger, counters, and a connection resolver.
        """
        self.__logger.set_references(references)
        self.__counters.set_references(references)
        self.__connection_resolver.set_references(references)

    def is_open(self) -> bool:
        """
        :return: whether or not this endpoint is open with an actively listening GRPC server.
        """
        return self.__server is not None

    def open(self, correlation_id: Optional[str], max_workers: int = 15):
        """
        Opens a connection using the parameters resolved by the referenced connection
        resolver and creates a GRPC server (service) using the set options and parameters.

        :param correlation_id: (optional) transaction id to trace execution through call chain.
        :param max_workers: max count of workers in thread pool.
        """
        if self.is_open():
            return

        connection = self.__connection_resolver.resolve(correlation_id)
        self.__uri = connection.get_as_string('uri')
        try:
            self.__connection_resolver.register(correlation_id)

            credentials = None

            if connection.get_as_string_with_default('protocol', 'http') == 'https':
                ssl_key_file = connection.get_as_nullable_string('ssl_key_file')
                ssl_crt_file = connection.get_as_nullable_string('ssl_crt_file')

                with open(ssl_key_file, 'rb') as file:
                    private_key = file.read()

                with open(ssl_crt_file, 'rb') as file:
                    certificate = file.read()

                credentials = grpc.ssl_server_credentials(((private_key, certificate),))

            # Create instance of express application
            if len(self.__interceptors) > 0:
                self.__server = grpc.server(futures.ThreadPoolExecutor(max_workers=max_workers),
                                            interceptors=self.__interceptors)
            else:
                self.__server = grpc.server(futures.ThreadPoolExecutor(max_workers=max_workers))

            if credentials:
                self.__server.add_secure_port(str(connection.get_as_string('host')) + ':' +
                                              str(connection.get_as_string('port')), credentials)
            else:
                self.__server.add_insecure_port(
                    str(connection.get_as_string('host')) + ':' + str(connection.get_as_string('port')))
            # Start operations
            self.__server.start()
            self.__connection_resolver.register(correlation_id)
            self.__perform_registrations()
            self.__logger.debug(correlation_id, 'Opened GRPC service at {}'.format(self.__uri))

        except Exception as ex:
            self.__server = None
            err = ConnectionException(correlation_id, 'CANNOT_CONNECT', 'Opening GRPC service failed').wrap(
                ex).with_details('url', self.__uri)
            raise err

    def close(self, correlation_id: Optional[str]):
        """
        Closes this endpoint and the GRPC server (service) that was opened earlier.

        :param correlation_id: (optional) transaction id to trace execution through call chain.
        """
        if self.__server is not None:
            self.__uri = None
            self.__commandable_methods = None
            self.__commandable_schemas = None
            self.__commandable_service = None

            # Eat exceptions
            try:
                self.__server.stop(None)
                self.__logger.debug(correlation_id, 'Closed GRPC service at {}'.format(self.__uri))
                self.__server = None
                GrpcEndpoint.__interceptors = []
                GrpcEndpoint.__registrations = []
                GrpcEndpoint.__connection_resolver = HttpConnectionResolver()
            except Exception as ex:
                self.__logger.warn(correlation_id, 'Failed while closing GRPC service: '.format(ex))

    def register(self, registration: IRegisterable):
        """
        Registers a registerable object for dynamic endpoint discovery.

        :param registration: the registration to add.
        """
        if registration is not None:
            self.__registrations.append(registration)

    def unregister(self, registration: IRegisterable):
        """
        Unregisters a registerable object, so that it is no longer used in dynamic
        endpoint discovery.

        :param registration: the registration to remove.
        """
        self.__registrations = list(filter(lambda r: r == registration, self.__registrations))

    def __perform_registrations(self):
        for registration in self.__registrations:
            registration.register()

        self.__register_commandable_service()

    def __register_commandable_service(self):
        if self.__commandable_methods is None:
            return
        self.__commandable_service = _CommandableMediator()
        self.__commandable_service.invoke_func(self.__invoke_commandable_method)

        self.register_service(self.__commandable_service)

    def __invoke_commandable_method(self, request: Any, context: Any):
        method = request.method
        action = self.__commandable_methods[method] if self.__commandable_methods else None
        correlation_id = request.correlation_id
        response = commandable_pb2.InvokeReply()

        # Handle method not found
        if action is None:
            err = InvocationException(correlation_id, 'METHOD_NOT_FOUND',
                                      'Method ' + method + ' was not found').with_details('method', method)
            resp_err = ErrorDescriptionFactory.create(err)
            response.error = commandable_pb2.ErrorDescription()
            response.error.category = resp_err.category
            response.error.code = resp_err.code
            response.error.correlation_id = resp_err.correlation_id
            response.error.status = resp_err.status
            response.error.message = resp_err.message
            response.error.cause = resp_err.cause
            response.error.stack_trace = resp_err.stack_trace
            response.error.details.addAll(resp_err.details)
            response.resultEmpty = True
            response.resultJson = ''

            return response

        try:
            # Convert arguments
            args_empty = request.args_empty
            args_json = request.args_json
            args = Parameters.from_json(args_json) if not args_empty and args_json else Parameters()

            try:
                result = action(correlation_id, args, method)
                response.result_empty = result is None
                if type(result) is DataPage:
                    response.result_json = json.dumps(result.to_json())
                else:
                    response.result_json = json.dumps(result) if result is not None else ''

                # TODO: Validate schema
                schema = self.__commandable_schemas[method]
                if schema:
                    pass

            except Exception as ex:
                # Process result and generate response
                resp_err = ErrorDescriptionFactory.create(ex)
                response.error.category = resp_err.category
                response.error.code = resp_err.code
                response.error.correlation_id = resp_err.correlation_id
                response.error.status = resp_err.status
                response.error.message = resp_err.message
                response.error.cause = resp_err.cause
                response.error.stack_trace = resp_err.stack_trace
                response.error.details.update(resp_err.details)
                response.result_empty = True
                response.result_json = ''

        except Exception as ex:
            # Handle unexpected exception
            err = InvocationException(correlation_id, 'METHOD_FAILED', 'Method ' + method + ' failed').wrap(
                ex).with_details('method', method)
            resp_err = ErrorDescriptionFactory.create(err)
            response.error.category = resp_err.category
            response.error.code = resp_err.code
            response.error.correlation_id = resp_err.correlation_id
            response.error.status = resp_err.status
            response.error.message = resp_err.message
            response.error.cause = resp_err.cause
            response.error.stack_trace = resp_err.stack_trace
            response.error.details.update(resp_err.details)
            response.result_empty = True
            response.result_json = ''

        return response

    def _register_method(self, name: str, schema: Schema, action: Callable[[Optional[str], Optional[str], Parameters], None]):
        """
        Registers a method in GRPC service.

        :param name: a method name
        :param schema: a validation schema to validate received parameters.
        :param action: an action function that is called when operation is invoked.
        """
        # TODO
        pass

    def register_service(self, service):
        """
        Registers a service with related implementation

        :param service: a GRPC service object.
        """
        service.add_servicer_to_server(self.__server)

    def _register_commandable_method(self, method: str, schema: Schema,
                                     action: Callable[[Optional[str], Optional[str], Parameters], None]):
        """
        Registers a commandable method in this objects GRPC server (service) by the given name.

        :param method: the GRPC method name.
        :param schema: the schema to use for parameter validation.
        :param action: the action to perform at the given route.
        """
        self.__commandable_methods = self.__commandable_methods or {}
        self.__commandable_methods[method] = action

        self.__commandable_schemas = self.__commandable_schemas or {}
        self.__commandable_schemas[method] = schema

    def _register_interceptor(self, interceptor: Callable):
        """
        Registers a middleware for methods in GRPC endpoint.

        :param interceptor: the middleware action to perform at the given route.
        """
        self.__interceptors.append(interceptor)
Ejemplo n.º 2
0
class GrpcClient(IOpenable, IReferenceable, IConfigurable):
    """
    Abstract client that calls remove endpoints using GRPC protocol.

    ### Configuration parameters ###
        - connection(s):
          - discovery_key:         (optional) a key to retrieve the connection from :func:`link`
          - protocol:              connection protocol: http or https
          - host:                  host name or IP address
          - port:                  port number
          - uri:                   resource URI or connection string with all parameters in it
        - options:
          - retries:               number of retries (default: 3)
          - connect_timeout:       connection timeout in milliseconds (default: 10 sec)
          - timeout:               invocation timeout in milliseconds (default: 10 sec)

    .. code-block:: python

        class MyGrpcClient(GrpcClient, IMyClient):
            ...
            def get_data(self, correlation_id, id ):
                timing = self.instrument(correlation_id, 'myclient.get_data')
                result = self.call("get_data", correlation_id, { id: id })
                timing.end_timing()
                return result
            ...

        client = MyGrpcClient()
        client.configure(ConfigParams.from_tuples(
            "connection.protocol", "http",
            "connection.host", "localhost",
            "connection.port", 8080
        ))
        result = client.get_data("123", "1")
    """

    _default_config = ConfigParams.from_tuples(
        "connection.protocol", "http", "connection.host", "0.0.0.0",
        "connection.port", 3000, "options.request_max_size", 1024 * 1024,
        "options.connect_timeout", 10000, "options.timeout", 10000,
        "options.retries", 3, "options.debug", True)

    def __init__(self, client_name: str):
        """
        Creates a new instance of the client.

        :param client_name: a client name.
        """
        self.__client = None
        self.__client_name = None

        # The GRPC client channel
        self._channel = None

        # The connection resolver.
        self._connection_resolver = HttpConnectionResolver()

        # The logger.
        self._logger = CompositeLogger()

        # The performance counters.
        self._counters = CompositeCounters()

        # The configuration options.
        self._options = ConfigParams()

        # The connection timeout in milliseconds.
        self._connection_timeout = 100000

        # The invocation timeout in milliseconds.
        self._timeout = 100000

        # The remote service uri which is calculated on open.
        self._uri: str = None

        self.__client_name = client_name

    def configure(self, config: ConfigParams):
        """
        Configures component by passing configuration parameters.

        :param config: configuration parameters to be set.
        """
        config = config.set_defaults(GrpcClient._default_config)
        self._connection_resolver.configure(config)
        self._options = self._options.override(config.get_section('options'))

        self._connection_timeout = config.get_as_integer_with_default(
            'options.connect_timeout', self._connection_timeout)
        self._timeout = config.get_as_integer_with_default(
            'options.timeout', self._timeout)

    def set_references(self, references: IReferences):
        """
        Sets references to dependent components.

        :param references: references to locate the component dependencies.
        """
        self._logger.set_references(references)
        self._counters.set_references(references)
        self._connection_resolver.set_references(references)

    def _instrument(self, correlation_id: Optional[str],
                    name: str) -> CounterTiming:
        """
        Adds instrumentation to log calls and measure call time.
        It returns a CounterTiming object that is used to end the time measurement.

        :param correlation_id: (optional) transaction id to trace execution through call chain.
        :param name: a method name.
        :return: CounterTiming object to end the time measurement.
        """
        self._logger.trace(correlation_id, 'Executing {} method'.format(name))
        self._counters.increment_one(name + '.call_count')
        return self._counters.begin_timing(name + '.call_time')

    def _instrument_error(self,
                          correlation_id: Optional[str],
                          name: str,
                          err: Exception,
                          reerror=False):
        """
        Adds instrumentation to error handling.

        :param correlation_id: (optional) transaction id to trace execution through call chain.
        :param name: a method name.
        :param err: an occured error
        :param reerror: if true - throw error
        """
        if err is not None:
            self._logger.error(correlation_id, err,
                               'Failed to call {} method'.format(name))
            self._counters.increment_one(name + '.call_errors')
            if reerror is not None and reerror is True:
                raise err

    def is_open(self) -> bool:
        """
        Checks if the component is opened.

        :return: Returns true if the component has been opened and false otherwise.
        """
        return self._channel is not None

    def open(self, correlation_id: Optional[str]):
        """
        Opens the component.

        :param correlation_id: (optional) transaction id to trace execution through call chain.
        """
        if self.is_open():
            return None

        try:
            connection = self._connection_resolver.resolve(correlation_id)
            self._uri = connection.get_as_string('uri')

            options = [('grpc.max_connection_idle_ms',
                        self._connection_timeout),
                       ('grpc.client_idle_timeout_ms', self._timeout)]

            if connection.get_as_string_with_default('protocol',
                                                     'http') == 'https':
                ssl_ca_file = connection.get_as_nullable_string('ssl_ca_file')
                with open(ssl_ca_file, 'rb') as file:
                    trusted_root = file.read()
                credentials = grpc.ssl_channel_credentials(trusted_root)
                channel = grpc.secure_channel(
                    str(connection.get_as_string('host')) + ':' +
                    str(connection.get_as_string('port')),
                    credentials=credentials,
                    options=options)
            else:
                channel = grpc.insecure_channel(
                    str(connection.get_as_string('host')) + ':' +
                    str(connection.get_as_string('port')),
                    options=options)
            self._channel = channel

        except Exception as ex:
            raise ConnectionException(
                correlation_id, 'CANNOT_CONNECT',
                'Opening GRPC client failed').wrap(ex).with_details(
                    'url', self._uri)

    def close(self, correlation_id: Optional[str]):
        """
        Closes component and frees used resources.

        :param correlation_id: (optional) transaction id to trace execution through call chain.
        """
        if self._channel is not None:
            # Eat exceptions
            try:
                self._logger.debug(
                    correlation_id,
                    'Closed GRPC service at {}'.format(self._uri))
            except Exception as ex:
                self._logger.warn(
                    correlation_id,
                    'Failed while closing GRPC service: {}'.format(ex))
            # if self.__client is not None:
            #     self.__client = None
            self._channel.close()
            self._channel = None
            self._uri = None
            GrpcClient._connection_resolver = HttpConnectionResolver()

    def call(self, method: str, client: Any, request: Any) -> Future:
        """
        Calls a remote method via GRPC protocol.

        :param method: name of the calling method
        :param client: current client
        :param request: (optional) request object.
        :return: (optional) future that receives result object or error.
        """

        client = client(self._channel)
        executor = futures.ThreadPoolExecutor(max_workers=1)
        response = executor.submit(client.__dict__[method], request)

        return response

    def _add_filter_params(self, params: Any, filter: Any) -> Any:
        """
        AddFilterParams method are adds filter parameters (with the same name as they defined)
        to invocation parameter map.

        :param params: invocation parameters.
        :param filter: (optional) filter parameters
        :return: invocation parameters with added filter parameters.
        """
        params = StringValueMap() if params is None else params

        if filter is not None:
            for k in filter.keys():
                params.put(k, filter[k])

        return params

    def _add_paging_params(self, params: Any, paging: Any) -> Any:
        """
        AddPagingParams method are adds paging parameters (skip, take, total) to invocation parameter map.

        :param params: invocation parameters.
        :param paging: (optional) paging parameters
        :return: invocation parameters with added paging parameters.
        """
        params = StringValueMap() if params is None else params

        if paging is not None:
            params.put('total', paging.total)
            if paging.skip is not None:
                params.put('skip', paging.skip)
            if paging.take is not None:
                params.put('take', paging.take)

        return params