def _init(self): """ Actually initializes the service. """ self.odb = self.worker_store.server.odb self.kvdb = self.worker_store.kvdb self.time.kvdb = self.kvdb self.pubsub = self.worker_store.pubsub self.slow_threshold = self.server.service_store.services[ self.impl_name]['slow_threshold'] # Queues out_amqp = PublisherFacade(self.broker_client) out_jms_wmq = WMQFacade(self.broker_client) out_zmq = ZMQFacade(self.server) # Patterns self.patterns = PatternsFacade(self) # SQL out_sql = self.worker_store.sql_pool_store # Regular outconns out_ftp, out_odoo, out_plain_http, out_soap = self.worker_store.worker_config.outgoing_connections( ) self.outgoing = Outgoing(out_amqp, out_ftp, out_jms_wmq, out_odoo, out_plain_http, out_soap, out_sql, self.worker_store.stomp_outconn_api, out_zmq) # Cloud self.cloud = Cloud() self.cloud.openstack.swift = self.worker_store.worker_config.cloud_openstack_swift self.cloud.aws.s3 = self.worker_store.worker_config.cloud_aws_s3 # Cassandra self.cassandra_conn = self.worker_store.cassandra_api self.cassandra_query = self.worker_store.cassandra_query_api # E-mail self.email = EMailAPI(self.worker_store.email_smtp_api, self.worker_store.email_imap_api) # Search self.search = SearchAPI(self.worker_store.search_es_api, self.worker_store.search_solr_api) is_sio = hasattr(self, 'SimpleIO') self.request.http.init(self.wsgi_environ) if is_sio: self.request.init(is_sio, self.cid, self.SimpleIO, self.data_format, self.transport, self.wsgi_environ) self.response.init(self.cid, self.SimpleIO, self.data_format) self.msg = MessageFacade(self.worker_store.msg_ns_store, self.worker_store.json_pointer_store, self.worker_store.xpath_store, self.worker_store.msg_ns_store, self.request.payload, self.time)
class Service(object): """ A base class for all services deployed on Zato servers, no matter the transport and protocol, be it plain HTTP, SOAP, IBM MQ or any other, regardless whether they're built-in or user-defined ones. """ _filter_by = None _enforce_service_invokes = None invokes = [] http_method_handlers = {} # Class-wide attributes shared by all services thus created here instead of assigning to self. cloud = Cloud() odb = None kvdb = None pubsub = None # type: PubSub cassandra_conn = None cassandra_query = None email = None search = None amqp = AMQPFacade() _worker_store = None _worker_config = None _msg_ns_store = None _ns_store = None _json_pointer_store = None _xpath_store = None _out_ftp = None _out_plain_http = None _req_resp_freq = 0 _has_before_job_hooks = None _has_after_job_hooks = None _before_job_hooks = [] _after_job_hooks = [] # User management and SSO sso = None # Crypto operations crypto = None # Audit log audit_pii = None # For invoking other servers directly servers = None def __init__(self, _get_logger=logging.getLogger, _Bunch=Bunch, _Request=Request, _Response=Response, _DictNav=DictNav, _ListNav=ListNav, _Outgoing=Outgoing, _WMQFacade=WMQFacade, _ZMQFacade=ZMQFacade, *ignored_args, **ignored_kwargs): self.name = self.__class__.__service_name # Will be set through .get_name by Service Store self.impl_name = self.__class__.__service_impl_name # Ditto self.logger = _get_logger(self.name) self.server = None self.broker_client = None self.channel = None self.cid = None self.in_reply_to = None self.data_format = None self.transport = None self.wsgi_environ = None self.job_type = None self.environ = _Bunch() self.request = _Request(self.logger) self.response = _Response(self.logger) self.invocation_time = None # When was the service invoked self.handle_return_time = None # When did its 'handle' method finished processing the request self.processing_time_raw = None # A timedelta object with the processing time up to microseconds self.processing_time = None # Processing time in milliseconds self.usage = 0 # How many times the service has been invoked self.slow_threshold = maxint # After how many ms to consider the response came too late self.msg = None self.time = None self.patterns = None self.user_config = None self.dictnav = _DictNav self.listnav = _ListNav self.has_validate_input = False self.has_validate_output = False self.cache = None self.out = self.outgoing = _Outgoing( self.amqp, self._out_ftp, _WMQFacade(self) if self.component_enabled_ibm_mq else None, self._worker_config.out_odoo, self._out_plain_http, self._worker_config.out_soap, self._worker_store.sql_pool_store, self._worker_store.stomp_outconn_api, ZMQFacade(self.server) if self.component_enabled_zeromq else None, self._worker_store.outgoing_web_sockets, self._worker_store.vault_conn_api, SMSAPI(self._worker_store.sms_twilio_api) if self.component_enabled_sms else None, ) @staticmethod def get_name_static(class_): return Service.get_name(class_) @classmethod def get_name(class_): """ Returns a service's name, settings its .name attribute along. This will be called once while the service is being deployed. """ if not hasattr(class_, '__service_name'): name = getattr(class_, 'name', None) if not name: name = service_name_from_impl(class_.get_impl_name()) name = class_.convert_impl_name(name) class_.__service_name = name return class_.__service_name @classmethod def get_impl_name(class_): if not hasattr(class_, '__service_impl_name'): class_.__service_impl_name = '{}.{}'.format( class_.__module__, class_.__name__) return class_.__service_impl_name @staticmethod def convert_impl_name(name): # TODO: Move the replace functionality over to uncamelify, possibly modifying its regexp split = uncamelify(name).split('.') path, class_name = split[:-1], split[-1] path = [elem.replace('_', '-') for elem in path] class_name = class_name[1:] if class_name.startswith( '-') else class_name class_name = class_name.replace('.-', '.').replace('_-', '_') return '{}.{}'.format('.'.join(path), class_name) @classmethod def add_http_method_handlers(class_): for name in dir(class_): if name.startswith('handle_'): if not getattr(class_, 'http_method_handlers', False): setattr(class_, 'http_method_handlers', {}) method = name.replace('handle_', '') class_.http_method_handlers[method] = getattr(class_, name) def _init(self, is_http=False): """ Actually initializes the service. """ self.slow_threshold = self.server.service_store.services[ self.impl_name]['slow_threshold'] # The if's below are meant to be written in this way because we don't want any unnecessary attribute lookups # and method calls in this method - it's invoked each time a service is executed. The attributes are set # for the whole of the Service class each time it is discovered they are needed. It cannot be done in ServiceStore # because at the time that ServiceStore executes the worker config may still not be ready. if self.component_enabled_cassandra: if not Service.cassandra_conn: Service.cassandra_conn = self._worker_store.cassandra_api if not Service.cassandra_query: Service.cassandra_query = self._worker_store.cassandra_query_api if self.component_enabled_email: if not Service.email: Service.email = EMailAPI(self._worker_store.email_smtp_api, self._worker_store.email_imap_api) if self.component_enabled_search: if not Service.search: Service.search = SearchAPI(self._worker_store.search_es_api, self._worker_store.search_solr_api) if self.component_enabled_msg_path: self.msg = MessageFacade(self._json_pointer_store, self._xpath_store, self._msg_ns_store, self.request.payload, self.time) if self.component_enabled_patterns: self.patterns = PatternsFacade(self) if is_http: self.request.http.init(self.wsgi_environ) # self.is_sio attribute is set by ServiceStore during deployment if self.has_sio: self.request.init(True, self.cid, self.SimpleIO, self.data_format, self.transport, self.wsgi_environ, self.server.encrypt) self.response.init(self.cid, self.SimpleIO, self.data_format) # Cache is always enabled self.cache = self._worker_store.cache_api def set_response_data(self, service, _raw_types=(basestring, dict, list, tuple, EtreeElement, ObjectifiedElement), **kwargs): response = service.response.payload if not isinstance(response, _raw_types): response = response.getvalue(serialize=kwargs['serialize']) if kwargs['as_bunch']: response = bunchify(response) service.response.payload = response return response def _invoke(self, service, channel, http_channels=(CHANNEL.HTTP_SOAP, CHANNEL.INVOKE)): # # If channel is HTTP and there are any per-HTTP verb methods, it means we want for the service to be a REST target. # Let's say it is POST. If we have handle_POST, it is invoked. If there is no handle_POST, # '405 Method Not Allowed is returned'. # # However, if we have 'handle' only, it means this is always invoked and no default 405 is returned. # # In short, implement handle_* if you want REST behaviour. Otherwise, keep everything in handle. # # Ok, this is HTTP if channel in http_channels: # We have at least one per-HTTP verb handler if service.http_method_handlers: # But do we have any handler matching current request's verb? if service.request.http.method in service.http_method_handlers: # Yes, call the handler service.http_method_handlers[service.request.http.method]( service) # No, return 405 else: service.response.status_code = METHOD_NOT_ALLOWED # We have no custom handlers so we always call 'handle' else: service.handle() # It's not HTTP so we simply call 'handle' else: service.handle() def extract_target(self, name): """ Splits a service's name into name and target, if the latter is provided on input at all. """ # It can be either a name or a name followed by the target to invoke the service on, # i.e. 'myservice' or 'myservice@mytarget'. if '@' in name: name, target = name.split('@') if not target: raise ZatoException( self.cid, 'Target must not be empty in `{}`'.format(name)) else: target = '' return name, target def update_handle(self, set_response_func, service, raw_request, channel, data_format, transport, server, broker_client, worker_store, cid, simple_io_config, _utcnow=datetime.utcnow, _call_hook_with_service=call_hook_with_service, _call_hook_no_service=call_hook_no_service, _CHANNEL_SCHEDULER=CHANNEL.SCHEDULER, _pattern_channels=(CHANNEL.FANOUT_CALL, CHANNEL.PARALLEL_EXEC_CALL), *args, **kwargs): wsgi_environ = kwargs.get('wsgi_environ', {}) payload = wsgi_environ.get('zato.request.payload') # Here's an edge case. If a SOAP request has a single child in Body and this child is an empty element # (though possibly with attributes), checking for 'not payload' alone won't suffice - this evaluates # to False so we'd be parsing the payload again superfluously. if not isinstance(payload, ObjectifiedElement) and not payload: payload = payload_from_request(cid, raw_request, data_format, transport) job_type = kwargs.get('job_type') channel_params = kwargs.get('channel_params', {}) merge_channel_params = kwargs.get('merge_channel_params', True) params_priority = kwargs.get('params_priority', PARAMS_PRIORITY.DEFAULT) service.update(service, channel, server, broker_client, worker_store, cid, payload, raw_request, transport, simple_io_config, data_format, wsgi_environ, job_type=job_type, channel_params=channel_params, merge_channel_params=merge_channel_params, params_priority=params_priority, in_reply_to=wsgi_environ.get( 'zato.request_ctx.in_reply_to', None), environ=kwargs.get('environ'), wmq_ctx=kwargs.get('wmq_ctx')) # It's possible the call will be completely filtered out. The uncommonly looking not self.accept shortcuts # if ServiceStore replaces self.accept with None in the most common case of this method's not being # implemented by user services. if (not self.accept) or service.accept(): # Assume everything goes fine e, exc_formatted = None, None try: if service.server.component_enabled.stats: service.usage = service.kvdb.conn.incr('{}{}'.format( KVDB.SERVICE_USAGE, service.name)) service.invocation_time = _utcnow() # All hooks are optional so we check if they have not been replaced with None by ServiceStore. # Call before job hooks if any are defined and we are called from the scheduler if service._has_before_job_hooks and self.channel.type == _CHANNEL_SCHEDULER: for elem in service._before_job_hooks: if elem: _call_hook_with_service(elem, service) # Called before .handle - catches exceptions if service.before_handle: _call_hook_no_service(service.before_handle) # Called before .handle - does not catch exceptions if service.validate_input: service.validate_input() # This is the place where the service is invoked self._invoke(service, channel) # Called after .handle - does not catch exceptions if service.validate_output: service.validate_output() # Called after .handle - catches exceptions if service.after_handle: _call_hook_no_service(service.after_handle) # Call after job hooks if any are defined and we are called from the scheduler if service._has_after_job_hooks and self.channel.type == _CHANNEL_SCHEDULER: for elem in service._after_job_hooks: if elem: _call_hook_with_service(elem, service) # Internal method - always defined and called service.post_handle() # Optional, almost never overridden. if service.finalize_handle: _call_hook_no_service(service.finalize_handle) except Exception, e: exc_formatted = format_exc(e) logger.warn(exc_formatted) finally:
class Service(object): """ A base class for all services deployed on Zato servers, no matter the transport and protocol, be it plain HTTP, SOAP, IBM MQ or any other, regardless whether they're built-in or user-defined ones. """ _filter_by = None _enforce_service_invokes = None invokes = [] http_method_handlers = {} # Class-wide attributes shared by all services thus created here instead of assigning to self. cloud = Cloud() odb = None kvdb = None pubsub = None # type: PubSub cassandra_conn = None cassandra_query = None email = None search = None amqp = AMQPFacade() _worker_store = None _worker_config = None _msg_ns_store = None _ns_store = None _json_pointer_store = None _xpath_store = None _out_ftp = None _out_plain_http = None _req_resp_freq = 0 _has_before_job_hooks = None _has_after_job_hooks = None _before_job_hooks = [] _after_job_hooks = [] # User management and SSO sso = None # Crypto operations crypto = None # Audit log audit_pii = None # For invoking other servers directly servers = None def __init__(self, _get_logger=logging.getLogger, _Bunch=Bunch, _Request=Request, _Response=Response, _DictNav=DictNav, _ListNav=ListNav, _Outgoing=Outgoing, _WMQFacade=WMQFacade, _ZMQFacade=ZMQFacade, *ignored_args, **ignored_kwargs): self.name = self.__class__.__service_name # Will be set through .get_name by Service Store self.impl_name = self.__class__.__service_impl_name # Ditto self.logger = _get_logger(self.name) self.server = None # type: ParallelServer self.broker_client = None self.channel = None self.cid = None self.in_reply_to = None self.data_format = None self.transport = None self.wsgi_environ = None self.job_type = None self.environ = _Bunch() self.request = _Request(self.logger) self.response = _Response(self.logger) self.invocation_time = None # When was the service invoked self.handle_return_time = None # When did its 'handle' method finished processing the request self.processing_time_raw = None # A timedelta object with the processing time up to microseconds self.processing_time = None # Processing time in milliseconds self.usage = 0 # How many times the service has been invoked self.slow_threshold = maxint # After how many ms to consider the response came too late self.msg = None self.time = None self.patterns = None self.user_config = None self.dictnav = _DictNav self.listnav = _ListNav self.has_validate_input = False self.has_validate_output = False self.cache = None self.out = self.outgoing = _Outgoing( self.amqp, self._out_ftp, _WMQFacade(self) if self.component_enabled_ibm_mq else None, self._worker_config.out_odoo, self._out_plain_http, self._worker_config.out_soap, self._worker_store.sql_pool_store, self._worker_store.stomp_outconn_api, ZMQFacade(self._worker_store.zmq_out_api) if self.component_enabled_zeromq else NO_DEFAULT_VALUE, self._worker_store.outconn_wsx, self._worker_store.vault_conn_api, SMSAPI(self._worker_store.sms_twilio_api) if self.component_enabled_sms else None, self._worker_config.out_sap, self._worker_config.out_sftp, self._worker_store.outconn_ldap, ) @staticmethod def get_name_static(class_): return Service.get_name(class_) @classmethod def get_name(class_): """ Returns a service's name, settings its .name attribute along. This will be called once while the service is being deployed. """ if not hasattr(class_, '__service_name'): name = getattr(class_, 'name', None) if not name: name = service_name_from_impl(class_.get_impl_name()) name = class_.convert_impl_name(name) class_.__service_name = name return class_.__service_name @classmethod def get_impl_name(class_): if not hasattr(class_, '__service_impl_name'): class_.__service_impl_name = '{}.{}'.format( class_.__module__, class_.__name__) return class_.__service_impl_name @staticmethod def convert_impl_name(name): # TODO: Move the replace functionality over to uncamelify, possibly modifying its regexp split = uncamelify(name).split('.') path, class_name = split[:-1], split[-1] path = [elem.replace('_', '-') for elem in path] class_name = class_name[1:] if class_name.startswith( '-') else class_name class_name = class_name.replace('.-', '.').replace('_-', '_') return '{}.{}'.format('.'.join(path), class_name) @classmethod def add_http_method_handlers(class_): for name in dir(class_): if name.startswith('handle_'): if not getattr(class_, 'http_method_handlers', False): setattr(class_, 'http_method_handlers', {}) method = name.replace('handle_', '') class_.http_method_handlers[method] = getattr(class_, name) def _init(self, may_have_wsgi_environ=False): """ Actually initializes the service. """ self.slow_threshold = self.server.service_store.services[ self.impl_name]['slow_threshold'] # The if's below are meant to be written in this way because we don't want any unnecessary attribute lookups # and method calls in this method - it's invoked each time a service is executed. The attributes are set # for the whole of the Service class each time it is discovered they are needed. It cannot be done in ServiceStore # because at the time that ServiceStore executes the worker config may still not be ready. if self.component_enabled_cassandra: if not Service.cassandra_conn: Service.cassandra_conn = self._worker_store.cassandra_api if not Service.cassandra_query: Service.cassandra_query = self._worker_store.cassandra_query_api if self.component_enabled_email: if not Service.email: Service.email = EMailAPI(self._worker_store.email_smtp_api, self._worker_store.email_imap_api) if self.component_enabled_search: if not Service.search: Service.search = SearchAPI(self._worker_store.search_es_api, self._worker_store.search_solr_api) if self.component_enabled_msg_path: self.msg = MessageFacade(self._json_pointer_store, self._xpath_store, self._msg_ns_store, self.request.payload, self.time) if self.component_enabled_patterns: self.patterns = PatternsFacade(self) if may_have_wsgi_environ: self.request.http.init(self.wsgi_environ) # self.is_sio attribute is set by ServiceStore during deployment if self.has_sio: self.request.init(True, self.cid, self.SimpleIO, self.data_format, self.transport, self.wsgi_environ, self.server.encrypt) self.response.init(self.cid, self.SimpleIO, self.data_format) # Cache is always enabled self.cache = self._worker_store.cache_api def set_response_data(self, service, _raw_types=(basestring, dict, list, tuple, EtreeElement, ObjectifiedElement), **kwargs): response = service.response.payload if not isinstance(response, _raw_types): response = response.getvalue(serialize=kwargs['serialize']) if kwargs['as_bunch']: response = bunchify(response) service.response.payload = response return response def _invoke(self, service, channel, http_channels=(CHANNEL.HTTP_SOAP, CHANNEL.INVOKE)): # # If channel is HTTP and there are any per-HTTP verb methods, it means we want for the service to be a REST target. # Let's say it is POST. If we have handle_POST, it is invoked. If there is no handle_POST, # '405 Method Not Allowed is returned'. # # However, if we have 'handle' only, it means this is always invoked and no default 405 is returned. # # In short, implement handle_* if you want REST behaviour. Otherwise, keep everything in handle. # # Ok, this is HTTP if channel in http_channels: # We have at least one per-HTTP verb handler if service.http_method_handlers: # But do we have any handler matching current request's verb? if service.request.http.method in service.http_method_handlers: # Yes, call the handler service.http_method_handlers[service.request.http.method]( service) # No, return 405 else: service.response.status_code = METHOD_NOT_ALLOWED # We have no custom handlers so we always call 'handle' else: service.handle() # It's not HTTP so we simply call 'handle' else: service.handle() def extract_target(self, name): """ Splits a service's name into name and target, if the latter is provided on input at all. """ # It can be either a name or a name followed by the target to invoke the service on, # i.e. 'myservice' or 'myservice@mytarget'. if '@' in name: name, target = name.split('@') if not target: raise ZatoException( self.cid, 'Target must not be empty in `{}`'.format(name)) else: target = '' return name, target def update_handle(self, set_response_func, service, raw_request, channel, data_format, transport, server, broker_client, worker_store, cid, simple_io_config, _utcnow=datetime.utcnow, _call_hook_with_service=call_hook_with_service, _call_hook_no_service=call_hook_no_service, _CHANNEL_SCHEDULER=CHANNEL.SCHEDULER, _pattern_channels=(CHANNEL.FANOUT_CALL, CHANNEL.PARALLEL_EXEC_CALL), *args, **kwargs): wsgi_environ = kwargs.get('wsgi_environ', {}) payload = wsgi_environ.get('zato.request.payload') # Here's an edge case. If a SOAP request has a single child in Body and this child is an empty element # (though possibly with attributes), checking for 'not payload' alone won't suffice - this evaluates # to False so we'd be parsing the payload again superfluously. if not isinstance(payload, ObjectifiedElement) and not payload: payload = payload_from_request(cid, raw_request, data_format, transport) job_type = kwargs.get('job_type') channel_params = kwargs.get('channel_params', {}) merge_channel_params = kwargs.get('merge_channel_params', True) params_priority = kwargs.get('params_priority', PARAMS_PRIORITY.DEFAULT) service.update(service, channel, server, broker_client, worker_store, cid, payload, raw_request, transport, simple_io_config, data_format, wsgi_environ, job_type=job_type, channel_params=channel_params, merge_channel_params=merge_channel_params, params_priority=params_priority, in_reply_to=wsgi_environ.get( 'zato.request_ctx.in_reply_to', None), environ=kwargs.get('environ'), wmq_ctx=kwargs.get('wmq_ctx')) # It's possible the call will be completely filtered out. The uncommonly looking not self.accept shortcuts # if ServiceStore replaces self.accept with None in the most common case of this method's not being # implemented by user services. if (not self.accept) or service.accept(): # Assumes it goes fine by default e, exc_formatted = None, None try: if service.server.component_enabled.stats: service.usage = service.kvdb.conn.incr('{}{}'.format( KVDB.SERVICE_USAGE, service.name)) service.invocation_time = _utcnow() # All hooks are optional so we check if they have not been replaced with None by ServiceStore. # Call before job hooks if any are defined and we are called from the scheduler if service._has_before_job_hooks and self.channel.type == _CHANNEL_SCHEDULER: for elem in service._before_job_hooks: if elem: _call_hook_with_service(elem, service) # Called before .handle - catches exceptions if service.before_handle: _call_hook_no_service(service.before_handle) # Called before .handle - does not catch exceptions if service.validate_input: service.validate_input() # This is the place where the service is invoked self._invoke(service, channel) # Called after .handle - does not catch exceptions if service.validate_output: service.validate_output() # Called after .handle - catches exceptions if service.after_handle: _call_hook_no_service(service.after_handle) # Call after job hooks if any are defined and we are called from the scheduler if service._has_after_job_hooks and self.channel.type == _CHANNEL_SCHEDULER: for elem in service._after_job_hooks: if elem: _call_hook_with_service(elem, service) # Internal method - always defined and called service.post_handle() # Optional, almost never overridden. if service.finalize_handle: _call_hook_no_service(service.finalize_handle) except Exception as ex: e = ex exc_formatted = format_exc() logger.warn(exc_formatted) finally: try: response = set_response_func(service, data_format=data_format, transport=transport, **kwargs) # If this was fan-out/fan-in we need to always notify our callbacks no matter the result if channel in _pattern_channels: func = self.patterns.fanout.on_call_finished if channel == CHANNEL.FANOUT_CALL else \ self.patterns.parallel.on_call_finished spawn(func, self, service.response.payload, exc_formatted) except Exception as resp_e: # If we already have an exception around, log the new one but don't overwrite the old one with it. logger.warn('Exception in service `%s`, e:`%s`', service.name, format_exc()) if e: if isinstance(e, Reportable): raise e else: raise Exception(exc_formatted) raise resp_e else: if e: raise e if isinstance(e, Exception) else Exception(e) # We don't accept it but some response needs to be returned anyway. else: response = service.response response.payload = '' response.status_code = BAD_REQUEST return response def invoke_by_impl_name(self, impl_name, payload='', channel=CHANNEL.INVOKE, data_format=DATA_FORMAT.DICT, transport=None, serialize=False, as_bunch=False, timeout=None, raise_timeout=True, **kwargs): """ Invokes a service synchronously by its implementation name (full dotted Python name). """ if self.component_enabled_target_matcher: orig_impl_name = impl_name impl_name, target = self.extract_target(impl_name) # It's possible we are being invoked through self.invoke or self.invoke_by_id target = target or kwargs.get('target', '') if not self._worker_store.target_matcher.is_allowed(target): raise ZatoException( self.cid, 'Invocation target `{}` not allowed ({})'.format( target, orig_impl_name)) if self.component_enabled_invoke_matcher: if not self._worker_store.invoke_matcher.is_allowed(impl_name): raise ZatoException( self.cid, 'Service `{}` (impl_name) cannot be invoked'.format( impl_name)) if self.impl_name == impl_name: msg = 'A service cannot invoke itself, name:[{}]'.format(self.name) self.logger.error(msg) raise ZatoException(self.cid, msg) service, is_active = self.server.service_store.new_instance(impl_name) if not is_active: raise Inactive(service.get_name()) set_response_func = kwargs.pop('set_response_func', service.set_response_data) invoke_args = (set_response_func, service, payload, channel, data_format, transport, self.server, self.broker_client, self._worker_store, kwargs.pop('cid', self.cid), self.request.simple_io_config) kwargs.update({'serialize': serialize, 'as_bunch': as_bunch}) try: if timeout: try: g = spawn(self.update_handle, *invoke_args, **kwargs) return g.get(block=True, timeout=timeout) except Timeout: g.kill() logger.warn('Service `%s` timed out (%s)', service.name, self.cid) if raise_timeout: raise else: return self.update_handle(*invoke_args, **kwargs) except Exception: logger.warn('Could not invoke `%s`, e:`%s`', service.name, format_exc()) raise def invoke(self, name, *args, **kwargs): """ Invokes a service synchronously by its name. """ if self.component_enabled_target_matcher: name, target = self.extract_target(name) kwargs['target'] = target if self._enforce_service_invokes and self.invokes: if name not in self.invokes: msg = 'Could not invoke `{}` which is not in `{}`'.format( name, self.invokes) self.logger.warn(msg) raise ValueError(msg) return self.invoke_by_impl_name( self.server.service_store.name_to_impl_name[name], *args, **kwargs) def invoke_by_id(self, service_id, *args, **kwargs): """ Invokes a service synchronously by its ID. """ if self.component_enabled_target_matcher: service_id, target = self.extract_target(service_id) kwargs['target'] = target return self.invoke_by_impl_name( self.server.service_store.id_to_impl_name[service_id], *args, **kwargs) def invoke_async(self, name, payload='', channel=CHANNEL.INVOKE_ASYNC, data_format=DATA_FORMAT.DICT, transport=None, expiration=BROKER.DEFAULT_EXPIRATION, to_json_string=False, cid=None, callback=None, zato_ctx={}, environ={}): """ Invokes a service asynchronously by its name. """ if self.component_enabled_target_matcher: name, target = self.extract_target(name) zato_ctx['zato.request_ctx.target'] = target else: target = None # Let's first find out if the service can be invoked at all impl_name = self.server.service_store.name_to_impl_name[name] if self.component_enabled_invoke_matcher: if not self._worker_store.invoke_matcher.is_allowed(impl_name): raise ZatoException( self.cid, 'Service `{}` (impl_name) cannot be invoked'.format( impl_name)) if to_json_string: payload = dumps(payload) cid = cid or new_cid() # If there is any callback at all, we need to figure out its name because that's how it will be invoked by. if callback: # The same service if callback is self: callback = self.name else: sink = '{}-async-callback'.format(self.name) if sink in self.server.service_store.name_to_impl_name: callback = sink # Otherwise the callback must be a string pointing to the actual service to reply to so we don't need to do anything. msg = {} msg['action'] = SERVICE.PUBLISH.value msg['service'] = name msg['payload'] = payload msg['cid'] = cid msg['channel'] = channel msg['data_format'] = data_format msg['transport'] = transport msg['is_async'] = True msg['callback'] = callback msg['zato_ctx'] = zato_ctx msg['environ'] = environ # If we have a target we need to invoke all the servers # and these which are not able to handle the target will drop the message. (self.broker_client.publish if target else self.broker_client.invoke_async)(msg, expiration=expiration) return cid def post_handle( self, _get_response_value=get_response_value, _utcnow=datetime.utcnow, _service_time_basic=KVDB.SERVICE_TIME_BASIC, _service_time_raw=KVDB.SERVICE_TIME_RAW, _service_time_raw_by_minute=KVDB.SERVICE_TIME_RAW_BY_MINUTE): """ An internal method executed after the service has completed and has a response ready to return. Updates its statistics and, optionally, stores a sample request/response pair. """ # # Statistics # self.handle_return_time = _utcnow() self.processing_time_raw = self.handle_return_time - self.invocation_time if self.server.component_enabled.stats: proc_time = self.processing_time_raw.total_seconds() * 1000.0 proc_time = proc_time if proc_time > 1 else 0 self.processing_time = int(round(proc_time)) with self.kvdb.conn.pipeline() as pipe: pipe.hset('%s%s' % (_service_time_basic, self.name), 'last', self.processing_time) pipe.rpush('%s%s' % (_service_time_raw, self.name), self.processing_time) key = '%s%s:%s' % ( _service_time_raw_by_minute, self.name, self.handle_return_time.strftime('%Y:%m:%d:%H:%M')) pipe.rpush(key, self.processing_time) # .. we'll have 5 minutes (5 * 60 seconds = 300 seconds) # to aggregate processing times for a given minute and then it will expire # Note that we need Redis 2.1.3+ otherwise the key has just been overwritten pipe.expire(key, 300) pipe.execute() # # Sample requests/responses # slow_response_enabled = self.server.component_enabled.slow_response needs_usage = self._req_resp_freq and self.usage % self._req_resp_freq == 0 if slow_response_enabled or needs_usage: raw_request = self.request.raw_request if not raw_request: req = '' else: req = raw_request if isinstance( raw_request, basestring) else repr(raw_request) if needs_usage: data = { 'cid': self.cid, 'req_ts': self.invocation_time.isoformat(), 'resp_ts': self.handle_return_time.isoformat(), 'req': req, 'resp': _get_response_value( self.response ), # TODO: Don't parse it here and a moment later below } self.kvdb.conn.hmset(key, data) # # Slow responses # if slow_response_enabled and self.slow_threshold: if self.processing_time > self.slow_threshold: raw_request = self.request.raw_request if not raw_request: req = '' else: req = raw_request if isinstance( raw_request, basestring) else repr(raw_request) data = { 'cid': self.cid, 'proc_time': self.processing_time, 'slow_threshold': self.slow_threshold, 'req_ts': self.invocation_time.isoformat(), 'resp_ts': self.handle_return_time.isoformat(), 'req': req, 'resp': _get_response_value( self.response ), # TODO: Don't parse it here and a moment earlier above } slow_response.store(self.kvdb, self.name, **data) def translate(self, *args, **kwargs): raise NotImplementedError('An initializer should override this method') def handle(self): """ The only method Zato services need to implement in order to process incoming requests. """ raise NotImplementedError( 'Should be overridden by subclasses (Service.handle)') def lock(self, name=None, *args, **kwargs): #ttl=20, block=10): """ Creates a distributed lock. name - defaults to self.name effectively making access to this service serialized ttl - defaults to 20 seconds and is the max time the lock will be held block - how long (in seconds) we will wait to acquire the lock before giving up """ # The relevant part of signature in 2.0 was `expires=20, timeout=10` # and the 3.0 -> 2.0 mapping is: ttl->expires, block=timeout if not args: ttl = kwargs.get('ttl') or kwargs.get('expires') or 20 block = kwargs.get('block') or kwargs.get('timeout') or 10 else: if len(args) == 1: ttl = args[0] block = 10 else: ttl = args[0] block = args[1] return self.server.zato_lock_manager(name or self.name, ttl=ttl, block=block) # ################################################################################################################################ def accept(self, _zato_no_op_marker=zato_no_op_marker): return True # ################################################################################################################################ @classmethod def before_add_to_store(cls, logger): """ Invoked right before the class is added to the service store. """ return True def before_job(self, _zato_no_op_marker=zato_no_op_marker): """ Invoked if the service has been defined as a job's invocation target, regardless of the job's type. """ def before_one_time_job(self, _zato_no_op_marker=zato_no_op_marker): """ Invoked if the service has been defined as a one-time job's invocation target. """ def before_interval_based_job(self, _zato_no_op_marker=zato_no_op_marker): """ Invoked if the service has been defined as an interval-based job's invocation target. """ def before_cron_style_job(self, _zato_no_op_marker=zato_no_op_marker): """ Invoked if the service has been defined as a cron-style job's invocation target. """ def before_handle(self, _zato_no_op_marker=zato_no_op_marker, *args, **kwargs): """ Invoked just before the actual service receives the request data. """ def after_job(self, _zato_no_op_marker=zato_no_op_marker): """ Invoked if the service has been defined as a job's invocation target, regardless of the job's type. """ def after_one_time_job(self, _zato_no_op_marker=zato_no_op_marker): """ Invoked if the service has been defined as a one-time job's invocation target. """ def after_interval_based_job(self, _zato_no_op_marker=zato_no_op_marker): """ Invoked if the service has been defined as an interval-based job's invocation target. """ def after_cron_style_job(self, _zato_no_op_marker=zato_no_op_marker): """ Invoked if the service has been defined as a cron-style job's invocation target. """ def after_handle(self, _zato_no_op_marker=zato_no_op_marker): """ Invoked right after the actual service has been invoked, regardless of whether the service raised an exception or not. """ def finalize_handle(self, _zato_no_op_marker=zato_no_op_marker): """ Offers the last chance to influence the service's operations. """ @staticmethod def after_add_to_store(logger): """ Invoked right after the class has been added to the service store. """ def validate_input(self, _zato_no_op_marker=zato_no_op_marker): """ Invoked right before handle. Any exception raised means handle will not be called. """ def validate_output(self, _zato_no_op_marker=zato_no_op_marker): """ Invoked right after handle. Any exception raised means further hooks will not be called. """ def get_request_hash(self, _zato_no_op_marker=zato_no_op_marker, *args, **kwargs): """ Lets services compute an incoming request's hash to decide whether i is already kept in cache, if one is configured for this request's channel. """ # ################################################################################################################################ def _log_input_output(self, user_msg, level, suppress_keys, is_response): suppress_keys = suppress_keys or [] suppressed_msg = '(suppressed)' container = 'response' if is_response else 'request' payload_key = '{}.payload'.format(container) user_msg = '{} '.format(user_msg) if user_msg else user_msg msg = {} if payload_key not in suppress_keys: msg[payload_key] = getattr(self, container).payload else: msg[payload_key] = suppressed_msg attrs = ('channel', 'cid', 'data_format', 'environ', 'impl_name', 'invocation_time', 'job_type', 'name', 'slow_threshold', 'usage', 'wsgi_environ') if is_response: attrs += ('handle_return_time', 'processing_time', 'processing_time_raw', 'zato.http.response.headers') for attr in attrs: if attr not in suppress_keys: msg[attr] = self.channel.type if attr == 'channel' else getattr( self, attr, '(None)') else: msg[attr] = suppressed_msg self.logger.log(level, '{}{}'.format(user_msg, msg)) return msg def log_input(self, user_msg='', level=logging.INFO, suppress_keys=None): return self._log_input_output(user_msg, level, suppress_keys, False) def log_output(self, user_msg='', level=logging.INFO, suppress_keys=('wsgi_environ', )): return self._log_input_output(user_msg, level, suppress_keys, True) # ################################################################################################################################ @staticmethod def update(service, channel_type, server, broker_client, _ignored, cid, payload, raw_request, transport=None, simple_io_config=None, data_format=None, wsgi_environ={}, job_type=None, channel_params=None, merge_channel_params=True, params_priority=None, in_reply_to=None, environ=None, init=True, wmq_ctx=None, _wsgi_channels=(CHANNEL.HTTP_SOAP, CHANNEL.INVOKE, CHANNEL.INVOKE_ASYNC), _AMQP=CHANNEL.AMQP, _WMQ=CHANNEL.WEBSPHERE_MQ): """ Takes a service instance and updates it with the current request's context data. """ service.server = server service.broker_client = broker_client service.cid = cid service.request.payload = payload service.request.raw_request = raw_request service.transport = transport service.request.simple_io_config = simple_io_config service.response.simple_io_config = simple_io_config service.data_format = data_format service.wsgi_environ = wsgi_environ service.job_type = job_type service.translate = server.kvdb.translate service.user_config = server.user_config service.static_config = server.static_config service.time = server.time_util if channel_params: service.request.channel_params.update(channel_params) service.request.merge_channel_params = merge_channel_params service.request.params_priority = params_priority service.in_reply_to = in_reply_to service.environ = environ or {} channel_item = wsgi_environ.get('zato.channel_item', {}) sec_def_info = wsgi_environ.get('zato.sec_def', {}) if channel_type == _AMQP: service.request.amqp = AMQPRequestData(channel_item['amqp_msg']) elif channel_type == _WMQ: service.request.wmq = service.request.ibm_mq = IBMMQRequestData( wmq_ctx) service.channel = service.chan = ChannelInfo( channel_item.get('id'), channel_item.get('name'), channel_type, channel_item.get('data_format'), channel_item.get('is_internal'), channel_item.get('match_target'), ChannelSecurityInfo(sec_def_info.get('id'), sec_def_info.get('name'), sec_def_info.get('type'), sec_def_info.get('username'), sec_def_info.get('impl')), channel_item) if init: service._init(channel_type in _wsgi_channels)