class Service(object): """ A base class for all services deployed on Zato servers, no matter the transport and protocol, be it plain HTTP, SOAP, WebSphere MQ or any other, regardless whether they're built-in or user-defined ones. """ http_method_handlers = {} def __init__(self, *ignored_args, **ignored_kwargs): self.logger = logging.getLogger(self.get_name()) self.server = None self.broker_client = None self.pubsub = None self.channel = None self.cid = None self.in_reply_to = None self.outgoing = None self.cloud = None self.worker_store = None self.odb = None self.data_format = None self.transport = None self.wsgi_environ = None self.job_type = None self.environ = Bunch() self.request = Request(self.logger) self.response = Response(self.logger) self.invocation_time = None # When was the service invoked self.handle_return_time = None # When did its 'handle' method finished processing the request self.processing_time_raw = None # A timedelta object with the processing time up to microseconds self.processing_time = None # Processing time in milliseconds self.usage = 0 # How many times the service has been invoked self.slow_threshold = maxint # After how many ms to consider the response came too late self.name = self.__class__.get_name() self.impl_name = self.__class__.get_impl_name() self.time = None self.patterns = None self.user_config = None self.dictnav = DictNav self.listnav = ListNav self.has_validate_input = False self.has_validate_output = False @staticmethod def get_name_static(class_): return Service.get_name(class_) @classmethod def get_name(class_): """ Returns a service's name, settings its .name attribute along. This will be called once while the service is being deployed. """ if not hasattr(class_, '__name'): name = getattr(class_, 'name', None) if not name: name = service_name_from_impl(class_.get_impl_name()) name = class_.convert_impl_name(name) class_.__name = name return class_.__name @classmethod def get_impl_name(class_): if not hasattr(class_, '__impl_name'): class_.__impl_name = '{}.{}'.format(class_.__module__, class_.__name__) return class_.__impl_name @staticmethod def convert_impl_name(name): # TODO: Move the replace functionality over to uncamelify, possibly modifying its regexp split = uncamelify(name).split('.') path, class_name = split[:-1], split[-1] path = [elem.replace('_', '-') for elem in path] class_name = class_name[1:] if class_name.startswith('-') else class_name class_name = class_name.replace('.-', '.').replace('_-', '_') return '{}.{}'.format('.'.join(path), class_name) @classmethod def add_http_method_handlers(class_): for name in dir(class_): if name.startswith('handle_'): if not getattr(class_, 'http_method_handlers', False): setattr(class_, 'http_method_handlers', {}) method = name.replace('handle_', '') class_.http_method_handlers[method] = getattr(class_, name) def _init(self): """ Actually initializes the service. """ self.odb = self.worker_store.server.odb self.kvdb = self.worker_store.kvdb self.pubsub = self.worker_store.pubsub self.slow_threshold = self.server.service_store.services[self.impl_name]['slow_threshold'] # Queues out_amqp = PublisherFacade(self.broker_client) out_jms_wmq = WMQFacade(self.broker_client) out_zmq = ZMQFacade(self.server) # Patterns self.patterns = PatternsFacade(self) # SQL out_sql = self.worker_store.sql_pool_store # Regular outconns out_ftp, out_odoo, out_plain_http, out_soap = self.worker_store.worker_config.outgoing_connections() self.outgoing = Outgoing( out_amqp, out_ftp, out_jms_wmq, out_odoo, out_plain_http, out_soap, None, None, out_zmq) self.outgoing = Outgoing( out_amqp, out_ftp, out_jms_wmq, out_odoo, out_plain_http, out_soap, out_sql, self.worker_store.stomp_outconn_api, out_zmq) # Cloud self.cloud = Cloud() self.cloud.openstack.swift = self.worker_store.worker_config.cloud_openstack_swift self.cloud.aws.s3 = self.worker_store.worker_config.cloud_aws_s3 # Cassandra self.cassandra_conn = self.worker_store.cassandra_api self.cassandra_query = self.worker_store.cassandra_query_api # E-mail self.email = EMailAPI(self.worker_store.email_smtp_api, self.worker_store.email_imap_api) # Search self.search = SearchAPI(self.worker_store.search_es_api, self.worker_store.search_solr_api) is_sio = hasattr(self, 'SimpleIO') self.request.http.init(self.wsgi_environ) if is_sio: self.request.init(is_sio, self.cid, self.SimpleIO, self.data_format, self.transport, self.wsgi_environ) self.response.init(self.cid, self.SimpleIO, self.data_format) self.msg = MessageFacade(self.worker_store.msg_ns_store, self.worker_store.json_pointer_store, self.worker_store.xpath_store, self.worker_store.msg_ns_store, self.request.payload, self.time) def set_response_data(self, service, **kwargs): response = service.response.payload if not isinstance(response, (basestring, dict, list, tuple, EtreeElement, ObjectifiedElement)): response = response.getvalue(serialize=kwargs['serialize']) if kwargs['as_bunch']: response = bunchify(response) service.response.payload = response return response def _invoke(self, service, channel, http_channels=(CHANNEL.HTTP_SOAP, CHANNEL.INVOKE)): # # If channel is HTTP and there are any per-HTTP verb methods, it means we want for the service to be a REST target. # Let's say it is POST. If we have handle_POST, it is invoked. If there is no handle_POST, # '405 Method Not Allowed is returned'. # # However, if we have 'handle' only, it means this is always invoked and no default 405 is returned. # # In short, implement handle_* if you want REST behaviour. Otherwise, keep everything in handle. # # Ok, this is HTTP if channel in http_channels: # We have at least one per-HTTP verb handler if service.http_method_handlers: # But do we have any handler matching current request's verb? if service.request.http.method in service.http_method_handlers: # Yes, call the handler service.http_method_handlers[service.request.http.method](service) # No, return 405 else: service.response.status_code = METHOD_NOT_ALLOWED # We have no custom handlers so we always call 'handle' else: service.handle() # It's not HTTP so we simply call 'handle' else: service.handle() def extract_target(self, name): """ Splits a service's name into name and target, if the latter is provided on input at all. """ # It can be either a name or a name followed by the target to invoke the service on, # i.e. 'myservice' or 'myservice@mytarget'. if '@' in name: name, target = name.split('@') if not target: raise ZatoException(self.cid, 'Target must not be empty in `{}`'.format(name)) else: target = '' return name, target def update_handle(self, set_response_func, service, raw_request, channel, data_format, transport, server, broker_client, worker_store, cid, simple_io_config, *args, **kwargs): wsgi_environ = kwargs.get('wsgi_environ', {}) payload = wsgi_environ.get('zato.request.payload') # Here's an edge case. If a SOAP request has a single child in Body and this child is an empty element # (though possibly with attributes), checking for 'not payload' alone won't suffice - this evaluates # to False so we'd be parsing the payload again superfluously. if not isinstance(payload, ObjectifiedElement) and not payload: payload = payload_from_request(cid, raw_request, data_format, transport) job_type = kwargs.get('job_type') channel_params = kwargs.get('channel_params', {}) merge_channel_params = kwargs.get('merge_channel_params', True) params_priority = kwargs.get('params_priority', PARAMS_PRIORITY.DEFAULT) service.update(service, channel, server, broker_client, worker_store, cid, payload, raw_request, transport, simple_io_config, data_format, wsgi_environ, job_type=job_type, channel_params=channel_params, merge_channel_params=merge_channel_params, params_priority=params_priority, in_reply_to=wsgi_environ.get('zato.request_ctx.in_reply_to', None), environ=kwargs.get('environ')) # It's possible the call will be completely filtered out if service.accept(): # Assume everything goes fine e, exc_formatted = None, None try: service.pre_handle() service.call_hooks('before') service.validate_input() self._invoke(service, channel) service.validate_output() service.call_hooks('after') service.post_handle() service.call_hooks('finalize') except Exception, e: exc_formatted = format_exc(e) logger.warn(exc_formatted) finally:
class Service(object): """ A base class for all services deployed on Zato servers, no matter the transport and protocol, be it plain HTTP, SOAP, WebSphere MQ or any other, regardless whether they're built-in or user-defined ones. """ http_method_handlers = {} def __init__(self, *ignored_args, **ignored_kwargs): self.logger = logging.getLogger(self.get_name()) self.server = None self.broker_client = None self.pubsub = None self.channel = None self.cid = None self.in_reply_to = None self.outgoing = None self.cloud = None self.worker_store = None self.odb = None self.data_format = None self.transport = None self.wsgi_environ = None self.job_type = None self.delivery_store = None self.environ = {} self.request = Request(self.logger) self.response = Response(self.logger) self.invocation_time = None # When was the service invoked self.handle_return_time = None # When did its 'handle' method finished processing the request self.processing_time_raw = None # A timedelta object with the processing time up to microseconds self.processing_time = None # Processing time in milliseconds self.usage = 0 # How many times the service has been invoked self.slow_threshold = maxint # After how many ms to consider the response came too late self.name = self.__class__.get_name() self.impl_name = self.__class__.get_impl_name() self.time = TimeUtil(None) self.patterns = None self.user_config = None self.dictnav = DictNav self.listnav = ListNav self.has_validate_input = False self.has_validate_output = False @staticmethod def get_name_static(class_): return Service.get_name(class_) @classmethod def get_name(class_): """ Returns a service's name, settings its .name attribute along. This will be called once while the service is being deployed. """ if not hasattr(class_, '__name'): name = getattr(class_, 'name', None) if not name: name = service_name_from_impl(class_.get_impl_name()) name = class_.convert_impl_name(name) class_.__name = name return class_.__name @classmethod def get_impl_name(class_): if not hasattr(class_, '__impl_name'): class_.__impl_name = '{}.{}'.format(class_.__module__, class_.__name__) return class_.__impl_name @staticmethod def convert_impl_name(name): # TODO: Move the replace functionality over to uncamelify, possibly modifying its regexp split = uncamelify(name).split('.') path, class_name = split[:-1], split[-1] path = [elem.replace('_', '-') for elem in path] class_name = class_name[1:] if class_name.startswith('-') else class_name class_name = class_name.replace('.-', '.').replace('_-', '_') return '{}.{}'.format('.'.join(path), class_name) @classmethod def add_http_method_handlers(class_): for name in dir(class_): if name.startswith('handle_'): if not getattr(class_, 'http_method_handlers', False): setattr(class_, 'http_method_handlers', {}) method = name.replace('handle_', '') class_.http_method_handlers[method] = getattr(class_, name) def _init(self): """ Actually initializes the service. """ self.odb = self.worker_store.server.odb self.kvdb = self.worker_store.kvdb self.time.kvdb = self.kvdb self.pubsub = self.worker_store.pubsub self.slow_threshold = self.server.service_store.services[self.impl_name]['slow_threshold'] # Queues out_amqp = PublisherFacade(self.broker_client, self.server.delivery_store) out_jms_wmq = WMQFacade(self.broker_client, self.server.delivery_store) out_zmq = ZMQFacade(self.server) # Patterns self.patterns = PatternsFacade(self) # SQL out_sql = self.worker_store.sql_pool_store # Regular outconns out_ftp, out_odoo, out_plain_http, out_soap = self.worker_store.worker_config.outgoing_connections() self.outgoing = Outgoing( out_amqp, out_ftp, out_jms_wmq, out_odoo, out_plain_http, out_soap, out_sql, self.worker_store.stomp_outconn_api, out_zmq) # Cloud self.cloud = Cloud() self.cloud.openstack.swift = self.worker_store.worker_config.cloud_openstack_swift self.cloud.aws.s3 = self.worker_store.worker_config.cloud_aws_s3 # Cassandra self.cassandra_conn = self.worker_store.cassandra_api self.cassandra_query = self.worker_store.cassandra_query_api # E-mail self.email = EMailAPI(self.worker_store.email_smtp_api, self.worker_store.email_imap_api) # Search self.search = SearchAPI(self.worker_store.search_es_api, self.worker_store.search_solr_api) is_sio = hasattr(self, 'SimpleIO') self.request.http.init(self.wsgi_environ) if is_sio: self.request.init(is_sio, self.cid, self.SimpleIO, self.data_format, self.transport, self.wsgi_environ) self.response.init(self.cid, self.SimpleIO, self.data_format) self.msg = MessageFacade(self.worker_store.msg_ns_store, self.worker_store.json_pointer_store, self.worker_store.xpath_store, self.worker_store.msg_ns_store, self.request.payload, self.time) def set_response_data(self, service, **kwargs): response = service.response.payload if not isinstance(response, (basestring, dict, list, tuple, EtreeElement, ObjectifiedElement)): response = response.getvalue(serialize=kwargs['serialize']) if kwargs['as_bunch']: response = bunchify(response) service.response.payload = response return response def _invoke(self, service, channel): # # If channel is HTTP and there are any per-HTTP verb methods, it means we want for the service to be a REST target. # Let's say it is POST. If we have handle_POST, it is invoked. If there is no handle_POST, # '405 Method Not Allowed is returned'. # # However, if we have 'handle' only, it means this is always invoked and no default 405 is returned. # # In short, implement handle_* if you want REST behaviour. Otherwise, keep everything in handle. # # Ok, this is HTTP if channel in (CHANNEL.HTTP_SOAP, CHANNEL.INVOKE): # We have at least one per-HTTP verb handler if service.http_method_handlers: # But do we have any handler matching current request's verb? if service.request.http.method in service.http_method_handlers: # Yes, call the handler service.http_method_handlers[service.request.http.method](service) # No, return 405 else: service.response.status_code = METHOD_NOT_ALLOWED # We have no custom handlers so we always call 'handle' else: service.handle() # It's not HTTP so we simply call 'handle' else: service.handle() def extract_target(self, name): """ Splits a service's name into name and target, if the latter is provided on input at all. """ # It can be either a name or a name followed by the target to invoke the service on, # i.e. 'myservice' or 'myservice@mytarget'. if '@' in name: name, target = name.split('@') if not target: raise ZatoException(self.cid, 'Target must not be empty in `{}`'.format(name)) else: target = '' return name, target def update_handle(self, set_response_func, service, raw_request, channel, data_format, transport, server, broker_client, worker_store, cid, simple_io_config, *args, **kwargs): wsgi_environ = kwargs.get('wsgi_environ', {}) payload = wsgi_environ.get('zato.request.payload') # Here's an edge case. If a SOAP request has a single child in Body and this child is an empty element # (though possibly with attributes), checking for 'not payload' alone won't suffice - this evaluates # to False so we'd be parsing the payload again superfluously. if not isinstance(payload, ObjectifiedElement) and not payload: payload = payload_from_request(cid, raw_request, data_format, transport) job_type = kwargs.get('job_type') channel_params = kwargs.get('channel_params', {}) merge_channel_params = kwargs.get('merge_channel_params', True) params_priority = kwargs.get('params_priority', PARAMS_PRIORITY.DEFAULT) service.update(service, channel, server, broker_client, worker_store, cid, payload, raw_request, transport, simple_io_config, data_format, wsgi_environ, job_type=job_type, channel_params=channel_params, merge_channel_params=merge_channel_params, params_priority=params_priority, in_reply_to=wsgi_environ.get('zato.request_ctx.in_reply_to', None), environ=kwargs.get('environ')) # It's possible the call will be completely filtered out if service.accept(): # Assume everything goes fine e, exc_formatted = None, None try: service.pre_handle() service.call_hooks('before') service.validate_input() self._invoke(service, channel) service.validate_output() service.call_hooks('after') service.post_handle() service.call_hooks('finalize') except Exception, e: exc_formatted = format_exc(e) logger.warn(exc_formatted) finally:
class Service(object): """ A base class for all services deployed on Zato servers, no matter the transport and protocol, be it plain HTTP, SOAP, WebSphere MQ or any other, regardless whether they're built-in or user-defined ones. """ passthrough_to = '' def __init__(self, *ignored_args, **ignored_kwargs): self.logger = logging.getLogger(self.get_name()) self.server = None self.broker_client = None self.pubsub = None self.channel = None self.cid = None self.outgoing = None self.cloud = None self.worker_store = None self.odb = None self.data_format = None self.transport = None self.wsgi_environ = None self.job_type = None self.delivery_store = None self.environ = {} self.request = Request(self.logger) self.response = Response(self.logger) self.invocation_time = None # When was the service invoked self.handle_return_time = None # When did its 'handle' method finished processing the request self.processing_time_raw = None # A timedelta object with the processing time up to microseconds self.processing_time = None # Processing time in milliseconds self.usage = 0 # How many times the service has been invoked self.slow_threshold = maxint # After how many ms to consider the response came too late self.name = self.__class__.get_name() self.impl_name = self.__class__.get_impl_name() self.time = TimeUtil(None) self.from_passthrough = False self.passthrough_request = None self.user_config = None self.dictnav = DictNav self.listnav = ListNav self.has_validate_input = False self.has_validate_output = False @staticmethod def get_name_static(class_): return Service.get_name(class_) @classmethod def get_name(class_): """ Returns a service's name, settings its .name attribute along. This will be called once while the service is being deployed. """ if not hasattr(class_, '__name'): name = getattr(class_, 'name', None) if not name: name = service_name_from_impl(class_.get_impl_name()) name = class_.convert_impl_name(name) class_.__name = name return class_.__name @classmethod def get_impl_name(class_): if not hasattr(class_, '__impl_name'): class_.__impl_name = '{}.{}'.format(class_.__module__, class_.__name__) return class_.__impl_name @staticmethod def convert_impl_name(name): # TODO: Move the replace functionality over to uncamelify, possibly modifying its regexp split = uncamelify(name).split('.') path, class_name = split[:-1], split[-1] path = [elem.replace('_', '-') for elem in path] class_name = class_name[1:] if class_name.startswith('-') else class_name class_name = class_name.replace('.-', '.').replace('_-', '_') return '{}.{}'.format('.'.join(path), class_name) def _init(self): """ Actually initializes the service. """ self.odb = self.worker_store.server.odb self.kvdb = self.worker_store.kvdb self.time.kvdb = self.kvdb self.pubsub = self.worker_store.pubsub self.slow_threshold = self.server.service_store.services[self.impl_name]['slow_threshold'] # Queues out_amqp = PublisherFacade(self.broker_client, self.server.delivery_store) out_jms_wmq = WMQFacade(self.broker_client, self.server.delivery_store) out_zmq = ZMQFacade(self.broker_client, self.server.delivery_store) # SQL out_sql = self.worker_store.sql_pool_store # Regular outconns out_ftp, out_plain_http, out_soap = self.worker_store.worker_config.outgoing_connections() self.outgoing = Outgoing(out_ftp, out_amqp, out_zmq, out_jms_wmq, out_sql, out_plain_http, out_soap) # Cloud self.cloud = Cloud() self.cloud.openstack.swift = self.worker_store.worker_config.cloud_openstack_swift self.cloud.aws.s3 = self.worker_store.worker_config.cloud_aws_s3 # Cassandra self.cassandra_conn = self.worker_store.cassandra_api self.cassandra_query = self.worker_store.cassandra_query_api # E-mail self.email = EMailAPI(self.worker_store.email_smtp_api, self.worker_store.email_imap_api) # Search self.search = SearchAPI(self.worker_store.search_es_api, self.worker_store.search_solr_api) is_sio = hasattr(self, 'SimpleIO') self.request.http.init(self.wsgi_environ) if is_sio: self.request.init(is_sio, self.cid, self.SimpleIO, self.data_format, self.transport, self.wsgi_environ) self.response.init(self.cid, self.SimpleIO, self.data_format) self.msg = MessageFacade(self.worker_store.msg_ns_store, self.worker_store.json_pointer_store, self.worker_store.xpath_store, self.worker_store.msg_ns_store, self.request.payload, self.time) def set_response_data(self, service, **kwargs): response = service.response.payload if not isinstance(response, (basestring, dict, list, tuple, EtreeElement, ObjectifiedElement)): response = response.getvalue(serialize=kwargs['serialize']) if kwargs['as_bunch']: response = bunchify(response) service.response.payload = response return response def update_handle(self, set_response_func, service, raw_request, channel, data_format, transport, server, broker_client, worker_store, cid, simple_io_config, *args, **kwargs): wsgi_environ = kwargs.get('wsgi_environ', {}) payload = wsgi_environ.get('zato.request.payload') # Here's an edge case. If a SOAP request has a single child in Body and this child is an empty element # (though possibly with attributes), checking for 'not payload' alone won't suffice - this evaluates # to False so we'd be parsing the payload again superfluously. if not isinstance(payload, ObjectifiedElement) and not payload: payload = payload_from_request(cid, raw_request, data_format, transport) job_type = kwargs.get('job_type') channel_params = kwargs.get('channel_params', {}) merge_channel_params = kwargs.get('merge_channel_params', True) params_priority = kwargs.get('params_priority', PARAMS_PRIORITY.DEFAULT) serialize = kwargs.get('serialize') as_bunch = kwargs.get('as_bunch') channel_item = kwargs.get('channel_item') service.update(service, channel, server, broker_client, worker_store, cid, payload, raw_request, transport, simple_io_config, data_format, wsgi_environ, job_type=job_type, channel_params=channel_params, merge_channel_params=merge_channel_params, params_priority=params_priority) # Depending on whether this is a pass-through service or not we invoke # the target services or the one we have in hand right now. Note that # hooks are always invoked for the one we have a handle to right now # even if it's a pass-through one. service.pre_handle() service.call_hooks('before') if service.passthrough_to: sio = getattr(service, 'SimpleIO', None) return self.invoke(service.passthrough_to, payload, channel, data_format, transport, serialize, as_bunch, sio=sio, from_passthrough=True, passthrough_request=self.request, set_response_func=set_response_func, channel_item=channel_item) else: service.validate_input() service.handle() service.validate_output() service.call_hooks('after') service.post_handle() service.call_hooks('finalize') return set_response_func(service, data_format=data_format, transport=transport, **kwargs) def invoke_by_impl_name(self, impl_name, payload='', channel=CHANNEL.INVOKE, data_format=DATA_FORMAT.DICT, transport=None, serialize=False, as_bunch=False, timeout=None, raise_timeout=True, **kwargs): """ Invokes a service synchronously by its implementation name (full dotted Python name). """ if self.impl_name == impl_name: msg = 'A service cannot invoke itself, name:[{}]'.format(self.name) self.logger.error(msg) raise ZatoException(self.cid, msg) service = self.server.service_store.new_instance(impl_name) service.from_passthrough = kwargs.get('from_passthrough', False) service.passthrough_request = kwargs.get('passthrough_request', None) if service.from_passthrough and kwargs.get('sio'): service.SimpleIO = kwargs['sio'] set_response_func = kwargs.pop('set_response_func', self.set_response_data) invoke_args = (set_response_func, service, payload, channel, data_format, transport, self.server, self.broker_client, self.worker_store, self.cid, self.request.simple_io_config) kwargs.update({"serialize":serialize, "as_bunch":as_bunch}) try: if timeout: try: g = spawn(self.update_handle, *invoke_args, **kwargs) return g.get(block=True, timeout=timeout) except Timeout: g.kill() logger.warn('Service `%s` timed out (%s)', service.name, self.cid) if raise_timeout: raise else: return self.update_handle(*invoke_args, **kwargs) except Exception, e: logger.warn('Could not invoke `%s`, e:`%s`', service.name, format_exc(e)) raise
class Service(object): """ A base class for all services deployed on Zato servers, no matter the transport and protocol, be it plain HTTP, SOAP, WebSphere MQ or any other, regardless whether they're built-in or user-defined ones. """ passthrough_to = '' def __init__(self, *ignored_args, **ignored_kwargs): self.logger = logging.getLogger(self.get_name()) self.server = None self.broker_client = None self.channel = None self.cid = None self.outgoing = None self.worker_store = None self.odb = None self.data_format = None self.transport = None self.wsgi_environ = None self.job_type = None self.delivery_store = None self.environ = {} self.request = Request(self.logger) self.response = Response(self.logger) self.invocation_time = None # When was the service invoked self.handle_return_time = None # When did its 'handle' method finished processing the request self.processing_time_raw = None # A timedelta object with the processing time up to microseconds self.processing_time = None # Processing time in milliseconds self.usage = 0 # How many times the service has been invoked self.slow_threshold = maxint # After how many ms to consider the response came too late self.name = self.__class__.get_name() self.impl_name = self.__class__.get_impl_name() self.time = TimeUtil(None) self.from_passthrough = False self.passthrough_request = None @classmethod def get_name(class_): """ Returns a service's name, settings its .name attribute along. This will be called once while the service is being deployed. """ if not hasattr(class_, '__name'): name = getattr(class_, 'name', None) if not name: name = service_name_from_impl(class_.get_impl_name()) name = class_.convert_impl_name(name) class_.__name = name return class_.__name @classmethod def get_impl_name(class_): if not hasattr(class_, '__impl_name'): class_.__impl_name = '{}.{}'.format(class_.__module__, class_.__name__) return class_.__impl_name @staticmethod def convert_impl_name(name): # TODO: Move the replace functionality over to uncamelify, possibly modifying its regexp split = uncamelify(name).split('.') path, class_name = split[:-1], split[-1] path = [elem.replace('_', '-') for elem in path] class_name = class_name[1:] if class_name.startswith('-') else class_name class_name = class_name.replace('.-', '.').replace('_-', '_') return '{}.{}'.format('.'.join(path), class_name) def _init(self): """ Actually initializes the service. """ self.odb = self.worker_store.server.odb self.kvdb = self.worker_store.kvdb self.time.kvdb = self.kvdb self.slow_threshold = self.server.service_store.services[self.impl_name]['slow_threshold'] out_amqp = PublisherFacade(self.broker_client, self.server.delivery_store) out_jms_wmq = WMQFacade(self.broker_client, self.server.delivery_store) out_zmq = ZMQFacade(self.broker_client, self.server.delivery_store) out_sql = self.worker_store.sql_pool_store out_ftp, out_plain_http, out_soap = self.worker_store.worker_config.outgoing_connections() self.outgoing = Outgoing(out_ftp, out_amqp, out_zmq, out_jms_wmq, out_sql, out_plain_http, out_soap) is_sio = hasattr(self, 'SimpleIO') if self.passthrough_request: self.request = self.passthrough_request self.request.http.init(self.wsgi_environ) if is_sio: self.request.init(is_sio, self.cid, self.SimpleIO, self.data_format, self.transport, self.wsgi_environ) self.response.init(self.cid, self.SimpleIO, self.data_format) self.msg = MessageFacade(self.worker_store.msg_ns_store, self.worker_store.elem_path_store, self.worker_store.xpath_store) def set_response_data(self, service, **kwargs): response = service.response.payload if not isinstance(response, (basestring, dict, list, tuple)): response = response.getvalue(serialize=kwargs['serialize']) if kwargs['as_bunch']: response = bunchify(response) service.response.payload = response return response def update_handle(self, set_response_func, service, raw_request, channel, data_format, transport, server, broker_client, worker_store, cid, simple_io_config, *args, **kwargs): payload = payload_from_request(cid, raw_request, data_format, transport) job_type = kwargs.get('job_type') channel_params = kwargs.get('channel_params', {}) merge_channel_params = kwargs.get('merge_channel_params', True) params_priority = kwargs.get('params_priority', PARAMS_PRIORITY.DEFAULT) wsgi_environ = kwargs.get('wsgi_environ', {}) serialize = kwargs.get('serialize') as_bunch = kwargs.get('as_bunch') service.update(service, channel, server, broker_client, worker_store, cid, payload, raw_request, transport, simple_io_config, data_format, wsgi_environ, job_type=job_type, channel_params=channel_params, merge_channel_params=merge_channel_params, params_priority=params_priority) # Depending on whether this is a pass-through service or not we invoke # the target services or the one we have in hand right now. Note that # hooks are always invoked for the one we have a handle to right now # even if it's a pass-through one. service.pre_handle() service.call_hooks('before') if service.passthrough_to: sio = getattr(service, 'SimpleIO', None) return self.invoke(service.passthrough_to, raw_request, channel, data_format, transport, serialize, as_bunch, sio=sio, from_passthrough=True, passthrough_request=self.request, set_response_func=set_response_func) else: service.handle() service.call_hooks('after') service.post_handle() service.call_hooks('finalize') return set_response_func(service, data_format=data_format, transport=transport, **kwargs) def invoke_by_impl_name(self, impl_name, payload='', channel=CHANNEL.INVOKE, data_format=DATA_FORMAT.DICT, transport=None, serialize=False, as_bunch=False, **kwargs): """ Invokes a service synchronously by its implementation name (full dotted Python name). """ if self.impl_name == impl_name: msg = 'A service cannot invoke itself, name:[{}]'.format(self.name) self.logger.error(msg) raise ZatoException(self.cid, msg) service = self.server.service_store.new_instance(impl_name) service.from_passthrough = kwargs.get('from_passthrough', False) service.passthrough_request = kwargs.get('passthrough_request', None) if service.from_passthrough and kwargs.get('sio'): service.SimpleIO = kwargs['sio'] set_response_func = kwargs.pop('set_response_func', self.set_response_data) return self.update_handle(set_response_func, service, payload, channel, data_format, transport, self.server, self.broker_client, self.worker_store, self.cid, self.request.simple_io_config, serialize=serialize, as_bunch=as_bunch, **kwargs) def invoke(self, name, *args, **kwargs): """ Invokes a service synchronously by its name. """ return self.invoke_by_impl_name(self.server.service_store.name_to_impl_name[name], *args, **kwargs) def invoke_by_id(self, service_id, *args, **kwargs): """ Invokes a service synchronously by its ID. """ return self.invoke_by_impl_name(self.server.service_store.id_to_impl_name[service_id], *args, **kwargs) def invoke_async(self, name, payload='', channel=CHANNEL.INVOKE_ASYNC, data_format=None, transport=None, expiration=BROKER.DEFAULT_EXPIRATION, to_json_string=False): """ Invokes a service asynchronously by its name. """ if to_json_string: payload = dumps(payload) cid = new_cid() msg = {} msg['action'] = SERVICE.PUBLISH msg['service'] = name msg['payload'] = payload msg['cid'] = cid msg['channel'] = channel msg['data_format'] = data_format msg['transport'] = transport self.broker_client.invoke_async(msg, expiration=expiration) return cid def deliver(self, def_name, payload, task_id=None, *args, **kwargs): """ Uses guaranteed delivery to send payload using a delivery definition known by def_name. *args and **kwargs will be passed directly as-is to the target behind the def_name. """ task_id = task_id or new_cid() self.delivery_store.deliver( self.server.cluster_id, def_name, payload, task_id, self.invoke, kwargs.pop('is_resubmit', False), kwargs.pop('is_auto', False), *args, **kwargs) return task_id def pre_handle(self): """ An internal method run just before the service sets to process the payload. Used for incrementing the service's usage count and storing the service invocation time. """ self.usage = self.kvdb.conn.incr('{}{}'.format(KVDB.SERVICE_USAGE, self.name)) self.invocation_time = datetime.utcnow() def post_handle(self): """ An internal method executed after the service has completed and has a response ready to return. Updates its statistics and, optionally, stores a sample request/response pair. """ # # Statistics # self.handle_return_time = datetime.utcnow() self.processing_time_raw = self.handle_return_time - self.invocation_time proc_time = self.processing_time_raw.total_seconds() * 1000.0 proc_time = proc_time if proc_time > 1 else 0 self.processing_time = int(round(proc_time)) self.kvdb.conn.hset('{}{}'.format(KVDB.SERVICE_TIME_BASIC, self.name), 'last', self.processing_time) self.kvdb.conn.rpush('{}{}'.format(KVDB.SERVICE_TIME_RAW, self.name), self.processing_time) key = '{}{}:{}'.format(KVDB.SERVICE_TIME_RAW_BY_MINUTE, self.name, self.handle_return_time.strftime('%Y:%m:%d:%H:%M')) self.kvdb.conn.rpush(key, self.processing_time) # .. we'll have 5 minutes (5 * 60 seconds = 300 seconds) # to aggregate processing times for a given minute and then it will expire # Note that we need Redis 2.1.3+ otherwise the key has just been overwritten self.kvdb.conn.expire(key, 300) # # Sample requests/responses # key, freq = request_response.should_store(self.kvdb, self.usage, self.name) if freq: # TODO: Don't parse it here and a moment later below resp = (self.response.payload.getvalue() if hasattr(self.response.payload, 'getvalue') else self.response.payload) or '' data = { 'cid': self.cid, 'req_ts': self.invocation_time.isoformat(), 'resp_ts': self.handle_return_time.isoformat(), 'req': self.request.raw_request or '', 'resp':resp, } request_response.store(self.kvdb, key, self.usage, freq, **data) # # Slow responses # if self.processing_time > self.slow_threshold: # TODO: Don't parse it here and a moment earlier above resp = (self.response.payload.getvalue() if hasattr(self.response.payload, 'getvalue') else self.response.payload) or '' data = { 'cid': self.cid, 'proc_time': self.processing_time, 'slow_threshold': self.slow_threshold, 'req_ts': self.invocation_time.isoformat(), 'resp_ts': self.handle_return_time.isoformat(), 'req': self.request.raw_request or '', 'resp': resp, } slow_response.store(self.kvdb, self.name, **data) def translate(self, *args, **kwargs): raise NotImplementedError('An initializer should override this method') def handle(self): """ The only method Zato services need to implement in order to process incoming requests. """ raise NotImplementedError('Should be overridden by subclasses') def lock(self, name=None, expires=20, timeout=0, backend=None): """ Creates a Redis-backed distributed lock. name - defaults to self.name effectively making access to this service serialized expires - defaults to 20 seconds and is the max time the lock will be held timeout - how long (in seconds) we will wait to acquire the lock before giving up and raising LockTimeout backend - a Redis connection object, defaults to self.kvdb.conn """ name = '{}{}'.format(KVDB.LOCK_SERVICE_PREFIX, name or self.name) backend = backend or self.kvdb.conn return Lock(name, expires, timeout, backend) # ############################################################################## def call_job_hooks(self, prefix): if self.channel == CHANNEL.SCHEDULER and prefix != 'finalize': try: getattr(self, '{}_job'.format(prefix))() except Exception, e: self.logger.error("Can't run {}_job, e:[{}]".format(prefix, format_exc(e))) else: try: func_name = '{}_{}_job'.format(prefix, self.job_type) func = getattr(self, func_name) func() except Exception, e: self.logger.error("Can't run {}, e:[{}]".format(func_name, format_exc(e)))
class Service(object): """ A base class for all services deployed on Zato servers, no matter the transport and protocol, be it plain HTTP, SOAP, WebSphere MQ or any other, regardless whether they're built-in or user-defined ones. """ passthrough_to = '' http_method_handlers = {} def __init__(self, *ignored_args, **ignored_kwargs): self.logger = logging.getLogger(self.get_name()) self.server = None self.broker_client = None self.pubsub = None self.channel = None self.cid = None self.outgoing = None self.cloud = None self.worker_store = None self.odb = None self.data_format = None self.transport = None self.wsgi_environ = None self.job_type = None self.delivery_store = None self.environ = {} self.request = Request(self.logger) self.response = Response(self.logger) self.invocation_time = None # When was the service invoked self.handle_return_time = None # When did its 'handle' method finished processing the request self.processing_time_raw = None # A timedelta object with the processing time up to microseconds self.processing_time = None # Processing time in milliseconds self.usage = 0 # How many times the service has been invoked self.slow_threshold = maxint # After how many ms to consider the response came too late self.name = self.__class__.get_name() self.impl_name = self.__class__.get_impl_name() self.time = TimeUtil(None) self.pattern = PatternsFacade(self) self.from_passthrough = False self.passthrough_request = None self.user_config = None self.dictnav = DictNav self.listnav = ListNav self.has_validate_input = False self.has_validate_output = False @staticmethod def get_name_static(class_): return Service.get_name(class_) @classmethod def get_name(class_): """ Returns a service's name, settings its .name attribute along. This will be called once while the service is being deployed. """ if not hasattr(class_, '__name'): name = getattr(class_, 'name', None) if not name: name = service_name_from_impl(class_.get_impl_name()) name = class_.convert_impl_name(name) class_.__name = name return class_.__name @classmethod def get_impl_name(class_): if not hasattr(class_, '__impl_name'): class_.__impl_name = '{}.{}'.format(class_.__module__, class_.__name__) return class_.__impl_name @staticmethod def convert_impl_name(name): # TODO: Move the replace functionality over to uncamelify, possibly modifying its regexp split = uncamelify(name).split('.') path, class_name = split[:-1], split[-1] path = [elem.replace('_', '-') for elem in path] class_name = class_name[1:] if class_name.startswith( '-') else class_name class_name = class_name.replace('.-', '.').replace('_-', '_') return '{}.{}'.format('.'.join(path), class_name) @classmethod def add_http_method_handlers(class_): for name in dir(class_): if name.startswith('handle_'): if not getattr(class_, 'http_method_handlers', False): setattr(class_, 'http_method_handlers', {}) method = name.replace('handle_', '') class_.http_method_handlers[method] = getattr(class_, name) def _init(self): """ Actually initializes the service. """ self.odb = self.worker_store.server.odb self.kvdb = self.worker_store.kvdb self.time.kvdb = self.kvdb self.pubsub = self.worker_store.pubsub self.slow_threshold = self.server.service_store.services[ self.impl_name]['slow_threshold'] # Queues out_amqp = PublisherFacade(self.broker_client, self.server.delivery_store) out_jms_wmq = WMQFacade(self.broker_client, self.server.delivery_store) out_zmq = ZMQFacade(self.broker_client, self.server.delivery_store) # SQL out_sql = self.worker_store.sql_pool_store # Regular outconns out_ftp, out_odoo, out_plain_http, out_soap = self.worker_store.worker_config.outgoing_connections( ) self.outgoing = Outgoing(out_amqp, out_ftp, out_jms_wmq, out_odoo, out_plain_http, out_soap, out_sql, out_zmq) # Cloud self.cloud = Cloud() self.cloud.openstack.swift = self.worker_store.worker_config.cloud_openstack_swift self.cloud.aws.s3 = self.worker_store.worker_config.cloud_aws_s3 # Cassandra self.cassandra_conn = self.worker_store.cassandra_api self.cassandra_query = self.worker_store.cassandra_query_api # E-mail self.email = EMailAPI(self.worker_store.email_smtp_api, self.worker_store.email_imap_api) # Search self.search = SearchAPI(self.worker_store.search_es_api, self.worker_store.search_solr_api) is_sio = hasattr(self, 'SimpleIO') self.request.http.init(self.wsgi_environ) if is_sio: self.request.init(is_sio, self.cid, self.SimpleIO, self.data_format, self.transport, self.wsgi_environ) self.response.init(self.cid, self.SimpleIO, self.data_format) self.msg = MessageFacade(self.worker_store.msg_ns_store, self.worker_store.json_pointer_store, self.worker_store.xpath_store, self.worker_store.msg_ns_store, self.request.payload, self.time) def set_response_data(self, service, **kwargs): response = service.response.payload if not isinstance( response, (basestring, dict, list, tuple, EtreeElement, ObjectifiedElement)): response = response.getvalue(serialize=kwargs['serialize']) if kwargs['as_bunch']: response = bunchify(response) service.response.payload = response return response def update_handle(self, set_response_func, service, raw_request, channel, data_format, transport, server, broker_client, worker_store, cid, simple_io_config, *args, **kwargs): wsgi_environ = kwargs.get('wsgi_environ', {}) payload = wsgi_environ.get('zato.request.payload') # Here's an edge case. If a SOAP request has a single child in Body and this child is an empty element # (though possibly with attributes), checking for 'not payload' alone won't suffice - this evaluates # to False so we'd be parsing the payload again superfluously. if not isinstance(payload, ObjectifiedElement) and not payload: payload = payload_from_request(cid, raw_request, data_format, transport) job_type = kwargs.get('job_type') channel_params = kwargs.get('channel_params', {}) merge_channel_params = kwargs.get('merge_channel_params', True) params_priority = kwargs.get('params_priority', PARAMS_PRIORITY.DEFAULT) serialize = kwargs.get('serialize') as_bunch = kwargs.get('as_bunch') channel_item = kwargs.get('channel_item') service.update(service, channel, server, broker_client, worker_store, cid, payload, raw_request, transport, simple_io_config, data_format, wsgi_environ, job_type=job_type, channel_params=channel_params, merge_channel_params=merge_channel_params, params_priority=params_priority) # Depending on whether this is a pass-through service or not we invoke # the target services or the one we have in hand right now. Note that # hooks are always invoked for the one we have a handle to right now # even if it's a pass-through one. service.pre_handle() service.call_hooks('before') if service.passthrough_to: sio = getattr(service, 'SimpleIO', None) return self.invoke(service.passthrough_to, payload, channel, data_format, transport, serialize, as_bunch, sio=sio, from_passthrough=True, passthrough_request=self.request, set_response_func=set_response_func, channel_item=channel_item) else: service.validate_input() # # If channel is HTTP and there are any per-HTTP verb methods, it means we want for the service to be a REST target. # Let's say it is POST. If we have handle_POST, it is invoked. If there is no handle_POST, # '405 Method Not Allowed is returned'. # # However, if we have 'handle' only, it means this is always invoked and no default 405 is returned. # # In short, implement handle_* if you want REST behaviour. Otherwise, keep everything in handle. # # Ok, this is HTTP if channel in (CHANNEL.HTTP_SOAP, CHANNEL.INVOKE): # We have at least one per-HTTP verb handler if service.http_method_handlers: # But do we have any handler matching current request's verb? if service.request.http.method in service.http_method_handlers: # Yes, call the handler service.http_method_handlers[ service.request.http.method](service) # No, return 405 else: service.response.status_code = METHOD_NOT_ALLOWED # We have no customer handlers so we always call 'handle' else: service.handle() # It's not HTTP so we simply call 'handle' else: service.handle() service.validate_output() service.call_hooks('after') service.post_handle() service.call_hooks('finalize') return set_response_func(service, data_format=data_format, transport=transport, **kwargs) def invoke_by_impl_name(self, impl_name, payload='', channel=CHANNEL.INVOKE, data_format=DATA_FORMAT.DICT, transport=None, serialize=False, as_bunch=False, timeout=None, raise_timeout=True, **kwargs): """ Invokes a service synchronously by its implementation name (full dotted Python name). """ if self.impl_name == impl_name: msg = 'A service cannot invoke itself, name:[{}]'.format(self.name) self.logger.error(msg) raise ZatoException(self.cid, msg) service = self.server.service_store.new_instance(impl_name) service.from_passthrough = kwargs.get('from_passthrough', False) service.passthrough_request = kwargs.get('passthrough_request', None) if service.from_passthrough and kwargs.get('sio'): service.SimpleIO = kwargs['sio'] set_response_func = kwargs.pop('set_response_func', self.set_response_data) invoke_args = (set_response_func, service, payload, channel, data_format, transport, self.server, self.broker_client, self.worker_store, self.cid, self.request.simple_io_config) kwargs.update({"serialize": serialize, "as_bunch": as_bunch}) try: if timeout: try: g = spawn(self.update_handle, *invoke_args, **kwargs) return g.get(block=True, timeout=timeout) except Timeout: g.kill() logger.warn('Service `%s` timed out (%s)', service.name, self.cid) if raise_timeout: raise else: return self.update_handle(*invoke_args, **kwargs) except Exception, e: logger.warn('Could not invoke `%s`, e:`%s`', service.name, format_exc(e)) raise