def __init__(self, *ignored_args, **ignored_kwargs):
self.logger = logging.getLogger(self.get_name())
self.server = None
self.broker_client = None
self.pubsub = None
self.channel = None
self.cid = None
self.in_reply_to = None
self.outgoing = None
self.cloud = None
self.worker_store = None
self.odb = None
self.data_format = None
self.transport = None
self.wsgi_environ = None
self.job_type = None
self.environ = Bunch()
self.request = Request(self.logger)
self.response = Response(self.logger)
self.invocation_time = None # When was the service invoked
self.handle_return_time = None # When did its 'handle' method finished processing the request
self.processing_time_raw = None # A timedelta object with the processing time up to microseconds
self.processing_time = None # Processing time in milliseconds
self.usage = 0 # How many times the service has been invoked
self.slow_threshold = maxint # After how many ms to consider the response came too late
self.name = self.__class__.get_name()
self.impl_name = self.__class__.get_impl_name()
self.time = None
self.patterns = None
self.user_config = None
self.dictnav = DictNav
self.listnav = ListNav
self.has_validate_input = False
self.has_validate_output = False
def __init__(self, *ignored_args, **ignored_kwargs):
self.logger = logging.getLogger(self.get_name())
self.server = None
self.broker_client = None
self.channel = None
self.cid = None
self.outgoing = None
self.worker_store = None
self.odb = None
self.data_format = None
self.transport = None
self.wsgi_environ = None
self.job_type = None
self.delivery_store = None
self.environ = {}
self.request = Request(self.logger)
self.response = Response(self.logger)
self.invocation_time = None # When was the service invoked
self.handle_return_time = None # When did its 'handle' method finished processing the request
self.processing_time_raw = None # A timedelta object with the processing time up to microseconds
self.processing_time = None # Processing time in milliseconds
self.usage = 0 # How many times the service has been invoked
self.slow_threshold = maxint # After how many ms to consider the response came too late
self.name = self.__class__.get_name()
self.impl_name = self.__class__.get_impl_name()
self.time = TimeUtil(None)
self.from_passthrough = False
self.passthrough_request = None
def __init__(self, *ignored_args, **ignored_kwargs):
self.logger = logging.getLogger(self.get_name())
self.server = None
self.broker_client = None
self.pubsub = None
self.channel = None
self.cid = None
self.in_reply_to = None
self.outgoing = None
self.cloud = None
self.worker_store = None
self.odb = None
self.data_format = None
self.transport = None
self.wsgi_environ = None
self.job_type = None
self.delivery_store = None
self.environ = {}
self.request = Request(self.logger)
self.response = Response(self.logger)
self.invocation_time = None # When was the service invoked
self.handle_return_time = None # When did its 'handle' method finished processing the request
self.processing_time_raw = None # A timedelta object with the processing time up to microseconds
self.processing_time = None # Processing time in milliseconds
self.usage = 0 # How many times the service has been invoked
self.slow_threshold = maxint # After how many ms to consider the response came too late
self.name = self.__class__.get_name()
self.impl_name = self.__class__.get_impl_name()
self.time = TimeUtil(None)
self.patterns = None
self.user_config = None
self.dictnav = DictNav
self.listnav = ListNav
self.has_validate_input = False
self.has_validate_output = False
class Service(object):
""" A base class for all services deployed on Zato servers, no matter
the transport and protocol, be it plain HTTP, SOAP, WebSphere MQ or any other,
regardless whether they're built-in or user-defined ones.
"""
http_method_handlers = {}
def __init__(self, *ignored_args, **ignored_kwargs):
self.logger = logging.getLogger(self.get_name())
self.server = None
self.broker_client = None
self.pubsub = None
self.channel = None
self.cid = None
self.in_reply_to = None
self.outgoing = None
self.cloud = None
self.worker_store = None
self.odb = None
self.data_format = None
self.transport = None
self.wsgi_environ = None
self.job_type = None
self.environ = Bunch()
self.request = Request(self.logger)
self.response = Response(self.logger)
self.invocation_time = None # When was the service invoked
self.handle_return_time = None # When did its 'handle' method finished processing the request
self.processing_time_raw = None # A timedelta object with the processing time up to microseconds
self.processing_time = None # Processing time in milliseconds
self.usage = 0 # How many times the service has been invoked
self.slow_threshold = maxint # After how many ms to consider the response came too late
self.name = self.__class__.get_name()
self.impl_name = self.__class__.get_impl_name()
self.time = None
self.patterns = None
self.user_config = None
self.dictnav = DictNav
self.listnav = ListNav
self.has_validate_input = False
self.has_validate_output = False
@staticmethod
def get_name_static(class_):
return Service.get_name(class_)
@classmethod
def get_name(class_):
""" Returns a service's name, settings its .name attribute along. This will
be called once while the service is being deployed.
"""
if not hasattr(class_, '__name'):
name = getattr(class_, 'name', None)
if not name:
name = service_name_from_impl(class_.get_impl_name())
name = class_.convert_impl_name(name)
class_.__name = name
return class_.__name
@classmethod
def get_impl_name(class_):
if not hasattr(class_, '__impl_name'):
class_.__impl_name = '{}.{}'.format(class_.__module__, class_.__name__)
return class_.__impl_name
@staticmethod
def convert_impl_name(name):
# TODO: Move the replace functionality over to uncamelify, possibly modifying its regexp
split = uncamelify(name).split('.')
path, class_name = split[:-1], split[-1]
path = [elem.replace('_', '-') for elem in path]
class_name = class_name[1:] if class_name.startswith('-') else class_name
class_name = class_name.replace('.-', '.').replace('_-', '_')
return '{}.{}'.format('.'.join(path), class_name)
@classmethod
def add_http_method_handlers(class_):
for name in dir(class_):
if name.startswith('handle_'):
if not getattr(class_, 'http_method_handlers', False):
setattr(class_, 'http_method_handlers', {})
method = name.replace('handle_', '')
class_.http_method_handlers[method] = getattr(class_, name)
def _init(self):
""" Actually initializes the service.
"""
self.odb = self.worker_store.server.odb
self.kvdb = self.worker_store.kvdb
self.pubsub = self.worker_store.pubsub
self.slow_threshold = self.server.service_store.services[self.impl_name]['slow_threshold']
# Queues
out_amqp = PublisherFacade(self.broker_client)
out_jms_wmq = WMQFacade(self.broker_client)
out_zmq = ZMQFacade(self.server)
# Patterns
self.patterns = PatternsFacade(self)
# SQL
out_sql = self.worker_store.sql_pool_store
# Regular outconns
out_ftp, out_odoo, out_plain_http, out_soap = self.worker_store.worker_config.outgoing_connections()
self.outgoing = Outgoing(
out_amqp, out_ftp, out_jms_wmq, out_odoo, out_plain_http, out_soap, None,
None, out_zmq)
self.outgoing = Outgoing(
out_amqp, out_ftp, out_jms_wmq, out_odoo, out_plain_http, out_soap, out_sql,
self.worker_store.stomp_outconn_api, out_zmq)
# Cloud
self.cloud = Cloud()
self.cloud.openstack.swift = self.worker_store.worker_config.cloud_openstack_swift
self.cloud.aws.s3 = self.worker_store.worker_config.cloud_aws_s3
# Cassandra
self.cassandra_conn = self.worker_store.cassandra_api
self.cassandra_query = self.worker_store.cassandra_query_api
# E-mail
self.email = EMailAPI(self.worker_store.email_smtp_api, self.worker_store.email_imap_api)
# Search
self.search = SearchAPI(self.worker_store.search_es_api, self.worker_store.search_solr_api)
is_sio = hasattr(self, 'SimpleIO')
self.request.http.init(self.wsgi_environ)
if is_sio:
self.request.init(is_sio, self.cid, self.SimpleIO, self.data_format, self.transport, self.wsgi_environ)
self.response.init(self.cid, self.SimpleIO, self.data_format)
self.msg = MessageFacade(self.worker_store.msg_ns_store,
self.worker_store.json_pointer_store, self.worker_store.xpath_store, self.worker_store.msg_ns_store,
self.request.payload, self.time)
def set_response_data(self, service, **kwargs):
response = service.response.payload
if not isinstance(response, (basestring, dict, list, tuple, EtreeElement, ObjectifiedElement)):
response = response.getvalue(serialize=kwargs['serialize'])
if kwargs['as_bunch']:
response = bunchify(response)
service.response.payload = response
return response
def _invoke(self, service, channel, http_channels=(CHANNEL.HTTP_SOAP, CHANNEL.INVOKE)):
#
# If channel is HTTP and there are any per-HTTP verb methods, it means we want for the service to be a REST target.
# Let's say it is POST. If we have handle_POST, it is invoked. If there is no handle_POST,
# '405 Method Not Allowed is returned'.
#
# However, if we have 'handle' only, it means this is always invoked and no default 405 is returned.
#
# In short, implement handle_* if you want REST behaviour. Otherwise, keep everything in handle.
#
# Ok, this is HTTP
if channel in http_channels:
# We have at least one per-HTTP verb handler
if service.http_method_handlers:
# But do we have any handler matching current request's verb?
if service.request.http.method in service.http_method_handlers:
# Yes, call the handler
service.http_method_handlers[service.request.http.method](service)
# No, return 405
else:
service.response.status_code = METHOD_NOT_ALLOWED
# We have no custom handlers so we always call 'handle'
else:
service.handle()
# It's not HTTP so we simply call 'handle'
else:
service.handle()
def extract_target(self, name):
""" Splits a service's name into name and target, if the latter is provided on input at all.
"""
# It can be either a name or a name followed by the target to invoke the service on,
# i.e. 'myservice' or 'myservice@mytarget'.
if '@' in name:
name, target = name.split('@')
if not target:
raise ZatoException(self.cid, 'Target must not be empty in `{}`'.format(name))
else:
target = ''
return name, target
def update_handle(self, set_response_func, service, raw_request, channel, data_format,
transport, server, broker_client, worker_store, cid, simple_io_config, *args, **kwargs):
wsgi_environ = kwargs.get('wsgi_environ', {})
payload = wsgi_environ.get('zato.request.payload')
# Here's an edge case. If a SOAP request has a single child in Body and this child is an empty element
# (though possibly with attributes), checking for 'not payload' alone won't suffice - this evaluates
# to False so we'd be parsing the payload again superfluously.
if not isinstance(payload, ObjectifiedElement) and not payload:
payload = payload_from_request(cid, raw_request, data_format, transport)
job_type = kwargs.get('job_type')
channel_params = kwargs.get('channel_params', {})
merge_channel_params = kwargs.get('merge_channel_params', True)
params_priority = kwargs.get('params_priority', PARAMS_PRIORITY.DEFAULT)
service.update(service, channel, server, broker_client,
worker_store, cid, payload, raw_request, transport,
simple_io_config, data_format, wsgi_environ,
job_type=job_type,
channel_params=channel_params,
merge_channel_params=merge_channel_params,
params_priority=params_priority, in_reply_to=wsgi_environ.get('zato.request_ctx.in_reply_to', None),
environ=kwargs.get('environ'))
# It's possible the call will be completely filtered out
if service.accept():
# Assume everything goes fine
e, exc_formatted = None, None
try:
service.pre_handle()
service.call_hooks('before')
service.validate_input()
self._invoke(service, channel)
service.validate_output()
service.call_hooks('after')
service.post_handle()
service.call_hooks('finalize')
except Exception, e:
exc_formatted = format_exc(e)
logger.warn(exc_formatted)
finally:
class Service(object):
""" A base class for all services deployed on Zato servers, no matter
the transport and protocol, be it plain HTTP, SOAP, WebSphere MQ or any other,
regardless whether they're built-in or user-defined ones.
"""
http_method_handlers = {}
def __init__(self, *ignored_args, **ignored_kwargs):
self.logger = logging.getLogger(self.get_name())
self.server = None
self.broker_client = None
self.pubsub = None
self.channel = None
self.cid = None
self.in_reply_to = None
self.outgoing = None
self.cloud = None
self.worker_store = None
self.odb = None
self.data_format = None
self.transport = None
self.wsgi_environ = None
self.job_type = None
self.delivery_store = None
self.environ = {}
self.request = Request(self.logger)
self.response = Response(self.logger)
self.invocation_time = None # When was the service invoked
self.handle_return_time = None # When did its 'handle' method finished processing the request
self.processing_time_raw = None # A timedelta object with the processing time up to microseconds
self.processing_time = None # Processing time in milliseconds
self.usage = 0 # How many times the service has been invoked
self.slow_threshold = maxint # After how many ms to consider the response came too late
self.name = self.__class__.get_name()
self.impl_name = self.__class__.get_impl_name()
self.time = TimeUtil(None)
self.patterns = None
self.user_config = None
self.dictnav = DictNav
self.listnav = ListNav
self.has_validate_input = False
self.has_validate_output = False
@staticmethod
def get_name_static(class_):
return Service.get_name(class_)
@classmethod
def get_name(class_):
""" Returns a service's name, settings its .name attribute along. This will
be called once while the service is being deployed.
"""
if not hasattr(class_, '__name'):
name = getattr(class_, 'name', None)
if not name:
name = service_name_from_impl(class_.get_impl_name())
name = class_.convert_impl_name(name)
class_.__name = name
return class_.__name
@classmethod
def get_impl_name(class_):
if not hasattr(class_, '__impl_name'):
class_.__impl_name = '{}.{}'.format(class_.__module__, class_.__name__)
return class_.__impl_name
@staticmethod
def convert_impl_name(name):
# TODO: Move the replace functionality over to uncamelify, possibly modifying its regexp
split = uncamelify(name).split('.')
path, class_name = split[:-1], split[-1]
path = [elem.replace('_', '-') for elem in path]
class_name = class_name[1:] if class_name.startswith('-') else class_name
class_name = class_name.replace('.-', '.').replace('_-', '_')
return '{}.{}'.format('.'.join(path), class_name)
@classmethod
def add_http_method_handlers(class_):
for name in dir(class_):
if name.startswith('handle_'):
if not getattr(class_, 'http_method_handlers', False):
setattr(class_, 'http_method_handlers', {})
method = name.replace('handle_', '')
class_.http_method_handlers[method] = getattr(class_, name)
def _init(self):
""" Actually initializes the service.
"""
self.odb = self.worker_store.server.odb
self.kvdb = self.worker_store.kvdb
self.time.kvdb = self.kvdb
self.pubsub = self.worker_store.pubsub
self.slow_threshold = self.server.service_store.services[self.impl_name]['slow_threshold']
# Queues
out_amqp = PublisherFacade(self.broker_client, self.server.delivery_store)
out_jms_wmq = WMQFacade(self.broker_client, self.server.delivery_store)
out_zmq = ZMQFacade(self.server)
# Patterns
self.patterns = PatternsFacade(self)
# SQL
out_sql = self.worker_store.sql_pool_store
# Regular outconns
out_ftp, out_odoo, out_plain_http, out_soap = self.worker_store.worker_config.outgoing_connections()
self.outgoing = Outgoing(
out_amqp, out_ftp, out_jms_wmq, out_odoo, out_plain_http, out_soap, out_sql,
self.worker_store.stomp_outconn_api, out_zmq)
# Cloud
self.cloud = Cloud()
self.cloud.openstack.swift = self.worker_store.worker_config.cloud_openstack_swift
self.cloud.aws.s3 = self.worker_store.worker_config.cloud_aws_s3
# Cassandra
self.cassandra_conn = self.worker_store.cassandra_api
self.cassandra_query = self.worker_store.cassandra_query_api
# E-mail
self.email = EMailAPI(self.worker_store.email_smtp_api, self.worker_store.email_imap_api)
# Search
self.search = SearchAPI(self.worker_store.search_es_api, self.worker_store.search_solr_api)
is_sio = hasattr(self, 'SimpleIO')
self.request.http.init(self.wsgi_environ)
if is_sio:
self.request.init(is_sio, self.cid, self.SimpleIO, self.data_format, self.transport, self.wsgi_environ)
self.response.init(self.cid, self.SimpleIO, self.data_format)
self.msg = MessageFacade(self.worker_store.msg_ns_store,
self.worker_store.json_pointer_store, self.worker_store.xpath_store, self.worker_store.msg_ns_store,
self.request.payload, self.time)
def set_response_data(self, service, **kwargs):
response = service.response.payload
if not isinstance(response, (basestring, dict, list, tuple, EtreeElement, ObjectifiedElement)):
response = response.getvalue(serialize=kwargs['serialize'])
if kwargs['as_bunch']:
response = bunchify(response)
service.response.payload = response
return response
def _invoke(self, service, channel):
#
# If channel is HTTP and there are any per-HTTP verb methods, it means we want for the service to be a REST target.
# Let's say it is POST. If we have handle_POST, it is invoked. If there is no handle_POST,
# '405 Method Not Allowed is returned'.
#
# However, if we have 'handle' only, it means this is always invoked and no default 405 is returned.
#
# In short, implement handle_* if you want REST behaviour. Otherwise, keep everything in handle.
#
# Ok, this is HTTP
if channel in (CHANNEL.HTTP_SOAP, CHANNEL.INVOKE):
# We have at least one per-HTTP verb handler
if service.http_method_handlers:
# But do we have any handler matching current request's verb?
if service.request.http.method in service.http_method_handlers:
# Yes, call the handler
service.http_method_handlers[service.request.http.method](service)
# No, return 405
else:
service.response.status_code = METHOD_NOT_ALLOWED
# We have no custom handlers so we always call 'handle'
else:
service.handle()
# It's not HTTP so we simply call 'handle'
else:
service.handle()
def extract_target(self, name):
""" Splits a service's name into name and target, if the latter is provided on input at all.
"""
# It can be either a name or a name followed by the target to invoke the service on,
# i.e. 'myservice' or 'myservice@mytarget'.
if '@' in name:
name, target = name.split('@')
if not target:
raise ZatoException(self.cid, 'Target must not be empty in `{}`'.format(name))
else:
target = ''
return name, target
def update_handle(self, set_response_func, service, raw_request, channel, data_format,
transport, server, broker_client, worker_store, cid, simple_io_config, *args, **kwargs):
wsgi_environ = kwargs.get('wsgi_environ', {})
payload = wsgi_environ.get('zato.request.payload')
# Here's an edge case. If a SOAP request has a single child in Body and this child is an empty element
# (though possibly with attributes), checking for 'not payload' alone won't suffice - this evaluates
# to False so we'd be parsing the payload again superfluously.
if not isinstance(payload, ObjectifiedElement) and not payload:
payload = payload_from_request(cid, raw_request, data_format, transport)
job_type = kwargs.get('job_type')
channel_params = kwargs.get('channel_params', {})
merge_channel_params = kwargs.get('merge_channel_params', True)
params_priority = kwargs.get('params_priority', PARAMS_PRIORITY.DEFAULT)
service.update(service, channel, server, broker_client,
worker_store, cid, payload, raw_request, transport,
simple_io_config, data_format, wsgi_environ,
job_type=job_type,
channel_params=channel_params,
merge_channel_params=merge_channel_params,
params_priority=params_priority, in_reply_to=wsgi_environ.get('zato.request_ctx.in_reply_to', None),
environ=kwargs.get('environ'))
# It's possible the call will be completely filtered out
if service.accept():
# Assume everything goes fine
e, exc_formatted = None, None
try:
service.pre_handle()
service.call_hooks('before')
service.validate_input()
self._invoke(service, channel)
service.validate_output()
service.call_hooks('after')
service.post_handle()
service.call_hooks('finalize')
except Exception, e:
exc_formatted = format_exc(e)
logger.warn(exc_formatted)
finally:
class Service(object):
""" A base class for all services deployed on Zato servers, no matter
the transport and protocol, be it plain HTTP, SOAP, WebSphere MQ or any other,
regardless whether they're built-in or user-defined ones.
"""
passthrough_to = ''
def __init__(self, *ignored_args, **ignored_kwargs):
self.logger = logging.getLogger(self.get_name())
self.server = None
self.broker_client = None
self.channel = None
self.cid = None
self.outgoing = None
self.worker_store = None
self.odb = None
self.data_format = None
self.transport = None
self.wsgi_environ = None
self.job_type = None
self.delivery_store = None
self.environ = {}
self.request = Request(self.logger)
self.response = Response(self.logger)
self.invocation_time = None # When was the service invoked
self.handle_return_time = None # When did its 'handle' method finished processing the request
self.processing_time_raw = None # A timedelta object with the processing time up to microseconds
self.processing_time = None # Processing time in milliseconds
self.usage = 0 # How many times the service has been invoked
self.slow_threshold = maxint # After how many ms to consider the response came too late
self.name = self.__class__.get_name()
self.impl_name = self.__class__.get_impl_name()
self.time = TimeUtil(None)
self.from_passthrough = False
self.passthrough_request = None
@classmethod
def get_name(class_):
""" Returns a service's name, settings its .name attribute along. This will
be called once while the service is being deployed.
"""
if not hasattr(class_, '__name'):
name = getattr(class_, 'name', None)
if not name:
name = service_name_from_impl(class_.get_impl_name())
name = class_.convert_impl_name(name)
class_.__name = name
return class_.__name
@classmethod
def get_impl_name(class_):
if not hasattr(class_, '__impl_name'):
class_.__impl_name = '{}.{}'.format(class_.__module__, class_.__name__)
return class_.__impl_name
@staticmethod
def convert_impl_name(name):
# TODO: Move the replace functionality over to uncamelify, possibly modifying its regexp
split = uncamelify(name).split('.')
path, class_name = split[:-1], split[-1]
path = [elem.replace('_', '-') for elem in path]
class_name = class_name[1:] if class_name.startswith('-') else class_name
class_name = class_name.replace('.-', '.').replace('_-', '_')
return '{}.{}'.format('.'.join(path), class_name)
def _init(self):
""" Actually initializes the service.
"""
self.odb = self.worker_store.server.odb
self.kvdb = self.worker_store.kvdb
self.time.kvdb = self.kvdb
self.slow_threshold = self.server.service_store.services[self.impl_name]['slow_threshold']
out_amqp = PublisherFacade(self.broker_client, self.server.delivery_store)
out_jms_wmq = WMQFacade(self.broker_client, self.server.delivery_store)
out_zmq = ZMQFacade(self.broker_client, self.server.delivery_store)
out_sql = self.worker_store.sql_pool_store
out_ftp, out_plain_http, out_soap = self.worker_store.worker_config.outgoing_connections()
self.outgoing = Outgoing(out_ftp, out_amqp, out_zmq, out_jms_wmq, out_sql, out_plain_http, out_soap)
is_sio = hasattr(self, 'SimpleIO')
if self.passthrough_request:
self.request = self.passthrough_request
self.request.http.init(self.wsgi_environ)
if is_sio:
self.request.init(is_sio, self.cid, self.SimpleIO, self.data_format, self.transport, self.wsgi_environ)
self.response.init(self.cid, self.SimpleIO, self.data_format)
self.msg = MessageFacade(self.worker_store.msg_ns_store,
self.worker_store.elem_path_store, self.worker_store.xpath_store)
def set_response_data(self, service, **kwargs):
response = service.response.payload
if not isinstance(response, (basestring, dict, list, tuple)):
response = response.getvalue(serialize=kwargs['serialize'])
if kwargs['as_bunch']:
response = bunchify(response)
service.response.payload = response
return response
def update_handle(self, set_response_func, service, raw_request, channel, data_format,
transport, server, broker_client, worker_store, cid, simple_io_config, *args, **kwargs):
payload = payload_from_request(cid, raw_request, data_format, transport)
job_type = kwargs.get('job_type')
channel_params = kwargs.get('channel_params', {})
merge_channel_params = kwargs.get('merge_channel_params', True)
params_priority = kwargs.get('params_priority', PARAMS_PRIORITY.DEFAULT)
wsgi_environ = kwargs.get('wsgi_environ', {})
serialize = kwargs.get('serialize')
as_bunch = kwargs.get('as_bunch')
service.update(service, channel, server, broker_client,
worker_store, cid, payload, raw_request, transport,
simple_io_config, data_format, wsgi_environ,
job_type=job_type,
channel_params=channel_params,
merge_channel_params=merge_channel_params,
params_priority=params_priority)
# Depending on whether this is a pass-through service or not we invoke
# the target services or the one we have in hand right now. Note that
# hooks are always invoked for the one we have a handle to right now
# even if it's a pass-through one.
service.pre_handle()
service.call_hooks('before')
if service.passthrough_to:
sio = getattr(service, 'SimpleIO', None)
return self.invoke(service.passthrough_to, raw_request, channel, data_format,
transport, serialize, as_bunch, sio=sio, from_passthrough=True,
passthrough_request=self.request, set_response_func=set_response_func)
else:
service.handle()
service.call_hooks('after')
service.post_handle()
service.call_hooks('finalize')
return set_response_func(service, data_format=data_format, transport=transport, **kwargs)
def invoke_by_impl_name(self, impl_name, payload='', channel=CHANNEL.INVOKE, data_format=DATA_FORMAT.DICT,
transport=None, serialize=False, as_bunch=False, **kwargs):
""" Invokes a service synchronously by its implementation name (full dotted Python name).
"""
if self.impl_name == impl_name:
msg = 'A service cannot invoke itself, name:[{}]'.format(self.name)
self.logger.error(msg)
raise ZatoException(self.cid, msg)
service = self.server.service_store.new_instance(impl_name)
service.from_passthrough = kwargs.get('from_passthrough', False)
service.passthrough_request = kwargs.get('passthrough_request', None)
if service.from_passthrough and kwargs.get('sio'):
service.SimpleIO = kwargs['sio']
set_response_func = kwargs.pop('set_response_func', self.set_response_data)
return self.update_handle(set_response_func, service, payload, channel,
data_format, transport, self.server, self.broker_client, self.worker_store,
self.cid, self.request.simple_io_config, serialize=serialize, as_bunch=as_bunch,
**kwargs)
def invoke(self, name, *args, **kwargs):
""" Invokes a service synchronously by its name.
"""
return self.invoke_by_impl_name(self.server.service_store.name_to_impl_name[name], *args, **kwargs)
def invoke_by_id(self, service_id, *args, **kwargs):
""" Invokes a service synchronously by its ID.
"""
return self.invoke_by_impl_name(self.server.service_store.id_to_impl_name[service_id], *args, **kwargs)
def invoke_async(self, name, payload='', channel=CHANNEL.INVOKE_ASYNC, data_format=None,
transport=None, expiration=BROKER.DEFAULT_EXPIRATION, to_json_string=False):
""" Invokes a service asynchronously by its name.
"""
if to_json_string:
payload = dumps(payload)
cid = new_cid()
msg = {}
msg['action'] = SERVICE.PUBLISH
msg['service'] = name
msg['payload'] = payload
msg['cid'] = cid
msg['channel'] = channel
msg['data_format'] = data_format
msg['transport'] = transport
self.broker_client.invoke_async(msg, expiration=expiration)
return cid
def deliver(self, def_name, payload, task_id=None, *args, **kwargs):
""" Uses guaranteed delivery to send payload using a delivery definition known by def_name.
*args and **kwargs will be passed directly as-is to the target behind the def_name.
"""
task_id = task_id or new_cid()
self.delivery_store.deliver(
self.server.cluster_id, def_name, payload, task_id, self.invoke,
kwargs.pop('is_resubmit', False),
kwargs.pop('is_auto', False),
*args, **kwargs)
return task_id
def pre_handle(self):
""" An internal method run just before the service sets to process the payload.
Used for incrementing the service's usage count and storing the service invocation time.
"""
self.usage = self.kvdb.conn.incr('{}{}'.format(KVDB.SERVICE_USAGE, self.name))
self.invocation_time = datetime.utcnow()
def post_handle(self):
""" An internal method executed after the service has completed and has
a response ready to return. Updates its statistics and, optionally, stores
a sample request/response pair.
"""
#
# Statistics
#
self.handle_return_time = datetime.utcnow()
self.processing_time_raw = self.handle_return_time - self.invocation_time
proc_time = self.processing_time_raw.total_seconds() * 1000.0
proc_time = proc_time if proc_time > 1 else 0
self.processing_time = int(round(proc_time))
self.kvdb.conn.hset('{}{}'.format(KVDB.SERVICE_TIME_BASIC, self.name), 'last', self.processing_time)
self.kvdb.conn.rpush('{}{}'.format(KVDB.SERVICE_TIME_RAW, self.name), self.processing_time)
key = '{}{}:{}'.format(KVDB.SERVICE_TIME_RAW_BY_MINUTE,
self.name, self.handle_return_time.strftime('%Y:%m:%d:%H:%M'))
self.kvdb.conn.rpush(key, self.processing_time)
# .. we'll have 5 minutes (5 * 60 seconds = 300 seconds)
# to aggregate processing times for a given minute and then it will expire
# Note that we need Redis 2.1.3+ otherwise the key has just been overwritten
self.kvdb.conn.expire(key, 300)
#
# Sample requests/responses
#
key, freq = request_response.should_store(self.kvdb, self.usage, self.name)
if freq:
# TODO: Don't parse it here and a moment later below
resp = (self.response.payload.getvalue() if hasattr(self.response.payload, 'getvalue') else self.response.payload) or ''
data = {
'cid': self.cid,
'req_ts': self.invocation_time.isoformat(),
'resp_ts': self.handle_return_time.isoformat(),
'req': self.request.raw_request or '',
'resp':resp,
}
request_response.store(self.kvdb, key, self.usage, freq, **data)
#
# Slow responses
#
if self.processing_time > self.slow_threshold:
# TODO: Don't parse it here and a moment earlier above
resp = (self.response.payload.getvalue() if hasattr(self.response.payload, 'getvalue') else self.response.payload) or ''
data = {
'cid': self.cid,
'proc_time': self.processing_time,
'slow_threshold': self.slow_threshold,
'req_ts': self.invocation_time.isoformat(),
'resp_ts': self.handle_return_time.isoformat(),
'req': self.request.raw_request or '',
'resp': resp,
}
slow_response.store(self.kvdb, self.name, **data)
def translate(self, *args, **kwargs):
raise NotImplementedError('An initializer should override this method')
def handle(self):
""" The only method Zato services need to implement in order to process
incoming requests.
"""
raise NotImplementedError('Should be overridden by subclasses')
def lock(self, name=None, expires=20, timeout=0, backend=None):
""" Creates a Redis-backed distributed lock.
name - defaults to self.name effectively making access to this service serialized
expires - defaults to 20 seconds and is the max time the lock will be held
timeout - how long (in seconds) we will wait to acquire the lock before giving up and raising LockTimeout
backend - a Redis connection object, defaults to self.kvdb.conn
"""
name = '{}{}'.format(KVDB.LOCK_SERVICE_PREFIX, name or self.name)
backend = backend or self.kvdb.conn
return Lock(name, expires, timeout, backend)
# ##############################################################################
def call_job_hooks(self, prefix):
if self.channel == CHANNEL.SCHEDULER and prefix != 'finalize':
try:
getattr(self, '{}_job'.format(prefix))()
except Exception, e:
self.logger.error("Can't run {}_job, e:[{}]".format(prefix, format_exc(e)))
else:
try:
func_name = '{}_{}_job'.format(prefix, self.job_type)
func = getattr(self, func_name)
func()
except Exception, e:
self.logger.error("Can't run {}, e:[{}]".format(func_name, format_exc(e)))
class Service(object):
""" A base class for all services deployed on Zato servers, no matter
the transport and protocol, be it plain HTTP, SOAP, WebSphere MQ or any other,
regardless whether they're built-in or user-defined ones.
"""
passthrough_to = ''
def __init__(self, *ignored_args, **ignored_kwargs):
self.logger = logging.getLogger(self.get_name())
self.server = None
self.broker_client = None
self.pubsub = None
self.channel = None
self.cid = None
self.outgoing = None
self.cloud = None
self.worker_store = None
self.odb = None
self.data_format = None
self.transport = None
self.wsgi_environ = None
self.job_type = None
self.delivery_store = None
self.environ = {}
self.request = Request(self.logger)
self.response = Response(self.logger)
self.invocation_time = None # When was the service invoked
self.handle_return_time = None # When did its 'handle' method finished processing the request
self.processing_time_raw = None # A timedelta object with the processing time up to microseconds
self.processing_time = None # Processing time in milliseconds
self.usage = 0 # How many times the service has been invoked
self.slow_threshold = maxint # After how many ms to consider the response came too late
self.name = self.__class__.get_name()
self.impl_name = self.__class__.get_impl_name()
self.time = TimeUtil(None)
self.from_passthrough = False
self.passthrough_request = None
self.user_config = None
self.dictnav = DictNav
self.listnav = ListNav
self.has_validate_input = False
self.has_validate_output = False
@staticmethod
def get_name_static(class_):
return Service.get_name(class_)
@classmethod
def get_name(class_):
""" Returns a service's name, settings its .name attribute along. This will
be called once while the service is being deployed.
"""
if not hasattr(class_, '__name'):
name = getattr(class_, 'name', None)
if not name:
name = service_name_from_impl(class_.get_impl_name())
name = class_.convert_impl_name(name)
class_.__name = name
return class_.__name
@classmethod
def get_impl_name(class_):
if not hasattr(class_, '__impl_name'):
class_.__impl_name = '{}.{}'.format(class_.__module__, class_.__name__)
return class_.__impl_name
@staticmethod
def convert_impl_name(name):
# TODO: Move the replace functionality over to uncamelify, possibly modifying its regexp
split = uncamelify(name).split('.')
path, class_name = split[:-1], split[-1]
path = [elem.replace('_', '-') for elem in path]
class_name = class_name[1:] if class_name.startswith('-') else class_name
class_name = class_name.replace('.-', '.').replace('_-', '_')
return '{}.{}'.format('.'.join(path), class_name)
def _init(self):
""" Actually initializes the service.
"""
self.odb = self.worker_store.server.odb
self.kvdb = self.worker_store.kvdb
self.time.kvdb = self.kvdb
self.pubsub = self.worker_store.pubsub
self.slow_threshold = self.server.service_store.services[self.impl_name]['slow_threshold']
# Queues
out_amqp = PublisherFacade(self.broker_client, self.server.delivery_store)
out_jms_wmq = WMQFacade(self.broker_client, self.server.delivery_store)
out_zmq = ZMQFacade(self.broker_client, self.server.delivery_store)
# SQL
out_sql = self.worker_store.sql_pool_store
# Regular outconns
out_ftp, out_plain_http, out_soap = self.worker_store.worker_config.outgoing_connections()
self.outgoing = Outgoing(out_ftp, out_amqp, out_zmq, out_jms_wmq, out_sql, out_plain_http, out_soap)
# Cloud
self.cloud = Cloud()
self.cloud.openstack.swift = self.worker_store.worker_config.cloud_openstack_swift
self.cloud.aws.s3 = self.worker_store.worker_config.cloud_aws_s3
# Cassandra
self.cassandra_conn = self.worker_store.cassandra_api
self.cassandra_query = self.worker_store.cassandra_query_api
# E-mail
self.email = EMailAPI(self.worker_store.email_smtp_api, self.worker_store.email_imap_api)
# Search
self.search = SearchAPI(self.worker_store.search_es_api, self.worker_store.search_solr_api)
is_sio = hasattr(self, 'SimpleIO')
self.request.http.init(self.wsgi_environ)
if is_sio:
self.request.init(is_sio, self.cid, self.SimpleIO, self.data_format, self.transport, self.wsgi_environ)
self.response.init(self.cid, self.SimpleIO, self.data_format)
self.msg = MessageFacade(self.worker_store.msg_ns_store,
self.worker_store.json_pointer_store, self.worker_store.xpath_store, self.worker_store.msg_ns_store,
self.request.payload, self.time)
def set_response_data(self, service, **kwargs):
response = service.response.payload
if not isinstance(response, (basestring, dict, list, tuple, EtreeElement, ObjectifiedElement)):
response = response.getvalue(serialize=kwargs['serialize'])
if kwargs['as_bunch']:
response = bunchify(response)
service.response.payload = response
return response
def update_handle(self, set_response_func, service, raw_request, channel, data_format,
transport, server, broker_client, worker_store, cid, simple_io_config, *args, **kwargs):
wsgi_environ = kwargs.get('wsgi_environ', {})
payload = wsgi_environ.get('zato.request.payload')
# Here's an edge case. If a SOAP request has a single child in Body and this child is an empty element
# (though possibly with attributes), checking for 'not payload' alone won't suffice - this evaluates
# to False so we'd be parsing the payload again superfluously.
if not isinstance(payload, ObjectifiedElement) and not payload:
payload = payload_from_request(cid, raw_request, data_format, transport)
job_type = kwargs.get('job_type')
channel_params = kwargs.get('channel_params', {})
merge_channel_params = kwargs.get('merge_channel_params', True)
params_priority = kwargs.get('params_priority', PARAMS_PRIORITY.DEFAULT)
serialize = kwargs.get('serialize')
as_bunch = kwargs.get('as_bunch')
channel_item = kwargs.get('channel_item')
service.update(service, channel, server, broker_client,
worker_store, cid, payload, raw_request, transport,
simple_io_config, data_format, wsgi_environ,
job_type=job_type,
channel_params=channel_params,
merge_channel_params=merge_channel_params,
params_priority=params_priority)
# Depending on whether this is a pass-through service or not we invoke
# the target services or the one we have in hand right now. Note that
# hooks are always invoked for the one we have a handle to right now
# even if it's a pass-through one.
service.pre_handle()
service.call_hooks('before')
if service.passthrough_to:
sio = getattr(service, 'SimpleIO', None)
return self.invoke(service.passthrough_to, payload, channel, data_format, transport, serialize, as_bunch,
sio=sio, from_passthrough=True, passthrough_request=self.request, set_response_func=set_response_func,
channel_item=channel_item)
else:
service.validate_input()
service.handle()
service.validate_output()
service.call_hooks('after')
service.post_handle()
service.call_hooks('finalize')
return set_response_func(service, data_format=data_format, transport=transport, **kwargs)
def invoke_by_impl_name(self, impl_name, payload='', channel=CHANNEL.INVOKE, data_format=DATA_FORMAT.DICT,
transport=None, serialize=False, as_bunch=False, timeout=None, raise_timeout=True, **kwargs):
""" Invokes a service synchronously by its implementation name (full dotted Python name).
"""
if self.impl_name == impl_name:
msg = 'A service cannot invoke itself, name:[{}]'.format(self.name)
self.logger.error(msg)
raise ZatoException(self.cid, msg)
service = self.server.service_store.new_instance(impl_name)
service.from_passthrough = kwargs.get('from_passthrough', False)
service.passthrough_request = kwargs.get('passthrough_request', None)
if service.from_passthrough and kwargs.get('sio'):
service.SimpleIO = kwargs['sio']
set_response_func = kwargs.pop('set_response_func', self.set_response_data)
invoke_args = (set_response_func, service, payload, channel, data_format, transport, self.server,
self.broker_client, self.worker_store, self.cid, self.request.simple_io_config)
kwargs.update({"serialize":serialize, "as_bunch":as_bunch})
try:
if timeout:
try:
g = spawn(self.update_handle, *invoke_args, **kwargs)
return g.get(block=True, timeout=timeout)
except Timeout:
g.kill()
logger.warn('Service `%s` timed out (%s)', service.name, self.cid)
if raise_timeout:
raise
else:
return self.update_handle(*invoke_args, **kwargs)
except Exception, e:
logger.warn('Could not invoke `%s`, e:`%s`', service.name, format_exc(e))
raise
class Service(object):
""" A base class for all services deployed on Zato servers, no matter
the transport and protocol, be it plain HTTP, SOAP, WebSphere MQ or any other,
regardless whether they're built-in or user-defined ones.
"""
passthrough_to = ''
http_method_handlers = {}
def __init__(self, *ignored_args, **ignored_kwargs):
self.logger = logging.getLogger(self.get_name())
self.server = None
self.broker_client = None
self.pubsub = None
self.channel = None
self.cid = None
self.outgoing = None
self.cloud = None
self.worker_store = None
self.odb = None
self.data_format = None
self.transport = None
self.wsgi_environ = None
self.job_type = None
self.delivery_store = None
self.environ = {}
self.request = Request(self.logger)
self.response = Response(self.logger)
self.invocation_time = None # When was the service invoked
self.handle_return_time = None # When did its 'handle' method finished processing the request
self.processing_time_raw = None # A timedelta object with the processing time up to microseconds
self.processing_time = None # Processing time in milliseconds
self.usage = 0 # How many times the service has been invoked
self.slow_threshold = maxint # After how many ms to consider the response came too late
self.name = self.__class__.get_name()
self.impl_name = self.__class__.get_impl_name()
self.time = TimeUtil(None)
self.pattern = PatternsFacade(self)
self.from_passthrough = False
self.passthrough_request = None
self.user_config = None
self.dictnav = DictNav
self.listnav = ListNav
self.has_validate_input = False
self.has_validate_output = False
@staticmethod
def get_name_static(class_):
return Service.get_name(class_)
@classmethod
def get_name(class_):
""" Returns a service's name, settings its .name attribute along. This will
be called once while the service is being deployed.
"""
if not hasattr(class_, '__name'):
name = getattr(class_, 'name', None)
if not name:
name = service_name_from_impl(class_.get_impl_name())
name = class_.convert_impl_name(name)
class_.__name = name
return class_.__name
@classmethod
def get_impl_name(class_):
if not hasattr(class_, '__impl_name'):
class_.__impl_name = '{}.{}'.format(class_.__module__,
class_.__name__)
return class_.__impl_name
@staticmethod
def convert_impl_name(name):
# TODO: Move the replace functionality over to uncamelify, possibly modifying its regexp
split = uncamelify(name).split('.')
path, class_name = split[:-1], split[-1]
path = [elem.replace('_', '-') for elem in path]
class_name = class_name[1:] if class_name.startswith(
'-') else class_name
class_name = class_name.replace('.-', '.').replace('_-', '_')
return '{}.{}'.format('.'.join(path), class_name)
@classmethod
def add_http_method_handlers(class_):
for name in dir(class_):
if name.startswith('handle_'):
if not getattr(class_, 'http_method_handlers', False):
setattr(class_, 'http_method_handlers', {})
method = name.replace('handle_', '')
class_.http_method_handlers[method] = getattr(class_, name)
def _init(self):
""" Actually initializes the service.
"""
self.odb = self.worker_store.server.odb
self.kvdb = self.worker_store.kvdb
self.time.kvdb = self.kvdb
self.pubsub = self.worker_store.pubsub
self.slow_threshold = self.server.service_store.services[
self.impl_name]['slow_threshold']
# Queues
out_amqp = PublisherFacade(self.broker_client,
self.server.delivery_store)
out_jms_wmq = WMQFacade(self.broker_client, self.server.delivery_store)
out_zmq = ZMQFacade(self.broker_client, self.server.delivery_store)
# SQL
out_sql = self.worker_store.sql_pool_store
# Regular outconns
out_ftp, out_odoo, out_plain_http, out_soap = self.worker_store.worker_config.outgoing_connections(
)
self.outgoing = Outgoing(out_amqp, out_ftp, out_jms_wmq, out_odoo,
out_plain_http, out_soap, out_sql, out_zmq)
# Cloud
self.cloud = Cloud()
self.cloud.openstack.swift = self.worker_store.worker_config.cloud_openstack_swift
self.cloud.aws.s3 = self.worker_store.worker_config.cloud_aws_s3
# Cassandra
self.cassandra_conn = self.worker_store.cassandra_api
self.cassandra_query = self.worker_store.cassandra_query_api
# E-mail
self.email = EMailAPI(self.worker_store.email_smtp_api,
self.worker_store.email_imap_api)
# Search
self.search = SearchAPI(self.worker_store.search_es_api,
self.worker_store.search_solr_api)
is_sio = hasattr(self, 'SimpleIO')
self.request.http.init(self.wsgi_environ)
if is_sio:
self.request.init(is_sio, self.cid, self.SimpleIO,
self.data_format, self.transport,
self.wsgi_environ)
self.response.init(self.cid, self.SimpleIO, self.data_format)
self.msg = MessageFacade(self.worker_store.msg_ns_store,
self.worker_store.json_pointer_store,
self.worker_store.xpath_store,
self.worker_store.msg_ns_store,
self.request.payload, self.time)
def set_response_data(self, service, **kwargs):
response = service.response.payload
if not isinstance(
response,
(basestring, dict, list, tuple, EtreeElement, ObjectifiedElement)):
response = response.getvalue(serialize=kwargs['serialize'])
if kwargs['as_bunch']:
response = bunchify(response)
service.response.payload = response
return response
def update_handle(self, set_response_func, service, raw_request, channel,
data_format, transport, server, broker_client,
worker_store, cid, simple_io_config, *args, **kwargs):
wsgi_environ = kwargs.get('wsgi_environ', {})
payload = wsgi_environ.get('zato.request.payload')
# Here's an edge case. If a SOAP request has a single child in Body and this child is an empty element
# (though possibly with attributes), checking for 'not payload' alone won't suffice - this evaluates
# to False so we'd be parsing the payload again superfluously.
if not isinstance(payload, ObjectifiedElement) and not payload:
payload = payload_from_request(cid, raw_request, data_format,
transport)
job_type = kwargs.get('job_type')
channel_params = kwargs.get('channel_params', {})
merge_channel_params = kwargs.get('merge_channel_params', True)
params_priority = kwargs.get('params_priority',
PARAMS_PRIORITY.DEFAULT)
serialize = kwargs.get('serialize')
as_bunch = kwargs.get('as_bunch')
channel_item = kwargs.get('channel_item')
service.update(service,
channel,
server,
broker_client,
worker_store,
cid,
payload,
raw_request,
transport,
simple_io_config,
data_format,
wsgi_environ,
job_type=job_type,
channel_params=channel_params,
merge_channel_params=merge_channel_params,
params_priority=params_priority)
# Depending on whether this is a pass-through service or not we invoke
# the target services or the one we have in hand right now. Note that
# hooks are always invoked for the one we have a handle to right now
# even if it's a pass-through one.
service.pre_handle()
service.call_hooks('before')
if service.passthrough_to:
sio = getattr(service, 'SimpleIO', None)
return self.invoke(service.passthrough_to,
payload,
channel,
data_format,
transport,
serialize,
as_bunch,
sio=sio,
from_passthrough=True,
passthrough_request=self.request,
set_response_func=set_response_func,
channel_item=channel_item)
else:
service.validate_input()
#
# If channel is HTTP and there are any per-HTTP verb methods, it means we want for the service to be a REST target.
# Let's say it is POST. If we have handle_POST, it is invoked. If there is no handle_POST,
# '405 Method Not Allowed is returned'.
#
# However, if we have 'handle' only, it means this is always invoked and no default 405 is returned.
#
# In short, implement handle_* if you want REST behaviour. Otherwise, keep everything in handle.
#
# Ok, this is HTTP
if channel in (CHANNEL.HTTP_SOAP, CHANNEL.INVOKE):
# We have at least one per-HTTP verb handler
if service.http_method_handlers:
# But do we have any handler matching current request's verb?
if service.request.http.method in service.http_method_handlers:
# Yes, call the handler
service.http_method_handlers[
service.request.http.method](service)
# No, return 405
else:
service.response.status_code = METHOD_NOT_ALLOWED
# We have no customer handlers so we always call 'handle'
else:
service.handle()
# It's not HTTP so we simply call 'handle'
else:
service.handle()
service.validate_output()
service.call_hooks('after')
service.post_handle()
service.call_hooks('finalize')
return set_response_func(service,
data_format=data_format,
transport=transport,
**kwargs)
def invoke_by_impl_name(self,
impl_name,
payload='',
channel=CHANNEL.INVOKE,
data_format=DATA_FORMAT.DICT,
transport=None,
serialize=False,
as_bunch=False,
timeout=None,
raise_timeout=True,
**kwargs):
""" Invokes a service synchronously by its implementation name (full dotted Python name).
"""
if self.impl_name == impl_name:
msg = 'A service cannot invoke itself, name:[{}]'.format(self.name)
self.logger.error(msg)
raise ZatoException(self.cid, msg)
service = self.server.service_store.new_instance(impl_name)
service.from_passthrough = kwargs.get('from_passthrough', False)
service.passthrough_request = kwargs.get('passthrough_request', None)
if service.from_passthrough and kwargs.get('sio'):
service.SimpleIO = kwargs['sio']
set_response_func = kwargs.pop('set_response_func',
self.set_response_data)
invoke_args = (set_response_func, service, payload, channel,
data_format, transport, self.server, self.broker_client,
self.worker_store, self.cid,
self.request.simple_io_config)
kwargs.update({"serialize": serialize, "as_bunch": as_bunch})
try:
if timeout:
try:
g = spawn(self.update_handle, *invoke_args, **kwargs)
return g.get(block=True, timeout=timeout)
except Timeout:
g.kill()
logger.warn('Service `%s` timed out (%s)', service.name,
self.cid)
if raise_timeout:
raise
else:
return self.update_handle(*invoke_args, **kwargs)
except Exception, e:
logger.warn('Could not invoke `%s`, e:`%s`', service.name,
format_exc(e))
raise
