def setUp(self):
self.eps = ElemPathStore()
self.cust_id = uuid4().hex
self.street_name1 = 'street-1-{}'.format(uuid4().hex)
self.street_name2 = 'street-2-{}'.format(uuid4().hex)
self.street_elems1_1 = 'street-1_1-{}'.format(uuid4().hex)
self.street_elems1_2 = 'street-1_2-{}'.format(uuid4().hex)
self.street_elems2_1 = 'street-2_1-{}'.format(uuid4().hex)
self.street_elems2_2 = 'street-2_2-{}'.format(uuid4().hex)
self.msg = Bunch()
self.msg.request = Bunch()
self.msg.request.customer = Bunch()
self.msg.request.customer.id = self.cust_id
self.msg.request.customer.address = []
self.msg.request.customer.address.append(
{'street_name': self.street_name1,
'elems': [self.street_elems1_1, self.street_elems1_2],
}
)
self.msg.request.customer.address.append(
{'street_name': self.street_name2,
'elems': [self.street_elems2_1, self.street_elems2_2],
}
)
self.expr1 = 'request.customer.id.text'
self.expr2 = '*.id'
self.expr3 = 'request.customer.id'
self.expr4 = 'request.customer.address.street_name'
self.expr5 = '*.address.street_name'
self.expr6 = 'request.customer.*.street_name'
self.expr7 = 'request.customer.address.street_name[1]'
self.expr8 = 'request.customer.address.street_name'
self.expr9 = 'request.customer.address.elems'
self.expressions = [self.expr1, self.expr2, self.expr3, self.expr4,
self.expr5, self.expr6, self.expr7, self.expr8, self.expr9]
for idx, expr in enumerate(self.expressions, 1):
config = Bunch()
config.name = str(idx)
config.value = expr
self.eps.create(config.name, config, {}, False)
def init(self):
# Statistics maintenance
self.stats_maint = MaintenanceTool(self.kvdb.conn)
self.msg_ns_store = NamespaceStore()
self.elem_path_store = ElemPathStore()
self.xpath_store = XPathStore()
# Message-related config - init_msg_ns_store must come before init_xpath_store
# so the latter has access to the former's namespace map.
self.init_msg_ns_store()
self.init_elem_path_store()
self.init_xpath_store()
# Request dispatcher - matches URLs, checks security and dispatches HTTP
# requests to services.
self.request_dispatcher = RequestDispatcher(simple_io_config=self.worker_config.simple_io)
self.request_dispatcher.url_data = URLData(
deepcopy(self.worker_config.http_soap),
self.server.odb.get_url_security(self.server.cluster_id, 'channel')[0],
self.worker_config.basic_auth, self.worker_config.oauth,
self.worker_config.tech_acc, self.worker_config.wss,
self.kvdb, self.broker_client, self.server.odb, self.elem_path_store, self.xpath_store)
self.request_dispatcher.request_handler = RequestHandler(self.server)
# Create all the expected connections
self.init_sql()
self.init_ftp()
self.init_http_soap()
# All set, whoever is waiting for us, if anyone at all, can now proceed
self.is_ready = True
def setUp(self):
self.eps = ElemPathStore()
self.cust_id = uuid4().hex
self.street_name1 = "street-1-{}".format(uuid4().hex)
self.street_name2 = "street-2-{}".format(uuid4().hex)
self.street_elems1_1 = "street-1_1-{}".format(uuid4().hex)
self.street_elems1_2 = "street-1_2-{}".format(uuid4().hex)
self.street_elems2_1 = "street-2_1-{}".format(uuid4().hex)
self.street_elems2_2 = "street-2_2-{}".format(uuid4().hex)
self.msg = Bunch()
self.msg.request = Bunch()
self.msg.request.customer = Bunch()
self.msg.request.customer.id = self.cust_id
self.msg.request.customer.address = []
self.msg.request.customer.address.append(
{"street_name": self.street_name1, "elems": [self.street_elems1_1, self.street_elems1_2]}
)
self.msg.request.customer.address.append(
{"street_name": self.street_name2, "elems": [self.street_elems2_1, self.street_elems2_2]}
)
self.expr1 = "request.customer.id.text"
self.expr2 = "*.id"
self.expr3 = "request.customer.id"
self.expr4 = "request.customer.address.street_name"
self.expr5 = "*.address.street_name"
self.expr6 = "request.customer.*.street_name"
self.expr7 = "request.customer.address.street_name[1]"
self.expr8 = "request.customer.address.street_name"
self.expr9 = "request.customer.address.elems"
self.expressions = [
self.expr1,
self.expr2,
self.expr3,
self.expr4,
self.expr5,
self.expr6,
self.expr7,
self.expr8,
self.expr9,
]
for idx, expr in enumerate(self.expressions, 1):
config = Bunch()
config.name = str(idx)
config.value = expr
self.eps.create(config.name, config, {}, False)
def init(self):
# Statistics maintenance
self.stats_maint = MaintenanceTool(self.kvdb.conn)
self.msg_ns_store = NamespaceStore()
self.elem_path_store = ElemPathStore()
self.xpath_store = XPathStore()
# Message-related config - init_msg_ns_store must come before init_xpath_store
# so the latter has access to the former's namespace map.
self.init_msg_ns_store()
self.init_elem_path_store()
self.init_xpath_store()
# Request dispatcher - matches URLs, checks security and dispatches HTTP
# requests to services.
self.request_dispatcher = RequestDispatcher(
simple_io_config=self.worker_config.simple_io)
self.request_dispatcher.url_data = URLData(
deepcopy(self.worker_config.http_soap),
self.server.odb.get_url_security(self.server.cluster_id,
'channel')[0],
self.worker_config.basic_auth, self.worker_config.ntlm,
self.worker_config.oauth, self.worker_config.tech_acc,
self.worker_config.wss, self.kvdb, self.broker_client,
self.server.odb, self.elem_path_store, self.xpath_store)
self.request_dispatcher.request_handler = RequestHandler(self.server)
# Create all the expected connections
self.init_sql()
self.init_ftp()
self.init_http_soap()
# All set, whoever is waiting for us, if anyone at all, can now proceed
self.is_ready = True
class WorkerStore(BrokerMessageReceiver):
""" Each worker thread has its own configuration store. The store is assigned
to the thread's threading.local variable. All the methods assume the data's
being already validated and sanitized by one of Zato's internal services.
There are exactly two threads willing to access the data at any time
- the worker thread this store belongs to
- the background ZeroMQ thread which may wish to update the store's configuration
hence the need for employing RLocks yet there shouldn't be much contention
because configuration updates are extremaly rare when compared to regular
access by worker threads.
"""
def __init__(self, worker_config=None, server=None):
self.logger = logging.getLogger(self.__class__.__name__)
self.is_ready = False
self.worker_config = worker_config
self.server = server
self.update_lock = RLock()
self.kvdb = server.kvdb
self.broker_client = None
def init(self):
# Statistics maintenance
self.stats_maint = MaintenanceTool(self.kvdb.conn)
self.msg_ns_store = NamespaceStore()
self.elem_path_store = ElemPathStore()
self.xpath_store = XPathStore()
# Message-related config - init_msg_ns_store must come before init_xpath_store
# so the latter has access to the former's namespace map.
self.init_msg_ns_store()
self.init_elem_path_store()
self.init_xpath_store()
# Request dispatcher - matches URLs, checks security and dispatches HTTP
# requests to services.
self.request_dispatcher = RequestDispatcher(simple_io_config=self.worker_config.simple_io)
self.request_dispatcher.url_data = URLData(
deepcopy(self.worker_config.http_soap),
self.server.odb.get_url_security(self.server.cluster_id, 'channel')[0],
self.worker_config.basic_auth, self.worker_config.oauth,
self.worker_config.tech_acc, self.worker_config.wss,
self.kvdb, self.broker_client, self.server.odb, self.elem_path_store, self.xpath_store)
self.request_dispatcher.request_handler = RequestHandler(self.server)
# Create all the expected connections
self.init_sql()
self.init_ftp()
self.init_http_soap()
# All set, whoever is waiting for us, if anyone at all, can now proceed
self.is_ready = True
def filter(self, msg):
# TODO: Fix it, worker doesn't need to accept all the messages
return True
def _http_soap_wrapper_from_config(self, config, has_sec_config=True):
""" Creates a new HTTP/SOAP connection wrapper out of a configuration
dictionary.
"""
security_name = config.get('security_name')
sec_config = {'security_name':security_name, 'sec_type':None, 'username':None,
'password':None, 'password_type':None}
_sec_config = None
# This will be set to True only if the method's invoked on a server's starting up
if has_sec_config:
# It's possible that there is no security config attached at all
if security_name:
_sec_config = config
else:
if security_name:
sec_type = config.sec_type
func = getattr(self.request_dispatcher.url_data, sec_type + '_get')
_sec_config = func(security_name).config
if logger.isEnabledFor(TRACE1):
logger.log(TRACE1, 'has_sec_config:[{}], security_name:[{}], _sec_config:[{}]'.format(
has_sec_config, security_name, _sec_config))
if _sec_config:
sec_config['sec_type'] = _sec_config['sec_type']
sec_config['username'] = _sec_config['username']
sec_config['password'] = _sec_config['password']
sec_config['password_type'] = _sec_config.get('password_type')
sec_config['salt'] = _sec_config.get('salt')
wrapper_config = {'id':config.id,
'is_active':config.is_active, 'method':config.method,
'data_format':config.get('data_format'),
'name':config.name, 'transport':config.transport,
'address_host':config.host,
'address_url_path':config.url_path,
'soap_action':config.soap_action, 'soap_version':config.soap_version,
'ping_method':config.ping_method, 'pool_size':config.pool_size,}
wrapper_config.update(sec_config)
return HTTPSOAPWrapper(wrapper_config)
# ##############################################################################
def init_sql(self):
""" Initializes SQL connections, first to ODB and then any user-defined ones.
"""
# We need a store first
self.sql_pool_store = PoolStore()
# Connect to ODB
self.sql_pool_store[ZATO_ODB_POOL_NAME] = self.worker_config.odb_data
self.odb = SessionWrapper()
self.odb.init_session(ZATO_ODB_POOL_NAME, self.worker_config.odb_data, self.sql_pool_store[ZATO_ODB_POOL_NAME].pool)
# Any user-defined SQL connections left?
for pool_name in self.worker_config.out_sql:
config = self.worker_config.out_sql[pool_name]['config']
self.sql_pool_store[pool_name] = config
def init_ftp(self):
""" Initializes FTP connetions. The method replaces whatever value self.out_ftp
previously had (initially this would be a ConfigDict of connection definitions).
"""
config_list = self.worker_config.out_ftp.get_config_list()
self.worker_config.out_ftp = FTPStore()
self.worker_config.out_ftp.add_params(config_list)
def init_http_soap(self):
""" Initializes plain HTTP/SOAP connections.
"""
for transport in('soap', 'plain_http'):
config_dict = getattr(self.worker_config, 'out_' + transport)
for name in config_dict:
config = config_dict[name].config
wrapper = self._http_soap_wrapper_from_config(config)
config_dict[name].conn = wrapper
# To make the API consistent with that of SQL connection pools
config_dict[name].ping = wrapper.ping
# ##############################################################################
def init_msg_ns_store(self):
for k, v in self.worker_config.msg_ns.items():
self.msg_ns_store.create(k, v.config)
def init_xpath_store(self):
for k, v in self.worker_config.xpath.items():
self.xpath_store.create(k, v.config, self.msg_ns_store.ns_map)
def init_elem_path_store(self):
for k, v in self.worker_config.elem_path.items():
self.elem_path_store.create(k, v.config, {})
# ##############################################################################
def _update_auth(self, msg, action_name, sec_type, visit_wrapper, keys=None):
""" A common method for updating auth-related configuration.
"""
with self.update_lock:
# Channels
handler = getattr(self.request_dispatcher.url_data, 'on_broker_msg_' + action_name)
handler(msg)
for transport in('soap', 'plain_http'):
config_dict = getattr(self.worker_config, 'out_' + transport)
# Wrappers and static configuration for outgoing connections
for name in config_dict.copy_keys():
config = config_dict[name].config
wrapper = config_dict[name].conn
if config['sec_type'] == sec_type:
if keys:
visit_wrapper(wrapper, msg, keys)
else:
visit_wrapper(wrapper, msg)
def _visit_wrapper_edit(self, wrapper, msg, keys):
""" Updates a given wrapper's security configuration.
"""
if wrapper.config['security_name'] == msg['old_name']:
for key in keys:
# All's good except for 'name', the msg's 'name' is known
# as 'security_name' in wrapper's config.
if key == 'name':
key1 = 'security_name'
key2 = key
else:
key1, key2 = key, key
wrapper.config[key1] = msg[key2]
wrapper.set_auth()
def _visit_wrapper_delete(self, wrapper, msg):
""" Deletes a wrapper.
"""
config_dict = getattr(self.worker_config, 'out_' + wrapper.config['transport'])
if wrapper.config['security_name'] == msg['name']:
del config_dict[wrapper.config['name']]
def _visit_wrapper_change_password(self, wrapper, msg):
""" Changes a wrapper's password.
"""
if wrapper.config['security_name'] == msg['name']:
wrapper.config['password'] = msg['password']
wrapper.set_auth()
# ##############################################################################
def basic_auth_get(self, name):
""" Returns the configuration of the HTTP Basic Auth security definition
of the given name.
"""
self.request_dispatcher.url_data.basic_auth_get(name)
def on_broker_msg_SECURITY_BASIC_AUTH_CREATE(self, msg, *args):
""" Creates a new HTTP Basic Auth security definition
"""
self.request_dispatcher.url_data.on_broker_msg_SECURITY_BASIC_AUTH_CREATE(msg, *args)
def on_broker_msg_SECURITY_BASIC_AUTH_EDIT(self, msg, *args):
""" Updates an existing HTTP Basic Auth security definition.
"""
self._update_auth(msg, code_to_name[msg.action], security_def_type.basic_auth,
self._visit_wrapper_edit, keys=('is_active', 'username', 'name'))
def on_broker_msg_SECURITY_BASIC_AUTH_DELETE(self, msg, *args):
""" Deletes an HTTP Basic Auth security definition.
"""
self._update_auth(msg, code_to_name[msg.action], security_def_type.basic_auth,
self._visit_wrapper_delete)
def on_broker_msg_SECURITY_BASIC_AUTH_CHANGE_PASSWORD(self, msg, *args):
""" Changes password of an HTTP Basic Auth security definition.
"""
self._update_auth(msg, code_to_name[msg.action], security_def_type.basic_auth,
self._visit_wrapper_change_password)
# ##############################################################################
def oauth_get(self, name):
""" Returns the configuration of the OAuth security definition
of the given name.
"""
self.request_dispatcher.url_data.oauth_get(name)
def on_broker_msg_SECURITY_OAUTH_CREATE(self, msg, *args):
""" Creates a new OAuth security definition
"""
self.request_dispatcher.url_data.on_broker_msg_SECURITY_OAUTH_CREATE(msg, *args)
def on_broker_msg_SECURITY_OAUTH_EDIT(self, msg, *args):
""" Updates an existing OAuth security definition.
"""
self._update_auth(msg, code_to_name[msg.action], security_def_type.oauth,
self._visit_wrapper_edit, keys=('is_active', 'username', 'name'))
def on_broker_msg_SECURITY_OAUTH_DELETE(self, msg, *args):
""" Deletes an OAuth security definition.
"""
self._update_auth(msg, code_to_name[msg.action], security_def_type.oauth,
self._visit_wrapper_delete)
def on_broker_msg_SECURITY_OAUTH_CHANGE_PASSWORD(self, msg, *args):
""" Changes password of an OAuth security definition.
"""
self._update_auth(msg, code_to_name[msg.action], security_def_type.oauth,
self._visit_wrapper_change_password)
# ##############################################################################
def tech_acc_get(self, name):
""" Returns the configuration of the technical account of the given name.
"""
self.request_dispatcher.url_data.tech_acc_get(name)
def on_broker_msg_SECURITY_TECH_ACC_CREATE(self, msg, *args):
""" Creates a new technical account.
"""
self.request_dispatcher.url_data.on_broker_msg_SECURITY_TECH_ACC_CREATE(msg, *args)
def on_broker_msg_SECURITY_TECH_ACC_EDIT(self, msg, *args):
""" Updates an existing technical account.
"""
self.request_dispatcher.url_data.on_broker_msg_SECURITY_TECH_ACC_EDIT(msg, *args)
def on_broker_msg_SECURITY_TECH_ACC_DELETE(self, msg, *args):
""" Deletes a technical account.
"""
self.request_dispatcher.url_data.on_broker_msg_SECURITY_TECH_ACC_DELETE(msg, *args)
def on_broker_msg_SECURITY_TECH_ACC_CHANGE_PASSWORD(self, msg, *args):
""" Changes the password of a technical account.
"""
self.request_dispatcher.url_data.on_broker_msg_SECURITY_TECH_ACC_CHANGE_PASSWORD(msg, *args)
# ##############################################################################
def wss_get(self, name):
""" Returns the configuration of the WSS definition of the given name.
"""
self.request_dispatcher.url_data.wss_get(name)
def on_broker_msg_SECURITY_WSS_CREATE(self, msg, *args):
""" Creates a new WS-Security definition.
"""
self.request_dispatcher.url_data.on_broker_msg_SECURITY_WSS_CREATE(msg, *args)
def on_broker_msg_SECURITY_WSS_EDIT(self, msg, *args):
""" Updates an existing WS-Security definition.
"""
self._update_auth(msg, code_to_name[msg.action], security_def_type.wss,
self._visit_wrapper_edit, keys=('is_active', 'username', 'name',
'nonce_freshness_time', 'reject_expiry_limit', 'password_type',
'reject_empty_nonce_creat', 'reject_stale_tokens'))
def on_broker_msg_SECURITY_WSS_DELETE(self, msg, *args):
""" Deletes a WS-Security definition.
"""
self._update_auth(msg, code_to_name[msg.action], security_def_type.wss,
self._visit_wrapper_delete)
def on_broker_msg_SECURITY_WSS_CHANGE_PASSWORD(self, msg, *args):
""" Changes the password of a WS-Security definition.
"""
self._update_auth(msg, code_to_name[msg.action], security_def_type.wss,
self._visit_wrapper_change_password)
# ##############################################################################
def _set_service_response_data(self, service, **ignored):
if not isinstance(service.response.payload, basestring):
service.response.payload = service.response.payload.getvalue()
def _on_message_invoke_service(self, msg, channel, action, args=None):
""" Triggered by external processes, such as AMQP or the singleton's scheduler,
creates a new service instance and invokes it.
"""
# WSGI environment is the best place we have to store raw msg in
wsgi_environ = {'zato.request_ctx.async_msg':msg}
service = self.server.service_store.new_instance_by_name(msg.service)
service.update_handle(self._set_service_response_data, service, msg.payload,
channel, msg.get('data_format'), msg.get('transport'), self.server,
self.broker_client, self, msg.cid, self.worker_config.simple_io,
job_type=msg.get('job_type'), wsgi_environ=wsgi_environ)
# ##############################################################################
def on_broker_msg_SCHEDULER_JOB_EXECUTED(self, msg, args=None):
return self._on_message_invoke_service(msg, CHANNEL.SCHEDULER, 'SCHEDULER_JOB_EXECUTED', args)
def on_broker_msg_CHANNEL_AMQP_MESSAGE_RECEIVED(self, msg, args=None):
return self._on_message_invoke_service(msg, CHANNEL.AMQP, 'CHANNEL_AMQP_MESSAGE_RECEIVED', args)
def on_broker_msg_CHANNEL_JMS_WMQ_MESSAGE_RECEIVED(self, msg, args=None):
return self._on_message_invoke_service(msg, CHANNEL.JMS_WMQ, 'CHANNEL_JMS_WMQ_MESSAGE_RECEIVED', args)
def on_broker_msg_CHANNEL_ZMQ_MESSAGE_RECEIVED(self, msg, args=None):
return self._on_message_invoke_service(msg, CHANNEL.ZMQ, 'CHANNEL_ZMQ_MESSAGE_RECEIVED', args)
# ##############################################################################
def on_broker_msg_OUTGOING_SQL_CREATE_EDIT(self, msg, *args):
""" Creates or updates an SQL connection, including changing its
password.
"""
# Is it a rename? If so, delete the connection first
if msg.get('old_name') and msg.get('old_name') != msg['name']:
del self.sql_pool_store[msg['old_name']]
self.sql_pool_store[msg['name']] = msg
def on_broker_msg_OUTGOING_SQL_CHANGE_PASSWORD(self, msg, *args):
""" Deletes an outgoing SQL connection pool and recreates it using the
new password.
"""
self.sql_pool_store.change_password(msg['name'], msg['password'])
def on_broker_msg_OUTGOING_SQL_DELETE(self, msg, *args):
""" Deletes an outgoing SQL connection pool.
"""
del self.sql_pool_store[msg['name']]
# ##############################################################################
def on_broker_msg_CHANNEL_HTTP_SOAP_CREATE_EDIT(self, msg, *args):
""" Creates or updates an HTTP/SOAP channel.
"""
self.request_dispatcher.url_data.on_broker_msg_CHANNEL_HTTP_SOAP_CREATE_EDIT(msg, *args)
def on_broker_msg_CHANNEL_HTTP_SOAP_DELETE(self, msg, *args):
""" Deletes an HTTP/SOAP channel.
"""
self.request_dispatcher.url_data.on_broker_msg_CHANNEL_HTTP_SOAP_DELETE(msg, *args)
# ##############################################################################
def _delete_outgoing_http_soap(self, name, transport, log_func):
""" Actually deletes an outgoing HTTP/SOAP connection.
"""
# Are we dealing with plain HTTP or SOAP?
config_dict = getattr(self.worker_config, 'out_' + transport)
# Delete the connection first, if it exists at all ..
try:
try:
wrapper = config_dict[name].conn
except (KeyError, AttributeError), e:
log_func('Could not access a wrapper, e:[{}]'.format(format_exc(e)))
else:
class TestElemPathStore(TestCase):
def setUp(self):
self.eps = ElemPathStore()
self.cust_id = uuid4().hex
self.street_name1 = 'street-1-{}'.format(uuid4().hex)
self.street_name2 = 'street-2-{}'.format(uuid4().hex)
self.street_elems1_1 = 'street-1_1-{}'.format(uuid4().hex)
self.street_elems1_2 = 'street-1_2-{}'.format(uuid4().hex)
self.street_elems2_1 = 'street-2_1-{}'.format(uuid4().hex)
self.street_elems2_2 = 'street-2_2-{}'.format(uuid4().hex)
self.msg = Bunch()
self.msg.request = Bunch()
self.msg.request.customer = Bunch()
self.msg.request.customer.id = self.cust_id
self.msg.request.customer.address = []
self.msg.request.customer.address.append(
{'street_name': self.street_name1,
'elems': [self.street_elems1_1, self.street_elems1_2],
}
)
self.msg.request.customer.address.append(
{'street_name': self.street_name2,
'elems': [self.street_elems2_1, self.street_elems2_2],
}
)
self.expr1 = 'request.customer.id.text'
self.expr2 = '*.id'
self.expr3 = 'request.customer.id'
self.expr4 = 'request.customer.address.street_name'
self.expr5 = '*.address.street_name'
self.expr6 = 'request.customer.*.street_name'
self.expr7 = 'request.customer.address.street_name[1]'
self.expr8 = 'request.customer.address.street_name'
self.expr9 = 'request.customer.address.elems'
self.expressions = [self.expr1, self.expr2, self.expr3, self.expr4,
self.expr5, self.expr6, self.expr7, self.expr8, self.expr9]
for idx, expr in enumerate(self.expressions, 1):
config = Bunch()
config.name = str(idx)
config.value = expr
self.eps.create(config.name, config, {}, False)
def test_invoke(self):
expected = {
'1': self.cust_id,
'2': [self.cust_id],
'3': [self.cust_id],
'4': [self.street_name1, self.street_name2],
'5': [self.street_name1, self.street_name2],
'6': [self.street_name1, self.street_name2],
'7': [self.street_name1, self.street_name2],
'8': [self.street_name1, self.street_name2],
'9': [self.street_elems1_1, self.street_elems1_2,
self.street_elems2_1, self.street_elems2_2],
}
for idx, expr in enumerate(self.expressions, 1):
name = str(idx)
result = self.eps.invoke(self.msg, name)
self.assertEquals(expected[name], result)
def test_conversion_roundtrip(self):
xml = self.eps.convert_dict_to_xml(self.msg)
msg = self.eps.convert_xml_to_dict(xml)
self.assertEquals(msg, self.msg)
def test_replace(self):
for idx, expr in enumerate(self.expressions, 1):
msg = deepcopy(self.msg)
new_value = uuid4().hex
name = str(idx)
replaced = self.eps.replace(msg, name, new_value)
result = self.eps.invoke(replaced, name)
if isinstance(result, basestring):
self.assertEquals(result, new_value)
else:
for item in result:
self.assertEquals(item, new_value)
class WorkerStore(BrokerMessageReceiver):
""" Each worker thread has its own configuration store. The store is assigned
to the thread's threading.local variable. All the methods assume the data's
being already validated and sanitized by one of Zato's internal services.
There are exactly two threads willing to access the data at any time
- the worker thread this store belongs to
- the background ZeroMQ thread which may wish to update the store's configuration
hence the need for employing RLocks yet there shouldn't be much contention
because configuration updates are extremaly rare when compared to regular
access by worker threads.
"""
def __init__(self, worker_config=None, server=None):
self.logger = logging.getLogger(self.__class__.__name__)
self.is_ready = False
self.worker_config = worker_config
self.server = server
self.update_lock = RLock()
self.kvdb = server.kvdb
self.broker_client = None
self.pubsub = None
""":type: zato.common.pubsub.PubSubAPI"""
def init(self):
# Statistics maintenance
self.stats_maint = MaintenanceTool(self.kvdb.conn)
self.msg_ns_store = NamespaceStore()
self.elem_path_store = ElemPathStore()
self.xpath_store = XPathStore()
# Message-related config - init_msg_ns_store must come before init_xpath_store
# so the latter has access to the former's namespace map.
self.init_msg_ns_store()
self.init_elem_path_store()
self.init_xpath_store()
# Request dispatcher - matches URLs, checks security and dispatches HTTP
# requests to services.
self.request_dispatcher = RequestDispatcher(simple_io_config=self.worker_config.simple_io)
self.request_dispatcher.url_data = URLData(
deepcopy(self.worker_config.http_soap),
self.server.odb.get_url_security(self.server.cluster_id, 'channel')[0],
self.worker_config.basic_auth, self.worker_config.ntlm, self.worker_config.oauth, self.worker_config.tech_acc,
self.worker_config.wss, self.worker_config.apikey, self.worker_config.aws, self.worker_config.openstack_security,
self.worker_config.xpath_sec, self.kvdb, self.broker_client, self.server.odb, self.elem_path_store, self.xpath_store)
self.request_dispatcher.request_handler = RequestHandler(self.server)
# Create all the expected connections and objects
self.init_sql()
self.init_ftp()
self.init_http_soap()
self.init_cloud()
self.init_pubsub()
self.init_notifiers()
# All set, whoever is waiting for us, if anyone at all, can now proceed
self.is_ready = True
def filter(self, msg):
# TODO: Fix it, worker doesn't need to accept all the messages
return True
def _update_aws_config(self, msg):
""" Parses the address to AWS we store into discrete components S3Connection objects expect.
Also turns metadata string into a dictionary
"""
url_info = urlparse(msg.address)
msg.is_secure = True if url_info.scheme == 'https' else False
msg.port = url_info.port if url_info.port else (443 if msg.is_secure else 80)
msg.host = url_info.netloc
msg.metadata = parse_extra_into_dict(msg.metadata_)
def _http_soap_wrapper_from_config(self, config, has_sec_config=True):
""" Creates a new HTTP/SOAP connection wrapper out of a configuration
dictionary.
"""
security_name = config.get('security_name')
sec_config = {'security_name':security_name, 'sec_type':None, 'username':None, 'password':None, 'password_type':None}
_sec_config = None
# This will be set to True only if the method's invoked on a server's starting up
if has_sec_config:
# It's possible that there is no security config attached at all
if security_name:
_sec_config = config
else:
if security_name:
sec_type = config.sec_type
func = getattr(self.request_dispatcher.url_data, sec_type + '_get')
_sec_config = func(security_name).config
if logger.isEnabledFor(TRACE1):
logger.log(TRACE1, 'has_sec_config:[{}], security_name:[{}], _sec_config:[{}]'.format(
has_sec_config, security_name, _sec_config))
if _sec_config:
sec_config['sec_type'] = _sec_config['sec_type']
sec_config['username'] = _sec_config['username']
sec_config['password'] = _sec_config['password']
sec_config['password_type'] = _sec_config.get('password_type')
sec_config['salt'] = _sec_config.get('salt')
wrapper_config = {'id':config.id,
'is_active':config.is_active, 'method':config.method,
'data_format':config.get('data_format'),
'name':config.name, 'transport':config.transport,
'address_host':config.host,
'address_url_path':config.url_path,
'soap_action':config.soap_action, 'soap_version':config.soap_version, 'ping_method':config.ping_method,
'pool_size':config.pool_size, 'serialization_type':config.serialization_type,
'timeout':config.timeout}
wrapper_config.update(sec_config)
if wrapper_config['serialization_type'] == HTTP_SOAP_SERIALIZATION_TYPE.SUDS.id:
wrapper_config['queue_build_cap'] = float(self.server.fs_server_config.misc.queue_build_cap)
wrapper = SudsSOAPWrapper(wrapper_config)
wrapper.build_client_queue()
return wrapper
return HTTPSOAPWrapper(wrapper_config)
# ################################################################################################################################
def init_sql(self):
""" Initializes SQL connections, first to ODB and then any user-defined ones.
"""
# We need a store first
self.sql_pool_store = PoolStore()
# Connect to ODB
self.sql_pool_store[ZATO_ODB_POOL_NAME] = self.worker_config.odb_data
self.odb = SessionWrapper()
self.odb.init_session(ZATO_ODB_POOL_NAME, self.worker_config.odb_data, self.sql_pool_store[ZATO_ODB_POOL_NAME].pool)
# Any user-defined SQL connections left?
for pool_name in self.worker_config.out_sql:
config = self.worker_config.out_sql[pool_name]['config']
self.sql_pool_store[pool_name] = config
def init_ftp(self):
""" Initializes FTP connetions. The method replaces whatever value self.out_ftp
previously had (initially this would be a ConfigDict of connection definitions).
"""
config_list = self.worker_config.out_ftp.get_config_list()
self.worker_config.out_ftp = FTPStore()
self.worker_config.out_ftp.add_params(config_list)
def init_http_soap(self):
""" Initializes plain HTTP/SOAP connections.
"""
for transport in('soap', 'plain_http'):
config_dict = getattr(self.worker_config, 'out_' + transport)
for name in config_dict:
config = config_dict[name].config
wrapper = self._http_soap_wrapper_from_config(config)
config_dict[name].conn = wrapper
# To make the API consistent with that of SQL connection pools
config_dict[name].ping = wrapper.ping
def init_cloud(self):
""" Initializes all the cloud connections.
"""
data = (
('cloud_openstack_swift', SwiftWrapper),
('cloud_aws_s3', S3Wrapper),
)
for config_key, wrapper in data:
config_attr = getattr(self.worker_config, config_key)
for name in config_attr:
config = config_attr[name]['config']
if isinstance(wrapper, S3Wrapper):
self._update_aws_config(config)
config.queue_build_cap = float(self.server.fs_server_config.misc.queue_build_cap)
config_attr[name].conn = wrapper(config)
config_attr[name].conn.build_queue()
def _update_cloud_openstack_swift_container(self, config_dict):
""" Makes sure OpenStack Swift containers always have a path to prefix queries with.
"""
config_dict.containers = [elem.split(':') for elem in config_dict.containers.splitlines()]
for item in config_dict.containers:
# No path specified so we use an empty string to catch everything.
if len(item) == 1:
item.append('')
item.append('{}:{}'.format(item[0], item[1]))
def init_notifiers(self):
for config_dict in self.worker_config.notif_cloud_openstack_swift.values():
self._update_cloud_openstack_swift_container(config_dict.config)
# ################################################################################################################################
def _topic_from_topic_data(self, data):
return Topic(data.name, data.is_active, True, data.max_depth)
def _add_pubsub_topic(self, data):
self.pubsub.add_topic(self._topic_from_topic_data(data))
def init_pubsub(self):
""" Initializes publish/subscribe mechanisms.
"""
self.pubsub.set_default_consumer(self.worker_config.pubsub.default_consumer)
self.pubsub.set_default_producer(self.worker_config.pubsub.default_producer)
for topic_name, topic_data in self.worker_config.pubsub.topics.items():
self._add_pubsub_topic(topic_data.config)
for list_value in self.worker_config.pubsub.producers.values():
for config in list_value:
self.pubsub.add_producer(Client(config.client_id, config.name, config.is_active), Topic(config.topic_name))
for list_value in self.worker_config.pubsub.consumers.values():
for config in list_value:
callback_type = PUB_SUB.CALLBACK_TYPE.OUTCONN_SOAP if bool(config.soap_version) else \
PUB_SUB.CALLBACK_TYPE.OUTCONN_PLAIN_HTP
self.pubsub.add_consumer(
Consumer(
config.client_id, config.name, config.is_active, config.sub_key, config.max_backlog,
config.delivery_mode, config.callback_id, config.callback_name, callback_type),
Topic(config.topic_name))
# ################################################################################################################################
def init_msg_ns_store(self):
for k, v in self.worker_config.msg_ns.items():
self.msg_ns_store.create(k, v.config)
def init_xpath_store(self):
for k, v in self.worker_config.xpath.items():
self.xpath_store.create(k, v.config, self.msg_ns_store.ns_map)
def init_elem_path_store(self):
for k, v in self.worker_config.elem_path.items():
self.elem_path_store.create(k, v.config, {})
# ################################################################################################################################
def _update_auth(self, msg, action_name, sec_type, visit_wrapper, keys=None):
""" A common method for updating auth-related configuration.
"""
with self.update_lock:
# Channels
handler = getattr(self.request_dispatcher.url_data, 'on_broker_msg_' + action_name)
handler(msg)
for transport in('soap', 'plain_http'):
config_dict = getattr(self.worker_config, 'out_' + transport)
# Wrappers and static configuration for outgoing connections
for name in config_dict.copy_keys():
config = config_dict[name].config
wrapper = config_dict[name].conn
if config['sec_type'] == sec_type:
if keys:
visit_wrapper(wrapper, msg, keys)
else:
visit_wrapper(wrapper, msg)
def _visit_wrapper_edit(self, wrapper, msg, keys):
""" Updates a given wrapper's security configuration.
"""
if wrapper.config['security_name'] == msg['old_name']:
for key in keys:
# All's good except for 'name', the msg's 'name' is known
# as 'security_name' in wrapper's config.
if key == 'name':
key1 = 'security_name'
key2 = key
else:
key1, key2 = key, key
wrapper.config[key1] = msg[key2]
wrapper.set_auth()
def _visit_wrapper_delete(self, wrapper, msg):
""" Deletes a wrapper.
"""
config_dict = getattr(self.worker_config, 'out_' + wrapper.config['transport'])
if wrapper.config['security_name'] == msg['name']:
del config_dict[wrapper.config['name']]
def _visit_wrapper_change_password(self, wrapper, msg):
""" Changes a wrapper's password.
"""
if wrapper.config['security_name'] == msg['name']:
wrapper.config['password'] = msg['password']
wrapper.set_auth()
# ################################################################################################################################
def apikey_get(self, name):
""" Returns the configuration of the API key of the given name.
"""
self.request_dispatcher.url_data.apikey_get(name)
def on_broker_msg_SECURITY_APIKEY_CREATE(self, msg, *args):
""" Creates a new API key security definition.
"""
self.request_dispatcher.url_data.on_broker_msg_SECURITY_APIKEY_CREATE(msg, *args)
def on_broker_msg_SECURITY_APIKEY_EDIT(self, msg, *args):
""" Updates an existing API key security definition.
"""
self._update_auth(msg, code_to_name[msg.action], SEC_DEF_TYPE.APIKEY,
self._visit_wrapper_edit, keys=('is_active', 'username', 'name'))
def on_broker_msg_SECURITY_APIKEY_DELETE(self, msg, *args):
""" Deletes an API key security definition.
"""
self._update_auth(msg, code_to_name[msg.action], SEC_DEF_TYPE.APIKEY,
self._visit_wrapper_delete)
def on_broker_msg_SECURITY_APIKEY_CHANGE_PASSWORD(self, msg, *args):
""" Changes password of an API key security definition.
"""
self._update_auth(msg, code_to_name[msg.action], SEC_DEF_TYPE.APIKEY,
self._visit_wrapper_change_password)
# ################################################################################################################################
def aws_get(self, name):
""" Returns the configuration of the AWS security definition
of the given name.
"""
self.request_dispatcher.url_data.aws_get(name)
def on_broker_msg_SECURITY_AWS_CREATE(self, msg, *args):
""" Creates a new AWS security definition
"""
self.request_dispatcher.url_data.on_broker_msg_SECURITY_AWS_CREATE(msg, *args)
def on_broker_msg_SECURITY_AWS_EDIT(self, msg, *args):
""" Updates an existing AWS security definition.
"""
self._update_auth(msg, code_to_name[msg.action], SEC_DEF_TYPE.AWS,
self._visit_wrapper_edit, keys=('is_active', 'username', 'name'))
def on_broker_msg_SECURITY_AWS_DELETE(self, msg, *args):
""" Deletes an AWS security definition.
"""
self._update_auth(msg, code_to_name[msg.action], SEC_DEF_TYPE.AWS,
self._visit_wrapper_delete)
def on_broker_msg_SECURITY_AWS_CHANGE_PASSWORD(self, msg, *args):
""" Changes password of an AWS security definition.
"""
self._update_auth(msg, code_to_name[msg.action], SEC_DEF_TYPE.AWS,
self._visit_wrapper_change_password)
# ################################################################################################################################
def openstack_get(self, name):
""" Returns the configuration of the OpenStack security definition
of the given name.
"""
self.request_dispatcher.url_data.openstack_get(name)
def on_broker_msg_SECURITY_OPENSTACK_CREATE(self, msg, *args):
""" Creates a new OpenStack security definition
"""
self.request_dispatcher.url_data.on_broker_msg_SECURITY_OPENSTACK_CREATE(msg, *args)
def on_broker_msg_SECURITY_OPENSTACK_EDIT(self, msg, *args):
""" Updates an existing OpenStack security definition.
"""
self._update_auth(msg, code_to_name[msg.action], SEC_DEF_TYPE.OPENSTACK,
self._visit_wrapper_edit, keys=('is_active', 'username', 'name'))
def on_broker_msg_SECURITY_OPENSTACK_DELETE(self, msg, *args):
""" Deletes an OpenStack security definition.
"""
self._update_auth(msg, code_to_name[msg.action], SEC_DEF_TYPE.OPENSTACK,
self._visit_wrapper_delete)
def on_broker_msg_SECURITY_OPENSTACK_CHANGE_PASSWORD(self, msg, *args):
""" Changes password of an OpenStack security definition.
"""
self._update_auth(msg, code_to_name[msg.action], SEC_DEF_TYPE.OPENSTACK,
self._visit_wrapper_change_password)
# ################################################################################################################################
def ntlm_get(self, name):
""" Returns the configuration of the NTLM security definition
of the given name.
"""
self.request_dispatcher.url_data.ntlm_get(name)
def on_broker_msg_SECURITY_NTLM_CREATE(self, msg, *args):
""" Creates a new NTLM security definition
"""
self.request_dispatcher.url_data.on_broker_msg_SECURITY_NTLM_CREATE(msg, *args)
def on_broker_msg_SECURITY_NTLM_EDIT(self, msg, *args):
""" Updates an existing NTLM security definition.
"""
self._update_auth(msg, code_to_name[msg.action], SEC_DEF_TYPE.NTLM,
self._visit_wrapper_edit, keys=('is_active', 'username', 'name'))
def on_broker_msg_SECURITY_NTLM_DELETE(self, msg, *args):
""" Deletes an NTLM security definition.
"""
self._update_auth(msg, code_to_name[msg.action], SEC_DEF_TYPE.NTLM,
self._visit_wrapper_delete)
def on_broker_msg_SECURITY_NTLM_CHANGE_PASSWORD(self, msg, *args):
""" Changes password of an NTLM security definition.
"""
self._update_auth(msg, code_to_name[msg.action], SEC_DEF_TYPE.NTLM,
self._visit_wrapper_change_password)
# ################################################################################################################################
def basic_auth_get(self, name):
""" Returns the configuration of the HTTP Basic Auth security definition
of the given name.
"""
self.request_dispatcher.url_data.basic_auth_get(name)
def on_broker_msg_SECURITY_BASIC_AUTH_CREATE(self, msg, *args):
""" Creates a new HTTP Basic Auth security definition
"""
self.request_dispatcher.url_data.on_broker_msg_SECURITY_BASIC_AUTH_CREATE(msg, *args)
def on_broker_msg_SECURITY_BASIC_AUTH_EDIT(self, msg, *args):
""" Updates an existing HTTP Basic Auth security definition.
"""
self._update_auth(msg, code_to_name[msg.action], SEC_DEF_TYPE.BASIC_AUTH,
self._visit_wrapper_edit, keys=('is_active', 'username', 'name'))
def on_broker_msg_SECURITY_BASIC_AUTH_DELETE(self, msg, *args):
""" Deletes an HTTP Basic Auth security definition.
"""
self._update_auth(msg, code_to_name[msg.action], SEC_DEF_TYPE.BASIC_AUTH,
self._visit_wrapper_delete)
def on_broker_msg_SECURITY_BASIC_AUTH_CHANGE_PASSWORD(self, msg, *args):
""" Changes password of an HTTP Basic Auth security definition.
"""
self._update_auth(msg, code_to_name[msg.action], SEC_DEF_TYPE.BASIC_AUTH,
self._visit_wrapper_change_password)
# ################################################################################################################################
def oauth_get(self, name):
""" Returns the configuration of the OAuth security definition
of the given name.
"""
self.request_dispatcher.url_data.oauth_get(name)
def on_broker_msg_SECURITY_OAUTH_CREATE(self, msg, *args):
""" Creates a new OAuth security definition
"""
self.request_dispatcher.url_data.on_broker_msg_SECURITY_OAUTH_CREATE(msg, *args)
def on_broker_msg_SECURITY_OAUTH_EDIT(self, msg, *args):
""" Updates an existing OAuth security definition.
"""
self._update_auth(msg, code_to_name[msg.action], SEC_DEF_TYPE.OAUTH,
self._visit_wrapper_edit, keys=('is_active', 'username', 'name'))
def on_broker_msg_SECURITY_OAUTH_DELETE(self, msg, *args):
""" Deletes an OAuth security definition.
"""
self._update_auth(msg, code_to_name[msg.action], SEC_DEF_TYPE.OAUTH,
self._visit_wrapper_delete)
def on_broker_msg_SECURITY_OAUTH_CHANGE_PASSWORD(self, msg, *args):
""" Changes password of an OAuth security definition.
"""
self._update_auth(msg, code_to_name[msg.action], SEC_DEF_TYPE.OAUTH,
self._visit_wrapper_change_password)
# ################################################################################################################################
def tech_acc_get(self, name):
""" Returns the configuration of the technical account of the given name.
"""
self.request_dispatcher.url_data.tech_acc_get(name)
def on_broker_msg_SECURITY_TECH_ACC_CREATE(self, msg, *args):
""" Creates a new technical account.
"""
self.request_dispatcher.url_data.on_broker_msg_SECURITY_TECH_ACC_CREATE(msg, *args)
def on_broker_msg_SECURITY_TECH_ACC_EDIT(self, msg, *args):
""" Updates an existing technical account.
"""
self.request_dispatcher.url_data.on_broker_msg_SECURITY_TECH_ACC_EDIT(msg, *args)
def on_broker_msg_SECURITY_TECH_ACC_DELETE(self, msg, *args):
""" Deletes a technical account.
"""
self.request_dispatcher.url_data.on_broker_msg_SECURITY_TECH_ACC_DELETE(msg, *args)
def on_broker_msg_SECURITY_TECH_ACC_CHANGE_PASSWORD(self, msg, *args):
""" Changes the password of a technical account.
"""
self.request_dispatcher.url_data.on_broker_msg_SECURITY_TECH_ACC_CHANGE_PASSWORD(msg, *args)
# ################################################################################################################################
def wss_get(self, name):
""" Returns the configuration of the WSS definition of the given name.
"""
self.request_dispatcher.url_data.wss_get(name)
def on_broker_msg_SECURITY_WSS_CREATE(self, msg, *args):
""" Creates a new WS-Security definition.
"""
self.request_dispatcher.url_data.on_broker_msg_SECURITY_WSS_CREATE(msg, *args)
def on_broker_msg_SECURITY_WSS_EDIT(self, msg, *args):
""" Updates an existing WS-Security definition.
"""
self._update_auth(msg, code_to_name[msg.action], SEC_DEF_TYPE.WSS,
self._visit_wrapper_edit, keys=('is_active', 'username', 'name',
'nonce_freshness_time', 'reject_expiry_limit', 'password_type',
'reject_empty_nonce_creat', 'reject_stale_tokens'))
def on_broker_msg_SECURITY_WSS_DELETE(self, msg, *args):
""" Deletes a WS-Security definition.
"""
self._update_auth(msg, code_to_name[msg.action], SEC_DEF_TYPE.WSS,
self._visit_wrapper_delete)
def on_broker_msg_SECURITY_WSS_CHANGE_PASSWORD(self, msg, *args):
""" Changes the password of a WS-Security definition.
"""
self._update_auth(msg, code_to_name[msg.action], SEC_DEF_TYPE.WSS,
self._visit_wrapper_change_password)
# ################################################################################################################################
def xpath_sec_get(self, name):
""" Returns the configuration of an XPath security definition of the given name.
"""
self.request_dispatcher.url_data.xpath_sec_get(name)
def on_broker_msg_SECURITY_XPATH_SEC_CREATE(self, msg, *args):
""" Creates a new XPath security definition
"""
self.request_dispatcher.url_data.on_broker_msg_SECURITY_XPATH_SEC_CREATE(msg, *args)
def on_broker_msg_SECURITY_XPATH_SEC_EDIT(self, msg, *args):
""" Updates an existing XPath security definition.
"""
self._update_auth(msg, code_to_name[msg.action], SEC_DEF_TYPE.XPATH_SEC,
self._visit_wrapper_edit, keys=('is_active', 'username', 'name'))
def on_broker_msg_SECURITY_XPATH_SEC_DELETE(self, msg, *args):
""" Deletes an XPath security definition.
"""
self._update_auth(msg, code_to_name[msg.action], SEC_DEF_TYPE.XPATH_SEC,
self._visit_wrapper_delete)
def on_broker_msg_SECURITY_XPATH_SEC_CHANGE_PASSWORD(self, msg, *args):
""" Changes password of an XPath security definition.
"""
self._update_auth(msg, code_to_name[msg.action], SEC_DEF_TYPE.XPATH_SEC,
self._visit_wrapper_change_password)
# ################################################################################################################################
def _set_service_response_data(self, service, **ignored):
if not isinstance(service.response.payload, basestring):
service.response.payload = service.response.payload.getvalue()
def _on_message_invoke_service(self, msg, channel, action, args=None):
""" Triggered by external processes, such as AMQP or the singleton's scheduler,
creates a new service instance and invokes it.
"""
# WSGI environment is the best place we have to store raw msg in
wsgi_environ = {'zato.request_ctx.async_msg':msg}
service = self.server.service_store.new_instance_by_name(msg['service'])
service.update_handle(self._set_service_response_data, service, msg['payload'],
channel, msg.get('data_format'), msg.get('transport'), self.server,
self.broker_client, self, msg['cid'], self.worker_config.simple_io,
job_type=msg.get('job_type'), wsgi_environ=wsgi_environ)
# ################################################################################################################################
def on_broker_msg_SCHEDULER_JOB_EXECUTED(self, msg, args=None):
return self._on_message_invoke_service(msg, CHANNEL.SCHEDULER, 'SCHEDULER_JOB_EXECUTED', args)
def on_broker_msg_CHANNEL_AMQP_MESSAGE_RECEIVED(self, msg, args=None):
return self._on_message_invoke_service(msg, CHANNEL.AMQP, 'CHANNEL_AMQP_MESSAGE_RECEIVED', args)
def on_broker_msg_CHANNEL_JMS_WMQ_MESSAGE_RECEIVED(self, msg, args=None):
return self._on_message_invoke_service(msg, CHANNEL.JMS_WMQ, 'CHANNEL_JMS_WMQ_MESSAGE_RECEIVED', args)
def on_broker_msg_CHANNEL_ZMQ_MESSAGE_RECEIVED(self, msg, args=None):
return self._on_message_invoke_service(msg, CHANNEL.ZMQ, 'CHANNEL_ZMQ_MESSAGE_RECEIVED', args)
# ################################################################################################################################
def on_broker_msg_OUTGOING_SQL_CREATE_EDIT(self, msg, *args):
""" Creates or updates an SQL connection, including changing its
password.
"""
# Is it a rename? If so, delete the connection first
if msg.get('old_name') and msg.get('old_name') != msg['name']:
del self.sql_pool_store[msg['old_name']]
self.sql_pool_store[msg['name']] = msg
def on_broker_msg_OUTGOING_SQL_CHANGE_PASSWORD(self, msg, *args):
""" Deletes an outgoing SQL connection pool and recreates it using the
new password.
"""
self.sql_pool_store.change_password(msg['name'], msg['password'])
def on_broker_msg_OUTGOING_SQL_DELETE(self, msg, *args):
""" Deletes an outgoing SQL connection pool.
"""
del self.sql_pool_store[msg['name']]
# ################################################################################################################################
def on_broker_msg_CHANNEL_HTTP_SOAP_CREATE_EDIT(self, msg, *args):
""" Creates or updates an HTTP/SOAP channel.
"""
self.request_dispatcher.url_data.on_broker_msg_CHANNEL_HTTP_SOAP_CREATE_EDIT(msg, *args)
def on_broker_msg_CHANNEL_HTTP_SOAP_DELETE(self, msg, *args):
""" Deletes an HTTP/SOAP channel.
"""
self.request_dispatcher.url_data.on_broker_msg_CHANNEL_HTTP_SOAP_DELETE(msg, *args)
# ################################################################################################################################
def _delete_config_close_wrapper(self, name, config_dict, conn_type, log_func):
""" Deletes a wrapper-based connection's config and closes its underlying wrapper.
"""
# Delete the connection first, if it exists at all ..
try:
try:
wrapper = config_dict[name].conn
except (KeyError, AttributeError), e:
log_func('Could not access wrapper, e:[{}]'.format(format_exc(e)))
else:
class TestElemPathStore(TestCase):
def setUp(self):
self.eps = ElemPathStore()
self.cust_id = uuid4().hex
self.street_name1 = "street-1-{}".format(uuid4().hex)
self.street_name2 = "street-2-{}".format(uuid4().hex)
self.street_elems1_1 = "street-1_1-{}".format(uuid4().hex)
self.street_elems1_2 = "street-1_2-{}".format(uuid4().hex)
self.street_elems2_1 = "street-2_1-{}".format(uuid4().hex)
self.street_elems2_2 = "street-2_2-{}".format(uuid4().hex)
self.msg = Bunch()
self.msg.request = Bunch()
self.msg.request.customer = Bunch()
self.msg.request.customer.id = self.cust_id
self.msg.request.customer.address = []
self.msg.request.customer.address.append(
{"street_name": self.street_name1, "elems": [self.street_elems1_1, self.street_elems1_2]}
)
self.msg.request.customer.address.append(
{"street_name": self.street_name2, "elems": [self.street_elems2_1, self.street_elems2_2]}
)
self.expr1 = "request.customer.id.text"
self.expr2 = "*.id"
self.expr3 = "request.customer.id"
self.expr4 = "request.customer.address.street_name"
self.expr5 = "*.address.street_name"
self.expr6 = "request.customer.*.street_name"
self.expr7 = "request.customer.address.street_name[1]"
self.expr8 = "request.customer.address.street_name"
self.expr9 = "request.customer.address.elems"
self.expressions = [
self.expr1,
self.expr2,
self.expr3,
self.expr4,
self.expr5,
self.expr6,
self.expr7,
self.expr8,
self.expr9,
]
for idx, expr in enumerate(self.expressions, 1):
config = Bunch()
config.name = str(idx)
config.value = expr
self.eps.create(config.name, config, {}, False)
def test_invoke(self):
expected = {
"1": self.cust_id,
"2": [self.cust_id],
"3": [self.cust_id],
"4": [self.street_name1, self.street_name2],
"5": [self.street_name1, self.street_name2],
"6": [self.street_name1, self.street_name2],
"7": [self.street_name1, self.street_name2],
"8": [self.street_name1, self.street_name2],
"9": [self.street_elems1_1, self.street_elems1_2, self.street_elems2_1, self.street_elems2_2],
}
for idx, expr in enumerate(self.expressions, 1):
name = str(idx)
result = self.eps.invoke(self.msg, name)
self.assertEquals(expected[name], result)
def test_conversion_roundtrip(self):
xml = self.eps.convert_dict_to_xml(self.msg)
msg = self.eps.convert_xml_to_dict(xml)
self.assertEquals(msg, self.msg)
def test_replace(self):
for idx, expr in enumerate(self.expressions, 1):
msg = deepcopy(self.msg)
new_value = uuid4().hex
name = str(idx)
replaced = self.eps.replace(msg, name, new_value)
result = self.eps.invoke(replaced, name)
if isinstance(result, basestring):
self.assertEquals(result, new_value)
else:
for item in result:
self.assertEquals(item, new_value)
class WorkerStore(BrokerMessageReceiver):
""" Each worker thread has its own configuration store. The store is assigned
to the thread's threading.local variable. All the methods assume the data's
being already validated and sanitized by one of Zato's internal services.
There are exactly two threads willing to access the data at any time
- the worker thread this store belongs to
- the background ZeroMQ thread which may wish to update the store's configuration
hence the need for employing RLocks yet there shouldn't be much contention
because configuration updates are extremaly rare when compared to regular
access by worker threads.
"""
def __init__(self, worker_config=None, server=None):
self.logger = logging.getLogger(self.__class__.__name__)
self.is_ready = False
self.worker_config = worker_config
self.server = server
self.update_lock = RLock()
self.kvdb = server.kvdb
self.broker_client = None
def init(self):
# Statistics maintenance
self.stats_maint = MaintenanceTool(self.kvdb.conn)
self.msg_ns_store = NamespaceStore()
self.elem_path_store = ElemPathStore()
self.xpath_store = XPathStore()
# Message-related config - init_msg_ns_store must come before init_xpath_store
# so the latter has access to the former's namespace map.
self.init_msg_ns_store()
self.init_elem_path_store()
self.init_xpath_store()
# Request dispatcher - matches URLs, checks security and dispatches HTTP
# requests to services.
self.request_dispatcher = RequestDispatcher(
simple_io_config=self.worker_config.simple_io)
self.request_dispatcher.url_data = URLData(
deepcopy(self.worker_config.http_soap),
self.server.odb.get_url_security(self.server.cluster_id,
'channel')[0],
self.worker_config.basic_auth, self.worker_config.ntlm,
self.worker_config.oauth, self.worker_config.tech_acc,
self.worker_config.wss, self.kvdb, self.broker_client,
self.server.odb, self.elem_path_store, self.xpath_store)
self.request_dispatcher.request_handler = RequestHandler(self.server)
# Create all the expected connections
self.init_sql()
self.init_ftp()
self.init_http_soap()
# All set, whoever is waiting for us, if anyone at all, can now proceed
self.is_ready = True
def filter(self, msg):
# TODO: Fix it, worker doesn't need to accept all the messages
return True
def _http_soap_wrapper_from_config(self, config, has_sec_config=True):
""" Creates a new HTTP/SOAP connection wrapper out of a configuration
dictionary.
"""
security_name = config.get('security_name')
sec_config = {
'security_name': security_name,
'sec_type': None,
'username': None,
'password': None,
'password_type': None
}
_sec_config = None
# This will be set to True only if the method's invoked on a server's starting up
if has_sec_config:
# It's possible that there is no security config attached at all
if security_name:
_sec_config = config
else:
if security_name:
sec_type = config.sec_type
func = getattr(self.request_dispatcher.url_data,
sec_type + '_get')
_sec_config = func(security_name).config
if logger.isEnabledFor(TRACE1):
logger.log(
TRACE1,
'has_sec_config:[{}], security_name:[{}], _sec_config:[{}]'.
format(has_sec_config, security_name, _sec_config))
if _sec_config:
sec_config['sec_type'] = _sec_config['sec_type']
sec_config['username'] = _sec_config['username']
sec_config['password'] = _sec_config['password']
sec_config['password_type'] = _sec_config.get('password_type')
sec_config['salt'] = _sec_config.get('salt')
wrapper_config = {
'id': config.id,
'is_active': config.is_active,
'method': config.method,
'data_format': config.get('data_format'),
'name': config.name,
'transport': config.transport,
'address_host': config.host,
'address_url_path': config.url_path,
'soap_action': config.soap_action,
'soap_version': config.soap_version,
'ping_method': config.ping_method,
'pool_size': config.pool_size,
'serialization_type': config.serialization_type
}
wrapper_config.update(sec_config)
if wrapper_config[
'serialization_type'] == HTTP_SOAP_SERIALIZATION_TYPE.SUDS.id:
wrapper_config['queue_build_cap'] = float(
self.server.fs_server_config.misc.suds_soap_queue_build_cap)
wrapper = SudsSOAPWrapper(wrapper_config)
wrapper.build_client_queue()
return wrapper
return HTTPSOAPWrapper(wrapper_config)
# ##############################################################################
def init_sql(self):
""" Initializes SQL connections, first to ODB and then any user-defined ones.
"""
# We need a store first
self.sql_pool_store = PoolStore()
# Connect to ODB
self.sql_pool_store[ZATO_ODB_POOL_NAME] = self.worker_config.odb_data
self.odb = SessionWrapper()
self.odb.init_session(ZATO_ODB_POOL_NAME, self.worker_config.odb_data,
self.sql_pool_store[ZATO_ODB_POOL_NAME].pool)
# Any user-defined SQL connections left?
for pool_name in self.worker_config.out_sql:
config = self.worker_config.out_sql[pool_name]['config']
self.sql_pool_store[pool_name] = config
def init_ftp(self):
""" Initializes FTP connetions. The method replaces whatever value self.out_ftp
previously had (initially this would be a ConfigDict of connection definitions).
"""
config_list = self.worker_config.out_ftp.get_config_list()
self.worker_config.out_ftp = FTPStore()
self.worker_config.out_ftp.add_params(config_list)
def init_http_soap(self):
""" Initializes plain HTTP/SOAP connections.
"""
for transport in ('soap', 'plain_http'):
config_dict = getattr(self.worker_config, 'out_' + transport)
for name in config_dict:
config = config_dict[name].config
wrapper = self._http_soap_wrapper_from_config(config)
config_dict[name].conn = wrapper
# To make the API consistent with that of SQL connection pools
config_dict[name].ping = wrapper.ping
# ##############################################################################
def init_msg_ns_store(self):
for k, v in self.worker_config.msg_ns.items():
self.msg_ns_store.create(k, v.config)
def init_xpath_store(self):
for k, v in self.worker_config.xpath.items():
self.xpath_store.create(k, v.config, self.msg_ns_store.ns_map)
def init_elem_path_store(self):
for k, v in self.worker_config.elem_path.items():
self.elem_path_store.create(k, v.config, {})
# ##############################################################################
def _update_auth(self,
msg,
action_name,
sec_type,
visit_wrapper,
keys=None):
""" A common method for updating auth-related configuration.
"""
with self.update_lock:
# Channels
handler = getattr(self.request_dispatcher.url_data,
'on_broker_msg_' + action_name)
handler(msg)
for transport in ('soap', 'plain_http'):
config_dict = getattr(self.worker_config, 'out_' + transport)
# Wrappers and static configuration for outgoing connections
for name in config_dict.copy_keys():
config = config_dict[name].config
wrapper = config_dict[name].conn
if config['sec_type'] == sec_type:
if keys:
visit_wrapper(wrapper, msg, keys)
else:
visit_wrapper(wrapper, msg)
def _visit_wrapper_edit(self, wrapper, msg, keys):
""" Updates a given wrapper's security configuration.
"""
if wrapper.config['security_name'] == msg['old_name']:
for key in keys:
# All's good except for 'name', the msg's 'name' is known
# as 'security_name' in wrapper's config.
if key == 'name':
key1 = 'security_name'
key2 = key
else:
key1, key2 = key, key
wrapper.config[key1] = msg[key2]
wrapper.set_auth()
def _visit_wrapper_delete(self, wrapper, msg):
""" Deletes a wrapper.
"""
config_dict = getattr(self.worker_config,
'out_' + wrapper.config['transport'])
if wrapper.config['security_name'] == msg['name']:
del config_dict[wrapper.config['name']]
def _visit_wrapper_change_password(self, wrapper, msg):
""" Changes a wrapper's password.
"""
if wrapper.config['security_name'] == msg['name']:
wrapper.config['password'] = msg['password']
wrapper.set_auth()
# ##############################################################################
def ntlm_get(self, name):
""" Returns the configuration of the NTLM security definition
of the given name.
"""
self.request_dispatcher.url_data.ntlm_get(name)
def on_broker_msg_SECURITY_NTLM_CREATE(self, msg, *args):
""" Creates a new NTLM security definition
"""
self.request_dispatcher.url_data.on_broker_msg_SECURITY_NTLM_CREATE(
msg, *args)
def on_broker_msg_SECURITY_NTLM_EDIT(self, msg, *args):
""" Updates an existing NTLM security definition.
"""
self._update_auth(msg,
code_to_name[msg.action],
security_def_type.ntlm,
self._visit_wrapper_edit,
keys=('is_active', 'username', 'name'))
def on_broker_msg_SECURITY_NTLM_DELETE(self, msg, *args):
""" Deletes an NTLM security definition.
"""
self._update_auth(msg, code_to_name[msg.action],
security_def_type.ntlm, self._visit_wrapper_delete)
def on_broker_msg_SECURITY_NTLM_CHANGE_PASSWORD(self, msg, *args):
""" Changes password of an NTLM security definition.
"""
self._update_auth(msg, code_to_name[msg.action],
security_def_type.ntlm,
self._visit_wrapper_change_password)
# ##############################################################################
def basic_auth_get(self, name):
""" Returns the configuration of the HTTP Basic Auth security definition
of the given name.
"""
self.request_dispatcher.url_data.basic_auth_get(name)
def on_broker_msg_SECURITY_BASIC_AUTH_CREATE(self, msg, *args):
""" Creates a new HTTP Basic Auth security definition
"""
self.request_dispatcher.url_data.on_broker_msg_SECURITY_BASIC_AUTH_CREATE(
msg, *args)
def on_broker_msg_SECURITY_BASIC_AUTH_EDIT(self, msg, *args):
""" Updates an existing HTTP Basic Auth security definition.
"""
self._update_auth(msg,
code_to_name[msg.action],
security_def_type.basic_auth,
self._visit_wrapper_edit,
keys=('is_active', 'username', 'name'))
def on_broker_msg_SECURITY_BASIC_AUTH_DELETE(self, msg, *args):
""" Deletes an HTTP Basic Auth security definition.
"""
self._update_auth(msg, code_to_name[msg.action],
security_def_type.basic_auth,
self._visit_wrapper_delete)
def on_broker_msg_SECURITY_BASIC_AUTH_CHANGE_PASSWORD(self, msg, *args):
""" Changes password of an HTTP Basic Auth security definition.
"""
self._update_auth(msg, code_to_name[msg.action],
security_def_type.basic_auth,
self._visit_wrapper_change_password)
# ##############################################################################
def oauth_get(self, name):
""" Returns the configuration of the OAuth security definition
of the given name.
"""
self.request_dispatcher.url_data.oauth_get(name)
def on_broker_msg_SECURITY_OAUTH_CREATE(self, msg, *args):
""" Creates a new OAuth security definition
"""
self.request_dispatcher.url_data.on_broker_msg_SECURITY_OAUTH_CREATE(
msg, *args)
def on_broker_msg_SECURITY_OAUTH_EDIT(self, msg, *args):
""" Updates an existing OAuth security definition.
"""
self._update_auth(msg,
code_to_name[msg.action],
security_def_type.oauth,
self._visit_wrapper_edit,
keys=('is_active', 'username', 'name'))
def on_broker_msg_SECURITY_OAUTH_DELETE(self, msg, *args):
""" Deletes an OAuth security definition.
"""
self._update_auth(msg, code_to_name[msg.action],
security_def_type.oauth, self._visit_wrapper_delete)
def on_broker_msg_SECURITY_OAUTH_CHANGE_PASSWORD(self, msg, *args):
""" Changes password of an OAuth security definition.
"""
self._update_auth(msg, code_to_name[msg.action],
security_def_type.oauth,
self._visit_wrapper_change_password)
# ##############################################################################
def tech_acc_get(self, name):
""" Returns the configuration of the technical account of the given name.
"""
self.request_dispatcher.url_data.tech_acc_get(name)
def on_broker_msg_SECURITY_TECH_ACC_CREATE(self, msg, *args):
""" Creates a new technical account.
"""
self.request_dispatcher.url_data.on_broker_msg_SECURITY_TECH_ACC_CREATE(
msg, *args)
def on_broker_msg_SECURITY_TECH_ACC_EDIT(self, msg, *args):
""" Updates an existing technical account.
"""
self.request_dispatcher.url_data.on_broker_msg_SECURITY_TECH_ACC_EDIT(
msg, *args)
def on_broker_msg_SECURITY_TECH_ACC_DELETE(self, msg, *args):
""" Deletes a technical account.
"""
self.request_dispatcher.url_data.on_broker_msg_SECURITY_TECH_ACC_DELETE(
msg, *args)
def on_broker_msg_SECURITY_TECH_ACC_CHANGE_PASSWORD(self, msg, *args):
""" Changes the password of a technical account.
"""
self.request_dispatcher.url_data.on_broker_msg_SECURITY_TECH_ACC_CHANGE_PASSWORD(
msg, *args)
# ##############################################################################
def wss_get(self, name):
""" Returns the configuration of the WSS definition of the given name.
"""
self.request_dispatcher.url_data.wss_get(name)
def on_broker_msg_SECURITY_WSS_CREATE(self, msg, *args):
""" Creates a new WS-Security definition.
"""
self.request_dispatcher.url_data.on_broker_msg_SECURITY_WSS_CREATE(
msg, *args)
def on_broker_msg_SECURITY_WSS_EDIT(self, msg, *args):
""" Updates an existing WS-Security definition.
"""
self._update_auth(msg,
code_to_name[msg.action],
security_def_type.wss,
self._visit_wrapper_edit,
keys=('is_active', 'username', 'name',
'nonce_freshness_time', 'reject_expiry_limit',
'password_type', 'reject_empty_nonce_creat',
'reject_stale_tokens'))
def on_broker_msg_SECURITY_WSS_DELETE(self, msg, *args):
""" Deletes a WS-Security definition.
"""
self._update_auth(msg, code_to_name[msg.action], security_def_type.wss,
self._visit_wrapper_delete)
def on_broker_msg_SECURITY_WSS_CHANGE_PASSWORD(self, msg, *args):
""" Changes the password of a WS-Security definition.
"""
self._update_auth(msg, code_to_name[msg.action], security_def_type.wss,
self._visit_wrapper_change_password)
# ##############################################################################
def _set_service_response_data(self, service, **ignored):
if not isinstance(service.response.payload, basestring):
service.response.payload = service.response.payload.getvalue()
def _on_message_invoke_service(self, msg, channel, action, args=None):
""" Triggered by external processes, such as AMQP or the singleton's scheduler,
creates a new service instance and invokes it.
"""
# WSGI environment is the best place we have to store raw msg in
wsgi_environ = {'zato.request_ctx.async_msg': msg}
service = self.server.service_store.new_instance_by_name(msg.service)
service.update_handle(self._set_service_response_data,
service,
msg.payload,
channel,
msg.get('data_format'),
msg.get('transport'),
self.server,
self.broker_client,
self,
msg.cid,
self.worker_config.simple_io,
job_type=msg.get('job_type'),
wsgi_environ=wsgi_environ)
# ##############################################################################
def on_broker_msg_SCHEDULER_JOB_EXECUTED(self, msg, args=None):
return self._on_message_invoke_service(msg, CHANNEL.SCHEDULER,
'SCHEDULER_JOB_EXECUTED', args)
def on_broker_msg_CHANNEL_AMQP_MESSAGE_RECEIVED(self, msg, args=None):
return self._on_message_invoke_service(
msg, CHANNEL.AMQP, 'CHANNEL_AMQP_MESSAGE_RECEIVED', args)
def on_broker_msg_CHANNEL_JMS_WMQ_MESSAGE_RECEIVED(self, msg, args=None):
return self._on_message_invoke_service(
msg, CHANNEL.JMS_WMQ, 'CHANNEL_JMS_WMQ_MESSAGE_RECEIVED', args)
def on_broker_msg_CHANNEL_ZMQ_MESSAGE_RECEIVED(self, msg, args=None):
return self._on_message_invoke_service(msg, CHANNEL.ZMQ,
'CHANNEL_ZMQ_MESSAGE_RECEIVED',
args)
# ##############################################################################
def on_broker_msg_OUTGOING_SQL_CREATE_EDIT(self, msg, *args):
""" Creates or updates an SQL connection, including changing its
password.
"""
# Is it a rename? If so, delete the connection first
if msg.get('old_name') and msg.get('old_name') != msg['name']:
del self.sql_pool_store[msg['old_name']]
self.sql_pool_store[msg['name']] = msg
def on_broker_msg_OUTGOING_SQL_CHANGE_PASSWORD(self, msg, *args):
""" Deletes an outgoing SQL connection pool and recreates it using the
new password.
"""
self.sql_pool_store.change_password(msg['name'], msg['password'])
def on_broker_msg_OUTGOING_SQL_DELETE(self, msg, *args):
""" Deletes an outgoing SQL connection pool.
"""
del self.sql_pool_store[msg['name']]
# ##############################################################################
def on_broker_msg_CHANNEL_HTTP_SOAP_CREATE_EDIT(self, msg, *args):
""" Creates or updates an HTTP/SOAP channel.
"""
self.request_dispatcher.url_data.on_broker_msg_CHANNEL_HTTP_SOAP_CREATE_EDIT(
msg, *args)
def on_broker_msg_CHANNEL_HTTP_SOAP_DELETE(self, msg, *args):
""" Deletes an HTTP/SOAP channel.
"""
self.request_dispatcher.url_data.on_broker_msg_CHANNEL_HTTP_SOAP_DELETE(
msg, *args)
# ##############################################################################
def _delete_outgoing_http_soap(self, name, transport, log_func):
""" Actually deletes an outgoing HTTP/SOAP connection.
"""
# Are we dealing with plain HTTP or SOAP?
config_dict = getattr(self.worker_config, 'out_' + transport)
# Delete the connection first, if it exists at all ..
try:
try:
wrapper = config_dict[name].conn
except (KeyError, AttributeError), e:
log_func('Could not access a wrapper, e:[{}]'.format(
format_exc(e)))
else: