def test_set(self):
msg = """
<root>
<a>123</a>
<b>456</b>
</root>
""".encode('utf-8')
config1 = Bunch()
config1.name = '1'
config1.value = '//a'
new_value = rand_string()
config2 = Bunch()
config2.name = '2'
config2.value = '/zzz'
xps = XPathStore()
xps.add(config1.name, config1)
xps.add(config2.name, config2)
doc = etree.fromstring(msg)
xps.set('1', doc, new_value)
value = xps.get('1', doc)
self.assertEquals(value, new_value)
xps.set('2', doc, new_value)
value = xps.get('2', doc)
self.assertEquals(value, None)
def test_store_replace(self):
expr1 = '/root/elem1'
expr2 = '//jt:elem2'
expr3 = '//list1/item1'
expr4 = '//item2/key'
ns_map = {'jt': 'just-testing'}
for idx, expr in enumerate([expr1, expr2, expr3, expr4]):
msg = """
<root>
<elem1>elem1</elem1>
<elem2 xmlns="just-testing">elem2</elem2>
<list1>
<item1>item-a</item1>
<item1>item-b</item1>
<item2>
<key>key</key>
</item2>
</list1>
</root>
""".encode('utf-8')
doc = etree.fromstring(msg)
new_value = uuid4().hex
config = Bunch()
config.name = str(idx)
config.value = expr
xps = XPathStore()
xps.add(config.name, config, ns_map=ns_map)
xps.set(config.name, doc, new_value, ns_map)
result = xps.get(config.name, doc)
self.assertTrue(len(result) > 0)
if isinstance(result, list):
for item in result:
logger.warn('%r %r %r %r %s', idx, expr, item, result,
etree.tostring(doc, pretty_print=1))
self.assertEquals(item, new_value)
else:
self.assertEquals(result, new_value)
def test_replace(self):
msg = """
<root>
<elem1>elem1</elem1>
<elem2 xmlns="just-testing">elem2</elem2>
<list1>
<item1>item</item1>
<item1>item</item1>
<item2>
<key>key</key>
</item2>
</list1>
</root>
""".encode('utf-8')
expr1 = '/root/elem1'
expr2 = '//jt:elem2'
expr3 = '//list1/item1'
expr4 = '//item2/key'
for idx, expr in enumerate([expr1, expr2, expr3, expr4]):
new_value = uuid4().hex
config = Bunch()
config.name = str(idx)
config.value = expr
xps = XPathStore()
xps.create(config.name, config, ns_map={'jt':'just-testing'})
replaced = xps.replace(msg, config.name, new_value)
result = xps.invoke(replaced, config.name, False)
self.assertTrue(len(result) > 0)
for item in result:
self.assertEquals(item.text, new_value)
def init(self):
# Statistics maintenance
self.stats_maint = MaintenanceTool(self.kvdb.conn)
self.msg_ns_store = NamespaceStore()
self.elem_path_store = ElemPathStore()
self.xpath_store = XPathStore()
# Message-related config - init_msg_ns_store must come before init_xpath_store
# so the latter has access to the former's namespace map.
self.init_msg_ns_store()
self.init_elem_path_store()
self.init_xpath_store()
# Request dispatcher - matches URLs, checks security and dispatches HTTP
# requests to services.
self.request_dispatcher = RequestDispatcher(
simple_io_config=self.worker_config.simple_io)
self.request_dispatcher.url_data = URLData(
deepcopy(self.worker_config.http_soap),
self.server.odb.get_url_security(self.server.cluster_id,
'channel')[0],
self.worker_config.basic_auth, self.worker_config.ntlm,
self.worker_config.oauth, self.worker_config.tech_acc,
self.worker_config.wss, self.kvdb, self.broker_client,
self.server.odb, self.elem_path_store, self.xpath_store)
self.request_dispatcher.request_handler = RequestHandler(self.server)
# Create all the expected connections
self.init_sql()
self.init_ftp()
self.init_http_soap()
# All set, whoever is waiting for us, if anyone at all, can now proceed
self.is_ready = True
def set_up_config(self, server):
# Which components are enabled
self.component_enabled.stats = asbool(self.fs_server_config.component_enabled.stats)
self.component_enabled.slow_response = asbool(self.fs_server_config.component_enabled.slow_response)
#
# Cassandra - start
#
query = self.odb.get_cassandra_conn_list(server.cluster.id, True)
self.config.cassandra_conn = ConfigDict.from_query('cassandra_conn', query, decrypt_func=self.decrypt)
query = self.odb.get_cassandra_query_list(server.cluster.id, True)
self.config.cassandra_query = ConfigDict.from_query('cassandra_query', query, decrypt_func=self.decrypt)
#
# Cassandra - end
#
#
# Search - start
#
query = self.odb.get_search_es_list(server.cluster.id, True)
self.config.search_es = ConfigDict.from_query('search_es', query, decrypt_func=self.decrypt)
query = self.odb.get_search_solr_list(server.cluster.id, True)
self.config.search_solr = ConfigDict.from_query('search_solr', query, decrypt_func=self.decrypt)
#
# Search - end
#
#
# SMS - start
#
query = self.odb.get_sms_twilio_list(server.cluster.id, True)
self.config.sms_twilio = ConfigDict.from_query('sms_twilio', query, decrypt_func=self.decrypt)
#
# SMS - end
#
#
# Cloud - start
#
# OpenStack - Swift
query = self.odb.get_cloud_openstack_swift_list(server.cluster.id, True)
self.config.cloud_openstack_swift = ConfigDict.from_query('cloud_openstack_swift', query, decrypt_func=self.decrypt)
query = self.odb.get_cloud_aws_s3_list(server.cluster.id, True)
self.config.cloud_aws_s3 = ConfigDict.from_query('cloud_aws_s3', query, decrypt_func=self.decrypt)
#
# Cloud - end
#
# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
# Services
query = self.odb.get_service_list(server.cluster.id, True)
self.config.service = ConfigDict.from_query('service_list', query, decrypt_func=self.decrypt)
#
# Definitions - start
#
# AMQP
query = self.odb.get_definition_amqp_list(server.cluster.id, True)
self.config.definition_amqp = ConfigDict.from_query('definition_amqp', query, decrypt_func=self.decrypt)
# IBM MQ
query = self.odb.get_definition_wmq_list(server.cluster.id, True)
self.config.definition_wmq = ConfigDict.from_query('definition_wmq', query, decrypt_func=self.decrypt)
#
# Definitions - end
#
#
# Channels - start
#
# AMQP
query = self.odb.get_channel_amqp_list(server.cluster.id, True)
self.config.channel_amqp = ConfigDict.from_query('channel_amqp', query, decrypt_func=self.decrypt)
# STOMP
query = self.odb.get_channel_stomp_list(server.cluster.id, True)
self.config.channel_stomp = ConfigDict.from_query('channel_stomp', query, decrypt_func=self.decrypt)
# IBM MQ
query = self.odb.get_channel_wmq_list(server.cluster.id, True)
self.config.channel_wmq = ConfigDict.from_query('channel_wmq', query, decrypt_func=self.decrypt)
#
# Channels - end
#
#
# Outgoing connections - start
#
# AMQP
query = self.odb.get_out_amqp_list(server.cluster.id, True)
self.config.out_amqp = ConfigDict.from_query('out_amqp', query, decrypt_func=self.decrypt)
# Caches
query = self.odb.get_cache_builtin_list(server.cluster.id, True)
self.config.cache_builtin = ConfigDict.from_query('cache_builtin', query, decrypt_func=self.decrypt)
query = self.odb.get_cache_memcached_list(server.cluster.id, True)
self.config.cache_memcached = ConfigDict.from_query('cache_memcached', query, decrypt_func=self.decrypt)
# FTP
query = self.odb.get_out_ftp_list(server.cluster.id, True)
self.config.out_ftp = ConfigDict.from_query('out_ftp', query, decrypt_func=self.decrypt)
# IBM MQ
query = self.odb.get_out_wmq_list(server.cluster.id, True)
self.config.out_wmq = ConfigDict.from_query('out_wmq', query, decrypt_func=self.decrypt)
# Odoo
query = self.odb.get_out_odoo_list(server.cluster.id, True)
self.config.out_odoo = ConfigDict.from_query('out_odoo', query, decrypt_func=self.decrypt)
# SAP RFC
query = self.odb.get_out_sap_list(server.cluster.id, True)
self.config.out_sap = ConfigDict.from_query('out_sap', query, decrypt_func=self.decrypt)
# REST
query = self.odb.get_http_soap_list(server.cluster.id, 'outgoing', 'plain_http', True)
self.config.out_plain_http = ConfigDict.from_query('out_plain_http', query, decrypt_func=self.decrypt)
# SFTP
query = self.odb.get_out_sftp_list(server.cluster.id, True)
self.config.out_sftp = ConfigDict.from_query('out_sftp', query, decrypt_func=self.decrypt, drop_opaque=True)
# SOAP
query = self.odb.get_http_soap_list(server.cluster.id, 'outgoing', 'soap', True)
self.config.out_soap = ConfigDict.from_query('out_soap', query, decrypt_func=self.decrypt)
# SQL
query = self.odb.get_out_sql_list(server.cluster.id, True)
self.config.out_sql = ConfigDict.from_query('out_sql', query, decrypt_func=self.decrypt)
# STOMP
query = self.odb.get_out_stomp_list(server.cluster.id, True)
self.config.out_stomp = ConfigDict.from_query('out_stomp', query, decrypt_func=self.decrypt)
# ZMQ channels
query = self.odb.get_channel_zmq_list(server.cluster.id, True)
self.config.channel_zmq = ConfigDict.from_query('channel_zmq', query, decrypt_func=self.decrypt)
# ZMQ outgoing
query = self.odb.get_out_zmq_list(server.cluster.id, True)
self.config.out_zmq = ConfigDict.from_query('out_zmq', query, decrypt_func=self.decrypt)
# WebSocket channels
query = self.odb.get_channel_web_socket_list(server.cluster.id, True)
self.config.channel_web_socket = ConfigDict.from_query('channel_web_socket', query, decrypt_func=self.decrypt)
#
# Outgoing connections - end
#
# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
#
# Generic - start
#
# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
# Connections
query = self.odb.get_generic_connection_list(server.cluster.id, True)
self.config.generic_connection = ConfigDict.from_query('generic_connection', query, decrypt_func=self.decrypt)
#
# Generic - end
#
# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
#
# Notifications - start
#
# OpenStack Swift
query = self.odb.get_notif_cloud_openstack_swift_list(server.cluster.id, True)
self.config.notif_cloud_openstack_swift = ConfigDict.from_query('notif_cloud_openstack_swift',
query, decrypt_func=self.decrypt)
# SQL
query = self.odb.get_notif_sql_list(server.cluster.id, True)
self.config.notif_sql = ConfigDict.from_query('notif_sql', query, decrypt_func=self.decrypt)
#
# Notifications - end
#
# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
#
# Security - start
#
# API keys
query = self.odb.get_apikey_security_list(server.cluster.id, True)
self.config.apikey = ConfigDict.from_query('apikey', query, decrypt_func=self.decrypt)
# AWS
query = self.odb.get_aws_security_list(server.cluster.id, True)
self.config.aws = ConfigDict.from_query('aws', query, decrypt_func=self.decrypt)
# HTTP Basic Auth
query = self.odb.get_basic_auth_list(server.cluster.id, None, True)
self.config.basic_auth = ConfigDict.from_query('basic_auth', query, decrypt_func=self.decrypt)
# JWT
query = self.odb.get_jwt_list(server.cluster.id, None, True)
self.config.jwt = ConfigDict.from_query('jwt', query, decrypt_func=self.decrypt)
# NTLM
query = self.odb.get_ntlm_list(server.cluster.id, True)
self.config.ntlm = ConfigDict.from_query('ntlm', query, decrypt_func=self.decrypt)
# OAuth
query = self.odb.get_oauth_list(server.cluster.id, True)
self.config.oauth = ConfigDict.from_query('oauth', query, decrypt_func=self.decrypt)
# OpenStack
query = self.odb.get_openstack_security_list(server.cluster.id, True)
self.config.openstack_security = ConfigDict.from_query('openstack_security', query, decrypt_func=self.decrypt)
# RBAC - permissions
query = self.odb.get_rbac_permission_list(server.cluster.id, True)
self.config.rbac_permission = ConfigDict.from_query('rbac_permission', query, decrypt_func=self.decrypt)
# RBAC - roles
query = self.odb.get_rbac_role_list(server.cluster.id, True)
self.config.rbac_role = ConfigDict.from_query('rbac_role', query, decrypt_func=self.decrypt)
# RBAC - client roles
query = self.odb.get_rbac_client_role_list(server.cluster.id, True)
self.config.rbac_client_role = ConfigDict.from_query('rbac_client_role', query, decrypt_func=self.decrypt)
# RBAC - role permission
query = self.odb.get_rbac_role_permission_list(server.cluster.id, True)
self.config.rbac_role_permission = ConfigDict.from_query('rbac_role_permission', query, decrypt_func=self.decrypt)
# TLS CA certs
query = self.odb.get_tls_ca_cert_list(server.cluster.id, True)
self.config.tls_ca_cert = ConfigDict.from_query('tls_ca_cert', query, decrypt_func=self.decrypt)
# TLS channel security
query = self.odb.get_tls_channel_sec_list(server.cluster.id, True)
self.config.tls_channel_sec = ConfigDict.from_query('tls_channel_sec', query, decrypt_func=self.decrypt)
# TLS key/cert pairs
query = self.odb.get_tls_key_cert_list(server.cluster.id, True)
self.config.tls_key_cert = ConfigDict.from_query('tls_key_cert', query, decrypt_func=self.decrypt)
# WS-Security
query = self.odb.get_wss_list(server.cluster.id, True)
self.config.wss = ConfigDict.from_query('wss', query, decrypt_func=self.decrypt)
# Vault connections
query = self.odb.get_vault_connection_list(server.cluster.id, True)
self.config.vault_conn_sec = ConfigDict.from_query('vault_conn_sec', query, decrypt_func=self.decrypt)
# XPath
query = self.odb.get_xpath_sec_list(server.cluster.id, True)
self.config.xpath_sec = ConfigDict.from_query('xpath_sec', query, decrypt_func=self.decrypt)
# Encrypt all secrets
self._encrypt_secrets()
#
# Security - end
#
# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
# All the HTTP/SOAP channels.
http_soap = []
for item in elems_with_opaque(self.odb.get_http_soap_list(server.cluster.id, 'channel')):
hs_item = {}
for key in item.keys():
hs_item[key] = getattr(item, key)
hs_item['match_target'] = get_match_target(hs_item, http_methods_allowed_re=self.http_methods_allowed_re)
hs_item['match_target_compiled'] = Matcher(hs_item['match_target'], hs_item.get('match_slash', ''))
http_soap.append(hs_item)
self.config.http_soap = http_soap
# Namespaces
query = self.odb.get_namespace_list(server.cluster.id, True)
self.config.msg_ns = ConfigDict.from_query('msg_ns', query, decrypt_func=self.decrypt)
# XPath
query = self.odb.get_xpath_list(server.cluster.id, True)
self.config.xpath = ConfigDict.from_query('msg_xpath', query, decrypt_func=self.decrypt)
# JSON Pointer
query = self.odb.get_json_pointer_list(server.cluster.id, True)
self.config.json_pointer = ConfigDict.from_query('json_pointer', query, decrypt_func=self.decrypt)
# SimpleIO
# In preparation for a SIO rewrite, we loaded SIO config from a file
# but actual code paths require the pre-3.0 format so let's prepare it here.
self.config.simple_io = ConfigDict('simple_io', Bunch())
int_exact = self.sio_config.int.exact
int_suffix = self.sio_config.int.suffix
bool_prefix = self.sio_config.bool.prefix
self.config.simple_io['int_parameters'] = int_exact if isinstance(int_exact, list) else [int_exact]
self.config.simple_io['int_parameter_suffixes'] = int_suffix if isinstance(int_suffix, list) else [int_suffix]
self.config.simple_io['bool_parameter_prefixes'] = bool_prefix if isinstance(bool_prefix, list) else [bool_prefix]
# Maintain backward-compatibility with pre-3.1 versions that did not specify any particular encoding
bytes_to_str = self.sio_config.get('bytes_to_str')
if not bytes_to_str:
bytes_to_str = {'encoding': None}
self.config.simple_io['bytes_to_str'] = bytes_to_str
# Pub/sub
self.config.pubsub = Bunch()
# Pub/sub - endpoints
query = self.odb.get_pubsub_endpoint_list(server.cluster.id, True)
self.config.pubsub_endpoint = ConfigDict.from_query('pubsub_endpoint', query, decrypt_func=self.decrypt)
# Pub/sub - topics
query = self.odb.get_pubsub_topic_list(server.cluster.id, True)
self.config.pubsub_topic = ConfigDict.from_query('pubsub_topic', query, decrypt_func=self.decrypt)
# Pub/sub - subscriptions
query = self.odb.get_pubsub_subscription_list(server.cluster.id, True)
self.config.pubsub_subscription = ConfigDict.from_query('pubsub_subscription', query, decrypt_func=self.decrypt)
# E-mail - SMTP
query = self.odb.get_email_smtp_list(server.cluster.id, True)
self.config.email_smtp = ConfigDict.from_query('email_smtp', query, decrypt_func=self.decrypt)
# E-mail - IMAP
query = self.odb.get_email_imap_list(server.cluster.id, True)
self.config.email_imap = ConfigDict.from_query('email_imap', query, decrypt_func=self.decrypt)
# Message paths
self.config.msg_ns_store = NamespaceStore()
self.config.json_pointer_store = JSONPointerStore()
self.config.xpath_store = XPathStore()
# HTTP access log should optionally ignore certain requests
access_log_ignore = self.fs_server_config.get('logging', {}).get('http_access_log_ignore')
if access_log_ignore:
access_log_ignore = access_log_ignore if isinstance(access_log_ignore, list) else [access_log_ignore]
self.needs_all_access_log = False
self.access_log_ignore.update(access_log_ignore)
# Assign config to worker
self.worker_store.worker_config = self.config
def set_up_config(self, server):
# Which components are enabled
self.component_enabled.stats = asbool(self.fs_server_config.component_enabled.stats)
self.component_enabled.slow_response = asbool(self.fs_server_config.component_enabled.slow_response)
self.component_enabled.live_msg_browser = asbool(self.fs_server_config.component_enabled.live_msg_browser)
# Details of what is enabled in live message browser
self.live_msg_browser = self.fs_server_config.live_msg_browser
self.live_msg_browser.include_internal = asbool(self.live_msg_browser.include_internal)
# Pub/sub
self.pubsub = PubSubAPI(RedisPubSub(self.kvdb.conn))
#
# Cassandra - start
#
query = self.odb.get_cassandra_conn_list(server.cluster.id, True)
self.config.cassandra_conn = ConfigDict.from_query('cassandra_conn', query)
query = self.odb.get_cassandra_query_list(server.cluster.id, True)
self.config.cassandra_query = ConfigDict.from_query('cassandra_query', query)
#
# Cassandra - end
#
#
# Search - start
#
query = self.odb.get_search_es_list(server.cluster.id, True)
self.config.search_es = ConfigDict.from_query('search_es', query)
query = self.odb.get_search_solr_list(server.cluster.id, True)
self.config.search_solr = ConfigDict.from_query('search_solr', query)
#
# Search - end
#
#
# Cloud - start
#
# OpenStack - Swift
query = self.odb.get_cloud_openstack_swift_list(server.cluster.id, True)
self.config.cloud_openstack_swift = ConfigDict.from_query('cloud_openstack_swift', query)
query = self.odb.get_cloud_aws_s3_list(server.cluster.id, True)
self.config.cloud_aws_s3 = ConfigDict.from_query('cloud_aws_s3', query)
#
# Cloud - end
#
# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
# Services
query = self.odb.get_service_list(server.cluster.id, True)
self.config.service = ConfigDict.from_query('service_list', query)
#
# Definitions - start
#
# AMQP
query = self.odb.get_definition_amqp_list(server.cluster.id, True)
self.config.definition_amqp = ConfigDict.from_query('definition_amqp', query)
#
# Definitions - end
#
#
# Channels - start
#
# AMQP
query = self.odb.get_channel_amqp_list(server.cluster.id, True)
self.config.channel_amqp = ConfigDict.from_query('channel_amqp', query)
# STOMP
query = self.odb.get_channel_stomp_list(server.cluster.id, True)
self.config.channel_stomp = ConfigDict.from_query('channel_stomp', query)
#
# Channels - end
#
#
# Outgoing connections - start
#
# AMQP
query = self.odb.get_out_amqp_list(server.cluster.id, True)
self.config.out_amqp = ConfigDict.from_query('out_amqp', query)
# FTP
query = self.odb.get_out_ftp_list(server.cluster.id, True)
self.config.out_ftp = ConfigDict.from_query('out_ftp', query)
# JMS WMQ
query = self.odb.get_out_jms_wmq_list(server.cluster.id, True)
self.config.out_jms_wmq = ConfigDict.from_query('out_jms_wmq', query)
# Odoo
query = self.odb.get_out_odoo_list(server.cluster.id, True)
self.config.out_odoo = ConfigDict.from_query('out_odoo', query)
# Plain HTTP
query = self.odb.get_http_soap_list(server.cluster.id, 'outgoing', 'plain_http', True)
self.config.out_plain_http = ConfigDict.from_query('out_plain_http', query)
# SOAP
query = self.odb.get_http_soap_list(server.cluster.id, 'outgoing', 'soap', True)
self.config.out_soap = ConfigDict.from_query('out_soap', query)
# SQL
query = self.odb.get_out_sql_list(server.cluster.id, True)
self.config.out_sql = ConfigDict.from_query('out_sql', query)
# STOMP
query = self.odb.get_out_stomp_list(server.cluster.id, True)
self.config.out_stomp = ConfigDict.from_query('out_stomp', query)
# ZMQ channels
query = self.odb.get_channel_zmq_list(server.cluster.id, True)
self.config.channel_zmq = ConfigDict.from_query('channel_zmq', query)
# ZMQ outgoing
query = self.odb.get_out_zmq_list(server.cluster.id, True)
self.config.out_zmq = ConfigDict.from_query('out_zmq', query)
# WebSocket channels
query = self.odb.get_channel_web_socket_list(server.cluster.id, True)
self.config.channel_web_socket = ConfigDict.from_query('channel_web_socket', query)
#
# Outgoing connections - end
#
# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
#
# Notifications - start
#
# OpenStack Swift
query = self.odb.get_notif_cloud_openstack_swift_list(server.cluster.id, True)
self.config.notif_cloud_openstack_swift = ConfigDict.from_query('notif_cloud_openstack_swift', query)
# SQL
query = self.odb.get_notif_sql_list(server.cluster.id, True)
self.config.notif_sql = ConfigDict.from_query('notif_sql', query)
#
# Notifications - end
#
# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
#
# Security - start
#
# API keys
query = self.odb.get_apikey_security_list(server.cluster.id, True)
self.config.apikey = ConfigDict.from_query('apikey', query)
# AWS
query = self.odb.get_aws_security_list(server.cluster.id, True)
self.config.aws = ConfigDict.from_query('aws', query)
# HTTP Basic Auth
query = self.odb.get_basic_auth_list(server.cluster.id, None, True)
self.config.basic_auth = ConfigDict.from_query('basic_auth', query)
# HTTP Basic Auth
query = self.odb.get_jwt_list(server.cluster.id, None, True)
self.config.jwt = ConfigDict.from_query('jwt', query)
# NTLM
query = self.odb.get_ntlm_list(server.cluster.id, True)
self.config.ntlm = ConfigDict.from_query('ntlm', query)
# OAuth
query = self.odb.get_oauth_list(server.cluster.id, True)
self.config.oauth = ConfigDict.from_query('oauth', query)
# OpenStack
query = self.odb.get_openstack_security_list(server.cluster.id, True)
self.config.openstack_security = ConfigDict.from_query('openstack_security', query)
# RBAC - permissions
query = self.odb.get_rbac_permission_list(server.cluster.id, True)
self.config.rbac_permission = ConfigDict.from_query('rbac_permission', query)
# RBAC - roles
query = self.odb.get_rbac_role_list(server.cluster.id, True)
self.config.rbac_role = ConfigDict.from_query('rbac_role', query)
# RBAC - client roles
query = self.odb.get_rbac_client_role_list(server.cluster.id, True)
self.config.rbac_client_role = ConfigDict.from_query('rbac_client_role', query)
# RBAC - role permission
query = self.odb.get_rbac_role_permission_list(server.cluster.id, True)
self.config.rbac_role_permission = ConfigDict.from_query('rbac_role_permission', query)
# Technical accounts
query = self.odb.get_tech_acc_list(server.cluster.id, True)
self.config.tech_acc = ConfigDict.from_query('tech_acc', query)
# TLS CA certs
query = self.odb.get_tls_ca_cert_list(server.cluster.id, True)
self.config.tls_ca_cert = ConfigDict.from_query('tls_ca_cert', query)
# TLS channel security
query = self.odb.get_tls_channel_sec_list(server.cluster.id, True)
self.config.tls_channel_sec = ConfigDict.from_query('tls_channel_sec', query)
# TLS key/cert pairs
query = self.odb.get_tls_key_cert_list(server.cluster.id, True)
self.config.tls_key_cert = ConfigDict.from_query('tls_key_cert', query)
# WS-Security
query = self.odb.get_wss_list(server.cluster.id, True)
self.config.wss = ConfigDict.from_query('wss', query)
# Vault connections
query = self.odb.get_vault_connection_list(server.cluster.id, True)
self.config.vault_conn_sec = ConfigDict.from_query('vault_conn_sec', query)
# XPath
query = self.odb.get_xpath_sec_list(server.cluster.id, True)
self.config.xpath_sec = ConfigDict.from_query('xpath_sec', query)
#
# Security - end
#
# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
# All the HTTP/SOAP channels.
http_soap = []
for item in self.odb.get_http_soap_list(server.cluster.id, 'channel'):
hs_item = {}
for key in item.keys():
hs_item[key] = getattr(item, key)
hs_item['replace_patterns_json_pointer'] = item.replace_patterns_json_pointer
hs_item['replace_patterns_xpath'] = item.replace_patterns_xpath
hs_item['match_target'] = '{}{}{}'.format(hs_item['soap_action'], MISC.SEPARATOR, hs_item['url_path'])
hs_item['match_target_compiled'] = Matcher(hs_item['match_target'])
http_soap.append(hs_item)
self.config.http_soap = http_soap
# Namespaces
query = self.odb.get_namespace_list(server.cluster.id, True)
self.config.msg_ns = ConfigDict.from_query('msg_ns', query)
# XPath
query = self.odb.get_xpath_list(server.cluster.id, True)
self.config.xpath = ConfigDict.from_query('msg_xpath', query)
# JSON Pointer
query = self.odb.get_json_pointer_list(server.cluster.id, True)
self.config.json_pointer = ConfigDict.from_query('json_pointer', query)
# SimpleIO
self.config.simple_io = ConfigDict('simple_io', Bunch())
self.config.simple_io['int_parameters'] = self.int_parameters
self.config.simple_io['int_parameter_suffixes'] = self.int_parameter_suffixes
self.config.simple_io['bool_parameter_prefixes'] = self.bool_parameter_prefixes
# Pub/sub config
self.config.pubsub = Bunch()
self.config.pubsub.default_consumer = Bunch()
self.config.pubsub.default_producer = Bunch()
query = self.odb.get_pubsub_topic_list(server.cluster.id, True)
self.config.pubsub.topics = ConfigDict.from_query('pubsub_topics', query)
id, name = self.odb.get_pubsub_default_client(server.cluster.id, 'zato.pubsub.default-consumer')
self.config.pubsub.default_consumer.id, self.config.pubsub.default_consumer.name = id, name
id, name = self.odb.get_pubsub_default_client(server.cluster.id, 'zato.pubsub.default-producer')
self.config.pubsub.default_producer.id, self.config.pubsub.default_producer.name = id, name
query = self.odb.get_pubsub_producer_list(server.cluster.id, True)
self.config.pubsub.producers = ConfigDict.from_query('pubsub_producers', query, list_config=True)
query = self.odb.get_pubsub_consumer_list(server.cluster.id, True)
self.config.pubsub.consumers = ConfigDict.from_query('pubsub_consumers', query, list_config=True)
# E-mail - SMTP
query = self.odb.get_email_smtp_list(server.cluster.id, True)
self.config.email_smtp = ConfigDict.from_query('email_smtp', query)
# E-mail - IMAP
query = self.odb.get_email_imap_list(server.cluster.id, True)
self.config.email_imap = ConfigDict.from_query('email_imap', query)
# Message paths
self.config.msg_ns_store = NamespaceStore()
self.config.json_pointer_store = JSONPointerStore()
self.config.xpath_store = XPathStore()
# Assign config to worker
self.worker_store.worker_config = self.config
self.worker_store.pubsub = self.pubsub
def set_up_config(self, server):
# Which components are enabled
self.component_enabled.stats = asbool(self.fs_server_config.component_enabled.stats)
self.component_enabled.slow_response = asbool(self.fs_server_config.component_enabled.slow_response)
self.component_enabled.live_msg_browser = asbool(self.fs_server_config.component_enabled.live_msg_browser)
# Details of what is enabled in live message browser
self.live_msg_browser = self.fs_server_config.live_msg_browser
self.live_msg_browser.include_internal = asbool(self.live_msg_browser.include_internal)
#
# Cassandra - start
#
query = self.odb.get_cassandra_conn_list(server.cluster.id, True)
self.config.cassandra_conn = ConfigDict.from_query('cassandra_conn', query, decrypt_func=self.decrypt)
query = self.odb.get_cassandra_query_list(server.cluster.id, True)
self.config.cassandra_query = ConfigDict.from_query('cassandra_query', query, decrypt_func=self.decrypt)
#
# Cassandra - end
#
#
# Search - start
#
query = self.odb.get_search_es_list(server.cluster.id, True)
self.config.search_es = ConfigDict.from_query('search_es', query, decrypt_func=self.decrypt)
query = self.odb.get_search_solr_list(server.cluster.id, True)
self.config.search_solr = ConfigDict.from_query('search_solr', query, decrypt_func=self.decrypt)
#
# Search - end
#
#
# SMS - start
#
query = self.odb.get_sms_twilio_list(server.cluster.id, True)
self.config.sms_twilio = ConfigDict.from_query('sms_twilio', query, decrypt_func=self.decrypt)
#
# SMS - end
#
#
# Cloud - start
#
# OpenStack - Swift
query = self.odb.get_cloud_openstack_swift_list(server.cluster.id, True)
self.config.cloud_openstack_swift = ConfigDict.from_query('cloud_openstack_swift', query, decrypt_func=self.decrypt)
query = self.odb.get_cloud_aws_s3_list(server.cluster.id, True)
self.config.cloud_aws_s3 = ConfigDict.from_query('cloud_aws_s3', query, decrypt_func=self.decrypt)
#
# Cloud - end
#
# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
# Services
query = self.odb.get_service_list(server.cluster.id, True)
self.config.service = ConfigDict.from_query('service_list', query, decrypt_func=self.decrypt)
#
# Definitions - start
#
# AMQP
query = self.odb.get_definition_amqp_list(server.cluster.id, True)
self.config.definition_amqp = ConfigDict.from_query('definition_amqp', query, decrypt_func=self.decrypt)
query = self.odb.get_definition_wmq_list(server.cluster.id, True)
self.config.definition_wmq = ConfigDict.from_query('definition_wmq', query, decrypt_func=self.decrypt)
#
# Definitions - end
#
#
# Channels - start
#
# AMQP
query = self.odb.get_channel_amqp_list(server.cluster.id, True)
self.config.channel_amqp = ConfigDict.from_query('channel_amqp', query, decrypt_func=self.decrypt)
# STOMP
query = self.odb.get_channel_stomp_list(server.cluster.id, True)
self.config.channel_stomp = ConfigDict.from_query('channel_stomp', query, decrypt_func=self.decrypt)
# IBM MQ
query = self.odb.get_channel_wmq_list(server.cluster.id, True)
self.config.channel_wmq = ConfigDict.from_query('channel_wmq', query, decrypt_func=self.decrypt)
#
# Channels - end
#
#
# Outgoing connections - start
#
# AMQP
query = self.odb.get_out_amqp_list(server.cluster.id, True)
self.config.out_amqp = ConfigDict.from_query('out_amqp', query, decrypt_func=self.decrypt)
# Caches
query = self.odb.get_cache_builtin_list(server.cluster.id, True)
self.config.cache_builtin = ConfigDict.from_query('cache_builtin', query, decrypt_func=self.decrypt)
query = self.odb.get_cache_memcached_list(server.cluster.id, True)
self.config.cache_memcached = ConfigDict.from_query('cache_memcached', query, decrypt_func=self.decrypt)
# FTP
query = self.odb.get_out_ftp_list(server.cluster.id, True)
self.config.out_ftp = ConfigDict.from_query('out_ftp', query, decrypt_func=self.decrypt)
# IBM MQ
query = self.odb.get_out_wmq_list(server.cluster.id, True)
self.config.out_wmq = ConfigDict.from_query('out_wmq', query, decrypt_func=self.decrypt)
# Odoo
query = self.odb.get_out_odoo_list(server.cluster.id, True)
self.config.out_odoo = ConfigDict.from_query('out_odoo', query, decrypt_func=self.decrypt)
# Plain HTTP
query = self.odb.get_http_soap_list(server.cluster.id, 'outgoing', 'plain_http', True)
self.config.out_plain_http = ConfigDict.from_query('out_plain_http', query, decrypt_func=self.decrypt)
# SOAP
query = self.odb.get_http_soap_list(server.cluster.id, 'outgoing', 'soap', True)
self.config.out_soap = ConfigDict.from_query('out_soap', query, decrypt_func=self.decrypt)
# SQL
query = self.odb.get_out_sql_list(server.cluster.id, True)
self.config.out_sql = ConfigDict.from_query('out_sql', query, decrypt_func=self.decrypt)
# STOMP
query = self.odb.get_out_stomp_list(server.cluster.id, True)
self.config.out_stomp = ConfigDict.from_query('out_stomp', query, decrypt_func=self.decrypt)
# ZMQ channels
query = self.odb.get_channel_zmq_list(server.cluster.id, True)
self.config.channel_zmq = ConfigDict.from_query('channel_zmq', query, decrypt_func=self.decrypt)
# ZMQ outgoing
query = self.odb.get_out_zmq_list(server.cluster.id, True)
self.config.out_zmq = ConfigDict.from_query('out_zmq', query, decrypt_func=self.decrypt)
# WebSocket channels
query = self.odb.get_channel_web_socket_list(server.cluster.id, True)
self.config.channel_web_socket = ConfigDict.from_query('channel_web_socket', query, decrypt_func=self.decrypt)
#
# Outgoing connections - end
#
# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
#
# Notifications - start
#
# OpenStack Swift
query = self.odb.get_notif_cloud_openstack_swift_list(server.cluster.id, True)
self.config.notif_cloud_openstack_swift = ConfigDict.from_query('notif_cloud_openstack_swift', query, decrypt_func=self.decrypt)
# SQL
query = self.odb.get_notif_sql_list(server.cluster.id, True)
self.config.notif_sql = ConfigDict.from_query('notif_sql', query, decrypt_func=self.decrypt)
#
# Notifications - end
#
# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
#
# Security - start
#
# API keys
query = self.odb.get_apikey_security_list(server.cluster.id, True)
self.config.apikey = ConfigDict.from_query('apikey', query, decrypt_func=self.decrypt)
# AWS
query = self.odb.get_aws_security_list(server.cluster.id, True)
self.config.aws = ConfigDict.from_query('aws', query, decrypt_func=self.decrypt)
# HTTP Basic Auth
query = self.odb.get_basic_auth_list(server.cluster.id, None, True)
self.config.basic_auth = ConfigDict.from_query('basic_auth', query, decrypt_func=self.decrypt)
# JWT
query = self.odb.get_jwt_list(server.cluster.id, None, True)
self.config.jwt = ConfigDict.from_query('jwt', query, decrypt_func=self.decrypt)
# NTLM
query = self.odb.get_ntlm_list(server.cluster.id, True)
self.config.ntlm = ConfigDict.from_query('ntlm', query, decrypt_func=self.decrypt)
# OAuth
query = self.odb.get_oauth_list(server.cluster.id, True)
self.config.oauth = ConfigDict.from_query('oauth', query, decrypt_func=self.decrypt)
# OpenStack
query = self.odb.get_openstack_security_list(server.cluster.id, True)
self.config.openstack_security = ConfigDict.from_query('openstack_security', query, decrypt_func=self.decrypt)
# RBAC - permissions
query = self.odb.get_rbac_permission_list(server.cluster.id, True)
self.config.rbac_permission = ConfigDict.from_query('rbac_permission', query, decrypt_func=self.decrypt)
# RBAC - roles
query = self.odb.get_rbac_role_list(server.cluster.id, True)
self.config.rbac_role = ConfigDict.from_query('rbac_role', query, decrypt_func=self.decrypt)
# RBAC - client roles
query = self.odb.get_rbac_client_role_list(server.cluster.id, True)
self.config.rbac_client_role = ConfigDict.from_query('rbac_client_role', query, decrypt_func=self.decrypt)
# RBAC - role permission
query = self.odb.get_rbac_role_permission_list(server.cluster.id, True)
self.config.rbac_role_permission = ConfigDict.from_query('rbac_role_permission', query, decrypt_func=self.decrypt)
# TLS CA certs
query = self.odb.get_tls_ca_cert_list(server.cluster.id, True)
self.config.tls_ca_cert = ConfigDict.from_query('tls_ca_cert', query, decrypt_func=self.decrypt)
# TLS channel security
query = self.odb.get_tls_channel_sec_list(server.cluster.id, True)
self.config.tls_channel_sec = ConfigDict.from_query('tls_channel_sec', query, decrypt_func=self.decrypt)
# TLS key/cert pairs
query = self.odb.get_tls_key_cert_list(server.cluster.id, True)
self.config.tls_key_cert = ConfigDict.from_query('tls_key_cert', query, decrypt_func=self.decrypt)
# WS-Security
query = self.odb.get_wss_list(server.cluster.id, True)
self.config.wss = ConfigDict.from_query('wss', query, decrypt_func=self.decrypt)
# Vault connections
query = self.odb.get_vault_connection_list(server.cluster.id, True)
self.config.vault_conn_sec = ConfigDict.from_query('vault_conn_sec', query, decrypt_func=self.decrypt)
# XPath
query = self.odb.get_xpath_sec_list(server.cluster.id, True)
self.config.xpath_sec = ConfigDict.from_query('xpath_sec', query, decrypt_func=self.decrypt)
# New in 3.0 - encrypt all old secrets
self._migrate_30_encrypt_secrets()
#
# Security - end
#
# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
# All the HTTP/SOAP channels.
http_soap = []
for item in self.odb.get_http_soap_list(server.cluster.id, 'channel'):
hs_item = {}
for key in item.keys():
hs_item[key] = getattr(item, key)
hs_item['replace_patterns_json_pointer'] = item.replace_patterns_json_pointer
hs_item['replace_patterns_xpath'] = item.replace_patterns_xpath
hs_item['match_target'] = '{}{}{}'.format(hs_item['soap_action'], MISC.SEPARATOR, hs_item['url_path'])
hs_item['match_target_compiled'] = Matcher(hs_item['match_target'])
http_soap.append(hs_item)
self.config.http_soap = http_soap
# Namespaces
query = self.odb.get_namespace_list(server.cluster.id, True)
self.config.msg_ns = ConfigDict.from_query('msg_ns', query, decrypt_func=self.decrypt)
# XPath
query = self.odb.get_xpath_list(server.cluster.id, True)
self.config.xpath = ConfigDict.from_query('msg_xpath', query, decrypt_func=self.decrypt)
# JSON Pointer
query = self.odb.get_json_pointer_list(server.cluster.id, True)
self.config.json_pointer = ConfigDict.from_query('json_pointer', query, decrypt_func=self.decrypt)
# SimpleIO
# In preparation for a SIO rewrite, we loaded SIO config from a file
# but actual code paths require the pre-3.0 format so let's prepare it here.
self.config.simple_io = ConfigDict('simple_io', Bunch())
self.config.simple_io['int_parameters'] = self.sio_config.int.exact
self.config.simple_io['int_parameter_suffixes'] = self.sio_config.int.suffix
self.config.simple_io['bool_parameter_prefixes'] = self.sio_config.bool.prefix
# Pub/sub
self.config.pubsub = Bunch()
# Pub/sub - endpoints
query = self.odb.get_pubsub_endpoint_list(server.cluster.id, True)
self.config.pubsub_endpoint = ConfigDict.from_query('pubsub_endpoint', query, decrypt_func=self.decrypt)
# Pub/sub - topics
query = self.odb.get_pubsub_topic_list(server.cluster.id, True)
self.config.pubsub_topic = ConfigDict.from_query('pubsub_topic', query, decrypt_func=self.decrypt)
# Pub/sub - subscriptions
query = self.odb.get_pubsub_subscription_list(server.cluster.id, True)
self.config.pubsub_subscription = ConfigDict.from_query('pubsub_subscription', query, decrypt_func=self.decrypt)
# E-mail - SMTP
query = self.odb.get_email_smtp_list(server.cluster.id, True)
self.config.email_smtp = ConfigDict.from_query('email_smtp', query, decrypt_func=self.decrypt)
# E-mail - IMAP
query = self.odb.get_email_imap_list(server.cluster.id, True)
self.config.email_imap = ConfigDict.from_query('email_imap', query, decrypt_func=self.decrypt)
# Message paths
self.config.msg_ns_store = NamespaceStore()
self.config.json_pointer_store = JSONPointerStore()
self.config.xpath_store = XPathStore()
# Assign config to worker
self.worker_store.worker_config = self.config
