def test_add(self): jps = JSONPointerStore() name1, expr1 = '1', config_value('/{}/{}'.format(*rand_string(2))) name2, expr2 = '2', config_value('/aaa/{}/{}'.format(*rand_string(2))) name3, expr3 = '3', config_value('/aaa/{}/{}'.format(*rand_string(2))) name4, expr4 = '2', config_value('/aaa/{}/{}'.format(*rand_string(2))) jps.add(name1, Bunch(value=expr1.value)) self.assertIn(name1, jps.data) self.assertEquals(expr1.value, jps.data[name1].path) jps.add(name2, Bunch(value=expr2.value)) self.assertIn(name2, jps.data) self.assertEquals(expr2.value, jps.data[name2].path) jps.add(name3, Bunch(value=expr3.value)) self.assertIn(name3, jps.data) self.assertEquals(expr3.value, jps.data[name3].path) # name4's value is '2' so it overrides 2 jps.add(name4, Bunch(value=expr4.value)) self.assertIn(name4, jps.data) self.assertEquals(expr4.value, jps.data[name2].path) self.assertEquals(expr4.value, jps.data[name4].path)
def test_set_in_place(self): jps = JSONPointerStore() doc = {'a': 'b'} value_random = rand_string() name1, expr1 = '1', config_value('/a') jps.add(name1, Bunch(value=expr1.value)) # in_place is False so a new doc is created and the previous one should be retained new_doc = jps.set(name1, doc, value_random, True, in_place=False) value = jps.get(name1, new_doc) self.assertEquals(value, value_random) value = jps.get(name1, doc) self.assertEquals(value, 'b')
def test_set_defaults(self): jps = JSONPointerStore() value1 = {'b': {}} value2 = {'c': {}} doc = {} name1, expr1 = '1', config_value('/a') name2, expr2 = '2', config_value('/a/b') jps.add(name1, Bunch(value=expr1.value)) jps.add(name2, Bunch(value=expr2.value)) jps.set(name1, doc, value1) value = jps.get(name1, doc) self.assertEquals(value, value1) jps.set(name2, doc, value2) value = jps.get(name2, doc) self.assertDictEqual(value, value2)
def test_set_create_missing(self): jps = JSONPointerStore() doc = {} name1, expr1, value1 = '1', config_value('/a/b/c/d'), rand_string() name2, expr2, value2 = '2', config_value('/a/b/c/dd'), rand_string() name3, expr3, value3 = '3', config_value('/a/b/cc/d'), rand_string() jps.add(name1, Bunch(value=expr1.value)) jps.add(name2, Bunch(value=expr2.value)) jps.add(name3, Bunch(value=expr3.value)) # Creates all the missing path parts in the empty document jps.set(name1, doc, value1) jps.set(name2, doc, value2) jps.set(name3, doc, value3) doc = bunchify(doc) self.assertEquals(doc.a.b.c.d, value1) self.assertEquals(doc.a.b.c.dd, value2) self.assertEquals(doc.a.b.cc.d, value3)
def test_set_skip_missing(self): jps = JSONPointerStore() doc = {} name1, expr1 = '1', config_value('/a') name2, expr2 = '2', config_value('/b') value1, value2 = rand_string(2) default1, default2 = rand_string(2) jps.add(name1, Bunch(value=expr1.value)) jps.add(name2, Bunch(value=expr2.value)) # value is equal to default1 because it is never set by jps.set jps.set(name1, doc, value1, True) value = jps.get(name1, doc, default1) self.assertEquals(value, default1) self.assertDictEqual(doc, {}) jps.set(name2, doc, value2) value = jps.get(name2, doc, default2) self.assertEquals(value, value2) self.assertDictEqual(doc, {'b': value2})
def set_up_config(self, server): # Which components are enabled self.component_enabled.stats = asbool(self.fs_server_config.component_enabled.stats) self.component_enabled.slow_response = asbool(self.fs_server_config.component_enabled.slow_response) # # Cassandra - start # query = self.odb.get_cassandra_conn_list(server.cluster.id, True) self.config.cassandra_conn = ConfigDict.from_query('cassandra_conn', query, decrypt_func=self.decrypt) query = self.odb.get_cassandra_query_list(server.cluster.id, True) self.config.cassandra_query = ConfigDict.from_query('cassandra_query', query, decrypt_func=self.decrypt) # # Cassandra - end # # # Search - start # query = self.odb.get_search_es_list(server.cluster.id, True) self.config.search_es = ConfigDict.from_query('search_es', query, decrypt_func=self.decrypt) query = self.odb.get_search_solr_list(server.cluster.id, True) self.config.search_solr = ConfigDict.from_query('search_solr', query, decrypt_func=self.decrypt) # # Search - end # # # SMS - start # query = self.odb.get_sms_twilio_list(server.cluster.id, True) self.config.sms_twilio = ConfigDict.from_query('sms_twilio', query, decrypt_func=self.decrypt) # # SMS - end # # # Cloud - start # # OpenStack - Swift query = self.odb.get_cloud_openstack_swift_list(server.cluster.id, True) self.config.cloud_openstack_swift = ConfigDict.from_query('cloud_openstack_swift', query, decrypt_func=self.decrypt) query = self.odb.get_cloud_aws_s3_list(server.cluster.id, True) self.config.cloud_aws_s3 = ConfigDict.from_query('cloud_aws_s3', query, decrypt_func=self.decrypt) # # Cloud - end # # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ # Services query = self.odb.get_service_list(server.cluster.id, True) self.config.service = ConfigDict.from_query('service_list', query, decrypt_func=self.decrypt) # # Definitions - start # # AMQP query = self.odb.get_definition_amqp_list(server.cluster.id, True) self.config.definition_amqp = ConfigDict.from_query('definition_amqp', query, decrypt_func=self.decrypt) # IBM MQ query = self.odb.get_definition_wmq_list(server.cluster.id, True) self.config.definition_wmq = ConfigDict.from_query('definition_wmq', query, decrypt_func=self.decrypt) # # Definitions - end # # # Channels - start # # AMQP query = self.odb.get_channel_amqp_list(server.cluster.id, True) self.config.channel_amqp = ConfigDict.from_query('channel_amqp', query, decrypt_func=self.decrypt) # STOMP query = self.odb.get_channel_stomp_list(server.cluster.id, True) self.config.channel_stomp = ConfigDict.from_query('channel_stomp', query, decrypt_func=self.decrypt) # IBM MQ query = self.odb.get_channel_wmq_list(server.cluster.id, True) self.config.channel_wmq = ConfigDict.from_query('channel_wmq', query, decrypt_func=self.decrypt) # # Channels - end # # # Outgoing connections - start # # AMQP query = self.odb.get_out_amqp_list(server.cluster.id, True) self.config.out_amqp = ConfigDict.from_query('out_amqp', query, decrypt_func=self.decrypt) # Caches query = self.odb.get_cache_builtin_list(server.cluster.id, True) self.config.cache_builtin = ConfigDict.from_query('cache_builtin', query, decrypt_func=self.decrypt) query = self.odb.get_cache_memcached_list(server.cluster.id, True) self.config.cache_memcached = ConfigDict.from_query('cache_memcached', query, decrypt_func=self.decrypt) # FTP query = self.odb.get_out_ftp_list(server.cluster.id, True) self.config.out_ftp = ConfigDict.from_query('out_ftp', query, decrypt_func=self.decrypt) # IBM MQ query = self.odb.get_out_wmq_list(server.cluster.id, True) self.config.out_wmq = ConfigDict.from_query('out_wmq', query, decrypt_func=self.decrypt) # Odoo query = self.odb.get_out_odoo_list(server.cluster.id, True) self.config.out_odoo = ConfigDict.from_query('out_odoo', query, decrypt_func=self.decrypt) # SAP RFC query = self.odb.get_out_sap_list(server.cluster.id, True) self.config.out_sap = ConfigDict.from_query('out_sap', query, decrypt_func=self.decrypt) # REST query = self.odb.get_http_soap_list(server.cluster.id, 'outgoing', 'plain_http', True) self.config.out_plain_http = ConfigDict.from_query('out_plain_http', query, decrypt_func=self.decrypt) # SFTP query = self.odb.get_out_sftp_list(server.cluster.id, True) self.config.out_sftp = ConfigDict.from_query('out_sftp', query, decrypt_func=self.decrypt, drop_opaque=True) # SOAP query = self.odb.get_http_soap_list(server.cluster.id, 'outgoing', 'soap', True) self.config.out_soap = ConfigDict.from_query('out_soap', query, decrypt_func=self.decrypt) # SQL query = self.odb.get_out_sql_list(server.cluster.id, True) self.config.out_sql = ConfigDict.from_query('out_sql', query, decrypt_func=self.decrypt) # STOMP query = self.odb.get_out_stomp_list(server.cluster.id, True) self.config.out_stomp = ConfigDict.from_query('out_stomp', query, decrypt_func=self.decrypt) # ZMQ channels query = self.odb.get_channel_zmq_list(server.cluster.id, True) self.config.channel_zmq = ConfigDict.from_query('channel_zmq', query, decrypt_func=self.decrypt) # ZMQ outgoing query = self.odb.get_out_zmq_list(server.cluster.id, True) self.config.out_zmq = ConfigDict.from_query('out_zmq', query, decrypt_func=self.decrypt) # WebSocket channels query = self.odb.get_channel_web_socket_list(server.cluster.id, True) self.config.channel_web_socket = ConfigDict.from_query('channel_web_socket', query, decrypt_func=self.decrypt) # # Outgoing connections - end # # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ # # Generic - start # # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ # Connections query = self.odb.get_generic_connection_list(server.cluster.id, True) self.config.generic_connection = ConfigDict.from_query('generic_connection', query, decrypt_func=self.decrypt) # # Generic - end # # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ # # Notifications - start # # OpenStack Swift query = self.odb.get_notif_cloud_openstack_swift_list(server.cluster.id, True) self.config.notif_cloud_openstack_swift = ConfigDict.from_query('notif_cloud_openstack_swift', query, decrypt_func=self.decrypt) # SQL query = self.odb.get_notif_sql_list(server.cluster.id, True) self.config.notif_sql = ConfigDict.from_query('notif_sql', query, decrypt_func=self.decrypt) # # Notifications - end # # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ # # Security - start # # API keys query = self.odb.get_apikey_security_list(server.cluster.id, True) self.config.apikey = ConfigDict.from_query('apikey', query, decrypt_func=self.decrypt) # AWS query = self.odb.get_aws_security_list(server.cluster.id, True) self.config.aws = ConfigDict.from_query('aws', query, decrypt_func=self.decrypt) # HTTP Basic Auth query = self.odb.get_basic_auth_list(server.cluster.id, None, True) self.config.basic_auth = ConfigDict.from_query('basic_auth', query, decrypt_func=self.decrypt) # JWT query = self.odb.get_jwt_list(server.cluster.id, None, True) self.config.jwt = ConfigDict.from_query('jwt', query, decrypt_func=self.decrypt) # NTLM query = self.odb.get_ntlm_list(server.cluster.id, True) self.config.ntlm = ConfigDict.from_query('ntlm', query, decrypt_func=self.decrypt) # OAuth query = self.odb.get_oauth_list(server.cluster.id, True) self.config.oauth = ConfigDict.from_query('oauth', query, decrypt_func=self.decrypt) # OpenStack query = self.odb.get_openstack_security_list(server.cluster.id, True) self.config.openstack_security = ConfigDict.from_query('openstack_security', query, decrypt_func=self.decrypt) # RBAC - permissions query = self.odb.get_rbac_permission_list(server.cluster.id, True) self.config.rbac_permission = ConfigDict.from_query('rbac_permission', query, decrypt_func=self.decrypt) # RBAC - roles query = self.odb.get_rbac_role_list(server.cluster.id, True) self.config.rbac_role = ConfigDict.from_query('rbac_role', query, decrypt_func=self.decrypt) # RBAC - client roles query = self.odb.get_rbac_client_role_list(server.cluster.id, True) self.config.rbac_client_role = ConfigDict.from_query('rbac_client_role', query, decrypt_func=self.decrypt) # RBAC - role permission query = self.odb.get_rbac_role_permission_list(server.cluster.id, True) self.config.rbac_role_permission = ConfigDict.from_query('rbac_role_permission', query, decrypt_func=self.decrypt) # TLS CA certs query = self.odb.get_tls_ca_cert_list(server.cluster.id, True) self.config.tls_ca_cert = ConfigDict.from_query('tls_ca_cert', query, decrypt_func=self.decrypt) # TLS channel security query = self.odb.get_tls_channel_sec_list(server.cluster.id, True) self.config.tls_channel_sec = ConfigDict.from_query('tls_channel_sec', query, decrypt_func=self.decrypt) # TLS key/cert pairs query = self.odb.get_tls_key_cert_list(server.cluster.id, True) self.config.tls_key_cert = ConfigDict.from_query('tls_key_cert', query, decrypt_func=self.decrypt) # WS-Security query = self.odb.get_wss_list(server.cluster.id, True) self.config.wss = ConfigDict.from_query('wss', query, decrypt_func=self.decrypt) # Vault connections query = self.odb.get_vault_connection_list(server.cluster.id, True) self.config.vault_conn_sec = ConfigDict.from_query('vault_conn_sec', query, decrypt_func=self.decrypt) # XPath query = self.odb.get_xpath_sec_list(server.cluster.id, True) self.config.xpath_sec = ConfigDict.from_query('xpath_sec', query, decrypt_func=self.decrypt) # Encrypt all secrets self._encrypt_secrets() # # Security - end # # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ # All the HTTP/SOAP channels. http_soap = [] for item in elems_with_opaque(self.odb.get_http_soap_list(server.cluster.id, 'channel')): hs_item = {} for key in item.keys(): hs_item[key] = getattr(item, key) hs_item['match_target'] = get_match_target(hs_item, http_methods_allowed_re=self.http_methods_allowed_re) hs_item['match_target_compiled'] = Matcher(hs_item['match_target'], hs_item.get('match_slash', '')) http_soap.append(hs_item) self.config.http_soap = http_soap # Namespaces query = self.odb.get_namespace_list(server.cluster.id, True) self.config.msg_ns = ConfigDict.from_query('msg_ns', query, decrypt_func=self.decrypt) # XPath query = self.odb.get_xpath_list(server.cluster.id, True) self.config.xpath = ConfigDict.from_query('msg_xpath', query, decrypt_func=self.decrypt) # JSON Pointer query = self.odb.get_json_pointer_list(server.cluster.id, True) self.config.json_pointer = ConfigDict.from_query('json_pointer', query, decrypt_func=self.decrypt) # SimpleIO # In preparation for a SIO rewrite, we loaded SIO config from a file # but actual code paths require the pre-3.0 format so let's prepare it here. self.config.simple_io = ConfigDict('simple_io', Bunch()) int_exact = self.sio_config.int.exact int_suffix = self.sio_config.int.suffix bool_prefix = self.sio_config.bool.prefix self.config.simple_io['int_parameters'] = int_exact if isinstance(int_exact, list) else [int_exact] self.config.simple_io['int_parameter_suffixes'] = int_suffix if isinstance(int_suffix, list) else [int_suffix] self.config.simple_io['bool_parameter_prefixes'] = bool_prefix if isinstance(bool_prefix, list) else [bool_prefix] # Maintain backward-compatibility with pre-3.1 versions that did not specify any particular encoding bytes_to_str = self.sio_config.get('bytes_to_str') if not bytes_to_str: bytes_to_str = {'encoding': None} self.config.simple_io['bytes_to_str'] = bytes_to_str # Pub/sub self.config.pubsub = Bunch() # Pub/sub - endpoints query = self.odb.get_pubsub_endpoint_list(server.cluster.id, True) self.config.pubsub_endpoint = ConfigDict.from_query('pubsub_endpoint', query, decrypt_func=self.decrypt) # Pub/sub - topics query = self.odb.get_pubsub_topic_list(server.cluster.id, True) self.config.pubsub_topic = ConfigDict.from_query('pubsub_topic', query, decrypt_func=self.decrypt) # Pub/sub - subscriptions query = self.odb.get_pubsub_subscription_list(server.cluster.id, True) self.config.pubsub_subscription = ConfigDict.from_query('pubsub_subscription', query, decrypt_func=self.decrypt) # E-mail - SMTP query = self.odb.get_email_smtp_list(server.cluster.id, True) self.config.email_smtp = ConfigDict.from_query('email_smtp', query, decrypt_func=self.decrypt) # E-mail - IMAP query = self.odb.get_email_imap_list(server.cluster.id, True) self.config.email_imap = ConfigDict.from_query('email_imap', query, decrypt_func=self.decrypt) # Message paths self.config.msg_ns_store = NamespaceStore() self.config.json_pointer_store = JSONPointerStore() self.config.xpath_store = XPathStore() # HTTP access log should optionally ignore certain requests access_log_ignore = self.fs_server_config.get('logging', {}).get('http_access_log_ignore') if access_log_ignore: access_log_ignore = access_log_ignore if isinstance(access_log_ignore, list) else [access_log_ignore] self.needs_all_access_log = False self.access_log_ignore.update(access_log_ignore) # Assign config to worker self.worker_store.worker_config = self.config
def set_up_config(self, server): # Which components are enabled self.component_enabled.stats = asbool(self.fs_server_config.component_enabled.stats) self.component_enabled.slow_response = asbool(self.fs_server_config.component_enabled.slow_response) self.component_enabled.live_msg_browser = asbool(self.fs_server_config.component_enabled.live_msg_browser) # Details of what is enabled in live message browser self.live_msg_browser = self.fs_server_config.live_msg_browser self.live_msg_browser.include_internal = asbool(self.live_msg_browser.include_internal) # Pub/sub self.pubsub = PubSubAPI(RedisPubSub(self.kvdb.conn)) # # Cassandra - start # query = self.odb.get_cassandra_conn_list(server.cluster.id, True) self.config.cassandra_conn = ConfigDict.from_query('cassandra_conn', query) query = self.odb.get_cassandra_query_list(server.cluster.id, True) self.config.cassandra_query = ConfigDict.from_query('cassandra_query', query) # # Cassandra - end # # # Search - start # query = self.odb.get_search_es_list(server.cluster.id, True) self.config.search_es = ConfigDict.from_query('search_es', query) query = self.odb.get_search_solr_list(server.cluster.id, True) self.config.search_solr = ConfigDict.from_query('search_solr', query) # # Search - end # # # Cloud - start # # OpenStack - Swift query = self.odb.get_cloud_openstack_swift_list(server.cluster.id, True) self.config.cloud_openstack_swift = ConfigDict.from_query('cloud_openstack_swift', query) query = self.odb.get_cloud_aws_s3_list(server.cluster.id, True) self.config.cloud_aws_s3 = ConfigDict.from_query('cloud_aws_s3', query) # # Cloud - end # # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ # Services query = self.odb.get_service_list(server.cluster.id, True) self.config.service = ConfigDict.from_query('service_list', query) # # Definitions - start # # AMQP query = self.odb.get_definition_amqp_list(server.cluster.id, True) self.config.definition_amqp = ConfigDict.from_query('definition_amqp', query) # # Definitions - end # # # Channels - start # # AMQP query = self.odb.get_channel_amqp_list(server.cluster.id, True) self.config.channel_amqp = ConfigDict.from_query('channel_amqp', query) # STOMP query = self.odb.get_channel_stomp_list(server.cluster.id, True) self.config.channel_stomp = ConfigDict.from_query('channel_stomp', query) # # Channels - end # # # Outgoing connections - start # # AMQP query = self.odb.get_out_amqp_list(server.cluster.id, True) self.config.out_amqp = ConfigDict.from_query('out_amqp', query) # FTP query = self.odb.get_out_ftp_list(server.cluster.id, True) self.config.out_ftp = ConfigDict.from_query('out_ftp', query) # JMS WMQ query = self.odb.get_out_jms_wmq_list(server.cluster.id, True) self.config.out_jms_wmq = ConfigDict.from_query('out_jms_wmq', query) # Odoo query = self.odb.get_out_odoo_list(server.cluster.id, True) self.config.out_odoo = ConfigDict.from_query('out_odoo', query) # Plain HTTP query = self.odb.get_http_soap_list(server.cluster.id, 'outgoing', 'plain_http', True) self.config.out_plain_http = ConfigDict.from_query('out_plain_http', query) # SOAP query = self.odb.get_http_soap_list(server.cluster.id, 'outgoing', 'soap', True) self.config.out_soap = ConfigDict.from_query('out_soap', query) # SQL query = self.odb.get_out_sql_list(server.cluster.id, True) self.config.out_sql = ConfigDict.from_query('out_sql', query) # STOMP query = self.odb.get_out_stomp_list(server.cluster.id, True) self.config.out_stomp = ConfigDict.from_query('out_stomp', query) # ZMQ channels query = self.odb.get_channel_zmq_list(server.cluster.id, True) self.config.channel_zmq = ConfigDict.from_query('channel_zmq', query) # ZMQ outgoing query = self.odb.get_out_zmq_list(server.cluster.id, True) self.config.out_zmq = ConfigDict.from_query('out_zmq', query) # WebSocket channels query = self.odb.get_channel_web_socket_list(server.cluster.id, True) self.config.channel_web_socket = ConfigDict.from_query('channel_web_socket', query) # # Outgoing connections - end # # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ # # Notifications - start # # OpenStack Swift query = self.odb.get_notif_cloud_openstack_swift_list(server.cluster.id, True) self.config.notif_cloud_openstack_swift = ConfigDict.from_query('notif_cloud_openstack_swift', query) # SQL query = self.odb.get_notif_sql_list(server.cluster.id, True) self.config.notif_sql = ConfigDict.from_query('notif_sql', query) # # Notifications - end # # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ # # Security - start # # API keys query = self.odb.get_apikey_security_list(server.cluster.id, True) self.config.apikey = ConfigDict.from_query('apikey', query) # AWS query = self.odb.get_aws_security_list(server.cluster.id, True) self.config.aws = ConfigDict.from_query('aws', query) # HTTP Basic Auth query = self.odb.get_basic_auth_list(server.cluster.id, None, True) self.config.basic_auth = ConfigDict.from_query('basic_auth', query) # HTTP Basic Auth query = self.odb.get_jwt_list(server.cluster.id, None, True) self.config.jwt = ConfigDict.from_query('jwt', query) # NTLM query = self.odb.get_ntlm_list(server.cluster.id, True) self.config.ntlm = ConfigDict.from_query('ntlm', query) # OAuth query = self.odb.get_oauth_list(server.cluster.id, True) self.config.oauth = ConfigDict.from_query('oauth', query) # OpenStack query = self.odb.get_openstack_security_list(server.cluster.id, True) self.config.openstack_security = ConfigDict.from_query('openstack_security', query) # RBAC - permissions query = self.odb.get_rbac_permission_list(server.cluster.id, True) self.config.rbac_permission = ConfigDict.from_query('rbac_permission', query) # RBAC - roles query = self.odb.get_rbac_role_list(server.cluster.id, True) self.config.rbac_role = ConfigDict.from_query('rbac_role', query) # RBAC - client roles query = self.odb.get_rbac_client_role_list(server.cluster.id, True) self.config.rbac_client_role = ConfigDict.from_query('rbac_client_role', query) # RBAC - role permission query = self.odb.get_rbac_role_permission_list(server.cluster.id, True) self.config.rbac_role_permission = ConfigDict.from_query('rbac_role_permission', query) # Technical accounts query = self.odb.get_tech_acc_list(server.cluster.id, True) self.config.tech_acc = ConfigDict.from_query('tech_acc', query) # TLS CA certs query = self.odb.get_tls_ca_cert_list(server.cluster.id, True) self.config.tls_ca_cert = ConfigDict.from_query('tls_ca_cert', query) # TLS channel security query = self.odb.get_tls_channel_sec_list(server.cluster.id, True) self.config.tls_channel_sec = ConfigDict.from_query('tls_channel_sec', query) # TLS key/cert pairs query = self.odb.get_tls_key_cert_list(server.cluster.id, True) self.config.tls_key_cert = ConfigDict.from_query('tls_key_cert', query) # WS-Security query = self.odb.get_wss_list(server.cluster.id, True) self.config.wss = ConfigDict.from_query('wss', query) # Vault connections query = self.odb.get_vault_connection_list(server.cluster.id, True) self.config.vault_conn_sec = ConfigDict.from_query('vault_conn_sec', query) # XPath query = self.odb.get_xpath_sec_list(server.cluster.id, True) self.config.xpath_sec = ConfigDict.from_query('xpath_sec', query) # # Security - end # # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ # All the HTTP/SOAP channels. http_soap = [] for item in self.odb.get_http_soap_list(server.cluster.id, 'channel'): hs_item = {} for key in item.keys(): hs_item[key] = getattr(item, key) hs_item['replace_patterns_json_pointer'] = item.replace_patterns_json_pointer hs_item['replace_patterns_xpath'] = item.replace_patterns_xpath hs_item['match_target'] = '{}{}{}'.format(hs_item['soap_action'], MISC.SEPARATOR, hs_item['url_path']) hs_item['match_target_compiled'] = Matcher(hs_item['match_target']) http_soap.append(hs_item) self.config.http_soap = http_soap # Namespaces query = self.odb.get_namespace_list(server.cluster.id, True) self.config.msg_ns = ConfigDict.from_query('msg_ns', query) # XPath query = self.odb.get_xpath_list(server.cluster.id, True) self.config.xpath = ConfigDict.from_query('msg_xpath', query) # JSON Pointer query = self.odb.get_json_pointer_list(server.cluster.id, True) self.config.json_pointer = ConfigDict.from_query('json_pointer', query) # SimpleIO self.config.simple_io = ConfigDict('simple_io', Bunch()) self.config.simple_io['int_parameters'] = self.int_parameters self.config.simple_io['int_parameter_suffixes'] = self.int_parameter_suffixes self.config.simple_io['bool_parameter_prefixes'] = self.bool_parameter_prefixes # Pub/sub config self.config.pubsub = Bunch() self.config.pubsub.default_consumer = Bunch() self.config.pubsub.default_producer = Bunch() query = self.odb.get_pubsub_topic_list(server.cluster.id, True) self.config.pubsub.topics = ConfigDict.from_query('pubsub_topics', query) id, name = self.odb.get_pubsub_default_client(server.cluster.id, 'zato.pubsub.default-consumer') self.config.pubsub.default_consumer.id, self.config.pubsub.default_consumer.name = id, name id, name = self.odb.get_pubsub_default_client(server.cluster.id, 'zato.pubsub.default-producer') self.config.pubsub.default_producer.id, self.config.pubsub.default_producer.name = id, name query = self.odb.get_pubsub_producer_list(server.cluster.id, True) self.config.pubsub.producers = ConfigDict.from_query('pubsub_producers', query, list_config=True) query = self.odb.get_pubsub_consumer_list(server.cluster.id, True) self.config.pubsub.consumers = ConfigDict.from_query('pubsub_consumers', query, list_config=True) # E-mail - SMTP query = self.odb.get_email_smtp_list(server.cluster.id, True) self.config.email_smtp = ConfigDict.from_query('email_smtp', query) # E-mail - IMAP query = self.odb.get_email_imap_list(server.cluster.id, True) self.config.email_imap = ConfigDict.from_query('email_imap', query) # Message paths self.config.msg_ns_store = NamespaceStore() self.config.json_pointer_store = JSONPointerStore() self.config.xpath_store = XPathStore() # Assign config to worker self.worker_store.worker_config = self.config self.worker_store.pubsub = self.pubsub
def test_get(self): jps = JSONPointerStore() c_value, d_value = rand_string(2) doc = { 'a': { 'b': [ { 'c': c_value }, { 'd': d_value }, ] }, 'e': None, 'f': 0 } name1, expr1 = '1', config_value('/a') name2, expr2 = '2', config_value('/a/b') name3, expr3 = '3', config_value('/a/b/0') name4, expr4 = '4', config_value('/a/b/1') name5, expr5 = '5', config_value('/a/b/0/c') # This will return default because the path points to None name6, expr6 = '6', config_value('/e') # This will return default because there is no such path name7, expr7 = '7', config_value('/e/e2/e3') # This will not return None because 0 is not None even though it's False in boolean sense name8, expr8 = '8', config_value('/f') jps.add(name1, Bunch(value=expr1.value)) value = jps.get(name1, doc) self.assertListEqual(value.keys(), ['b']) jps.add(name2, Bunch(value=expr2.value)) value = jps.get(name2, doc) self.assertDictEqual(value[0], {'c': c_value}) self.assertDictEqual(value[1], {'d': d_value}) jps.add(name3, Bunch(value=expr3.value)) value = jps.get(name3, doc) self.assertDictEqual(value, {'c': c_value}) jps.add(name4, Bunch(value=expr4.value)) value = jps.get(name4, doc) self.assertDictEqual(value, {'d': d_value}) jps.add(name5, Bunch(value=expr5.value)) value = jps.get(name5, doc) self.assertEquals(value, c_value) default1 = rand_string() default2 = rand_string() jps.add(name6, Bunch(value=expr6.value)) value = jps.get(name6, doc, default1) self.assertEquals(value, default1) jps.add(name7, Bunch(value=expr7.value)) value = jps.get(name7, doc, default2) self.assertEquals(value, default2) jps.add(name8, Bunch(value=expr8.value)) value = jps.get(name8, doc) self.assertEquals(value, 0)
def set_up_config(self, server): # Which components are enabled self.component_enabled.stats = asbool(self.fs_server_config.component_enabled.stats) self.component_enabled.slow_response = asbool(self.fs_server_config.component_enabled.slow_response) self.component_enabled.live_msg_browser = asbool(self.fs_server_config.component_enabled.live_msg_browser) # Details of what is enabled in live message browser self.live_msg_browser = self.fs_server_config.live_msg_browser self.live_msg_browser.include_internal = asbool(self.live_msg_browser.include_internal) # # Cassandra - start # query = self.odb.get_cassandra_conn_list(server.cluster.id, True) self.config.cassandra_conn = ConfigDict.from_query('cassandra_conn', query, decrypt_func=self.decrypt) query = self.odb.get_cassandra_query_list(server.cluster.id, True) self.config.cassandra_query = ConfigDict.from_query('cassandra_query', query, decrypt_func=self.decrypt) # # Cassandra - end # # # Search - start # query = self.odb.get_search_es_list(server.cluster.id, True) self.config.search_es = ConfigDict.from_query('search_es', query, decrypt_func=self.decrypt) query = self.odb.get_search_solr_list(server.cluster.id, True) self.config.search_solr = ConfigDict.from_query('search_solr', query, decrypt_func=self.decrypt) # # Search - end # # # SMS - start # query = self.odb.get_sms_twilio_list(server.cluster.id, True) self.config.sms_twilio = ConfigDict.from_query('sms_twilio', query, decrypt_func=self.decrypt) # # SMS - end # # # Cloud - start # # OpenStack - Swift query = self.odb.get_cloud_openstack_swift_list(server.cluster.id, True) self.config.cloud_openstack_swift = ConfigDict.from_query('cloud_openstack_swift', query, decrypt_func=self.decrypt) query = self.odb.get_cloud_aws_s3_list(server.cluster.id, True) self.config.cloud_aws_s3 = ConfigDict.from_query('cloud_aws_s3', query, decrypt_func=self.decrypt) # # Cloud - end # # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ # Services query = self.odb.get_service_list(server.cluster.id, True) self.config.service = ConfigDict.from_query('service_list', query, decrypt_func=self.decrypt) # # Definitions - start # # AMQP query = self.odb.get_definition_amqp_list(server.cluster.id, True) self.config.definition_amqp = ConfigDict.from_query('definition_amqp', query, decrypt_func=self.decrypt) query = self.odb.get_definition_wmq_list(server.cluster.id, True) self.config.definition_wmq = ConfigDict.from_query('definition_wmq', query, decrypt_func=self.decrypt) # # Definitions - end # # # Channels - start # # AMQP query = self.odb.get_channel_amqp_list(server.cluster.id, True) self.config.channel_amqp = ConfigDict.from_query('channel_amqp', query, decrypt_func=self.decrypt) # STOMP query = self.odb.get_channel_stomp_list(server.cluster.id, True) self.config.channel_stomp = ConfigDict.from_query('channel_stomp', query, decrypt_func=self.decrypt) # IBM MQ query = self.odb.get_channel_wmq_list(server.cluster.id, True) self.config.channel_wmq = ConfigDict.from_query('channel_wmq', query, decrypt_func=self.decrypt) # # Channels - end # # # Outgoing connections - start # # AMQP query = self.odb.get_out_amqp_list(server.cluster.id, True) self.config.out_amqp = ConfigDict.from_query('out_amqp', query, decrypt_func=self.decrypt) # Caches query = self.odb.get_cache_builtin_list(server.cluster.id, True) self.config.cache_builtin = ConfigDict.from_query('cache_builtin', query, decrypt_func=self.decrypt) query = self.odb.get_cache_memcached_list(server.cluster.id, True) self.config.cache_memcached = ConfigDict.from_query('cache_memcached', query, decrypt_func=self.decrypt) # FTP query = self.odb.get_out_ftp_list(server.cluster.id, True) self.config.out_ftp = ConfigDict.from_query('out_ftp', query, decrypt_func=self.decrypt) # IBM MQ query = self.odb.get_out_wmq_list(server.cluster.id, True) self.config.out_wmq = ConfigDict.from_query('out_wmq', query, decrypt_func=self.decrypt) # Odoo query = self.odb.get_out_odoo_list(server.cluster.id, True) self.config.out_odoo = ConfigDict.from_query('out_odoo', query, decrypt_func=self.decrypt) # Plain HTTP query = self.odb.get_http_soap_list(server.cluster.id, 'outgoing', 'plain_http', True) self.config.out_plain_http = ConfigDict.from_query('out_plain_http', query, decrypt_func=self.decrypt) # SOAP query = self.odb.get_http_soap_list(server.cluster.id, 'outgoing', 'soap', True) self.config.out_soap = ConfigDict.from_query('out_soap', query, decrypt_func=self.decrypt) # SQL query = self.odb.get_out_sql_list(server.cluster.id, True) self.config.out_sql = ConfigDict.from_query('out_sql', query, decrypt_func=self.decrypt) # STOMP query = self.odb.get_out_stomp_list(server.cluster.id, True) self.config.out_stomp = ConfigDict.from_query('out_stomp', query, decrypt_func=self.decrypt) # ZMQ channels query = self.odb.get_channel_zmq_list(server.cluster.id, True) self.config.channel_zmq = ConfigDict.from_query('channel_zmq', query, decrypt_func=self.decrypt) # ZMQ outgoing query = self.odb.get_out_zmq_list(server.cluster.id, True) self.config.out_zmq = ConfigDict.from_query('out_zmq', query, decrypt_func=self.decrypt) # WebSocket channels query = self.odb.get_channel_web_socket_list(server.cluster.id, True) self.config.channel_web_socket = ConfigDict.from_query('channel_web_socket', query, decrypt_func=self.decrypt) # # Outgoing connections - end # # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ # # Notifications - start # # OpenStack Swift query = self.odb.get_notif_cloud_openstack_swift_list(server.cluster.id, True) self.config.notif_cloud_openstack_swift = ConfigDict.from_query('notif_cloud_openstack_swift', query, decrypt_func=self.decrypt) # SQL query = self.odb.get_notif_sql_list(server.cluster.id, True) self.config.notif_sql = ConfigDict.from_query('notif_sql', query, decrypt_func=self.decrypt) # # Notifications - end # # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ # # Security - start # # API keys query = self.odb.get_apikey_security_list(server.cluster.id, True) self.config.apikey = ConfigDict.from_query('apikey', query, decrypt_func=self.decrypt) # AWS query = self.odb.get_aws_security_list(server.cluster.id, True) self.config.aws = ConfigDict.from_query('aws', query, decrypt_func=self.decrypt) # HTTP Basic Auth query = self.odb.get_basic_auth_list(server.cluster.id, None, True) self.config.basic_auth = ConfigDict.from_query('basic_auth', query, decrypt_func=self.decrypt) # JWT query = self.odb.get_jwt_list(server.cluster.id, None, True) self.config.jwt = ConfigDict.from_query('jwt', query, decrypt_func=self.decrypt) # NTLM query = self.odb.get_ntlm_list(server.cluster.id, True) self.config.ntlm = ConfigDict.from_query('ntlm', query, decrypt_func=self.decrypt) # OAuth query = self.odb.get_oauth_list(server.cluster.id, True) self.config.oauth = ConfigDict.from_query('oauth', query, decrypt_func=self.decrypt) # OpenStack query = self.odb.get_openstack_security_list(server.cluster.id, True) self.config.openstack_security = ConfigDict.from_query('openstack_security', query, decrypt_func=self.decrypt) # RBAC - permissions query = self.odb.get_rbac_permission_list(server.cluster.id, True) self.config.rbac_permission = ConfigDict.from_query('rbac_permission', query, decrypt_func=self.decrypt) # RBAC - roles query = self.odb.get_rbac_role_list(server.cluster.id, True) self.config.rbac_role = ConfigDict.from_query('rbac_role', query, decrypt_func=self.decrypt) # RBAC - client roles query = self.odb.get_rbac_client_role_list(server.cluster.id, True) self.config.rbac_client_role = ConfigDict.from_query('rbac_client_role', query, decrypt_func=self.decrypt) # RBAC - role permission query = self.odb.get_rbac_role_permission_list(server.cluster.id, True) self.config.rbac_role_permission = ConfigDict.from_query('rbac_role_permission', query, decrypt_func=self.decrypt) # TLS CA certs query = self.odb.get_tls_ca_cert_list(server.cluster.id, True) self.config.tls_ca_cert = ConfigDict.from_query('tls_ca_cert', query, decrypt_func=self.decrypt) # TLS channel security query = self.odb.get_tls_channel_sec_list(server.cluster.id, True) self.config.tls_channel_sec = ConfigDict.from_query('tls_channel_sec', query, decrypt_func=self.decrypt) # TLS key/cert pairs query = self.odb.get_tls_key_cert_list(server.cluster.id, True) self.config.tls_key_cert = ConfigDict.from_query('tls_key_cert', query, decrypt_func=self.decrypt) # WS-Security query = self.odb.get_wss_list(server.cluster.id, True) self.config.wss = ConfigDict.from_query('wss', query, decrypt_func=self.decrypt) # Vault connections query = self.odb.get_vault_connection_list(server.cluster.id, True) self.config.vault_conn_sec = ConfigDict.from_query('vault_conn_sec', query, decrypt_func=self.decrypt) # XPath query = self.odb.get_xpath_sec_list(server.cluster.id, True) self.config.xpath_sec = ConfigDict.from_query('xpath_sec', query, decrypt_func=self.decrypt) # New in 3.0 - encrypt all old secrets self._migrate_30_encrypt_secrets() # # Security - end # # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ # All the HTTP/SOAP channels. http_soap = [] for item in self.odb.get_http_soap_list(server.cluster.id, 'channel'): hs_item = {} for key in item.keys(): hs_item[key] = getattr(item, key) hs_item['replace_patterns_json_pointer'] = item.replace_patterns_json_pointer hs_item['replace_patterns_xpath'] = item.replace_patterns_xpath hs_item['match_target'] = '{}{}{}'.format(hs_item['soap_action'], MISC.SEPARATOR, hs_item['url_path']) hs_item['match_target_compiled'] = Matcher(hs_item['match_target']) http_soap.append(hs_item) self.config.http_soap = http_soap # Namespaces query = self.odb.get_namespace_list(server.cluster.id, True) self.config.msg_ns = ConfigDict.from_query('msg_ns', query, decrypt_func=self.decrypt) # XPath query = self.odb.get_xpath_list(server.cluster.id, True) self.config.xpath = ConfigDict.from_query('msg_xpath', query, decrypt_func=self.decrypt) # JSON Pointer query = self.odb.get_json_pointer_list(server.cluster.id, True) self.config.json_pointer = ConfigDict.from_query('json_pointer', query, decrypt_func=self.decrypt) # SimpleIO # In preparation for a SIO rewrite, we loaded SIO config from a file # but actual code paths require the pre-3.0 format so let's prepare it here. self.config.simple_io = ConfigDict('simple_io', Bunch()) self.config.simple_io['int_parameters'] = self.sio_config.int.exact self.config.simple_io['int_parameter_suffixes'] = self.sio_config.int.suffix self.config.simple_io['bool_parameter_prefixes'] = self.sio_config.bool.prefix # Pub/sub self.config.pubsub = Bunch() # Pub/sub - endpoints query = self.odb.get_pubsub_endpoint_list(server.cluster.id, True) self.config.pubsub_endpoint = ConfigDict.from_query('pubsub_endpoint', query, decrypt_func=self.decrypt) # Pub/sub - topics query = self.odb.get_pubsub_topic_list(server.cluster.id, True) self.config.pubsub_topic = ConfigDict.from_query('pubsub_topic', query, decrypt_func=self.decrypt) # Pub/sub - subscriptions query = self.odb.get_pubsub_subscription_list(server.cluster.id, True) self.config.pubsub_subscription = ConfigDict.from_query('pubsub_subscription', query, decrypt_func=self.decrypt) # E-mail - SMTP query = self.odb.get_email_smtp_list(server.cluster.id, True) self.config.email_smtp = ConfigDict.from_query('email_smtp', query, decrypt_func=self.decrypt) # E-mail - IMAP query = self.odb.get_email_imap_list(server.cluster.id, True) self.config.email_imap = ConfigDict.from_query('email_imap', query, decrypt_func=self.decrypt) # Message paths self.config.msg_ns_store = NamespaceStore() self.config.json_pointer_store = JSONPointerStore() self.config.xpath_store = XPathStore() # Assign config to worker self.worker_store.worker_config = self.config