def deploy_missing_services(self, locally_deployed): """ Deploys services that exist on other servers but not on ours. """ # The locally_deployed list are all the services that we could import based on our current # understanding of the contents of the cluster. However, it's possible that we have # been shut down for a long time and during that time other servers deployed services # we don't know anything about. They are not stored locally because we were down. # Hence we need to check out if there are any other servers in the cluster and if so, # grab their list of services, compare it with what we have deployed and deploy # any that are missing. # Continue only if there is more than one running server in the cluster. other_servers = self.odb.get_servers() if other_servers: other_server = other_servers[0] # Index 0 is as random as any other because the list is not sorted. missing = self.odb.get_missing_services(other_server, locally_deployed) if missing: logger.info('Found extra services to deploy: %s', ', '.join(sorted(item.name for item in missing))) # (file_name, source_path) -> a list of services it contains modules = {} # Coalesce all service modules - it is possible that each one has multiple services # so we do want to deploy the same module over for each service found. for service_id, name, source_path, source in missing: file_name = os.path.basename(source_path) _, tmp_full_path = mkstemp(suffix='-'+ file_name) # Module names are unique so they can serve as keys key = file_name if key not in modules: modules[key] = { 'tmp_full_path': tmp_full_path, 'services': [name] # We can append initial name already in this 'if' branch } # Save the source code only once here f = open(tmp_full_path, 'wb') f.write(source) f.close() else: modules[key]['services'].append(name) # Create a deployment package in ODB out of which all the services will be picked up .. for file_name, values in modules.items(): msg = Bunch() msg.action = HOT_DEPLOY.CREATE_SERVICE.value msg.msg_type = MESSAGE_TYPE.TO_PARALLEL_ALL msg.package_id = hot_deploy(self, file_name, values['tmp_full_path'], notify=False) # .. and tell the worker to actually deploy all the services the package contains. #gevent.spawn(self.worker_store.on_broker_msg_HOT_DEPLOY_CREATE_SERVICE, msg) self.worker_store.on_broker_msg_HOT_DEPLOY_CREATE_SERVICE(msg) logger.info('Deployed extra services found: %s', sorted(values['services']))
def test_top_level_skip_empty_input_single(self): class MyService(Service): class SimpleIO: input = 'aaa', 'bbb', '-ccc', '-ddd', '-eee', '-fff' default_value = NotGiven class SkipEmpty: input = 'ccc' CySimpleIO.attach_sio(self.get_server_config(), MyService) data = Bunch() data.aaa = 'aaa' data.bbb = 'bbb' input = MyService._sio.parse_input(data, DATA_FORMAT.JSON) self.assertIsInstance(input, Bunch) self.assertDictEqual( input, { 'aaa': 'aaa', 'bbb': 'bbb', 'ddd': NotGiven, 'eee': NotGiven, 'fff': NotGiven, })
def deploy_missing_services(self, locally_deployed): """ Deploys services that exist on other servers but not on ours. """ # The locally_deployed list are all the services that we could import based on our current # understanding of the contents of the cluster. However, it's possible that we have # been shut down for a long time and during that time other servers deployed services # we don't know anything about. They are not stored locally because we were down. # Hence we need to check out if there are any other servers in the cluster and if so, # grab their list of services, compare it with what we have deployed and deploy # any that are missing. # Continue only if there is more than one running server in the cluster. other_servers = self.odb.get_servers() if other_servers: other_server = other_servers[ 0] # Index 0 is as random as any other because the list is not sorted. missing = self.odb.get_missing_services(other_server, locally_deployed) if missing: logger.info('Found extra services to deploy: %s', ', '.join(sorted(item.name for item in missing))) for service_id, name, source_path, source in missing: file_name = os.path.basename(source_path) _, full_path = mkstemp(suffix='-' + file_name) f = open(full_path, 'wb') f.write(source) f.close() # Create a deployment package in ODB out of which all the services will be picked up .. msg = Bunch() msg.action = HOT_DEPLOY.CREATE_SERVICE.value msg.msg_type = MESSAGE_TYPE.TO_PARALLEL_ALL msg.package_id = hot_deploy(self, file_name, full_path, notify=False) # .. and tell the worker to actually deploy all the services the package contains. gevent.spawn( self.worker_store. on_broker_msg_HOT_DEPLOY_CREATE_SERVICE, msg) logger.info('Deployed an extra service found: %s (%s)', name, service_id)
def __init__(self, *ignored_args, **ignored_kwargs): self.logger = logging.getLogger(self.get_name()) self.server = None self.broker_client = None self.pubsub = None self.channel = None self.cid = None self.in_reply_to = None self.outgoing = None self.cloud = None self.worker_store = None self.odb = None self.data_format = None self.transport = None self.wsgi_environ = None self.job_type = None self.environ = Bunch() self.request = Request(self.logger) self.response = Response(self.logger) self.invocation_time = None # When was the service invoked self.handle_return_time = None # When did its 'handle' method finished processing the request self.processing_time_raw = None # A timedelta object with the processing time up to microseconds self.processing_time = None # Processing time in milliseconds self.usage = 0 # How many times the service has been invoked self.slow_threshold = maxint # After how many ms to consider the response came too late self.name = self.__class__.get_name() self.impl_name = self.__class__.get_impl_name() self.time = None self.patterns = None self.user_config = None self.dictnav = DictNav self.listnav = ListNav self.has_validate_input = False self.has_validate_output = False
def set_up_pickup(self): empty = [] # Fix up booleans and paths for stanza, stanza_config in self.pickup_config.items(): # user_config_items is empty by default if not stanza_config: empty.append(stanza) continue stanza_config.read_on_pickup = asbool(stanza_config.get('read_on_pickup', True)) stanza_config.parse_on_pickup = asbool(stanza_config.get('parse_on_pickup', True)) stanza_config.delete_after_pickup = asbool(stanza_config.get('delete_after_pickup', True)) stanza_config.case_insensitive = asbool(stanza_config.get('case_insensitive', True)) stanza_config.pickup_from = absolutize(stanza_config.pickup_from, self.base_dir) stanza_config.is_service_hot_deploy = False mpt = stanza_config.get('move_processed_to') stanza_config.move_processed_to = absolutize(mpt, self.base_dir) if mpt else None services = stanza_config.get('services') or [] stanza_config.services = [services] if not isinstance(services, list) else services topics = stanza_config.get('topics') or [] stanza_config.topics = [topics] if not isinstance(topics, list) else topics flags = globre.EXACT if stanza_config.case_insensitive: flags |= IGNORECASE patterns = stanza_config.patterns stanza_config.patterns = [patterns] if not isinstance(patterns, list) else patterns stanza_config.patterns = [globre.compile(elem, flags) for elem in stanza_config.patterns] if not os.path.exists(stanza_config.pickup_from): logger.warn('Pickup dir `%s` does not exist (%s)', stanza_config.pickup_from, stanza) for item in empty: del self.pickup_config[item] # Ok, now that we have configured everything that pickup.conf had # we still need to make it aware of services and how to pick them up from FS. stanza = 'zato_internal_service_hot_deploy' stanza_config = Bunch({ 'pickup_from': self.hot_deploy_config.pickup_dir, 'patterns': [globre.compile('*.py', globre.EXACT | IGNORECASE)], 'read_on_pickup': False, 'parse_on_pickup': False, 'delete_after_pickup': self.hot_deploy_config.delete_after_pickup, 'is_service_hot_deploy': True, }) self.pickup_config[stanza] = stanza_config self.pickup = PickupManager(self, self.pickup_config) spawn_greenlet(self.pickup.run)
def from_query(name, query_data, impl_class=Bunch, item_class=Bunch, list_config=False): """ Return a new ConfigDict with items taken from an SQL query. """ config_dict = ConfigDict(name) config_dict._impl = impl_class() if query_data: query, attrs = query_data for item in query: if hasattr(item, 'name'): item_name = item.name else: item_name = item.get_name() if list_config: list_dict = Bunch() if item_name not in config_dict._impl: config_dict._impl[item_name] = [] config_dict._impl[item_name].append(list_dict) else: config_dict._impl[item_name] = item_class() if list_config: for attr_name in attrs.keys(): list_dict[attr_name] = getattr(item, attr_name) else: config_dict._impl[item_name].config = item_class() for attr_name in attrs.keys(): config_dict._impl[item_name]['config'][attr_name] = getattr(item, attr_name) return config_dict
def from_query(name, query_data, impl_class=Bunch, item_class=Bunch, list_config=False, decrypt_func=None, drop_opaque=False): """ Return a new ConfigDict with items taken from an SQL query. """ config_dict = ConfigDict(name) config_dict._impl = impl_class() if query_data: query, attrs = query_data for item in query: if hasattr(item, 'name'): item_name = item.name else: item_name = item.get_name() if list_config: list_dict = Bunch() if item_name not in config_dict._impl: config_dict._impl[item_name] = [] config_dict._impl[item_name].append(list_dict) else: config_dict._impl[item_name] = item_class() if list_config: for attr_name in attrs.keys(): list_dict[attr_name] = getattr(item, attr_name) else: config_dict._impl[item_name].config = item_class() for attr_name in attrs.keys(): config = config_dict._impl[item_name]['config'] original = value = getattr(item, attr_name) value = resolve_value(attr_name, value, decrypt_func) config[attr_name] = value # Temporarily, add a flag to indicate whether the password in ODB was encrypted or not. if attr_name in SECRETS.PARAMS: if original is None: original = '' config['_encryption_needed'] = True if original.startswith(SECRETS.PREFIX): config['_encrypted_in_odb'] = True else: config['_encrypted_in_odb'] = False # Post-process data before it is returned to resolve any opaque attributes for value in config_dict.values(): value_config = value['config'] if ElemsWithOpaqueMaker.has_opaque_data(value_config): ElemsWithOpaqueMaker.process_config_dict( value_config, drop_opaque) return config_dict
def test_parse_nested_dict_customer_no_defaults(self): locality = Dict('locality', 'type', 'name') address = Dict('address', locality, 'street') email = Dict('email', 'personal', 'business') customer = Dict('customer', 'name', email, address) class MyService(Service): class SimpleIO: input = customer CySimpleIO.attach_sio(self.get_server_config(), MyService) data = Bunch() data.customer = Bunch() data.customer.name = 'my-name' data.customer.email = Bunch() data.customer.email.personal = 'my-personal-email' data.customer.email.business = 'my-business-email' data.customer.address = Bunch() data.customer.address.street = 'my-street' data.customer.address.locality = Bunch() data.customer.address.locality.type = 'my-type' data.customer.address.locality.name = 'my-name' input = MyService._sio.parse_input(data, DATA_FORMAT.JSON) self.assertIsInstance(input, Bunch) self.assertEquals(input.customer.name, data.customer.name) self.assertEquals(input.customer.email.personal, data.customer.email.personal) self.assertEquals(input.customer.email.business, data.customer.email.business) self.assertEquals(input.customer.address.street, data.customer.address.street) self.assertEquals(input.customer.address.locality.type, data.customer.address.locality.type) self.assertEquals(input.customer.address.locality.name, data.customer.address.locality.name)
def copy(self): """ Returns a new instance of ConfigDict with items copied over from self. """ with self.lock: config_dict = ConfigDict(self.name) config_dict._impl = Bunch() config_dict._impl.update(deepcopy(self._impl)) return config_dict
def test_top_level_skip_empty_input_true_no_force_empty_with_attribute( self): class MyService(Service): class SimpleIO: input = 'aaa', 'bbb', '-ccc', '-ddd' skip_empty_keys = True CySimpleIO.attach_sio(self.get_server_config(), MyService) data = Bunch() data.aaa = 'aaa' data.bbb = 'bbb' input = MyService._sio.parse_input(data, DATA_FORMAT.JSON) self.assertIsInstance(input, Bunch) self.assertDictEqual(input, { 'aaa': 'aaa', 'bbb': 'bbb', })
def server(self): server = ParallelServer() server.odb = self.odb_manager() server.service_store = self.service_store() server.service_store.server = server server.sql_pool_store = self.sql_pool_store() server.service_modules = self.service_modules() server.kvdb = self.kvdb() server.user_config = Bunch() return server
def test_top_level_skip_empty_input_true_with_force_empty_single(self): class MyService(Service): class SimpleIO: input = 'aaa', 'bbb', '-ccc', '-ddd', '-eee', '-fff' class SkipEmpty: input = True force_empty_input = 'eee' CySimpleIO.attach_sio(self.get_server_config(), MyService) data = Bunch() data.aaa = 'aaa' data.bbb = 'bbb' input = MyService._sio.parse_input(data, DATA_FORMAT.JSON) self.assertIsInstance(input, Bunch) self.assertDictEqual(input, { 'aaa': 'aaa', 'bbb': 'bbb', 'eee': backward_compat_default_value, })
def from_query(name, query_data, impl_class=Bunch, item_class=Bunch, list_config=False, decrypt_func=None): """ Return a new ConfigDict with items taken from an SQL query. """ config_dict = ConfigDict(name) config_dict._impl = impl_class() if query_data: query, attrs = query_data for item in query: if hasattr(item, 'name'): item_name = item.name else: item_name = item.get_name() if list_config: list_dict = Bunch() if item_name not in config_dict._impl: config_dict._impl[item_name] = [] config_dict._impl[item_name].append(list_dict) else: config_dict._impl[item_name] = item_class() if list_config: for attr_name in attrs.keys(): list_dict[attr_name] = getattr(item, attr_name) else: config_dict._impl[item_name].config = item_class() for attr_name in attrs.keys(): config = config_dict._impl[item_name]['config'] original = value = getattr(item, attr_name) value = resolve_value(attr_name, value, decrypt_func) config[attr_name] = value # Temporarily, add a flag to indicate whether the password in ODB was encrypted or not. if attr_name in SECRETS.PARAMS: config['_encryption_needed'] = True if original.startswith(SECRETS.PREFIX): config['_encrypted_in_odb'] = True else: config['_encrypted_in_odb'] = False return config_dict
def test_parse_nested_dict_all_sio_elems(self): locality = Dict('locality', Int('type'), Text('name'), AsIs('coords'), Decimal('geo_skip'), Float('geo_diff')) address = Dict('address', locality, UUID('street_id'), CSV('prefs'), DateTime('since'), List('types'), Opaque('opaque1')) email = Dict('email', Text('value'), Bool('is_business'), Date('join_date'), DictList('preferred_order', 'name', 'pos')) customer = Dict('customer', 'name', email, address) class MyService(Service): class SimpleIO: input = customer CySimpleIO.attach_sio(self.get_server_config(), MyService) data = Bunch() data.customer = Bunch() data.customer.name = 'my-name' data.customer.email = Bunch() data.customer.email.value = 'my-email' data.customer.email.is_business = True data.customer.email.join_date = '1999-12-31' data.customer.email.preferred_order = [{'name':'address2', 'pos':'2'}, {'name':'address1', 'pos':'1'}] data.customer.address = Bunch() data.customer.address.locality = Bunch() data.customer.address.locality.type = '111' data.customer.address.locality.name = 'my-locality' data.customer.address.locality.coords = object() data.customer.address.locality.geo_skip = '123.456' data.customer.address.locality.geo_diff = '999.777' data.customer.address.street_id = uuid4().hex data.customer.address.prefs = '1,2,3,4' data.customer.address.since = '27-11-1988T11:22:33' data.customer.address.types = ['a', 'b', 'c', 'd'] data.customer.address.opaque1 = object() input = MyService._sio.parse_input(data, DATA_FORMAT.JSON) self.assertIsInstance(input, Bunch) self.assertEquals(input.customer.name, data.customer.name) self.assertEquals(input.customer.email.value, data.customer.email.value) self.assertEquals(input.customer.email.is_business, data.customer.email.is_business) self.assertEquals(input.customer.email.join_date, dt_parse(data.customer.email.join_date)) self.assertListEqual(input.customer.email.preferred_order, data.customer.email.preferred_order) self.assertEquals(input.customer.address.locality.type, int(data.customer.address.locality.type)) self.assertEquals(input.customer.address.locality.name, data.customer.address.locality.name) self.assertIs(input.customer.address.locality.coords, data.customer.address.locality.coords) self.assertEquals(input.customer.address.locality.geo_skip, decimal_Decimal(data.customer.address.locality.geo_skip)) self.assertEquals(input.customer.address.locality.geo_diff, float(data.customer.address.locality.geo_diff)) self.assertEquals(input.customer.address.street_id, uuid_UUID(data.customer.address.street_id)) self.assertEquals(input.customer.address.prefs, data.customer.address.prefs.split(',')) self.assertEquals(input.customer.address.since, dt_parse(data.customer.address.since)) self.assertEquals(input.customer.address.types, data.customer.address.types) self.assertIs(input.customer.address.opaque1, data.customer.address.opaque1)
def parallel_server(self): server = ParallelServer() server.odb = self.odb_manager() server.service_store = self.service_store() server.sql_pool_store = self.sql_pool_store() server.int_parameters = self.int_parameters() server.int_parameter_suffixes = self.int_parameter_suffixes() server.bool_parameter_prefixes = self.bool_parameter_prefixes() server.internal_service_modules = self.internal_service_modules() server.service_modules = self.service_modules() server.kvdb = self.kvdb() server.user_config = Bunch() return server
def test_parse_nested_dict_customer_deep_defaults_elem_level(self): locality_default = object() locality = Dict('locality', '-type', '-name', default=locality_default) address = Dict('address', locality, '-street') email = Dict('email', 'personal', 'business') customer = Dict('customer', 'name', email, address) _default_input_value = 'default-input-value' class MyService(Service): class SimpleIO: input = customer default_input_value = 'default-input-value' CySimpleIO.attach_sio(self.get_server_config(), MyService) # Note that this locality has no type nor name but we expect for that Dict's default value to be used, # also, address has no street but since this Dict has no default value, again, SimpleIO one will be used. data = Bunch() data.customer = Bunch() data.customer.name = 'my-name' data.customer.email = Bunch() data.customer.email.personal = 'my-personal-email' data.customer.email.business = 'my-business-email' data.customer.address = Bunch() data.customer.address.locality = Bunch() input = MyService._sio.parse_input(data, DATA_FORMAT.JSON) self.assertIsInstance(input, Bunch) self.assertEquals(input.customer.name, data.customer.name) self.assertEquals(input.customer.email.personal, data.customer.email.personal) self.assertEquals(input.customer.email.business, data.customer.email.business) self.assertEquals(input.customer.address.street, _default_input_value) self.assertEquals(input.customer.address.locality.type, locality_default) self.assertEquals(input.customer.address.locality.name, locality_default)
def test_parse_nested_dict_customer_deep_defaults_sio_level(self): locality = Dict('locality', '-type', '-name') address = Dict('address', locality, 'street') email = Dict('email', 'personal', 'business') customer = Dict('customer', 'name', email, address) _default_input_value = 'default-input-value' class MyService(Service): class SimpleIO: input = customer default_input_value = 'default-input-value' CySimpleIO.attach_sio(self.get_server_config(), MyService) # Note that locality has no type nor name and we expect for the SimpleIO-level default value to be used data = Bunch() data.customer = Bunch() data.customer.name = 'my-name' data.customer.email = Bunch() data.customer.email.personal = 'my-personal-email' data.customer.email.business = 'my-business-email' data.customer.address = Bunch() data.customer.address.street = 'my-street' data.customer.address.locality = Bunch() input = MyService._sio.parse_input(data, DATA_FORMAT.JSON) self.assertIsInstance(input, Bunch) self.assertEquals(input.customer.name, data.customer.name) self.assertEquals(input.customer.email.personal, data.customer.email.personal) self.assertEquals(input.customer.email.business, data.customer.email.business) self.assertEquals(input.customer.address.street, data.customer.address.street) self.assertEquals(input.customer.address.locality.type, _default_input_value) self.assertEquals(input.customer.address.locality.name, _default_input_value)
def __init__(self): self.host = None self.port = None self.crypto_manager = None self.odb = None self.odb_data = None self.config = None self.repo_location = None self.user_conf_location = None self.sql_pool_store = None self.soap11_content_type = None self.soap12_content_type = None self.plain_xml_content_type = None self.json_content_type = None self.internal_service_modules = None # Zato's own internal services self.service_modules = None # Set programmatically in Spring self.service_sources = None # Set in a config file self.base_dir = None self.tls_dir = None self.static_dir = None self.hot_deploy_config = None self.pickup = None self.fs_server_config = None self.fs_sql_config = None self.pickup_config = None self.logging_config = None self.logging_conf_path = None self.sio_config = None self.sso_config = None self.connector_server_grace_time = None self.id = None self.name = None self.worker_id = None self.worker_pid = None self.cluster = None self.cluster_id = None self.kvdb = None self.startup_jobs = None self.worker_store = None self.request_dispatcher_dispatch = None self.deployment_lock_expires = None self.deployment_lock_timeout = None self.deployment_key = '' self.app_context = None self.has_gevent = None self.delivery_store = None self.static_config = None self.component_enabled = Bunch() self.client_address_headers = [ 'HTTP_X_ZATO_FORWARDED_FOR', 'HTTP_X_FORWARDED_FOR', 'REMOTE_ADDR' ] self.broker_client = None self.return_tracebacks = None self.default_error_message = None self.time_util = None self.preferred_address = None self.crypto_use_tls = None self.servers = None self.zato_lock_manager = None self.pid = None self.sync_internal = None self.ipc_api = IPCAPI(False) self.ipc_forwarder = IPCAPI(True) self.wmq_ipc_tcp_port = None self.fifo_response_buffer_size = None # Will be in megabytes self.is_first_worker = None self.shmem_size = -1.0 self.server_startup_ipc = ServerStartupIPC() self.keyutils = KeyUtils() self.sso_api = None self.is_sso_enabled = False self.audit_pii = audit_pii self.startup_callable_tool = None self.default_internal_pubsub_endpoint_id = None self._hash_secret_method = None self._hash_secret_rounds = None self._hash_secret_salt_size = None # Allows users store arbitrary data across service invocations self.user_ctx = Bunch() self.user_ctx_lock = gevent.lock.RLock() self.access_logger = logging.getLogger('zato_access_log') self.access_logger_log = self.access_logger._log self.needs_access_log = self.access_logger.isEnabledFor(INFO) self.has_pubsub_audit_log = logging.getLogger( 'zato_pubsub_audit').isEnabledFor('INFO') self.is_enabled_for_warn = logging.getLogger('zato').isEnabledFor( 'WARN') # The main config store self.config = ConfigStore() gevent.signal(signal.SIGINT, self.destroy)
def start_server(parallel_server, zato_deployment_key=None): # Easier to type self = parallel_server # This cannot be done in __init__ because each sub-process obviously has its own PID self.pid = os.getpid() # This also cannot be done in __init__ which doesn't have this variable yet self.is_first_worker = int(os.environ['ZATO_SERVER_WORKER_IDX']) == 0 # Used later on use_tls = asbool(self.fs_server_config.crypto.use_tls) # Will be None if we are not running in background. if not zato_deployment_key: zato_deployment_key = '{}.{}'.format(datetime.utcnow().isoformat(), uuid4().hex) self.deployment_key = zato_deployment_key register_diag_handlers() # Create all POSIX IPC objects now that we have the deployment key self.shmem_size = int(float(self.fs_server_config.shmem.size) * 10**6) # Convert to megabytes as integer self.server_startup_ipc.create(self.deployment_key, self.shmem_size) # Store the ODB configuration, create an ODB connection pool and have self.odb use it self.config.odb_data = self.get_config_odb_data(self) self.set_up_odb() # Now try grabbing the basic server's data from the ODB. No point # in doing anything else if we can't get past this point. server = self.odb.fetch_server(self.config.odb_data) if not server: raise Exception('Server does not exist in the ODB') # Set up the server-wide default lock manager odb_data = self.config.odb_data backend_type = 'fcntl' if odb_data.engine == 'sqlite' else odb_data.engine self.zato_lock_manager = LockManager(backend_type, 'zato', self.odb.session) # Just to make sure distributed locking is configured correctly with self.zato_lock_manager(uuid4().hex): pass # Basic metadata self.id = server.id self.name = server.name self.cluster_id = server.cluster_id self.cluster = self.odb.cluster self.worker_id = '{}.{}.{}.{}'.format(self.cluster_id, self.id, self.worker_pid, new_cid()) # Looked up upfront here and assigned to services in their store self.enforce_service_invokes = asbool( self.fs_server_config.misc.enforce_service_invokes) # For server-to-server communication self.servers = Servers(self.odb, self.cluster.name, self.decrypt) logger.info( 'Preferred address of `%s@%s` (pid: %s) is `http%s://%s:%s`', self.name, self.cluster.name, self.pid, 's' if use_tls else '', self.preferred_address, self.port) # Reads in all configuration from ODB self.worker_store = WorkerStore(self.config, self) self.worker_store.invoke_matcher.read_config( self.fs_server_config.invoke_patterns_allowed) self.worker_store.target_matcher.read_config( self.fs_server_config.invoke_target_patterns_allowed) self.set_up_config(server) # Deploys services is_first, locally_deployed = self._after_init_common(server) # Initializes worker store, including connectors self.worker_store.init() self.request_dispatcher_dispatch = self.worker_store.request_dispatcher.dispatch # Normalize hot-deploy configuration self.hot_deploy_config = Bunch() self.hot_deploy_config.work_dir = os.path.normpath( os.path.join(self.repo_location, self.fs_server_config.hot_deploy.work_dir)) self.hot_deploy_config.backup_history = int( self.fs_server_config.hot_deploy.backup_history) self.hot_deploy_config.backup_format = self.fs_server_config.hot_deploy.backup_format # Configure remaining parts of SSO self.configure_sso() # Cannot be done in __init__ because self.sso_config is not available there yet salt_size = self.sso_config.hash_secret.salt_size self.crypto_manager.add_hash_scheme('zato.default', self.sso_config.hash_secret.rounds, salt_size) for name in ('current_work_dir', 'backup_work_dir', 'last_backup_work_dir', 'delete_after_pick_up'): # New in 2.0 if name == 'delete_after_pick_up': value = asbool(self.fs_server_config.hot_deploy.get( name, True)) self.hot_deploy_config[name] = value else: self.hot_deploy_config[name] = os.path.normpath( os.path.join(self.hot_deploy_config.work_dir, self.fs_server_config.hot_deploy[name])) broker_callbacks = { TOPICS[MESSAGE_TYPE.TO_PARALLEL_ANY]: self.worker_store.on_broker_msg, TOPICS[MESSAGE_TYPE.TO_PARALLEL_ALL]: self.worker_store.on_broker_msg, } self.broker_client = BrokerClient(self.kvdb, 'parallel', broker_callbacks, self.get_lua_programs()) self.worker_store.set_broker_client(self.broker_client) self._after_init_accepted(locally_deployed) self.odb.server_up_down(server.token, SERVER_UP_STATUS.RUNNING, True, self.host, self.port, self.preferred_address, use_tls) if is_first: logger.info('First worker of `%s` is %s', self.name, self.pid) self.startup_callable_tool.invoke( SERVER_STARTUP.PHASE.IN_PROCESS_FIRST, kwargs={ 'parallel_server': self, }) # Startup services self.invoke_startup_services(is_first) spawn_greenlet(self.set_up_pickup) # IPC ipc_forwarder_name = '{}-{}'.format(self.cluster.name, self.name) ipc_forwarder_name = fs_safe_name(ipc_forwarder_name) self.ipc_forwarder.name = ipc_forwarder_name self.ipc_forwarder.pid = self.pid spawn_greenlet(self.ipc_forwarder.run) # Set up IBM MQ connections if that component is enabled if self.fs_server_config.component_enabled.ibm_mq: # Will block for a few seconds at most, until is_ok is returned # which indicates that a connector started or not. is_ok = self.start_ibm_mq_connector( int(self.fs_server_config.ibm_mq.ipc_tcp_start_port)) if is_ok: self.create_initial_wmq_definitions( self.worker_store.worker_config.definition_wmq) self.create_initial_wmq_outconns( self.worker_store.worker_config.out_wmq) self.create_initial_wmq_channels( self.worker_store.worker_config.channel_wmq) else: self.startup_callable_tool.invoke( SERVER_STARTUP.PHASE.IN_PROCESS_OTHER, kwargs={ 'parallel_server': self, }) # IPC self.ipc_api.name = self.name self.ipc_api.pid = self.pid self.ipc_api.on_message_callback = self.worker_store.on_ipc_message spawn_greenlet(self.ipc_api.run) self.startup_callable_tool.invoke(SERVER_STARTUP.PHASE.AFTER_STARTED, kwargs={ 'parallel_server': self, }) logger.info('Started `%s@%s` (pid: %s)', server.name, server.cluster.name, self.pid)
def __init__(self): self.host = None self.port = None self.crypto_manager = None self.odb = None self.odb_data = None self.config = None self.repo_location = None self.user_conf_location = None self.sql_pool_store = None self.soap11_content_type = None self.soap12_content_type = None self.plain_xml_content_type = None self.json_content_type = None self.service_modules = None # Set programmatically in Spring self.service_sources = None # Set in a config file self.base_dir = None # type: unicode self.tls_dir = None # type: unicode self.static_dir = None # type: unicode self.json_schema_dir = None # type: unicode self.hot_deploy_config = None self.pickup = None self.fs_server_config = None self.fs_sql_config = None self.pickup_config = None self.logging_config = None self.logging_conf_path = None self.sio_config = None self.sso_config = None self.connector_server_grace_time = None self.id = None self.name = None self.worker_id = None self.worker_pid = None self.cluster = None self.cluster_id = None self.kvdb = None self.startup_jobs = None self.worker_store = None # type: WorkerStore self.service_store = None # type: ServiceStore self.request_dispatcher_dispatch = None self.deployment_lock_expires = None self.deployment_lock_timeout = None self.deployment_key = '' self.has_gevent = None self.delivery_store = None self.static_config = None self.component_enabled = Bunch() self.client_address_headers = ['HTTP_X_ZATO_FORWARDED_FOR', 'HTTP_X_FORWARDED_FOR', 'REMOTE_ADDR'] self.broker_client = None self.return_tracebacks = None self.default_error_message = None self.time_util = None self.preferred_address = None self.crypto_use_tls = None self.servers = None self.zato_lock_manager = None self.pid = None self.sync_internal = None self.ipc_api = IPCAPI() self.fifo_response_buffer_size = None # Will be in megabytes self.is_first_worker = None self.shmem_size = -1.0 self.server_startup_ipc = ServerStartupIPC() self.connector_config_ipc = ConnectorConfigIPC() self.keyutils = KeyUtils() self.sso_api = None self.is_sso_enabled = False self.audit_pii = audit_pii self.has_fg = False self.startup_callable_tool = None self.default_internal_pubsub_endpoint_id = None self._hash_secret_method = None self._hash_secret_rounds = None self._hash_secret_salt_size = None # Our arbiter may potentially call the cleanup procedure multiple times # and this will be set to True the first time around. self._is_process_closing = False # Allows users store arbitrary data across service invocations self.user_ctx = Bunch() self.user_ctx_lock = gevent.lock.RLock() # Connectors self.connector_ibm_mq = IBMMQIPC(self) self.connector_sftp = SFTPIPC(self) # HTTP methods allowed as a Python list self.http_methods_allowed = [] # As above, but as a regular expression pattern self.http_methods_allowed_re = '' self.access_logger = logging.getLogger('zato_access_log') self.access_logger_log = self.access_logger._log self.needs_access_log = self.access_logger.isEnabledFor(INFO) self.has_pubsub_audit_log = logging.getLogger('zato_pubsub_audit').isEnabledFor(INFO) self.is_enabled_for_warn = logging.getLogger('zato').isEnabledFor(WARN) # The main config store self.config = ConfigStore()
def copy(self): """ Creates a copy of this ConfigStore. All configuration data is copied over except for SQL connections. """ config_store = ConfigStore() # Grab all ConfigDicts - even if they're actually ZATO_NONE - and make their copies for attr_name in dir(self): attr = getattr(self, attr_name) if isinstance(attr, ConfigDict): copy_func = getattr(attr, 'copy') setattr(config_store, attr_name, copy_func()) elif attr is ZATO_NONE: setattr(config_store, attr_name, ZATO_NONE) http_soap = MultiDict() dict_of_lists = self.http_soap.dict_of_lists() for url_path, lists in dict_of_lists.items(): _info = Bunch() for elem in lists: for soap_action, item in elem.items(): _info[soap_action] = Bunch() _info[soap_action].id = item.id _info[soap_action].name = item.name _info[soap_action].is_active = item.is_active _info[soap_action].is_internal = item.is_internal _info[soap_action].url_path = item.url_path _info[soap_action].method = item.method _info[soap_action].soap_version = item.soap_version _info[soap_action].service_id = item.service_id _info[soap_action].service_name = item.service_name _info[soap_action].impl_name = item.impl_name _info[soap_action].transport = item.transport _info[soap_action].connection = item.connection http_soap.add(url_path, _info) config_store.http_soap = http_soap config_store.url_sec = self.url_sec config_store.broker_config = self.broker_config config_store.odb_data = deepcopy(self.odb_data) return config_store
def start_server(parallel_server, zato_deployment_key=None): # Easier to type self = parallel_server # type: ParallelServer # This cannot be done in __init__ because each sub-process obviously has its own PID self.pid = os.getpid() # This also cannot be done in __init__ which doesn't have this variable yet self.is_first_worker = int(os.environ['ZATO_SERVER_WORKER_IDX']) == 0 # Used later on use_tls = asbool(self.fs_server_config.crypto.use_tls) # Will be None if we are not running in background. if not zato_deployment_key: zato_deployment_key = '{}.{}'.format(datetime.utcnow().isoformat(), uuid4().hex) self.deployment_key = zato_deployment_key register_diag_handlers() # Create all POSIX IPC objects now that we have the deployment key self.shmem_size = int(float(self.fs_server_config.shmem.size) * 10**6) # Convert to megabytes as integer self.server_startup_ipc.create(self.deployment_key, self.shmem_size) self.connector_config_ipc.create(self.deployment_key, self.shmem_size) # Store the ODB configuration, create an ODB connection pool and have self.odb use it self.config.odb_data = self.get_config_odb_data(self) self.set_up_odb() # Now try grabbing the basic server's data from the ODB. No point # in doing anything else if we can't get past this point. server = self.odb.fetch_server(self.config.odb_data) if not server: raise Exception('Server does not exist in the ODB') # Set up the server-wide default lock manager odb_data = self.config.odb_data backend_type = 'fcntl' if odb_data.engine == 'sqlite' else odb_data.engine self.zato_lock_manager = LockManager(backend_type, 'zato', self.odb.session) # Just to make sure distributed locking is configured correctly with self.zato_lock_manager(uuid4().hex): pass # Basic metadata self.id = server.id self.name = server.name self.cluster_id = server.cluster_id self.cluster = self.odb.cluster self.worker_id = '{}.{}.{}.{}'.format(self.cluster_id, self.id, self.worker_pid, new_cid()) # Looked up upfront here and assigned to services in their store self.enforce_service_invokes = asbool(self.fs_server_config.misc.enforce_service_invokes) # For server-to-server communication self.servers = Servers(self.odb, self.cluster.name, self.decrypt) logger.info('Preferred address of `%s@%s` (pid: %s) is `http%s://%s:%s`', self.name, self.cluster.name, self.pid, 's' if use_tls else '', self.preferred_address, self.port) # Configure which HTTP methods can be invoked via REST or SOAP channels methods_allowed = self.fs_server_config.http.methods_allowed methods_allowed = methods_allowed if isinstance(methods_allowed, list) else [methods_allowed] self.http_methods_allowed.extend(methods_allowed) # As above, as a regular expression to be used in pattern matching http_methods_allowed_re = '|'.join(self.http_methods_allowed) self.http_methods_allowed_re = '({})'.format(http_methods_allowed_re) # Reads in all configuration from ODB self.worker_store = WorkerStore(self.config, self) self.worker_store.invoke_matcher.read_config(self.fs_server_config.invoke_patterns_allowed) self.worker_store.target_matcher.read_config(self.fs_server_config.invoke_target_patterns_allowed) self.set_up_config(server) # Normalize hot-deploy configuration self.hot_deploy_config = Bunch() self.hot_deploy_config.pickup_dir = absolutize(self.fs_server_config.hot_deploy.pickup_dir, self.repo_location) self.hot_deploy_config.work_dir = os.path.normpath(os.path.join( self.repo_location, self.fs_server_config.hot_deploy.work_dir)) self.hot_deploy_config.backup_history = int(self.fs_server_config.hot_deploy.backup_history) self.hot_deploy_config.backup_format = self.fs_server_config.hot_deploy.backup_format # Added in 3.1, hence optional max_batch_size = int(self.fs_server_config.hot_deploy.get('max_batch_size', 1000)) # Turn it into megabytes max_batch_size = max_batch_size * 1000 # Finally, assign it to ServiceStore self.service_store.max_batch_size = max_batch_size # Deploys services is_first, locally_deployed = self._after_init_common(server) # Initializes worker store, including connectors self.worker_store.init() self.request_dispatcher_dispatch = self.worker_store.request_dispatcher.dispatch # Configure remaining parts of SSO self.configure_sso() # Cannot be done in __init__ because self.sso_config is not available there yet salt_size = self.sso_config.hash_secret.salt_size self.crypto_manager.add_hash_scheme('zato.default', self.sso_config.hash_secret.rounds, salt_size) for name in('current_work_dir', 'backup_work_dir', 'last_backup_work_dir', 'delete_after_pickup'): # New in 2.0 if name == 'delete_after_pickup': # For backward compatibility, we need to support both names old_name = 'delete_after_pick_up' if old_name in self.fs_server_config.hot_deploy: _name = old_name else: _name = name value = asbool(self.fs_server_config.hot_deploy.get(_name, True)) self.hot_deploy_config[name] = value else: self.hot_deploy_config[name] = os.path.normpath(os.path.join( self.hot_deploy_config.work_dir, self.fs_server_config.hot_deploy[name])) broker_callbacks = { TOPICS[MESSAGE_TYPE.TO_PARALLEL_ANY]: self.worker_store.on_broker_msg, TOPICS[MESSAGE_TYPE.TO_PARALLEL_ALL]: self.worker_store.on_broker_msg, } self.broker_client = BrokerClient(self.kvdb, 'parallel', broker_callbacks, self.get_lua_programs()) self.worker_store.set_broker_client(self.broker_client) # Make sure that broker client's connection is ready before continuing # to rule out edge cases where, for instance, hot deployment would # try to publish a locally found package (one of extra packages found) # before the client's thread connected to KVDB. if not self.broker_client.ready: start = now = datetime.utcnow() max_seconds = 120 until = now + timedelta(seconds=max_seconds) while not self.broker_client.ready: now = datetime.utcnow() delta = (now - start).total_seconds() if now < until: # Do not log too early so as not to clutter logs if delta > 2: logger.info('Waiting for broker client to become ready (%s, max:%s)', delta, max_seconds) gevent.sleep(0.5) else: raise Exception('Broker client did not become ready within {} seconds'.format(max_seconds)) self._after_init_accepted(locally_deployed) self.odb.server_up_down( server.token, SERVER_UP_STATUS.RUNNING, True, self.host, self.port, self.preferred_address, use_tls) if is_first: logger.info('First worker of `%s` is %s', self.name, self.pid) self.startup_callable_tool.invoke(SERVER_STARTUP.PHASE.IN_PROCESS_FIRST, kwargs={ 'parallel_server': self, }) # Clean up any old WSX connections possibly registered for this server # which may be still linger around, for instance, if the server was previously # shut down forcibly and did not have an opportunity to run self.cleanup_on_stop self.cleanup_wsx() # Startup services self.invoke_startup_services(is_first) spawn_greenlet(self.set_up_pickup) # Set up subprocess-based IBM MQ connections if that component is enabled if self.fs_server_config.component_enabled.ibm_mq: # Will block for a few seconds at most, until is_ok is returned # which indicates that a connector started or not. is_ok = self.connector_ibm_mq.start_ibm_mq_connector(int(self.fs_server_config.ibm_mq.ipc_tcp_start_port)) try: if is_ok: self.connector_ibm_mq.create_initial_wmq_definitions(self.worker_store.worker_config.definition_wmq) self.connector_ibm_mq.create_initial_wmq_outconns(self.worker_store.worker_config.out_wmq) self.connector_ibm_mq.create_initial_wmq_channels(self.worker_store.worker_config.channel_wmq) except Exception as e: logger.warn('Could not create initial IBM MQ objects, e:`%s`', e) # Set up subprocess-based SFTP connections is_ok = self.connector_sftp.start_sftp_connector(int(self.fs_server_config.ibm_mq.ipc_tcp_start_port)) if is_ok: self.connector_sftp.create_initial_sftp_outconns(self.worker_store.worker_config.out_sftp) else: self.startup_callable_tool.invoke(SERVER_STARTUP.PHASE.IN_PROCESS_OTHER, kwargs={ 'parallel_server': self, }) # IPC self.ipc_api.name = self.ipc_api.get_endpoint_name(self.cluster.name, self.name, self.pid) self.ipc_api.pid = self.pid self.ipc_api.on_message_callback = self.worker_store.on_ipc_message spawn_greenlet(self.ipc_api.run) self.startup_callable_tool.invoke(SERVER_STARTUP.PHASE.AFTER_STARTED, kwargs={ 'parallel_server': self, }) logger.info('Started `%s@%s` (pid: %s)', server.name, server.cluster.name, self.pid)
def test_parse_nested_dict_all_sio_elems_some_missing(self): _default_input_value = 'default-input-value' default_locality = 'default-locality' default_address = 'default-address' locality = Dict('locality', Int('type'), Text('name'), AsIs('-coords'), Decimal('geo_skip'), Float('geo_diff'), default=default_locality) address = Dict('address', locality, UUID('-street_id'), CSV('prefs'), DateTime('since'), List('types'), Opaque('opaque1'), default=default_address) email = Dict('email', Text('value'), Bool('is_business'), Date('-join_date'), DictList('preferred_order', 'name', '-pos')) customer = Dict('customer', 'name', email, address) class MyService(Service): class SimpleIO: input = customer default_input_value = 'default-input-value' CySimpleIO.attach_sio(self.get_server_config(), MyService) # Note that 'join_date', 'street_id', 'coords' and one of 'pos' keys are missing in input below, # the test ensures that default values are used in their place. data = Bunch() data.customer = Bunch() data.customer.name = 'my-name' data.customer.email = Bunch() data.customer.email.value = 'my-email' data.customer.email.is_business = True data.customer.email.preferred_order = [{ 'name': 'address2', 'pos': '2' }, { 'name': 'address1' }] data.customer.address = Bunch() data.customer.address.locality = Bunch() data.customer.address.locality.type = '111' data.customer.address.locality.name = 'my-locality' data.customer.address.locality.geo_skip = '123.456' data.customer.address.locality.geo_diff = '999.777' data.customer.address.prefs = '1,2,3,4' data.customer.address.since = '27-11-1988T11:22:33' data.customer.address.types = ['a', 'b', 'c', 'd'] data.customer.address.opaque1 = object() input = MyService._sio.parse_input(data, DATA_FORMAT.JSON) self.assertIsInstance(input, Bunch) self.assertEquals(input.customer.name, data.customer.name) self.assertEquals(input.customer.email.value, data.customer.email.value) self.assertEquals(input.customer.email.is_business, data.customer.email.is_business) self.assertEquals(input.customer.email.join_date, _default_input_value) self.assertDictEqual(input.customer.email.preferred_order[0], data.customer.email.preferred_order[0]) self.assertEquals(input.customer.email.preferred_order[1].name, data.customer.email.preferred_order[1]['name']) self.assertEquals(input.customer.email.preferred_order[1].pos, _default_input_value) self.assertEquals(input.customer.address.locality.type, int(data.customer.address.locality.type)) self.assertEquals(input.customer.address.locality.name, data.customer.address.locality.name) self.assertEquals(input.customer.address.locality.coords, default_locality) self.assertEquals( input.customer.address.locality.geo_skip, decimal_Decimal(data.customer.address.locality.geo_skip)) self.assertEquals(input.customer.address.locality.geo_diff, float(data.customer.address.locality.geo_diff)) self.assertEquals(input.customer.address.street_id, default_address) self.assertEquals(input.customer.address.prefs, data.customer.address.prefs.split(',')) self.assertEquals(input.customer.address.since, dt_parse(data.customer.address.since)) self.assertEquals(input.customer.address.types, data.customer.address.types) self.assertIs(input.customer.address.opaque1, data.customer.address.opaque1)
def set_up_config(self, server): # Which components are enabled self.component_enabled.stats = asbool(self.fs_server_config.component_enabled.stats) self.component_enabled.slow_response = asbool(self.fs_server_config.component_enabled.slow_response) self.component_enabled.live_msg_browser = asbool(self.fs_server_config.component_enabled.live_msg_browser) # Details of what is enabled in live message browser self.live_msg_browser = self.fs_server_config.live_msg_browser self.live_msg_browser.include_internal = asbool(self.live_msg_browser.include_internal) # Pub/sub self.pubsub = PubSubAPI(RedisPubSub(self.kvdb.conn)) # # Cassandra - start # query = self.odb.get_cassandra_conn_list(server.cluster.id, True) self.config.cassandra_conn = ConfigDict.from_query('cassandra_conn', query) query = self.odb.get_cassandra_query_list(server.cluster.id, True) self.config.cassandra_query = ConfigDict.from_query('cassandra_query', query) # # Cassandra - end # # # Search - start # query = self.odb.get_search_es_list(server.cluster.id, True) self.config.search_es = ConfigDict.from_query('search_es', query) query = self.odb.get_search_solr_list(server.cluster.id, True) self.config.search_solr = ConfigDict.from_query('search_solr', query) # # Search - end # # # Cloud - start # # OpenStack - Swift query = self.odb.get_cloud_openstack_swift_list(server.cluster.id, True) self.config.cloud_openstack_swift = ConfigDict.from_query('cloud_openstack_swift', query) query = self.odb.get_cloud_aws_s3_list(server.cluster.id, True) self.config.cloud_aws_s3 = ConfigDict.from_query('cloud_aws_s3', query) # # Cloud - end # # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ # Services query = self.odb.get_service_list(server.cluster.id, True) self.config.service = ConfigDict.from_query('service_list', query) # # Definitions - start # # AMQP query = self.odb.get_definition_amqp_list(server.cluster.id, True) self.config.definition_amqp = ConfigDict.from_query('definition_amqp', query) # # Definitions - end # # # Channels - start # # AMQP query = self.odb.get_channel_amqp_list(server.cluster.id, True) self.config.channel_amqp = ConfigDict.from_query('channel_amqp', query) # STOMP query = self.odb.get_channel_stomp_list(server.cluster.id, True) self.config.channel_stomp = ConfigDict.from_query('channel_stomp', query) # # Channels - end # # # Outgoing connections - start # # AMQP query = self.odb.get_out_amqp_list(server.cluster.id, True) self.config.out_amqp = ConfigDict.from_query('out_amqp', query) # FTP query = self.odb.get_out_ftp_list(server.cluster.id, True) self.config.out_ftp = ConfigDict.from_query('out_ftp', query) # JMS WMQ query = self.odb.get_out_jms_wmq_list(server.cluster.id, True) self.config.out_jms_wmq = ConfigDict.from_query('out_jms_wmq', query) # Odoo query = self.odb.get_out_odoo_list(server.cluster.id, True) self.config.out_odoo = ConfigDict.from_query('out_odoo', query) # Plain HTTP query = self.odb.get_http_soap_list(server.cluster.id, 'outgoing', 'plain_http', True) self.config.out_plain_http = ConfigDict.from_query('out_plain_http', query) # SOAP query = self.odb.get_http_soap_list(server.cluster.id, 'outgoing', 'soap', True) self.config.out_soap = ConfigDict.from_query('out_soap', query) # SQL query = self.odb.get_out_sql_list(server.cluster.id, True) self.config.out_sql = ConfigDict.from_query('out_sql', query) # STOMP query = self.odb.get_out_stomp_list(server.cluster.id, True) self.config.out_stomp = ConfigDict.from_query('out_stomp', query) # ZMQ channels query = self.odb.get_channel_zmq_list(server.cluster.id, True) self.config.channel_zmq = ConfigDict.from_query('channel_zmq', query) # ZMQ outgoing query = self.odb.get_out_zmq_list(server.cluster.id, True) self.config.out_zmq = ConfigDict.from_query('out_zmq', query) # WebSocket channels query = self.odb.get_channel_web_socket_list(server.cluster.id, True) self.config.channel_web_socket = ConfigDict.from_query('channel_web_socket', query) # # Outgoing connections - end # # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ # # Notifications - start # # OpenStack Swift query = self.odb.get_notif_cloud_openstack_swift_list(server.cluster.id, True) self.config.notif_cloud_openstack_swift = ConfigDict.from_query('notif_cloud_openstack_swift', query) # SQL query = self.odb.get_notif_sql_list(server.cluster.id, True) self.config.notif_sql = ConfigDict.from_query('notif_sql', query) # # Notifications - end # # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ # # Security - start # # API keys query = self.odb.get_apikey_security_list(server.cluster.id, True) self.config.apikey = ConfigDict.from_query('apikey', query) # AWS query = self.odb.get_aws_security_list(server.cluster.id, True) self.config.aws = ConfigDict.from_query('aws', query) # HTTP Basic Auth query = self.odb.get_basic_auth_list(server.cluster.id, None, True) self.config.basic_auth = ConfigDict.from_query('basic_auth', query) # HTTP Basic Auth query = self.odb.get_jwt_list(server.cluster.id, None, True) self.config.jwt = ConfigDict.from_query('jwt', query) # NTLM query = self.odb.get_ntlm_list(server.cluster.id, True) self.config.ntlm = ConfigDict.from_query('ntlm', query) # OAuth query = self.odb.get_oauth_list(server.cluster.id, True) self.config.oauth = ConfigDict.from_query('oauth', query) # OpenStack query = self.odb.get_openstack_security_list(server.cluster.id, True) self.config.openstack_security = ConfigDict.from_query('openstack_security', query) # RBAC - permissions query = self.odb.get_rbac_permission_list(server.cluster.id, True) self.config.rbac_permission = ConfigDict.from_query('rbac_permission', query) # RBAC - roles query = self.odb.get_rbac_role_list(server.cluster.id, True) self.config.rbac_role = ConfigDict.from_query('rbac_role', query) # RBAC - client roles query = self.odb.get_rbac_client_role_list(server.cluster.id, True) self.config.rbac_client_role = ConfigDict.from_query('rbac_client_role', query) # RBAC - role permission query = self.odb.get_rbac_role_permission_list(server.cluster.id, True) self.config.rbac_role_permission = ConfigDict.from_query('rbac_role_permission', query) # Technical accounts query = self.odb.get_tech_acc_list(server.cluster.id, True) self.config.tech_acc = ConfigDict.from_query('tech_acc', query) # TLS CA certs query = self.odb.get_tls_ca_cert_list(server.cluster.id, True) self.config.tls_ca_cert = ConfigDict.from_query('tls_ca_cert', query) # TLS channel security query = self.odb.get_tls_channel_sec_list(server.cluster.id, True) self.config.tls_channel_sec = ConfigDict.from_query('tls_channel_sec', query) # TLS key/cert pairs query = self.odb.get_tls_key_cert_list(server.cluster.id, True) self.config.tls_key_cert = ConfigDict.from_query('tls_key_cert', query) # WS-Security query = self.odb.get_wss_list(server.cluster.id, True) self.config.wss = ConfigDict.from_query('wss', query) # Vault connections query = self.odb.get_vault_connection_list(server.cluster.id, True) self.config.vault_conn_sec = ConfigDict.from_query('vault_conn_sec', query) # XPath query = self.odb.get_xpath_sec_list(server.cluster.id, True) self.config.xpath_sec = ConfigDict.from_query('xpath_sec', query) # # Security - end # # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ # All the HTTP/SOAP channels. http_soap = [] for item in self.odb.get_http_soap_list(server.cluster.id, 'channel'): hs_item = {} for key in item.keys(): hs_item[key] = getattr(item, key) hs_item['replace_patterns_json_pointer'] = item.replace_patterns_json_pointer hs_item['replace_patterns_xpath'] = item.replace_patterns_xpath hs_item['match_target'] = '{}{}{}'.format(hs_item['soap_action'], MISC.SEPARATOR, hs_item['url_path']) hs_item['match_target_compiled'] = Matcher(hs_item['match_target']) http_soap.append(hs_item) self.config.http_soap = http_soap # Namespaces query = self.odb.get_namespace_list(server.cluster.id, True) self.config.msg_ns = ConfigDict.from_query('msg_ns', query) # XPath query = self.odb.get_xpath_list(server.cluster.id, True) self.config.xpath = ConfigDict.from_query('msg_xpath', query) # JSON Pointer query = self.odb.get_json_pointer_list(server.cluster.id, True) self.config.json_pointer = ConfigDict.from_query('json_pointer', query) # SimpleIO self.config.simple_io = ConfigDict('simple_io', Bunch()) self.config.simple_io['int_parameters'] = self.int_parameters self.config.simple_io['int_parameter_suffixes'] = self.int_parameter_suffixes self.config.simple_io['bool_parameter_prefixes'] = self.bool_parameter_prefixes # Pub/sub config self.config.pubsub = Bunch() self.config.pubsub.default_consumer = Bunch() self.config.pubsub.default_producer = Bunch() query = self.odb.get_pubsub_topic_list(server.cluster.id, True) self.config.pubsub.topics = ConfigDict.from_query('pubsub_topics', query) id, name = self.odb.get_pubsub_default_client(server.cluster.id, 'zato.pubsub.default-consumer') self.config.pubsub.default_consumer.id, self.config.pubsub.default_consumer.name = id, name id, name = self.odb.get_pubsub_default_client(server.cluster.id, 'zato.pubsub.default-producer') self.config.pubsub.default_producer.id, self.config.pubsub.default_producer.name = id, name query = self.odb.get_pubsub_producer_list(server.cluster.id, True) self.config.pubsub.producers = ConfigDict.from_query('pubsub_producers', query, list_config=True) query = self.odb.get_pubsub_consumer_list(server.cluster.id, True) self.config.pubsub.consumers = ConfigDict.from_query('pubsub_consumers', query, list_config=True) # E-mail - SMTP query = self.odb.get_email_smtp_list(server.cluster.id, True) self.config.email_smtp = ConfigDict.from_query('email_smtp', query) # E-mail - IMAP query = self.odb.get_email_imap_list(server.cluster.id, True) self.config.email_imap = ConfigDict.from_query('email_imap', query) # Message paths self.config.msg_ns_store = NamespaceStore() self.config.json_pointer_store = JSONPointerStore() self.config.xpath_store = XPathStore() # Assign config to worker self.worker_store.worker_config = self.config self.worker_store.pubsub = self.pubsub
def set_up_config(self, server): # Which components are enabled self.component_enabled.stats = asbool(self.fs_server_config.component_enabled.stats) self.component_enabled.slow_response = asbool(self.fs_server_config.component_enabled.slow_response) # # Cassandra - start # query = self.odb.get_cassandra_conn_list(server.cluster.id, True) self.config.cassandra_conn = ConfigDict.from_query('cassandra_conn', query, decrypt_func=self.decrypt) query = self.odb.get_cassandra_query_list(server.cluster.id, True) self.config.cassandra_query = ConfigDict.from_query('cassandra_query', query, decrypt_func=self.decrypt) # # Cassandra - end # # # Search - start # query = self.odb.get_search_es_list(server.cluster.id, True) self.config.search_es = ConfigDict.from_query('search_es', query, decrypt_func=self.decrypt) query = self.odb.get_search_solr_list(server.cluster.id, True) self.config.search_solr = ConfigDict.from_query('search_solr', query, decrypt_func=self.decrypt) # # Search - end # # # SMS - start # query = self.odb.get_sms_twilio_list(server.cluster.id, True) self.config.sms_twilio = ConfigDict.from_query('sms_twilio', query, decrypt_func=self.decrypt) # # SMS - end # # # Cloud - start # # OpenStack - Swift query = self.odb.get_cloud_openstack_swift_list(server.cluster.id, True) self.config.cloud_openstack_swift = ConfigDict.from_query('cloud_openstack_swift', query, decrypt_func=self.decrypt) query = self.odb.get_cloud_aws_s3_list(server.cluster.id, True) self.config.cloud_aws_s3 = ConfigDict.from_query('cloud_aws_s3', query, decrypt_func=self.decrypt) # # Cloud - end # # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ # Services query = self.odb.get_service_list(server.cluster.id, True) self.config.service = ConfigDict.from_query('service_list', query, decrypt_func=self.decrypt) # # Definitions - start # # AMQP query = self.odb.get_definition_amqp_list(server.cluster.id, True) self.config.definition_amqp = ConfigDict.from_query('definition_amqp', query, decrypt_func=self.decrypt) # IBM MQ query = self.odb.get_definition_wmq_list(server.cluster.id, True) self.config.definition_wmq = ConfigDict.from_query('definition_wmq', query, decrypt_func=self.decrypt) # # Definitions - end # # # Channels - start # # AMQP query = self.odb.get_channel_amqp_list(server.cluster.id, True) self.config.channel_amqp = ConfigDict.from_query('channel_amqp', query, decrypt_func=self.decrypt) # STOMP query = self.odb.get_channel_stomp_list(server.cluster.id, True) self.config.channel_stomp = ConfigDict.from_query('channel_stomp', query, decrypt_func=self.decrypt) # IBM MQ query = self.odb.get_channel_wmq_list(server.cluster.id, True) self.config.channel_wmq = ConfigDict.from_query('channel_wmq', query, decrypt_func=self.decrypt) # # Channels - end # # # Outgoing connections - start # # AMQP query = self.odb.get_out_amqp_list(server.cluster.id, True) self.config.out_amqp = ConfigDict.from_query('out_amqp', query, decrypt_func=self.decrypt) # Caches query = self.odb.get_cache_builtin_list(server.cluster.id, True) self.config.cache_builtin = ConfigDict.from_query('cache_builtin', query, decrypt_func=self.decrypt) query = self.odb.get_cache_memcached_list(server.cluster.id, True) self.config.cache_memcached = ConfigDict.from_query('cache_memcached', query, decrypt_func=self.decrypt) # FTP query = self.odb.get_out_ftp_list(server.cluster.id, True) self.config.out_ftp = ConfigDict.from_query('out_ftp', query, decrypt_func=self.decrypt) # IBM MQ query = self.odb.get_out_wmq_list(server.cluster.id, True) self.config.out_wmq = ConfigDict.from_query('out_wmq', query, decrypt_func=self.decrypt) # Odoo query = self.odb.get_out_odoo_list(server.cluster.id, True) self.config.out_odoo = ConfigDict.from_query('out_odoo', query, decrypt_func=self.decrypt) # SAP RFC query = self.odb.get_out_sap_list(server.cluster.id, True) self.config.out_sap = ConfigDict.from_query('out_sap', query, decrypt_func=self.decrypt) # REST query = self.odb.get_http_soap_list(server.cluster.id, 'outgoing', 'plain_http', True) self.config.out_plain_http = ConfigDict.from_query('out_plain_http', query, decrypt_func=self.decrypt) # SFTP query = self.odb.get_out_sftp_list(server.cluster.id, True) self.config.out_sftp = ConfigDict.from_query('out_sftp', query, decrypt_func=self.decrypt, drop_opaque=True) # SOAP query = self.odb.get_http_soap_list(server.cluster.id, 'outgoing', 'soap', True) self.config.out_soap = ConfigDict.from_query('out_soap', query, decrypt_func=self.decrypt) # SQL query = self.odb.get_out_sql_list(server.cluster.id, True) self.config.out_sql = ConfigDict.from_query('out_sql', query, decrypt_func=self.decrypt) # STOMP query = self.odb.get_out_stomp_list(server.cluster.id, True) self.config.out_stomp = ConfigDict.from_query('out_stomp', query, decrypt_func=self.decrypt) # ZMQ channels query = self.odb.get_channel_zmq_list(server.cluster.id, True) self.config.channel_zmq = ConfigDict.from_query('channel_zmq', query, decrypt_func=self.decrypt) # ZMQ outgoing query = self.odb.get_out_zmq_list(server.cluster.id, True) self.config.out_zmq = ConfigDict.from_query('out_zmq', query, decrypt_func=self.decrypt) # WebSocket channels query = self.odb.get_channel_web_socket_list(server.cluster.id, True) self.config.channel_web_socket = ConfigDict.from_query('channel_web_socket', query, decrypt_func=self.decrypt) # # Outgoing connections - end # # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ # # Generic - start # # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ # Connections query = self.odb.get_generic_connection_list(server.cluster.id, True) self.config.generic_connection = ConfigDict.from_query('generic_connection', query, decrypt_func=self.decrypt) # # Generic - end # # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ # # Notifications - start # # OpenStack Swift query = self.odb.get_notif_cloud_openstack_swift_list(server.cluster.id, True) self.config.notif_cloud_openstack_swift = ConfigDict.from_query('notif_cloud_openstack_swift', query, decrypt_func=self.decrypt) # SQL query = self.odb.get_notif_sql_list(server.cluster.id, True) self.config.notif_sql = ConfigDict.from_query('notif_sql', query, decrypt_func=self.decrypt) # # Notifications - end # # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ # # Security - start # # API keys query = self.odb.get_apikey_security_list(server.cluster.id, True) self.config.apikey = ConfigDict.from_query('apikey', query, decrypt_func=self.decrypt) # AWS query = self.odb.get_aws_security_list(server.cluster.id, True) self.config.aws = ConfigDict.from_query('aws', query, decrypt_func=self.decrypt) # HTTP Basic Auth query = self.odb.get_basic_auth_list(server.cluster.id, None, True) self.config.basic_auth = ConfigDict.from_query('basic_auth', query, decrypt_func=self.decrypt) # JWT query = self.odb.get_jwt_list(server.cluster.id, None, True) self.config.jwt = ConfigDict.from_query('jwt', query, decrypt_func=self.decrypt) # NTLM query = self.odb.get_ntlm_list(server.cluster.id, True) self.config.ntlm = ConfigDict.from_query('ntlm', query, decrypt_func=self.decrypt) # OAuth query = self.odb.get_oauth_list(server.cluster.id, True) self.config.oauth = ConfigDict.from_query('oauth', query, decrypt_func=self.decrypt) # OpenStack query = self.odb.get_openstack_security_list(server.cluster.id, True) self.config.openstack_security = ConfigDict.from_query('openstack_security', query, decrypt_func=self.decrypt) # RBAC - permissions query = self.odb.get_rbac_permission_list(server.cluster.id, True) self.config.rbac_permission = ConfigDict.from_query('rbac_permission', query, decrypt_func=self.decrypt) # RBAC - roles query = self.odb.get_rbac_role_list(server.cluster.id, True) self.config.rbac_role = ConfigDict.from_query('rbac_role', query, decrypt_func=self.decrypt) # RBAC - client roles query = self.odb.get_rbac_client_role_list(server.cluster.id, True) self.config.rbac_client_role = ConfigDict.from_query('rbac_client_role', query, decrypt_func=self.decrypt) # RBAC - role permission query = self.odb.get_rbac_role_permission_list(server.cluster.id, True) self.config.rbac_role_permission = ConfigDict.from_query('rbac_role_permission', query, decrypt_func=self.decrypt) # TLS CA certs query = self.odb.get_tls_ca_cert_list(server.cluster.id, True) self.config.tls_ca_cert = ConfigDict.from_query('tls_ca_cert', query, decrypt_func=self.decrypt) # TLS channel security query = self.odb.get_tls_channel_sec_list(server.cluster.id, True) self.config.tls_channel_sec = ConfigDict.from_query('tls_channel_sec', query, decrypt_func=self.decrypt) # TLS key/cert pairs query = self.odb.get_tls_key_cert_list(server.cluster.id, True) self.config.tls_key_cert = ConfigDict.from_query('tls_key_cert', query, decrypt_func=self.decrypt) # WS-Security query = self.odb.get_wss_list(server.cluster.id, True) self.config.wss = ConfigDict.from_query('wss', query, decrypt_func=self.decrypt) # Vault connections query = self.odb.get_vault_connection_list(server.cluster.id, True) self.config.vault_conn_sec = ConfigDict.from_query('vault_conn_sec', query, decrypt_func=self.decrypt) # XPath query = self.odb.get_xpath_sec_list(server.cluster.id, True) self.config.xpath_sec = ConfigDict.from_query('xpath_sec', query, decrypt_func=self.decrypt) # Encrypt all secrets self._encrypt_secrets() # # Security - end # # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ # All the HTTP/SOAP channels. http_soap = [] for item in elems_with_opaque(self.odb.get_http_soap_list(server.cluster.id, 'channel')): hs_item = {} for key in item.keys(): hs_item[key] = getattr(item, key) hs_item['match_target'] = get_match_target(hs_item, http_methods_allowed_re=self.http_methods_allowed_re) hs_item['match_target_compiled'] = Matcher(hs_item['match_target'], hs_item.get('match_slash', '')) http_soap.append(hs_item) self.config.http_soap = http_soap # Namespaces query = self.odb.get_namespace_list(server.cluster.id, True) self.config.msg_ns = ConfigDict.from_query('msg_ns', query, decrypt_func=self.decrypt) # XPath query = self.odb.get_xpath_list(server.cluster.id, True) self.config.xpath = ConfigDict.from_query('msg_xpath', query, decrypt_func=self.decrypt) # JSON Pointer query = self.odb.get_json_pointer_list(server.cluster.id, True) self.config.json_pointer = ConfigDict.from_query('json_pointer', query, decrypt_func=self.decrypt) # SimpleIO # In preparation for a SIO rewrite, we loaded SIO config from a file # but actual code paths require the pre-3.0 format so let's prepare it here. self.config.simple_io = ConfigDict('simple_io', Bunch()) int_exact = self.sio_config.int.exact int_suffix = self.sio_config.int.suffix bool_prefix = self.sio_config.bool.prefix self.config.simple_io['int_parameters'] = int_exact if isinstance(int_exact, list) else [int_exact] self.config.simple_io['int_parameter_suffixes'] = int_suffix if isinstance(int_suffix, list) else [int_suffix] self.config.simple_io['bool_parameter_prefixes'] = bool_prefix if isinstance(bool_prefix, list) else [bool_prefix] # Maintain backward-compatibility with pre-3.1 versions that did not specify any particular encoding bytes_to_str = self.sio_config.get('bytes_to_str') if not bytes_to_str: bytes_to_str = {'encoding': None} self.config.simple_io['bytes_to_str'] = bytes_to_str # Pub/sub self.config.pubsub = Bunch() # Pub/sub - endpoints query = self.odb.get_pubsub_endpoint_list(server.cluster.id, True) self.config.pubsub_endpoint = ConfigDict.from_query('pubsub_endpoint', query, decrypt_func=self.decrypt) # Pub/sub - topics query = self.odb.get_pubsub_topic_list(server.cluster.id, True) self.config.pubsub_topic = ConfigDict.from_query('pubsub_topic', query, decrypt_func=self.decrypt) # Pub/sub - subscriptions query = self.odb.get_pubsub_subscription_list(server.cluster.id, True) self.config.pubsub_subscription = ConfigDict.from_query('pubsub_subscription', query, decrypt_func=self.decrypt) # E-mail - SMTP query = self.odb.get_email_smtp_list(server.cluster.id, True) self.config.email_smtp = ConfigDict.from_query('email_smtp', query, decrypt_func=self.decrypt) # E-mail - IMAP query = self.odb.get_email_imap_list(server.cluster.id, True) self.config.email_imap = ConfigDict.from_query('email_imap', query, decrypt_func=self.decrypt) # Message paths self.config.msg_ns_store = NamespaceStore() self.config.json_pointer_store = JSONPointerStore() self.config.xpath_store = XPathStore() # HTTP access log should optionally ignore certain requests access_log_ignore = self.fs_server_config.get('logging', {}).get('http_access_log_ignore') if access_log_ignore: access_log_ignore = access_log_ignore if isinstance(access_log_ignore, list) else [access_log_ignore] self.needs_all_access_log = False self.access_log_ignore.update(access_log_ignore) # Assign config to worker self.worker_store.worker_config = self.config
def set_up_config(self, server): # Which components are enabled self.component_enabled.stats = asbool(self.fs_server_config.component_enabled.stats) self.component_enabled.slow_response = asbool(self.fs_server_config.component_enabled.slow_response) self.component_enabled.live_msg_browser = asbool(self.fs_server_config.component_enabled.live_msg_browser) # Details of what is enabled in live message browser self.live_msg_browser = self.fs_server_config.live_msg_browser self.live_msg_browser.include_internal = asbool(self.live_msg_browser.include_internal) # # Cassandra - start # query = self.odb.get_cassandra_conn_list(server.cluster.id, True) self.config.cassandra_conn = ConfigDict.from_query('cassandra_conn', query, decrypt_func=self.decrypt) query = self.odb.get_cassandra_query_list(server.cluster.id, True) self.config.cassandra_query = ConfigDict.from_query('cassandra_query', query, decrypt_func=self.decrypt) # # Cassandra - end # # # Search - start # query = self.odb.get_search_es_list(server.cluster.id, True) self.config.search_es = ConfigDict.from_query('search_es', query, decrypt_func=self.decrypt) query = self.odb.get_search_solr_list(server.cluster.id, True) self.config.search_solr = ConfigDict.from_query('search_solr', query, decrypt_func=self.decrypt) # # Search - end # # # SMS - start # query = self.odb.get_sms_twilio_list(server.cluster.id, True) self.config.sms_twilio = ConfigDict.from_query('sms_twilio', query, decrypt_func=self.decrypt) # # SMS - end # # # Cloud - start # # OpenStack - Swift query = self.odb.get_cloud_openstack_swift_list(server.cluster.id, True) self.config.cloud_openstack_swift = ConfigDict.from_query('cloud_openstack_swift', query, decrypt_func=self.decrypt) query = self.odb.get_cloud_aws_s3_list(server.cluster.id, True) self.config.cloud_aws_s3 = ConfigDict.from_query('cloud_aws_s3', query, decrypt_func=self.decrypt) # # Cloud - end # # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ # Services query = self.odb.get_service_list(server.cluster.id, True) self.config.service = ConfigDict.from_query('service_list', query, decrypt_func=self.decrypt) # # Definitions - start # # AMQP query = self.odb.get_definition_amqp_list(server.cluster.id, True) self.config.definition_amqp = ConfigDict.from_query('definition_amqp', query, decrypt_func=self.decrypt) query = self.odb.get_definition_wmq_list(server.cluster.id, True) self.config.definition_wmq = ConfigDict.from_query('definition_wmq', query, decrypt_func=self.decrypt) # # Definitions - end # # # Channels - start # # AMQP query = self.odb.get_channel_amqp_list(server.cluster.id, True) self.config.channel_amqp = ConfigDict.from_query('channel_amqp', query, decrypt_func=self.decrypt) # STOMP query = self.odb.get_channel_stomp_list(server.cluster.id, True) self.config.channel_stomp = ConfigDict.from_query('channel_stomp', query, decrypt_func=self.decrypt) # IBM MQ query = self.odb.get_channel_wmq_list(server.cluster.id, True) self.config.channel_wmq = ConfigDict.from_query('channel_wmq', query, decrypt_func=self.decrypt) # # Channels - end # # # Outgoing connections - start # # AMQP query = self.odb.get_out_amqp_list(server.cluster.id, True) self.config.out_amqp = ConfigDict.from_query('out_amqp', query, decrypt_func=self.decrypt) # Caches query = self.odb.get_cache_builtin_list(server.cluster.id, True) self.config.cache_builtin = ConfigDict.from_query('cache_builtin', query, decrypt_func=self.decrypt) query = self.odb.get_cache_memcached_list(server.cluster.id, True) self.config.cache_memcached = ConfigDict.from_query('cache_memcached', query, decrypt_func=self.decrypt) # FTP query = self.odb.get_out_ftp_list(server.cluster.id, True) self.config.out_ftp = ConfigDict.from_query('out_ftp', query, decrypt_func=self.decrypt) # IBM MQ query = self.odb.get_out_wmq_list(server.cluster.id, True) self.config.out_wmq = ConfigDict.from_query('out_wmq', query, decrypt_func=self.decrypt) # Odoo query = self.odb.get_out_odoo_list(server.cluster.id, True) self.config.out_odoo = ConfigDict.from_query('out_odoo', query, decrypt_func=self.decrypt) # Plain HTTP query = self.odb.get_http_soap_list(server.cluster.id, 'outgoing', 'plain_http', True) self.config.out_plain_http = ConfigDict.from_query('out_plain_http', query, decrypt_func=self.decrypt) # SOAP query = self.odb.get_http_soap_list(server.cluster.id, 'outgoing', 'soap', True) self.config.out_soap = ConfigDict.from_query('out_soap', query, decrypt_func=self.decrypt) # SQL query = self.odb.get_out_sql_list(server.cluster.id, True) self.config.out_sql = ConfigDict.from_query('out_sql', query, decrypt_func=self.decrypt) # STOMP query = self.odb.get_out_stomp_list(server.cluster.id, True) self.config.out_stomp = ConfigDict.from_query('out_stomp', query, decrypt_func=self.decrypt) # ZMQ channels query = self.odb.get_channel_zmq_list(server.cluster.id, True) self.config.channel_zmq = ConfigDict.from_query('channel_zmq', query, decrypt_func=self.decrypt) # ZMQ outgoing query = self.odb.get_out_zmq_list(server.cluster.id, True) self.config.out_zmq = ConfigDict.from_query('out_zmq', query, decrypt_func=self.decrypt) # WebSocket channels query = self.odb.get_channel_web_socket_list(server.cluster.id, True) self.config.channel_web_socket = ConfigDict.from_query('channel_web_socket', query, decrypt_func=self.decrypt) # # Outgoing connections - end # # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ # # Notifications - start # # OpenStack Swift query = self.odb.get_notif_cloud_openstack_swift_list(server.cluster.id, True) self.config.notif_cloud_openstack_swift = ConfigDict.from_query('notif_cloud_openstack_swift', query, decrypt_func=self.decrypt) # SQL query = self.odb.get_notif_sql_list(server.cluster.id, True) self.config.notif_sql = ConfigDict.from_query('notif_sql', query, decrypt_func=self.decrypt) # # Notifications - end # # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ # # Security - start # # API keys query = self.odb.get_apikey_security_list(server.cluster.id, True) self.config.apikey = ConfigDict.from_query('apikey', query, decrypt_func=self.decrypt) # AWS query = self.odb.get_aws_security_list(server.cluster.id, True) self.config.aws = ConfigDict.from_query('aws', query, decrypt_func=self.decrypt) # HTTP Basic Auth query = self.odb.get_basic_auth_list(server.cluster.id, None, True) self.config.basic_auth = ConfigDict.from_query('basic_auth', query, decrypt_func=self.decrypt) # JWT query = self.odb.get_jwt_list(server.cluster.id, None, True) self.config.jwt = ConfigDict.from_query('jwt', query, decrypt_func=self.decrypt) # NTLM query = self.odb.get_ntlm_list(server.cluster.id, True) self.config.ntlm = ConfigDict.from_query('ntlm', query, decrypt_func=self.decrypt) # OAuth query = self.odb.get_oauth_list(server.cluster.id, True) self.config.oauth = ConfigDict.from_query('oauth', query, decrypt_func=self.decrypt) # OpenStack query = self.odb.get_openstack_security_list(server.cluster.id, True) self.config.openstack_security = ConfigDict.from_query('openstack_security', query, decrypt_func=self.decrypt) # RBAC - permissions query = self.odb.get_rbac_permission_list(server.cluster.id, True) self.config.rbac_permission = ConfigDict.from_query('rbac_permission', query, decrypt_func=self.decrypt) # RBAC - roles query = self.odb.get_rbac_role_list(server.cluster.id, True) self.config.rbac_role = ConfigDict.from_query('rbac_role', query, decrypt_func=self.decrypt) # RBAC - client roles query = self.odb.get_rbac_client_role_list(server.cluster.id, True) self.config.rbac_client_role = ConfigDict.from_query('rbac_client_role', query, decrypt_func=self.decrypt) # RBAC - role permission query = self.odb.get_rbac_role_permission_list(server.cluster.id, True) self.config.rbac_role_permission = ConfigDict.from_query('rbac_role_permission', query, decrypt_func=self.decrypt) # TLS CA certs query = self.odb.get_tls_ca_cert_list(server.cluster.id, True) self.config.tls_ca_cert = ConfigDict.from_query('tls_ca_cert', query, decrypt_func=self.decrypt) # TLS channel security query = self.odb.get_tls_channel_sec_list(server.cluster.id, True) self.config.tls_channel_sec = ConfigDict.from_query('tls_channel_sec', query, decrypt_func=self.decrypt) # TLS key/cert pairs query = self.odb.get_tls_key_cert_list(server.cluster.id, True) self.config.tls_key_cert = ConfigDict.from_query('tls_key_cert', query, decrypt_func=self.decrypt) # WS-Security query = self.odb.get_wss_list(server.cluster.id, True) self.config.wss = ConfigDict.from_query('wss', query, decrypt_func=self.decrypt) # Vault connections query = self.odb.get_vault_connection_list(server.cluster.id, True) self.config.vault_conn_sec = ConfigDict.from_query('vault_conn_sec', query, decrypt_func=self.decrypt) # XPath query = self.odb.get_xpath_sec_list(server.cluster.id, True) self.config.xpath_sec = ConfigDict.from_query('xpath_sec', query, decrypt_func=self.decrypt) # New in 3.0 - encrypt all old secrets self._migrate_30_encrypt_secrets() # # Security - end # # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ # All the HTTP/SOAP channels. http_soap = [] for item in self.odb.get_http_soap_list(server.cluster.id, 'channel'): hs_item = {} for key in item.keys(): hs_item[key] = getattr(item, key) hs_item['replace_patterns_json_pointer'] = item.replace_patterns_json_pointer hs_item['replace_patterns_xpath'] = item.replace_patterns_xpath hs_item['match_target'] = '{}{}{}'.format(hs_item['soap_action'], MISC.SEPARATOR, hs_item['url_path']) hs_item['match_target_compiled'] = Matcher(hs_item['match_target']) http_soap.append(hs_item) self.config.http_soap = http_soap # Namespaces query = self.odb.get_namespace_list(server.cluster.id, True) self.config.msg_ns = ConfigDict.from_query('msg_ns', query, decrypt_func=self.decrypt) # XPath query = self.odb.get_xpath_list(server.cluster.id, True) self.config.xpath = ConfigDict.from_query('msg_xpath', query, decrypt_func=self.decrypt) # JSON Pointer query = self.odb.get_json_pointer_list(server.cluster.id, True) self.config.json_pointer = ConfigDict.from_query('json_pointer', query, decrypt_func=self.decrypt) # SimpleIO # In preparation for a SIO rewrite, we loaded SIO config from a file # but actual code paths require the pre-3.0 format so let's prepare it here. self.config.simple_io = ConfigDict('simple_io', Bunch()) self.config.simple_io['int_parameters'] = self.sio_config.int.exact self.config.simple_io['int_parameter_suffixes'] = self.sio_config.int.suffix self.config.simple_io['bool_parameter_prefixes'] = self.sio_config.bool.prefix # Pub/sub self.config.pubsub = Bunch() # Pub/sub - endpoints query = self.odb.get_pubsub_endpoint_list(server.cluster.id, True) self.config.pubsub_endpoint = ConfigDict.from_query('pubsub_endpoint', query, decrypt_func=self.decrypt) # Pub/sub - topics query = self.odb.get_pubsub_topic_list(server.cluster.id, True) self.config.pubsub_topic = ConfigDict.from_query('pubsub_topic', query, decrypt_func=self.decrypt) # Pub/sub - subscriptions query = self.odb.get_pubsub_subscription_list(server.cluster.id, True) self.config.pubsub_subscription = ConfigDict.from_query('pubsub_subscription', query, decrypt_func=self.decrypt) # E-mail - SMTP query = self.odb.get_email_smtp_list(server.cluster.id, True) self.config.email_smtp = ConfigDict.from_query('email_smtp', query, decrypt_func=self.decrypt) # E-mail - IMAP query = self.odb.get_email_imap_list(server.cluster.id, True) self.config.email_imap = ConfigDict.from_query('email_imap', query, decrypt_func=self.decrypt) # Message paths self.config.msg_ns_store = NamespaceStore() self.config.json_pointer_store = JSONPointerStore() self.config.xpath_store = XPathStore() # Assign config to worker self.worker_store.worker_config = self.config
def start_server(parallel_server, zato_deployment_key=None): # Easier to type self = parallel_server # This cannot be done in __init__ because each sub-process obviously has its own PID self.pid = os.getpid() # Used later on use_tls = asbool(self.fs_server_config.crypto.use_tls) # Will be None if we are not running in background. if not zato_deployment_key: zato_deployment_key = '{}.{}'.format(datetime.utcnow().isoformat(), uuid4().hex) self.deployment_key = zato_deployment_key register_diag_handlers() # Store the ODB configuration, create an ODB connection pool and have self.odb use it self.config.odb_data = self.get_config_odb_data(self) self.set_odb_pool() # Now try grabbing the basic server's data from the ODB. No point # in doing anything else if we can't get past this point. server = self.odb.fetch_server(self.config.odb_data) if not server: raise Exception('Server does not exist in the ODB') # Set up the server-wide default lock manager odb_data = self.config.odb_data backend_type = 'fcntl' if odb_data.engine == 'sqlite' else odb_data.engine self.zato_lock_manager = LockManager(backend_type, 'zato', self.odb.session) # Just to make sure distributed locking is configured correctly with self.zato_lock_manager(uuid4().hex): pass # Basic metadata self.id = server.id self.name = server.name self.cluster_id = server.cluster_id self.cluster = self.odb.cluster # Looked up upfront here and assigned to services in their store self.enforce_service_invokes = asbool( self.fs_server_config.misc.enforce_service_invokes) # For server-to-server communication self.servers = Servers(self.odb, self.cluster.name) logger.info( 'Preferred address of `%s@%s` (pid: %s) is `http%s://%s:%s`', self.name, self.cluster.name, self.pid, 's' if use_tls else '', self.preferred_address, self.port) # Reads in all configuration from ODB self.worker_store = WorkerStore(self.config, self) self.worker_store.invoke_matcher.read_config( self.fs_server_config.invoke_patterns_allowed) self.worker_store.target_matcher.read_config( self.fs_server_config.invoke_target_patterns_allowed) self.set_up_config(server) # Deploys services is_first, locally_deployed = self._after_init_common(server) # Initializes worker store, including connectors self.worker_store.init() self.request_dispatcher_dispatch = self.worker_store.request_dispatcher.dispatch # Normalize hot-deploy configuration self.hot_deploy_config = Bunch() self.hot_deploy_config.work_dir = os.path.normpath( os.path.join(self.repo_location, self.fs_server_config.hot_deploy.work_dir)) self.hot_deploy_config.backup_history = int( self.fs_server_config.hot_deploy.backup_history) self.hot_deploy_config.backup_format = self.fs_server_config.hot_deploy.backup_format for name in ('current_work_dir', 'backup_work_dir', 'last_backup_work_dir', 'delete_after_pick_up'): # New in 2.0 if name == 'delete_after_pick_up': value = asbool(self.fs_server_config.hot_deploy.get( name, True)) self.hot_deploy_config[name] = value else: self.hot_deploy_config[name] = os.path.normpath( os.path.join(self.hot_deploy_config.work_dir, self.fs_server_config.hot_deploy[name])) self._after_init_accepted(locally_deployed) broker_callbacks = { TOPICS[MESSAGE_TYPE.TO_PARALLEL_ANY]: self.worker_store.on_broker_msg, TOPICS[MESSAGE_TYPE.TO_PARALLEL_ALL]: self.worker_store.on_broker_msg, } self.broker_client = BrokerClient(self.kvdb, 'parallel', broker_callbacks, self.get_lua_programs()) self.worker_store.set_broker_client(self.broker_client) self.odb.server_up_down(server.token, SERVER_UP_STATUS.RUNNING, True, self.host, self.port, self.preferred_address, use_tls) # Startup services if is_first: self.invoke_startup_services(is_first) spawn_greenlet(self.set_up_pickup) # IPC if is_first: self.ipc_forwarder.name = self.name self.ipc_forwarder.pid = self.pid spawn_greenlet(self.ipc_forwarder.run) # IPC self.ipc_api.name = self.name self.ipc_api.pid = self.pid self.ipc_api.on_message_callback = self.worker_store.on_ipc_message spawn_greenlet(self.ipc_api.run) logger.info('Started `%s@%s` (pid: %s)', server.name, server.cluster.name, self.pid)
def __init__(self): self.host = None self.port = None self.crypto_manager = None self.odb = None self.odb_data = None self.config = None self.repo_location = None self.user_conf_location = None self.sql_pool_store = None self.int_parameters = None self.int_parameter_suffixes = None self.bool_parameter_prefixes = None self.soap11_content_type = None self.soap12_content_type = None self.plain_xml_content_type = None self.json_content_type = None self.internal_service_modules = None # Zato's own internal services self.service_modules = None # Set programmatically in Spring self.service_sources = None # Set in a config file self.base_dir = None self.tls_dir = None self.hot_deploy_config = None self.pickup = None self.fs_server_config = None self.pickup_config = None self.connector_server_grace_time = None self.id = None self.name = None self.cluster = None self.cluster_id = None self.kvdb = None self.startup_jobs = None self.worker_store = None self.request_dispatcher_dispatch = None self.deployment_lock_expires = None self.deployment_lock_timeout = None self.deployment_key = '' self.app_context = None self.has_gevent = None self.delivery_store = None self.static_config = None self.component_enabled = Bunch() self.client_address_headers = [ 'HTTP_X_ZATO_FORWARDED_FOR', 'HTTP_X_FORWARDED_FOR', 'REMOTE_ADDR' ] self.broker_client = None self.return_tracebacks = None self.default_error_message = None self.time_util = None self.preferred_address = None self.crypto_use_tls = None self.servers = None self.zato_lock_manager = None self.pid = None self.sync_internal = None self.ipc_api = IPCAPI(False) self.ipc_forwarder = IPCAPI(True) self.fifo_response_buffer_size = 0.1 # In megabytes self.live_msg_browser = None # Allows users store arbitrary data across service invocations self.user_ctx = Bunch() self.user_ctx_lock = gevent.lock.RLock() self.access_logger = logging.getLogger('zato_access_log') self.access_logger_log = self.access_logger._log self.needs_access_log = self.access_logger.isEnabledFor(INFO) # The main config store self.config = ConfigStore() gevent.signal(signal.SIGINT, self.destroy)
def get_config_odb_data(self, parallel_server): """ Returns configuration with regards to ODB data. """ odb_data = Bunch() odb_data.db_name = parallel_server.odb_data['db_name'] odb_data.extra = parallel_server.odb_data['extra'] odb_data.engine = parallel_server.odb_data['engine'] odb_data.token = parallel_server.fs_server_config.main.token odb_data.is_odb = True if odb_data.engine != 'sqlite': odb_data.password = parallel_server.odb_data['password'] odb_data.host = parallel_server.odb_data['host'] odb_data.port = parallel_server.odb_data['port'] odb_data.engine = parallel_server.odb_data['engine'] odb_data.pool_size = parallel_server.odb_data['pool_size'] odb_data.username = parallel_server.odb_data['username'] # Note that we don't read is_active off of anywhere - ODB always must # be active and it's not a regular connection pool anyway. odb_data.is_active = True return odb_data
def become_cluster_wide(self, connector_server_keep_alive_job_time, connector_server_grace_time, server_id, cluster_id, starting_up): """ Attempts to become a connector server, the one to start the connector processes. """ base_job_data = Bunch({ 'id': 0, # Dummy ID 'weeks': None, 'days': None, 'hours': None, 'minutes': None, 'seconds': connector_server_keep_alive_job_time, 'repeats': None, 'extra': 'server_id:{};cluster_id:{}'.format(server_id, cluster_id), }) job_data = None if self.parallel_server.odb.become_cluster_wide(connector_server_grace_time): self.is_cluster_wide = True # Schedule a job for letting the other servers know we're still alive job_data = Bunch(base_job_data.copy()) job_data.start_date = datetime.utcnow() job_data.name = 'zato.server.cluster-wide-singleton-keep-alive' job_data.service = 'zato.server.cluster-wide-singleton-keep-alive' else: # All other singleton servers that are just starting up get this job # for checking whether the connector server is alive or not if starting_up: job_data = Bunch(base_job_data.copy()) job_data.start_date = datetime.utcnow() + timedelta(seconds=10) # Let's give the other server some time to warm up job_data.name = ENSURE_SINGLETON_JOB job_data.service = 'zato.server.ensure-cluster-wide-singleton' if job_data: self.scheduler.create_interval_based(job_data, MESSAGE_TYPE.TO_AMQP_PUBLISHING_CONNECTOR_ALL) return self.is_cluster_wide