def __init__(self):
# PyMQI is an optional dependency so let's import it here rather than on module level
try:
import pymqi
except ImportError:
self.pymqi = None
else:
self.pymqi = pymqi
self.host = '127.0.0.1'
self.port = None
self.username = None
self.password = None
self.server_auth = None
self.basic_auth_expected = None
self.server_port = None
self.server_path = None
self.server_address = 'http://127.0.0.1:{}{}'
self.lock = RLock()
self.logger = None
self.parent_pid = getppid()
self.keyutils = KeyUtils('zato-wmq', self.parent_pid)
self.connections = {}
self.outconns = {}
self.channels = {}
self.outconn_id_to_def_id = {} # Maps outgoing connection IDs to their underlying definition IDs
self.channel_id_to_def_id = {} # Ditto but for channels
self.outconn_name_to_id = {} # Maps outgoing connection names to their IDs
self.set_config()
def __init__(self):
self.host = None
self.port = None
self.crypto_manager = None
self.odb = None
self.odb_data = None
self.config = None
self.repo_location = None
self.user_conf_location = None
self.sql_pool_store = None
self.soap11_content_type = None
self.soap12_content_type = None
self.plain_xml_content_type = None
self.json_content_type = None
self.internal_service_modules = None # Zato's own internal services
self.service_modules = None # Set programmatically in Spring
self.service_sources = None # Set in a config file
self.base_dir = None
self.tls_dir = None
self.static_dir = None
self.hot_deploy_config = None
self.pickup = None
self.fs_server_config = None
self.fs_sql_config = None
self.pickup_config = None
self.logging_config = None
self.logging_conf_path = None
self.sio_config = None
self.sso_config = None
self.connector_server_grace_time = None
self.id = None
self.name = None
self.worker_id = None
self.worker_pid = None
self.cluster = None
self.cluster_id = None
self.kvdb = None
self.startup_jobs = None
self.worker_store = None
self.request_dispatcher_dispatch = None
self.deployment_lock_expires = None
self.deployment_lock_timeout = None
self.deployment_key = ''
self.app_context = None
self.has_gevent = None
self.delivery_store = None
self.static_config = None
self.component_enabled = Bunch()
self.client_address_headers = [
'HTTP_X_ZATO_FORWARDED_FOR', 'HTTP_X_FORWARDED_FOR', 'REMOTE_ADDR'
]
self.broker_client = None
self.return_tracebacks = None
self.default_error_message = None
self.time_util = None
self.preferred_address = None
self.crypto_use_tls = None
self.servers = None
self.zato_lock_manager = None
self.pid = None
self.sync_internal = None
self.ipc_api = IPCAPI(False)
self.ipc_forwarder = IPCAPI(True)
self.wmq_ipc_tcp_port = None
self.fifo_response_buffer_size = None # Will be in megabytes
self.is_first_worker = None
self.shmem_size = -1.0
self.server_startup_ipc = ServerStartupIPC()
self.keyutils = KeyUtils()
self.sso_api = None
self.is_sso_enabled = False
self.audit_pii = audit_pii
self.startup_callable_tool = None
self.default_internal_pubsub_endpoint_id = None
self._hash_secret_method = None
self._hash_secret_rounds = None
self._hash_secret_salt_size = None
# Allows users store arbitrary data across service invocations
self.user_ctx = Bunch()
self.user_ctx_lock = gevent.lock.RLock()
self.access_logger = logging.getLogger('zato_access_log')
self.access_logger_log = self.access_logger._log
self.needs_access_log = self.access_logger.isEnabledFor(INFO)
self.has_pubsub_audit_log = logging.getLogger(
'zato_pubsub_audit').isEnabledFor('INFO')
self.is_enabled_for_warn = logging.getLogger('zato').isEnabledFor(
'WARN')
# The main config store
self.config = ConfigStore()
gevent.signal(signal.SIGINT, self.destroy)
def __init__(self):
self.host = None
self.port = None
self.crypto_manager = None
self.odb = None
self.odb_data = None
self.config = None
self.repo_location = None
self.user_conf_location = None
self.sql_pool_store = None
self.soap11_content_type = None
self.soap12_content_type = None
self.plain_xml_content_type = None
self.json_content_type = None
self.service_modules = None # Set programmatically in Spring
self.service_sources = None # Set in a config file
self.base_dir = None # type: unicode
self.tls_dir = None # type: unicode
self.static_dir = None # type: unicode
self.json_schema_dir = None # type: unicode
self.hot_deploy_config = None
self.pickup = None
self.fs_server_config = None
self.fs_sql_config = None
self.pickup_config = None
self.logging_config = None
self.logging_conf_path = None
self.sio_config = None
self.sso_config = None
self.connector_server_grace_time = None
self.id = None
self.name = None
self.worker_id = None
self.worker_pid = None
self.cluster = None
self.cluster_id = None
self.kvdb = None
self.startup_jobs = None
self.worker_store = None # type: WorkerStore
self.service_store = None # type: ServiceStore
self.request_dispatcher_dispatch = None
self.deployment_lock_expires = None
self.deployment_lock_timeout = None
self.deployment_key = ''
self.has_gevent = None
self.delivery_store = None
self.static_config = None
self.component_enabled = Bunch()
self.client_address_headers = ['HTTP_X_ZATO_FORWARDED_FOR', 'HTTP_X_FORWARDED_FOR', 'REMOTE_ADDR']
self.broker_client = None
self.return_tracebacks = None
self.default_error_message = None
self.time_util = None
self.preferred_address = None
self.crypto_use_tls = None
self.servers = None
self.zato_lock_manager = None
self.pid = None
self.sync_internal = None
self.ipc_api = IPCAPI()
self.fifo_response_buffer_size = None # Will be in megabytes
self.is_first_worker = None
self.shmem_size = -1.0
self.server_startup_ipc = ServerStartupIPC()
self.connector_config_ipc = ConnectorConfigIPC()
self.keyutils = KeyUtils()
self.sso_api = None
self.is_sso_enabled = False
self.audit_pii = audit_pii
self.has_fg = False
self.startup_callable_tool = None
self.default_internal_pubsub_endpoint_id = None
self._hash_secret_method = None
self._hash_secret_rounds = None
self._hash_secret_salt_size = None
# Our arbiter may potentially call the cleanup procedure multiple times
# and this will be set to True the first time around.
self._is_process_closing = False
# Allows users store arbitrary data across service invocations
self.user_ctx = Bunch()
self.user_ctx_lock = gevent.lock.RLock()
# Connectors
self.connector_ibm_mq = IBMMQIPC(self)
self.connector_sftp = SFTPIPC(self)
# HTTP methods allowed as a Python list
self.http_methods_allowed = []
# As above, but as a regular expression pattern
self.http_methods_allowed_re = ''
self.access_logger = logging.getLogger('zato_access_log')
self.access_logger_log = self.access_logger._log
self.needs_access_log = self.access_logger.isEnabledFor(INFO)
self.has_pubsub_audit_log = logging.getLogger('zato_pubsub_audit').isEnabledFor(INFO)
self.is_enabled_for_warn = logging.getLogger('zato').isEnabledFor(WARN)
# The main config store
self.config = ConfigStore()
class ConnectionContainer(object):
def __init__(self):
# PyMQI is an optional dependency so let's import it here rather than on module level
try:
import pymqi
except ImportError:
self.pymqi = None
else:
self.pymqi = pymqi
self.host = '127.0.0.1'
self.port = None
self.username = None
self.password = None
self.server_auth = None
self.basic_auth_expected = None
self.server_port = None
self.server_path = None
self.server_address = 'http://127.0.0.1:{}{}'
self.lock = RLock()
self.logger = None
self.parent_pid = getppid()
self.keyutils = KeyUtils('zato-wmq', self.parent_pid)
self.connections = {}
self.outconns = {}
self.channels = {}
self.outconn_id_to_def_id = {} # Maps outgoing connection IDs to their underlying definition IDs
self.channel_id_to_def_id = {} # Ditto but for channels
self.outconn_name_to_id = {} # Maps outgoing connection names to their IDs
self.set_config()
def set_config(self):
""" Sets self attributes, as configured in keyring by our parent process.
"""
config = self.keyutils.user_get()
config = loads(config)
config = bunchify(config)
self.username = config.username
self.password = config.password
self.server_auth = (self.username, self.password)
self.base_dir = config.base_dir
self.port = config.port
self.server_port = config.server_port
self.server_path = config.server_path
self.server_address = self.server_address.format(self.server_port, self.server_path)
with open(config.logging_conf_path) as f:
logging_config = yaml.load(f)
# IBM MQ logging configuration is new in Zato 3.0, so it's optional.
if not 'zato_ibm_mq' in logging_config['loggers']:
logging_config = default_logging_config
self.set_up_logging(logging_config)
# ################################################################################################################################
def set_up_logging(self, config):
logger_conf = config['loggers']['zato_ibm_mq']
wmq_handler_conf = config['handlers']['ibm_mq']
del wmq_handler_conf['formatter']
wmq_handler_conf.pop('class', False)
formatter_conf = config['formatters']['default']['format']
self.logger = getLogger(logger_conf['qualname'])
self.logger.setLevel(getattr(logging, logger_conf['level']))
formatter = Formatter(formatter_conf)
wmq_handler_conf['filename'] = path.abspath(path.join(self.base_dir, wmq_handler_conf['filename']))
wmq_handler = RotatingFileHandler(**wmq_handler_conf)
wmq_handler.setFormatter(formatter)
stdout_handler = StreamHandler(sys.stdout)
stdout_handler.setFormatter(formatter)
self.logger.addHandler(wmq_handler)
self.logger.addHandler(stdout_handler)
# ################################################################################################################################
def on_mq_message_received(self, msg_ctx, _post=post):
_post(self.server_address, data=dumps({
'msg': msg_ctx.mq_msg.to_dict(),
'channel_id': msg_ctx.channel_id,
'queue_name': msg_ctx.queue_name,
'service_name': msg_ctx.service_name,
'data_format': msg_ctx.data_format,
}), auth=self.server_auth)
# ################################################################################################################################
def _create_definition(self, msg, needs_connect=True):
""" A low-level method to create connection definitions. Must be called with self.lock held.
"""
msg.pop('name')
msg.pop('cluster_id', None)
msg.pop('old_name', None)
id = msg.pop('id')
msg['needs_jms'] = msg.pop('use_jms', False)
msg.pop('_encryption_needed', False)
msg.pop('_encrypted_in_odb', False)
# We always create and add a connetion ..
conn = WebSphereMQConnection(**msg)
self.connections[id] = conn
# .. because even if it fails here, it will be eventually established during one of .send or .receive,
# however, it is possible that our caller already knows that the connection will fail so we need
# to take it into account too.
if needs_connect:
conn.connect()
return conn
# ################################################################################################################################
def _on_DEFINITION_WMQ_CREATE(self, msg):
""" Creates a new connection to IBM MQ.
"""
if not self.pymqi:
return Response(_http_503, 'Could not find pymqi module, MQ connections will not start')
with self.lock:
try:
self._create_definition(msg)
except Exception as e:
self.logger.warn(format_exc())
return Response(_http_503, str(e.message))
else:
return Response()
# ################################################################################################################################
def _on_DEFINITION_WMQ_EDIT(self, msg):
""" Updates an existing definition - close the current one, including channels and outconns,
and creates a new one in its place.
"""
with self.lock:
def_id = msg.id
old_conn = self.connections[def_id]
# Edit messages don't carry passwords
msg.password = old_conn.password
# It's possible that we are editing a connection that has no connected yet,
# e.g. if password was invalid, so this needs to be guarded by an if.
if old_conn.is_connected:
self.connections[def_id].close()
# Overwrites the previous connection object
new_conn = self._create_definition(msg, old_conn.is_connected)
# Stop and start all channels using this definition.
for channel_id, _def_id in self.channel_id_to_def_id.items():
if def_id == _def_id:
channel = self.channels[channel_id]
channel.stop()
channel.conn = new_conn
channel.start()
return Response()
# ################################################################################################################################
def _on_DEFINITION_WMQ_DELETE(self, msg):
""" Deletes an IBM MQ MQ definition along with its associated outconns and channels.
"""
with self.lock:
def_id = msg.id
# Stop all connections ..
try:
self.connections[def_id].close()
except Exception:
self.logger.warn(format_exc())
finally:
try:
del self.connections[def_id]
except Exception:
self.logger.warn(format_exc())
# .. continue to delete outconns regardless of errors above ..
for outconn_id, outconn_def_id in self.outconn_id_to_def_id.items():
if outconn_def_id == def_id:
del self.outconn_id_to_def_id[outconn_id]
del self.outconns[outconn_id]
# .. delete channels too.
for channel_id, channel_def_id in self.channel_id_to_def_id.items():
if channel_def_id == def_id:
del self.channel_id_to_def_id[channel_id]
del self.channels[channel_id]
return Response()
# ################################################################################################################################
def _on_DEFINITION_WMQ_CHANGE_PASSWORD(self, msg):
with self.lock:
try:
conn = self.connections[msg.id]
conn.close()
conn.password = msg.password
conn.connect()
except Exception as e:
self.logger.warn(format_exc())
return Response(_http_503, str(e.message), 'text/plain')
else:
return Response()
# ################################################################################################################################
def _on_DEFINITION_WMQ_PING(self, msg):
""" Pings a remote IBM MQ manager.
"""
try:
self.connections[msg.id].ping()
except WebSphereMQException, e:
return Response(_http_503, str(e.message), 'text/plain')
else: