def set_up_pickup(self):
empty = []
# Fix up booleans and paths
for stanza, stanza_config in self.pickup_config.items():
# user_config_items is empty by default
if not stanza_config:
empty.append(stanza)
continue
stanza_config.read_on_pickup = asbool(stanza_config.get('read_on_pickup', True))
stanza_config.parse_on_pickup = asbool(stanza_config.get('parse_on_pickup', True))
stanza_config.delete_after_pickup = asbool(stanza_config.get('delete_after_pickup', True))
stanza_config.case_insensitive = asbool(stanza_config.get('case_insensitive', True))
stanza_config.pickup_from = absolutize(stanza_config.pickup_from, self.base_dir)
stanza_config.is_service_hot_deploy = False
mpt = stanza_config.get('move_processed_to')
stanza_config.move_processed_to = absolutize(mpt, self.base_dir) if mpt else None
services = stanza_config.get('services') or []
stanza_config.services = [services] if not isinstance(services, list) else services
topics = stanza_config.get('topics') or []
stanza_config.topics = [topics] if not isinstance(topics, list) else topics
flags = globre.EXACT
if stanza_config.case_insensitive:
flags |= IGNORECASE
patterns = stanza_config.patterns
stanza_config.patterns = [patterns] if not isinstance(patterns, list) else patterns
stanza_config.patterns = [globre.compile(elem, flags) for elem in stanza_config.patterns]
if not os.path.exists(stanza_config.pickup_from):
logger.warn('Pickup dir `%s` does not exist (%s)', stanza_config.pickup_from, stanza)
for item in empty:
del self.pickup_config[item]
# Ok, now that we have configured everything that pickup.conf had
# we still need to make it aware of services and how to pick them up from FS.
stanza = 'zato_internal_service_hot_deploy'
stanza_config = Bunch({
'pickup_from': self.hot_deploy_config.pickup_dir,
'patterns': [globre.compile('*.py', globre.EXACT | IGNORECASE)],
'read_on_pickup': False,
'parse_on_pickup': False,
'delete_after_pickup': self.hot_deploy_config.delete_after_pickup,
'is_service_hot_deploy': True,
})
self.pickup_config[stanza] = stanza_config
self.pickup = PickupManager(self, self.pickup_config)
spawn_greenlet(self.pickup.run)
def start_server(parallel_server, zato_deployment_key=None):
# Easier to type
self = parallel_server # type: ParallelServer
# This cannot be done in __init__ because each sub-process obviously has its own PID
self.pid = os.getpid()
# This also cannot be done in __init__ which doesn't have this variable yet
self.is_first_worker = int(os.environ['ZATO_SERVER_WORKER_IDX']) == 0
# Used later on
use_tls = asbool(self.fs_server_config.crypto.use_tls)
# Will be None if we are not running in background.
if not zato_deployment_key:
zato_deployment_key = '{}.{}'.format(datetime.utcnow().isoformat(), uuid4().hex)
self.deployment_key = zato_deployment_key
register_diag_handlers()
# Create all POSIX IPC objects now that we have the deployment key
self.shmem_size = int(float(self.fs_server_config.shmem.size) * 10**6) # Convert to megabytes as integer
self.server_startup_ipc.create(self.deployment_key, self.shmem_size)
self.connector_config_ipc.create(self.deployment_key, self.shmem_size)
# Store the ODB configuration, create an ODB connection pool and have self.odb use it
self.config.odb_data = self.get_config_odb_data(self)
self.set_up_odb()
# Now try grabbing the basic server's data from the ODB. No point
# in doing anything else if we can't get past this point.
server = self.odb.fetch_server(self.config.odb_data)
if not server:
raise Exception('Server does not exist in the ODB')
# Set up the server-wide default lock manager
odb_data = self.config.odb_data
backend_type = 'fcntl' if odb_data.engine == 'sqlite' else odb_data.engine
self.zato_lock_manager = LockManager(backend_type, 'zato', self.odb.session)
# Just to make sure distributed locking is configured correctly
with self.zato_lock_manager(uuid4().hex):
pass
# Basic metadata
self.id = server.id
self.name = server.name
self.cluster_id = server.cluster_id
self.cluster = self.odb.cluster
self.worker_id = '{}.{}.{}.{}'.format(self.cluster_id, self.id, self.worker_pid, new_cid())
# Looked up upfront here and assigned to services in their store
self.enforce_service_invokes = asbool(self.fs_server_config.misc.enforce_service_invokes)
# For server-to-server communication
self.servers = Servers(self.odb, self.cluster.name, self.decrypt)
logger.info('Preferred address of `%s@%s` (pid: %s) is `http%s://%s:%s`', self.name,
self.cluster.name, self.pid, 's' if use_tls else '', self.preferred_address,
self.port)
# Configure which HTTP methods can be invoked via REST or SOAP channels
methods_allowed = self.fs_server_config.http.methods_allowed
methods_allowed = methods_allowed if isinstance(methods_allowed, list) else [methods_allowed]
self.http_methods_allowed.extend(methods_allowed)
# As above, as a regular expression to be used in pattern matching
http_methods_allowed_re = '|'.join(self.http_methods_allowed)
self.http_methods_allowed_re = '({})'.format(http_methods_allowed_re)
# Reads in all configuration from ODB
self.worker_store = WorkerStore(self.config, self)
self.worker_store.invoke_matcher.read_config(self.fs_server_config.invoke_patterns_allowed)
self.worker_store.target_matcher.read_config(self.fs_server_config.invoke_target_patterns_allowed)
self.set_up_config(server)
# Normalize hot-deploy configuration
self.hot_deploy_config = Bunch()
self.hot_deploy_config.pickup_dir = absolutize(self.fs_server_config.hot_deploy.pickup_dir, self.repo_location)
self.hot_deploy_config.work_dir = os.path.normpath(os.path.join(
self.repo_location, self.fs_server_config.hot_deploy.work_dir))
self.hot_deploy_config.backup_history = int(self.fs_server_config.hot_deploy.backup_history)
self.hot_deploy_config.backup_format = self.fs_server_config.hot_deploy.backup_format
# Added in 3.1, hence optional
max_batch_size = int(self.fs_server_config.hot_deploy.get('max_batch_size', 1000))
# Turn it into megabytes
max_batch_size = max_batch_size * 1000
# Finally, assign it to ServiceStore
self.service_store.max_batch_size = max_batch_size
# Deploys services
is_first, locally_deployed = self._after_init_common(server)
# Initializes worker store, including connectors
self.worker_store.init()
self.request_dispatcher_dispatch = self.worker_store.request_dispatcher.dispatch
# Configure remaining parts of SSO
self.configure_sso()
# Cannot be done in __init__ because self.sso_config is not available there yet
salt_size = self.sso_config.hash_secret.salt_size
self.crypto_manager.add_hash_scheme('zato.default', self.sso_config.hash_secret.rounds, salt_size)
for name in('current_work_dir', 'backup_work_dir', 'last_backup_work_dir', 'delete_after_pickup'):
# New in 2.0
if name == 'delete_after_pickup':
# For backward compatibility, we need to support both names
old_name = 'delete_after_pick_up'
if old_name in self.fs_server_config.hot_deploy:
_name = old_name
else:
_name = name
value = asbool(self.fs_server_config.hot_deploy.get(_name, True))
self.hot_deploy_config[name] = value
else:
self.hot_deploy_config[name] = os.path.normpath(os.path.join(
self.hot_deploy_config.work_dir, self.fs_server_config.hot_deploy[name]))
broker_callbacks = {
TOPICS[MESSAGE_TYPE.TO_PARALLEL_ANY]: self.worker_store.on_broker_msg,
TOPICS[MESSAGE_TYPE.TO_PARALLEL_ALL]: self.worker_store.on_broker_msg,
}
self.broker_client = BrokerClient(self.kvdb, 'parallel', broker_callbacks, self.get_lua_programs())
self.worker_store.set_broker_client(self.broker_client)
# Make sure that broker client's connection is ready before continuing
# to rule out edge cases where, for instance, hot deployment would
# try to publish a locally found package (one of extra packages found)
# before the client's thread connected to KVDB.
if not self.broker_client.ready:
start = now = datetime.utcnow()
max_seconds = 120
until = now + timedelta(seconds=max_seconds)
while not self.broker_client.ready:
now = datetime.utcnow()
delta = (now - start).total_seconds()
if now < until:
# Do not log too early so as not to clutter logs
if delta > 2:
logger.info('Waiting for broker client to become ready (%s, max:%s)', delta, max_seconds)
gevent.sleep(0.5)
else:
raise Exception('Broker client did not become ready within {} seconds'.format(max_seconds))
self._after_init_accepted(locally_deployed)
self.odb.server_up_down(
server.token, SERVER_UP_STATUS.RUNNING, True, self.host, self.port, self.preferred_address, use_tls)
if is_first:
logger.info('First worker of `%s` is %s', self.name, self.pid)
self.startup_callable_tool.invoke(SERVER_STARTUP.PHASE.IN_PROCESS_FIRST, kwargs={
'parallel_server': self,
})
# Clean up any old WSX connections possibly registered for this server
# which may be still linger around, for instance, if the server was previously
# shut down forcibly and did not have an opportunity to run self.cleanup_on_stop
self.cleanup_wsx()
# Startup services
self.invoke_startup_services(is_first)
spawn_greenlet(self.set_up_pickup)
# Set up subprocess-based IBM MQ connections if that component is enabled
if self.fs_server_config.component_enabled.ibm_mq:
# Will block for a few seconds at most, until is_ok is returned
# which indicates that a connector started or not.
is_ok = self.connector_ibm_mq.start_ibm_mq_connector(int(self.fs_server_config.ibm_mq.ipc_tcp_start_port))
try:
if is_ok:
self.connector_ibm_mq.create_initial_wmq_definitions(self.worker_store.worker_config.definition_wmq)
self.connector_ibm_mq.create_initial_wmq_outconns(self.worker_store.worker_config.out_wmq)
self.connector_ibm_mq.create_initial_wmq_channels(self.worker_store.worker_config.channel_wmq)
except Exception as e:
logger.warn('Could not create initial IBM MQ objects, e:`%s`', e)
# Set up subprocess-based SFTP connections
is_ok = self.connector_sftp.start_sftp_connector(int(self.fs_server_config.ibm_mq.ipc_tcp_start_port))
if is_ok:
self.connector_sftp.create_initial_sftp_outconns(self.worker_store.worker_config.out_sftp)
else:
self.startup_callable_tool.invoke(SERVER_STARTUP.PHASE.IN_PROCESS_OTHER, kwargs={
'parallel_server': self,
})
# IPC
self.ipc_api.name = self.ipc_api.get_endpoint_name(self.cluster.name, self.name, self.pid)
self.ipc_api.pid = self.pid
self.ipc_api.on_message_callback = self.worker_store.on_ipc_message
spawn_greenlet(self.ipc_api.run)
self.startup_callable_tool.invoke(SERVER_STARTUP.PHASE.AFTER_STARTED, kwargs={
'parallel_server': self,
})
logger.info('Started `%s@%s` (pid: %s)', server.name, server.cluster.name, self.pid)