def execute(self, args):
repo_dir = os.path.join(os.path.abspath(os.path.join(self.original_dir, args.path)), 'config', 'repo')
config = get_config(repo_dir, 'server.conf')
headers = {}
if args.headers:
for pair in args.headers.strip().split(';'):
k, v = pair.strip().split('=', 1)
headers[k] = v
repo_dir = os.path.join(os.path.abspath(os.path.join(args.path)), 'config', 'repo')
config = get_config(repo_dir, 'server.conf')
client = AnyServiceInvoker(
'http://{}'.format(config.main.gunicorn_bind), args.url_path, self.get_server_client_auth(config, repo_dir),
max_response_repr=int(args.max_response_repr), max_cid_repr=int(args.max_cid_repr))
# Prevents attempts to convert/escape XML into JSON
to_json = True if args.data_format == DATA_FORMAT.JSON else False
func = client.invoke_async if args.async else client.invoke
response = func(args.name, args.payload, headers, args.channel, args.data_format, args.transport, to_json=to_json)
if response.ok:
self.logger.info(response.data or '(None)')
else:
self.logger.error(response.details)
if args.verbose:
self.logger.debug('inner.text:[{}]'.format(response.inner.text))
self.logger.debug('response:[{}]'.format(response))
def _get_sso_config(self, args, repo_location, secrets_conf):
# type: (Namespace, unicode, Bunch) -> UserAPI
sso_conf = get_config(repo_location,
'sso.conf',
needs_user_config=False)
normalize_password_reject_list(sso_conf)
crypto_manager = CryptoManager.from_secret_key(
secrets_conf.secret_keys.key1)
crypto_manager.add_hash_scheme('sso.super-user',
sso_conf.hash_secret.rounds,
sso_conf.hash_secret.salt_size)
server_conf = get_config(repo_location,
'server.conf',
needs_user_config=False,
crypto_manager=crypto_manager,
secrets_conf=secrets_conf)
def _get_session():
return self.get_odb_session_from_server_config(server_conf, None)
def _hash_secret(_secret):
return crypto_manager.hash_secret(_secret, 'sso.super-user')
return UserAPI(None, sso_conf, _get_session, crypto_manager.encrypt,
crypto_manager.decrypt, _hash_secret, None, new_user_id)
def main():
# Always attempt to store the PID file first
store_pidfile(os.path.abspath('.'))
# Capture warnings to log files
logging.captureWarnings(True)
config = Config()
repo_location = os.path.join('.', 'config', 'repo')
# Logging configuration
with open(os.path.join(repo_location, 'logging.conf')) as f:
dictConfig(yaml.load(f))
# Read config in and extend it with ODB-specific information
config.main = get_config(repo_location, 'scheduler.conf')
config.main.odb.fs_sql_config = get_config(repo_location,
'sql.conf',
needs_user_config=False)
# Make all paths absolute
if config.main.crypto.use_tls:
config.main.crypto.ca_certs_location = absjoin(
repo_location, config.main.crypto.ca_certs_location)
config.main.crypto.priv_key_location = absjoin(
repo_location, config.main.crypto.priv_key_location)
config.main.crypto.cert_location = absjoin(
repo_location, config.main.crypto.cert_location)
logger = logging.getLogger(__name__)
logger.info('Scheduler starting (http{}://{}:{})'.format(
's' if config.main.crypto.use_tls else '', config.main.bind.host,
config.main.bind.port))
# Fix up configuration so it uses the format internal utilities expect
for name, job_config in iteritems(
get_config(repo_location,
'startup_jobs.conf',
needs_user_config=False)):
job_config['name'] = name
config.startup_jobs.append(job_config)
# Run the scheduler server
try:
SchedulerServer(config, repo_location).serve_forever()
except Exception:
logger.warn(format_exc())
def set_zato_client(self):
repo_dir = os.path.join(
os.path.abspath(os.path.join(self.server_path)), 'config', 'repo')
config = get_config(repo_dir, 'server.conf')
self.client = ZatoClient('http://{}'.format(config.main.gunicorn_bind),
'/zato/admin/invoke',
get_server_client_auth(config, repo_dir),
max_response_repr=15000)
session = get_odb_session_from_server_config(
config, get_crypto_manager_from_server_config(config, repo_dir))
self.client.cluster_id = session.query(odb.model.Server).\
filter(odb.model.Server.token == config.main.token).\
one().cluster_id
self.client.odb_session = session
# Sanity check
self.client.invoke('zato.ping')
# ################################################################################################################################
def execute(self, args):
# type: (Namespace) -> object
repo_location = os.path.join(args.path, 'config', 'repo')
secrets_conf = get_config(repo_location,
'secrets.conf',
needs_user_config=False)
# This file must exist, otherwise it's not a path to a server
if not secrets_conf:
self.logger.warn('Could not find file `secrets.conf` in `%s`',
repo_location)
return self.SYS_ERROR.NOT_A_ZATO_SERVER
user_api = self._get_sso_config(args, repo_location, secrets_conf)
if self.user_required:
user = user_api.get_user_by_username(_cid, args.username)
if not user:
self.logger.warn('No such user `%s`', args.username)
return self.SYS_ERROR.NO_SUCH_SSO_USER
else:
user = None
return self._on_sso_command(args, user, user_api)
def _after_init_common(self, server):
""" Initializes parts of the server that don't depend on whether the
server's been allowed to join the cluster or not.
"""
# Patterns to match during deployment
self.service_store.patterns_matcher.read_config(
self.fs_server_config.deploy_patterns_allowed)
# Static config files
self.static_config = StaticConfig(
os.path.join(self.repo_location, 'static'))
# Key-value DB
kvdb_config = get_kvdb_config_for_log(self.fs_server_config.kvdb)
kvdb_logger.info('Worker config `%s`', kvdb_config)
self.kvdb.config = self.fs_server_config.kvdb
self.kvdb.server = self
self.kvdb.decrypt_func = self.crypto_manager.decrypt
self.kvdb.init()
kvdb_logger.info('Worker config `%s`', kvdb_config)
# Lua programs, both internal and user defined ones.
for name, program in self.get_lua_programs():
self.kvdb.lua_container.add_lua_program(name, program)
# TimeUtil needs self.kvdb so it can be set now
self.time_util = TimeUtil(self.kvdb)
# Service sources
self.service_sources = []
for name in open(
os.path.join(self.repo_location,
self.fs_server_config.main.service_sources)):
name = name.strip()
if name and not name.startswith('#'):
if not os.path.isabs(name):
name = os.path.normpath(os.path.join(self.base_dir, name))
self.service_sources.append(name)
# User-config from ./config/repo/user-config
for file_name in os.listdir(self.user_conf_location):
conf = get_config(self.user_conf_location, file_name)
# Not used at all in this type of configuration
conf.pop('user_config_items', None)
self.user_config[get_user_config_name(file_name)] = conf
# Convert size of FIFO response buffers to megabytes
self.fifo_response_buffer_size = int(
float(self.fs_server_config.misc.fifo_response_buffer_size) *
megabyte)
is_first, locally_deployed = self.maybe_on_first_worker(
server, self.kvdb.conn)
return is_first, locally_deployed
def execute(self, args):
repo_dir = os.path.join(os.path.abspath(os.path.join(self.original_dir, args.path)), 'config', 'repo')
config = get_config(repo_dir, 'server.conf')
priv_key_location = os.path.abspath(os.path.join(repo_dir, config.crypto.priv_key_location))
cm = CryptoManager(priv_key_location=priv_key_location)
cm.load_keys()
engine_args = Bunch()
engine_args.odb_type = config.odb.engine
engine_args.odb_user = config.odb.username
engine_args.odb_password = cm.decrypt(config.odb.password)
engine_args.odb_host = config.odb.host
engine_args.odb_db_name = config.odb.db_name
engine = self._get_engine(engine_args)
session = self._get_session(engine)
auth = None
headers = {}
with closing(session) as session:
cluster = session.query(Server).\
filter(Server.token == config.main.token).\
one().cluster
channel = session.query(HTTPSOAP).\
filter(HTTPSOAP.cluster_id == cluster.id).\
filter(HTTPSOAP.url_path == args.url_path).\
one()
if channel.security_id:
security = session.query(HTTPBasicAuth).\
filter(HTTPBasicAuth.id == channel.security_id).\
first()
if security:
auth = (security.username, security.password)
if args.headers:
for pair in args.headers.strip().split(';'):
k, v = pair.strip().split('=', 1)
headers[k] = v
client = AnyServiceInvoker('http://{}'.format(config.main.gunicorn_bind), args.url_path, auth,
max_response_repr=int(args.max_response_repr), max_cid_repr=int(args.max_cid_repr))
func = client.invoke_async if args.async else client.invoke
response = func(args.name, args.payload, headers, args.channel, args.data_format, args.transport)
if response.ok:
self.logger.info(response.data or '(None)')
else:
self.logger.error(response.details)
if args.verbose:
self.logger.debug('inner.text:[{}]'.format(response.inner.text))
self.logger.debug('response:[{}]'.format(response))
def _init(self):
""" Initializes all the basic run-time data structures and connects
to the Zato broker.
"""
fs_server_config = get_config(self.repo_location, 'server.conf')
app_context = get_app_context(fs_server_config)
crypto_manager = get_crypto_manager(self.repo_location, app_context,
fs_server_config)
config_odb = fs_server_config.odb
self.odb = app_context.get_object('odb_manager')
self.odb.crypto_manager = crypto_manager
self.odb.token = fs_server_config.main.token
# Key-value DB
self.kvdb = KVDB()
self.kvdb.config = fs_server_config.kvdb
self.kvdb.decrypt_func = self.odb.crypto_manager.decrypt
self.kvdb.init()
# Broker client
self.broker_client = BrokerClient(self.kvdb, self.broker_client_id,
self.broker_callbacks)
self.broker_client.start()
# ODB
#
# Ticket #35 Don't ignore odb_port when creating an ODB
# https://github.com/zatosource/zato/issues/35
#
engine = config_odb.engine
port = config_odb['port']
self.odb_config = Bunch()
self.odb_config.db_name = config_odb.db_name
self.odb_config.is_active = True
self.odb_config.engine = engine
self.odb_config.extra = config_odb.extra
self.odb_config.host = config_odb.host
self.odb_config.port = port
self.odb_config.password = self.odb.crypto_manager.decrypt(
config_odb.password)
self.odb_config.pool_size = config_odb.pool_size
self.odb_config.username = config_odb.username
self.odb_config.is_odb = True
self.sql_pool_store = app_context.get_object('sql_pool_store')
self.sql_pool_store[ZATO_ODB_POOL_NAME] = self.odb_config
self.odb.pool = self.sql_pool_store[ZATO_ODB_POOL_NAME].pool
self._setup_odb()
# Delivery store
self.delivery_store = DeliveryStore(
self.kvdb, self.broker_client, self.odb,
float(fs_server_config.misc.delivery_lock_timeout))
def load_secrets(secrets, secret_names, crypto_manager, conf_location, conf_file_name):
conf = get_config(os.path.dirname(conf_location), conf_file_name, False)
for name in secret_names:
k, v = name.split(':')
if conf[k][v]:
secrets[name] = crypto_manager.decrypt(conf[k][v])
return secrets, conf
def load_secrets(secrets, secret_names, crypto_manager, conf_location, conf_file_name):
conf = get_config(os.path.dirname(conf_location), conf_file_name, False)
for name in secret_names:
k, v = name.split(':')
if conf[k][v]:
secrets[name] = crypto_manager.decrypt(conf[k][v])
return secrets, conf
def execute(self, args):
repo_dir = os.path.join(
os.path.abspath(os.path.join(self.original_dir, args.path)),
'config', 'repo')
config = get_config(repo_dir, 'server.conf')
headers = {}
if args.headers:
for pair in args.headers.strip().split(';'):
k, v = pair.strip().split('=', 1)
headers[k] = v
repo_dir = os.path.join(os.path.abspath(os.path.join(args.path)),
'config', 'repo')
config = get_config(repo_dir, 'server.conf')
client = AnyServiceInvoker(
'http://{}'.format(config.main.gunicorn_bind),
args.url_path,
self.get_server_client_auth(config, repo_dir),
max_response_repr=int(args.max_response_repr),
max_cid_repr=int(args.max_cid_repr))
# Prevents attempts to convert/escape XML into JSON
to_json = True if args.data_format == DATA_FORMAT.JSON else False
func = client.invoke_async if args. async else client.invoke
response = func(args.name,
args.payload,
headers,
args.channel,
args.data_format,
args.transport,
to_json=to_json)
if response.ok:
self.logger.info(response.data or '(None)')
else:
self.logger.error(response.details)
if args.verbose:
self.logger.debug('inner.text:[{}]'.format(response.inner.text))
self.logger.debug('response:[{}]'.format(response))
def _init(self):
""" Initializes all the basic run-time data structures and connects
to the Zato broker.
"""
fs_server_config = get_config(self.repo_location, 'server.conf')
app_context = get_app_context(fs_server_config)
crypto_manager = get_crypto_manager(self.repo_location, app_context, fs_server_config)
config_odb = fs_server_config.odb
self.odb = app_context.get_object('odb_manager')
self.odb.crypto_manager = crypto_manager
self.odb.token = fs_server_config.main.token
# Key-value DB
self.kvdb = KVDB()
self.kvdb.config = fs_server_config.kvdb
self.kvdb.decrypt_func = self.odb.crypto_manager.decrypt
self.kvdb.init()
# Broker client
self.broker_client = BrokerClient(self.kvdb, self.broker_client_id, self.broker_callbacks)
self.broker_client.start()
# ODB
#
# Ticket #35 Don't ignore odb_port when creating an ODB
# https://github.com/zatosource/zato/issues/35
#
engine = config_odb.engine
port = config_odb.get('port')
if not port:
port = 5432 if engine == 'postgresql' else 1521
self.odb_config = Bunch()
self.odb_config.db_name = config_odb.db_name
self.odb_config.is_active = True
self.odb_config.engine = engine
self.odb_config.extra = config_odb.extra
self.odb_config.host = config_odb.host
self.odb_config.port = port
self.odb_config.password = self.odb.crypto_manager.decrypt(config_odb.password)
self.odb_config.pool_size = config_odb.pool_size
self.odb_config.username = config_odb.username
self.odb_config.is_odb = True
self.sql_pool_store = app_context.get_object('sql_pool_store')
self.sql_pool_store[ZATO_ODB_POOL_NAME] = self.odb_config
self.odb.pool = self.sql_pool_store[ZATO_ODB_POOL_NAME].pool
self._setup_odb()
# Delivery store
self.delivery_store = DeliveryStore(self.kvdb, self.broker_client, self.odb, float(fs_server_config.misc.delivery_lock_timeout))
def run(base_dir):
os.chdir(base_dir)
# We're doing it here even if someone doesn't use PostgreSQL at all
# so we're not suprised when someone suddenly starts using PG.
# TODO: Make sure it's registered for each of the subprocess
psycopg2.extensions.register_type(psycopg2.extensions.UNICODE)
psycopg2.extensions.register_type(psycopg2.extensions.UNICODEARRAY)
repo_location = os.path.join(base_dir, 'config', 'repo')
# Configure the logging first, before configuring the actual server.
logging.addLevelName('TRACE1', TRACE1)
logging.config.fileConfig(os.path.join(repo_location, 'logging.conf'))
config = get_config(repo_location, 'server.conf')
app_context = get_app_context(config)
crypto_manager = get_crypto_manager(repo_location, app_context, config)
parallel_server = app_context.get_object('parallel_server')
zato_gunicorn_app = ZatoGunicornApplication(
parallel_server, repo_location, config.main, config.crypto)
parallel_server.crypto_manager = crypto_manager
parallel_server.odb_data = config.odb
parallel_server.host = zato_gunicorn_app.zato_host
parallel_server.port = zato_gunicorn_app.zato_port
parallel_server.repo_location = repo_location
parallel_server.base_dir = base_dir
parallel_server.fs_server_config = config
parallel_server.startup_jobs = app_context.get_object('startup_jobs')
parallel_server.app_context = app_context
# Remove all locks possibly left over by previous server instances
clear_locks(app_context.get_object('kvdb'), config.main.token, config.kvdb, crypto_manager.decrypt)
# Turn the repo dir into an actual repository and commit any new/modified files
RepoManager(repo_location).ensure_repo_consistency()
# This is new in 1.2 so is optional
profiler_enabled = config.get('profiler', {}).get('enabled', False)
if asbool(profiler_enabled):
profiler_dir = os.path.abspath(os.path.join(base_dir, config.profiler.profiler_dir))
parallel_server.on_wsgi_request = ProfileMiddleware(
parallel_server.on_wsgi_request,
log_filename = os.path.join(profiler_dir, config.profiler.log_filename),
cachegrind_filename = os.path.join(profiler_dir, config.profiler.cachegrind_filename),
discard_first_request = config.profiler.discard_first_request,
flush_at_shutdown = config.profiler.flush_at_shutdown,
path = config.profiler.url_path,
unwind = config.profiler.unwind)
# Run the app at last
zato_gunicorn_app.run()
def from_server_conf(server_dir, cache_name, is_https):
# type: (str, str, bool) -> Client
repo_dir = get_repo_dir_from_component_dir(server_dir)
cm = ServerCryptoManager.from_repo_dir(None, repo_dir, None)
secrets_conf = get_config(repo_dir,
'secrets.conf',
needs_user_config=False)
config = get_config(repo_dir,
'server.conf',
crypto_manager=cm,
secrets_conf=secrets_conf)
session = None
password = None
try:
session = get_odb_session_from_server_config(config, None, False)
cluster = session.query(Server).\
filter(Server.token == config.main.token).\
one().cluster # type: Cluster
security = session.query(HTTPBasicAuth).\
filter(Cluster.id == HTTPBasicAuth.cluster_id).\
filter(HTTPBasicAuth.username == CACHE.API_USERNAME).\
filter(HTTPBasicAuth.cluster_id == cluster.id).\
first() # type: HTTPBasicAuth
if security:
password = security.password
finally:
if session:
session.close()
return Client.from_dict({
'username': CACHE.API_USERNAME,
'password': password,
'address': config.main.gunicorn_bind,
'cache_name': cache_name,
'is_https': is_https,
})
def handle(self):
input = self.request.input
with self.lock('{}-{}-{}'.format(self.name, self.server.name,
input.data)):
with open(
os.path.join(self.server.user_conf_location,
input.file_name), 'wb') as f:
f.write(input.data)
conf = get_config(self.server.user_conf_location, input.file_name)
entry = self.server.user_config.setdefault(
get_user_config_name(input.file_name), Bunch())
entry.clear()
entry.update(conf)
def _init(self):
""" Initializes all the basic run-time data structures and connects
to the Zato broker.
"""
fs_server_config = get_config(self.repo_location, 'server.conf')
app_context = get_app_context(fs_server_config)
crypto_manager = get_crypto_manager(self.repo_location, app_context, fs_server_config)
config_odb = fs_server_config.odb
self.odb = app_context.get_object('odb_manager')
self.odb.crypto_manager = crypto_manager
self.odb.token = fs_server_config.main.token
# Key-value DB
self.kvdb = KVDB()
self.kvdb.config = fs_server_config.kvdb
self.kvdb.decrypt_func = self.odb.crypto_manager.decrypt
self.kvdb.init()
# Broker client
self.broker_client = BrokerClient(self.kvdb, self.broker_client_id, self.broker_callbacks)
self.broker_client.start()
# ODB
self.odb_config = Bunch()
self.odb_config.db_name = config_odb.db_name
self.odb_config.is_active = True
self.odb_config.engine = config_odb.engine
self.odb_config.extra = config_odb.extra
self.odb_config.host = config_odb.host
self.odb_config.password = self.odb.crypto_manager.decrypt(config_odb.password)
self.odb_config.pool_size = config_odb.pool_size
self.odb_config.username = config_odb.username
self.odb_config.is_odb = True
self.sql_pool_store = app_context.get_object('sql_pool_store')
self.sql_pool_store[ZATO_ODB_POOL_NAME] = self.odb_config
self.odb.pool = self.sql_pool_store[ZATO_ODB_POOL_NAME].pool
self._setup_odb()
# Delivery store
self.delivery_store = DeliveryStore(self.kvdb, self.broker_client, self.odb, float(fs_server_config.misc.delivery_lock_timeout))
def set_zato_client(self):
repo_dir = os.path.join(os.path.abspath(os.path.join(self.server_path)), 'config', 'repo')
config = get_config(repo_dir, 'server.conf')
self.client = ZatoClient('http://{}'.format(config.main.gunicorn_bind),
'/zato/admin/invoke', get_server_client_auth(config, repo_dir), max_response_repr=15000)
session = get_odb_session_from_server_config(
config, get_crypto_manager_from_server_config(config, repo_dir))
self.client.cluster_id = session.query(odb.model.Server).\
filter(odb.model.Server.token == config.main.token).\
one().cluster_id
self.client.odb_session = session
# Sanity check
self.client.invoke('zato.ping')
def run(base_dir):
os.chdir(base_dir)
# We're doing it here even if someone doesn't use PostgreSQL at all
# so we're not suprised when someone suddenly starts using PG.
# TODO: Make sure it's registered for each of the subprocess
psycopg2.extensions.register_type(psycopg2.extensions.UNICODE)
psycopg2.extensions.register_type(psycopg2.extensions.UNICODEARRAY)
repo_location = os.path.join(base_dir, 'config', 'repo')
# Configure the logging first, before configuring the actual server.
logging.addLevelName('TRACE1', TRACE1)
logging.config.fileConfig(os.path.join(repo_location, 'logging.conf'))
config = get_config(repo_location, 'server.conf')
app_context = get_app_context(config)
crypto_manager = get_crypto_manager(repo_location, app_context, config)
parallel_server = app_context.get_object('parallel_server')
zato_gunicorn_app = ZatoGunicornApplication(parallel_server, config.main)
parallel_server.crypto_manager = crypto_manager
parallel_server.odb_data = config.odb
parallel_server.host = zato_gunicorn_app.zato_host
parallel_server.port = zato_gunicorn_app.zato_port
parallel_server.repo_location = repo_location
parallel_server.base_dir = base_dir
parallel_server.fs_server_config = config
parallel_server.stats_jobs = app_context.get_object('stats_jobs')
parallel_server.app_context = app_context
# Remove all locks possibly left over by previous server instances
clear_locks(app_context.get_object('kvdb'), config.main.token, config.kvdb,
crypto_manager.decrypt)
# Turn the repo dir into an actual repository and commit any new/modified files
RepoManager(repo_location).ensure_repo_consistency()
# Run the app at last
zato_gunicorn_app.run()
def run(base_dir):
os.chdir(base_dir)
# We're doing it here even if someone doesn't use PostgreSQL at all
# so we're not suprised when someone suddenly starts using PG.
# TODO: Make sure it's registered for each of the subprocess when the code's
# finally modified to use subprocesses.
psycopg2.extensions.register_type(psycopg2.extensions.UNICODE)
psycopg2.extensions.register_type(psycopg2.extensions.UNICODEARRAY)
repo_location = os.path.join(base_dir, 'config', 'repo')
# Configure the logging first, before configuring the actual server.
logging.addLevelName('TRACE1', TRACE1)
logging.config.fileConfig(os.path.join(repo_location, 'logging.conf'))
config = get_config(repo_location, 'server.conf')
app_context = get_app_context(config)
crypto_manager = get_crypto_manager(repo_location, app_context, config)
parallel_server = app_context.get_object('parallel_server')
zato_gunicorn_app = ZatoGunicornApplication(parallel_server, config.main)
parallel_server.crypto_manager = crypto_manager
parallel_server.odb_data = config.odb
parallel_server.host = zato_gunicorn_app.zato_host
parallel_server.port = zato_gunicorn_app.zato_port
parallel_server.repo_location = repo_location
parallel_server.base_dir = base_dir
parallel_server.fs_server_config = config
parallel_server.stats_jobs = app_context.get_object('stats_jobs')
parallel_server.app_context = app_context
# Remove all locks possibly left over by previous server instances
clear_locks(app_context.get_object('kvdb'), config.main.token, config.kvdb, crypto_manager.decrypt)
# Turn the repo dir into an actual repository and commit any new/modified files
RepoManager(repo_location).ensure_repo_consistency()
# Run the app at last
zato_gunicorn_app.run()
def _init(self):
""" Initializes all the basic run-time data structures and connects
to the Zato broker.
"""
fs_server_config = get_config(self.repo_location, 'server.conf')
app_context = get_app_context(fs_server_config)
crypto_manager = get_crypto_manager(self.repo_location, app_context,
fs_server_config)
config_odb = fs_server_config.odb
self.odb = app_context.get_object('odb_manager')
self.odb.crypto_manager = crypto_manager
self.odb.token = fs_server_config.main.token
# Key-value DB
self.kvdb = KVDB()
self.kvdb.config = fs_server_config.kvdb
self.kvdb.decrypt_func = self.odb.crypto_manager.decrypt
self.kvdb.init()
# Broker client
self.broker_client = BrokerClient(self.kvdb, self.broker_client_id,
self.broker_callbacks)
self.broker_client.start()
# ODB
self.odb_config = Bunch()
self.odb_config.db_name = config_odb.db_name
self.odb_config.is_active = True
self.odb_config.engine = config_odb.engine
self.odb_config.extra = config_odb.extra
self.odb_config.host = config_odb.host
self.odb_config.password = self.odb.crypto_manager.decrypt(
config_odb.password)
self.odb_config.pool_size = config_odb.pool_size
self.odb_config.username = config_odb.username
self.odb_config.is_odb = True
self.sql_pool_store = app_context.get_object('sql_pool_store')
self.sql_pool_store[ZATO_ODB_POOL_NAME] = self.odb_config
self.odb.pool = self.sql_pool_store[ZATO_ODB_POOL_NAME].pool
self._setup_odb()
def run(host, port, base_dir, start_singleton):
repo_location = os.path.join(base_dir, 'config', 'repo')
# Configure the logging first, before configuring the actual server.
logging.addLevelName('TRACE1', TRACE1)
logging.config.fileConfig(os.path.join(repo_location, 'logging.conf'))
config = get_config(repo_location, 'server.conf')
app_context = get_app_context(config, ZatoContext)
crypto_manager = get_crypto_manager(repo_location, app_context, config)
parallel_server = app_context.get_object('parallel_server')
parallel_server.crypto_manager = crypto_manager
parallel_server.odb.crypto_manager = crypto_manager
parallel_server.odb.odb_data = config['odb']
parallel_server.host = host
parallel_server.port = port
parallel_server.repo_location = repo_location
if start_singleton:
singleton_server = app_context.get_object('singleton_server')
singleton_server.initial_sleep_time = int(config['singleton']['initial_sleep_time']) / 1000.
parallel_server.singleton_server = singleton_server
# Wow, this line looks weird. What it does is simply assigning a parallel
# server instance to the singleton server.
parallel_server.singleton_server.parallel_server = parallel_server
parallel_server.after_init()
print('OK..')
parallel_server.run_forever()
# $ sudo netstat -an | grep TIME_WAIT | wc -l
# cat /proc/sys/net/core/wmem_max - 109568
# /sbin/sysctl net.ipv4.tcp_mem="109568 109568 109568"
# echo 0 > /proc/sys/net/ipv4/conf/eth0/rp_filter
'''
def _init(self):
""" Initializes all the basic run-time data structures and connects
to the Zato broker.
"""
# Imported here to avoid circular dependencies
from zato.server.config.app import ZatoContext
config = get_config(self.repo_location, 'server.conf')
app_context = get_app_context(config, ZatoContext)
crypto_manager = get_crypto_manager(self.repo_location, app_context, config)
self.odb = app_context.get_object('odb_manager')
self.odb.crypto_manager = crypto_manager
self.odb.odb_data = config['odb']
self._setup_odb()
self.worker_data = Bunch()
self.worker_data.broker_config = Bunch()
self.worker_data.broker_config.name = self.broker_client_name
self.worker_data.broker_config.broker_token = self.server.cluster.broker_token
self.worker_data.broker_config.zmq_context = zmq.Context()
broker_push_client_pull = 'tcp://{0}:{1}'.format(self.server.cluster.broker_host,
self.server.cluster.broker_start_port + self.broker_push_client_pull_port)
client_push_broker_pull = 'tcp://{0}:{1}'.format(self.server.cluster.broker_host,
self.server.cluster.broker_start_port + self.client_push_broker_pull_port)
broker_pub_client_sub = 'tcp://{0}:{1}'.format(self.server.cluster.broker_host,
self.server.cluster.broker_start_port + self.broker_pub_client_sub_port)
self.worker_data.broker_config.broker_push_client_pull = broker_push_client_pull
self.worker_data.broker_config.client_push_broker_pull = client_push_broker_pull
self.worker_data.broker_config.broker_pub_client_sub = broker_pub_client_sub
# Connects to the broker
super(BaseConnector, self)._init()
def _init(self):
""" Initializes all the basic run-time data structures and connects
to the Zato broker.
"""
fs_server_config = get_config(self.repo_location, 'server.conf')
app_context = get_app_context(fs_server_config)
crypto_manager = get_crypto_manager(self.repo_location, app_context, fs_server_config)
config_odb = fs_server_config.odb
self.odb = app_context.get_object('odb_manager')
self.odb.crypto_manager = crypto_manager
self.odb.token = fs_server_config.main.token
# Key-value DB
self.kvdb = KVDB()
self.kvdb.config = fs_server_config.kvdb
self.kvdb.decrypt_func = self.odb.crypto_manager.decrypt
self.kvdb.init()
odb_data = Bunch()
odb_data.db_name = config_odb.db_name
odb_data.engine = config_odb.engine
odb_data.extra = config_odb.extra
odb_data.host = config_odb.host
odb_data.password = self.odb.crypto_manager.decrypt(config_odb.password)
odb_data.pool_size = config_odb.pool_size
odb_data.username = config_odb.username
odb_data.is_odb = True
self.sql_pool_store = app_context.get_object('sql_pool_store')
self.sql_pool_store[ZATO_ODB_POOL_NAME] = odb_data
self.odb.pool = self.sql_pool_store[ZATO_ODB_POOL_NAME]
self._setup_odb()
# Connects to the broker
super(BaseConnector, self)._init()
def run(base_dir, start_gunicorn_app=True):
# Store a pidfile before doing anything else
store_pidfile(base_dir)
# For dumping stacktraces
register_diag_handlers()
# Start initializing the server now
os.chdir(base_dir)
try:
import pymysql
pymysql.install_as_MySQLdb()
except ImportError:
pass
# We're doing it here even if someone doesn't use PostgreSQL at all
# so we're not suprised when someone suddenly starts using PG.
# TODO: Make sure it's registered for each of the subprocess
psycopg2.extensions.register_type(psycopg2.extensions.UNICODE)
psycopg2.extensions.register_type(psycopg2.extensions.UNICODEARRAY)
repo_location = os.path.join(base_dir, 'config', 'repo')
# Configure the logging first, before configuring the actual server.
logging.addLevelName('TRACE1', TRACE1)
logging.config.fileConfig(os.path.join(repo_location, 'logging.conf'))
logger = logging.getLogger(__name__)
kvdb_logger = logging.getLogger('zato_kvdb')
config = get_config(repo_location, 'server.conf')
# New in 2.0 - Start monitoring as soon as possible
if config.get('newrelic', {}).get('config'):
import newrelic.agent
newrelic.agent.initialize(
config.newrelic.config, config.newrelic.environment or None, config.newrelic.ignore_errors or None,
config.newrelic.log_file or None, config.newrelic.log_level or None)
# Store KVDB config in logs, possibly replacing its password if told to
kvdb_config = get_kvdb_config_for_log(config.kvdb)
kvdb_logger.info('Master process config `%s`', kvdb_config)
# New in 2.0 hence optional
user_locale = config.misc.get('locale', None)
if user_locale:
locale.setlocale(locale.LC_ALL, user_locale)
value = 12345
logger.info('Locale is `%s`, amount of %s -> `%s`', user_locale, value,
locale.currency(value, grouping=True).decode('utf-8'))
# Spring Python
app_context = get_app_context(config)
# Makes queries against Postgres asynchronous
if asbool(config.odb.use_async_driver) and config.odb.engine == 'postgresql':
make_psycopg_green()
# New in 2.0 - Put HTTP_PROXY in os.environ.
http_proxy = config.misc.get('http_proxy', False)
if http_proxy:
os.environ['http_proxy'] = http_proxy
crypto_manager = get_crypto_manager(repo_location, app_context, config)
parallel_server = app_context.get_object('parallel_server')
zato_gunicorn_app = ZatoGunicornApplication(parallel_server, repo_location, config.main, config.crypto)
parallel_server.crypto_manager = crypto_manager
parallel_server.odb_data = config.odb
parallel_server.host = zato_gunicorn_app.zato_host
parallel_server.port = zato_gunicorn_app.zato_port
parallel_server.repo_location = repo_location
parallel_server.base_dir = base_dir
parallel_server.fs_server_config = config
parallel_server.user_config.update(config.user_config_items)
parallel_server.startup_jobs = app_context.get_object('startup_jobs')
parallel_server.app_context = app_context
# Remove all locks possibly left over by previous server instances
kvdb = app_context.get_object('kvdb')
kvdb.component = 'master-proc'
clear_locks(kvdb, config.main.token, config.kvdb, crypto_manager.decrypt)
# Turn the repo dir into an actual repository and commit any new/modified files
RepoManager(repo_location).ensure_repo_consistency()
# New in 2.0 so it's optional.
profiler_enabled = config.get('profiler', {}).get('enabled', False)
if asbool(profiler_enabled):
profiler_dir = os.path.abspath(os.path.join(base_dir, config.profiler.profiler_dir))
parallel_server.on_wsgi_request = ProfileMiddleware(
parallel_server.on_wsgi_request,
log_filename = os.path.join(profiler_dir, config.profiler.log_filename),
cachegrind_filename = os.path.join(profiler_dir, config.profiler.cachegrind_filename),
discard_first_request = config.profiler.discard_first_request,
flush_at_shutdown = config.profiler.flush_at_shutdown,
path = config.profiler.url_path,
unwind = config.profiler.unwind)
# Run the app at last we execute from command line
if start_gunicorn_app:
zato_gunicorn_app.run()
else:
return zato_gunicorn_app.zato_wsgi_app
def run(base_dir, start_gunicorn_app=True):
# Filter out warnings we are not interested in
warnings.filterwarnings('ignore', 'Mean of empty slice.')
warnings.filterwarnings('ignore',
'invalid value encountered in double_scalars')
# Store a pidfile before doing anything else
store_pidfile(base_dir)
# For dumping stacktraces
register_diag_handlers()
# Start initializing the server now
os.chdir(base_dir)
try:
import pymysql
pymysql.install_as_MySQLdb()
except ImportError:
pass
# We're doing it here even if someone doesn't use PostgreSQL at all
# so we're not suprised when someone suddenly starts using PG.
# TODO: Make sure it's registered for each of the subprocess
psycopg2.extensions.register_type(psycopg2.extensions.UNICODE)
psycopg2.extensions.register_type(psycopg2.extensions.UNICODEARRAY)
repo_location = os.path.join(base_dir, 'config', 'repo')
# Configure the logging first, before configuring the actual server.
logging.addLevelName('TRACE1', TRACE1)
with open(os.path.join(repo_location, 'logging.conf')) as f:
dictConfig(yaml.load(f))
logger = logging.getLogger(__name__)
kvdb_logger = logging.getLogger('zato_kvdb')
config = get_config(repo_location, 'server.conf')
# New in 2.0 - Start monitoring as soon as possible
if config.get('newrelic', {}).get('config'):
import newrelic.agent
newrelic.agent.initialize(config.newrelic.config,
config.newrelic.environment or None,
config.newrelic.ignore_errors or None,
config.newrelic.log_file or None,
config.newrelic.log_level or None)
# New in 2.0 - override gunicorn-set Server HTTP header
gunicorn.SERVER_SOFTWARE = config.misc.get('http_server_header', 'Zato')
# Store KVDB config in logs, possibly replacing its password if told to
kvdb_config = get_kvdb_config_for_log(config.kvdb)
kvdb_logger.info('Master process config `%s`', kvdb_config)
# New in 2.0 hence optional
user_locale = config.misc.get('locale', None)
if user_locale:
locale.setlocale(locale.LC_ALL, user_locale)
value = 12345
logger.info('Locale is `%s`, amount of %s -> `%s`', user_locale, value,
locale.currency(value, grouping=True).decode('utf-8'))
# Spring Python
app_context = get_app_context(config)
# Makes queries against Postgres asynchronous
if asbool(
config.odb.use_async_driver) and config.odb.engine == 'postgresql':
make_psycopg_green()
# New in 2.0 - Put HTTP_PROXY in os.environ.
http_proxy = config.misc.get('http_proxy', False)
if http_proxy:
os.environ['http_proxy'] = http_proxy
crypto_manager = get_crypto_manager(repo_location, app_context, config)
parallel_server = app_context.get_object('parallel_server')
zato_gunicorn_app = ZatoGunicornApplication(parallel_server, repo_location,
config.main, config.crypto)
parallel_server.crypto_manager = crypto_manager
parallel_server.odb_data = config.odb
parallel_server.host = zato_gunicorn_app.zato_host
parallel_server.port = zato_gunicorn_app.zato_port
parallel_server.repo_location = repo_location
parallel_server.base_dir = base_dir
parallel_server.tls_dir = os.path.join(parallel_server.base_dir, 'config',
'repo', 'tls')
parallel_server.fs_server_config = config
parallel_server.user_config.update(config.user_config_items)
parallel_server.startup_jobs = app_context.get_object('startup_jobs')
parallel_server.app_context = app_context
# Remove all locks possibly left over by previous server instances
kvdb = app_context.get_object('kvdb')
kvdb.component = 'master-proc'
clear_locks(kvdb, config.main.token, config.kvdb, crypto_manager.decrypt)
# Turn the repo dir into an actual repository and commit any new/modified files
RepoManager(repo_location).ensure_repo_consistency()
# New in 2.0 so it's optional.
profiler_enabled = config.get('profiler', {}).get('enabled', False)
# New in 2.0 so it's optional.
sentry_config = config.get('sentry')
dsn = sentry_config.pop('dsn', None)
if dsn:
from raven import Client
from raven.conf import setup_logging
from raven.handlers.logging import SentryHandler
handler_level = sentry_config.pop('level')
client = Client(dsn, **sentry_config)
handler = SentryHandler(client=client)
handler.setLevel(getattr(logging, handler_level))
logger = logging.getLogger('')
logger.addHandler(handler)
for name in logging.Logger.manager.loggerDict:
if name.startswith('zato'):
logger = logging.getLogger(name)
logger.addHandler(handler)
if asbool(profiler_enabled):
profiler_dir = os.path.abspath(
os.path.join(base_dir, config.profiler.profiler_dir))
parallel_server.on_wsgi_request = ProfileMiddleware(
parallel_server.on_wsgi_request,
log_filename=os.path.join(profiler_dir,
config.profiler.log_filename),
cachegrind_filename=os.path.join(
profiler_dir, config.profiler.cachegrind_filename),
discard_first_request=config.profiler.discard_first_request,
flush_at_shutdown=config.profiler.flush_at_shutdown,
path=config.profiler.url_path,
unwind=config.profiler.unwind)
# Run the app at last we execute from command line
if start_gunicorn_app:
zato_gunicorn_app.run()
else:
return zato_gunicorn_app.zato_wsgi_app
def run(base_dir, start_gunicorn_app=True):
# Store a pidfile before doing anything else
store_pidfile(base_dir)
# For dumping stacktraces
register_diag_handlers()
# Capture warnings to log files
logging.captureWarnings(True)
# Start initializing the server now
os.chdir(base_dir)
try:
import pymysql
pymysql.install_as_MySQLdb()
except ImportError:
pass
# We're doing it here even if someone doesn't use PostgreSQL at all
# so we're not suprised when someone suddenly starts using PG.
# TODO: Make sure it's registered for each of the subprocess
psycopg2.extensions.register_type(psycopg2.extensions.UNICODE)
psycopg2.extensions.register_type(psycopg2.extensions.UNICODEARRAY)
# We know we don't need warnings because users may explicitly configure no certificate validation.
# We don't want for urllib3 to warn us about it.
import requests as _r
_r.packages.urllib3.disable_warnings()
repo_location = os.path.join(base_dir, 'config', 'repo')
# Configure the logging first, before configuring the actual server.
logging.addLevelName('TRACE1', TRACE1)
with open(os.path.join(repo_location, 'logging.conf')) as f:
dictConfig(yaml.load(f))
logger = logging.getLogger(__name__)
kvdb_logger = logging.getLogger('zato_kvdb')
config = get_config(repo_location, 'server.conf')
# New in 2.0 - Start monitoring as soon as possible
if config.get('newrelic', {}).get('config'):
import newrelic.agent
newrelic.agent.initialize(
config.newrelic.config, config.newrelic.environment or None, config.newrelic.ignore_errors or None,
config.newrelic.log_file or None, config.newrelic.log_level or None)
# New in 2.0 - override gunicorn-set Server HTTP header
gunicorn.SERVER_SOFTWARE = config.misc.get('http_server_header', 'Zato')
# Store KVDB config in logs, possibly replacing its password if told to
kvdb_config = get_kvdb_config_for_log(config.kvdb)
kvdb_logger.info('Master process config `%s`', kvdb_config)
# New in 2.0 hence optional
user_locale = config.misc.get('locale', None)
if user_locale:
locale.setlocale(locale.LC_ALL, user_locale)
value = 12345
logger.info('Locale is `%s`, amount of %s -> `%s`', user_locale, value,
locale.currency(value, grouping=True).decode('utf-8'))
# Spring Python
app_context = get_app_context(config)
# Makes queries against Postgres asynchronous
if asbool(config.odb.use_async_driver) and config.odb.engine == 'postgresql':
make_psycopg_green()
# New in 2.0 - Put HTTP_PROXY in os.environ.
http_proxy = config.misc.get('http_proxy', False)
if http_proxy:
os.environ['http_proxy'] = http_proxy
crypto_manager = get_crypto_manager(repo_location, app_context, config)
parallel_server = app_context.get_object('parallel_server')
zato_gunicorn_app = ZatoGunicornApplication(parallel_server, repo_location, config.main, config.crypto)
parallel_server.crypto_manager = crypto_manager
parallel_server.odb_data = config.odb
parallel_server.host = zato_gunicorn_app.zato_host
parallel_server.port = zato_gunicorn_app.zato_port
parallel_server.repo_location = repo_location
parallel_server.base_dir = base_dir
parallel_server.tls_dir = os.path.join(parallel_server.base_dir, 'config', 'repo', 'tls')
parallel_server.fs_server_config = config
parallel_server.user_config.update(config.user_config_items)
parallel_server.startup_jobs = app_context.get_object('startup_jobs')
parallel_server.app_context = app_context
# Remove all locks possibly left over by previous server instances
kvdb = app_context.get_object('kvdb')
kvdb.component = 'master-proc'
clear_locks(kvdb, config.main.token, config.kvdb, crypto_manager.decrypt)
# Turn the repo dir into an actual repository and commit any new/modified files
RepoManager(repo_location).ensure_repo_consistency()
# New in 2.0 so it's optional.
profiler_enabled = config.get('profiler', {}).get('enabled', False)
# New in 2.0 so it's optional.
sentry_config = config.get('sentry')
dsn = sentry_config.pop('dsn', None)
if dsn:
from raven import Client
from raven.handlers.logging import SentryHandler
handler_level = sentry_config.pop('level')
client = Client(dsn, **sentry_config)
handler = SentryHandler(client=client)
handler.setLevel(getattr(logging, handler_level))
logger = logging.getLogger('')
logger.addHandler(handler)
for name in logging.Logger.manager.loggerDict:
if name.startswith('zato'):
logger = logging.getLogger(name)
logger.addHandler(handler)
if asbool(profiler_enabled):
profiler_dir = os.path.abspath(os.path.join(base_dir, config.profiler.profiler_dir))
parallel_server.on_wsgi_request = ProfileMiddleware(
parallel_server.on_wsgi_request,
log_filename = os.path.join(profiler_dir, config.profiler.log_filename),
cachegrind_filename = os.path.join(profiler_dir, config.profiler.cachegrind_filename),
discard_first_request = config.profiler.discard_first_request,
flush_at_shutdown = config.profiler.flush_at_shutdown,
path = config.profiler.url_path,
unwind = config.profiler.unwind)
# New in 2.0 - set environmet variables for servers to inherit
os_environ = config.get('os_environ', {})
for key, value in os_environ.items():
os.environ[key] = value
# Run the app at last
if start_gunicorn_app:
zato_gunicorn_app.run()
else:
return zato_gunicorn_app.zato_wsgi_app
def run(base_dir, start_gunicorn_app=True, options=None):
options = options or {}
# Store a pidfile before doing anything else
store_pidfile(base_dir)
# For dumping stacktraces
register_diag_handlers()
# Capture warnings to log files
logging.captureWarnings(True)
# Start initializing the server now
os.chdir(base_dir)
try:
import pymysql
pymysql.install_as_MySQLdb()
except ImportError:
pass
# We're doing it here even if someone doesn't use PostgreSQL at all
# so we're not suprised when someone suddenly starts using PG.
# TODO: Make sure it's registered for each of the subprocess
psycopg2.extensions.register_type(psycopg2.extensions.UNICODE)
psycopg2.extensions.register_type(psycopg2.extensions.UNICODEARRAY)
# We know we don't need warnings because users may explicitly configure no certificate validation.
# We don't want for urllib3 to warn us about it.
import requests as _r
_r.packages.urllib3.disable_warnings()
repo_location = os.path.join(base_dir, 'config', 'repo')
# Configure the logging first, before configuring the actual server.
logging.addLevelName('TRACE1', TRACE1)
with open(os.path.join(repo_location, 'logging.conf')) as f:
dictConfig(yaml.load(f))
logger = logging.getLogger(__name__)
kvdb_logger = logging.getLogger('zato_kvdb')
config = get_config(repo_location, 'server.conf')
pickup_config = get_config(repo_location, 'pickup.conf')
# Do not proceed unless we can be certain our own preferred address or IP can be obtained.
preferred_address = config.preferred_address.get('address')
if not preferred_address:
preferred_address = get_preferred_ip(config.main.gunicorn_bind,
config.preferred_address)
if not preferred_address and not config.server_to_server.boot_if_preferred_not_found:
msg = 'Unable to start the server. Could not obtain a preferred address, please configure [bind_options] in server.conf'
logger.warn(msg)
raise Exception(msg)
# New in 2.0 - Start monitoring as soon as possible
if config.get('newrelic', {}).get('config'):
import newrelic.agent
newrelic.agent.initialize(config.newrelic.config,
config.newrelic.environment or None,
config.newrelic.ignore_errors or None,
config.newrelic.log_file or None,
config.newrelic.log_level or None)
# New in 2.0 - override gunicorn-set Server HTTP header
gunicorn.SERVER_SOFTWARE = config.misc.get('http_server_header', 'Zato')
# Store KVDB config in logs, possibly replacing its password if told to
kvdb_config = get_kvdb_config_for_log(config.kvdb)
kvdb_logger.info('Master process config `%s`', kvdb_config)
# New in 2.0 hence optional
user_locale = config.misc.get('locale', None)
if user_locale:
locale.setlocale(locale.LC_ALL, user_locale)
value = 12345
logger.info('Locale is `%s`, amount of %s -> `%s`', user_locale, value,
locale.currency(value, grouping=True).decode('utf-8'))
# Spring Python
app_context = get_app_context(config)
# Makes queries against Postgres asynchronous
if asbool(
config.odb.use_async_driver) and config.odb.engine == 'postgresql':
make_psycopg_green()
# New in 2.0 - Put HTTP_PROXY in os.environ.
http_proxy = config.misc.get('http_proxy', False)
if http_proxy:
os.environ['http_proxy'] = http_proxy
crypto_manager = get_crypto_manager(repo_location, app_context, config)
server = app_context.get_object('server')
zato_gunicorn_app = ZatoGunicornApplication(server, repo_location,
config.main, config.crypto)
server.crypto_manager = crypto_manager
server.odb_data = config.odb
server.host = zato_gunicorn_app.zato_host
server.port = zato_gunicorn_app.zato_port
server.repo_location = repo_location
server.user_conf_location = os.path.join(server.repo_location, 'user-conf')
server.base_dir = base_dir
server.tls_dir = os.path.join(server.base_dir, 'config', 'repo', 'tls')
server.fs_server_config = config
server.pickup_config = pickup_config
server.user_config.update(config.user_config_items)
server.app_context = app_context
server.preferred_address = preferred_address
server.sync_internal = options['sync_internal']
server.jwt_secret = server.fs_server_config.misc.jwt_secret.encode('utf8')
# Remove all locks possibly left over by previous server instances
kvdb = app_context.get_object('kvdb')
kvdb.component = 'master-proc'
clear_locks(kvdb, config.main.token, config.kvdb, crypto_manager.decrypt)
# New in 2.0.8
server.return_tracebacks = asbool(
config.misc.get('return_tracebacks', True))
server.default_error_message = config.misc.get('default_error_message',
'An error has occurred')
# Turn the repo dir into an actual repository and commit any new/modified files
RepoManager(repo_location).ensure_repo_consistency()
# New in 2.0 so it's optional.
profiler_enabled = config.get('profiler', {}).get('enabled', False)
# New in 2.0 so it's optional.
sentry_config = config.get('sentry')
dsn = sentry_config.pop('dsn', None)
if dsn:
from raven import Client
from raven.handlers.logging import SentryHandler
handler_level = sentry_config.pop('level')
client = Client(dsn, **sentry_config)
handler = SentryHandler(client=client)
handler.setLevel(getattr(logging, handler_level))
logger = logging.getLogger('')
logger.addHandler(handler)
for name in logging.Logger.manager.loggerDict:
if name.startswith('zato'):
logger = logging.getLogger(name)
logger.addHandler(handler)
if asbool(profiler_enabled):
profiler_dir = os.path.abspath(
os.path.join(base_dir, config.profiler.profiler_dir))
server.on_wsgi_request = ProfileMiddleware(
server.on_wsgi_request,
log_filename=os.path.join(profiler_dir,
config.profiler.log_filename),
cachegrind_filename=os.path.join(
profiler_dir, config.profiler.cachegrind_filename),
discard_first_request=config.profiler.discard_first_request,
flush_at_shutdown=config.profiler.flush_at_shutdown,
path=config.profiler.url_path,
unwind=config.profiler.unwind)
# New in 2.0 - set environmet variables for servers to inherit
os_environ = config.get('os_environ', {})
for key, value in os_environ.items():
os.environ[key] = value
# Run the app at last
if start_gunicorn_app:
zato_gunicorn_app.run()
else:
return zato_gunicorn_app.zato_wsgi_app
def run(base_dir, start_gunicorn_app=True, options=None):
options = options or {}
# Store a pidfile before doing anything else
store_pidfile(base_dir)
# For dumping stacktraces
register_diag_handlers()
# Capture warnings to log files
logging.captureWarnings(True)
# Start initializing the server now
os.chdir(base_dir)
try:
import pymysql
pymysql.install_as_MySQLdb()
except ImportError:
pass
# We're doing it here even if someone doesn't use PostgreSQL at all
# so we're not suprised when someone suddenly starts using PG.
# TODO: Make sure it's registered for each of the subprocess
psycopg2.extensions.register_type(psycopg2.extensions.UNICODE)
psycopg2.extensions.register_type(psycopg2.extensions.UNICODEARRAY)
# We know we don't need warnings because users may explicitly configure no certificate validation.
# We don't want for urllib3 to warn us about it.
import requests as _r
_r.packages.urllib3.disable_warnings()
repo_location = os.path.join(base_dir, 'config', 'repo')
# Configure the logging first, before configuring the actual server.
logging.addLevelName('TRACE1', TRACE1)
logging_conf_path = os.path.join(repo_location, 'logging.conf')
with open(logging_conf_path) as f:
logging_config = yaml.load(f)
dictConfig(logging_config)
logger = logging.getLogger(__name__)
kvdb_logger = logging.getLogger('zato_kvdb')
crypto_manager = ServerCryptoManager(repo_location,
secret_key=options['secret_key'],
stdin_data=read_stdin_data())
secrets_config = ConfigObj(os.path.join(repo_location, 'secrets.conf'),
use_zato=False)
server_config = get_config(repo_location,
'server.conf',
crypto_manager=crypto_manager,
secrets_conf=secrets_config)
pickup_config = get_config(repo_location, 'pickup.conf')
sio_config = get_config(repo_location,
'simple-io.conf',
needs_user_config=False)
sso_config = get_config(repo_location, 'sso.conf', needs_user_config=False)
normalize_sso_config(sso_config)
# Now that we have access to server.conf, greenify libraries required to be made greenlet-friendly,
# assuming that there are any - otherwise do not do anything.
to_greenify = []
for key, value in server_config.get('greenify', {}).items():
if asbool(value):
if not os.path.exists(key):
raise ValueError('No such path `{}`'.format(key))
else:
to_greenify.append(key)
# Go ahead only if we actually have anything to greenify
if to_greenify:
import greenify
greenify.greenify()
for name in to_greenify:
result = greenify.patch_lib(name)
if not result:
raise ValueError(
'Library `{}` could not be greenified'.format(name))
else:
logger.info('Greenified library `%s`', name)
server_config.main.token = server_config.main.token.encode('utf8')
# Do not proceed unless we can be certain our own preferred address or IP can be obtained.
preferred_address = server_config.preferred_address.get('address')
if not preferred_address:
preferred_address = get_preferred_ip(server_config.main.gunicorn_bind,
server_config.preferred_address)
if not preferred_address and not server_config.server_to_server.boot_if_preferred_not_found:
msg = 'Unable to start the server. Could not obtain a preferred address, please configure [bind_options] in server.conf'
logger.warn(msg)
raise Exception(msg)
# Create the startup callable tool as soon as practical
startup_callable_tool = StartupCallableTool(server_config)
# Run the hook before there is any server object created
startup_callable_tool.invoke(SERVER_STARTUP.PHASE.FS_CONFIG_ONLY,
kwargs={
'server_config': server_config,
'pickup_config': pickup_config,
'sio_config': sio_config,
'sso_config': sso_config,
})
# New in 2.0 - Start monitoring as soon as possible
if server_config.get('newrelic', {}).get('config'):
import newrelic.agent
newrelic.agent.initialize(server_config.newrelic.config,
server_config.newrelic.environment or None,
server_config.newrelic.ignore_errors or None,
server_config.newrelic.log_file or None,
server_config.newrelic.log_level or None)
zunicorn.SERVER_SOFTWARE = server_config.misc.get('http_server_header',
'Zato')
# Store KVDB config in logs, possibly replacing its password if told to
kvdb_config = get_kvdb_config_for_log(server_config.kvdb)
kvdb_logger.info('Main process config `%s`', kvdb_config)
# New in 2.0 hence optional
user_locale = server_config.misc.get('locale', None)
if user_locale:
locale.setlocale(locale.LC_ALL, user_locale)
value = 12345
logger.info('Locale is `%s`, amount of %s -> `%s`', user_locale, value,
locale.currency(value, grouping=True).decode('utf-8'))
# Makes queries against Postgres asynchronous
if asbool(server_config.odb.use_async_driver
) and server_config.odb.engine == 'postgresql':
make_psycopg_green()
if server_config.misc.http_proxy:
os.environ['http_proxy'] = server_config.misc.http_proxy
# Basic components needed for the server to boot up
kvdb = KVDB()
odb_manager = ODBManager(well_known_data=ZATO_CRYPTO_WELL_KNOWN_DATA)
sql_pool_store = PoolStore()
service_store = ServiceStore()
service_store.odb = odb_manager
service_store.services = {}
server = ParallelServer()
server.odb = odb_manager
server.service_store = service_store
server.service_store.server = server
server.sql_pool_store = sql_pool_store
server.service_modules = []
server.kvdb = kvdb
# Assigned here because it is a circular dependency
odb_manager.parallel_server = server
zato_gunicorn_app = ZatoGunicornApplication(server, repo_location,
server_config.main,
server_config.crypto)
server.has_fg = options.get('fg')
server.crypto_manager = crypto_manager
server.odb_data = server_config.odb
server.host = zato_gunicorn_app.zato_host
server.port = zato_gunicorn_app.zato_port
server.repo_location = repo_location
server.user_conf_location = os.path.join(server.repo_location, 'user-conf')
server.base_dir = base_dir
server.logs_dir = os.path.join(server.base_dir, 'logs')
server.tls_dir = os.path.join(server.base_dir, 'config', 'repo', 'tls')
server.static_dir = os.path.join(server.base_dir, 'config', 'repo',
'static')
server.json_schema_dir = os.path.join(server.base_dir, 'config', 'repo',
'schema', 'json')
server.fs_server_config = server_config
server.fs_sql_config = get_config(repo_location,
'sql.conf',
needs_user_config=False)
server.pickup_config = pickup_config
server.logging_config = logging_config
server.logging_conf_path = logging_conf_path
server.sio_config = sio_config
server.sso_config = sso_config
server.user_config.update(server_config.user_config_items)
server.preferred_address = preferred_address
server.sync_internal = options['sync_internal']
server.jwt_secret = server.fs_server_config.misc.jwt_secret.encode('utf8')
server.startup_callable_tool = startup_callable_tool
server.is_sso_enabled = server.fs_server_config.component_enabled.sso
if server.is_sso_enabled:
server.sso_api = SSOAPI(server, sso_config, None,
crypto_manager.encrypt, crypto_manager.decrypt,
crypto_manager.hash_secret,
crypto_manager.verify_hash, new_user_id)
# Remove all locks possibly left over by previous server instances
kvdb.component = 'master-proc'
clear_locks(kvdb, server_config.main.token, server_config.kvdb,
crypto_manager.decrypt)
# New in 2.0.8
server.return_tracebacks = asbool(
server_config.misc.get('return_tracebacks', True))
server.default_error_message = server_config.misc.get(
'default_error_message', 'An error has occurred')
# Turn the repo dir into an actual repository and commit any new/modified files
RepoManager(repo_location).ensure_repo_consistency()
# New in 2.0 so it's optional.
profiler_enabled = server_config.get('profiler', {}).get('enabled', False)
# New in 2.0 so it's optional.
sentry_config = server_config.get('sentry')
dsn = sentry_config.pop('dsn', None)
if dsn:
from raven import Client
from raven.handlers.logging import SentryHandler
handler_level = sentry_config.pop('level')
client = Client(dsn, **sentry_config)
handler = SentryHandler(client=client)
handler.setLevel(getattr(logging, handler_level))
logger = logging.getLogger('')
logger.addHandler(handler)
for name in logging.Logger.manager.loggerDict:
if name.startswith('zato'):
logger = logging.getLogger(name)
logger.addHandler(handler)
if asbool(profiler_enabled):
profiler_dir = os.path.abspath(
os.path.join(base_dir, server_config.profiler.profiler_dir))
server.on_wsgi_request = ProfileMiddleware(
server.on_wsgi_request,
log_filename=os.path.join(profiler_dir,
server_config.profiler.log_filename),
cachegrind_filename=os.path.join(
profiler_dir, server_config.profiler.cachegrind_filename),
discard_first_request=server_config.profiler.discard_first_request,
flush_at_shutdown=server_config.profiler.flush_at_shutdown,
path=server_config.profiler.url_path,
unwind=server_config.profiler.unwind)
# New in 2.0 - set environmet variables for servers to inherit
os_environ = server_config.get('os_environ', {})
for key, value in os_environ.items():
os.environ[key] = value
# Run the hook right before the Gunicorn-level server actually starts
startup_callable_tool.invoke(SERVER_STARTUP.PHASE.IMPL_BEFORE_RUN,
kwargs={
'zato_gunicorn_app': zato_gunicorn_app,
})
# Run the app at last
if start_gunicorn_app:
zato_gunicorn_app.run()
else:
return zato_gunicorn_app.zato_wsgi_app
def execute(self, args):
repo_dir = os.path.join(
os.path.abspath(os.path.join(self.original_dir, args.path)),
'config', 'repo')
config = get_config(repo_dir, 'server.conf')
priv_key_location = os.path.abspath(
os.path.join(repo_dir, config.crypto.priv_key_location))
cm = CryptoManager(priv_key_location=priv_key_location)
cm.load_keys()
engine_args = Bunch()
engine_args.odb_type = config.odb.engine
engine_args.odb_user = config.odb.username
engine_args.odb_password = cm.decrypt(config.odb.password)
engine_args.odb_host = config.odb.host
engine_args.odb_db_name = config.odb.db_name
engine = self._get_engine(engine_args)
session = self._get_session(engine)
auth = None
headers = {}
with closing(session) as session:
cluster = session.query(Server).\
filter(Server.token == config.main.token).\
one().cluster
channel = session.query(HTTPSOAP).\
filter(HTTPSOAP.cluster_id == cluster.id).\
filter(HTTPSOAP.url_path == args.url_path).\
one()
if channel.security_id:
security = session.query(HTTPBasicAuth).\
filter(HTTPBasicAuth.id == channel.security_id).\
first()
if security:
auth = (security.username, security.password)
if args.headers:
for pair in args.headers.strip().split(';'):
k, v = pair.strip().split('=', 1)
headers[k] = v
client = AnyServiceInvoker(
'http://{}'.format(config.main.gunicorn_bind),
args.url_path,
auth,
max_response_repr=int(args.max_response_repr),
max_cid_repr=int(args.max_cid_repr))
func = client.invoke_async if args. async else client.invoke
response = func(args.name, args.payload, headers, args.channel,
args.data_format, args.transport)
if response.ok:
self.logger.info(response.data or '(None)')
else:
self.logger.error(response.details)
if args.verbose:
self.logger.debug('inner.text:[{}]'.format(response.inner.text))
self.logger.debug('response:[{}]'.format(response))