def update_globals(config, base_dir='.'):
globals()['DATABASES'] = {'default': {}}
for name in 'zato_secret_key', 'well_known_data', 'DATABASE_PASSWORD', 'SECRET_KEY', 'ADMIN_INVOKE_PASSWORD':
config[name] = config[name].encode('utf8')
# If secret key is not given directly in the config file, we will expect to find it
# on command line.
zato_secret_key = config['zato_secret_key']
zato_secret_key = resolve_secret_key(zato_secret_key)
config['zato_secret_key'] = zato_secret_key
cm = WebAdminCryptoManager.from_secret_key(config['zato_secret_key'],
config['well_known_data'],
stdin_data=read_stdin_data())
for k, v in config.items():
if k.startswith('DATABASE_'):
default = globals()['DATABASES']['default']
k = k.replace('DATABASE_', '', 1)
if k == 'PASSWORD' and config['db_type'] != 'sqlite':
v = cm.decrypt(v)
default[k] = str(v)
else:
if k == 'ADMIN_INVOKE_PASSWORD':
v = cm.decrypt(v)
elif k == 'log_config':
v = os.path.join(base_dir, v)
globals()[k] = v
def __init__(self, args):
self.args = args
self.original_dir = os.getcwd()
self.show_output = False if 'ZATO_CLI_DONT_SHOW_OUTPUT' in os.environ else True
self.verbose = args.verbose
self.reset_logger(args)
self.engine = None
if args.store_config:
self.store_config(args)
# Get input from sys.stdin, if any was provided at all. It needs to be read once here,
# because subprocesses will not be able to do it once we read it all in in the parent
# one, so we read it here and give other processes explicitly on input, if they need it.
self.stdin_data = read_stdin_data()
def __init__(self, config, repo_location):
self.odb = config.odb
self.config = config
self.repo_location = repo_location
self.sql_pool_store = PoolStore()
# Set up the crypto manager that will be used by both ODB and, possibly, KVDB
self.config.crypto_manager = SchedulerCryptoManager(
self.repo_location, stdin_data=read_stdin_data())
# ODB connection
self.odb = ODBManager()
if self.config.main.odb.engine != 'sqlite':
self.config.main.odb.password = self.config.crypto_manager.decrypt(
config.main.odb.password)
self.config.main.odb.host = config.main.odb.host
self.config.main.odb.pool_size = config.main.odb.pool_size
self.config.main.odb.username = config.main.odb.username
self.sql_pool_store[ZATO_ODB_POOL_NAME] = self.config.main.odb
main = self.config.main
if main.crypto.use_tls:
priv_key, cert = main.crypto.priv_key_location, main.crypto.cert_location
else:
priv_key, cert = None, None
# API server
self.api_server = WSGIServer((main.bind.host, int(main.bind.port)),
self,
keyfile=priv_key,
certfile=cert)
self.odb.pool = self.sql_pool_store[ZATO_ODB_POOL_NAME].pool
self.odb.init_session(ZATO_ODB_POOL_NAME, self.config.main.odb,
self.odb.pool, False)
self.config.odb = self.odb
# Scheduler
self.scheduler = Scheduler(self.config)
def update_globals(config, base_dir='.'):
globals()['DATABASES'] = {'default': {}}
cm = WebAdminCryptoManager.from_secret_key(config['zato_secret_key'],
config['well_known_data'],
stdin_data=read_stdin_data())
for k, v in config.items():
if k.startswith('DATABASE_'):
default = globals()['DATABASES']['default']
k = k.replace('DATABASE_', '', 1)
if k == 'PASSWORD' and config['db_type'] != 'sqlite':
v = cm.decrypt(v)
default[k] = str(v)
else:
if k == 'ADMIN_INVOKE_PASSWORD':
v = cm.decrypt(v)
elif k == 'log_config':
v = os.path.join(base_dir, v)
globals()[k] = v
def run(base_dir, start_gunicorn_app=True, options=None):
options = options or {}
# Store a pidfile before doing anything else
store_pidfile(base_dir)
# For dumping stacktraces
register_diag_handlers()
# Capture warnings to log files
logging.captureWarnings(True)
# Start initializing the server now
os.chdir(base_dir)
try:
import pymysql
pymysql.install_as_MySQLdb()
except ImportError:
pass
# We're doing it here even if someone doesn't use PostgreSQL at all
# so we're not suprised when someone suddenly starts using PG.
# TODO: Make sure it's registered for each of the subprocess
psycopg2.extensions.register_type(psycopg2.extensions.UNICODE)
psycopg2.extensions.register_type(psycopg2.extensions.UNICODEARRAY)
# We know we don't need warnings because users may explicitly configure no certificate validation.
# We don't want for urllib3 to warn us about it.
import requests as _r
_r.packages.urllib3.disable_warnings()
repo_location = os.path.join(base_dir, 'config', 'repo')
# Configure the logging first, before configuring the actual server.
logging.addLevelName('TRACE1', TRACE1)
logging_conf_path = os.path.join(repo_location, 'logging.conf')
with open(logging_conf_path) as f:
logging_config = yaml.load(f)
dictConfig(logging_config)
logger = logging.getLogger(__name__)
kvdb_logger = logging.getLogger('zato_kvdb')
crypto_manager = ServerCryptoManager(repo_location,
secret_key=options['secret_key'],
stdin_data=read_stdin_data())
secrets_config = ConfigObj(os.path.join(repo_location, 'secrets.conf'),
use_zato=False)
server_config = get_config(repo_location,
'server.conf',
crypto_manager=crypto_manager,
secrets_conf=secrets_config)
pickup_config = get_config(repo_location, 'pickup.conf')
sio_config = get_config(repo_location,
'simple-io.conf',
needs_user_config=False)
sso_config = get_config(repo_location, 'sso.conf', needs_user_config=False)
normalize_sso_config(sso_config)
# Now that we have access to server.conf, greenify libraries required to be made greenlet-friendly,
# assuming that there are any - otherwise do not do anything.
to_greenify = []
for key, value in server_config.get('greenify', {}).items():
if asbool(value):
if not os.path.exists(key):
raise ValueError('No such path `{}`'.format(key))
else:
to_greenify.append(key)
# Go ahead only if we actually have anything to greenify
if to_greenify:
import greenify
greenify.greenify()
for name in to_greenify:
result = greenify.patch_lib(name)
if not result:
raise ValueError(
'Library `{}` could not be greenified'.format(name))
else:
logger.info('Greenified library `%s`', name)
server_config.main.token = server_config.main.token.encode('utf8')
# Do not proceed unless we can be certain our own preferred address or IP can be obtained.
preferred_address = server_config.preferred_address.get('address')
if not preferred_address:
preferred_address = get_preferred_ip(server_config.main.gunicorn_bind,
server_config.preferred_address)
if not preferred_address and not server_config.server_to_server.boot_if_preferred_not_found:
msg = 'Unable to start the server. Could not obtain a preferred address, please configure [bind_options] in server.conf'
logger.warn(msg)
raise Exception(msg)
# Create the startup callable tool as soon as practical
startup_callable_tool = StartupCallableTool(server_config)
# Run the hook before there is any server object created
startup_callable_tool.invoke(SERVER_STARTUP.PHASE.FS_CONFIG_ONLY,
kwargs={
'server_config': server_config,
'pickup_config': pickup_config,
'sio_config': sio_config,
'sso_config': sso_config,
})
# New in 2.0 - Start monitoring as soon as possible
if server_config.get('newrelic', {}).get('config'):
import newrelic.agent
newrelic.agent.initialize(server_config.newrelic.config,
server_config.newrelic.environment or None,
server_config.newrelic.ignore_errors or None,
server_config.newrelic.log_file or None,
server_config.newrelic.log_level or None)
zunicorn.SERVER_SOFTWARE = server_config.misc.get('http_server_header',
'Zato')
# Store KVDB config in logs, possibly replacing its password if told to
kvdb_config = get_kvdb_config_for_log(server_config.kvdb)
kvdb_logger.info('Main process config `%s`', kvdb_config)
# New in 2.0 hence optional
user_locale = server_config.misc.get('locale', None)
if user_locale:
locale.setlocale(locale.LC_ALL, user_locale)
value = 12345
logger.info('Locale is `%s`, amount of %s -> `%s`', user_locale, value,
locale.currency(value, grouping=True).decode('utf-8'))
# Makes queries against Postgres asynchronous
if asbool(server_config.odb.use_async_driver
) and server_config.odb.engine == 'postgresql':
make_psycopg_green()
if server_config.misc.http_proxy:
os.environ['http_proxy'] = server_config.misc.http_proxy
# Basic components needed for the server to boot up
kvdb = KVDB()
odb_manager = ODBManager(well_known_data=ZATO_CRYPTO_WELL_KNOWN_DATA)
sql_pool_store = PoolStore()
service_store = ServiceStore()
service_store.odb = odb_manager
service_store.services = {}
server = ParallelServer()
server.odb = odb_manager
server.service_store = service_store
server.service_store.server = server
server.sql_pool_store = sql_pool_store
server.service_modules = []
server.kvdb = kvdb
# Assigned here because it is a circular dependency
odb_manager.parallel_server = server
zato_gunicorn_app = ZatoGunicornApplication(server, repo_location,
server_config.main,
server_config.crypto)
server.has_fg = options.get('fg')
server.crypto_manager = crypto_manager
server.odb_data = server_config.odb
server.host = zato_gunicorn_app.zato_host
server.port = zato_gunicorn_app.zato_port
server.repo_location = repo_location
server.user_conf_location = os.path.join(server.repo_location, 'user-conf')
server.base_dir = base_dir
server.logs_dir = os.path.join(server.base_dir, 'logs')
server.tls_dir = os.path.join(server.base_dir, 'config', 'repo', 'tls')
server.static_dir = os.path.join(server.base_dir, 'config', 'repo',
'static')
server.json_schema_dir = os.path.join(server.base_dir, 'config', 'repo',
'schema', 'json')
server.fs_server_config = server_config
server.fs_sql_config = get_config(repo_location,
'sql.conf',
needs_user_config=False)
server.pickup_config = pickup_config
server.logging_config = logging_config
server.logging_conf_path = logging_conf_path
server.sio_config = sio_config
server.sso_config = sso_config
server.user_config.update(server_config.user_config_items)
server.preferred_address = preferred_address
server.sync_internal = options['sync_internal']
server.jwt_secret = server.fs_server_config.misc.jwt_secret.encode('utf8')
server.startup_callable_tool = startup_callable_tool
server.is_sso_enabled = server.fs_server_config.component_enabled.sso
if server.is_sso_enabled:
server.sso_api = SSOAPI(server, sso_config, None,
crypto_manager.encrypt, crypto_manager.decrypt,
crypto_manager.hash_secret,
crypto_manager.verify_hash, new_user_id)
# Remove all locks possibly left over by previous server instances
kvdb.component = 'master-proc'
clear_locks(kvdb, server_config.main.token, server_config.kvdb,
crypto_manager.decrypt)
# New in 2.0.8
server.return_tracebacks = asbool(
server_config.misc.get('return_tracebacks', True))
server.default_error_message = server_config.misc.get(
'default_error_message', 'An error has occurred')
# Turn the repo dir into an actual repository and commit any new/modified files
RepoManager(repo_location).ensure_repo_consistency()
# New in 2.0 so it's optional.
profiler_enabled = server_config.get('profiler', {}).get('enabled', False)
# New in 2.0 so it's optional.
sentry_config = server_config.get('sentry')
dsn = sentry_config.pop('dsn', None)
if dsn:
from raven import Client
from raven.handlers.logging import SentryHandler
handler_level = sentry_config.pop('level')
client = Client(dsn, **sentry_config)
handler = SentryHandler(client=client)
handler.setLevel(getattr(logging, handler_level))
logger = logging.getLogger('')
logger.addHandler(handler)
for name in logging.Logger.manager.loggerDict:
if name.startswith('zato'):
logger = logging.getLogger(name)
logger.addHandler(handler)
if asbool(profiler_enabled):
profiler_dir = os.path.abspath(
os.path.join(base_dir, server_config.profiler.profiler_dir))
server.on_wsgi_request = ProfileMiddleware(
server.on_wsgi_request,
log_filename=os.path.join(profiler_dir,
server_config.profiler.log_filename),
cachegrind_filename=os.path.join(
profiler_dir, server_config.profiler.cachegrind_filename),
discard_first_request=server_config.profiler.discard_first_request,
flush_at_shutdown=server_config.profiler.flush_at_shutdown,
path=server_config.profiler.url_path,
unwind=server_config.profiler.unwind)
# New in 2.0 - set environmet variables for servers to inherit
os_environ = server_config.get('os_environ', {})
for key, value in os_environ.items():
os.environ[key] = value
# Run the hook right before the Gunicorn-level server actually starts
startup_callable_tool.invoke(SERVER_STARTUP.PHASE.IMPL_BEFORE_RUN,
kwargs={
'zato_gunicorn_app': zato_gunicorn_app,
})
# Run the app at last
if start_gunicorn_app:
zato_gunicorn_app.run()
else:
return zato_gunicorn_app.zato_wsgi_app
def run(base_dir, start_gunicorn_app=True, options=None):
options = options or {}
# Store a pidfile before doing anything else
store_pidfile(base_dir)
# For dumping stacktraces
register_diag_handlers()
# Capture warnings to log files
logging.captureWarnings(True)
# Start initializing the server now
os.chdir(base_dir)
try:
import pymysql
pymysql.install_as_MySQLdb()
except ImportError:
pass
# We're doing it here even if someone doesn't use PostgreSQL at all
# so we're not suprised when someone suddenly starts using PG.
# TODO: Make sure it's registered for each of the subprocess
psycopg2.extensions.register_type(psycopg2.extensions.UNICODE)
psycopg2.extensions.register_type(psycopg2.extensions.UNICODEARRAY)
# We know we don't need warnings because users may explicitly configure no certificate validation.
# We don't want for urllib3 to warn us about it.
import requests as _r
_r.packages.urllib3.disable_warnings()
repo_location = os.path.join(base_dir, 'config', 'repo')
# Configure the logging first, before configuring the actual server.
logging.addLevelName('TRACE1', TRACE1)
logging_conf_path = os.path.join(repo_location, 'logging.conf')
with open(logging_conf_path) as f:
logging_config = yaml.load(f)
dictConfig(logging_config)
logger = logging.getLogger(__name__)
kvdb_logger = logging.getLogger('zato_kvdb')
crypto_manager = ServerCryptoManager(repo_location, secret_key=options['secret_key'], stdin_data=read_stdin_data())
secrets_config = ConfigObj(os.path.join(repo_location, 'secrets.conf'), use_zato=False)
server_config = get_config(repo_location, 'server.conf', crypto_manager=crypto_manager, secrets_conf=secrets_config)
pickup_config = get_config(repo_location, 'pickup.conf')
sio_config = get_config(repo_location, 'simple-io.conf', needs_user_config=False)
sso_config = get_config(repo_location, 'sso.conf', needs_user_config=False)
normalize_sso_config(sso_config)
# Do not proceed unless we can be certain our own preferred address or IP can be obtained.
preferred_address = server_config.preferred_address.get('address')
if not preferred_address:
preferred_address = get_preferred_ip(server_config.main.gunicorn_bind, server_config.preferred_address)
if not preferred_address and not server_config.server_to_server.boot_if_preferred_not_found:
msg = 'Unable to start the server. Could not obtain a preferred address, please configure [bind_options] in server.conf'
logger.warn(msg)
raise Exception(msg)
# New in 2.0 - Start monitoring as soon as possible
if server_config.get('newrelic', {}).get('config'):
import newrelic.agent
newrelic.agent.initialize(
server_config.newrelic.config, server_config.newrelic.environment or None, server_config.newrelic.ignore_errors or None,
server_config.newrelic.log_file or None, server_config.newrelic.log_level or None)
# New in 2.0 - override gunicorn-set Server HTTP header
gunicorn.SERVER_SOFTWARE = server_config.misc.get('http_server_header', 'Zato')
# Store KVDB config in logs, possibly replacing its password if told to
kvdb_config = get_kvdb_config_for_log(server_config.kvdb)
kvdb_logger.info('Main process config `%s`', kvdb_config)
# New in 2.0 hence optional
user_locale = server_config.misc.get('locale', None)
if user_locale:
locale.setlocale(locale.LC_ALL, user_locale)
value = 12345
logger.info('Locale is `%s`, amount of %s -> `%s`', user_locale, value, locale.currency(
value, grouping=True).decode('utf-8'))
# Spring Python
app_context = get_app_context(server_config)
# Makes queries against Postgres asynchronous
if asbool(server_config.odb.use_async_driver) and server_config.odb.engine == 'postgresql':
make_psycopg_green()
if server_config.misc.http_proxy:
os.environ['http_proxy'] = server_config.misc.http_proxy
server = app_context.get_object('server')
zato_gunicorn_app = ZatoGunicornApplication(server, repo_location, server_config.main, server_config.crypto)
server.crypto_manager = crypto_manager
server.odb_data = server_config.odb
server.host = zato_gunicorn_app.zato_host
server.port = zato_gunicorn_app.zato_port
server.repo_location = repo_location
server.user_conf_location = os.path.join(server.repo_location, 'user-conf')
server.base_dir = base_dir
server.logs_dir = os.path.join(server.base_dir, 'logs')
server.tls_dir = os.path.join(server.base_dir, 'config', 'repo', 'tls')
server.fs_server_config = server_config
server.fs_sql_config = get_config(repo_location, 'sql.conf', needs_user_config=False)
server.pickup_config = pickup_config
server.logging_config = logging_config
server.logging_conf_path = logging_conf_path
server.sio_config = sio_config
server.sso_config = sso_config
server.user_config.update(server_config.user_config_items)
server.app_context = app_context
server.preferred_address = preferred_address
server.sync_internal = options['sync_internal']
server.jwt_secret = server.fs_server_config.misc.jwt_secret.encode('utf8')
server.is_sso_enabled = server.fs_server_config.component_enabled.sso
if server.is_sso_enabled:
server.sso_api = SSOAPI(server, sso_config, None, crypto_manager.encrypt, crypto_manager.decrypt,
crypto_manager.hash_secret, crypto_manager.verify_hash, new_user_id)
# Remove all locks possibly left over by previous server instances
kvdb = app_context.get_object('kvdb')
kvdb.component = 'master-proc'
clear_locks(kvdb, server_config.main.token, server_config.kvdb, crypto_manager.decrypt)
# New in 2.0.8
server.return_tracebacks = asbool(server_config.misc.get('return_tracebacks', True))
server.default_error_message = server_config.misc.get('default_error_message', 'An error has occurred')
# Turn the repo dir into an actual repository and commit any new/modified files
RepoManager(repo_location).ensure_repo_consistency()
# New in 2.0 so it's optional.
profiler_enabled = server_config.get('profiler', {}).get('enabled', False)
# New in 2.0 so it's optional.
sentry_config = server_config.get('sentry')
dsn = sentry_config.pop('dsn', None)
if dsn:
from raven import Client
from raven.handlers.logging import SentryHandler
handler_level = sentry_config.pop('level')
client = Client(dsn, **sentry_config)
handler = SentryHandler(client=client)
handler.setLevel(getattr(logging, handler_level))
logger = logging.getLogger('')
logger.addHandler(handler)
for name in logging.Logger.manager.loggerDict:
if name.startswith('zato'):
logger = logging.getLogger(name)
logger.addHandler(handler)
if asbool(profiler_enabled):
profiler_dir = os.path.abspath(os.path.join(base_dir, server_config.profiler.profiler_dir))
server.on_wsgi_request = ProfileMiddleware(
server.on_wsgi_request,
log_filename = os.path.join(profiler_dir, server_config.profiler.log_filename),
cachegrind_filename = os.path.join(profiler_dir, server_config.profiler.cachegrind_filename),
discard_first_request = server_config.profiler.discard_first_request,
flush_at_shutdown = server_config.profiler.flush_at_shutdown,
path = server_config.profiler.url_path,
unwind = server_config.profiler.unwind)
# New in 2.0 - set environmet variables for servers to inherit
os_environ = server_config.get('os_environ', {})
for key, value in os_environ.items():
os.environ[key] = value
# Run the app at last
if start_gunicorn_app:
zato_gunicorn_app.run()
else:
return zato_gunicorn_app.zato_wsgi_app
