def execute(self,
args,
show_output=True,
admin_password=None,
needs_admin_created_flag=False):
try:
import pymysql
pymysql.install_as_MySQLdb()
except ImportError:
pass
# stdlib
import os, json
from random import getrandbits
from uuid import uuid4
# Django
from django.core.management import call_command
# Python 2/3 compatibility
from past.builtins import unicode
# Zato
# TODO: There really shouldn't be any direct dependency between zato-cli and zato-web-admin
from zato.admin.zato_settings import update_globals
from zato.cli import common_logging_conf_contents, is_arg_given
from zato.common.crypto.api import WebAdminCryptoManager
from zato.common.crypto.const import well_known_data
from zato.common.defaults import web_admin_host, web_admin_port
os.chdir(self.target_dir)
repo_dir = os.path.join(self.target_dir, 'config', 'repo')
web_admin_conf_path = os.path.join(repo_dir, 'web-admin.conf')
initial_data_json_path = os.path.join(repo_dir, 'initial-data.json')
os.mkdir(os.path.join(self.target_dir, 'logs'))
os.mkdir(os.path.join(self.target_dir, 'config'))
os.mkdir(repo_dir)
user_name = 'admin'
admin_password = admin_password if admin_password else WebAdminCryptoManager.generate_password(
)
# If we have a CA's certificate then it implicitly means that there is some CA
# which tells us that we are to trust both the CA and the certificates that it issues,
# and the only certificate we are interested in is the one to the load-balancer.
# This is why, if we get ca_certs_path, it must be because we are to use TLS
# in communication with the load-balancer's agent which in turn means that we have crypto material on input.
has_crypto = is_arg_given(args, 'ca_certs_path')
if has_crypto:
self.copy_web_admin_crypto(repo_dir, args)
zato_secret_key = WebAdminCryptoManager.generate_key()
cm = WebAdminCryptoManager.from_secret_key(zato_secret_key)
django_secret_key = uuid4().hex.encode('utf8')
django_site_id = getrandbits(20)
admin_invoke_password = getattr(args, 'admin_invoke_password', None)
if not admin_invoke_password:
admin_invoke_password = '******' + uuid4().hex
if isinstance(admin_invoke_password, unicode):
admin_invoke_password = admin_invoke_password.encode('utf8')
odb_password = args.odb_password or ''
odb_password = odb_password.encode('utf8')
config = {
'host': web_admin_host,
'port': web_admin_port,
'db_type': args.odb_type,
'log_config': 'logging.conf',
'lb_agent_use_tls': 'true' if has_crypto else 'false',
'zato_secret_key': zato_secret_key,
'well_known_data': cm.encrypt(well_known_data.encode('utf8')),
'DATABASE_NAME': args.odb_db_name or args.sqlite_path,
'DATABASE_USER': args.odb_user or '',
'DATABASE_PASSWORD': cm.encrypt(odb_password),
'DATABASE_HOST': args.odb_host or '',
'DATABASE_PORT': args.odb_port or '',
'SITE_ID': django_site_id,
'SECRET_KEY': cm.encrypt(django_secret_key),
'ADMIN_INVOKE_NAME': 'admin.invoke',
'ADMIN_INVOKE_PASSWORD': cm.encrypt(admin_invoke_password),
}
for name in 'zato_secret_key', 'well_known_data', 'DATABASE_PASSWORD', 'SECRET_KEY', 'ADMIN_INVOKE_PASSWORD':
config[name] = config[name].decode('utf8')
open(os.path.join(repo_dir, 'logging.conf'), 'w').write(
common_logging_conf_contents.format(
log_path='./logs/web-admin.log'))
open(web_admin_conf_path, 'w').write(config_template.format(**config))
open(initial_data_json_path,
'w').write(initial_data_json.format(**config))
# Initial info
self.store_initial_info(self.target_dir,
self.COMPONENTS.WEB_ADMIN.code)
config = json.loads(
open(os.path.join(repo_dir, 'web-admin.conf')).read())
config['config_dir'] = self.target_dir
update_globals(config, self.target_dir)
os.environ['DJANGO_SETTINGS_MODULE'] = 'zato.admin.settings'
import django
django.setup()
self.reset_logger(args, True)
# Can't import these without DJANGO_SETTINGS_MODULE being set
from django.contrib.auth.models import User
from django.db import connection
from django.db.utils import IntegrityError
call_command('migrate',
run_syncdb=True,
interactive=False,
verbosity=0)
call_command('loaddata', initial_data_json_path, verbosity=0)
try:
call_command('createsuperuser',
interactive=False,
username=user_name,
first_name='admin-first-name',
last_name='admin-last-name',
email='*****@*****.**')
admin_created = True
user = User.objects.get(username=user_name)
user.set_password(admin_password)
user.save()
except IntegrityError:
# This will happen if user 'admin' already exists, e.g. if this is not the first cluster in this database
admin_created = False
connection._rollback()
# Needed because Django took over our logging config
self.reset_logger(args, True)
if show_output:
if self.verbose:
msg = """Successfully created a web admin instance.
You can start it with the 'zato start {path}' command.""".format(
path=os.path.abspath(
os.path.join(os.getcwd(), self.target_dir)))
self.logger.debug(msg)
else:
self.logger.info('OK')
# We return it only when told to explicitly so when the command runs from CLI
# it doesn't return a non-zero exit code.
if needs_admin_created_flag:
return admin_created
def execute(self, args, default_http_port=None, show_output=True, return_server_id=False):
# stdlib
import os
from datetime import datetime
from traceback import format_exc
# Cryptography
from cryptography.fernet import Fernet
# SQLAlchemy
from sqlalchemy.exc import IntegrityError
# Python 2/3 compatibility
from six import PY3
# Zato
from zato.cli._apispec_default import apispec_files
from zato.common.api import SERVER_JOIN_STATUS
from zato.common.crypto.const import well_known_data
from zato.common.defaults import http_plain_server_port
from zato.common.odb.model import Cluster, Server
default_http_port = default_http_port or http_plain_server_port
engine = self._get_engine(args)
session = self._get_session(engine)
cluster = session.query(Cluster).\
filter(Cluster.name == args.cluster_name).\
first()
if not cluster:
self.logger.error("Cluster `%s` doesn't exist in ODB", args.cluster_name)
return self.SYS_ERROR.NO_SUCH_CLUSTER
server = Server(cluster=cluster)
server.name = args.server_name
if isinstance(self.token, (bytes, bytearray)):
server.token = self.token.decode('utf8')
else:
server.token = self.token
server.last_join_status = SERVER_JOIN_STATUS.ACCEPTED
server.last_join_mod_by = self._get_user_host()
server.last_join_mod_date = datetime.utcnow()
session.add(server)
try:
if not self.dirs_prepared:
self.prepare_directories(show_output)
repo_dir = os.path.join(self.target_dir, 'config', 'repo')
# Note that server crypto material is optional so if none was given on input
# this command will be a no-op.
self.copy_server_crypto(repo_dir, args)
if show_output:
self.logger.debug('Created a repo in {}'.format(repo_dir))
self.logger.debug('Creating files..')
for file_name, contents in sorted(files.items()):
file_name = os.path.join(self.target_dir, file_name)
if show_output:
self.logger.debug('Creating {}'.format(file_name))
f = open(file_name, 'w')
f.write(contents)
f.close()
logging_conf_loc = os.path.join(self.target_dir, 'config/repo/logging.conf')
logging_conf = open(logging_conf_loc).read()
open(logging_conf_loc, 'w').write(logging_conf.format(
log_path=os.path.join(self.target_dir, 'logs', 'zato.log')))
if show_output:
self.logger.debug('Logging configuration stored in {}'.format(logging_conf_loc))
odb_engine=args.odb_type
if odb_engine.startswith('postgresql'):
odb_engine = 'postgresql+pg8000'
server_conf_loc = os.path.join(self.target_dir, 'config/repo/server.conf')
server_conf = open(server_conf_loc, 'w')
server_conf.write(
server_conf_template.format(
port=getattr(args, 'http_port', None) or default_http_port,
gunicorn_workers=1,
odb_db_name=args.odb_db_name or args.sqlite_path,
odb_engine=odb_engine,
odb_host=args.odb_host or '',
odb_port=args.odb_port or '',
odb_pool_size=default_odb_pool_size,
odb_user=args.odb_user or '',
kvdb_host=args.kvdb_host,
kvdb_port=args.kvdb_port,
initial_cluster_name=args.cluster_name,
initial_server_name=args.server_name,
))
server_conf.close()
pickup_conf_loc = os.path.join(self.target_dir, 'config/repo/pickup.conf')
pickup_conf_file = open(pickup_conf_loc, 'w')
pickup_conf_file.write(pickup_conf)
pickup_conf_file.close()
user_conf_loc = os.path.join(self.target_dir, 'config/repo/user.conf')
user_conf = open(user_conf_loc, 'w')
user_conf.write(user_conf_contents)
user_conf.close()
sso_conf_loc = os.path.join(self.target_dir, 'config/repo/sso.conf')
sso_conf = open(sso_conf_loc, 'w')
sso_conf.write(sso_conf_contents)
sso_conf.close()
# There will be multiple keys in future releases to allow for key rotation
key1 = args.secret_key or Fernet.generate_key()
fernet1 = Fernet(key1)
secrets_conf_loc = os.path.join(self.target_dir, 'config/repo/secrets.conf')
secrets_conf = open(secrets_conf_loc, 'w')
kvdb_password = args.kvdb_password or ''
kvdb_password = kvdb_password.encode('utf8')
kvdb_password = fernet1.encrypt(kvdb_password)
kvdb_password = kvdb_password.decode('utf8')
odb_password = args.odb_password or ''
odb_password = odb_password.encode('utf8')
odb_password = fernet1.encrypt(odb_password)
odb_password = odb_password.decode('utf8')
zato_well_known_data = fernet1.encrypt(well_known_data.encode('utf8'))
zato_well_known_data = zato_well_known_data.decode('utf8')
if isinstance(key1, (bytes, bytearray)):
key1 = key1.decode('utf8')
zato_main_token = fernet1.encrypt(self.token)
zato_main_token = zato_main_token.decode('utf8')
zato_misc_jwt_secret = getattr(args, 'jwt_secret', None)
if not zato_misc_jwt_secret:
zato_misc_jwt_secret = Fernet.generate_key()
if not isinstance(zato_misc_jwt_secret, bytes):
zato_misc_jwt_secret = zato_misc_jwt_secret.encode('utf8')
zato_misc_jwt_secret = fernet1.encrypt(zato_misc_jwt_secret)
if isinstance(zato_misc_jwt_secret, bytes):
zato_misc_jwt_secret = zato_misc_jwt_secret.decode('utf8')
secrets_conf.write(secrets_conf_template.format(
keys_key1=key1,
zato_well_known_data=zato_well_known_data,
zato_kvdb_password=kvdb_password,
zato_main_token=zato_main_token,
zato_misc_jwt_secret=zato_misc_jwt_secret,
zato_odb_password=odb_password,
))
secrets_conf.close()
bytes_to_str_encoding = 'utf8' if PY3 else ''
simple_io_conf_loc = os.path.join(self.target_dir, 'config/repo/simple-io.conf')
simple_io_conf = open(simple_io_conf_loc, 'w')
simple_io_conf.write(simple_io_conf_contents.format(
bytes_to_str_encoding=bytes_to_str_encoding
))
simple_io_conf.close()
if show_output:
self.logger.debug('Core configuration stored in {}'.format(server_conf_loc))
# Sphinx APISpec files
for file_path, contents in apispec_files.items():
full_path = os.path.join(self.target_dir, 'config/repo/static/sphinxdoc/apispec', file_path)
dir_name = os.path.dirname(full_path)
try:
os.makedirs(dir_name, 0o770)
except OSError:
# That is fine, the directory must have already created in one of previous iterations
pass
finally:
api_file = open(full_path, 'w')
api_file.write(contents)
api_file.close()
# Initial info
self.store_initial_info(self.target_dir, self.COMPONENTS.SERVER.code)
session.commit()
except IntegrityError:
msg = 'Server name `{}` already exists'.format(args.server_name)
if self.verbose:
msg += '. Caught an exception:`{}`'.format(format_exc())
self.logger.error(msg)
session.rollback()
return self.SYS_ERROR.SERVER_NAME_ALREADY_EXISTS
except Exception:
self.logger.error('Could not create the server, e:`%s`', format_exc())
session.rollback()
else:
if show_output:
self.logger.debug('Server added to the ODB')
if show_output:
if self.verbose:
msg = """Successfully created a new server.
You can now start it with the 'zato start {}' command.""".format(self.target_dir)
self.logger.debug(msg)
else:
self.logger.info('OK')
# This is optional - need only by quickstart.py and needs to be requested explicitly,
# otherwise it would be construed as a non-0 return code from this process.
if return_server_id:
return server.id
def execute(self, args, show_output=True, needs_created_flag=False):
os.chdir(self.target_dir)
repo_dir = os.path.join(self.target_dir, 'config', 'repo')
conf_path = os.path.join(repo_dir, 'scheduler.conf')
startup_jobs_conf_path = os.path.join(repo_dir, 'startup_jobs.conf')
sql_conf_path = os.path.join(repo_dir, 'sql.conf')
os.mkdir(os.path.join(self.target_dir, 'logs'))
os.mkdir(os.path.join(self.target_dir, 'config'))
os.mkdir(repo_dir)
self.copy_scheduler_crypto(repo_dir, args)
if hasattr(args, 'get'):
secret_key = args.get('secret_key')
else:
secret_key = args.secret_key
secret_key = secret_key or SchedulerCryptoManager.generate_key()
cm = SchedulerCryptoManager.from_secret_key(secret_key)
odb_engine=args.odb_type
if odb_engine.startswith('postgresql'):
odb_engine = 'postgresql+pg8000'
if args.cluster_id:
cluster_id = args.cluster_id
else:
cluster_id = self._get_cluster_id_by_name(args, args.cluster_name)
odb_password = args.odb_password or ''
odb_password = odb_password.encode('utf8')
odb_password = cm.encrypt(odb_password)
odb_password = odb_password.decode('utf8')
kvdb_password = args.kvdb_password or ''
kvdb_password = kvdb_password.encode('utf8')
kvdb_password = cm.encrypt(kvdb_password)
kvdb_password = kvdb_password.decode('utf8')
user1_password = cm.generate_password()
user1_password = cm.encrypt(user1_password)
user1_password = user1_password.decode('utf8')
zato_well_known_data = well_known_data.encode('utf8')
zato_well_known_data = cm.encrypt(zato_well_known_data)
zato_well_known_data = zato_well_known_data.decode('utf8')
if isinstance(secret_key, (bytes, bytearray)):
secret_key = secret_key.decode('utf8')
# We will use TLS only if we were given crypto material on input
use_tls = is_arg_given(args, 'priv_key_path')
config = {
'odb_db_name': args.odb_db_name or args.sqlite_path,
'odb_engine': odb_engine,
'odb_host': args.odb_host or '',
'odb_port': args.odb_port or '',
'odb_password': odb_password,
'odb_username': args.odb_user or '',
'broker_host': args.kvdb_host,
'broker_port': args.kvdb_port,
'broker_password': kvdb_password,
'user1_password': user1_password,
'cluster_id': cluster_id,
'cluster_name': args.cluster_name,
'secret_key1': secret_key,
'well_known_data': zato_well_known_data,
'use_tls': 'true' if use_tls else 'false'
}
open(os.path.join(repo_dir, 'logging.conf'), 'w').write(
common_logging_conf_contents.format(log_path='./logs/scheduler.log'))
open(conf_path, 'w').write(config_template.format(**config))
open(startup_jobs_conf_path, 'w').write(startup_jobs)
open(sql_conf_path, 'w').write(sql_conf_contents)
# Initial info
self.store_initial_info(self.target_dir, self.COMPONENTS.SCHEDULER.code)
if show_output:
if self.verbose:
msg = """Successfully created a scheduler instance.
You can start it with the 'zato start {path}' command.""".format(path=os.path.abspath(os.path.join(os.getcwd(), self.target_dir)))
self.logger.debug(msg)
else:
self.logger.info('OK')
# We return it only when told to explicitly so when the command runs from CLI
# it doesn't return a non-zero exit code.
if needs_created_flag:
return True