def main():
deploy_cfg = configparser.ConfigParser()
try:
deploy_cfg.read(deploy_cfg_file)
if len(deploy_cfg.sections()) == 0:
raise FileNotFoundError("can't read: {}".format(deploy_cfg_file))
Database.address = get_no_empty(deploy_cfg, "Database", "IPv4-Address")
Database.ssh_root_user = get_no_empty(deploy_cfg, "Database",
"SSH-Root-User")
Database.ssh_port = get_no_empty(deploy_cfg, "Database", "SSH-Port")
Database.mysql_port = get_no_empty(deploy_cfg, "Database",
"MySql-Port")
Storage.address = get_no_empty(deploy_cfg, "Storage", "IPv4-Address")
Storage.rtt_dir = get_no_empty(deploy_cfg, "Storage", "RTT-Files-dir")
Storage.ssh_config = get_no_empty(deploy_cfg, "Storage", "SSH-Config")
Storage.acc_name = get_no_empty(deploy_cfg, "Storage", "Storage-User")
Storage.acc_chroot = get_no_empty(deploy_cfg, "Storage",
"Storage-Chroot")
except BaseException as e:
print_error("Configuration file: {}".format(e))
sys.exit(1)
# Sanity checks
try:
check_paths_abs(
{Storage.acc_chroot, Storage.CHROOT_HOME_DIR, Storage.rtt_dir})
check_paths_rel({
Storage.CHROOT_CONF_DIR, Storage.CHROOT_DATA_DIR, Storage.SSH_DIR,
Storage.COMMON_FILES_DIR, Storage.CREDENTIALS_DIR
})
check_files_exists(
{Storage.ssh_config, CommonConst.STORAGE_CLEAN_CACHE})
except AssertionError as e:
print_error("Invalid configuration. {}".format(e))
sys.exit(1)
# Declaring all variables that
# will be used in this script.
# These are mostly absolute paths put
# together from the settings.
# Path related to storage user home
Storage.home_dir = \
"{}{}".format(Storage.acc_chroot, Storage.CHROOT_HOME_DIR)
Storage.ssh_dir = \
os.path.join(Storage.home_dir, Storage.SSH_DIR)
Storage.authorized_keys_file = \
os.path.join(Storage.ssh_dir, Storage.AUTH_KEYS_FILE)
Storage.data_dir = \
os.path.join(Storage.home_dir, Storage.CHROOT_DATA_DIR)
Storage.config_dir = \
os.path.join(Storage.home_dir, Storage.CHROOT_CONF_DIR)
# Paths related to rtt files on server
Storage.rtt_file_store_ini = \
os.path.join(Storage.rtt_dir, Storage.STORE_CONFIG_FILE)
Storage.rtt_file_clean_cache = \
os.path.join(Storage.rtt_dir, Storage.CLEAN_CACHE_SCRIPT)
Storage.rtt_file_clean_cache_log = \
os.path.join(Storage.rtt_dir, Storage.CLEAN_CACHE_LOG)
Storage.rtt_common_dir = \
os.path.join(Storage.rtt_dir, Storage.COMMON_FILES_DIR)
Storage.rtt_credentials_dir \
= os.path.join(Storage.rtt_dir, Storage.CREDENTIALS_DIR)
Storage.rtt_file_mysql_cred = \
os.path.join(Storage.rtt_credentials_dir, Storage.MYSQL_CREDENTIALS_FILE)
try:
# Creating sftp jail for account
# Adding rtt-admin group that is intended to manage
# directories and files related to rtt without root access
exec_sys_call_check("groupadd {}".format(Storage.RTT_ADMIN_GROUP),
acc_codes=[0, 9])
# Adding user for access
exec_sys_call_check("useradd -d {} -s /usr/sbin/nologin {}".format(
Storage.CHROOT_HOME_DIR, Storage.acc_name),
acc_codes=[0, 9])
# Configuring ssh server
sshd_config_append = "\n\n\n\n" \
"Match User {0}\n" \
"\tChrootDirectory {1}\n" \
"\tForceCommand internal-sftp\n" \
"\tAllowTcpForwarding no\n" \
"\tPermitTunnel no\n" \
"\tX11Forwarding no\n" \
"\tAuthorizedKeysFile {1}{2}\n" \
"\tPasswordAuthentication no\n" \
"\n".format(Storage.acc_name, Storage.acc_chroot,
os.path.join(Storage.CHROOT_HOME_DIR,
Storage.SSH_DIR, Storage.AUTH_KEYS_FILE))
with open(Storage.ssh_config, "a") as f:
f.write(sshd_config_append)
exec_sys_call_check("service sshd restart")
# Creating sftp jail for accessing storage
create_dir(Storage.acc_chroot, 0o755)
create_dir(Storage.home_dir,
0o700,
own=Storage.acc_name,
grp=Storage.acc_name)
create_dir(Storage.ssh_dir,
0o700,
own=Storage.acc_name,
grp=Storage.acc_name)
create_dir(Storage.data_dir,
0o700,
own=Storage.acc_name,
grp=Storage.acc_name)
create_dir(Storage.config_dir,
0o700,
own=Storage.acc_name,
grp=Storage.acc_name)
create_file(Storage.authorized_keys_file,
0o600,
own=Storage.acc_name,
grp=Storage.acc_name)
# Creating directory for rtt files on the server
create_dir(Storage.rtt_dir, 0o2770, grp=Storage.RTT_ADMIN_GROUP)
# Set ACL on top directory - ensures all new files will have correct permissions
exec_sys_call_check("setfacl -R -d -m g::rwx {}".format(
Storage.rtt_dir))
exec_sys_call_check("setfacl -R -d -m o::--- {}".format(
Storage.rtt_dir))
create_dir(Storage.rtt_credentials_dir,
0o2770,
grp=Storage.RTT_ADMIN_GROUP)
# Copying script for cache cleaning
shutil.copy(CommonConst.STORAGE_CLEAN_CACHE,
Storage.rtt_file_clean_cache)
chmod_chown(Storage.rtt_file_clean_cache,
0o770,
grp=Storage.RTT_ADMIN_GROUP)
# Copying common scripts into directory
# with rtt files
if os.path.exists(Storage.rtt_common_dir):
shutil.rmtree(Storage.rtt_common_dir)
shutil.copytree(CommonConst.COMMON_FILES_DIR, Storage.rtt_common_dir)
recursive_chmod_chown(Storage.rtt_common_dir,
mod_f=0o660,
mod_d=0o2770,
grp=Storage.RTT_ADMIN_GROUP)
# Creating configuration file for storage server scripts
ini_cfg = configparser.ConfigParser()
ini_cfg.add_section("MySQL-Database")
ini_cfg.set("MySQL-Database", "Name", Database.MYSQL_DB_NAME)
ini_cfg.set("MySQL-Database", "Address", Database.address)
ini_cfg.set("MySQL-Database", "Port", Database.mysql_port)
ini_cfg.set("MySQL-Database", "Credentials-file",
Storage.rtt_file_mysql_cred)
ini_cfg.add_section("Local-cache")
ini_cfg.set("Local-cache", "Data-directory", Storage.data_dir)
ini_cfg.set("Local-cache", "Config-directory", Storage.config_dir)
with open(Storage.rtt_file_store_ini, "w") as f:
ini_cfg.write(f)
# Creating credentials file for database access
cred_db_password = get_rnd_pwd()
cred_cfg = configparser.ConfigParser()
cred_cfg.add_section("Credentials")
cred_cfg.set("Credentials", "Username", Storage.MYSQL_STORAGE_USER)
cred_cfg.set("Credentials", "Password", cred_db_password)
with open(Storage.rtt_file_mysql_cred, "w") as f:
cred_cfg.write(f)
# Installing required packages
install_debian_pkg("libmysqlclient-dev")
install_debian_pkg("python3-pip")
install_debian_pkg("python3-cryptography")
install_debian_pkg("python3-paramiko")
install_python_pkg("mysqlclient")
# This can be done only after installing cryptography and paramiko
from common.rtt_registration import register_db_user
register_db_user(Database.ssh_root_user,
Database.address,
Database.ssh_port,
Storage.MYSQL_STORAGE_USER,
cred_db_password,
Storage.address,
Database.MYSQL_ROOT_USERNAME,
Database.MYSQL_DB_NAME,
priv_select=True)
# Adding new job to cron - cache cleaning script
add_cron_job(Storage.rtt_file_clean_cache, Storage.rtt_file_store_ini,
Storage.rtt_file_clean_cache_log)
# All configured here.
except BaseException as e:
print_error("{}. Fix error and run the script again.".format(e))
def main():
parser = argparse.ArgumentParser(description='Worker deployment')
parser.add_argument('--metacentrum',
dest='metacentrum',
action='store_const',
const=True,
default=False,
help='Metacetrum deployment')
parser.add_argument('--docker',
dest='docker',
action='store_const',
const=True,
default=False,
help='Docker deployment')
parser.add_argument(
'--db-passwd',
dest='db_passwd',
help='DB password to use, if given, skips DB registration')
parser.add_argument(
'--ssh-passphrase',
dest='ssh_passphrase',
help='SSH passphrase to use to protect the private key')
parser.add_argument(
'--ssh-priv',
dest='ssh_priv',
help=
'SSH private key to use instead of generated one, has to have .pub counterpart'
)
parser.add_argument('--no-db-reg',
dest='no_db_reg',
action='store_const',
const=True,
default=False,
help='Skip MySQL user registration')
parser.add_argument('--no-ssh-reg',
dest='no_ssh_reg',
action='store_const',
const=True,
default=False,
help='Skip SSH key registration at storage server')
parser.add_argument('--no-email',
dest='no_email',
action='store_const',
const=True,
default=False,
help='Skip email registration')
parser.add_argument('--no-cron',
dest='no_cron',
action='store_const',
const=True,
default=False,
help='Skip cron setup')
parser.add_argument('--sys-python',
dest='sys_python',
action='store_const',
const=True,
default=False,
help='Use system python')
parser.add_argument(
'--ph4-rtt',
dest='ph4_rtt',
action='store_const',
const=True,
default=False,
help='Use Ph4r05 fork of RTT - required for metacentrum')
parser.add_argument('--pub-storage',
dest='public_storage',
action='store_const',
const=True,
default=False,
help='Use public storage address')
parser.add_argument('--local-db',
dest='local_db',
action='store_const',
const=True,
default=False,
help='DB server is on the same machine')
parser.add_argument('--mysql-pass',
dest='mysql_pass',
action='store_const',
const=True,
default=False,
help='DB password to use')
parser.add_argument('--mysql-pass-file',
dest='mysql_pass_file',
action='store_const',
const=True,
default=False,
help='DB password file to use')
parser.add_argument('--deb9',
dest='deb9',
type=int,
default=0,
help='Compile for debian 9')
parser.add_argument('--deb10',
dest='deb10',
type=int,
default=1,
help='Compile for debian 10')
parser.add_argument('--build-only-rtt',
dest='build_only_rtt',
type=int,
default=0,
help='Compile RTT only')
parser.add_argument('--build-only-cryptostreams',
dest='build_only_cryptostreams',
type=int,
default=0,
help='Compile cryptostreams only')
parser.add_argument('--build-only-batteries',
dest='build_only_batteries',
type=int,
default=0,
help='Compile batteries only')
parser.add_argument('-j',
dest='buildj',
type=int,
default=2,
help='Parallel compilation')
parser.add_argument('--config',
dest='config',
default='deployment_settings.ini',
help='Path to deployment_settings.ini')
parser.add_argument('backend_id',
default=None,
help='Backend ID to deploy')
args = parser.parse_args()
deploy_cfg_file = args.config
wbare = not args.metacentrum
if args.metacentrum or args.docker:
args.ph4_rtt = True
args.public_storage = True
# Get path to main config from console
if not args.backend_id:
print("\nUsage: ./deploy_backend.py <backend-id>\n")
print(
"<backend-id> must be entered according to config with deployment settings.\n"
" Configuration file \"{}\" must\n"
" contain one and only one section named\n"
" \"Backend-<backend-id>\"\n".format(deploy_cfg_file))
sys.exit(1)
deploy_cfg = configparser.ConfigParser()
try:
current_dir = os.path.abspath(os.path.curdir)
deploy_cfg.read(deploy_cfg_file)
if len(deploy_cfg.sections()) == 0:
raise FileNotFoundError("can't read: {}".format(deploy_cfg_file))
backend_sec = "Backend-" + args.backend_id
Backend.address = get_no_empty(deploy_cfg, backend_sec, "IPv4-Address")
Backend.rtt_files_dir = get_no_empty(deploy_cfg, backend_sec,
"RTT-Files-dir")
Backend.exec_max_tests = get_no_empty(deploy_cfg, backend_sec,
"Maximum-parallel-tests")
Backend.exec_test_timeout = get_no_empty(deploy_cfg, backend_sec,
"Maximum-seconds-per-test")
Backend.backend_id = deploy_cfg.get(backend_sec,
"backend-id",
fallback=None)
Backend.backend_name = deploy_cfg.get(backend_sec,
"backend-name",
fallback=backend_sec)
Backend.backend_loc = deploy_cfg.get(backend_sec,
"backend-loc",
fallback='')
Backend.backend_longterm = deploy_cfg.get(backend_sec,
"backend-longterm",
fallback=1)
Backend.backend_aux = deploy_cfg.get(backend_sec,
"backend-aux",
fallback='{}')
Backend.log_dir = deploy_cfg.get(backend_sec, "log-dir", fallback=None)
if not Backend.backend_id:
Backend.backend_id = hashlib.md5(str(
time.time()).encode('utf8')).hexdigest()
Database.address = get_no_empty(deploy_cfg, "Database", "IPv4-Address")
Database.mysql_port = get_no_empty(deploy_cfg, "Database",
"MySQL-port")
Database.ssh_port = get_no_empty(deploy_cfg, "Database", "SSH-Port")
Database.ssh_root_user = get_no_empty(deploy_cfg, "Database",
"SSH-Root-User")
Storage.address_private = get_no_empty(deploy_cfg, "Storage",
"IPv4-Address")
Storage.address_public = deploy_cfg.get(
"Storage", "IPv4-Address-Public") or Storage.address_private
Storage.address = Storage.address_public if args.public_storage else Storage.address_private
Storage.ssh_root_user = get_no_empty(deploy_cfg, "Storage",
"SSH-Root-User")
Storage.acc_chroot = get_no_empty(deploy_cfg, "Storage",
"Storage-Chroot")
Storage.storage_user = get_no_empty(deploy_cfg, "Storage",
"Storage-User")
Storage.ssh_port = get_no_empty(deploy_cfg, "Storage", "SSH-port")
except Exception as e:
print_error("Configuration file: {}".format(e))
sys.exit(1)
# Sanity checks
try:
check_paths_abs({Backend.rtt_files_dir})
check_paths_rel({
Backend.COMMON_FILES_DIR,
Backend.CACHE_DATA_DIR,
Backend.CACHE_CONFIG_DIR,
Backend.CREDENTIALS_DIR,
Backend.RTT_EXECUTION_DIR,
Backend.RANDOMNESS_TESTING_TOOLKIT_SRC_DIR,
Backend.RTT_STATISTICAL_BATTERIES_SRC_DIR,
})
check_files_exists({
CommonConst.BACKEND_CLEAN_CACHE_SCRIPT,
CommonConst.BACKEND_RUN_JOBS_SCRIPT
})
except AssertionError as e:
print_error("Invalid configuration. {}".format(e))
sys.exit(1)
os.makedirs(Backend.rtt_files_dir, 0o771, True)
# Defined absolute paths to directories and files
Backend.rand_test_tool_src_dir = \
join(Backend.rtt_files_dir, Backend.RANDOMNESS_TESTING_TOOLKIT_SRC_DIR)
Backend.rand_test_tool_dl_zip = \
join(Backend.rtt_files_dir, Backend.RANDOMNESS_TESTING_TOOLKIT_GIT_NAME + ".zip")
Backend.stat_batt_src_dir = \
join(Backend.rtt_files_dir, Backend.RTT_STATISTICAL_BATTERIES_SRC_DIR)
Backend.stat_batt_dl_zip = \
join(Backend.rtt_files_dir, Backend.RTT_STATISTICAL_BATTERIES_GIT_NAME + ".zip")
Backend.common_files_dir = \
join(Backend.rtt_files_dir, Backend.COMMON_FILES_DIR)
Backend.cache_conf_dir = \
join(Backend.rtt_files_dir, Backend.CACHE_CONFIG_DIR)
Backend.cache_data_dir = \
join(Backend.rtt_files_dir, Backend.CACHE_DATA_DIR)
Backend.credentials_dir = \
join(Backend.rtt_files_dir, Backend.CREDENTIALS_DIR)
Backend.rtt_exec_dir = \
join(Backend.rtt_files_dir, Backend.RTT_EXECUTION_DIR)
Backend.rtt_exec_nist_exp_dir = \
join(Backend.rtt_exec_dir, Backend.NIST_STS_EXPERIMENTS_DIR)
Backend.rtt_exec_nist_temp_dir = \
join(Backend.rtt_exec_dir, Backend.NIST_STS_TEMPLATES_DIR)
Backend.ssh_store_pkey = \
join(Backend.credentials_dir, Backend.SSH_CREDENTIALS_KEY)
Backend.ssh_store_pubkey = \
join(Backend.credentials_dir, Backend.SSH_CREDENTIALS_KEY + ".pub")
Backend.run_jobs_path = \
join(Backend.rtt_files_dir, Backend.RUN_JOBS_SCRIPT)
Backend.clean_cache_path = \
join(Backend.rtt_files_dir, Backend.CLEAN_CACHE_SCRIPT)
Backend.rtt_binary_path = \
join(Backend.rtt_exec_dir, os.path.basename(Backend.RTT_BINARY_PATH))
Backend.cryptostreams_binary_dir = Backend.rtt_exec_dir
Backend.mysql_cred_ini_path = \
join(Backend.credentials_dir, Backend.MYSQL_CREDENTIALS_FILE_INI)
Backend.mysql_cred_json_path = \
join(Backend.credentials_dir, Backend.MYSQL_CREDENTIALS_FILE_JSON)
Backend.ssh_cred_ini_path = \
join(Backend.credentials_dir, Backend.SSH_CREDENTIALS_FILE)
Backend.config_ini_path = \
join(Backend.rtt_files_dir, Backend.BACKEND_CONFIG_FILE)
wgrp = Backend.RTT_ADMIN_GROUP
try:
# Install essential packages
install_debian_pkgs([
"acl", "sudo", "wget", "unzip", "rsync", "git", "curl",
"openssh-client"
])
# Adding rtt-admin group that is intended to manage
# directories and files related to rtt without root access
exec_sys_call_check("groupadd {}".format(Backend.RTT_ADMIN_GROUP),
acc_codes=[0, 9])
# Remove directories that was created previously
if os.path.exists(Backend.rtt_files_dir):
shutil.rmtree(Backend.rtt_files_dir)
# Create and copy needed files into rtt-files
create_dir(Backend.rtt_files_dir, 0o2770, grp=wgrp)
# Set ACL on top directory - ensures all new files will have correct permissions
exec_sys_call_check("setfacl -R -d -m g::rwx {}".format(
Backend.rtt_files_dir))
exec_sys_call_check("setfacl -R -d -m o::--- {}".format(
Backend.rtt_files_dir))
create_dir(Backend.cache_conf_dir, 0o2770, grp=wgrp)
create_dir(Backend.cache_data_dir, 0o2770, grp=wgrp)
create_dir(Backend.credentials_dir, 0o2770, grp=wgrp)
create_dir(Backend.rtt_exec_dir, 0o2770, grp=wgrp)
shutil.copy(CommonConst.BACKEND_CLEAN_CACHE_SCRIPT,
Backend.clean_cache_path)
chmod_chown(Backend.clean_cache_path, 0o770, grp=wgrp)
shutil.copy(CommonConst.BACKEND_RUN_JOBS_SCRIPT, Backend.run_jobs_path)
chmod_chown(Backend.run_jobs_path, 0o770, grp=wgrp)
if os.path.exists(Backend.common_files_dir):
shutil.rmtree(Backend.common_files_dir)
shutil.copytree(CommonConst.COMMON_FILES_DIR, Backend.common_files_dir)
recursive_chmod_chown(Backend.common_files_dir,
mod_f=0o660,
mod_d=0o2770,
grp=wgrp)
# Install packages
install_debian_pkg("mailutils")
if not args.no_email:
install_debian_pkg("postfix")
install_debian_pkgs(["libmysqlcppconn-dev"])
if args.deb10:
install_debian_pkgs(["libunbound-dev", "libunistring-dev"])
if args.deb9:
install_debian_pkgs(["libffi-dev"])
install_debian_pkg_at_least_one(
["default-libmysqlclient-dev", "libmysqlclient-dev"])
python3, pip3 = setup_python3(use_system=args.sys_python,
buildj=args.buildj)
install_python_pkg("pip", no_cache=False, pip3=pip3)
install_python_pkgs([
"paramiko",
"cryptography",
"mysqlclient",
"sarge",
"requests",
"shellescape",
"coloredlogs",
"filelock",
"jsonpath-ng",
"sshtunnel",
"randomgen",
"numpy",
"booltest",
"booltest-rtt",
"rtt-data-gen",
],
pip3=pip3)
# Get current versions of needed tools from git
# Statistical batteries
if os.path.exists(Backend.stat_batt_src_dir):
shutil.rmtree(Backend.stat_batt_src_dir)
exec_sys_call_check("wget {} -O {}".format(
Backend.RTT_STATISTICAL_BATTERIES_ZIP_URL,
Backend.stat_batt_dl_zip))
exec_sys_call_check("unzip {} -d {}".format(Backend.stat_batt_dl_zip,
Backend.rtt_files_dir))
os.remove(Backend.stat_batt_dl_zip)
os.rename(
join(Backend.rtt_files_dir,
Backend.RTT_STATISTICAL_BATTERIES_GIT_NAME),
Backend.stat_batt_src_dir)
# Randomness testing toolkit
if os.path.exists(Backend.rand_test_tool_src_dir):
shutil.rmtree(Backend.rand_test_tool_src_dir)
if args.ph4_rtt:
exec_sys_call_check(
"git clone --recursive https://github.com/ph4r05/randomness-testing-toolkit.git %s"
% Backend.rand_test_tool_src_dir)
else:
exec_sys_call_check("wget {} -O {}".format(
Backend.RANDOMNESS_TESTING_TOOLKIT_ZIP_URL,
Backend.rand_test_tool_dl_zip))
exec_sys_call_check("unzip {} -d {}".format(
Backend.rand_test_tool_dl_zip, Backend.rtt_files_dir))
os.remove(Backend.rand_test_tool_dl_zip)
os.rename(
join(Backend.rtt_files_dir,
Backend.RANDOMNESS_TESTING_TOOLKIT_GIT_NAME),
Backend.rand_test_tool_src_dir)
# Change into directory rtt-src and rtt-stat-batt-src and call make and ./INSTALL respectively.
# Build statistical batteries
os.chdir(Backend.stat_batt_src_dir)
build_only_used = args.build_only_rtt or args.build_only_cryptostreams or args.build_only_batteries
build_batteries = not args.build_only_rtt and not args.build_only_cryptostreams
build_rtt = not args.build_only_cryptostreams and not args.build_only_batteries
build_cs = not args.build_only_rtt and not args.build_only_batteries
if build_batteries:
exec_sys_call_check("chmod +x INSTALL")
exec_sys_call_check("./INSTALL",
env=get_make_env(buildj=args.buildj))
recursive_chmod_chown(Backend.stat_batt_src_dir,
mod_f=0o660,
mod_d=0o2770,
grp=wgrp)
chmod_chown(Backend.DIEHARDER_BINARY_PATH, 0o770)
chmod_chown(Backend.NIST_STS_BINARY_PATH, 0o770)
chmod_chown(Backend.TESTU01_BINARY_PATH, 0o770)
build_static_dieharder(Backend.stat_batt_src_dir,
buildj=args.buildj)
os.chdir(Backend.stat_batt_src_dir)
# Build randomness testing toolkit
os.chdir(Backend.rand_test_tool_src_dir)
rtt_env = None
if args.ph4_rtt:
# p11 libs required by gnutls, required by libmaria
p11libs = None
try:
p11libs = build_p11_lib(
buildj=args.buildj) if args.deb10 else None
except Exception as e:
logger.warning("P11 build failed, using dynamic: %s" % (e, ),
exc_info=e)
lib_data = copy_rtt_libs(Backend.rand_test_tool_src_dir)
rtt_env = get_rtt_build_env(Backend.rand_test_tool_src_dir,
lib_data, args.deb10, p11libs)
os.chdir(Backend.rand_test_tool_src_dir)
try:
if build_rtt:
exec_sys_call_check("make -j%s" % (args.buildj, ),
env=rtt_env,
acc_codes=[0, 1, 2])
except Exception as e:
if args.ph4_rtt:
logger.warning(
"Could not build RTT statically, trying dynamic build")
print(
"[ERROR] Could not build RTT statically, trying dynamic build"
)
if build_rtt:
exec_sys_call_check("make -j%s" % (args.buildj, ),
acc_codes=[0, 1, 2])
else:
raise e
recursive_chmod_chown(Backend.rand_test_tool_src_dir,
mod_f=0o660,
mod_d=0o2770,
grp=wgrp)
chmod_chown(Backend.RTT_BINARY_PATH, 0o770)
# Build cryptostreams
cstreams = None
if build_cs:
cstreams = cryptostreams_complete_deploy(
ph4=args.ph4_rtt,
res_bin_dir=Backend.cryptostreams_binary_dir,
src_dir=Backend.rtt_files_dir,
buildj=args.buildj)
# Build finished, go into original directory
os.chdir(current_dir)
if build_only_used:
logger.info("Using --build-only-x, terminating call")
return
# Link RTT binary into execution directory
os.symlink(
join(Backend.rand_test_tool_src_dir, Backend.RTT_BINARY_PATH),
Backend.rtt_binary_path)
# Copy needed directories and files into execution directory
shutil.copytree(
join(Backend.stat_batt_src_dir, Backend.NIST_STS_TEMPLATES_DIR),
join(Backend.rtt_exec_dir,
os.path.basename(Backend.NIST_STS_TEMPLATES_DIR)))
shutil.copytree(
join(Backend.stat_batt_src_dir, Backend.NIST_STS_EXPERIMENTS_DIR),
join(Backend.rtt_exec_dir,
os.path.basename(Backend.NIST_STS_EXPERIMENTS_DIR)))
# Booltest-rtt, rtt-data-gen
try:
booltest_rtt_binary = subprocess.check_output(
['which', 'booltest_rtt']).decode('utf8').strip()
except Exception as e:
booltest_rtt_binary = ""
try:
rtt_data_gen_binary = subprocess.check_output(
['which', 'rtt-data-gen']).decode('utf8').strip()
except Exception as e:
rtt_data_gen_binary = ""
if booltest_rtt_binary and os.path.exists(booltest_rtt_binary):
_spath = os.path.join(Backend.rtt_exec_dir, 'booltest_rtt')
try_remove(_spath)
os.symlink(booltest_rtt_binary, _spath)
if rtt_data_gen_binary and os.path.exists(rtt_data_gen_binary):
_spath = os.path.join(Backend.rtt_exec_dir, 'rtt-data-gen')
try_remove(_spath)
os.symlink(rtt_data_gen_binary, _spath)
rtt_settings = {
"toolkit-settings": {
"logger": {
"dir-prefix":
join(Backend.rtt_files_dir, Backend.EXEC_LOGS_TOP_DIR),
"run-log-dir":
Backend.EXEC_LOGS_RUN_LOG_DIR,
"dieharder-dir":
Backend.EXEC_LOGS_DIEHARDER_DIR,
"nist-sts-dir":
Backend.EXEC_LOGS_NIST_STS_DIR,
"tu01-smallcrush-dir":
Backend.EXEC_LOGS_SMALLCRUSH_DIR,
"tu01-crush-dir":
Backend.EXEC_LOGS_CRUSH_DIR,
"tu01-bigcrush-dir":
Backend.EXEC_LOGS_BIGCRUSH_DIR,
"tu01-rabbit-dir":
Backend.EXEC_LOGS_RABBIT_DIR,
"tu01-alphabit-dir":
Backend.EXEC_LOGS_ALPHABIT_DIR,
"tu01-blockalphabit-dir":
Backend.EXEC_LOGS_BLOCKALPHABIT_DIR
},
"result-storage": {
"file": {
"main-file":
join(Backend.rtt_files_dir,
Backend.EXEC_REPS_MAIN_FILE),
"dir-prefix":
join(Backend.rtt_files_dir, Backend.EXEC_REPS_TOP_DIR),
"dieharder-dir":
Backend.EXEC_REPS_DIEHARDER_DIR,
"nist-sts-dir":
Backend.EXEC_REPS_NIST_STS_DIR,
"tu01-smallcrush-dir":
Backend.EXEC_REPS_SMALLCRUSH_DIR,
"tu01-crush-dir":
Backend.EXEC_REPS_CRUSH_DIR,
"tu01-bigcrush-dir":
Backend.EXEC_REPS_BIGCRUSH_DIR,
"tu01-rabbit-dir":
Backend.EXEC_REPS_RABBIT_DIR,
"tu01-alphabit-dir":
Backend.EXEC_REPS_ALPHABIT_DIR,
"tu01-blockalphabit-dir":
Backend.EXEC_REPS_ALPHABIT_DIR
},
"mysql-db": {
"address": Database.address,
"port": Database.mysql_port,
"name": Database.MYSQL_DB_NAME,
"credentials-file": Backend.mysql_cred_json_path
}
},
"binaries": {
"nist-sts":
join(Backend.stat_batt_src_dir,
Backend.NIST_STS_BINARY_PATH),
"dieharder":
join(Backend.stat_batt_src_dir,
Backend.DIEHARDER_BINARY_PATH),
"testu01":
join(Backend.stat_batt_src_dir,
Backend.TESTU01_BINARY_PATH),
"cryptostreams":
cstreams[0],
},
"miscellaneous": {
"nist-sts": {
"main-result-dir":
join(Backend.rtt_exec_dir,
Backend.NIST_MAIN_RESULT_DIR)
}
},
"execution": {
"max-parallel-tests": int(Backend.exec_max_tests),
"test-timeout-seconds": int(Backend.exec_test_timeout)
},
"booltest": {
"default-cli":
"--no-summary --json-out --log-prints --top 128 --no-comb-and --only-top-comb --only-top-deg --no-term-map --topterm-heap --topterm-heap-k 256 --best-x-combs 512",
"strategies": [{
"name":
"v1",
"cli":
"",
"variations": [{
"bl": [128, 256, 384, 512],
"deg": [1, 2, 3],
"cdeg": [1, 2, 3],
"exclusions": []
}]
}, {
"name":
"halving",
"cli":
"--halving",
"variations": [{
"bl": [128, 256, 384, 512],
"deg": [1, 2, 3],
"cdeg": [1, 2, 3],
"exclusions": []
}]
}]
}
}
}
with open(join(Backend.rtt_exec_dir, Backend.RTT_SETTINGS_JSON),
"w") as f:
json.dump(rtt_settings, f, indent=4)
# Get email configuration
# Add configuration to file
# inet_interface = loopback-only
# inet_protocol = ipv4
if not args.no_email:
with open(Backend.POSTFIX_CFG_PATH) as mail_cfg:
for line in mail_cfg.readlines():
if line.startswith(Backend.POSTFIX_HOST_OPT):
Backend.sender_email = line.split(sep=" = ")[1]
if not args.no_email and Backend.sender_email is None:
print_error("can't find option {} in file {}".format(
Backend.POSTFIX_CFG_PATH, Backend.POSTFIX_HOST_OPT))
sys.exit(1)
Backend.sender_email = (
"root@" + Backend.sender_email
) if not args.no_email else '*****@*****.**'
# Create backend configuration file
backend_ini_cfg = configparser.ConfigParser()
backend_ini_cfg.add_section("MySQL-Database")
backend_ini_cfg.set("MySQL-Database", "Address", Database.address)
backend_ini_cfg.set("MySQL-Database", "Port", Database.mysql_port)
backend_ini_cfg.set("MySQL-Database", "Name", Database.MYSQL_DB_NAME)
backend_ini_cfg.set("MySQL-Database", "Credentials-file",
Backend.mysql_cred_ini_path)
backend_ini_cfg.add_section("Local-cache")
backend_ini_cfg.set("Local-cache", "Data-directory",
Backend.cache_data_dir)
backend_ini_cfg.set("Local-cache", "Config-directory",
Backend.cache_conf_dir)
backend_ini_cfg.add_section("Backend")
backend_ini_cfg.set("Backend", "Sender-email", Backend.sender_email)
backend_ini_cfg.set("Backend", "Maximum-seconds-per-test",
Backend.exec_test_timeout)
backend_ini_cfg.set("Backend", "Maximum-parallel-tests",
Backend.exec_max_tests)
backend_ini_cfg.set("Backend", "backend-id", Backend.backend_id)
backend_ini_cfg.set("Backend", "backend-name", Backend.backend_name)
backend_ini_cfg.set("Backend", "backend-loc", Backend.backend_loc)
backend_ini_cfg.set("Backend", "backend-longterm",
Backend.backend_longterm)
backend_ini_cfg.set("Backend", "backend-aux", Backend.backend_aux)
if Backend.log_dir:
backend_ini_cfg.set("Backend", "log-dir", Backend.log_dir)
backend_ini_cfg.add_section("Storage")
backend_ini_cfg.set("Storage", "Address", Storage.address)
backend_ini_cfg.set("Storage", "Port", Storage.ssh_port)
backend_ini_cfg.set(
"Storage", "Data-directory",
join(Storage.CHROOT_HOME_DIR, Storage.CHROOT_DATA_DIR))
backend_ini_cfg.set(
"Storage", "Config-directory",
join(Storage.CHROOT_HOME_DIR, Storage.CHROOT_CONF_DIR))
backend_ini_cfg.set("Storage", "Credentials-file",
Backend.ssh_cred_ini_path)
backend_ini_cfg.add_section("RTT-Binary")
backend_ini_cfg.set("RTT-Binary", "Binary-path",
Backend.rtt_binary_path)
backend_ini_cfg.set("RTT-Binary", "booltest-rtt-path",
booltest_rtt_binary) # TODO: auto-detect
backend_ini_cfg.set("RTT-Binary", "rtt-data-gen-path",
rtt_data_gen_binary) # TODO: auto-detect
with open(Backend.config_ini_path, "w") as f:
backend_ini_cfg.write(f)
from common.rtt_registration import register_db_user
from common.rtt_registration import add_authorized_key_to_server
from common.rtt_registration import get_db_reg_command
# Register machine to database
db_pwd = args.db_passwd if args.db_passwd else get_rnd_pwd()
write_db_credentials(Backend.MYSQL_BACKEND_USER, db_pwd,
Backend.mysql_cred_ini_path)
write_db_credentials_json(Backend.MYSQL_BACKEND_USER, db_pwd,
Backend.mysql_cred_json_path)
post_install_info = []
db_addr_from = Backend.address if wbare else '%'
if not args.no_db_reg:
db_def_passwd = get_mysql_password_args(args)
register_db_user(Database.ssh_root_user,
Database.address,
Database.ssh_port,
Backend.MYSQL_BACKEND_USER,
db_pwd,
db_addr_from,
Database.MYSQL_ROOT_USERNAME,
Database.MYSQL_DB_NAME,
priv_select=True,
priv_insert=True,
priv_update=True,
priv_create=True,
db_def_passwd=db_def_passwd,
db_no_pass=args.local_db)
else:
sql = get_db_reg_command(username=Database.MYSQL_ROOT_USERNAME,
password=None,
db_name=Database.MYSQL_DB_NAME,
reg_name=Backend.MYSQL_BACKEND_USER,
reg_address=db_addr_from,
reg_pwd=db_pwd,
priv_select=True,
priv_insert=True,
priv_update=True,
priv_create=True,
db_host=Database.address,
db_port=Database.ssh_port)
post_install_info.append(
'* DB user not registered to the DB server. Make sure the following user:password has access: '
)
post_install_info.append(sql + '\n')
# Register machine to storage
key_pwd = args.ssh_passphrase if args.ssh_passphrase else get_rnd_pwd()
if args.ssh_priv:
shutil.copy(args.ssh_priv, Backend.ssh_store_pkey)
shutil.copy(args.ssh_priv + '.pub', Backend.ssh_store_pubkey)
else:
exec_sys_call_check(
"ssh-keygen -q -b 2048 -t rsa -N {} -f {}".format(
key_pwd, Backend.ssh_store_pkey))
chmod_chown(Backend.ssh_store_pkey, 0o600, grp=wgrp)
chmod_chown(Backend.ssh_store_pubkey, 0o660, grp=wgrp)
with open(Backend.ssh_store_pubkey) as f:
pub_key = f.read().rstrip()
write_ssh_credentials(Storage.storage_user, key_pwd,
Backend.ssh_store_pkey,
Backend.ssh_cred_ini_path)
authorized_keys_path = "{}{}".format(
Storage.acc_chroot,
join(Storage.CHROOT_HOME_DIR, Storage.SSH_DIR,
Storage.AUTH_KEYS_FILE))
if args.no_ssh_reg:
post_install_info.append(
'* Register the following key on the storage server at %s' %
(authorized_keys_path, ))
post_install_info.append('%s' % (pub_key, ))
post_install_info.append('')
else:
add_authorized_key_to_server(Storage.ssh_root_user,
Storage.address, Storage.ssh_port,
pub_key, authorized_keys_path)
# Add cron jobs for cache cleaning and job running script
if not args.no_cron:
install_debian_pkg("cron")
add_cron_job(Backend.clean_cache_path,
Backend.config_ini_path,
join(Backend.rtt_files_dir, Backend.CLEAN_CACHE_LOG),
python3=python3)
add_cron_job(Backend.run_jobs_path,
Backend.config_ini_path,
join(Backend.rtt_files_dir, Backend.RUN_JOBS_LOG),
python3=python3)
exec_sys_call_check("service cron restart")
if not args.docker:
service_enable("cron.service")
try:
adj_workers = '/root/adjust_workers.py'
os.chdir(current_dir)
shutil.copy('files/adjust_workers.py', adj_workers)
chmod_chown(adj_workers, 0o770, grp=wgrp)
except Exception as e:
logger.error("Could not install adjust_workers: %s" % (e, ))
if post_install_info:
print('=' * 80)
for x in post_install_info:
print(x)
except BaseException as e:
print_error("{}. Fix error and run the script again.".format(e))
traceback.print_exc()
return 2
def main():
parser = argparse.ArgumentParser(description='Frontend deployment')
parser.add_argument('--docker',
dest='docker',
action='store_const',
const=True,
default=False,
help='Docker deployment')
parser.add_argument('--ph4',
dest='ph4',
action='store_const',
const=True,
default=False,
help='Use Ph4 forks of tools')
parser.add_argument('--no-chroot',
dest='no_chroot',
action='store_const',
const=True,
default=False,
help='No chroot install, install to root')
parser.add_argument('--local-db',
dest='local_db',
action='store_const',
const=True,
default=False,
help='DB server is on the same machine')
parser.add_argument('--no-ssh-server',
dest='no_ssh_server',
action='store_const',
const=True,
default=False,
help='DB server is on the same machine')
parser.add_argument('--mysql-pass',
dest='mysql_pass',
action='store_const',
const=True,
default=False,
help='DB password to use')
parser.add_argument('--mysql-pass-file',
dest='mysql_pass_file',
action='store_const',
const=True,
default=False,
help='DB password file to use')
parser.add_argument('--sys-python',
dest='sys_python',
action='store_const',
const=True,
default=False,
help='Use system python')
parser.add_argument('-j',
dest='buildj',
type=int,
default=2,
help='Parallel compilation')
parser.add_argument('--config',
dest='config',
default='deployment_settings.ini',
help='Path to deployment_settings.ini')
args = parser.parse_args()
deploy_cfg_file = args.config
if args.no_chroot:
args.no_ssh_server = True
deploy_cfg = configparser.ConfigParser()
try:
deploy_cfg.read(deploy_cfg_file)
if len(deploy_cfg.sections()) == 0:
raise FileNotFoundError("can't read: {}".format(deploy_cfg_file))
Frontend.address = get_no_empty(deploy_cfg, "Frontend", "IPv4-Address")
Frontend.rtt_users_chroot = get_no_empty(deploy_cfg, "Frontend",
"RTT-Users-Chroot")
Frontend.ssh_config = get_no_empty(deploy_cfg, "Frontend",
"SSH-Config")
Database.address = get_no_empty(deploy_cfg, "Database", "IPv4-Address")
Database.mysql_port = get_no_empty(deploy_cfg, "Database",
"MySQL-port")
Database.ssh_port = get_no_empty(deploy_cfg, "Database", "SSH-Port")
Database.ssh_root_user = get_no_empty(deploy_cfg, "Database",
"SSH-Root-User")
Storage.address = get_no_empty(deploy_cfg, "Storage", "IPv4-Address")
Storage.ssh_root_user = get_no_empty(deploy_cfg, "Storage",
"SSH-Root-User")
Storage.acc_chroot = get_no_empty(deploy_cfg, "Storage",
"Storage-Chroot")
Storage.storage_user = get_no_empty(deploy_cfg, "Storage",
"Storage-User")
Storage.ssh_port = get_no_empty(deploy_cfg, "Storage", "SSH-port")
except BaseException as e:
print_error("Configuration file: {}".format(e))
sys.exit(1)
# Sanity checks
try:
check_paths_abs({
Frontend.rtt_users_chroot, Frontend.CHROOT_RTT_FILES,
Frontend.CHROOT_RTT_USERS_HOME
})
check_paths_rel({
Frontend.CREDENTIALS_DIR, Frontend.COMMON_FILES_DIR,
Frontend.SSH_DIR
})
check_files_exists({
Frontend.ssh_config, Frontend.FSTAB_FILE,
CommonConst.FRONTEND_SUBMIT_EXPERIMENT_SCRIPT,
CommonConst.FRONTEND_ADD_USER_SCRIPT
})
except AssertionError as e:
print_error("Invalid configuration. {}".format(e))
sys.exit(1)
# Setting of paths used in this script
Frontend.abs_rtt_files = \
Frontend.rtt_users_chroot + Frontend.CHROOT_RTT_FILES
if args.no_chroot:
Frontend.abs_rtt_files = Frontend.CHROOT_RTT_FILES
Frontend.abs_config_ini = \
os.path.join(Frontend.abs_rtt_files, Frontend.FRONT_CONFIG_FILE)
Frontend.abs_submit_exp_script = \
os.path.join(Frontend.abs_rtt_files, Frontend.SUBMIT_EXPERIMENT_SCRIPT)
Frontend.abs_add_user_script = \
os.path.join(Frontend.abs_rtt_files, Frontend.ADD_USER_SCRIPT)
Frontend.submit_exp_base_name = \
os.path.splitext(Frontend.SUBMIT_EXPERIMENT_SCRIPT)[0]
Frontend.abs_cred_dir = \
os.path.join(Frontend.abs_rtt_files, Frontend.CREDENTIALS_DIR)
Frontend.rel_cred_dir = \
os.path.join(Frontend.CHROOT_RTT_FILES, Frontend.CREDENTIALS_DIR)
Frontend.abs_common_files = \
os.path.join(Frontend.abs_rtt_files, Frontend.COMMON_FILES_DIR)
Frontend.abs_cred_mysql_ini = \
os.path.join(Frontend.abs_cred_dir, Frontend.MYSQL_CREDENTIALS_FILE)
Frontend.rel_cred_mysql_ini = \
os.path.join(Frontend.rel_cred_dir, Frontend.MYSQL_CREDENTIALS_FILE)
Frontend.abs_cred_store_ini = \
os.path.join(Frontend.abs_cred_dir, Frontend.SSH_CREDENTIALS_FILE)
Frontend.rel_cred_store_ini = \
os.path.join(Frontend.rel_cred_dir, Frontend.SSH_CREDENTIALS_FILE)
Frontend.abs_cred_store_key = \
os.path.join(Frontend.abs_cred_dir, Frontend.SSH_CREDENTIALS_KEY)
Frontend.rel_cred_store_key = \
os.path.join(Frontend.rel_cred_dir, Frontend.SSH_CREDENTIALS_KEY)
try:
install_debian_pkgs(
["acl", "sudo", "wget", "unzip", "rsync", "openssh-client", "git"])
# Adding rtt-admin group that is intended to manage
# directories and files related to rtt without root access
exec_sys_call_check("groupadd {}".format(Frontend.RTT_ADMIN_GROUP),
acc_codes=[0, 9])
rtt_admin_grp_gid = grp.getgrnam(Frontend.RTT_ADMIN_GROUP).gr_gid
# Adding group for users of rtt
exec_sys_call_check("groupadd {}".format(Frontend.RTT_USER_GROUP),
acc_codes=[0, 9])
rtt_user_grp_gid = grp.getgrnam(Frontend.RTT_USER_GROUP).gr_gid
# Installing debootstrap used for ssh jail
install_debian_pkg("debootstrap")
# Delete chroot directory if it exists
if os.path.exists(Frontend.rtt_users_chroot):
print("Deleting %s" % Frontend.rtt_users_chroot)
try_fnc(lambda: exec_sys_call("umount %s" % os.path.join(
Frontend.rtt_users_chroot, "proc")))
try_fnc(lambda: exec_sys_call("umount %s" % os.path.join(
Frontend.rtt_users_chroot, "sys")))
shutil.rmtree(Frontend.rtt_users_chroot)
if os.path.exists(Frontend.abs_rtt_files):
print("Deleting %s" % Frontend.abs_rtt_files)
shutil.rmtree(Frontend.abs_rtt_files)
# Building chroot jail for rtt users
create_dir(Frontend.rtt_users_chroot,
0o775,
grp=Frontend.RTT_ADMIN_GROUP)
if not args.no_chroot:
exec_sys_call_check("debootstrap {} {}".format(
Frontend.CHROOT_DEBIAN_VERSION, Frontend.rtt_users_chroot))
with open(Frontend.FSTAB_FILE, "a") as f:
f.write("proc {} proc defaults 0 0\n".format(
os.path.join(Frontend.rtt_users_chroot, "proc")))
f.write("sysfs {} sysfs defaults 0 0\n".format(
os.path.join(Frontend.rtt_users_chroot, "sys")))
exec_sys_call_check("mount proc {} -t proc".format(
os.path.join(Frontend.rtt_users_chroot, "proc")))
exec_sys_call_check("mount sysfs {} -t sysfs".format(
os.path.join(Frontend.rtt_users_chroot, "sys")))
shutil.copy("/etc/hosts",
os.path.join(Frontend.rtt_users_chroot, "etc/hosts"))
create_dir(Frontend.abs_rtt_files,
0o2775,
grp=Frontend.RTT_ADMIN_GROUP)
# Set ACL on top directory - ensures all new files will have correct permissions
exec_sys_call_check("setfacl -R -d -m g::rwx {}".format(
Frontend.abs_rtt_files))
exec_sys_call_check("setfacl -R -d -m o::--- {}".format(
Frontend.abs_rtt_files))
create_dir(Frontend.abs_cred_dir, 0o2770, grp=Frontend.RTT_ADMIN_GROUP)
frontend_ini_cfg = configparser.ConfigParser()
frontend_ini_cfg.add_section("MySQL-Database")
frontend_ini_cfg.set("MySQL-Database", "Name", Database.MYSQL_DB_NAME)
frontend_ini_cfg.set("MySQL-Database", "Address", Database.address)
frontend_ini_cfg.set("MySQL-Database", "Port", Database.mysql_port)
frontend_ini_cfg.set("MySQL-Database", "Credentials-file",
Frontend.rel_cred_mysql_ini)
frontend_ini_cfg.add_section("Storage")
frontend_ini_cfg.set("Storage", "Address", Storage.address)
frontend_ini_cfg.set("Storage", "Port", Storage.ssh_port)
frontend_ini_cfg.set(
"Storage", "Data-directory",
os.path.join(Storage.CHROOT_HOME_DIR, Storage.CHROOT_DATA_DIR))
frontend_ini_cfg.set(
"Storage", "Config-directory",
os.path.join(Storage.CHROOT_HOME_DIR, Storage.CHROOT_CONF_DIR))
frontend_ini_cfg.set("Storage", "Credentials-file",
Frontend.rel_cred_store_ini)
frontend_ini_cfg.add_section("Frontend")
frontend_ini_cfg.set("Frontend", "RTT-Users-Chroot",
Frontend.rtt_users_chroot)
with open(Frontend.abs_config_ini, "w") as f:
frontend_ini_cfg.write(f)
shutil.copy(CommonConst.FRONTEND_SUBMIT_EXPERIMENT_SCRIPT,
Frontend.abs_submit_exp_script)
chmod_chown(Frontend.abs_submit_exp_script,
0o660,
grp=Frontend.RTT_ADMIN_GROUP)
shutil.copy(CommonConst.FRONTEND_ADD_USER_SCRIPT,
Frontend.abs_add_user_script)
chmod_chown(Frontend.abs_add_user_script,
0o770,
grp=Frontend.RTT_ADMIN_GROUP)
if os.path.exists(Frontend.abs_common_files):
shutil.rmtree(Frontend.abs_common_files)
shutil.copytree(CommonConst.COMMON_FILES_DIR,
Frontend.abs_common_files)
recursive_chmod_chown(Frontend.abs_common_files,
mod_f=0o660,
mod_d=0o2770,
grp=Frontend.RTT_ADMIN_GROUP)
# Entering chroot jail
real_root = os.open("/", os.O_RDONLY)
if not args.no_chroot:
os.chroot(Frontend.rtt_users_chroot)
# Adding groups - instead there should be two-way sync!!!
exec_sys_call_check("groupadd -g {} {}".format(
rtt_admin_grp_gid, Frontend.RTT_ADMIN_GROUP),
acc_codes=[0, 9])
exec_sys_call_check("groupadd -g {} {}".format(
rtt_user_grp_gid, Frontend.RTT_USER_GROUP),
acc_codes=[0, 9])
# Installing needed packages inside jail
install_debian_pkg_at_least_one(
["default-libmysqlclient-dev", "libmysqlclient-dev"])
install_debian_pkgs(["build-essential"])
python3, pip3 = setup_python3(use_system=args.sys_python,
buildj=args.buildj)
if args.sys_python:
install_debian_pkgs([
"python3-dev", "python3-setuptools", "python3-cryptography",
"python3-paramiko", "python3-pip"
])
install_python_pkg("pip", no_cache=False, pip3=pip3)
install_python_pkgs([
"setuptools",
"paramiko",
"cryptography",
"pyinstaller",
"filelock",
"jsonpath-ng",
"booltest",
"booltest-rtt",
],
pip3=pip3)
os.chdir(Frontend.CHROOT_RTT_FILES)
submit_bin = submit_experiment_deploy(Frontend.CHROOT_RTT_FILES)
submit_bin_dst = os.path.join('/usr/bin',
Frontend.SUBMIT_EXPERIMENT_BINARY)
try_fnc(lambda: os.unlink(submit_bin_dst))
os.symlink(submit_bin, submit_bin_dst)
# CryptoStreams
cryptostreams_complete_deploy(ph4=args.ph4, buildj=args.buildj)
# Exiting chroot jail
if not args.no_chroot:
os.fchdir(real_root)
os.chroot(".")
os.close(real_root)
sshd_config_append = "\n\n\n\n" \
"Match Group {0}\n" \
"\tChrootDirectory {1}\n" \
"\tPasswordAuthentication yes\n" \
"\tAllowTcpForwarding yes\n" \
"\tPermitTunnel yes\n" \
"\tX11Forwarding no\n" \
"\tAuthorizedKeysFile {1}{2}\n" \
"\n".format(Frontend.RTT_USER_GROUP, Frontend.rtt_users_chroot,
os.path.join(Frontend.CHROOT_RTT_USERS_HOME, "%u",
Frontend.SSH_DIR, Frontend.AUTH_KEYS_FILE))
if not args.no_ssh_server:
install_debian_pkgs(["openssh-server"])
with open(Frontend.ssh_config, "a") as f:
f.write(sshd_config_append)
exec_sys_call_check("service ssh restart")
install_debian_pkgs(
["python3-pip", "python3-cryptography", "python3-paramiko"])
from common.rtt_registration import register_db_user
from common.rtt_registration import add_authorized_key_to_server
# Register frontend user at the database
cred_mysql_db_password = get_rnd_pwd()
write_db_credentials(Frontend.MYSQL_FRONTEND_USER,
cred_mysql_db_password,
Frontend.abs_cred_mysql_ini)
db_def_passwd = get_mysql_password_args(args)
register_db_user(Database.ssh_root_user,
Database.address,
Database.ssh_port,
Frontend.MYSQL_FRONTEND_USER,
cred_mysql_db_password,
Frontend.address,
Database.MYSQL_ROOT_USERNAME,
Database.MYSQL_DB_NAME,
priv_insert=True,
priv_select=True,
db_def_passwd=db_def_passwd,
db_no_pass=args.local_db)
# Register frontend at the storage
cred_store_ssh_key_password = get_rnd_pwd()
write_ssh_credentials(Storage.storage_user,
cred_store_ssh_key_password,
Frontend.rel_cred_store_key,
Frontend.abs_cred_store_ini)
exec_sys_call_check("ssh-keygen -q -b 2048 -t rsa -N {} -f {}".format(
cred_store_ssh_key_password, Frontend.abs_cred_store_key))
chmod_chown(Frontend.abs_cred_store_key,
0o660,
grp=Frontend.RTT_ADMIN_GROUP)
chmod_chown(Frontend.abs_cred_store_key + ".pub",
0o660,
grp=Frontend.RTT_ADMIN_GROUP)
with open("{}.pub".format(Frontend.abs_cred_store_key),
"r") as pub_key_f:
pub_key = pub_key_f.read().rstrip()
add_authorized_key_to_server(
Storage.ssh_root_user, Storage.address, Storage.ssh_port, pub_key,
"{}{}".format(
Storage.acc_chroot,
os.path.join(Storage.CHROOT_HOME_DIR, Storage.SSH_DIR,
Storage.AUTH_KEYS_FILE)))
if not args.docker:
service_enable("ssh.service")
# Everything should be okay now.
except BaseException as e:
print_error("{}. Fix error and run the script again.".format(e))
traceback.print_exc()
return 2
def main():
if len(sys.argv) != 2:
print("\nUsage: ./deploy_backend.py <backend-id>\n")
print(
"<backend-id> must be entered according to config with deployment settings.\n"
" Configuration file \"{}\" must\n"
" contain one and only one section named\n"
" \"Backend-<backend-id>\"\n".format(deploy_cfg_file))
sys.exit(1)
deploy_cfg = configparser.ConfigParser()
try:
deploy_cfg.read(deploy_cfg_file)
if len(deploy_cfg.sections()) == 0:
raise FileNotFoundError("can't read: {}".format(deploy_cfg_file))
backend_sec = "Backend-" + sys.argv[1]
Backend.address = get_no_empty(deploy_cfg, backend_sec, "IPv4-Address")
Backend.rtt_files_dir = get_no_empty(deploy_cfg, backend_sec,
"RTT-Files-dir")
Backend.exec_max_tests = get_no_empty(deploy_cfg, backend_sec,
"Maximum-parallel-tests")
Backend.exec_test_timeout = get_no_empty(deploy_cfg, backend_sec,
"Maximum-seconds-per-test")
Database.address = get_no_empty(deploy_cfg, "Database", "IPv4-Address")
Database.mysql_port = get_no_empty(deploy_cfg, "Database",
"MySQL-port")
Database.ssh_port = get_no_empty(deploy_cfg, "Database", "SSH-Port")
Database.ssh_root_user = get_no_empty(deploy_cfg, "Database",
"SSH-Root-User")
Storage.address = get_no_empty(deploy_cfg, "Storage", "IPv4-Address")
Storage.ssh_root_user = get_no_empty(deploy_cfg, "Storage",
"SSH-Root-User")
Storage.acc_chroot = get_no_empty(deploy_cfg, "Storage",
"Storage-Chroot")
Storage.storage_user = get_no_empty(deploy_cfg, "Storage",
"Storage-User")
Storage.ssh_port = get_no_empty(deploy_cfg, "Storage", "SSH-port")
except Exception as e:
print_error("Configuration file: {}".format(e))
sys.exit(1)
# Sanity checks
try:
check_paths_abs({Backend.rtt_files_dir})
check_paths_rel({
Backend.COMMON_FILES_DIR,
Backend.CACHE_DATA_DIR,
Backend.CACHE_CONFIG_DIR,
Backend.CREDENTIALS_DIR,
Backend.RTT_EXECUTION_DIR,
Backend.RANDOMNESS_TESTING_TOOLKIT_SRC_DIR,
Backend.RTT_STATISTICAL_BATTERIES_SRC_DIR,
})
check_files_exists({
CommonConst.BACKEND_CLEAN_CACHE_SCRIPT,
CommonConst.BACKEND_RUN_JOBS_SCRIPT
})
except AssertionError as e:
print_error("Invalid configuration. {}".format(e))
sys.exit(1)
# Defined absolute paths to directories and files
Backend.rand_test_tool_src_dir = \
join(Backend.rtt_files_dir, Backend.RANDOMNESS_TESTING_TOOLKIT_SRC_DIR)
Backend.rand_test_tool_dl_zip = \
join(Backend.rtt_files_dir, Backend.RANDOMNESS_TESTING_TOOLKIT_GIT_NAME + ".zip")
Backend.stat_batt_src_dir = \
join(Backend.rtt_files_dir, Backend.RTT_STATISTICAL_BATTERIES_SRC_DIR)
Backend.stat_batt_dl_zip = \
join(Backend.rtt_files_dir, Backend.RTT_STATISTICAL_BATTERIES_GIT_NAME + ".zip")
Backend.common_files_dir = \
join(Backend.rtt_files_dir, Backend.COMMON_FILES_DIR)
Backend.cache_conf_dir = \
join(Backend.rtt_files_dir, Backend.CACHE_CONFIG_DIR)
Backend.cache_data_dir = \
join(Backend.rtt_files_dir, Backend.CACHE_DATA_DIR)
Backend.credentials_dir = \
join(Backend.rtt_files_dir, Backend.CREDENTIALS_DIR)
Backend.rtt_exec_dir = \
join(Backend.rtt_files_dir, Backend.RTT_EXECUTION_DIR)
Backend.rtt_exec_nist_exp_dir = \
join(Backend.rtt_exec_dir, Backend.NIST_STS_EXPERIMENTS_DIR)
Backend.rtt_exec_nist_temp_dir = \
join(Backend.rtt_exec_dir, Backend.NIST_STS_TEMPLATES_DIR)
Backend.ssh_store_pkey = \
join(Backend.credentials_dir, Backend.SSH_CREDENTIALS_KEY)
Backend.ssh_store_pubkey = \
join(Backend.credentials_dir, Backend.SSH_CREDENTIALS_KEY + ".pub")
Backend.run_jobs_path = \
join(Backend.rtt_files_dir, Backend.RUN_JOBS_SCRIPT)
Backend.clean_cache_path = \
join(Backend.rtt_files_dir, Backend.CLEAN_CACHE_SCRIPT)
Backend.rtt_binary_path = \
join(Backend.rtt_exec_dir, os.path.basename(Backend.RTT_BINARY_PATH))
Backend.mysql_cred_ini_path = \
join(Backend.credentials_dir, Backend.MYSQL_CREDENTIALS_FILE_INI)
Backend.mysql_cred_json_path = \
join(Backend.credentials_dir, Backend.MYSQL_CREDENTIALS_FILE_JSON)
Backend.ssh_cred_ini_path = \
join(Backend.credentials_dir, Backend.SSH_CREDENTIALS_FILE)
Backend.config_ini_path = \
join(Backend.rtt_files_dir, Backend.BACKEND_CONFIG_FILE)
try:
# Adding rtt-admin group that is intended to manage
# directories and files related to rtt without root access
exec_sys_call_check("groupadd {}".format(Backend.RTT_ADMIN_GROUP),
acc_codes=[0, 9])
# Remove directories that was created previously
if os.path.exists(Backend.rtt_files_dir):
shutil.rmtree(Backend.rtt_files_dir)
# Create and copy needed files into rtt-files
create_dir(Backend.rtt_files_dir, 0o2770, grp=Backend.RTT_ADMIN_GROUP)
# Set ACL on top directory - ensures all new files will have correct permissions
exec_sys_call_check("setfacl -R -d -m g::rwx {}".format(
Backend.rtt_files_dir))
exec_sys_call_check("setfacl -R -d -m o::--- {}".format(
Backend.rtt_files_dir))
create_dir(Backend.cache_conf_dir, 0o2770, grp=Backend.RTT_ADMIN_GROUP)
create_dir(Backend.cache_data_dir, 0o2770, grp=Backend.RTT_ADMIN_GROUP)
create_dir(Backend.credentials_dir,
0o2770,
grp=Backend.RTT_ADMIN_GROUP)
create_dir(Backend.rtt_exec_dir, 0o2770, grp=Backend.RTT_ADMIN_GROUP)
shutil.copy(CommonConst.BACKEND_CLEAN_CACHE_SCRIPT,
Backend.clean_cache_path)
chmod_chown(Backend.clean_cache_path,
0o770,
grp=Backend.RTT_ADMIN_GROUP)
shutil.copy(CommonConst.BACKEND_RUN_JOBS_SCRIPT, Backend.run_jobs_path)
chmod_chown(Backend.run_jobs_path, 0o770, grp=Backend.RTT_ADMIN_GROUP)
if os.path.exists(Backend.common_files_dir):
shutil.rmtree(Backend.common_files_dir)
shutil.copytree(CommonConst.COMMON_FILES_DIR, Backend.common_files_dir)
recursive_chmod_chown(Backend.common_files_dir,
mod_f=0o660,
mod_d=0o2770,
grp=Backend.RTT_ADMIN_GROUP)
# Install packages
install_debian_pkg("mailutils")
install_debian_pkg("postfix")
install_debian_pkg("libmysqlcppconn-dev")
install_debian_pkg("libmysqlclient-dev")
install_debian_pkg("python3-pip")
install_debian_pkg("python3-cryptography")
install_debian_pkg("python3-paramiko")
install_python_pkg("mysqlclient")
# Get current versions of needed tools from git
# Statistical batteries
if os.path.exists(Backend.stat_batt_src_dir):
shutil.rmtree(Backend.stat_batt_src_dir)
exec_sys_call_check("wget {} -O {}".format(
Backend.RTT_STATISTICAL_BATTERIES_ZIP_URL,
Backend.stat_batt_dl_zip))
exec_sys_call_check("unzip {} -d {}".format(Backend.stat_batt_dl_zip,
Backend.rtt_files_dir))
os.remove(Backend.stat_batt_dl_zip)
os.rename(
join(Backend.rtt_files_dir,
Backend.RTT_STATISTICAL_BATTERIES_GIT_NAME),
Backend.stat_batt_src_dir)
# Randomness testing toolkit
if os.path.exists(Backend.rand_test_tool_src_dir):
shutil.rmtree(Backend.rand_test_tool_src_dir)
exec_sys_call_check("wget {} -O {}".format(
Backend.RANDOMNESS_TESTING_TOOLKIT_ZIP_URL,
Backend.rand_test_tool_dl_zip))
exec_sys_call_check("unzip {} -d {}".format(
Backend.rand_test_tool_dl_zip, Backend.rtt_files_dir))
os.remove(Backend.rand_test_tool_dl_zip)
os.rename(
join(Backend.rtt_files_dir,
Backend.RANDOMNESS_TESTING_TOOLKIT_GIT_NAME),
Backend.rand_test_tool_src_dir)
# Change into directory rtt-src and rtt-stat-batt-src and call make and ./INSTALL respectively.
current_dir = os.path.abspath(os.path.curdir)
# Build statistical batteries
os.chdir(Backend.stat_batt_src_dir)
exec_sys_call_check("./INSTALL")
recursive_chmod_chown(Backend.stat_batt_src_dir,
mod_f=0o660,
mod_d=0o2770,
grp=Backend.RTT_ADMIN_GROUP)
chmod_chown(Backend.DIEHARDER_BINARY_PATH, 0o770)
chmod_chown(Backend.NIST_STS_BINARY_PATH, 0o770)
chmod_chown(Backend.TESTU01_BINARY_PATH, 0o770)
# Build randomness testing toolkit
os.chdir(Backend.rand_test_tool_src_dir)
exec_sys_call_check("make")
recursive_chmod_chown(Backend.rand_test_tool_src_dir,
mod_f=0o660,
mod_d=0o2770,
grp=Backend.RTT_ADMIN_GROUP)
chmod_chown(Backend.RTT_BINARY_PATH, 0o770)
# Build finished, go into original directory
os.chdir(current_dir)
# Link rtt binary into execution directory
os.symlink(
join(Backend.rand_test_tool_src_dir, Backend.RTT_BINARY_PATH),
Backend.rtt_binary_path)
# Copy needed directories and files into execution directory
shutil.copytree(
join(Backend.stat_batt_src_dir, Backend.NIST_STS_TEMPLATES_DIR),
join(Backend.rtt_exec_dir,
os.path.basename(Backend.NIST_STS_TEMPLATES_DIR)))
shutil.copytree(
join(Backend.stat_batt_src_dir, Backend.NIST_STS_EXPERIMENTS_DIR),
join(Backend.rtt_exec_dir,
os.path.basename(Backend.NIST_STS_EXPERIMENTS_DIR)))
rtt_settings = {
"toolkit-settings": {
"logger": {
"dir-prefix":
join(Backend.rtt_files_dir, Backend.EXEC_LOGS_TOP_DIR),
"run-log-dir":
Backend.EXEC_LOGS_RUN_LOG_DIR,
"dieharder-dir":
Backend.EXEC_LOGS_DIEHARDER_DIR,
"nist-sts-dir":
Backend.EXEC_LOGS_NIST_STS_DIR,
"tu01-smallcrush-dir":
Backend.EXEC_LOGS_SMALLCRUSH_DIR,
"tu01-crush-dir":
Backend.EXEC_LOGS_CRUSH_DIR,
"tu01-bigcrush-dir":
Backend.EXEC_LOGS_BIGCRUSH_DIR,
"tu01-rabbit-dir":
Backend.EXEC_LOGS_RABBIT_DIR,
"tu01-alphabit-dir":
Backend.EXEC_LOGS_ALPHABIT_DIR,
"tu01-blockalphabit-dir":
Backend.EXEC_LOGS_BLOCKALPHABIT_DIR
},
"result-storage": {
"file": {
"main-file":
join(Backend.rtt_files_dir,
Backend.EXEC_REPS_MAIN_FILE),
"dir-prefix":
join(Backend.rtt_files_dir, Backend.EXEC_REPS_TOP_DIR),
"dieharder-dir":
Backend.EXEC_REPS_DIEHARDER_DIR,
"nist-sts-dir":
Backend.EXEC_REPS_NIST_STS_DIR,
"tu01-smallcrush-dir":
Backend.EXEC_REPS_SMALLCRUSH_DIR,
"tu01-crush-dir":
Backend.EXEC_REPS_CRUSH_DIR,
"tu01-bigcrush-dir":
Backend.EXEC_REPS_BIGCRUSH_DIR,
"tu01-rabbit-dir":
Backend.EXEC_REPS_RABBIT_DIR,
"tu01-alphabit-dir":
Backend.EXEC_REPS_ALPHABIT_DIR,
"tu01-blockalphabit-dir":
Backend.EXEC_REPS_ALPHABIT_DIR
},
"mysql-db": {
"address": Database.address,
"port": Database.mysql_port,
"name": Database.MYSQL_DB_NAME,
"credentials-file": Backend.mysql_cred_json_path
}
},
"binaries": {
"nist-sts":
join(Backend.stat_batt_src_dir,
Backend.NIST_STS_BINARY_PATH),
"dieharder":
join(Backend.stat_batt_src_dir,
Backend.DIEHARDER_BINARY_PATH),
"testu01":
join(Backend.stat_batt_src_dir,
Backend.TESTU01_BINARY_PATH)
},
"miscellaneous": {
"nist-sts": {
"main-result-dir":
join(Backend.rtt_exec_dir,
Backend.NIST_MAIN_RESULT_DIR)
}
},
"execution": {
"max-parallel-tests": int(Backend.exec_max_tests),
"test-timeout-seconds": int(Backend.exec_test_timeout)
}
}
}
with open(join(Backend.rtt_exec_dir, Backend.RTT_SETTINGS_JSON),
"w") as f:
json.dump(rtt_settings, f, indent=4)
# Get email configuration
# Add configuration to file
# inet_interface = loopback-only
# inet_protocol = ipv4
with open(Backend.POSTFIX_CFG_PATH) as mail_cfg:
for line in mail_cfg.readlines():
if line.startswith(Backend.POSTFIX_HOST_OPT):
Backend.sender_email = line.split(sep=" = ")[1]
if Backend.sender_email is None:
print_error("can't find option {} in file {}".format(
Backend.POSTFIX_CFG_PATH, Backend.POSTFIX_HOST_OPT))
sys.exit(1)
Backend.sender_email = "root@" + Backend.sender_email
# Create backend configuration file
backend_ini_cfg = configparser.ConfigParser()
backend_ini_cfg.add_section("MySQL-Database")
backend_ini_cfg.set("MySQL-Database", "Address", Database.address)
backend_ini_cfg.set("MySQL-Database", "Port", Database.mysql_port)
backend_ini_cfg.set("MySQL-Database", "Name", Database.MYSQL_DB_NAME)
backend_ini_cfg.set("MySQL-Database", "Credentials-file",
Backend.mysql_cred_ini_path)
backend_ini_cfg.add_section("Local-cache")
backend_ini_cfg.set("Local-cache", "Data-directory",
Backend.cache_data_dir)
backend_ini_cfg.set("Local-cache", "Config-directory",
Backend.cache_conf_dir)
backend_ini_cfg.add_section("Backend")
backend_ini_cfg.set("Backend", "Sender-email", Backend.sender_email)
backend_ini_cfg.add_section("Storage")
backend_ini_cfg.set("Storage", "Address", Storage.address)
backend_ini_cfg.set("Storage", "Port", Storage.ssh_port)
backend_ini_cfg.set(
"Storage", "Data-directory",
join(Storage.CHROOT_HOME_DIR, Storage.CHROOT_DATA_DIR))
backend_ini_cfg.set(
"Storage", "Config-directory",
join(Storage.CHROOT_HOME_DIR, Storage.CHROOT_CONF_DIR))
backend_ini_cfg.set("Storage", "Credentials-file",
Backend.ssh_cred_ini_path)
backend_ini_cfg.add_section("RTT-Binary")
backend_ini_cfg.set("RTT-Binary", "Binary-path",
Backend.rtt_binary_path)
with open(Backend.config_ini_path, "w") as f:
backend_ini_cfg.write(f)
from common.rtt_registration import register_db_user
from common.rtt_registration import add_authorized_key_to_server
# Register machine to database
db_pwd = get_rnd_pwd()
cred_mysql_db_ini = configparser.ConfigParser()
cred_mysql_db_ini.add_section("Credentials")
cred_mysql_db_ini.set("Credentials", "Username",
Backend.MYSQL_BACKEND_USER)
cred_mysql_db_ini.set("Credentials", "Password", db_pwd)
with open(Backend.mysql_cred_ini_path, "w") as f:
cred_mysql_db_ini.write(f)
cred_mysql_db_json = {
"credentials": {
"username": Backend.MYSQL_BACKEND_USER,
"password": db_pwd
}
}
with open(Backend.mysql_cred_json_path, "w") as f:
json.dump(cred_mysql_db_json, f, indent=4)
register_db_user(Database.ssh_root_user,
Database.address,
Database.ssh_port,
Backend.MYSQL_BACKEND_USER,
db_pwd,
Backend.address,
Database.MYSQL_ROOT_USERNAME,
Database.MYSQL_DB_NAME,
priv_select=True,
priv_insert=True,
priv_update=True)
# Register machine to storage
key_pwd = get_rnd_pwd()
exec_sys_call_check("ssh-keygen -q -b 2048 -t rsa -N {} -f {}".format(
key_pwd, Backend.ssh_store_pkey))
chmod_chown(Backend.ssh_store_pkey, 0o660, grp=Backend.RTT_ADMIN_GROUP)
chmod_chown(Backend.ssh_store_pubkey,
0o660,
grp=Backend.RTT_ADMIN_GROUP)
with open(Backend.ssh_store_pubkey) as f:
pub_key = f.read().rstrip()
cred_ssh_store_ini = configparser.ConfigParser()
cred_ssh_store_ini.add_section("Credentials")
cred_ssh_store_ini.set("Credentials", "Username", Storage.storage_user)
cred_ssh_store_ini.set("Credentials", "Private-key-file",
Backend.ssh_store_pkey)
cred_ssh_store_ini.set("Credentials", "Private-key-password", key_pwd)
with open(Backend.ssh_cred_ini_path, "w") as f:
cred_ssh_store_ini.write(f)
add_authorized_key_to_server(
Storage.ssh_root_user, Storage.address, Storage.ssh_port, pub_key,
"{}{}".format(
Storage.acc_chroot,
join(Storage.CHROOT_HOME_DIR, Storage.SSH_DIR,
Storage.AUTH_KEYS_FILE)))
# Add cron jobs for cache cleaning and job running script
add_cron_job(Backend.clean_cache_path, Backend.config_ini_path,
join(Backend.rtt_files_dir, Backend.CLEAN_CACHE_LOG))
add_cron_job(Backend.run_jobs_path, Backend.config_ini_path,
join(Backend.rtt_files_dir, Backend.RUN_JOBS_LOG))
except BaseException as e:
print_error("{}. Fix error and run the script again.".format(e))
def main():
parser = argparse.ArgumentParser(description='Storage deployment')
parser.add_argument('--docker',
dest='docker',
action='store_const',
const=True,
default=False,
help='Docker deployment')
parser.add_argument('--local-db',
dest='local_db',
action='store_const',
const=True,
default=False,
help='DB server is on the same machine')
parser.add_argument('--mysql-pass',
dest='mysql_pass',
action='store_const',
const=True,
default=False,
help='DB password to use')
parser.add_argument('--mysql-pass-file',
dest='mysql_pass_file',
action='store_const',
const=True,
default=False,
help='DB password file to use')
parser.add_argument('--sys-python',
dest='sys_python',
action='store_const',
const=True,
default=False,
help='Use system python')
parser.add_argument('-j',
dest='buildj',
type=int,
default=2,
help='Parallel compilation')
parser.add_argument('--config',
dest='config',
default='deployment_settings.ini',
help='Path to deployment_settings.ini')
args = parser.parse_args()
deploy_cfg_file = args.config
deploy_cfg = configparser.ConfigParser()
try:
deploy_cfg.read(deploy_cfg_file)
if len(deploy_cfg.sections()) == 0:
raise FileNotFoundError("can't read: {}".format(deploy_cfg_file))
Database.address = get_no_empty(deploy_cfg, "Database", "IPv4-Address")
Database.ssh_root_user = get_no_empty(deploy_cfg, "Database",
"SSH-Root-User")
Database.ssh_port = get_no_empty(deploy_cfg, "Database", "SSH-Port")
Database.mysql_port = get_no_empty(deploy_cfg, "Database",
"MySql-Port")
Storage.address = get_no_empty(deploy_cfg, "Storage", "IPv4-Address")
Storage.rtt_dir = get_no_empty(deploy_cfg, "Storage", "RTT-Files-dir")
Storage.ssh_config = get_no_empty(deploy_cfg, "Storage", "SSH-Config")
Storage.acc_name = get_no_empty(deploy_cfg, "Storage", "Storage-User")
Storage.acc_chroot = get_no_empty(deploy_cfg, "Storage",
"Storage-Chroot")
except BaseException as e:
print_error("Configuration file: {}".format(e))
sys.exit(1)
# Sanity checks
try:
check_paths_abs(
{Storage.acc_chroot, Storage.CHROOT_HOME_DIR, Storage.rtt_dir})
check_paths_rel({
Storage.CHROOT_CONF_DIR, Storage.CHROOT_DATA_DIR, Storage.SSH_DIR,
Storage.COMMON_FILES_DIR, Storage.CREDENTIALS_DIR
})
check_files_exists({CommonConst.STORAGE_CLEAN_CACHE})
except AssertionError as e:
print_error("Invalid configuration. {}".format(e))
sys.exit(1)
# Declaring all variables that
# will be used in this script.
# These are mostly absolute paths put
# together from the settings.
# Path related to storage user home
Storage.home_dir = \
"{}{}".format(Storage.acc_chroot, Storage.CHROOT_HOME_DIR)
Storage.ssh_dir = \
os.path.join(Storage.home_dir, Storage.SSH_DIR)
Storage.authorized_keys_file = \
os.path.join(Storage.ssh_dir, Storage.AUTH_KEYS_FILE)
Storage.data_dir = \
os.path.join(Storage.home_dir, Storage.CHROOT_DATA_DIR)
Storage.config_dir = \
os.path.join(Storage.home_dir, Storage.CHROOT_CONF_DIR)
# Paths related to rtt files on server
Storage.rtt_file_store_ini = \
os.path.join(Storage.rtt_dir, Storage.STORE_CONFIG_FILE)
Storage.rtt_file_clean_cache = \
os.path.join(Storage.rtt_dir, Storage.CLEAN_CACHE_SCRIPT)
Storage.rtt_file_clean_cache_log = \
os.path.join(Storage.rtt_dir, Storage.CLEAN_CACHE_LOG)
Storage.rtt_common_dir = \
os.path.join(Storage.rtt_dir, Storage.COMMON_FILES_DIR)
Storage.rtt_credentials_dir \
= os.path.join(Storage.rtt_dir, Storage.CREDENTIALS_DIR)
Storage.rtt_file_mysql_cred = \
os.path.join(Storage.rtt_credentials_dir, Storage.MYSQL_CREDENTIALS_FILE)
try:
install_debian_pkgs([
"acl", "sudo", "wget", "unzip", "rsync", "cron", "openssh-client"
])
install_debian_pkg("openssh-server")
# Creating sftp jail for account
# Adding rtt-admin group that is intended to manage
# directories and files related to rtt without root access
exec_sys_call_check("groupadd {}".format(Storage.RTT_ADMIN_GROUP),
acc_codes=[0, 9])
# Adding user for access
exec_sys_call_check("useradd -d {} -s /usr/sbin/nologin {}".format(
Storage.CHROOT_HOME_DIR, Storage.acc_name),
acc_codes=[0, 9])
# Configuring ssh server
sshd_config_append = "\n\n" \
"\tClientAliveInterval 120\n\n" \
"Match User {0}\n" \
"\tChrootDirectory {1}\n" \
"\tForceCommand internal-sftp\n" \
"\tAllowTcpForwarding yes\n" \
"\tPermitTunnel yes\n" \
"\tX11Forwarding no\n" \
"\tAuthorizedKeysFile {1}{2}\n" \
"\tPasswordAuthentication no\n" \
"\n".format(Storage.acc_name, Storage.acc_chroot,
os.path.join(Storage.CHROOT_HOME_DIR,
Storage.SSH_DIR, Storage.AUTH_KEYS_FILE))
with open(Storage.ssh_config, "a") as f:
f.write(sshd_config_append)
exec_sys_call_check("service ssh restart")
# Creating sftp jail for accessing storage
create_dir(Storage.acc_chroot, 0o755)
create_dir(Storage.home_dir,
0o700,
own=Storage.acc_name,
grp=Storage.acc_name)
create_dir(Storage.ssh_dir,
0o700,
own=Storage.acc_name,
grp=Storage.acc_name)
create_dir(Storage.data_dir,
0o700,
own=Storage.acc_name,
grp=Storage.acc_name)
create_dir(Storage.config_dir,
0o700,
own=Storage.acc_name,
grp=Storage.acc_name)
create_file(Storage.authorized_keys_file,
0o600,
own=Storage.acc_name,
grp=Storage.acc_name)
# Creating directory for rtt files on the server
create_dir(Storage.rtt_dir, 0o2770, grp=Storage.RTT_ADMIN_GROUP)
# Set ACL on top directory - ensures all new files will have correct permissions
exec_sys_call_check("setfacl -R -d -m g::rwx {}".format(
Storage.rtt_dir))
exec_sys_call_check("setfacl -R -d -m o::--- {}".format(
Storage.rtt_dir))
create_dir(Storage.rtt_credentials_dir,
0o2770,
grp=Storage.RTT_ADMIN_GROUP)
# Copying script for cache cleaning
shutil.copy(CommonConst.STORAGE_CLEAN_CACHE,
Storage.rtt_file_clean_cache)
chmod_chown(Storage.rtt_file_clean_cache,
0o770,
grp=Storage.RTT_ADMIN_GROUP)
# Copying common scripts into directory
# with rtt files
if os.path.exists(Storage.rtt_common_dir):
shutil.rmtree(Storage.rtt_common_dir)
shutil.copytree(CommonConst.COMMON_FILES_DIR, Storage.rtt_common_dir)
recursive_chmod_chown(Storage.rtt_common_dir,
mod_f=0o660,
mod_d=0o2770,
grp=Storage.RTT_ADMIN_GROUP)
# Creating configuration file for storage server scripts
ini_cfg = configparser.ConfigParser()
ini_cfg.add_section("MySQL-Database")
ini_cfg.set("MySQL-Database", "Name", Database.MYSQL_DB_NAME)
ini_cfg.set("MySQL-Database", "Address", Database.address)
ini_cfg.set("MySQL-Database", "Port", Database.mysql_port)
ini_cfg.set("MySQL-Database", "Credentials-file",
Storage.rtt_file_mysql_cred)
ini_cfg.add_section("Local-cache")
ini_cfg.set("Local-cache", "Data-directory", Storage.data_dir)
ini_cfg.set("Local-cache", "Config-directory", Storage.config_dir)
with open(Storage.rtt_file_store_ini, "w") as f:
ini_cfg.write(f)
# Creating credentials file for database access
cred_db_password = get_rnd_pwd()
write_db_credentials(Storage.MYSQL_STORAGE_USER, cred_db_password,
Storage.rtt_file_mysql_cred)
# Installing required packages
install_debian_pkg("libmysqlcppconn-dev")
install_debian_pkg_at_least_one(
["default-libmysqlclient-dev", "libmysqlclient-dev"])
python3, pip3 = setup_python3(use_system=args.sys_python,
buildj=args.buildj)
install_python_pkg("pip", no_cache=False, pip3=pip3)
install_python_pkgs(["mysqlclient", "paramiko"], pip3=pip3)
# This can be done only after installing cryptography and paramiko
from common.rtt_registration import register_db_user
db_def_passwd = get_mysql_password_args(args)
register_db_user(Database.ssh_root_user,
Database.address,
Database.ssh_port,
Storage.MYSQL_STORAGE_USER,
cred_db_password,
Storage.address,
Database.MYSQL_ROOT_USERNAME,
Database.MYSQL_DB_NAME,
priv_select=True,
db_def_passwd=db_def_passwd,
db_no_pass=args.local_db)
# Adding new job to cron - cache cleaning script
install_debian_pkg("cron")
add_cron_job(Storage.rtt_file_clean_cache,
Storage.rtt_file_store_ini,
Storage.rtt_file_clean_cache_log,
python3=python3)
exec_sys_call_check("service cron restart")
if not args.docker:
service_enable("ssh.service")
service_enable("cron.service")
# All configured here.
except BaseException as e:
print_error("{}. Fix error and run the script again.".format(e))
traceback.print_exc()
return 2
def main():
parser = argparse.ArgumentParser(description='Web view deployment')
parser.add_argument('--docker',
dest='docker',
action='store_const',
const=True,
default=False,
help='Docker deployment')
parser.add_argument('--ph4',
dest='ph4',
action='store_const',
const=True,
default=False,
help='Use Ph4 forks of tools')
parser.add_argument('--local-db',
dest='local_db',
action='store_const',
const=True,
default=False,
help='DB server is on the same machine')
parser.add_argument('--mysql-pass',
dest='mysql_pass',
action='store_const',
const=True,
default=False,
help='DB password to use')
parser.add_argument('--mysql-pass-file',
dest='mysql_pass_file',
action='store_const',
const=True,
default=False,
help='DB password file to use')
parser.add_argument('--config',
dest='config',
default='deployment_settings.ini',
help='Path to deployment_settings.ini')
args = parser.parse_args()
deploy_cfg_file = args.config
deploy_cfg = configparser.ConfigParser()
current_dir = os.path.abspath(os.path.curdir)
try:
deploy_cfg.read(deploy_cfg_file)
if len(deploy_cfg.sections()) == 0:
raise FileNotFoundError("can't read: {}".format(deploy_cfg_file))
Database.address = get_no_empty(deploy_cfg, "Database", "IPv4-Address")
Database.mysql_port = get_no_empty(deploy_cfg, "Database",
"MySQL-port")
Database.ssh_port = get_no_empty(deploy_cfg, "Database", "SSH-Port")
Database.ssh_root_user = get_no_empty(deploy_cfg, "Database",
"SSH-Root-User")
Storage.address = get_no_empty(deploy_cfg, "Storage", "IPv4-Address")
Storage.ssh_root_user = get_no_empty(deploy_cfg, "Storage",
"SSH-Root-User")
Storage.acc_chroot = get_no_empty(deploy_cfg, "Storage",
"Storage-Chroot")
Storage.storage_user = get_no_empty(deploy_cfg, "Storage",
"Storage-User")
Storage.ssh_port = get_no_empty(deploy_cfg, "Storage", "SSH-port")
RTTWeb.address = get_no_empty(deploy_cfg, "Web", "IPv4-Address")
except BaseException as e:
print_error("Configuration file: {}".format(e))
sys.exit(1)
# Sanity checks
try:
check_files_exists({
RTTWeb.APACHE_CONFIG, CommonConst.FRONTEND_SUBMIT_EXPERIMENT_SCRIPT
})
except AssertionError as e:
print_error("Invalid configuration. {}".format(e))
sys.exit(1)
try:
install_debian_pkgs([
"acl", "sudo", "wget", "unzip", "rsync", "openssh-client",
"python3-pip", "python3-venv", "apache2",
"libapache2-mod-wsgi-py3", "certbot"
])
python_packages = [
"wheel", "django==2.0.8", "django-bootstrap3",
"django-bootstrap-form", "django-datetime-widget", "mysqlclient",
"sarge", "requests", "shellescape", "coloredlogs", "filelock",
"configparser", "cryptography", "pyinstaller", "filelock",
"jsonpath-ng"
]
install_python_pkg("pip", no_cache=False)
install_python_pkgs(python_packages)
# Adding rtt-admin group that is intended to manage
# directories and files related to rtt without root access
exec_sys_call_check("groupadd {}".format(Frontend.RTT_ADMIN_GROUP),
acc_codes=[0, 9])
rtt_admin_grp_gid = grp.getgrnam(Frontend.RTT_ADMIN_GROUP).gr_gid
# Adding group for users of rtt
exec_sys_call_check("groupadd {}".format(Frontend.RTT_USER_GROUP),
acc_codes=[0, 9])
rtt_user_grp_gid = grp.getgrnam(Frontend.RTT_USER_GROUP).gr_gid
wusr = '******'
wgrp = 'www-data'
# Lets encrypt http-auth
os.makedirs('/var/www/html/.well-known/acme-challenge', 0o777, True)
recursive_chmod_chown('/var/www/html/.well-known', 0o660, 0o771, wusr,
wgrp)
dst_dir = RTTWeb.RTT_WEB_PATH
if os.path.exists(dst_dir):
shutil.rmtree(dst_dir)
rttweb_repo = RTTWeb.WEB_REPO_PH4 if args.ph4 else RTTWeb.WEB_REPO
exec_sys_call_check("git clone --recursive %s %s" %
(rttweb_repo, dst_dir))
os.chdir(dst_dir)
exec_sys_call_check("python3 -m venv %s" % RTTWeb.RTT_WEB_ENV)
pip3_venv = os.path.abspath(
os.path.join(RTTWeb.RTT_WEB_ENV, 'bin', 'pip3'))
install_python_pkgs(python_packages, pip3=pip3_venv)
# Credentials
from common.rtt_registration import register_db_user
from common.rtt_registration import add_authorized_key_to_server
from common.rtt_registration import db_server_cmd, get_db_command
credsdir = os.path.join(dst_dir, RTTWeb.RTT_WEB_CREDENTIALS)
if os.path.exists(credsdir):
shutil.rmtree(credsdir)
os.makedirs(credsdir, 0o777, True)
sec_key = get_rnd_pwd()
sec_file = os.path.join(credsdir,
RTTWeb.RTT_WEB_CREDENTIALS_SECRET_KEY)
with create_file_wperms(sec_file, mask=0o640, mode='w') as fh:
fh.write(sec_key)
chmod_chown(sec_file, 0o640, own=wusr, grp=wgrp)
# Create database
db_def_passwd = get_mysql_password_args(args)
print("Creating Web database")
create_cmd = get_db_command(Database.MYSQL_ROOT_USERNAME,
db_def_passwd,
'"CREATE DATABASE %s"' % RTTWeb.MYSQL_DB)
db_server_cmd(Database.ssh_root_user,
Database.address,
Database.ssh_port,
command=create_cmd)
# Register user for results preview
cred_mysql_db_password = get_rnd_pwd()
creds_db_path = os.path.join(credsdir, RTTWeb.MYSQL_RTT_CONFIG)
creds_db_path2 = os.path.join(credsdir, RTTWeb.MYSQL_RTT_CONFIG2)
write_db_credentials(RTTWeb.MYSQL_RTT_USER, cred_mysql_db_password,
creds_db_path)
shutil.copy(creds_db_path, creds_db_path2)
chmod_chown(creds_db_path, 0o640, own=wusr, grp=wgrp)
chmod_chown(creds_db_path2, 0o640, own=wusr, grp=wgrp)
db_addr_from = RTTWeb.address if not args.docker else '%'
register_db_user(
Database.ssh_root_user,
Database.address,
Database.ssh_port,
RTTWeb.MYSQL_RTT_USER,
cred_mysql_db_password,
db_addr_from,
Database.MYSQL_ROOT_USERNAME,
Database.MYSQL_DB_NAME,
priv_select=True,
priv_insert=True, # insert for submit_experiment
db_def_passwd=db_def_passwd,
db_no_pass=args.local_db)
# Register web user
creds_mysql_web_pass = get_rnd_pwd()
creds_db_path_web = os.path.join(credsdir, RTTWeb.WEB_DB_CONFIG)
write_db_credentials_web(RTTWeb.MYSQL_USER,
creds_mysql_web_pass,
RTTWeb.MYSQL_DB,
creds_db_path_web,
address=Database.address)
chmod_chown(creds_db_path_web, 0o640, own=wusr, grp=wgrp)
register_db_user(Database.ssh_root_user,
Database.address,
Database.ssh_port,
RTTWeb.MYSQL_USER,
creds_mysql_web_pass,
db_addr_from,
Database.MYSQL_ROOT_USERNAME,
RTTWeb.MYSQL_DB,
priv_select=True,
priv_insert=True,
priv_update=True,
priv_delete=True,
priv_create=True,
priv_alter=True,
priv_index=True,
db_def_passwd=db_def_passwd,
db_no_pass=args.local_db)
# Register machine to storage
rtt_ssh_pkey = os.path.join(credsdir, RTTWeb.SSH_CREDENTIALS_KEY)
key_pwd = get_rnd_pwd()
exec_sys_call_check("ssh-keygen -q -b 2048 -t rsa -N {} -f {}".format(
key_pwd, rtt_ssh_pkey))
chmod_chown(rtt_ssh_pkey, 0o600, own=wusr, grp=wgrp)
chmod_chown(rtt_ssh_pkey + ".pub", 0o640, own=wusr, grp=wgrp)
with open(rtt_ssh_pkey + ".pub") as f:
pub_key = f.read().rstrip()
rtt_ssh_cfg = os.path.join(credsdir, RTTWeb.SSH_CREDENTIALS_FILE)
write_ssh_credentials(Storage.storage_user, key_pwd, rtt_ssh_pkey,
rtt_ssh_cfg)
chmod_chown(rtt_ssh_cfg, 0o640, own=wusr, grp=wgrp)
authorized_keys_path = "{}{}".format(
Storage.acc_chroot,
os.path.join(Storage.CHROOT_HOME_DIR, Storage.SSH_DIR,
Storage.AUTH_KEYS_FILE))
add_authorized_key_to_server(Storage.ssh_root_user, Storage.address,
Storage.ssh_port, pub_key,
authorized_keys_path)
# Submit experiment
submdir = os.path.join(dst_dir, RTTWeb.RTT_WEB_SUBMIT_EXP)
if os.path.exists(submdir):
shutil.rmtree(submdir)
os.makedirs(submdir, 0o771, True)
os.chdir(submdir)
# - create frontend.ini
frontend_ini_cfg = configparser.ConfigParser()
frontend_ini_cfg.add_section("MySQL-Database")
frontend_ini_cfg.set("MySQL-Database", "Name", Database.MYSQL_DB_NAME)
frontend_ini_cfg.set("MySQL-Database", "Address", Database.address)
frontend_ini_cfg.set("MySQL-Database", "Port", Database.mysql_port)
frontend_ini_cfg.set("MySQL-Database", "Credentials-file",
os.path.abspath(creds_db_path))
frontend_ini_cfg.add_section("Storage")
frontend_ini_cfg.set("Storage", "Address", Storage.address)
frontend_ini_cfg.set("Storage", "Port", Storage.ssh_port)
frontend_ini_cfg.set(
"Storage", "Data-directory",
os.path.join(Storage.CHROOT_HOME_DIR, Storage.CHROOT_DATA_DIR))
frontend_ini_cfg.set(
"Storage", "Config-directory",
os.path.join(Storage.CHROOT_HOME_DIR, Storage.CHROOT_CONF_DIR))
frontend_ini_cfg.set("Storage", "Credentials-file",
os.path.abspath(rtt_ssh_cfg))
with open(Frontend.FRONT_CONFIG_FILE, "w") as f:
frontend_ini_cfg.write(f)
shutil.copy(
os.path.join(current_dir,
CommonConst.FRONTEND_SUBMIT_EXPERIMENT_SCRIPT),
Frontend.SUBMIT_EXPERIMENT_SCRIPT)
chmod_chown(Frontend.SUBMIT_EXPERIMENT_SCRIPT,
0o660,
own=wusr,
grp=wgrp)
if os.path.exists(Frontend.COMMON_FILES_DIR):
shutil.rmtree(Frontend.COMMON_FILES_DIR)
shutil.copytree(
os.path.join(current_dir, CommonConst.COMMON_FILES_DIR),
Frontend.COMMON_FILES_DIR)
recursive_chmod_chown(Frontend.COMMON_FILES_DIR,
mod_f=0o660,
mod_d=0o2770,
own=wusr,
grp=wgrp)
submit_exp_base_name = os.path.splitext(
Frontend.SUBMIT_EXPERIMENT_SCRIPT)[0]
exec_sys_call_check("pyinstaller -F {}".format(
Frontend.SUBMIT_EXPERIMENT_SCRIPT))
shutil.move("dist/{}".format(submit_exp_base_name),
Frontend.SUBMIT_EXPERIMENT_BINARY)
chmod_chown(Frontend.SUBMIT_EXPERIMENT_BINARY,
0o6775,
own=wusr,
grp=wgrp)
shutil.rmtree("dist")
shutil.rmtree("build")
shutil.rmtree("__pycache__")
os.remove("{}.spec".format(submit_exp_base_name))
# Migrate
os.chdir(RTTWeb.RTT_WEB_PATH)
python_venv = os.path.abspath(
os.path.join(RTTWeb.RTT_WEB_PATH, RTTWeb.RTT_WEB_ENV, 'bin',
'python3'))
exec_sys_call_check("%s manage.py migrate" % python_venv)
# Chown all
print("Chmoding...")
exec_sys_call_check("chown -R \"%s:%s\" %s" % (wusr, wgrp, dst_dir))
# Apache config file
shutil.copy(os.path.join(current_dir, RTTWeb.APACHE_CONFIG),
'/etc/apache2/sites-available/000-default.conf')
# Restart apache
exec_sys_call_check("service apache2 restart")
if not args.docker:
service_enable("apache2.service")
# Everything should be okay now.
except BaseException as e:
print_error("{}. Fix error and run the script again.".format(e))
traceback.print_exc()
def main():
deploy_cfg = configparser.ConfigParser()
try:
deploy_cfg.read(deploy_cfg_file)
if len(deploy_cfg.sections()) == 0:
raise FileNotFoundError("can't read: {}".format(deploy_cfg_file))
Frontend.address = get_no_empty(deploy_cfg, "Frontend", "IPv4-Address")
Frontend.rtt_users_chroot = get_no_empty(deploy_cfg, "Frontend",
"RTT-Users-Chroot")
Frontend.ssh_config = get_no_empty(deploy_cfg, "Frontend",
"SSH-Config")
Database.address = get_no_empty(deploy_cfg, "Database", "IPv4-Address")
Database.mysql_port = get_no_empty(deploy_cfg, "Database",
"MySQL-port")
Database.ssh_port = get_no_empty(deploy_cfg, "Database", "SSH-Port")
Database.ssh_root_user = get_no_empty(deploy_cfg, "Database",
"SSH-Root-User")
Storage.address = get_no_empty(deploy_cfg, "Storage", "IPv4-Address")
Storage.ssh_root_user = get_no_empty(deploy_cfg, "Storage",
"SSH-Root-User")
Storage.acc_chroot = get_no_empty(deploy_cfg, "Storage",
"Storage-Chroot")
Storage.storage_user = get_no_empty(deploy_cfg, "Storage",
"Storage-User")
Storage.ssh_port = get_no_empty(deploy_cfg, "Storage", "SSH-port")
except BaseException as e:
print_error("Configuration file: {}".format(e))
sys.exit(1)
# Sanity checks
try:
check_paths_abs({
Frontend.rtt_users_chroot, Frontend.CHROOT_RTT_FILES,
Frontend.CHROOT_RTT_USERS_HOME
})
check_paths_rel({
Frontend.CREDENTIALS_DIR, Frontend.COMMON_FILES_DIR,
Frontend.SSH_DIR
})
check_files_exists({
Frontend.ssh_config, Frontend.FSTAB_FILE,
CommonConst.FRONTEND_SUBMIT_EXPERIMENT_SCRIPT,
CommonConst.FRONTEND_ADD_USER_SCRIPT
})
except AssertionError as e:
print_error("Invalid configuration. {}".format(e))
sys.exit(1)
# Setting of paths used in this script
Frontend.abs_rtt_files = \
Frontend.rtt_users_chroot + Frontend.CHROOT_RTT_FILES
Frontend.abs_config_ini = \
os.path.join(Frontend.abs_rtt_files, Frontend.FRONT_CONFIG_FILE)
Frontend.abs_submit_exp_script = \
os.path.join(Frontend.abs_rtt_files, Frontend.SUBMIT_EXPERIMENT_SCRIPT)
Frontend.abs_add_user_script = \
os.path.join(Frontend.abs_rtt_files, Frontend.ADD_USER_SCRIPT)
Frontend.submit_exp_base_name = \
os.path.splitext(Frontend.SUBMIT_EXPERIMENT_SCRIPT)[0]
Frontend.abs_cred_dir = \
os.path.join(Frontend.abs_rtt_files, Frontend.CREDENTIALS_DIR)
Frontend.rel_cred_dir = \
os.path.join(Frontend.CHROOT_RTT_FILES, Frontend.CREDENTIALS_DIR)
Frontend.abs_common_files = \
os.path.join(Frontend.abs_rtt_files, Frontend.COMMON_FILES_DIR)
Frontend.abs_cred_mysql_ini = \
os.path.join(Frontend.abs_cred_dir, Frontend.MYSQL_CREDENTIALS_FILE)
Frontend.rel_cred_mysql_ini = \
os.path.join(Frontend.rel_cred_dir, Frontend.MYSQL_CREDENTIALS_FILE)
Frontend.abs_cred_store_ini = \
os.path.join(Frontend.abs_cred_dir, Frontend.SSH_CREDENTIALS_FILE)
Frontend.rel_cred_store_ini = \
os.path.join(Frontend.rel_cred_dir, Frontend.SSH_CREDENTIALS_FILE)
Frontend.abs_cred_store_key = \
os.path.join(Frontend.abs_cred_dir, Frontend.SSH_CREDENTIALS_KEY)
Frontend.rel_cred_store_key = \
os.path.join(Frontend.rel_cred_dir, Frontend.SSH_CREDENTIALS_KEY)
try:
# Adding rtt-admin group that is intended to manage
# directories and files related to rtt without root access
exec_sys_call_check("groupadd {}".format(Frontend.RTT_ADMIN_GROUP),
acc_codes=[0, 9])
rtt_admin_grp_gid = grp.getgrnam(Frontend.RTT_ADMIN_GROUP).gr_gid
# Adding group for users of rtt
exec_sys_call_check("groupadd {}".format(Frontend.RTT_USER_GROUP),
acc_codes=[0, 9])
rtt_user_grp_gid = grp.getgrnam(Frontend.RTT_USER_GROUP).gr_gid
# Installing debootstrap used for ssh jail
install_debian_pkg("debootstrap")
# Delete chroot directory if it exists
if os.path.exists(Frontend.rtt_users_chroot):
shutil.rmtree(Frontend.rtt_users_chroot)
# Building chroot jail for rtt users
create_dir(Frontend.rtt_users_chroot,
0o775,
grp=Frontend.RTT_ADMIN_GROUP)
exec_sys_call_check("debootstrap {} {}".format(
Frontend.CHROOT_DEBIAN_VERSION, Frontend.rtt_users_chroot))
with open(Frontend.FSTAB_FILE, "a") as f:
f.write("proc {} proc defaults 0 0\n".format(
os.path.join(Frontend.rtt_users_chroot, "proc")))
f.write("sysfs {} sysfs defaults 0 0\n".format(
os.path.join(Frontend.rtt_users_chroot, "sys")))
exec_sys_call_check("mount proc {} -t proc".format(
os.path.join(Frontend.rtt_users_chroot, "proc")))
exec_sys_call_check("mount sysfs {} -t sysfs".format(
os.path.join(Frontend.rtt_users_chroot, "sys")))
shutil.copy("/etc/hosts",
os.path.join(Frontend.rtt_users_chroot, "etc/hosts"))
create_dir(Frontend.abs_rtt_files,
0o2775,
grp=Frontend.RTT_ADMIN_GROUP)
# Set ACL on top directory - ensures all new files will have correct permissions
exec_sys_call_check("setfacl -R -d -m g::rwx {}".format(
Frontend.abs_rtt_files))
exec_sys_call_check("setfacl -R -d -m o::--- {}".format(
Frontend.abs_rtt_files))
create_dir(Frontend.abs_cred_dir, 0o2770, grp=Frontend.RTT_ADMIN_GROUP)
frontend_ini_cfg = configparser.ConfigParser()
frontend_ini_cfg.add_section("MySQL-Database")
frontend_ini_cfg.set("MySQL-Database", "Name", Database.MYSQL_DB_NAME)
frontend_ini_cfg.set("MySQL-Database", "Address", Database.address)
frontend_ini_cfg.set("MySQL-Database", "Port", Database.mysql_port)
frontend_ini_cfg.set("MySQL-Database", "Credentials-file",
Frontend.rel_cred_mysql_ini)
frontend_ini_cfg.add_section("Storage")
frontend_ini_cfg.set("Storage", "Address", Storage.address)
frontend_ini_cfg.set("Storage", "Port", Storage.ssh_port)
frontend_ini_cfg.set(
"Storage", "Data-directory",
os.path.join(Storage.CHROOT_HOME_DIR, Storage.CHROOT_DATA_DIR))
frontend_ini_cfg.set(
"Storage", "Config-directory",
os.path.join(Storage.CHROOT_HOME_DIR, Storage.CHROOT_CONF_DIR))
frontend_ini_cfg.set("Storage", "Credentials-file",
Frontend.rel_cred_store_ini)
frontend_ini_cfg.add_section("Frontend")
frontend_ini_cfg.set("Frontend", "RTT-Users-Chroot",
Frontend.rtt_users_chroot)
with open(Frontend.abs_config_ini, "w") as f:
frontend_ini_cfg.write(f)
shutil.copy(CommonConst.FRONTEND_SUBMIT_EXPERIMENT_SCRIPT,
Frontend.abs_submit_exp_script)
chmod_chown(Frontend.abs_submit_exp_script,
0o660,
grp=Frontend.RTT_ADMIN_GROUP)
shutil.copy(CommonConst.FRONTEND_ADD_USER_SCRIPT,
Frontend.abs_add_user_script)
chmod_chown(Frontend.abs_add_user_script,
0o770,
grp=Frontend.RTT_ADMIN_GROUP)
if os.path.exists(Frontend.abs_common_files):
shutil.rmtree(Frontend.abs_common_files)
shutil.copytree(CommonConst.COMMON_FILES_DIR,
Frontend.abs_common_files)
recursive_chmod_chown(Frontend.abs_common_files,
mod_f=0o660,
mod_d=0o2770,
grp=Frontend.RTT_ADMIN_GROUP)
# Entering chroot jail
real_root = os.open("/", os.O_RDONLY)
os.chroot(Frontend.rtt_users_chroot)
# Adding groups - instead there should be two-way sync!!!
exec_sys_call_check("groupadd -g {} {}".format(
rtt_admin_grp_gid, Frontend.RTT_ADMIN_GROUP),
acc_codes=[0, 9])
exec_sys_call_check("groupadd -g {} {}".format(
rtt_user_grp_gid, Frontend.RTT_USER_GROUP),
acc_codes=[0, 9])
# Installing needed packages inside jail
install_debian_pkg("python3")
install_debian_pkg("python3-dev")
install_debian_pkg("python3-setuptools")
install_debian_pkg("libmysqlclient-dev")
install_debian_pkg("build-essential")
install_debian_pkg("python3-cryptography")
install_debian_pkg("python3-paramiko")
install_debian_pkg("python3-pip")
install_python_pkg("pyinstaller")
install_python_pkg("mysqlclient")
os.chdir(Frontend.CHROOT_RTT_FILES)
exec_sys_call_check("pyinstaller -F {}".format(
Frontend.SUBMIT_EXPERIMENT_SCRIPT))
shutil.move("dist/{}".format(Frontend.submit_exp_base_name),
Frontend.SUBMIT_EXPERIMENT_BINARY)
chmod_chown(Frontend.SUBMIT_EXPERIMENT_BINARY,
0o2775,
grp=Frontend.RTT_ADMIN_GROUP)
shutil.rmtree("dist")
shutil.rmtree("build")
shutil.rmtree("__pycache__")
os.remove("{}.spec".format(Frontend.submit_exp_base_name))
# Exiting chroot jail
os.fchdir(real_root)
os.chroot(".")
os.close(real_root)
sshd_config_append = "\n\n\n\n" \
"Match Group {0}\n" \
"\tChrootDirectory {1}\n" \
"\tPasswordAuthentication yes\n" \
"\tAllowTcpForwarding no\n" \
"\tPermitTunnel no\n" \
"\tX11Forwarding no\n" \
"\tAuthorizedKeysFile {1}{2}\n" \
"\n".format(Frontend.RTT_USER_GROUP, Frontend.rtt_users_chroot,
os.path.join(Frontend.CHROOT_RTT_USERS_HOME, "%u",
Frontend.SSH_DIR, Frontend.AUTH_KEYS_FILE))
with open(Frontend.ssh_config, "a") as f:
f.write(sshd_config_append)
exec_sys_call_check("service sshd restart")
install_debian_pkg("python3-cryptography")
install_debian_pkg("python3-paramiko")
from common.rtt_registration import register_db_user
from common.rtt_registration import add_authorized_key_to_server
# Register frontend user at the database
cred_mysql_db_password = get_rnd_pwd()
cred_mysql_db_cfg = configparser.ConfigParser()
cred_mysql_db_cfg.add_section("Credentials")
cred_mysql_db_cfg.set("Credentials", "Username",
Frontend.MYSQL_FRONTEND_USER)
cred_mysql_db_cfg.set("Credentials", "Password",
cred_mysql_db_password)
with open(Frontend.abs_cred_mysql_ini, "w") as f:
cred_mysql_db_cfg.write(f)
register_db_user(Database.ssh_root_user,
Database.address,
Database.ssh_port,
Frontend.MYSQL_FRONTEND_USER,
cred_mysql_db_password,
Frontend.address,
Database.MYSQL_ROOT_USERNAME,
Database.MYSQL_DB_NAME,
priv_insert=True,
priv_select=True)
# Register frontend at the storage
cred_store_ssh_key_password = get_rnd_pwd()
cred_store_ssh_cfg = configparser.ConfigParser()
cred_store_ssh_cfg.add_section("Credentials")
cred_store_ssh_cfg.set("Credentials", "Username", Storage.storage_user)
cred_store_ssh_cfg.set("Credentials", "Private-key-file",
Frontend.rel_cred_store_key)
cred_store_ssh_cfg.set("Credentials", "Private-key-password",
cred_store_ssh_key_password)
with open(Frontend.abs_cred_store_ini, "w") as f:
cred_store_ssh_cfg.write(f)
exec_sys_call_check("ssh-keygen -q -b 2048 -t rsa -N {} -f {}".format(
cred_store_ssh_key_password, Frontend.abs_cred_store_key))
chmod_chown(Frontend.abs_cred_store_key,
0o660,
grp=Frontend.RTT_ADMIN_GROUP)
chmod_chown(Frontend.abs_cred_store_key + ".pub",
0o660,
grp=Frontend.RTT_ADMIN_GROUP)
with open("{}.pub".format(Frontend.abs_cred_store_key),
"r") as pub_key_f:
pub_key = pub_key_f.read().rstrip()
add_authorized_key_to_server(
Storage.ssh_root_user, Storage.address, Storage.ssh_port, pub_key,
"{}{}".format(
Storage.acc_chroot,
os.path.join(Storage.CHROOT_HOME_DIR, Storage.SSH_DIR,
Storage.AUTH_KEYS_FILE)))
# Everything should be okay now.
except BaseException as e:
print_error("{}. Fix error and run the script again.".format(e))
