def main(): parser = argparse.ArgumentParser(description='Worker deployment') parser.add_argument('--metacentrum', dest='metacentrum', action='store_const', const=True, default=False, help='Metacetrum deployment') parser.add_argument('--docker', dest='docker', action='store_const', const=True, default=False, help='Docker deployment') parser.add_argument( '--db-passwd', dest='db_passwd', help='DB password to use, if given, skips DB registration') parser.add_argument( '--ssh-passphrase', dest='ssh_passphrase', help='SSH passphrase to use to protect the private key') parser.add_argument( '--ssh-priv', dest='ssh_priv', help= 'SSH private key to use instead of generated one, has to have .pub counterpart' ) parser.add_argument('--no-db-reg', dest='no_db_reg', action='store_const', const=True, default=False, help='Skip MySQL user registration') parser.add_argument('--no-ssh-reg', dest='no_ssh_reg', action='store_const', const=True, default=False, help='Skip SSH key registration at storage server') parser.add_argument('--no-email', dest='no_email', action='store_const', const=True, default=False, help='Skip email registration') parser.add_argument('--no-cron', dest='no_cron', action='store_const', const=True, default=False, help='Skip cron setup') parser.add_argument('--sys-python', dest='sys_python', action='store_const', const=True, default=False, help='Use system python') parser.add_argument( '--ph4-rtt', dest='ph4_rtt', action='store_const', const=True, default=False, help='Use Ph4r05 fork of RTT - required for metacentrum') parser.add_argument('--pub-storage', dest='public_storage', action='store_const', const=True, default=False, help='Use public storage address') parser.add_argument('--local-db', dest='local_db', action='store_const', const=True, default=False, help='DB server is on the same machine') parser.add_argument('--mysql-pass', dest='mysql_pass', action='store_const', const=True, default=False, help='DB password to use') parser.add_argument('--mysql-pass-file', dest='mysql_pass_file', action='store_const', const=True, default=False, help='DB password file to use') parser.add_argument('--deb9', dest='deb9', type=int, default=0, help='Compile for debian 9') parser.add_argument('--deb10', dest='deb10', type=int, default=1, help='Compile for debian 10') parser.add_argument('--build-only-rtt', dest='build_only_rtt', type=int, default=0, help='Compile RTT only') parser.add_argument('--build-only-cryptostreams', dest='build_only_cryptostreams', type=int, default=0, help='Compile cryptostreams only') parser.add_argument('--build-only-batteries', dest='build_only_batteries', type=int, default=0, help='Compile batteries only') parser.add_argument('-j', dest='buildj', type=int, default=2, help='Parallel compilation') parser.add_argument('--config', dest='config', default='deployment_settings.ini', help='Path to deployment_settings.ini') parser.add_argument('backend_id', default=None, help='Backend ID to deploy') args = parser.parse_args() deploy_cfg_file = args.config wbare = not args.metacentrum if args.metacentrum or args.docker: args.ph4_rtt = True args.public_storage = True # Get path to main config from console if not args.backend_id: print("\nUsage: ./deploy_backend.py <backend-id>\n") print( "<backend-id> must be entered according to config with deployment settings.\n" " Configuration file \"{}\" must\n" " contain one and only one section named\n" " \"Backend-<backend-id>\"\n".format(deploy_cfg_file)) sys.exit(1) deploy_cfg = configparser.ConfigParser() try: current_dir = os.path.abspath(os.path.curdir) deploy_cfg.read(deploy_cfg_file) if len(deploy_cfg.sections()) == 0: raise FileNotFoundError("can't read: {}".format(deploy_cfg_file)) backend_sec = "Backend-" + args.backend_id Backend.address = get_no_empty(deploy_cfg, backend_sec, "IPv4-Address") Backend.rtt_files_dir = get_no_empty(deploy_cfg, backend_sec, "RTT-Files-dir") Backend.exec_max_tests = get_no_empty(deploy_cfg, backend_sec, "Maximum-parallel-tests") Backend.exec_test_timeout = get_no_empty(deploy_cfg, backend_sec, "Maximum-seconds-per-test") Backend.backend_id = deploy_cfg.get(backend_sec, "backend-id", fallback=None) Backend.backend_name = deploy_cfg.get(backend_sec, "backend-name", fallback=backend_sec) Backend.backend_loc = deploy_cfg.get(backend_sec, "backend-loc", fallback='') Backend.backend_longterm = deploy_cfg.get(backend_sec, "backend-longterm", fallback=1) Backend.backend_aux = deploy_cfg.get(backend_sec, "backend-aux", fallback='{}') Backend.log_dir = deploy_cfg.get(backend_sec, "log-dir", fallback=None) if not Backend.backend_id: Backend.backend_id = hashlib.md5(str( time.time()).encode('utf8')).hexdigest() Database.address = get_no_empty(deploy_cfg, "Database", "IPv4-Address") Database.mysql_port = get_no_empty(deploy_cfg, "Database", "MySQL-port") Database.ssh_port = get_no_empty(deploy_cfg, "Database", "SSH-Port") Database.ssh_root_user = get_no_empty(deploy_cfg, "Database", "SSH-Root-User") Storage.address_private = get_no_empty(deploy_cfg, "Storage", "IPv4-Address") Storage.address_public = deploy_cfg.get( "Storage", "IPv4-Address-Public") or Storage.address_private Storage.address = Storage.address_public if args.public_storage else Storage.address_private Storage.ssh_root_user = get_no_empty(deploy_cfg, "Storage", "SSH-Root-User") Storage.acc_chroot = get_no_empty(deploy_cfg, "Storage", "Storage-Chroot") Storage.storage_user = get_no_empty(deploy_cfg, "Storage", "Storage-User") Storage.ssh_port = get_no_empty(deploy_cfg, "Storage", "SSH-port") except Exception as e: print_error("Configuration file: {}".format(e)) sys.exit(1) # Sanity checks try: check_paths_abs({Backend.rtt_files_dir}) check_paths_rel({ Backend.COMMON_FILES_DIR, Backend.CACHE_DATA_DIR, Backend.CACHE_CONFIG_DIR, Backend.CREDENTIALS_DIR, Backend.RTT_EXECUTION_DIR, Backend.RANDOMNESS_TESTING_TOOLKIT_SRC_DIR, Backend.RTT_STATISTICAL_BATTERIES_SRC_DIR, }) check_files_exists({ CommonConst.BACKEND_CLEAN_CACHE_SCRIPT, CommonConst.BACKEND_RUN_JOBS_SCRIPT }) except AssertionError as e: print_error("Invalid configuration. {}".format(e)) sys.exit(1) os.makedirs(Backend.rtt_files_dir, 0o771, True) # Defined absolute paths to directories and files Backend.rand_test_tool_src_dir = \ join(Backend.rtt_files_dir, Backend.RANDOMNESS_TESTING_TOOLKIT_SRC_DIR) Backend.rand_test_tool_dl_zip = \ join(Backend.rtt_files_dir, Backend.RANDOMNESS_TESTING_TOOLKIT_GIT_NAME + ".zip") Backend.stat_batt_src_dir = \ join(Backend.rtt_files_dir, Backend.RTT_STATISTICAL_BATTERIES_SRC_DIR) Backend.stat_batt_dl_zip = \ join(Backend.rtt_files_dir, Backend.RTT_STATISTICAL_BATTERIES_GIT_NAME + ".zip") Backend.common_files_dir = \ join(Backend.rtt_files_dir, Backend.COMMON_FILES_DIR) Backend.cache_conf_dir = \ join(Backend.rtt_files_dir, Backend.CACHE_CONFIG_DIR) Backend.cache_data_dir = \ join(Backend.rtt_files_dir, Backend.CACHE_DATA_DIR) Backend.credentials_dir = \ join(Backend.rtt_files_dir, Backend.CREDENTIALS_DIR) Backend.rtt_exec_dir = \ join(Backend.rtt_files_dir, Backend.RTT_EXECUTION_DIR) Backend.rtt_exec_nist_exp_dir = \ join(Backend.rtt_exec_dir, Backend.NIST_STS_EXPERIMENTS_DIR) Backend.rtt_exec_nist_temp_dir = \ join(Backend.rtt_exec_dir, Backend.NIST_STS_TEMPLATES_DIR) Backend.ssh_store_pkey = \ join(Backend.credentials_dir, Backend.SSH_CREDENTIALS_KEY) Backend.ssh_store_pubkey = \ join(Backend.credentials_dir, Backend.SSH_CREDENTIALS_KEY + ".pub") Backend.run_jobs_path = \ join(Backend.rtt_files_dir, Backend.RUN_JOBS_SCRIPT) Backend.clean_cache_path = \ join(Backend.rtt_files_dir, Backend.CLEAN_CACHE_SCRIPT) Backend.rtt_binary_path = \ join(Backend.rtt_exec_dir, os.path.basename(Backend.RTT_BINARY_PATH)) Backend.cryptostreams_binary_dir = Backend.rtt_exec_dir Backend.mysql_cred_ini_path = \ join(Backend.credentials_dir, Backend.MYSQL_CREDENTIALS_FILE_INI) Backend.mysql_cred_json_path = \ join(Backend.credentials_dir, Backend.MYSQL_CREDENTIALS_FILE_JSON) Backend.ssh_cred_ini_path = \ join(Backend.credentials_dir, Backend.SSH_CREDENTIALS_FILE) Backend.config_ini_path = \ join(Backend.rtt_files_dir, Backend.BACKEND_CONFIG_FILE) wgrp = Backend.RTT_ADMIN_GROUP try: # Install essential packages install_debian_pkgs([ "acl", "sudo", "wget", "unzip", "rsync", "git", "curl", "openssh-client" ]) # Adding rtt-admin group that is intended to manage # directories and files related to rtt without root access exec_sys_call_check("groupadd {}".format(Backend.RTT_ADMIN_GROUP), acc_codes=[0, 9]) # Remove directories that was created previously if os.path.exists(Backend.rtt_files_dir): shutil.rmtree(Backend.rtt_files_dir) # Create and copy needed files into rtt-files create_dir(Backend.rtt_files_dir, 0o2770, grp=wgrp) # Set ACL on top directory - ensures all new files will have correct permissions exec_sys_call_check("setfacl -R -d -m g::rwx {}".format( Backend.rtt_files_dir)) exec_sys_call_check("setfacl -R -d -m o::--- {}".format( Backend.rtt_files_dir)) create_dir(Backend.cache_conf_dir, 0o2770, grp=wgrp) create_dir(Backend.cache_data_dir, 0o2770, grp=wgrp) create_dir(Backend.credentials_dir, 0o2770, grp=wgrp) create_dir(Backend.rtt_exec_dir, 0o2770, grp=wgrp) shutil.copy(CommonConst.BACKEND_CLEAN_CACHE_SCRIPT, Backend.clean_cache_path) chmod_chown(Backend.clean_cache_path, 0o770, grp=wgrp) shutil.copy(CommonConst.BACKEND_RUN_JOBS_SCRIPT, Backend.run_jobs_path) chmod_chown(Backend.run_jobs_path, 0o770, grp=wgrp) if os.path.exists(Backend.common_files_dir): shutil.rmtree(Backend.common_files_dir) shutil.copytree(CommonConst.COMMON_FILES_DIR, Backend.common_files_dir) recursive_chmod_chown(Backend.common_files_dir, mod_f=0o660, mod_d=0o2770, grp=wgrp) # Install packages install_debian_pkg("mailutils") if not args.no_email: install_debian_pkg("postfix") install_debian_pkgs(["libmysqlcppconn-dev"]) if args.deb10: install_debian_pkgs(["libunbound-dev", "libunistring-dev"]) if args.deb9: install_debian_pkgs(["libffi-dev"]) install_debian_pkg_at_least_one( ["default-libmysqlclient-dev", "libmysqlclient-dev"]) python3, pip3 = setup_python3(use_system=args.sys_python, buildj=args.buildj) install_python_pkg("pip", no_cache=False, pip3=pip3) install_python_pkgs([ "paramiko", "cryptography", "mysqlclient", "sarge", "requests", "shellescape", "coloredlogs", "filelock", "jsonpath-ng", "sshtunnel", "randomgen", "numpy", "booltest", "booltest-rtt", "rtt-data-gen", ], pip3=pip3) # Get current versions of needed tools from git # Statistical batteries if os.path.exists(Backend.stat_batt_src_dir): shutil.rmtree(Backend.stat_batt_src_dir) exec_sys_call_check("wget {} -O {}".format( Backend.RTT_STATISTICAL_BATTERIES_ZIP_URL, Backend.stat_batt_dl_zip)) exec_sys_call_check("unzip {} -d {}".format(Backend.stat_batt_dl_zip, Backend.rtt_files_dir)) os.remove(Backend.stat_batt_dl_zip) os.rename( join(Backend.rtt_files_dir, Backend.RTT_STATISTICAL_BATTERIES_GIT_NAME), Backend.stat_batt_src_dir) # Randomness testing toolkit if os.path.exists(Backend.rand_test_tool_src_dir): shutil.rmtree(Backend.rand_test_tool_src_dir) if args.ph4_rtt: exec_sys_call_check( "git clone --recursive https://github.com/ph4r05/randomness-testing-toolkit.git %s" % Backend.rand_test_tool_src_dir) else: exec_sys_call_check("wget {} -O {}".format( Backend.RANDOMNESS_TESTING_TOOLKIT_ZIP_URL, Backend.rand_test_tool_dl_zip)) exec_sys_call_check("unzip {} -d {}".format( Backend.rand_test_tool_dl_zip, Backend.rtt_files_dir)) os.remove(Backend.rand_test_tool_dl_zip) os.rename( join(Backend.rtt_files_dir, Backend.RANDOMNESS_TESTING_TOOLKIT_GIT_NAME), Backend.rand_test_tool_src_dir) # Change into directory rtt-src and rtt-stat-batt-src and call make and ./INSTALL respectively. # Build statistical batteries os.chdir(Backend.stat_batt_src_dir) build_only_used = args.build_only_rtt or args.build_only_cryptostreams or args.build_only_batteries build_batteries = not args.build_only_rtt and not args.build_only_cryptostreams build_rtt = not args.build_only_cryptostreams and not args.build_only_batteries build_cs = not args.build_only_rtt and not args.build_only_batteries if build_batteries: exec_sys_call_check("chmod +x INSTALL") exec_sys_call_check("./INSTALL", env=get_make_env(buildj=args.buildj)) recursive_chmod_chown(Backend.stat_batt_src_dir, mod_f=0o660, mod_d=0o2770, grp=wgrp) chmod_chown(Backend.DIEHARDER_BINARY_PATH, 0o770) chmod_chown(Backend.NIST_STS_BINARY_PATH, 0o770) chmod_chown(Backend.TESTU01_BINARY_PATH, 0o770) build_static_dieharder(Backend.stat_batt_src_dir, buildj=args.buildj) os.chdir(Backend.stat_batt_src_dir) # Build randomness testing toolkit os.chdir(Backend.rand_test_tool_src_dir) rtt_env = None if args.ph4_rtt: # p11 libs required by gnutls, required by libmaria p11libs = None try: p11libs = build_p11_lib( buildj=args.buildj) if args.deb10 else None except Exception as e: logger.warning("P11 build failed, using dynamic: %s" % (e, ), exc_info=e) lib_data = copy_rtt_libs(Backend.rand_test_tool_src_dir) rtt_env = get_rtt_build_env(Backend.rand_test_tool_src_dir, lib_data, args.deb10, p11libs) os.chdir(Backend.rand_test_tool_src_dir) try: if build_rtt: exec_sys_call_check("make -j%s" % (args.buildj, ), env=rtt_env, acc_codes=[0, 1, 2]) except Exception as e: if args.ph4_rtt: logger.warning( "Could not build RTT statically, trying dynamic build") print( "[ERROR] Could not build RTT statically, trying dynamic build" ) if build_rtt: exec_sys_call_check("make -j%s" % (args.buildj, ), acc_codes=[0, 1, 2]) else: raise e recursive_chmod_chown(Backend.rand_test_tool_src_dir, mod_f=0o660, mod_d=0o2770, grp=wgrp) chmod_chown(Backend.RTT_BINARY_PATH, 0o770) # Build cryptostreams cstreams = None if build_cs: cstreams = cryptostreams_complete_deploy( ph4=args.ph4_rtt, res_bin_dir=Backend.cryptostreams_binary_dir, src_dir=Backend.rtt_files_dir, buildj=args.buildj) # Build finished, go into original directory os.chdir(current_dir) if build_only_used: logger.info("Using --build-only-x, terminating call") return # Link RTT binary into execution directory os.symlink( join(Backend.rand_test_tool_src_dir, Backend.RTT_BINARY_PATH), Backend.rtt_binary_path) # Copy needed directories and files into execution directory shutil.copytree( join(Backend.stat_batt_src_dir, Backend.NIST_STS_TEMPLATES_DIR), join(Backend.rtt_exec_dir, os.path.basename(Backend.NIST_STS_TEMPLATES_DIR))) shutil.copytree( join(Backend.stat_batt_src_dir, Backend.NIST_STS_EXPERIMENTS_DIR), join(Backend.rtt_exec_dir, os.path.basename(Backend.NIST_STS_EXPERIMENTS_DIR))) # Booltest-rtt, rtt-data-gen try: booltest_rtt_binary = subprocess.check_output( ['which', 'booltest_rtt']).decode('utf8').strip() except Exception as e: booltest_rtt_binary = "" try: rtt_data_gen_binary = subprocess.check_output( ['which', 'rtt-data-gen']).decode('utf8').strip() except Exception as e: rtt_data_gen_binary = "" if booltest_rtt_binary and os.path.exists(booltest_rtt_binary): _spath = os.path.join(Backend.rtt_exec_dir, 'booltest_rtt') try_remove(_spath) os.symlink(booltest_rtt_binary, _spath) if rtt_data_gen_binary and os.path.exists(rtt_data_gen_binary): _spath = os.path.join(Backend.rtt_exec_dir, 'rtt-data-gen') try_remove(_spath) os.symlink(rtt_data_gen_binary, _spath) rtt_settings = { "toolkit-settings": { "logger": { "dir-prefix": join(Backend.rtt_files_dir, Backend.EXEC_LOGS_TOP_DIR), "run-log-dir": Backend.EXEC_LOGS_RUN_LOG_DIR, "dieharder-dir": Backend.EXEC_LOGS_DIEHARDER_DIR, "nist-sts-dir": Backend.EXEC_LOGS_NIST_STS_DIR, "tu01-smallcrush-dir": Backend.EXEC_LOGS_SMALLCRUSH_DIR, "tu01-crush-dir": Backend.EXEC_LOGS_CRUSH_DIR, "tu01-bigcrush-dir": Backend.EXEC_LOGS_BIGCRUSH_DIR, "tu01-rabbit-dir": Backend.EXEC_LOGS_RABBIT_DIR, "tu01-alphabit-dir": Backend.EXEC_LOGS_ALPHABIT_DIR, "tu01-blockalphabit-dir": Backend.EXEC_LOGS_BLOCKALPHABIT_DIR }, "result-storage": { "file": { "main-file": join(Backend.rtt_files_dir, Backend.EXEC_REPS_MAIN_FILE), "dir-prefix": join(Backend.rtt_files_dir, Backend.EXEC_REPS_TOP_DIR), "dieharder-dir": Backend.EXEC_REPS_DIEHARDER_DIR, "nist-sts-dir": Backend.EXEC_REPS_NIST_STS_DIR, "tu01-smallcrush-dir": Backend.EXEC_REPS_SMALLCRUSH_DIR, "tu01-crush-dir": Backend.EXEC_REPS_CRUSH_DIR, "tu01-bigcrush-dir": Backend.EXEC_REPS_BIGCRUSH_DIR, "tu01-rabbit-dir": Backend.EXEC_REPS_RABBIT_DIR, "tu01-alphabit-dir": Backend.EXEC_REPS_ALPHABIT_DIR, "tu01-blockalphabit-dir": Backend.EXEC_REPS_ALPHABIT_DIR }, "mysql-db": { "address": Database.address, "port": Database.mysql_port, "name": Database.MYSQL_DB_NAME, "credentials-file": Backend.mysql_cred_json_path } }, "binaries": { "nist-sts": join(Backend.stat_batt_src_dir, Backend.NIST_STS_BINARY_PATH), "dieharder": join(Backend.stat_batt_src_dir, Backend.DIEHARDER_BINARY_PATH), "testu01": join(Backend.stat_batt_src_dir, Backend.TESTU01_BINARY_PATH), "cryptostreams": cstreams[0], }, "miscellaneous": { "nist-sts": { "main-result-dir": join(Backend.rtt_exec_dir, Backend.NIST_MAIN_RESULT_DIR) } }, "execution": { "max-parallel-tests": int(Backend.exec_max_tests), "test-timeout-seconds": int(Backend.exec_test_timeout) }, "booltest": { "default-cli": "--no-summary --json-out --log-prints --top 128 --no-comb-and --only-top-comb --only-top-deg --no-term-map --topterm-heap --topterm-heap-k 256 --best-x-combs 512", "strategies": [{ "name": "v1", "cli": "", "variations": [{ "bl": [128, 256, 384, 512], "deg": [1, 2, 3], "cdeg": [1, 2, 3], "exclusions": [] }] }, { "name": "halving", "cli": "--halving", "variations": [{ "bl": [128, 256, 384, 512], "deg": [1, 2, 3], "cdeg": [1, 2, 3], "exclusions": [] }] }] } } } with open(join(Backend.rtt_exec_dir, Backend.RTT_SETTINGS_JSON), "w") as f: json.dump(rtt_settings, f, indent=4) # Get email configuration # Add configuration to file # inet_interface = loopback-only # inet_protocol = ipv4 if not args.no_email: with open(Backend.POSTFIX_CFG_PATH) as mail_cfg: for line in mail_cfg.readlines(): if line.startswith(Backend.POSTFIX_HOST_OPT): Backend.sender_email = line.split(sep=" = ")[1] if not args.no_email and Backend.sender_email is None: print_error("can't find option {} in file {}".format( Backend.POSTFIX_CFG_PATH, Backend.POSTFIX_HOST_OPT)) sys.exit(1) Backend.sender_email = ( "root@" + Backend.sender_email ) if not args.no_email else '*****@*****.**' # Create backend configuration file backend_ini_cfg = configparser.ConfigParser() backend_ini_cfg.add_section("MySQL-Database") backend_ini_cfg.set("MySQL-Database", "Address", Database.address) backend_ini_cfg.set("MySQL-Database", "Port", Database.mysql_port) backend_ini_cfg.set("MySQL-Database", "Name", Database.MYSQL_DB_NAME) backend_ini_cfg.set("MySQL-Database", "Credentials-file", Backend.mysql_cred_ini_path) backend_ini_cfg.add_section("Local-cache") backend_ini_cfg.set("Local-cache", "Data-directory", Backend.cache_data_dir) backend_ini_cfg.set("Local-cache", "Config-directory", Backend.cache_conf_dir) backend_ini_cfg.add_section("Backend") backend_ini_cfg.set("Backend", "Sender-email", Backend.sender_email) backend_ini_cfg.set("Backend", "Maximum-seconds-per-test", Backend.exec_test_timeout) backend_ini_cfg.set("Backend", "Maximum-parallel-tests", Backend.exec_max_tests) backend_ini_cfg.set("Backend", "backend-id", Backend.backend_id) backend_ini_cfg.set("Backend", "backend-name", Backend.backend_name) backend_ini_cfg.set("Backend", "backend-loc", Backend.backend_loc) backend_ini_cfg.set("Backend", "backend-longterm", Backend.backend_longterm) backend_ini_cfg.set("Backend", "backend-aux", Backend.backend_aux) if Backend.log_dir: backend_ini_cfg.set("Backend", "log-dir", Backend.log_dir) backend_ini_cfg.add_section("Storage") backend_ini_cfg.set("Storage", "Address", Storage.address) backend_ini_cfg.set("Storage", "Port", Storage.ssh_port) backend_ini_cfg.set( "Storage", "Data-directory", join(Storage.CHROOT_HOME_DIR, Storage.CHROOT_DATA_DIR)) backend_ini_cfg.set( "Storage", "Config-directory", join(Storage.CHROOT_HOME_DIR, Storage.CHROOT_CONF_DIR)) backend_ini_cfg.set("Storage", "Credentials-file", Backend.ssh_cred_ini_path) backend_ini_cfg.add_section("RTT-Binary") backend_ini_cfg.set("RTT-Binary", "Binary-path", Backend.rtt_binary_path) backend_ini_cfg.set("RTT-Binary", "booltest-rtt-path", booltest_rtt_binary) # TODO: auto-detect backend_ini_cfg.set("RTT-Binary", "rtt-data-gen-path", rtt_data_gen_binary) # TODO: auto-detect with open(Backend.config_ini_path, "w") as f: backend_ini_cfg.write(f) from common.rtt_registration import register_db_user from common.rtt_registration import add_authorized_key_to_server from common.rtt_registration import get_db_reg_command # Register machine to database db_pwd = args.db_passwd if args.db_passwd else get_rnd_pwd() write_db_credentials(Backend.MYSQL_BACKEND_USER, db_pwd, Backend.mysql_cred_ini_path) write_db_credentials_json(Backend.MYSQL_BACKEND_USER, db_pwd, Backend.mysql_cred_json_path) post_install_info = [] db_addr_from = Backend.address if wbare else '%' if not args.no_db_reg: db_def_passwd = get_mysql_password_args(args) register_db_user(Database.ssh_root_user, Database.address, Database.ssh_port, Backend.MYSQL_BACKEND_USER, db_pwd, db_addr_from, Database.MYSQL_ROOT_USERNAME, Database.MYSQL_DB_NAME, priv_select=True, priv_insert=True, priv_update=True, priv_create=True, db_def_passwd=db_def_passwd, db_no_pass=args.local_db) else: sql = get_db_reg_command(username=Database.MYSQL_ROOT_USERNAME, password=None, db_name=Database.MYSQL_DB_NAME, reg_name=Backend.MYSQL_BACKEND_USER, reg_address=db_addr_from, reg_pwd=db_pwd, priv_select=True, priv_insert=True, priv_update=True, priv_create=True, db_host=Database.address, db_port=Database.ssh_port) post_install_info.append( '* DB user not registered to the DB server. Make sure the following user:password has access: ' ) post_install_info.append(sql + '\n') # Register machine to storage key_pwd = args.ssh_passphrase if args.ssh_passphrase else get_rnd_pwd() if args.ssh_priv: shutil.copy(args.ssh_priv, Backend.ssh_store_pkey) shutil.copy(args.ssh_priv + '.pub', Backend.ssh_store_pubkey) else: exec_sys_call_check( "ssh-keygen -q -b 2048 -t rsa -N {} -f {}".format( key_pwd, Backend.ssh_store_pkey)) chmod_chown(Backend.ssh_store_pkey, 0o600, grp=wgrp) chmod_chown(Backend.ssh_store_pubkey, 0o660, grp=wgrp) with open(Backend.ssh_store_pubkey) as f: pub_key = f.read().rstrip() write_ssh_credentials(Storage.storage_user, key_pwd, Backend.ssh_store_pkey, Backend.ssh_cred_ini_path) authorized_keys_path = "{}{}".format( Storage.acc_chroot, join(Storage.CHROOT_HOME_DIR, Storage.SSH_DIR, Storage.AUTH_KEYS_FILE)) if args.no_ssh_reg: post_install_info.append( '* Register the following key on the storage server at %s' % (authorized_keys_path, )) post_install_info.append('%s' % (pub_key, )) post_install_info.append('') else: add_authorized_key_to_server(Storage.ssh_root_user, Storage.address, Storage.ssh_port, pub_key, authorized_keys_path) # Add cron jobs for cache cleaning and job running script if not args.no_cron: install_debian_pkg("cron") add_cron_job(Backend.clean_cache_path, Backend.config_ini_path, join(Backend.rtt_files_dir, Backend.CLEAN_CACHE_LOG), python3=python3) add_cron_job(Backend.run_jobs_path, Backend.config_ini_path, join(Backend.rtt_files_dir, Backend.RUN_JOBS_LOG), python3=python3) exec_sys_call_check("service cron restart") if not args.docker: service_enable("cron.service") try: adj_workers = '/root/adjust_workers.py' os.chdir(current_dir) shutil.copy('files/adjust_workers.py', adj_workers) chmod_chown(adj_workers, 0o770, grp=wgrp) except Exception as e: logger.error("Could not install adjust_workers: %s" % (e, )) if post_install_info: print('=' * 80) for x in post_install_info: print(x) except BaseException as e: print_error("{}. Fix error and run the script again.".format(e)) traceback.print_exc() return 2
def main(): parser = argparse.ArgumentParser(description='Frontend deployment') parser.add_argument('--docker', dest='docker', action='store_const', const=True, default=False, help='Docker deployment') parser.add_argument('--ph4', dest='ph4', action='store_const', const=True, default=False, help='Use Ph4 forks of tools') parser.add_argument('--no-chroot', dest='no_chroot', action='store_const', const=True, default=False, help='No chroot install, install to root') parser.add_argument('--local-db', dest='local_db', action='store_const', const=True, default=False, help='DB server is on the same machine') parser.add_argument('--no-ssh-server', dest='no_ssh_server', action='store_const', const=True, default=False, help='DB server is on the same machine') parser.add_argument('--mysql-pass', dest='mysql_pass', action='store_const', const=True, default=False, help='DB password to use') parser.add_argument('--mysql-pass-file', dest='mysql_pass_file', action='store_const', const=True, default=False, help='DB password file to use') parser.add_argument('--sys-python', dest='sys_python', action='store_const', const=True, default=False, help='Use system python') parser.add_argument('-j', dest='buildj', type=int, default=2, help='Parallel compilation') parser.add_argument('--config', dest='config', default='deployment_settings.ini', help='Path to deployment_settings.ini') args = parser.parse_args() deploy_cfg_file = args.config if args.no_chroot: args.no_ssh_server = True deploy_cfg = configparser.ConfigParser() try: deploy_cfg.read(deploy_cfg_file) if len(deploy_cfg.sections()) == 0: raise FileNotFoundError("can't read: {}".format(deploy_cfg_file)) Frontend.address = get_no_empty(deploy_cfg, "Frontend", "IPv4-Address") Frontend.rtt_users_chroot = get_no_empty(deploy_cfg, "Frontend", "RTT-Users-Chroot") Frontend.ssh_config = get_no_empty(deploy_cfg, "Frontend", "SSH-Config") Database.address = get_no_empty(deploy_cfg, "Database", "IPv4-Address") Database.mysql_port = get_no_empty(deploy_cfg, "Database", "MySQL-port") Database.ssh_port = get_no_empty(deploy_cfg, "Database", "SSH-Port") Database.ssh_root_user = get_no_empty(deploy_cfg, "Database", "SSH-Root-User") Storage.address = get_no_empty(deploy_cfg, "Storage", "IPv4-Address") Storage.ssh_root_user = get_no_empty(deploy_cfg, "Storage", "SSH-Root-User") Storage.acc_chroot = get_no_empty(deploy_cfg, "Storage", "Storage-Chroot") Storage.storage_user = get_no_empty(deploy_cfg, "Storage", "Storage-User") Storage.ssh_port = get_no_empty(deploy_cfg, "Storage", "SSH-port") except BaseException as e: print_error("Configuration file: {}".format(e)) sys.exit(1) # Sanity checks try: check_paths_abs({ Frontend.rtt_users_chroot, Frontend.CHROOT_RTT_FILES, Frontend.CHROOT_RTT_USERS_HOME }) check_paths_rel({ Frontend.CREDENTIALS_DIR, Frontend.COMMON_FILES_DIR, Frontend.SSH_DIR }) check_files_exists({ Frontend.ssh_config, Frontend.FSTAB_FILE, CommonConst.FRONTEND_SUBMIT_EXPERIMENT_SCRIPT, CommonConst.FRONTEND_ADD_USER_SCRIPT }) except AssertionError as e: print_error("Invalid configuration. {}".format(e)) sys.exit(1) # Setting of paths used in this script Frontend.abs_rtt_files = \ Frontend.rtt_users_chroot + Frontend.CHROOT_RTT_FILES if args.no_chroot: Frontend.abs_rtt_files = Frontend.CHROOT_RTT_FILES Frontend.abs_config_ini = \ os.path.join(Frontend.abs_rtt_files, Frontend.FRONT_CONFIG_FILE) Frontend.abs_submit_exp_script = \ os.path.join(Frontend.abs_rtt_files, Frontend.SUBMIT_EXPERIMENT_SCRIPT) Frontend.abs_add_user_script = \ os.path.join(Frontend.abs_rtt_files, Frontend.ADD_USER_SCRIPT) Frontend.submit_exp_base_name = \ os.path.splitext(Frontend.SUBMIT_EXPERIMENT_SCRIPT)[0] Frontend.abs_cred_dir = \ os.path.join(Frontend.abs_rtt_files, Frontend.CREDENTIALS_DIR) Frontend.rel_cred_dir = \ os.path.join(Frontend.CHROOT_RTT_FILES, Frontend.CREDENTIALS_DIR) Frontend.abs_common_files = \ os.path.join(Frontend.abs_rtt_files, Frontend.COMMON_FILES_DIR) Frontend.abs_cred_mysql_ini = \ os.path.join(Frontend.abs_cred_dir, Frontend.MYSQL_CREDENTIALS_FILE) Frontend.rel_cred_mysql_ini = \ os.path.join(Frontend.rel_cred_dir, Frontend.MYSQL_CREDENTIALS_FILE) Frontend.abs_cred_store_ini = \ os.path.join(Frontend.abs_cred_dir, Frontend.SSH_CREDENTIALS_FILE) Frontend.rel_cred_store_ini = \ os.path.join(Frontend.rel_cred_dir, Frontend.SSH_CREDENTIALS_FILE) Frontend.abs_cred_store_key = \ os.path.join(Frontend.abs_cred_dir, Frontend.SSH_CREDENTIALS_KEY) Frontend.rel_cred_store_key = \ os.path.join(Frontend.rel_cred_dir, Frontend.SSH_CREDENTIALS_KEY) try: install_debian_pkgs( ["acl", "sudo", "wget", "unzip", "rsync", "openssh-client", "git"]) # Adding rtt-admin group that is intended to manage # directories and files related to rtt without root access exec_sys_call_check("groupadd {}".format(Frontend.RTT_ADMIN_GROUP), acc_codes=[0, 9]) rtt_admin_grp_gid = grp.getgrnam(Frontend.RTT_ADMIN_GROUP).gr_gid # Adding group for users of rtt exec_sys_call_check("groupadd {}".format(Frontend.RTT_USER_GROUP), acc_codes=[0, 9]) rtt_user_grp_gid = grp.getgrnam(Frontend.RTT_USER_GROUP).gr_gid # Installing debootstrap used for ssh jail install_debian_pkg("debootstrap") # Delete chroot directory if it exists if os.path.exists(Frontend.rtt_users_chroot): print("Deleting %s" % Frontend.rtt_users_chroot) try_fnc(lambda: exec_sys_call("umount %s" % os.path.join( Frontend.rtt_users_chroot, "proc"))) try_fnc(lambda: exec_sys_call("umount %s" % os.path.join( Frontend.rtt_users_chroot, "sys"))) shutil.rmtree(Frontend.rtt_users_chroot) if os.path.exists(Frontend.abs_rtt_files): print("Deleting %s" % Frontend.abs_rtt_files) shutil.rmtree(Frontend.abs_rtt_files) # Building chroot jail for rtt users create_dir(Frontend.rtt_users_chroot, 0o775, grp=Frontend.RTT_ADMIN_GROUP) if not args.no_chroot: exec_sys_call_check("debootstrap {} {}".format( Frontend.CHROOT_DEBIAN_VERSION, Frontend.rtt_users_chroot)) with open(Frontend.FSTAB_FILE, "a") as f: f.write("proc {} proc defaults 0 0\n".format( os.path.join(Frontend.rtt_users_chroot, "proc"))) f.write("sysfs {} sysfs defaults 0 0\n".format( os.path.join(Frontend.rtt_users_chroot, "sys"))) exec_sys_call_check("mount proc {} -t proc".format( os.path.join(Frontend.rtt_users_chroot, "proc"))) exec_sys_call_check("mount sysfs {} -t sysfs".format( os.path.join(Frontend.rtt_users_chroot, "sys"))) shutil.copy("/etc/hosts", os.path.join(Frontend.rtt_users_chroot, "etc/hosts")) create_dir(Frontend.abs_rtt_files, 0o2775, grp=Frontend.RTT_ADMIN_GROUP) # Set ACL on top directory - ensures all new files will have correct permissions exec_sys_call_check("setfacl -R -d -m g::rwx {}".format( Frontend.abs_rtt_files)) exec_sys_call_check("setfacl -R -d -m o::--- {}".format( Frontend.abs_rtt_files)) create_dir(Frontend.abs_cred_dir, 0o2770, grp=Frontend.RTT_ADMIN_GROUP) frontend_ini_cfg = configparser.ConfigParser() frontend_ini_cfg.add_section("MySQL-Database") frontend_ini_cfg.set("MySQL-Database", "Name", Database.MYSQL_DB_NAME) frontend_ini_cfg.set("MySQL-Database", "Address", Database.address) frontend_ini_cfg.set("MySQL-Database", "Port", Database.mysql_port) frontend_ini_cfg.set("MySQL-Database", "Credentials-file", Frontend.rel_cred_mysql_ini) frontend_ini_cfg.add_section("Storage") frontend_ini_cfg.set("Storage", "Address", Storage.address) frontend_ini_cfg.set("Storage", "Port", Storage.ssh_port) frontend_ini_cfg.set( "Storage", "Data-directory", os.path.join(Storage.CHROOT_HOME_DIR, Storage.CHROOT_DATA_DIR)) frontend_ini_cfg.set( "Storage", "Config-directory", os.path.join(Storage.CHROOT_HOME_DIR, Storage.CHROOT_CONF_DIR)) frontend_ini_cfg.set("Storage", "Credentials-file", Frontend.rel_cred_store_ini) frontend_ini_cfg.add_section("Frontend") frontend_ini_cfg.set("Frontend", "RTT-Users-Chroot", Frontend.rtt_users_chroot) with open(Frontend.abs_config_ini, "w") as f: frontend_ini_cfg.write(f) shutil.copy(CommonConst.FRONTEND_SUBMIT_EXPERIMENT_SCRIPT, Frontend.abs_submit_exp_script) chmod_chown(Frontend.abs_submit_exp_script, 0o660, grp=Frontend.RTT_ADMIN_GROUP) shutil.copy(CommonConst.FRONTEND_ADD_USER_SCRIPT, Frontend.abs_add_user_script) chmod_chown(Frontend.abs_add_user_script, 0o770, grp=Frontend.RTT_ADMIN_GROUP) if os.path.exists(Frontend.abs_common_files): shutil.rmtree(Frontend.abs_common_files) shutil.copytree(CommonConst.COMMON_FILES_DIR, Frontend.abs_common_files) recursive_chmod_chown(Frontend.abs_common_files, mod_f=0o660, mod_d=0o2770, grp=Frontend.RTT_ADMIN_GROUP) # Entering chroot jail real_root = os.open("/", os.O_RDONLY) if not args.no_chroot: os.chroot(Frontend.rtt_users_chroot) # Adding groups - instead there should be two-way sync!!! exec_sys_call_check("groupadd -g {} {}".format( rtt_admin_grp_gid, Frontend.RTT_ADMIN_GROUP), acc_codes=[0, 9]) exec_sys_call_check("groupadd -g {} {}".format( rtt_user_grp_gid, Frontend.RTT_USER_GROUP), acc_codes=[0, 9]) # Installing needed packages inside jail install_debian_pkg_at_least_one( ["default-libmysqlclient-dev", "libmysqlclient-dev"]) install_debian_pkgs(["build-essential"]) python3, pip3 = setup_python3(use_system=args.sys_python, buildj=args.buildj) if args.sys_python: install_debian_pkgs([ "python3-dev", "python3-setuptools", "python3-cryptography", "python3-paramiko", "python3-pip" ]) install_python_pkg("pip", no_cache=False, pip3=pip3) install_python_pkgs([ "setuptools", "paramiko", "cryptography", "pyinstaller", "filelock", "jsonpath-ng", "booltest", "booltest-rtt", ], pip3=pip3) os.chdir(Frontend.CHROOT_RTT_FILES) submit_bin = submit_experiment_deploy(Frontend.CHROOT_RTT_FILES) submit_bin_dst = os.path.join('/usr/bin', Frontend.SUBMIT_EXPERIMENT_BINARY) try_fnc(lambda: os.unlink(submit_bin_dst)) os.symlink(submit_bin, submit_bin_dst) # CryptoStreams cryptostreams_complete_deploy(ph4=args.ph4, buildj=args.buildj) # Exiting chroot jail if not args.no_chroot: os.fchdir(real_root) os.chroot(".") os.close(real_root) sshd_config_append = "\n\n\n\n" \ "Match Group {0}\n" \ "\tChrootDirectory {1}\n" \ "\tPasswordAuthentication yes\n" \ "\tAllowTcpForwarding yes\n" \ "\tPermitTunnel yes\n" \ "\tX11Forwarding no\n" \ "\tAuthorizedKeysFile {1}{2}\n" \ "\n".format(Frontend.RTT_USER_GROUP, Frontend.rtt_users_chroot, os.path.join(Frontend.CHROOT_RTT_USERS_HOME, "%u", Frontend.SSH_DIR, Frontend.AUTH_KEYS_FILE)) if not args.no_ssh_server: install_debian_pkgs(["openssh-server"]) with open(Frontend.ssh_config, "a") as f: f.write(sshd_config_append) exec_sys_call_check("service ssh restart") install_debian_pkgs( ["python3-pip", "python3-cryptography", "python3-paramiko"]) from common.rtt_registration import register_db_user from common.rtt_registration import add_authorized_key_to_server # Register frontend user at the database cred_mysql_db_password = get_rnd_pwd() write_db_credentials(Frontend.MYSQL_FRONTEND_USER, cred_mysql_db_password, Frontend.abs_cred_mysql_ini) db_def_passwd = get_mysql_password_args(args) register_db_user(Database.ssh_root_user, Database.address, Database.ssh_port, Frontend.MYSQL_FRONTEND_USER, cred_mysql_db_password, Frontend.address, Database.MYSQL_ROOT_USERNAME, Database.MYSQL_DB_NAME, priv_insert=True, priv_select=True, db_def_passwd=db_def_passwd, db_no_pass=args.local_db) # Register frontend at the storage cred_store_ssh_key_password = get_rnd_pwd() write_ssh_credentials(Storage.storage_user, cred_store_ssh_key_password, Frontend.rel_cred_store_key, Frontend.abs_cred_store_ini) exec_sys_call_check("ssh-keygen -q -b 2048 -t rsa -N {} -f {}".format( cred_store_ssh_key_password, Frontend.abs_cred_store_key)) chmod_chown(Frontend.abs_cred_store_key, 0o660, grp=Frontend.RTT_ADMIN_GROUP) chmod_chown(Frontend.abs_cred_store_key + ".pub", 0o660, grp=Frontend.RTT_ADMIN_GROUP) with open("{}.pub".format(Frontend.abs_cred_store_key), "r") as pub_key_f: pub_key = pub_key_f.read().rstrip() add_authorized_key_to_server( Storage.ssh_root_user, Storage.address, Storage.ssh_port, pub_key, "{}{}".format( Storage.acc_chroot, os.path.join(Storage.CHROOT_HOME_DIR, Storage.SSH_DIR, Storage.AUTH_KEYS_FILE))) if not args.docker: service_enable("ssh.service") # Everything should be okay now. except BaseException as e: print_error("{}. Fix error and run the script again.".format(e)) traceback.print_exc() return 2
def main(): parser = argparse.ArgumentParser(description='Web view deployment') parser.add_argument('--docker', dest='docker', action='store_const', const=True, default=False, help='Docker deployment') parser.add_argument('--ph4', dest='ph4', action='store_const', const=True, default=False, help='Use Ph4 forks of tools') parser.add_argument('--local-db', dest='local_db', action='store_const', const=True, default=False, help='DB server is on the same machine') parser.add_argument('--mysql-pass', dest='mysql_pass', action='store_const', const=True, default=False, help='DB password to use') parser.add_argument('--mysql-pass-file', dest='mysql_pass_file', action='store_const', const=True, default=False, help='DB password file to use') parser.add_argument('--config', dest='config', default='deployment_settings.ini', help='Path to deployment_settings.ini') args = parser.parse_args() deploy_cfg_file = args.config deploy_cfg = configparser.ConfigParser() current_dir = os.path.abspath(os.path.curdir) try: deploy_cfg.read(deploy_cfg_file) if len(deploy_cfg.sections()) == 0: raise FileNotFoundError("can't read: {}".format(deploy_cfg_file)) Database.address = get_no_empty(deploy_cfg, "Database", "IPv4-Address") Database.mysql_port = get_no_empty(deploy_cfg, "Database", "MySQL-port") Database.ssh_port = get_no_empty(deploy_cfg, "Database", "SSH-Port") Database.ssh_root_user = get_no_empty(deploy_cfg, "Database", "SSH-Root-User") Storage.address = get_no_empty(deploy_cfg, "Storage", "IPv4-Address") Storage.ssh_root_user = get_no_empty(deploy_cfg, "Storage", "SSH-Root-User") Storage.acc_chroot = get_no_empty(deploy_cfg, "Storage", "Storage-Chroot") Storage.storage_user = get_no_empty(deploy_cfg, "Storage", "Storage-User") Storage.ssh_port = get_no_empty(deploy_cfg, "Storage", "SSH-port") RTTWeb.address = get_no_empty(deploy_cfg, "Web", "IPv4-Address") except BaseException as e: print_error("Configuration file: {}".format(e)) sys.exit(1) # Sanity checks try: check_files_exists({ RTTWeb.APACHE_CONFIG, CommonConst.FRONTEND_SUBMIT_EXPERIMENT_SCRIPT }) except AssertionError as e: print_error("Invalid configuration. {}".format(e)) sys.exit(1) try: install_debian_pkgs([ "acl", "sudo", "wget", "unzip", "rsync", "openssh-client", "python3-pip", "python3-venv", "apache2", "libapache2-mod-wsgi-py3", "certbot" ]) python_packages = [ "wheel", "django==2.0.8", "django-bootstrap3", "django-bootstrap-form", "django-datetime-widget", "mysqlclient", "sarge", "requests", "shellescape", "coloredlogs", "filelock", "configparser", "cryptography", "pyinstaller", "filelock", "jsonpath-ng" ] install_python_pkg("pip", no_cache=False) install_python_pkgs(python_packages) # Adding rtt-admin group that is intended to manage # directories and files related to rtt without root access exec_sys_call_check("groupadd {}".format(Frontend.RTT_ADMIN_GROUP), acc_codes=[0, 9]) rtt_admin_grp_gid = grp.getgrnam(Frontend.RTT_ADMIN_GROUP).gr_gid # Adding group for users of rtt exec_sys_call_check("groupadd {}".format(Frontend.RTT_USER_GROUP), acc_codes=[0, 9]) rtt_user_grp_gid = grp.getgrnam(Frontend.RTT_USER_GROUP).gr_gid wusr = '******' wgrp = 'www-data' # Lets encrypt http-auth os.makedirs('/var/www/html/.well-known/acme-challenge', 0o777, True) recursive_chmod_chown('/var/www/html/.well-known', 0o660, 0o771, wusr, wgrp) dst_dir = RTTWeb.RTT_WEB_PATH if os.path.exists(dst_dir): shutil.rmtree(dst_dir) rttweb_repo = RTTWeb.WEB_REPO_PH4 if args.ph4 else RTTWeb.WEB_REPO exec_sys_call_check("git clone --recursive %s %s" % (rttweb_repo, dst_dir)) os.chdir(dst_dir) exec_sys_call_check("python3 -m venv %s" % RTTWeb.RTT_WEB_ENV) pip3_venv = os.path.abspath( os.path.join(RTTWeb.RTT_WEB_ENV, 'bin', 'pip3')) install_python_pkgs(python_packages, pip3=pip3_venv) # Credentials from common.rtt_registration import register_db_user from common.rtt_registration import add_authorized_key_to_server from common.rtt_registration import db_server_cmd, get_db_command credsdir = os.path.join(dst_dir, RTTWeb.RTT_WEB_CREDENTIALS) if os.path.exists(credsdir): shutil.rmtree(credsdir) os.makedirs(credsdir, 0o777, True) sec_key = get_rnd_pwd() sec_file = os.path.join(credsdir, RTTWeb.RTT_WEB_CREDENTIALS_SECRET_KEY) with create_file_wperms(sec_file, mask=0o640, mode='w') as fh: fh.write(sec_key) chmod_chown(sec_file, 0o640, own=wusr, grp=wgrp) # Create database db_def_passwd = get_mysql_password_args(args) print("Creating Web database") create_cmd = get_db_command(Database.MYSQL_ROOT_USERNAME, db_def_passwd, '"CREATE DATABASE %s"' % RTTWeb.MYSQL_DB) db_server_cmd(Database.ssh_root_user, Database.address, Database.ssh_port, command=create_cmd) # Register user for results preview cred_mysql_db_password = get_rnd_pwd() creds_db_path = os.path.join(credsdir, RTTWeb.MYSQL_RTT_CONFIG) creds_db_path2 = os.path.join(credsdir, RTTWeb.MYSQL_RTT_CONFIG2) write_db_credentials(RTTWeb.MYSQL_RTT_USER, cred_mysql_db_password, creds_db_path) shutil.copy(creds_db_path, creds_db_path2) chmod_chown(creds_db_path, 0o640, own=wusr, grp=wgrp) chmod_chown(creds_db_path2, 0o640, own=wusr, grp=wgrp) db_addr_from = RTTWeb.address if not args.docker else '%' register_db_user( Database.ssh_root_user, Database.address, Database.ssh_port, RTTWeb.MYSQL_RTT_USER, cred_mysql_db_password, db_addr_from, Database.MYSQL_ROOT_USERNAME, Database.MYSQL_DB_NAME, priv_select=True, priv_insert=True, # insert for submit_experiment db_def_passwd=db_def_passwd, db_no_pass=args.local_db) # Register web user creds_mysql_web_pass = get_rnd_pwd() creds_db_path_web = os.path.join(credsdir, RTTWeb.WEB_DB_CONFIG) write_db_credentials_web(RTTWeb.MYSQL_USER, creds_mysql_web_pass, RTTWeb.MYSQL_DB, creds_db_path_web, address=Database.address) chmod_chown(creds_db_path_web, 0o640, own=wusr, grp=wgrp) register_db_user(Database.ssh_root_user, Database.address, Database.ssh_port, RTTWeb.MYSQL_USER, creds_mysql_web_pass, db_addr_from, Database.MYSQL_ROOT_USERNAME, RTTWeb.MYSQL_DB, priv_select=True, priv_insert=True, priv_update=True, priv_delete=True, priv_create=True, priv_alter=True, priv_index=True, db_def_passwd=db_def_passwd, db_no_pass=args.local_db) # Register machine to storage rtt_ssh_pkey = os.path.join(credsdir, RTTWeb.SSH_CREDENTIALS_KEY) key_pwd = get_rnd_pwd() exec_sys_call_check("ssh-keygen -q -b 2048 -t rsa -N {} -f {}".format( key_pwd, rtt_ssh_pkey)) chmod_chown(rtt_ssh_pkey, 0o600, own=wusr, grp=wgrp) chmod_chown(rtt_ssh_pkey + ".pub", 0o640, own=wusr, grp=wgrp) with open(rtt_ssh_pkey + ".pub") as f: pub_key = f.read().rstrip() rtt_ssh_cfg = os.path.join(credsdir, RTTWeb.SSH_CREDENTIALS_FILE) write_ssh_credentials(Storage.storage_user, key_pwd, rtt_ssh_pkey, rtt_ssh_cfg) chmod_chown(rtt_ssh_cfg, 0o640, own=wusr, grp=wgrp) authorized_keys_path = "{}{}".format( Storage.acc_chroot, os.path.join(Storage.CHROOT_HOME_DIR, Storage.SSH_DIR, Storage.AUTH_KEYS_FILE)) add_authorized_key_to_server(Storage.ssh_root_user, Storage.address, Storage.ssh_port, pub_key, authorized_keys_path) # Submit experiment submdir = os.path.join(dst_dir, RTTWeb.RTT_WEB_SUBMIT_EXP) if os.path.exists(submdir): shutil.rmtree(submdir) os.makedirs(submdir, 0o771, True) os.chdir(submdir) # - create frontend.ini frontend_ini_cfg = configparser.ConfigParser() frontend_ini_cfg.add_section("MySQL-Database") frontend_ini_cfg.set("MySQL-Database", "Name", Database.MYSQL_DB_NAME) frontend_ini_cfg.set("MySQL-Database", "Address", Database.address) frontend_ini_cfg.set("MySQL-Database", "Port", Database.mysql_port) frontend_ini_cfg.set("MySQL-Database", "Credentials-file", os.path.abspath(creds_db_path)) frontend_ini_cfg.add_section("Storage") frontend_ini_cfg.set("Storage", "Address", Storage.address) frontend_ini_cfg.set("Storage", "Port", Storage.ssh_port) frontend_ini_cfg.set( "Storage", "Data-directory", os.path.join(Storage.CHROOT_HOME_DIR, Storage.CHROOT_DATA_DIR)) frontend_ini_cfg.set( "Storage", "Config-directory", os.path.join(Storage.CHROOT_HOME_DIR, Storage.CHROOT_CONF_DIR)) frontend_ini_cfg.set("Storage", "Credentials-file", os.path.abspath(rtt_ssh_cfg)) with open(Frontend.FRONT_CONFIG_FILE, "w") as f: frontend_ini_cfg.write(f) shutil.copy( os.path.join(current_dir, CommonConst.FRONTEND_SUBMIT_EXPERIMENT_SCRIPT), Frontend.SUBMIT_EXPERIMENT_SCRIPT) chmod_chown(Frontend.SUBMIT_EXPERIMENT_SCRIPT, 0o660, own=wusr, grp=wgrp) if os.path.exists(Frontend.COMMON_FILES_DIR): shutil.rmtree(Frontend.COMMON_FILES_DIR) shutil.copytree( os.path.join(current_dir, CommonConst.COMMON_FILES_DIR), Frontend.COMMON_FILES_DIR) recursive_chmod_chown(Frontend.COMMON_FILES_DIR, mod_f=0o660, mod_d=0o2770, own=wusr, grp=wgrp) submit_exp_base_name = os.path.splitext( Frontend.SUBMIT_EXPERIMENT_SCRIPT)[0] exec_sys_call_check("pyinstaller -F {}".format( Frontend.SUBMIT_EXPERIMENT_SCRIPT)) shutil.move("dist/{}".format(submit_exp_base_name), Frontend.SUBMIT_EXPERIMENT_BINARY) chmod_chown(Frontend.SUBMIT_EXPERIMENT_BINARY, 0o6775, own=wusr, grp=wgrp) shutil.rmtree("dist") shutil.rmtree("build") shutil.rmtree("__pycache__") os.remove("{}.spec".format(submit_exp_base_name)) # Migrate os.chdir(RTTWeb.RTT_WEB_PATH) python_venv = os.path.abspath( os.path.join(RTTWeb.RTT_WEB_PATH, RTTWeb.RTT_WEB_ENV, 'bin', 'python3')) exec_sys_call_check("%s manage.py migrate" % python_venv) # Chown all print("Chmoding...") exec_sys_call_check("chown -R \"%s:%s\" %s" % (wusr, wgrp, dst_dir)) # Apache config file shutil.copy(os.path.join(current_dir, RTTWeb.APACHE_CONFIG), '/etc/apache2/sites-available/000-default.conf') # Restart apache exec_sys_call_check("service apache2 restart") if not args.docker: service_enable("apache2.service") # Everything should be okay now. except BaseException as e: print_error("{}. Fix error and run the script again.".format(e)) traceback.print_exc()
def main(): if len(sys.argv) != 2: print("\nUsage: ./deploy_backend.py <backend-id>\n") print( "<backend-id> must be entered according to config with deployment settings.\n" " Configuration file \"{}\" must\n" " contain one and only one section named\n" " \"Backend-<backend-id>\"\n".format(deploy_cfg_file)) sys.exit(1) deploy_cfg = configparser.ConfigParser() try: deploy_cfg.read(deploy_cfg_file) if len(deploy_cfg.sections()) == 0: raise FileNotFoundError("can't read: {}".format(deploy_cfg_file)) backend_sec = "Backend-" + sys.argv[1] Backend.address = get_no_empty(deploy_cfg, backend_sec, "IPv4-Address") Backend.rtt_files_dir = get_no_empty(deploy_cfg, backend_sec, "RTT-Files-dir") Backend.exec_max_tests = get_no_empty(deploy_cfg, backend_sec, "Maximum-parallel-tests") Backend.exec_test_timeout = get_no_empty(deploy_cfg, backend_sec, "Maximum-seconds-per-test") Database.address = get_no_empty(deploy_cfg, "Database", "IPv4-Address") Database.mysql_port = get_no_empty(deploy_cfg, "Database", "MySQL-port") Database.ssh_port = get_no_empty(deploy_cfg, "Database", "SSH-Port") Database.ssh_root_user = get_no_empty(deploy_cfg, "Database", "SSH-Root-User") Storage.address = get_no_empty(deploy_cfg, "Storage", "IPv4-Address") Storage.ssh_root_user = get_no_empty(deploy_cfg, "Storage", "SSH-Root-User") Storage.acc_chroot = get_no_empty(deploy_cfg, "Storage", "Storage-Chroot") Storage.storage_user = get_no_empty(deploy_cfg, "Storage", "Storage-User") Storage.ssh_port = get_no_empty(deploy_cfg, "Storage", "SSH-port") except Exception as e: print_error("Configuration file: {}".format(e)) sys.exit(1) # Sanity checks try: check_paths_abs({Backend.rtt_files_dir}) check_paths_rel({ Backend.COMMON_FILES_DIR, Backend.CACHE_DATA_DIR, Backend.CACHE_CONFIG_DIR, Backend.CREDENTIALS_DIR, Backend.RTT_EXECUTION_DIR, Backend.RANDOMNESS_TESTING_TOOLKIT_SRC_DIR, Backend.RTT_STATISTICAL_BATTERIES_SRC_DIR, }) check_files_exists({ CommonConst.BACKEND_CLEAN_CACHE_SCRIPT, CommonConst.BACKEND_RUN_JOBS_SCRIPT }) except AssertionError as e: print_error("Invalid configuration. {}".format(e)) sys.exit(1) # Defined absolute paths to directories and files Backend.rand_test_tool_src_dir = \ join(Backend.rtt_files_dir, Backend.RANDOMNESS_TESTING_TOOLKIT_SRC_DIR) Backend.rand_test_tool_dl_zip = \ join(Backend.rtt_files_dir, Backend.RANDOMNESS_TESTING_TOOLKIT_GIT_NAME + ".zip") Backend.stat_batt_src_dir = \ join(Backend.rtt_files_dir, Backend.RTT_STATISTICAL_BATTERIES_SRC_DIR) Backend.stat_batt_dl_zip = \ join(Backend.rtt_files_dir, Backend.RTT_STATISTICAL_BATTERIES_GIT_NAME + ".zip") Backend.common_files_dir = \ join(Backend.rtt_files_dir, Backend.COMMON_FILES_DIR) Backend.cache_conf_dir = \ join(Backend.rtt_files_dir, Backend.CACHE_CONFIG_DIR) Backend.cache_data_dir = \ join(Backend.rtt_files_dir, Backend.CACHE_DATA_DIR) Backend.credentials_dir = \ join(Backend.rtt_files_dir, Backend.CREDENTIALS_DIR) Backend.rtt_exec_dir = \ join(Backend.rtt_files_dir, Backend.RTT_EXECUTION_DIR) Backend.rtt_exec_nist_exp_dir = \ join(Backend.rtt_exec_dir, Backend.NIST_STS_EXPERIMENTS_DIR) Backend.rtt_exec_nist_temp_dir = \ join(Backend.rtt_exec_dir, Backend.NIST_STS_TEMPLATES_DIR) Backend.ssh_store_pkey = \ join(Backend.credentials_dir, Backend.SSH_CREDENTIALS_KEY) Backend.ssh_store_pubkey = \ join(Backend.credentials_dir, Backend.SSH_CREDENTIALS_KEY + ".pub") Backend.run_jobs_path = \ join(Backend.rtt_files_dir, Backend.RUN_JOBS_SCRIPT) Backend.clean_cache_path = \ join(Backend.rtt_files_dir, Backend.CLEAN_CACHE_SCRIPT) Backend.rtt_binary_path = \ join(Backend.rtt_exec_dir, os.path.basename(Backend.RTT_BINARY_PATH)) Backend.mysql_cred_ini_path = \ join(Backend.credentials_dir, Backend.MYSQL_CREDENTIALS_FILE_INI) Backend.mysql_cred_json_path = \ join(Backend.credentials_dir, Backend.MYSQL_CREDENTIALS_FILE_JSON) Backend.ssh_cred_ini_path = \ join(Backend.credentials_dir, Backend.SSH_CREDENTIALS_FILE) Backend.config_ini_path = \ join(Backend.rtt_files_dir, Backend.BACKEND_CONFIG_FILE) try: # Adding rtt-admin group that is intended to manage # directories and files related to rtt without root access exec_sys_call_check("groupadd {}".format(Backend.RTT_ADMIN_GROUP), acc_codes=[0, 9]) # Remove directories that was created previously if os.path.exists(Backend.rtt_files_dir): shutil.rmtree(Backend.rtt_files_dir) # Create and copy needed files into rtt-files create_dir(Backend.rtt_files_dir, 0o2770, grp=Backend.RTT_ADMIN_GROUP) # Set ACL on top directory - ensures all new files will have correct permissions exec_sys_call_check("setfacl -R -d -m g::rwx {}".format( Backend.rtt_files_dir)) exec_sys_call_check("setfacl -R -d -m o::--- {}".format( Backend.rtt_files_dir)) create_dir(Backend.cache_conf_dir, 0o2770, grp=Backend.RTT_ADMIN_GROUP) create_dir(Backend.cache_data_dir, 0o2770, grp=Backend.RTT_ADMIN_GROUP) create_dir(Backend.credentials_dir, 0o2770, grp=Backend.RTT_ADMIN_GROUP) create_dir(Backend.rtt_exec_dir, 0o2770, grp=Backend.RTT_ADMIN_GROUP) shutil.copy(CommonConst.BACKEND_CLEAN_CACHE_SCRIPT, Backend.clean_cache_path) chmod_chown(Backend.clean_cache_path, 0o770, grp=Backend.RTT_ADMIN_GROUP) shutil.copy(CommonConst.BACKEND_RUN_JOBS_SCRIPT, Backend.run_jobs_path) chmod_chown(Backend.run_jobs_path, 0o770, grp=Backend.RTT_ADMIN_GROUP) if os.path.exists(Backend.common_files_dir): shutil.rmtree(Backend.common_files_dir) shutil.copytree(CommonConst.COMMON_FILES_DIR, Backend.common_files_dir) recursive_chmod_chown(Backend.common_files_dir, mod_f=0o660, mod_d=0o2770, grp=Backend.RTT_ADMIN_GROUP) # Install packages install_debian_pkg("mailutils") install_debian_pkg("postfix") install_debian_pkg("libmysqlcppconn-dev") install_debian_pkg("libmysqlclient-dev") install_debian_pkg("python3-pip") install_debian_pkg("python3-cryptography") install_debian_pkg("python3-paramiko") install_python_pkg("mysqlclient") # Get current versions of needed tools from git # Statistical batteries if os.path.exists(Backend.stat_batt_src_dir): shutil.rmtree(Backend.stat_batt_src_dir) exec_sys_call_check("wget {} -O {}".format( Backend.RTT_STATISTICAL_BATTERIES_ZIP_URL, Backend.stat_batt_dl_zip)) exec_sys_call_check("unzip {} -d {}".format(Backend.stat_batt_dl_zip, Backend.rtt_files_dir)) os.remove(Backend.stat_batt_dl_zip) os.rename( join(Backend.rtt_files_dir, Backend.RTT_STATISTICAL_BATTERIES_GIT_NAME), Backend.stat_batt_src_dir) # Randomness testing toolkit if os.path.exists(Backend.rand_test_tool_src_dir): shutil.rmtree(Backend.rand_test_tool_src_dir) exec_sys_call_check("wget {} -O {}".format( Backend.RANDOMNESS_TESTING_TOOLKIT_ZIP_URL, Backend.rand_test_tool_dl_zip)) exec_sys_call_check("unzip {} -d {}".format( Backend.rand_test_tool_dl_zip, Backend.rtt_files_dir)) os.remove(Backend.rand_test_tool_dl_zip) os.rename( join(Backend.rtt_files_dir, Backend.RANDOMNESS_TESTING_TOOLKIT_GIT_NAME), Backend.rand_test_tool_src_dir) # Change into directory rtt-src and rtt-stat-batt-src and call make and ./INSTALL respectively. current_dir = os.path.abspath(os.path.curdir) # Build statistical batteries os.chdir(Backend.stat_batt_src_dir) exec_sys_call_check("./INSTALL") recursive_chmod_chown(Backend.stat_batt_src_dir, mod_f=0o660, mod_d=0o2770, grp=Backend.RTT_ADMIN_GROUP) chmod_chown(Backend.DIEHARDER_BINARY_PATH, 0o770) chmod_chown(Backend.NIST_STS_BINARY_PATH, 0o770) chmod_chown(Backend.TESTU01_BINARY_PATH, 0o770) # Build randomness testing toolkit os.chdir(Backend.rand_test_tool_src_dir) exec_sys_call_check("make") recursive_chmod_chown(Backend.rand_test_tool_src_dir, mod_f=0o660, mod_d=0o2770, grp=Backend.RTT_ADMIN_GROUP) chmod_chown(Backend.RTT_BINARY_PATH, 0o770) # Build finished, go into original directory os.chdir(current_dir) # Link rtt binary into execution directory os.symlink( join(Backend.rand_test_tool_src_dir, Backend.RTT_BINARY_PATH), Backend.rtt_binary_path) # Copy needed directories and files into execution directory shutil.copytree( join(Backend.stat_batt_src_dir, Backend.NIST_STS_TEMPLATES_DIR), join(Backend.rtt_exec_dir, os.path.basename(Backend.NIST_STS_TEMPLATES_DIR))) shutil.copytree( join(Backend.stat_batt_src_dir, Backend.NIST_STS_EXPERIMENTS_DIR), join(Backend.rtt_exec_dir, os.path.basename(Backend.NIST_STS_EXPERIMENTS_DIR))) rtt_settings = { "toolkit-settings": { "logger": { "dir-prefix": join(Backend.rtt_files_dir, Backend.EXEC_LOGS_TOP_DIR), "run-log-dir": Backend.EXEC_LOGS_RUN_LOG_DIR, "dieharder-dir": Backend.EXEC_LOGS_DIEHARDER_DIR, "nist-sts-dir": Backend.EXEC_LOGS_NIST_STS_DIR, "tu01-smallcrush-dir": Backend.EXEC_LOGS_SMALLCRUSH_DIR, "tu01-crush-dir": Backend.EXEC_LOGS_CRUSH_DIR, "tu01-bigcrush-dir": Backend.EXEC_LOGS_BIGCRUSH_DIR, "tu01-rabbit-dir": Backend.EXEC_LOGS_RABBIT_DIR, "tu01-alphabit-dir": Backend.EXEC_LOGS_ALPHABIT_DIR, "tu01-blockalphabit-dir": Backend.EXEC_LOGS_BLOCKALPHABIT_DIR }, "result-storage": { "file": { "main-file": join(Backend.rtt_files_dir, Backend.EXEC_REPS_MAIN_FILE), "dir-prefix": join(Backend.rtt_files_dir, Backend.EXEC_REPS_TOP_DIR), "dieharder-dir": Backend.EXEC_REPS_DIEHARDER_DIR, "nist-sts-dir": Backend.EXEC_REPS_NIST_STS_DIR, "tu01-smallcrush-dir": Backend.EXEC_REPS_SMALLCRUSH_DIR, "tu01-crush-dir": Backend.EXEC_REPS_CRUSH_DIR, "tu01-bigcrush-dir": Backend.EXEC_REPS_BIGCRUSH_DIR, "tu01-rabbit-dir": Backend.EXEC_REPS_RABBIT_DIR, "tu01-alphabit-dir": Backend.EXEC_REPS_ALPHABIT_DIR, "tu01-blockalphabit-dir": Backend.EXEC_REPS_ALPHABIT_DIR }, "mysql-db": { "address": Database.address, "port": Database.mysql_port, "name": Database.MYSQL_DB_NAME, "credentials-file": Backend.mysql_cred_json_path } }, "binaries": { "nist-sts": join(Backend.stat_batt_src_dir, Backend.NIST_STS_BINARY_PATH), "dieharder": join(Backend.stat_batt_src_dir, Backend.DIEHARDER_BINARY_PATH), "testu01": join(Backend.stat_batt_src_dir, Backend.TESTU01_BINARY_PATH) }, "miscellaneous": { "nist-sts": { "main-result-dir": join(Backend.rtt_exec_dir, Backend.NIST_MAIN_RESULT_DIR) } }, "execution": { "max-parallel-tests": int(Backend.exec_max_tests), "test-timeout-seconds": int(Backend.exec_test_timeout) } } } with open(join(Backend.rtt_exec_dir, Backend.RTT_SETTINGS_JSON), "w") as f: json.dump(rtt_settings, f, indent=4) # Get email configuration # Add configuration to file # inet_interface = loopback-only # inet_protocol = ipv4 with open(Backend.POSTFIX_CFG_PATH) as mail_cfg: for line in mail_cfg.readlines(): if line.startswith(Backend.POSTFIX_HOST_OPT): Backend.sender_email = line.split(sep=" = ")[1] if Backend.sender_email is None: print_error("can't find option {} in file {}".format( Backend.POSTFIX_CFG_PATH, Backend.POSTFIX_HOST_OPT)) sys.exit(1) Backend.sender_email = "root@" + Backend.sender_email # Create backend configuration file backend_ini_cfg = configparser.ConfigParser() backend_ini_cfg.add_section("MySQL-Database") backend_ini_cfg.set("MySQL-Database", "Address", Database.address) backend_ini_cfg.set("MySQL-Database", "Port", Database.mysql_port) backend_ini_cfg.set("MySQL-Database", "Name", Database.MYSQL_DB_NAME) backend_ini_cfg.set("MySQL-Database", "Credentials-file", Backend.mysql_cred_ini_path) backend_ini_cfg.add_section("Local-cache") backend_ini_cfg.set("Local-cache", "Data-directory", Backend.cache_data_dir) backend_ini_cfg.set("Local-cache", "Config-directory", Backend.cache_conf_dir) backend_ini_cfg.add_section("Backend") backend_ini_cfg.set("Backend", "Sender-email", Backend.sender_email) backend_ini_cfg.add_section("Storage") backend_ini_cfg.set("Storage", "Address", Storage.address) backend_ini_cfg.set("Storage", "Port", Storage.ssh_port) backend_ini_cfg.set( "Storage", "Data-directory", join(Storage.CHROOT_HOME_DIR, Storage.CHROOT_DATA_DIR)) backend_ini_cfg.set( "Storage", "Config-directory", join(Storage.CHROOT_HOME_DIR, Storage.CHROOT_CONF_DIR)) backend_ini_cfg.set("Storage", "Credentials-file", Backend.ssh_cred_ini_path) backend_ini_cfg.add_section("RTT-Binary") backend_ini_cfg.set("RTT-Binary", "Binary-path", Backend.rtt_binary_path) with open(Backend.config_ini_path, "w") as f: backend_ini_cfg.write(f) from common.rtt_registration import register_db_user from common.rtt_registration import add_authorized_key_to_server # Register machine to database db_pwd = get_rnd_pwd() cred_mysql_db_ini = configparser.ConfigParser() cred_mysql_db_ini.add_section("Credentials") cred_mysql_db_ini.set("Credentials", "Username", Backend.MYSQL_BACKEND_USER) cred_mysql_db_ini.set("Credentials", "Password", db_pwd) with open(Backend.mysql_cred_ini_path, "w") as f: cred_mysql_db_ini.write(f) cred_mysql_db_json = { "credentials": { "username": Backend.MYSQL_BACKEND_USER, "password": db_pwd } } with open(Backend.mysql_cred_json_path, "w") as f: json.dump(cred_mysql_db_json, f, indent=4) register_db_user(Database.ssh_root_user, Database.address, Database.ssh_port, Backend.MYSQL_BACKEND_USER, db_pwd, Backend.address, Database.MYSQL_ROOT_USERNAME, Database.MYSQL_DB_NAME, priv_select=True, priv_insert=True, priv_update=True) # Register machine to storage key_pwd = get_rnd_pwd() exec_sys_call_check("ssh-keygen -q -b 2048 -t rsa -N {} -f {}".format( key_pwd, Backend.ssh_store_pkey)) chmod_chown(Backend.ssh_store_pkey, 0o660, grp=Backend.RTT_ADMIN_GROUP) chmod_chown(Backend.ssh_store_pubkey, 0o660, grp=Backend.RTT_ADMIN_GROUP) with open(Backend.ssh_store_pubkey) as f: pub_key = f.read().rstrip() cred_ssh_store_ini = configparser.ConfigParser() cred_ssh_store_ini.add_section("Credentials") cred_ssh_store_ini.set("Credentials", "Username", Storage.storage_user) cred_ssh_store_ini.set("Credentials", "Private-key-file", Backend.ssh_store_pkey) cred_ssh_store_ini.set("Credentials", "Private-key-password", key_pwd) with open(Backend.ssh_cred_ini_path, "w") as f: cred_ssh_store_ini.write(f) add_authorized_key_to_server( Storage.ssh_root_user, Storage.address, Storage.ssh_port, pub_key, "{}{}".format( Storage.acc_chroot, join(Storage.CHROOT_HOME_DIR, Storage.SSH_DIR, Storage.AUTH_KEYS_FILE))) # Add cron jobs for cache cleaning and job running script add_cron_job(Backend.clean_cache_path, Backend.config_ini_path, join(Backend.rtt_files_dir, Backend.CLEAN_CACHE_LOG)) add_cron_job(Backend.run_jobs_path, Backend.config_ini_path, join(Backend.rtt_files_dir, Backend.RUN_JOBS_LOG)) except BaseException as e: print_error("{}. Fix error and run the script again.".format(e))
def main(): deploy_cfg = configparser.ConfigParser() try: deploy_cfg.read(deploy_cfg_file) if len(deploy_cfg.sections()) == 0: raise FileNotFoundError("can't read: {}".format(deploy_cfg_file)) Frontend.address = get_no_empty(deploy_cfg, "Frontend", "IPv4-Address") Frontend.rtt_users_chroot = get_no_empty(deploy_cfg, "Frontend", "RTT-Users-Chroot") Frontend.ssh_config = get_no_empty(deploy_cfg, "Frontend", "SSH-Config") Database.address = get_no_empty(deploy_cfg, "Database", "IPv4-Address") Database.mysql_port = get_no_empty(deploy_cfg, "Database", "MySQL-port") Database.ssh_port = get_no_empty(deploy_cfg, "Database", "SSH-Port") Database.ssh_root_user = get_no_empty(deploy_cfg, "Database", "SSH-Root-User") Storage.address = get_no_empty(deploy_cfg, "Storage", "IPv4-Address") Storage.ssh_root_user = get_no_empty(deploy_cfg, "Storage", "SSH-Root-User") Storage.acc_chroot = get_no_empty(deploy_cfg, "Storage", "Storage-Chroot") Storage.storage_user = get_no_empty(deploy_cfg, "Storage", "Storage-User") Storage.ssh_port = get_no_empty(deploy_cfg, "Storage", "SSH-port") except BaseException as e: print_error("Configuration file: {}".format(e)) sys.exit(1) # Sanity checks try: check_paths_abs({ Frontend.rtt_users_chroot, Frontend.CHROOT_RTT_FILES, Frontend.CHROOT_RTT_USERS_HOME }) check_paths_rel({ Frontend.CREDENTIALS_DIR, Frontend.COMMON_FILES_DIR, Frontend.SSH_DIR }) check_files_exists({ Frontend.ssh_config, Frontend.FSTAB_FILE, CommonConst.FRONTEND_SUBMIT_EXPERIMENT_SCRIPT, CommonConst.FRONTEND_ADD_USER_SCRIPT }) except AssertionError as e: print_error("Invalid configuration. {}".format(e)) sys.exit(1) # Setting of paths used in this script Frontend.abs_rtt_files = \ Frontend.rtt_users_chroot + Frontend.CHROOT_RTT_FILES Frontend.abs_config_ini = \ os.path.join(Frontend.abs_rtt_files, Frontend.FRONT_CONFIG_FILE) Frontend.abs_submit_exp_script = \ os.path.join(Frontend.abs_rtt_files, Frontend.SUBMIT_EXPERIMENT_SCRIPT) Frontend.abs_add_user_script = \ os.path.join(Frontend.abs_rtt_files, Frontend.ADD_USER_SCRIPT) Frontend.submit_exp_base_name = \ os.path.splitext(Frontend.SUBMIT_EXPERIMENT_SCRIPT)[0] Frontend.abs_cred_dir = \ os.path.join(Frontend.abs_rtt_files, Frontend.CREDENTIALS_DIR) Frontend.rel_cred_dir = \ os.path.join(Frontend.CHROOT_RTT_FILES, Frontend.CREDENTIALS_DIR) Frontend.abs_common_files = \ os.path.join(Frontend.abs_rtt_files, Frontend.COMMON_FILES_DIR) Frontend.abs_cred_mysql_ini = \ os.path.join(Frontend.abs_cred_dir, Frontend.MYSQL_CREDENTIALS_FILE) Frontend.rel_cred_mysql_ini = \ os.path.join(Frontend.rel_cred_dir, Frontend.MYSQL_CREDENTIALS_FILE) Frontend.abs_cred_store_ini = \ os.path.join(Frontend.abs_cred_dir, Frontend.SSH_CREDENTIALS_FILE) Frontend.rel_cred_store_ini = \ os.path.join(Frontend.rel_cred_dir, Frontend.SSH_CREDENTIALS_FILE) Frontend.abs_cred_store_key = \ os.path.join(Frontend.abs_cred_dir, Frontend.SSH_CREDENTIALS_KEY) Frontend.rel_cred_store_key = \ os.path.join(Frontend.rel_cred_dir, Frontend.SSH_CREDENTIALS_KEY) try: # Adding rtt-admin group that is intended to manage # directories and files related to rtt without root access exec_sys_call_check("groupadd {}".format(Frontend.RTT_ADMIN_GROUP), acc_codes=[0, 9]) rtt_admin_grp_gid = grp.getgrnam(Frontend.RTT_ADMIN_GROUP).gr_gid # Adding group for users of rtt exec_sys_call_check("groupadd {}".format(Frontend.RTT_USER_GROUP), acc_codes=[0, 9]) rtt_user_grp_gid = grp.getgrnam(Frontend.RTT_USER_GROUP).gr_gid # Installing debootstrap used for ssh jail install_debian_pkg("debootstrap") # Delete chroot directory if it exists if os.path.exists(Frontend.rtt_users_chroot): shutil.rmtree(Frontend.rtt_users_chroot) # Building chroot jail for rtt users create_dir(Frontend.rtt_users_chroot, 0o775, grp=Frontend.RTT_ADMIN_GROUP) exec_sys_call_check("debootstrap {} {}".format( Frontend.CHROOT_DEBIAN_VERSION, Frontend.rtt_users_chroot)) with open(Frontend.FSTAB_FILE, "a") as f: f.write("proc {} proc defaults 0 0\n".format( os.path.join(Frontend.rtt_users_chroot, "proc"))) f.write("sysfs {} sysfs defaults 0 0\n".format( os.path.join(Frontend.rtt_users_chroot, "sys"))) exec_sys_call_check("mount proc {} -t proc".format( os.path.join(Frontend.rtt_users_chroot, "proc"))) exec_sys_call_check("mount sysfs {} -t sysfs".format( os.path.join(Frontend.rtt_users_chroot, "sys"))) shutil.copy("/etc/hosts", os.path.join(Frontend.rtt_users_chroot, "etc/hosts")) create_dir(Frontend.abs_rtt_files, 0o2775, grp=Frontend.RTT_ADMIN_GROUP) # Set ACL on top directory - ensures all new files will have correct permissions exec_sys_call_check("setfacl -R -d -m g::rwx {}".format( Frontend.abs_rtt_files)) exec_sys_call_check("setfacl -R -d -m o::--- {}".format( Frontend.abs_rtt_files)) create_dir(Frontend.abs_cred_dir, 0o2770, grp=Frontend.RTT_ADMIN_GROUP) frontend_ini_cfg = configparser.ConfigParser() frontend_ini_cfg.add_section("MySQL-Database") frontend_ini_cfg.set("MySQL-Database", "Name", Database.MYSQL_DB_NAME) frontend_ini_cfg.set("MySQL-Database", "Address", Database.address) frontend_ini_cfg.set("MySQL-Database", "Port", Database.mysql_port) frontend_ini_cfg.set("MySQL-Database", "Credentials-file", Frontend.rel_cred_mysql_ini) frontend_ini_cfg.add_section("Storage") frontend_ini_cfg.set("Storage", "Address", Storage.address) frontend_ini_cfg.set("Storage", "Port", Storage.ssh_port) frontend_ini_cfg.set( "Storage", "Data-directory", os.path.join(Storage.CHROOT_HOME_DIR, Storage.CHROOT_DATA_DIR)) frontend_ini_cfg.set( "Storage", "Config-directory", os.path.join(Storage.CHROOT_HOME_DIR, Storage.CHROOT_CONF_DIR)) frontend_ini_cfg.set("Storage", "Credentials-file", Frontend.rel_cred_store_ini) frontend_ini_cfg.add_section("Frontend") frontend_ini_cfg.set("Frontend", "RTT-Users-Chroot", Frontend.rtt_users_chroot) with open(Frontend.abs_config_ini, "w") as f: frontend_ini_cfg.write(f) shutil.copy(CommonConst.FRONTEND_SUBMIT_EXPERIMENT_SCRIPT, Frontend.abs_submit_exp_script) chmod_chown(Frontend.abs_submit_exp_script, 0o660, grp=Frontend.RTT_ADMIN_GROUP) shutil.copy(CommonConst.FRONTEND_ADD_USER_SCRIPT, Frontend.abs_add_user_script) chmod_chown(Frontend.abs_add_user_script, 0o770, grp=Frontend.RTT_ADMIN_GROUP) if os.path.exists(Frontend.abs_common_files): shutil.rmtree(Frontend.abs_common_files) shutil.copytree(CommonConst.COMMON_FILES_DIR, Frontend.abs_common_files) recursive_chmod_chown(Frontend.abs_common_files, mod_f=0o660, mod_d=0o2770, grp=Frontend.RTT_ADMIN_GROUP) # Entering chroot jail real_root = os.open("/", os.O_RDONLY) os.chroot(Frontend.rtt_users_chroot) # Adding groups - instead there should be two-way sync!!! exec_sys_call_check("groupadd -g {} {}".format( rtt_admin_grp_gid, Frontend.RTT_ADMIN_GROUP), acc_codes=[0, 9]) exec_sys_call_check("groupadd -g {} {}".format( rtt_user_grp_gid, Frontend.RTT_USER_GROUP), acc_codes=[0, 9]) # Installing needed packages inside jail install_debian_pkg("python3") install_debian_pkg("python3-dev") install_debian_pkg("python3-setuptools") install_debian_pkg("libmysqlclient-dev") install_debian_pkg("build-essential") install_debian_pkg("python3-cryptography") install_debian_pkg("python3-paramiko") install_debian_pkg("python3-pip") install_python_pkg("pyinstaller") install_python_pkg("mysqlclient") os.chdir(Frontend.CHROOT_RTT_FILES) exec_sys_call_check("pyinstaller -F {}".format( Frontend.SUBMIT_EXPERIMENT_SCRIPT)) shutil.move("dist/{}".format(Frontend.submit_exp_base_name), Frontend.SUBMIT_EXPERIMENT_BINARY) chmod_chown(Frontend.SUBMIT_EXPERIMENT_BINARY, 0o2775, grp=Frontend.RTT_ADMIN_GROUP) shutil.rmtree("dist") shutil.rmtree("build") shutil.rmtree("__pycache__") os.remove("{}.spec".format(Frontend.submit_exp_base_name)) # Exiting chroot jail os.fchdir(real_root) os.chroot(".") os.close(real_root) sshd_config_append = "\n\n\n\n" \ "Match Group {0}\n" \ "\tChrootDirectory {1}\n" \ "\tPasswordAuthentication yes\n" \ "\tAllowTcpForwarding no\n" \ "\tPermitTunnel no\n" \ "\tX11Forwarding no\n" \ "\tAuthorizedKeysFile {1}{2}\n" \ "\n".format(Frontend.RTT_USER_GROUP, Frontend.rtt_users_chroot, os.path.join(Frontend.CHROOT_RTT_USERS_HOME, "%u", Frontend.SSH_DIR, Frontend.AUTH_KEYS_FILE)) with open(Frontend.ssh_config, "a") as f: f.write(sshd_config_append) exec_sys_call_check("service sshd restart") install_debian_pkg("python3-cryptography") install_debian_pkg("python3-paramiko") from common.rtt_registration import register_db_user from common.rtt_registration import add_authorized_key_to_server # Register frontend user at the database cred_mysql_db_password = get_rnd_pwd() cred_mysql_db_cfg = configparser.ConfigParser() cred_mysql_db_cfg.add_section("Credentials") cred_mysql_db_cfg.set("Credentials", "Username", Frontend.MYSQL_FRONTEND_USER) cred_mysql_db_cfg.set("Credentials", "Password", cred_mysql_db_password) with open(Frontend.abs_cred_mysql_ini, "w") as f: cred_mysql_db_cfg.write(f) register_db_user(Database.ssh_root_user, Database.address, Database.ssh_port, Frontend.MYSQL_FRONTEND_USER, cred_mysql_db_password, Frontend.address, Database.MYSQL_ROOT_USERNAME, Database.MYSQL_DB_NAME, priv_insert=True, priv_select=True) # Register frontend at the storage cred_store_ssh_key_password = get_rnd_pwd() cred_store_ssh_cfg = configparser.ConfigParser() cred_store_ssh_cfg.add_section("Credentials") cred_store_ssh_cfg.set("Credentials", "Username", Storage.storage_user) cred_store_ssh_cfg.set("Credentials", "Private-key-file", Frontend.rel_cred_store_key) cred_store_ssh_cfg.set("Credentials", "Private-key-password", cred_store_ssh_key_password) with open(Frontend.abs_cred_store_ini, "w") as f: cred_store_ssh_cfg.write(f) exec_sys_call_check("ssh-keygen -q -b 2048 -t rsa -N {} -f {}".format( cred_store_ssh_key_password, Frontend.abs_cred_store_key)) chmod_chown(Frontend.abs_cred_store_key, 0o660, grp=Frontend.RTT_ADMIN_GROUP) chmod_chown(Frontend.abs_cred_store_key + ".pub", 0o660, grp=Frontend.RTT_ADMIN_GROUP) with open("{}.pub".format(Frontend.abs_cred_store_key), "r") as pub_key_f: pub_key = pub_key_f.read().rstrip() add_authorized_key_to_server( Storage.ssh_root_user, Storage.address, Storage.ssh_port, pub_key, "{}{}".format( Storage.acc_chroot, os.path.join(Storage.CHROOT_HOME_DIR, Storage.SSH_DIR, Storage.AUTH_KEYS_FILE))) # Everything should be okay now. except BaseException as e: print_error("{}. Fix error and run the script again.".format(e))