def install_problem(problem_path, allow_reinstall=False): """ Install a problem from a source directory. Args: problem_path: path to the problem source directory """ problem_obj = get_problem(problem_path) if ( os.path.isdir(get_problem_root_hashed(problem_obj, absolute=True)) and not allow_reinstall ): logger.error( f"Problem {problem_obj['unique_name']} is already installed. You may specify --reinstall to reinstall an updated version from the specified directory." ) return logger.info(f"Installing problem {problem_obj['unique_name']}...") acquire_lock() staging_dir_path = generate_staging_directory( problem_name=problem_obj["unique_name"] ) logger.debug( f"{problem_obj['unique_name']}: created staging directory" + f" ({staging_dir_path})" ) generated_deb_path = package_problem( problem_path, staging_path=staging_dir_path, out_path=DEB_ROOT ) logger.debug(f"{problem_obj['unique_name']}: created debian package") try: subprocess.run( "DEBIAN_FRONTEND=noninteractive apt-get -y install " + f"--reinstall {generated_deb_path}", shell=True, check=True, stdout=subprocess.PIPE, ) except subprocess.CalledProcessError: logger.error("An error occurred while installing problem packages.") raise FatalException finally: release_lock() logger.debug(f"{problem_obj['unique_name']}: installed package") logger.info(f"{problem_obj['unique_name']} installed successfully")
def deploy_problems(args, config): """ Main entrypoint for problem deployment """ global deploy_config, port_map, inv_port_map deploy_config = config need_restart_xinetd = False try: user = getpwnam(deploy_config.default_user) except KeyError as e: logger.info("default_user '%s' does not exist. Creating the user now.", deploy_config.default_user) create_user(deploy_config.default_user) if args.deployment_directory is not None and (len(args.problem_paths) > 1 or args.num_instances > 1): logger.error( "Cannot specify deployment directory if deploying multiple problems or instances." ) raise FatalException if args.secret: deploy_config.deploy_secret = args.secret logger.warning( "Overriding deploy_secret with user supplied secret '%s'.", args.secret) problem_names = args.problem_paths if args.bundle: bundle_problems = [] for bundle_path in args.problem_paths: if os.path.isfile(bundle_path): bundle = get_bundle(bundle_path) bundle_problems.extend(bundle["problems"]) else: bundle_sources_path = get_bundle_root( bundle_path, absolute=True) if os.path.isdir(bundle_sources_path): bundle = get_bundle(bundle_sources_path) bundle_problems.extend(bundle["problems"]) else: logger.error("Could not find bundle at '%s'.", bundle_path) raise FatalException problem_names = bundle_problems # before deploying problems, load in port_map and already_deployed instances already_deployed = {} for path, problem in get_all_problems().items(): already_deployed[path] = [] for instance in get_all_problem_instances(path): already_deployed[path].append(instance["instance_number"]) if "port" in instance: port_map[instance["port"]] = (problem["name"], instance["instance_number"]) inv_port_map[(problem["name"], instance["instance_number"])] = instance["port"] lock_file = join(HACKSPORTS_ROOT, "deploy.lock") if os.path.isfile(lock_file): logger.error( "Cannot deploy while other deployment in progress. If you believe this is an error, " "run 'shell_manager clean'") raise FatalException logger.debug("Obtaining deployment lock file %s", lock_file) with open(lock_file, "w") as f: f.write("1") if args.instances: instance_list = args.instances else: instance_list = list(range(0, args.num_instances)) try: for problem_name in problem_names: if isdir(get_problem_root(problem_name, absolute=True)): # problem_name is already an installed package deploy_location = get_problem_root(problem_name, absolute=True) elif isdir(problem_name) and args.dry: # dry run - avoid installing package deploy_location = problem_name elif isdir(problem_name): # problem_name is a source dir - convert to .deb and install try: if not os.path.isdir(TEMP_DEB_DIR): os.mkdir(TEMP_DEB_DIR) generated_deb_path = package_problem(problem_name, out_path=TEMP_DEB_DIR) except FatalException: logger.error("An error occurred while packaging %s.", problem_name) raise try: # reinstall flag ensures package will be overwritten if version is the same, # maintaining previous 'dpkg -i' behavior subprocess.run('apt-get install --reinstall {}'.format(generated_deb_path), shell=True, check=True, stdout=subprocess.PIPE) except subprocess.CalledProcessError: logger.error("An error occurred while installing problem packages.") raise FatalException deploy_location = get_problem_root_hashed(get_problem(problem_name), absolute=True) else: logger.error("'%s' is neither an installed package, nor a valid problem directory", problem_name) raise FatalException # Avoid redeploying already-deployed instances if args.redeploy: todo_instance_list = instance_list else: todo_instance_list = list( set(instance_list) - set(already_deployed.get(problem_name, []))) need_restart_xinetd = deploy_problem( deploy_location, instances=todo_instance_list, test=args.dry, deployment_directory=args.deployment_directory, debug=args.debug, restart_xinetd=False) finally: # Restart xinetd unless specified. Service must be manually restarted if not args.no_restart and need_restart_xinetd: execute(["service", "xinetd", "restart"], timeout=60) logger.debug("Releasing lock file %s", lock_file) os.remove(lock_file)
def postinst_dependencies(problem, problem_path, debian_path, install_path): """ Handles the generation of the postinst script for additional dependencies. Args: problem: the problem object. problem_path: the problem directory. debian_path: the deb's DEBIAN directory. """ postinst_template = ["#!/bin/bash"] requirements_path = join(problem_path, "requirements.txt") dependencies_path = join(problem_path, "install_dependencies") staging_requirements_path = join(install_path, "requirements.txt") deployed_requirements_path = join( get_problem_root_hashed(problem, absolute=True), "__files", "requirements.txt") deployed_setup_path = join( get_problem_root_hashed(problem, absolute=True), "__files", "install_dependencies", ) listed_requirements = problem.get("pip_requirements", []) pip_python_version = problem.get("pip_python_version") valid_pip_python_versions = ["2", "3", "3.7"] if pip_python_version not in valid_pip_python_versions: pip_python_version = "3" # Write or copy the requirements to the staging directory. if len(listed_requirements) > 0: if isfile(requirements_path): logger.error( "Problem '%s' has both a pip_requirements field and requirements.txt.", problem["name"], ) raise FatalException with open(staging_requirements_path, "w") as f: f.writelines("\n".join(listed_requirements)) elif isfile(requirements_path): copy(requirements_path, staging_requirements_path) if logger.getEffectiveLevel() <= logging.DEBUG and isfile( staging_requirements_path): with open(staging_requirements_path, "r") as f: logger.debug("python requirements:\n%s", f.read()) if isfile(staging_requirements_path): postinst_template.append("python{ver} -m pip install -r {path}".format( ver=pip_python_version, path=deployed_requirements_path)) if isfile(dependencies_path): copy(dependencies_path, join(install_path, "install_dependencies")) # Ensure it is executable chmod(join(install_path, "install_dependencies"), 0o500) postinst_template.append("bash -c '{}'".format(deployed_setup_path)) chmod(debian_path, 0o775) postinst_path = join(debian_path, "postinst") with open(postinst_path, "w") as f: chmod(postinst_path, 0o775) contents = "\n".join(postinst_template) f.write(contents) # post_template always has a she-bang. if len(postinst_template) > 1: logger.debug("post install:\n%s", contents)
def package_problem(problem_path, staging_path=None, out_path=None, ignore_files=None): """ Does the work of packaging a single problem. Args: problem_path (str): path to the problem directory staging_path (str, optional): path to a temporary. staging directory for packaging this problem. out_path (str, optional): path to an output directory for the resultant .deb package. ignore_files (list of str, optional): filenames to exclude when packaging this problem. Returns: str: the absolute path to the packaged problem """ if ignore_files is None: ignore_files = [] problem = get_problem(problem_path) logger.debug("Starting to package: '%s'.", problem["name"]) # Create staging directories needed for packaging paths = {} if staging_path is None: paths["staging"] = join(problem_path, "__staging") else: paths["staging"] = join(staging_path, "__staging") paths["debian"] = join(paths["staging"], "DEBIAN") paths["data"] = join(paths["staging"], get_problem_root_hashed(problem)) paths["install_data"] = join(paths["data"], "__files") for path in paths.values(): if not isdir(path): makedirs(path) # Copy the problem files to the staging directory ignore_files.append("__staging") full_copy(problem_path, paths["data"], ignore=ignore_files) # note that this chmod does not work correct if on a vagrant shared folder, # so we need to package the problems elsewhere chmod(paths["data"], 0o750) problem_to_control(problem, paths["debian"]) postinst_dependencies(problem, problem_path, paths["debian"], paths["install_data"]) # Package the staging directory as a .deb def format_deb_file_name(problem): """ Prepare the file name of the deb package according to deb policy. Args: problem: the problem object Returns: An acceptable file name for the problem. """ raw_package_name = "{}.deb".format( sanitize_name(problem["unique_name"])) return raw_package_name deb_directory = out_path if out_path is not None else getcwd() deb_path = join(deb_directory, format_deb_file_name(problem)) shell = spur.LocalShell() result = shell.run( ["fakeroot", "dpkg-deb", "--build", paths["staging"], deb_path]) if result.return_code != 0: logger.error("Error building problem deb for '%s'.", problem["name"]) logger.error(result.output) raise FatalException else: logger.debug("Problem '%s' packaged successfully.", problem["unique_name"]) # Remove the staging directory logger.debug("Cleaning up '%s' staging directory '%s'.", problem["name"], paths["staging"]) rmtree(paths["staging"]) return os.path.abspath(deb_path)