def install_problem(problem_path, allow_reinstall=False):
"""
Install a problem from a source directory.
Args:
problem_path: path to the problem source directory
"""
problem_obj = get_problem(problem_path)
if (
os.path.isdir(get_problem_root_hashed(problem_obj, absolute=True))
and not allow_reinstall
):
logger.error(
f"Problem {problem_obj['unique_name']} is already installed. You may specify --reinstall to reinstall an updated version from the specified directory."
)
return
logger.info(f"Installing problem {problem_obj['unique_name']}...")
acquire_lock()
staging_dir_path = generate_staging_directory(
problem_name=problem_obj["unique_name"]
)
logger.debug(
f"{problem_obj['unique_name']}: created staging directory"
+ f" ({staging_dir_path})"
)
generated_deb_path = package_problem(
problem_path, staging_path=staging_dir_path, out_path=DEB_ROOT
)
logger.debug(f"{problem_obj['unique_name']}: created debian package")
try:
subprocess.run(
"DEBIAN_FRONTEND=noninteractive apt-get -y install "
+ f"--reinstall {generated_deb_path}",
shell=True,
check=True,
stdout=subprocess.PIPE,
)
except subprocess.CalledProcessError:
logger.error("An error occurred while installing problem packages.")
raise FatalException
finally:
release_lock()
logger.debug(f"{problem_obj['unique_name']}: installed package")
logger.info(f"{problem_obj['unique_name']} installed successfully")
def deploy_problems(args, config):
""" Main entrypoint for problem deployment """
global deploy_config, port_map, inv_port_map
deploy_config = config
need_restart_xinetd = False
try:
user = getpwnam(deploy_config.default_user)
except KeyError as e:
logger.info("default_user '%s' does not exist. Creating the user now.",
deploy_config.default_user)
create_user(deploy_config.default_user)
if args.deployment_directory is not None and (len(args.problem_paths) > 1 or
args.num_instances > 1):
logger.error(
"Cannot specify deployment directory if deploying multiple problems or instances."
)
raise FatalException
if args.secret:
deploy_config.deploy_secret = args.secret
logger.warning(
"Overriding deploy_secret with user supplied secret '%s'.",
args.secret)
problem_names = args.problem_paths
if args.bundle:
bundle_problems = []
for bundle_path in args.problem_paths:
if os.path.isfile(bundle_path):
bundle = get_bundle(bundle_path)
bundle_problems.extend(bundle["problems"])
else:
bundle_sources_path = get_bundle_root(
bundle_path, absolute=True)
if os.path.isdir(bundle_sources_path):
bundle = get_bundle(bundle_sources_path)
bundle_problems.extend(bundle["problems"])
else:
logger.error("Could not find bundle at '%s'.", bundle_path)
raise FatalException
problem_names = bundle_problems
# before deploying problems, load in port_map and already_deployed instances
already_deployed = {}
for path, problem in get_all_problems().items():
already_deployed[path] = []
for instance in get_all_problem_instances(path):
already_deployed[path].append(instance["instance_number"])
if "port" in instance:
port_map[instance["port"]] = (problem["name"],
instance["instance_number"])
inv_port_map[(problem["name"],
instance["instance_number"])] = instance["port"]
lock_file = join(HACKSPORTS_ROOT, "deploy.lock")
if os.path.isfile(lock_file):
logger.error(
"Cannot deploy while other deployment in progress. If you believe this is an error, "
"run 'shell_manager clean'")
raise FatalException
logger.debug("Obtaining deployment lock file %s", lock_file)
with open(lock_file, "w") as f:
f.write("1")
if args.instances:
instance_list = args.instances
else:
instance_list = list(range(0, args.num_instances))
try:
for problem_name in problem_names:
if isdir(get_problem_root(problem_name, absolute=True)):
# problem_name is already an installed package
deploy_location = get_problem_root(problem_name, absolute=True)
elif isdir(problem_name) and args.dry:
# dry run - avoid installing package
deploy_location = problem_name
elif isdir(problem_name):
# problem_name is a source dir - convert to .deb and install
try:
if not os.path.isdir(TEMP_DEB_DIR):
os.mkdir(TEMP_DEB_DIR)
generated_deb_path = package_problem(problem_name, out_path=TEMP_DEB_DIR)
except FatalException:
logger.error("An error occurred while packaging %s.", problem_name)
raise
try:
# reinstall flag ensures package will be overwritten if version is the same,
# maintaining previous 'dpkg -i' behavior
subprocess.run('apt-get install --reinstall {}'.format(generated_deb_path), shell=True, check=True, stdout=subprocess.PIPE)
except subprocess.CalledProcessError:
logger.error("An error occurred while installing problem packages.")
raise FatalException
deploy_location = get_problem_root_hashed(get_problem(problem_name), absolute=True)
else:
logger.error("'%s' is neither an installed package, nor a valid problem directory",
problem_name)
raise FatalException
# Avoid redeploying already-deployed instances
if args.redeploy:
todo_instance_list = instance_list
else:
todo_instance_list = list(
set(instance_list) -
set(already_deployed.get(problem_name, [])))
need_restart_xinetd = deploy_problem(
deploy_location,
instances=todo_instance_list,
test=args.dry,
deployment_directory=args.deployment_directory,
debug=args.debug,
restart_xinetd=False)
finally:
# Restart xinetd unless specified. Service must be manually restarted
if not args.no_restart and need_restart_xinetd:
execute(["service", "xinetd", "restart"], timeout=60)
logger.debug("Releasing lock file %s", lock_file)
os.remove(lock_file)
def postinst_dependencies(problem, problem_path, debian_path, install_path):
"""
Handles the generation of the postinst script for additional dependencies.
Args:
problem: the problem object.
problem_path: the problem directory.
debian_path: the deb's DEBIAN directory.
"""
postinst_template = ["#!/bin/bash"]
requirements_path = join(problem_path, "requirements.txt")
dependencies_path = join(problem_path, "install_dependencies")
staging_requirements_path = join(install_path, "requirements.txt")
deployed_requirements_path = join(
get_problem_root_hashed(problem, absolute=True), "__files",
"requirements.txt")
deployed_setup_path = join(
get_problem_root_hashed(problem, absolute=True),
"__files",
"install_dependencies",
)
listed_requirements = problem.get("pip_requirements", [])
pip_python_version = problem.get("pip_python_version")
valid_pip_python_versions = ["2", "3", "3.7"]
if pip_python_version not in valid_pip_python_versions:
pip_python_version = "3"
# Write or copy the requirements to the staging directory.
if len(listed_requirements) > 0:
if isfile(requirements_path):
logger.error(
"Problem '%s' has both a pip_requirements field and requirements.txt.",
problem["name"],
)
raise FatalException
with open(staging_requirements_path, "w") as f:
f.writelines("\n".join(listed_requirements))
elif isfile(requirements_path):
copy(requirements_path, staging_requirements_path)
if logger.getEffectiveLevel() <= logging.DEBUG and isfile(
staging_requirements_path):
with open(staging_requirements_path, "r") as f:
logger.debug("python requirements:\n%s", f.read())
if isfile(staging_requirements_path):
postinst_template.append("python{ver} -m pip install -r {path}".format(
ver=pip_python_version, path=deployed_requirements_path))
if isfile(dependencies_path):
copy(dependencies_path, join(install_path, "install_dependencies"))
# Ensure it is executable
chmod(join(install_path, "install_dependencies"), 0o500)
postinst_template.append("bash -c '{}'".format(deployed_setup_path))
chmod(debian_path, 0o775)
postinst_path = join(debian_path, "postinst")
with open(postinst_path, "w") as f:
chmod(postinst_path, 0o775)
contents = "\n".join(postinst_template)
f.write(contents)
# post_template always has a she-bang.
if len(postinst_template) > 1:
logger.debug("post install:\n%s", contents)
def package_problem(problem_path,
staging_path=None,
out_path=None,
ignore_files=None):
"""
Does the work of packaging a single problem.
Args:
problem_path (str): path to the problem directory
staging_path (str, optional): path to a temporary.
staging directory for packaging this problem.
out_path (str, optional): path to an output directory
for the resultant .deb package.
ignore_files (list of str, optional): filenames to exclude
when packaging this problem.
Returns:
str: the absolute path to the packaged problem
"""
if ignore_files is None:
ignore_files = []
problem = get_problem(problem_path)
logger.debug("Starting to package: '%s'.", problem["name"])
# Create staging directories needed for packaging
paths = {}
if staging_path is None:
paths["staging"] = join(problem_path, "__staging")
else:
paths["staging"] = join(staging_path, "__staging")
paths["debian"] = join(paths["staging"], "DEBIAN")
paths["data"] = join(paths["staging"], get_problem_root_hashed(problem))
paths["install_data"] = join(paths["data"], "__files")
for path in paths.values():
if not isdir(path):
makedirs(path)
# Copy the problem files to the staging directory
ignore_files.append("__staging")
full_copy(problem_path, paths["data"], ignore=ignore_files)
# note that this chmod does not work correct if on a vagrant shared folder,
# so we need to package the problems elsewhere
chmod(paths["data"], 0o750)
problem_to_control(problem, paths["debian"])
postinst_dependencies(problem, problem_path, paths["debian"],
paths["install_data"])
# Package the staging directory as a .deb
def format_deb_file_name(problem):
"""
Prepare the file name of the deb package according to deb policy.
Args:
problem: the problem object
Returns:
An acceptable file name for the problem.
"""
raw_package_name = "{}.deb".format(
sanitize_name(problem["unique_name"]))
return raw_package_name
deb_directory = out_path if out_path is not None else getcwd()
deb_path = join(deb_directory, format_deb_file_name(problem))
shell = spur.LocalShell()
result = shell.run(
["fakeroot", "dpkg-deb", "--build", paths["staging"], deb_path])
if result.return_code != 0:
logger.error("Error building problem deb for '%s'.", problem["name"])
logger.error(result.output)
raise FatalException
else:
logger.debug("Problem '%s' packaged successfully.",
problem["unique_name"])
# Remove the staging directory
logger.debug("Cleaning up '%s' staging directory '%s'.", problem["name"],
paths["staging"])
rmtree(paths["staging"])
return os.path.abspath(deb_path)