def containerize_problems(args):
""" Main entrypoint for problem containerization """
# determine what we are deploying
problem_names = args.problem_names
if args.instances:
instance_list = args.instances
else:
instance_list = list(range(0, args.num_instances))
logger.debug(f"Containerizing: {problem_names} {instance_list}")
# build base images required
ensure_base_images()
deploy_init(contain=True)
flag_fmt = args.flag_format if args.flag_format else FLAG_FMT
for name in problem_names:
if not os.path.isdir(get_problem_root(name, absolute=True)):
logger.error(f"'{name}' is not an installed problem")
continue
logger.debug(f"Problem : {name}")
src = get_problem_root(name, absolute=True)
metadata = get_problem(src)
cur_instances = [
i["instance_number"] for i in get_all_problem_instances(name)
]
logger.debug(f"Existing : {cur_instances}")
origwd = os.getcwd()
for instance in instance_list:
if instance in cur_instances:
logger.warn(f"Instance already deployed: {instance}")
continue
logger.debug(f"Instance : {instance}")
# copy source files to a staging directory and switch to it
staging = generate_staging_directory(problem_name=name,
instance_number=instance)
dst = os.path.join(staging, "_containerize")
shutil.copytree(src, dst)
os.chdir(dst)
# build the image
containerize(metadata, instance, flag_fmt)
# return to the orginal directory
os.chdir(origwd)
def get_all_problems():
""" Returns a dictionary of name-hash:object mappings """
problems = {}
if os.path.isdir(PROBLEM_ROOT):
for name in os.listdir(PROBLEM_ROOT):
try:
problem = get_problem(get_problem_root(name, absolute=True))
problems[name] = problem
except FileNotFoundError as e:
pass
return problems
def get_all_problems():
""" Returns a dictionary of name:object mappings """
problems = {}
if os.path.isdir(PROBLEM_ROOT):
for name in os.listdir(PROBLEM_ROOT):
try:
problem = get_problem(get_problem_root(name, absolute=True))
problems[name] = problem
except FileNotFoundError as e:
pass
return problems
def undeploy_problems(args):
"""
Main entrypoint for problem undeployment
Does not remove the installed packages (apt-get remove [sanitized name with hash]).
Does not remove the problem from the web server (delete it from the mongo db).
"""
problem_names = args.problem_names
if len(problem_names) == 0:
logger.error("No problem name(s) specified")
raise FatalException
if len(problem_names) == 1 and problem_names[0] == "all":
# Shortcut to undeploy n instances of all problems
problem_names = [
v["unique_name"] for k, v in get_all_problems().items()
]
acquire_lock()
if args.instances:
instance_list = args.instances
else:
instance_list = list(range(0, args.num_instances))
try:
for problem_name in problem_names:
if not isdir(get_problem_root(problem_name, absolute=True)):
logger.error(f"'{problem_name}' is not an installed problem")
continue
instances_to_remove = copy(instance_list)
deployed_instances = set()
for instance in get_all_problem_instances(problem_name):
deployed_instances.add(instance["instance_number"])
instances_to_remove = list(
set(instances_to_remove).intersection(deployed_instances))
if len(instances_to_remove) == 0:
logger.warning(
f"No deployed instances found for {problem_name}")
continue
remove_instances(problem_name, instances_to_remove)
finally:
execute(["service", "xinetd", "restart"], timeout=60)
release_lock()
def postinst_dependencies(problem, problem_path, debian_path, install_path):
"""
Handles the generation of the postinst script for additional dependencies.
Args:
problem: the problem object.
problem_path: the problem directory.
debian_path: the deb's DEBIAN directory.
"""
postinst_template = ["#!/bin/bash"]
requirements_path = join(problem_path, "requirements.txt")
dependencies_path = join(problem_path, "install_dependencies")
staging_requirements_path = join(install_path, "requirements.txt")
deployed_requirements_path = join(
get_problem_root(problem["name"], absolute=True), "__files",
"requirements.txt")
deployed_setup_path = join(
get_problem_root(problem["name"], absolute=True), "__files",
"install_dependencies")
listed_requirements = problem.get("pip_requirements", [])
pip_python_version = problem.get("pip_python_version")
valid_pip_python_versions = ["2", "3", "3.6"]
if pip_python_version not in valid_pip_python_versions:
pip_python_version = "3"
# Write or copy the requirements to the staging directory.
if len(listed_requirements) > 0:
if isfile(requirements_path):
logger.error(
"Problem '%s' has both a pip_requirements field and requirements.txt.",
problem["name"])
raise FatalException
with open(staging_requirements_path, "w") as f:
f.writelines("\n".join(listed_requirements))
elif isfile(requirements_path):
copy(requirements_path, staging_requirements_path)
if logger.getEffectiveLevel() <= logging.DEBUG and isfile(
staging_requirements_path):
with open(staging_requirements_path, "r") as f:
logger.debug("python requirements:\n%s", f.read())
if isfile(staging_requirements_path):
postinst_template.append("python{ver} -m pip install -r {path}".format(
ver=pip_python_version, path=deployed_requirements_path))
if isfile(dependencies_path):
copy(dependencies_path, join(install_path, "install_dependencies"))
# Ensure it is executable
chmod(join(install_path, "install_dependencies"), 0o500)
postinst_template.append("bash -c '{}'".format(deployed_setup_path))
chmod(debian_path, 0o775)
postinst_path = join(debian_path, "postinst")
with open(postinst_path, "w") as f:
chmod(postinst_path, 0o775)
contents = "\n".join(postinst_template)
f.write(contents)
# post_template always has a she-bang.
if len(postinst_template) > 1:
logger.debug("post install:\n%s", contents)
def problem_builder(args, config):
"""
Main entrypoint for package building operations.
"""
if not args.problem_paths:
print(
"usage: shell_manager package [-h] [-s STAGING_DIR] [-o OUT] [-i IGNORE] problem_path"
)
print(
"shell_manager bundle: error: the following arguments are required: problem_path"
)
raise FatalException
# Grab a problem_path
problem_base_path = args.problem_paths.pop()
problem_paths = find_problems(problem_base_path)
if len(problem_paths) == 0:
logging.critical("No problems found under '%s'!", problem_base_path)
raise FatalException
for problem_path in problem_paths:
problem = get_problem(problem_path)
logger.debug("Starting to package: '%s'.", problem["name"])
paths = {}
if args.staging_dir is None:
paths["staging"] = join(problem_path, "__staging")
else:
paths["staging"] = join(args.staging_dir, "__staging")
paths["debian"] = join(paths["staging"], "DEBIAN")
paths["data"] = join(paths["staging"],
get_problem_root(problem["name"]))
paths["install_data"] = join(paths["data"], "__files")
# Make all of the directories, order does not matter with makedirs
[
makedirs(staging_path) for _, staging_path in paths.items()
if not isdir(staging_path)
]
args.ignore.append("__staging")
full_copy(problem_path, paths["data"], ignore=args.ignore)
# note that this chmod does not work correct if on a vagrant shared folder,
# so we need to package the problems elsewhere
chmod(paths["data"], 0o750)
problem_to_control(problem, paths["debian"])
postinst_dependencies(problem, problem_path, paths["debian"],
paths["install_data"])
deb_directory = args.out if args.out is not None else getcwd()
def format_deb_file_name(problem):
"""
Prepare the file name of the deb package according to deb policy.
Args:
problem: the problem object
Returns:
An acceptable file name for the problem.
"""
raw_package_name = "{}-{}-{}.deb".format(
sanitize_name(problem.get("organization", "ctf")),
sanitize_name(problem.get("pkg_name", problem["name"])),
sanitize_name(problem.get("version", "1.0-0")))
return raw_package_name
deb_path = join(deb_directory, format_deb_file_name(problem))
shell = spur.LocalShell()
result = shell.run(
["fakeroot", "dpkg-deb", "--build", paths["staging"], deb_path])
if result.return_code != 0:
logger.error("Error building problem deb for '%s'.",
problem["name"])
logger.error(result.output)
else:
logger.info("Problem '%s' packaged successfully.", problem["name"])
logger.debug("Clearning up '%s' staging directory '%s'.",
problem["name"], paths["staging"])
rmtree(paths["staging"])
if len(args.problem_paths) >= 1:
return problem_builder(args, config)
def deploy_problems(args, config):
""" Main entrypoint for problem deployment """
global deploy_config, port_map, inv_port_map
deploy_config = config
need_restart_xinetd = False
try:
user = getpwnam(deploy_config.default_user)
except KeyError as e:
logger.info("default_user '%s' does not exist. Creating the user now.",
deploy_config.default_user)
create_user(deploy_config.default_user)
if args.deployment_directory is not None and (len(args.problem_paths) > 1 or
args.num_instances > 1):
logger.error(
"Cannot specify deployment directory if deploying multiple problems or instances."
)
raise FatalException
if args.secret:
deploy_config.deploy_secret = args.secret
logger.warning(
"Overriding deploy_secret with user supplied secret '%s'.",
args.secret)
problem_names = args.problem_paths
if args.bundle:
bundle_problems = []
for bundle_path in args.problem_paths:
if os.path.isfile(bundle_path):
bundle = get_bundle(bundle_path)
bundle_problems.extend(bundle["problems"])
else:
bundle_sources_path = get_bundle_root(
bundle_path, absolute=True)
if os.path.isdir(bundle_sources_path):
bundle = get_bundle(bundle_sources_path)
bundle_problems.extend(bundle["problems"])
else:
logger.error("Could not find bundle at '%s'.", bundle_path)
raise FatalException
problem_names = bundle_problems
# before deploying problems, load in port_map and already_deployed instances
already_deployed = {}
for path, problem in get_all_problems().items():
already_deployed[path] = []
for instance in get_all_problem_instances(path):
already_deployed[path].append(instance["instance_number"])
if "port" in instance:
port_map[instance["port"]] = (problem["name"],
instance["instance_number"])
inv_port_map[(problem["name"],
instance["instance_number"])] = instance["port"]
lock_file = join(HACKSPORTS_ROOT, "deploy.lock")
if os.path.isfile(lock_file):
logger.error(
"Cannot deploy while other deployment in progress. If you believe this is an error, "
"run 'shell_manager clean'")
raise FatalException
logger.debug("Obtaining deployment lock file %s", lock_file)
with open(lock_file, "w") as f:
f.write("1")
if args.instances:
instance_list = args.instances
else:
instance_list = list(range(0, args.num_instances))
try:
for problem_name in problem_names:
if isdir(get_problem_root(problem_name, absolute=True)):
# problem_name is already an installed package
deploy_location = get_problem_root(problem_name, absolute=True)
elif isdir(problem_name) and args.dry:
# dry run - avoid installing package
deploy_location = problem_name
elif isdir(problem_name):
# problem_name is a source dir - convert to .deb and install
try:
if not os.path.isdir(TEMP_DEB_DIR):
os.mkdir(TEMP_DEB_DIR)
generated_deb_path = package_problem(problem_name, out_path=TEMP_DEB_DIR)
except FatalException:
logger.error("An error occurred while packaging %s.", problem_name)
raise
try:
# reinstall flag ensures package will be overwritten if version is the same,
# maintaining previous 'dpkg -i' behavior
subprocess.run('apt-get install --reinstall {}'.format(generated_deb_path), shell=True, check=True, stdout=subprocess.PIPE)
except subprocess.CalledProcessError:
logger.error("An error occurred while installing problem packages.")
raise FatalException
deploy_location = get_problem_root_hashed(get_problem(problem_name), absolute=True)
else:
logger.error("'%s' is neither an installed package, nor a valid problem directory",
problem_name)
raise FatalException
# Avoid redeploying already-deployed instances
if args.redeploy:
todo_instance_list = instance_list
else:
todo_instance_list = list(
set(instance_list) -
set(already_deployed.get(problem_name, [])))
need_restart_xinetd = deploy_problem(
deploy_location,
instances=todo_instance_list,
test=args.dry,
deployment_directory=args.deployment_directory,
debug=args.debug,
restart_xinetd=False)
finally:
# Restart xinetd unless specified. Service must be manually restarted
if not args.no_restart and need_restart_xinetd:
execute(["service", "xinetd", "restart"], timeout=60)
logger.debug("Releasing lock file %s", lock_file)
os.remove(lock_file)
def deploy_problems(args):
""" Main entrypoint for problem deployment """
global FLAG_FMT
if args.flag_format:
FLAG_FMT = args.flag_format
logger.info(f"Deploying with custom flag format: {FLAG_FMT}")
shared_config, local_config, port_map = deploy_init(args.containerize)
need_restart_xinetd = False
try:
user = getpwnam(shared_config.default_user)
except KeyError as e:
logger.info(
"default_user '%s' does not exist. Creating the user now.",
shared_config.default_user,
)
create_user(shared_config.default_user)
problem_names = args.problem_names
if len(problem_names) == 1 and problem_names[0] == "all":
# Shortcut to deploy n instances of all problems
problem_names = [
v["unique_name"] for k, v in get_all_problems().items()
]
if args.instances:
instance_list = args.instances
else:
instance_list = list(range(0, args.num_instances))
if args.containerize and (len(problem_names) > 1
or len(instance_list) > 1):
logger.error("can only deploy a single instance per container")
return
acquire_lock()
try:
for problem_name in problem_names:
if not isdir(get_problem_root(problem_name, absolute=True)):
logger.error(f"'{problem_name}' is not an installed problem")
continue
source_location = get_problem_root(problem_name, absolute=True)
problem_object = get_problem(source_location)
instances_to_deploy = copy(instance_list)
is_static_flag = problem_object.get("static_flag", False)
if is_static_flag is True:
instances_to_deploy = [0]
# Avoid redeploying already-deployed instances
if not args.redeploy:
already_deployed = set()
for instance in get_all_problem_instances(problem_name):
already_deployed.add(instance["instance_number"])
instances_to_deploy = list(
set(instances_to_deploy) - already_deployed)
if instances_to_deploy:
deploy_problem(source_location,
instances=instances_to_deploy,
test=args.dry,
debug=args.debug,
restart_xinetd=False,
containerize=args.containerize)
else:
logger.info(
"No additional instances to deploy for '%s'.",
problem_object["unique_name"],
)
finally:
# Restart xinetd unless specified. Service must be manually restarted
if not args.no_restart:
execute(["service", "xinetd", "restart"], timeout=60)
# Write out updated port map
with open(PORT_MAP_PATH, "w") as f:
stringified_port_map = {repr(k): v for k, v in port_map.items()}
json.dump(stringified_port_map, f)
release_lock()
def bundle_problems(args, config):
"""
Main entrypoint for generating problem bundles.
"""
bundle_path = args.bundle_path
if os.path.isdir(args.bundle_path):
bundle = get_bundle(args.bundle_path)
bundle_path = join(args.bundle_path, "bundle.json")
elif os.path.isfile(args.bundle_path):
bundle = json.loads(open(args.bundle_path).read())
else:
logger.error("No bundle could be found at '%s'", args.bundle_path)
raise FatalException
logger.debug("Starting to bundle: '%s'.", bundle["name"])
for problem_name in bundle["problems"]:
installed_path = get_problem_root(problem_name, absolute=True)
if not isdir(installed_path) or not get_problem(installed_path):
logger.error("'%s' is not an installed problem.", problem_name)
raise FatalException
paths = {"working": getcwd() if args.out is None else args.out}
if args.staging_dir:
paths["staging"] = join(args.staging_dir, "__staging")
else:
paths["staging"] = join(paths["working"], "__staging")
paths["debian"] = join(paths["staging"], "DEBIAN")
paths["bundle_root"] = join(paths["staging"],
get_bundle_root(bundle["name"]))
[
makedirs(staging_path)
for _, staging_path in paths.items()
if not isdir(staging_path)
]
# note that this chmod does not work correct if on a vagrant shared folder,
# so we need to package the problems elsewhere
chmod(dirname(paths["bundle_root"]), 0o750)
bundle_to_control(bundle, paths["debian"])
copied_bundle_path = join(paths["bundle_root"], "bundle.json")
copyfile(bundle_path, copied_bundle_path)
def format_deb_file_name(bundle):
"""
Prepare the file name of the deb package according to deb policy.
Args:
bundle: the bundle object
Returns:
An acceptable file name for the bundle.
"""
raw_package_name = "{}-{}-bundle-{}.deb".format(
sanitize_name(bundle.get("organization", "ctf")),
sanitize_name(bundle["name"]),
sanitize_name(bundle.get("version", "1.0-0")))
return raw_package_name
deb_path = join(paths["working"], format_deb_file_name(bundle))
shell = spur.LocalShell()
result = shell.run(
["fakeroot", "dpkg-deb", "--build", paths["staging"], deb_path])
if result.return_code != 0:
logger.error("Error building bundle deb for '%s'.", bundle["name"])
logger.error(result.output)
else:
logger.info("Bundle '%s' packaged successfully.", bundle["name"])
logger.debug("Clearning up '%s' staging directory '%s'.", bundle["name"],
paths["staging"])
rmtree(paths["staging"])
def undeploy_problems(args, config):
""" Main entrypoint for problem undeployment """
problem_names = args.problem_paths
if args.bundle:
bundle_problems = []
for bundle_path in args.problem_paths:
if isfile(bundle_path):
bundle = get_bundle(bundle_path)
bundle_problems.extend(bundle["problems"])
else:
bundle_sources_path = get_bundle_root(bundle_path, absolute=True)
if isdir(bundle_sources_path):
bundle = get_bundle(bundle_sources_path)
bundle_problems.extend(bundle["problems"])
else:
logger.error("Could not find bundle at '%s'.", bundle_path)
raise FatalException
problem_names = bundle_problems
# before deploying problems, load in already_deployed instances
already_deployed = {}
for path, problem in get_all_problems().items():
already_deployed[problem["name"]] = []
for instance in get_all_problem_instances(path):
already_deployed[problem["name"]].append(instance["instance_number"])
lock_file = join(HACKSPORTS_ROOT, "deploy.lock")
if os.path.isfile(lock_file):
logger.error("Cannot undeploy while other deployment in progress. If you believe this is an error, "
"run 'shell_manager clean'")
raise FatalException
logger.debug("Obtaining deployment lock file %s", lock_file)
with open(lock_file, "w") as f:
f.write("1")
if args.instances:
instance_list = args.instances
else:
instance_list = list(range(0, args.num_instances))
try:
for problem_name in problem_names:
problem_root = get_problem_root(problem_name, absolute=True)
if isdir(problem_root):
problem = get_problem(problem_root)
instances = list(filter(lambda x: x in already_deployed[problem["name"]], instance_list))
if len(instances) == 0:
logger.warn("No deployed instances %s were found for problem '%s'.", instance_list, problem["name"])
else:
logger.debug("Undeploying problem '%s'.", problem["name"])
remove_instances(problem_name, instance_list)
logger.info("Problem instances %s were successfully removed from '%s'.", instances, problem["name"])
else:
logger.error("Problem '%s' doesn't appear to be installed.", problem_name)
raise FatalException
finally:
logger.debug("Releasing lock file %s", lock_file)
os.remove(lock_file)
def deploy_problems(args, config):
""" Main entrypoint for problem deployment """
global deploy_config, port_map, inv_port_map
deploy_config = config
try:
user = getpwnam(deploy_config.default_user)
except KeyError as e:
logger.info("default_user '%s' does not exist. Creating the user now.", deploy_config.default_user)
create_user(deploy_config.default_user)
if args.deployment_directory is not None and (len(args.problem_paths) > 1 or args.num_instances > 1):
logger.error("Cannot specify deployment directory if deploying multiple problems or instances.")
raise FatalException
if args.secret:
deploy_config.deploy_secret = args.secret
logger.warn("Overriding deploy_secret with user supplied secret '%s'.", args.secret)
problem_names = args.problem_paths
if args.bundle:
bundle_problems = []
for bundle_path in args.problem_paths:
if os.path.isfile(bundle_path):
bundle = get_bundle(bundle_path)
bundle_problems.extend(bundle["problems"])
else:
bundle_sources_path = get_bundle_root(bundle_path, absolute=True)
if os.path.isdir(bundle_sources_path):
bundle = get_bundle(bundle_sources_path)
bundle_problems.extend(bundle["problems"])
else:
logger.error("Could not find bundle at '%s'.", bundle_path)
raise FatalException
problem_names = bundle_problems
# before deploying problems, load in port_map and already_deployed instances
already_deployed = {}
for path, problem in get_all_problems().items():
already_deployed[path] = []
for instance in get_all_problem_instances(path):
already_deployed[path].append(instance["instance_number"])
if "port" in instance:
port_map[instance["port"]] = (problem["name"], instance["instance_number"])
inv_port_map[(problem["name"], instance["instance_number"])] = instance["port"]
lock_file = join(HACKSPORTS_ROOT, "deploy.lock")
if os.path.isfile(lock_file):
logger.error("Cannot deploy while other deployment in progress. If you believe this is an error, "
"run 'shell_manager clean'")
raise FatalException
logger.debug("Obtaining deployment lock file %s", lock_file)
with open(lock_file, "w") as f:
f.write("1")
if args.instances:
instance_list = args.instances
else:
instance_list = list(range(0, args.num_instances))
try:
for problem_name in problem_names:
if args.redeploy:
todo_instance_list = instance_list
else:
# remove already deployed instances
todo_instance_list = list(set(instance_list) - set(already_deployed.get(problem_name, [])))
if args.dry and isdir(problem_name):
deploy_problem(problem_name, instances=todo_instance_list, test=args.dry,
deployment_directory=args.deployment_directory, debug=args.debug)
elif isdir(join(get_problem_root(problem_name, absolute=True))):
deploy_problem(join(get_problem_root(problem_name, absolute=True)), instances=todo_instance_list,
test=args.dry, deployment_directory=args.deployment_directory, debug=args.debug)
else:
logger.error("Problem '%s' doesn't appear to be installed.", problem_name)
raise FatalException
finally:
logger.debug("Releasing lock file %s", lock_file)
if not args.dry:
os.remove(lock_file)
def deploy_problems(args):
""" Main entrypoint for problem deployment """
global shared_config, local_config, port_map
shared_config = get_shared_config()
local_config = get_local_config()
need_restart_xinetd = False
try:
user = getpwnam(shared_config.default_user)
except KeyError as e:
logger.info("default_user '%s' does not exist. Creating the user now.",
shared_config.default_user)
create_user(shared_config.default_user)
problem_names = args.problem_names
if len(problem_names) == 1 and problem_names[0] == 'all':
# Shortcut to deploy n instances of all problems
problem_names = [
v['unique_name'] for k, v in get_all_problems().items()
]
# Attempt to load the port_map from file
try:
port_map_path = join(SHARED_ROOT, 'port_map.json')
with open(port_map_path, 'r') as f:
port_map = json.load(f)
port_map = {literal_eval(k): v for k, v in port_map.items()}
except FileNotFoundError:
# If it does not exist, create it
for path, problem in get_all_problems().items():
for instance in get_all_problem_instances(path):
port_map[(problem["unique_name"],
instance["instance_number"])] = instance.get(
"port", None)
with open(port_map_path, 'w') as f:
stringified_port_map = {repr(k): v for k, v in port_map.items()}
json.dump(stringified_port_map, f)
except IOError:
logger.error(f"Error loading port map from {port_map_path}")
raise
acquire_lock()
if args.instances:
instance_list = args.instances
else:
instance_list = list(range(0, args.num_instances))
try:
for problem_name in problem_names:
if not isdir(get_problem_root(problem_name, absolute=True)):
logger.error(f"'{problem_name}' is not an installed problem")
continue
source_location = get_problem_root(problem_name, absolute=True)
problem_object = get_problem(source_location)
instances_to_deploy = copy(instance_list)
is_static_flag = problem_object.get("static_flag", False)
if is_static_flag is True:
instances_to_deploy = [0]
# Avoid redeploying already-deployed instances
if not args.redeploy or is_static_flag:
already_deployed = set()
for instance in get_all_problem_instances(problem_name):
already_deployed.add(instance["instance_number"])
instances_to_deploy = list(
set(instances_to_deploy) - already_deployed)
if instances_to_deploy:
deploy_problem(source_location,
instances=instances_to_deploy,
test=args.dry,
debug=args.debug,
restart_xinetd=False)
else:
logger.info("No additional instances to deploy for '%s'.",
problem_object["unique_name"])
finally:
# Restart xinetd unless specified. Service must be manually restarted
if not args.no_restart:
execute(["service", "xinetd", "restart"], timeout=60)
# Write out updated port map
with open(port_map_path, 'w') as f:
stringified_port_map = {repr(k): v for k, v in port_map.items()}
json.dump(stringified_port_map, f)
release_lock()
def package_problem(problem_path,
staging_path=None,
out_path=None,
ignore_files=[]):
"""
Does the work of packaging a single problem.
Args:
problem_path (str): path to the problem directory
staging_path (str, optional): path to a temporary.
staging directory for packaging this problem.
out_path (str, optional): path to an output directory
for the resultant .deb package.
ignore_files (list of str, optional): filenames to exclude
when packaging this problem.
Returns:
tuple (str, str): the name of the package,
the absolute path to the packaged problem
"""
problem = get_problem(problem_path)
logger.debug("Starting to package: '%s'.", problem["name"])
# Create staging directories needed for packaging
paths = {}
if staging_path is None:
paths["staging"] = join(problem_path, "__staging")
else:
paths["staging"] = join(staging_path, "__staging")
paths["debian"] = join(paths["staging"], "DEBIAN")
paths["data"] = join(paths["staging"], get_problem_root(problem["name"]))
paths["install_data"] = join(paths["data"], "__files")
for path in paths.values():
if not isdir(path):
makedirs(path)
# Copy the problem files to the staging directory
ignore_files.append("__staging")
full_copy(problem_path, paths["data"], ignore=ignore_files)
# note that this chmod does not work correct if on a vagrant shared folder,
# so we need to package the problems elsewhere
chmod(paths["data"], 0o750)
problem_to_control(problem, paths["debian"])
postinst_dependencies(problem, problem_path, paths["debian"],
paths["install_data"])
# Package the staging directory as a .deb
def format_deb_file_name(problem):
"""
Prepare the file name of the deb package according to deb policy.
Args:
problem: the problem object
Returns:
An acceptable file name for the problem.
"""
raw_package_name = "{}-{}-{}.deb".format(
sanitize_name(problem.get("organization", "ctf")),
sanitize_name(problem.get("pkg_name", problem["name"])),
sanitize_name(problem.get("version", "1.0-0")))
return raw_package_name
deb_directory = out_path if out_path is not None else getcwd()
deb_path = join(deb_directory, format_deb_file_name(problem))
shell = spur.LocalShell()
result = shell.run(
["fakeroot", "dpkg-deb", "--build", paths["staging"], deb_path])
if result.return_code != 0:
logger.error("Error building problem deb for '%s'.", problem["name"])
logger.error(result.output)
raise FatalException
else:
logger.info("Problem '%s' packaged successfully.", problem["name"])
# Remove the staging directory
logger.debug("Cleaning up '%s' staging directory '%s'.", problem["name"],
paths["staging"])
rmtree(paths["staging"])
return (sanitize_name(problem.get("pkg_name", problem["name"])),
os.path.abspath(deb_path))
def postinst_dependencies(problem, problem_path, debian_path, install_path):
"""
Handles the generation of the postinst script for additional dependencies.
Args:
problem: the problem object.
problem_path: the problem directory.
debian_path: the deb's DEBIAN directory.
"""
postinst_template = ["#!/bin/bash"]
requirements_path = join(problem_path, "requirements.txt")
dependencies_path = join(problem_path, "install_dependencies")
staging_requirements_path = join(install_path, "requirements.txt")
deployed_requirements_path = join(get_problem_root(problem["name"], absolute=True),
"__files", "requirements.txt")
deployed_setup_path = join(get_problem_root(problem["name"], absolute=True),
"__files", "install_dependencies")
listed_requirements = problem.get("pip_requirements", [])
#Write or copy the requirements to the staging directory.
if len(listed_requirements) > 0:
if isfile(requirements_path):
logger.error("Problem '%s' has both a pip_requirements field and requirements.txt.", problem["name"])
raise FatalException
with open(staging_requirements_path, "w") as f:
f.writelines("\n".join(listed_requirements))
elif isfile(requirements_path):
copy(requirements_path, staging_requirements_path)
if logger.getEffectiveLevel() <= logging.DEBUG and isfile(staging_requirements_path):
with open(staging_requirements_path, "r") as f:
logger.debug("python requirements:\n%s", f.read())
if isfile(staging_requirements_path):
postinst_template.append("pip3 install -r {}".format(deployed_requirements_path))
if isfile(dependencies_path):
copy(dependencies_path, join(install_path, "install_dependencies"))
#Ensure it is executable
chmod(join(install_path, "install_dependencies"), 0o500)
postinst_template.append("bash -c '{}'".format(deployed_setup_path))
chmod(debian_path, 0o775)
postinst_path = join(debian_path, "postinst")
with open(postinst_path, "w") as f:
chmod(postinst_path, 0o775)
contents = "\n".join(postinst_template)
f.write(contents)
#post_template always has a she-bang.
if len(postinst_template) > 1:
logger.debug("post install:\n%s", contents)
def problem_builder(args, config):
"""
Main entrypoint for package building operations.
"""
#Grab a problem_path
problem_base_path = args.problem_paths.pop()
problem_paths = find_problems(problem_base_path)
if len(problem_paths) == 0:
logging.critical("No problems found under '%s'!", problem_base_path)
raise FatalException
for problem_path in problem_paths:
problem = get_problem(problem_path)
logger.debug("Starting to package: '%s'.", problem["name"])
paths = {}
if args.staging_dir is None:
paths["staging"] = join(problem_path, "__staging")
else:
paths["staging"] = join(args.staging_dir, "__staging")
paths["debian"] = join(paths["staging"], "DEBIAN")
paths["data"] = join(paths["staging"], get_problem_root(problem["name"]))
paths["install_data"] = join(paths["data"], "__files")
#Make all of the directories, order does not matter with makedirs
[makedirs(staging_path) for _, staging_path in paths.items() if not isdir(staging_path)]
args.ignore.append("__staging")
full_copy(problem_path, paths["data"], ignore=args.ignore)
# note that this chmod does not work correct if on a vagrant shared folder,
# so we need to package the problems elsewhere
chmod(paths["data"], 0o750)
problem_to_control(problem, paths["debian"])
postinst_dependencies(problem, problem_path,
paths["debian"], paths["install_data"])
deb_directory = args.out if args.out is not None else getcwd()
def format_deb_file_name(problem):
"""
Prepare the file name of the deb package according to deb policy.
Args:
problem: the problem object
Returns:
An acceptable file name for the problem.
"""
raw_package_name = "{}-{}-{}.deb".format(
sanitize_name(problem.get("organization", "ctf")),
sanitize_name(problem.get("pkg_name", problem["name"])),
sanitize_name(problem.get("version", "1.0-0"))
)
return raw_package_name
deb_path = join(deb_directory, format_deb_file_name(problem))
shell = spur.LocalShell()
result = shell.run(["fakeroot", "dpkg-deb", "--build", paths["staging"], deb_path])
if result.return_code != 0:
logger.error("Error building problem deb for '%s'.", problem["name"])
logger.error(result.output)
else:
logger.info("Problem '%s' packaged successfully.", problem["name"])
logger.debug("Clearning up '%s' staging directory '%s'.", problem["name"], paths["staging"])
rmtree(paths["staging"])
if len(args.problem_paths) >= 1:
return problem_builder(args, config)
def deploy_problems(args, config):
""" Main entrypoint for problem deployment """
global deploy_config, port_map, inv_port_map
deploy_config = config
try:
user = getpwnam(deploy_config.default_user)
except KeyError as e:
logger.info("default_user '%s' does not exist. Creating the user now.",
deploy_config.default_user)
create_user(deploy_config.default_user)
if args.deployment_directory is not None and (len(args.problem_paths) > 1
or args.num_instances > 1):
logger.error(
"Cannot specify deployment directory if deploying multiple problems or instances."
)
raise FatalException
if args.secret:
deploy_config.deploy_secret = args.secret
logger.warn("Overriding deploy_secret with user supplied secret '%s'.",
args.secret)
problem_names = args.problem_paths
if args.bundle:
bundle_problems = []
for bundle_path in args.problem_paths:
if os.path.isfile(bundle_path):
bundle = get_bundle(bundle_path)
bundle_problems.extend(bundle["problems"])
else:
bundle_sources_path = get_bundle_root(bundle_path,
absolute=True)
if os.path.isdir(bundle_sources_path):
bundle = get_bundle(bundle_sources_path)
bundle_problems.extend(bundle["problems"])
else:
logger.error("Could not find bundle at '%s'.", bundle_path)
raise FatalException
problem_names = bundle_problems
# before deploying problems, load in port_map and already_deployed instances
already_deployed = {}
for path, problem in get_all_problems().items():
already_deployed[path] = []
for instance in get_all_problem_instances(path):
already_deployed[path].append(instance["instance_number"])
if "port" in instance:
port_map[instance["port"]] = (problem["name"],
instance["instance_number"])
inv_port_map[(problem["name"],
instance["instance_number"])] = instance["port"]
lock_file = join(HACKSPORTS_ROOT, "deploy.lock")
if os.path.isfile(lock_file):
logger.error(
"Cannot deploy while other deployment in progress. If you believe this is an error, "
"run 'shell_manager clean'")
raise FatalException
logger.debug("Obtaining deployment lock file %s", lock_file)
with open(lock_file, "w") as f:
f.write("1")
if args.instances:
instance_list = args.instances
else:
instance_list = list(range(0, args.num_instances))
try:
for problem_name in problem_names:
if args.redeploy:
todo_instance_list = instance_list
else:
# remove already deployed instances
todo_instance_list = list(
set(instance_list) -
set(already_deployed.get(problem_name, [])))
if args.dry and isdir(problem_name):
deploy_problem(problem_name,
instances=todo_instance_list,
test=args.dry,
deployment_directory=args.deployment_directory,
debug=args.debug)
elif isdir(join(get_problem_root(problem_name, absolute=True))):
deploy_problem(join(
get_problem_root(problem_name, absolute=True)),
instances=todo_instance_list,
test=args.dry,
deployment_directory=args.deployment_directory,
debug=args.debug)
else:
logger.error("Problem '%s' doesn't appear to be installed.",
problem_name)
raise FatalException
finally:
logger.debug("Releasing lock file %s", lock_file)
if not args.dry:
os.remove(lock_file)
def undeploy_problems(args, config):
""" Main entrypoint for problem undeployment """
problem_names = args.problem_paths
if args.bundle:
bundle_problems = []
for bundle_path in args.problem_paths:
if isfile(bundle_path):
bundle = get_bundle(bundle_path)
bundle_problems.extend(bundle["problems"])
else:
bundle_sources_path = get_bundle_root(bundle_path,
absolute=True)
if isdir(bundle_sources_path):
bundle = get_bundle(bundle_sources_path)
bundle_problems.extend(bundle["problems"])
else:
logger.error("Could not find bundle at '%s'.", bundle_path)
raise FatalException
problem_names = bundle_problems
# before deploying problems, load in already_deployed instances
already_deployed = {}
for path, problem in get_all_problems().items():
already_deployed[problem["name"]] = []
for instance in get_all_problem_instances(path):
already_deployed[problem["name"]].append(
instance["instance_number"])
lock_file = join(HACKSPORTS_ROOT, "deploy.lock")
if os.path.isfile(lock_file):
logger.error(
"Cannot undeploy while other deployment in progress. If you believe this is an error, "
"run 'shell_manager clean'")
raise FatalException
logger.debug("Obtaining deployment lock file %s", lock_file)
with open(lock_file, "w") as f:
f.write("1")
if args.instances:
instance_list = args.instances
else:
instance_list = list(range(0, args.num_instances))
try:
for problem_name in problem_names:
problem_root = get_problem_root(problem_name, absolute=True)
if isdir(problem_root):
problem = get_problem(problem_root)
instances = list(
filter(lambda x: x in already_deployed[problem["name"]],
instance_list))
if len(instances) == 0:
logger.warn(
"No deployed instances %s were found for problem '%s'.",
instance_list, problem["name"])
else:
logger.debug("Undeploying problem '%s'.", problem["name"])
remove_instances(problem_name, instance_list)
logger.info(
"Problem instances %s were successfully removed from '%s'.",
instances, problem["name"])
else:
logger.error("Problem '%s' doesn't appear to be installed.",
problem_name)
raise FatalException
finally:
logger.debug("Releasing lock file %s", lock_file)
os.remove(lock_file)
def bundle_problems(args, config):
"""
Main entrypoint for generating problem bundles.
"""
bundle_path = args.bundle_path
if os.path.isdir(args.bundle_path):
bundle = get_bundle(args.bundle_path)
bundle_path = join(args.bundle_path, "bundle.json")
elif os.path.isfile(args.bundle_path):
bundle = json.loads(open(args.bundle_path).read())
else:
raise Exception("No bundle {}".format(args.bundle_path))
for problem_name in bundle["problems"]:
installed_path = get_problem_root(problem_name, absolute=True)
if not isdir(installed_path) or not get_problem(installed_path):
raise Exception("'{}' is not an installed problem.".format(problem_name))
paths = {"working": getcwd() if args.out is None else args.out}
if args.staging_dir:
paths["staging"] = join(args.staging_dir, "__staging")
else:
paths["staging"] = join(paths["working"], "__staging")
paths["debian"] = join(paths["staging"], "DEBIAN")
paths["bundle_root"] = join(paths["staging"], get_bundle_root(bundle["name"]))
for _, staging_path in path.items():
if not isdir(staging_path):
makedirs(staging_path)
# note that this chmod does not work correct if on a vagrant shared folder,
# so we need to package the problems elsewhere
chmod(dirname(paths["bundle_root"]), 0o750)
bundle_to_control(bundle, paths["debian"])
copied_bundle_path = join(paths["bundle_root"], "bundle.json")
copyfile(bundle_path, copied_bundle_path)
def format_deb_file_name(bundle):
"""
Prepare the file name of the deb package according to deb policy.
Args:
bundle: the bundle object
Returns:
An acceptable file name for the bundle.
"""
raw_package_name = "{}-{}-bundle-{}.deb".format(
bundle.get("organization", "ctf"), bundle["name"], bundle.get("version", "1.0-0")
)
return sanitize_name(raw_package_name)
deb_path = join(paths["working"], format_deb_file_name(bundle))
shell = spur.LocalShell()
result = shell.run(["fakeroot", "dpkg-deb", "--build", paths["staging"], deb_path])
if result.return_code != 0:
print("Error building bundle deb for '{}'".format(bundle["name"]))
print(result.output)
else:
print("Bundle '{}' packaged successfully.".format(bundle["name"]))
print("Cleaning up staging directory '{}'.".format(paths["staging"]))
rmtree(paths["staging"])
