def _popen(self, command,
stdin=None, stdout=None, stderr=None,
close_fds=True):
"""Execute the given command in the sandbox using
subprocess.Popen, assigning the corresponding standard file
descriptors.
command ([string]): executable filename and arguments of the
command.
stdin (int|None): a file descriptor.
stdout (int|None): a file descriptor.
stderr (int|None): a file descriptor.
close_fds (bool): close all file descriptor before executing.
return (Popen): popen object.
"""
self.exec_num += 1
self.log = None
args = [self.box_exec] + self.build_box_options() + ["--"] + command
logger.debug("Executing program in sandbox with command: `%s'.",
pretty_print_cmdline(args))
with io.open(self.relative_path(self.cmd_file), 'at') as commands:
commands.write("%s\n" % (pretty_print_cmdline(args)))
try:
p = subprocess.Popen(args,
stdin=stdin, stdout=stdout, stderr=stderr,
close_fds=close_fds)
except OSError:
logger.critical("Failed to execute program in sandbox "
"with command: %s", pretty_print_cmdline(args),
exc_info=True)
raise
return p
def _execute(self, command):
"""Execute the given command in the sandbox.
command (list): executable filename and arguments of the
command.
return (bool): True if the sandbox didn't report errors
(caused by the sandbox itself), False otherwise
"""
self.exec_num += 1
self.log = None
args = [self.box_exec] + self.build_box_options() + ["--"] + command
logger.debug("Executing program in sandbox with command: `%s'.",
pretty_print_cmdline(args))
with io.open(self.relative_path(self.cmd_file), 'at') as commands:
commands.write("%s\n" % (pretty_print_cmdline(args)))
return self.translate_box_exitcode(subprocess.call(args))
def _execute(self, command):
"""Execute the given command in the sandbox.
command (list): executable filename and arguments of the
command.
return (bool): True if the sandbox didn't report errors
(caused by the sandbox itself), False otherwise
"""
self.exec_num += 1
self.log = None
args = [self.box_exec] + self.build_box_options() + ["--"] + command
logger.debug("Executing program in sandbox with command: `%s'.",
pretty_print_cmdline(args))
with io.open(self.relative_path(self.cmd_file), 'at') as commands:
commands.write("%s\n" % (pretty_print_cmdline(args)))
return self.translate_box_exitcode(subprocess.call(args))
def initialize_isolate(self):
"""Initialize isolate's box."""
init_cmd = ([self.box_exec] + (["--cg"] if self.cgroup else []) +
["--box-id=%d" % self.box_id, "--init"])
ret = subprocess.call(init_cmd)
if ret != 0:
raise SandboxInterfaceException(
"Failed to initialize sandbox with command: %s "
"(error %d)" % (pretty_print_cmdline(init_cmd), ret))
def sh(cmdline, ignore_failure=False):
"""Execute a simple command.
cmd ([str]): the (unescaped) command to execute.
ignore_failure (bool): whether to suppress failures.
raise (TestException): if the command failed and ignore_failure was False.
"""
if CONFIG["VERBOSITY"] >= 1:
logger.info('$ %s', pretty_print_cmdline(cmdline))
kwargs = dict()
if CONFIG["VERBOSITY"] >= 3:
kwargs["stdout"] = subprocess.DEVNULL
kwargs["stderr"] = subprocess.STDOUT
ret = subprocess.call(cmdline, **kwargs)
if not ignore_failure and ret != 0:
raise TestException(
"Execution failed with %d/%d. Tried to execute:\n%s\n" %
(ret & 0xff, ret >> 8, pretty_print_cmdline(cmdline)))
def _popen(self,
command,
stdin=None,
stdout=None,
stderr=None,
preexec_fn=None,
close_fds=True):
"""Execute the given command in the sandbox using
subprocess.Popen, assigning the corresponding standard file
descriptors.
command ([string]): executable filename and arguments of the
command.
stdin (file|None): a file descriptor/object or None.
stdout (file|None): a file descriptor/object or None.
stderr (file|None): a file descriptor/object or None.
preexec_fn (function|None): to be called just before execve()
or None.
close_fds (bool): close all file descriptor before executing.
return (object): popen object.
"""
self.exec_time = None
self.exec_num += 1
logger.debug("Executing program in sandbox with command: `%s'.",
" ".join(command))
with io.open(self.relative_path(self.cmd_file), 'at') as commands:
commands.write("%s\n" % (pretty_print_cmdline(command)))
try:
p = subprocess.Popen(command,
stdin=stdin,
stdout=stdout,
stderr=stderr,
preexec_fn=preexec_fn,
close_fds=close_fds)
except OSError:
logger.critical(
"Failed to execute program in sandbox "
"with command: `%s'.",
" ".join(command),
exc_info=True)
raise
return p
def sh(cmdline, ignore_failure=False):
"""Execute a simple command.
cmd ([str]): the (unescaped) command to execute.
ignore_failure (bool): whether to suppress failures.
raise (TestException): if the command failed and ignore_failure was False.
"""
if CONFIG["VERBOSITY"] >= 1:
logger.info('$ %s', pretty_print_cmdline(cmdline))
kwargs = dict()
if CONFIG["VERBOSITY"] >= 3:
kwargs["stdout"] = subprocess.DEVNULL
kwargs["stderr"] = subprocess.STDOUT
kwargs["check"] = not ignore_failure
try:
subprocess.run(cmdline, **kwargs)
except subprocess.CalledProcessError as e:
raise TestException("Execution failed") from e
def _popen(self, command,
stdin=None, stdout=None, stderr=None,
preexec_fn=None, close_fds=True):
"""Execute the given command in the sandbox using
subprocess.Popen, assigning the corresponding standard file
descriptors.
command ([string]): executable filename and arguments of the
command.
stdin (file|None): a file descriptor/object or None.
stdout (file|None): a file descriptor/object or None.
stderr (file|None): a file descriptor/object or None.
preexec_fn (function|None): to be called just before execve()
or None.
close_fds (bool): close all file descriptor before executing.
return (object): popen object.
"""
self.exec_time = None
self.exec_num += 1
logger.debug("Executing program in sandbox with command: `%s'.",
" ".join(command))
with io.open(self.relative_path(self.cmd_file), 'at') as commands:
commands.write("%s\n" % (pretty_print_cmdline(command)))
try:
p = subprocess.Popen(command,
stdin=stdin, stdout=stdout, stderr=stderr,
preexec_fn=preexec_fn, close_fds=close_fds)
except OSError:
logger.critical("Failed to execute program in sandbox "
"with command: `%s'.",
" ".join(command), exc_info=True)
raise
return p
def test_double_quotes(self):
self.assertEqual(
pretty_print_cmdline(["ls", "-al", "file\"with\"dblquotes"]),
"""ls -al 'file"with"dblquotes'""")
def test_quotes(self):
self.assertEqual(
pretty_print_cmdline(["ls", "-al", "file'with'quotes"]),
"""ls -al 'file'"'"'with'"'"'quotes'""")
def test_spaces(self):
self.assertEqual(
pretty_print_cmdline(["ls", "-al", "file with spaces"]),
"ls -al 'file with spaces'")
def test_success(self):
self.assertEqual(
pretty_print_cmdline(["ls", "-al", "file"]),
"ls -al file")
def __init__(self, multithreaded, file_cacher, temp_dir=None):
"""Initialization.
For arguments documentation, see SandboxBase.__init__.
"""
SandboxBase.__init__(self, multithreaded, file_cacher, temp_dir)
# Isolate only accepts ids between 0 and 99. We assign the
# range [(shard+1)*10, (shard+2)*10) to each Worker and keep
# the range [0, 10) for other uses (command-line scripts like
# cmsMake or direct console users of isolate). Inside each
# range ids are assigned sequentially, with a wrap-around.
# FIXME This is the only use of FileCacher.service, and it's an
# improper use! Avoid it!
if file_cacher is not None and file_cacher.service is not None:
box_id = ((file_cacher.service.shard + 1) * 10
+ (IsolateSandbox.next_id % 10)) % 100
else:
box_id = IsolateSandbox.next_id % 10
IsolateSandbox.next_id += 1
# We create a directory "tmp" inside the outer temporary directory,
# because the sandbox will bind-mount the inner one. The sandbox also
# runs code as a different user, and so we need to ensure that they can
# read and write to the directory. But we don't want everybody on the
# system to, which is why the outer directory exists with no read
# permissions.
self.inner_temp_dir = "/tmp"
self.outer_temp_dir = tempfile.mkdtemp(dir=self.temp_dir)
# Don't use os.path.join here, because the absoluteness of /tmp will
# bite you.
self.path = self.outer_temp_dir + self.inner_temp_dir
os.mkdir(self.path)
self.allow_writing_all()
self.exec_name = 'isolate'
self.box_exec = self.detect_box_executable()
self.info_basename = "run.log" # Used for -M
self.log = None
self.exec_num = -1
logger.debug("Sandbox in `%s' created, using box `%s'.",
self.path, self.box_exec)
# Default parameters for isolate
self.box_id = box_id # -b
self.cgroup = config.use_cgroups # --cg
self.chdir = self.inner_temp_dir # -c
self.dirs = [] # -d
self.dirs += [(self.inner_temp_dir, self.path, "rw")]
self.preserve_env = False # -e
self.inherit_env = [] # -E
self.set_env = {} # -E
self.fsize = None # -f
self.stdin_file = None # -i
self.stack_space = None # -k
self.address_space = None # -m
self.stdout_file = None # -o
self.stderr_file = None # -r
self.timeout = None # -t
self.verbosity = 0 # -v
self.wallclock_timeout = None # -w
self.extra_timeout = None # -x
# Set common environment variables.
# Specifically needed by Python, that searches the home for
# packages.
self.set_env["HOME"] = "./"
# Needed on Ubuntu by PHP (and more, ) that
# have in /usr/bin only a symlink to one out of many
# alternatives.
if os.path.isdir("/etc/alternatives"):
self.add_mapped_directories(["/etc/alternatives"])
# Tell isolate to get the sandbox ready.
box_cmd = [self.box_exec] + (["--cg"] if self.cgroup else []) \
+ ["--box-id=%d" % self.box_id] + ["--init"]
ret = subprocess.call(box_cmd)
if ret != 0:
raise SandboxInterfaceException(
"Failed to initialize sandbox with command: %s "
"(error %d)" % (pretty_print_cmdline(box_cmd), ret))
self._has_cleanedup = False
def _popen(self, command,
stdin=None, stdout=None, stderr=None,
close_fds=True):
"""Execute the given command in the sandbox using
subprocess.Popen, assigning the corresponding standard file
descriptors.
command ([string]): executable filename and arguments of the
command.
stdin (int|None): a file descriptor.
stdout (int|None): a file descriptor.
stderr (int|None): a file descriptor.
close_fds (bool): close all file descriptor before executing.
return (Popen): popen object.
"""
self.log = None
self.exec_num += 1
# We run a selection of commands without isolate, as they need
# to create new files. This is safe because these commands do
# not depend on the user input.
if command[0] in IsolateSandbox.SECURE_COMMANDS:
logger.debug("Executing non-securely: %s at %s",
pretty_print_cmdline(command), self.path)
try:
prev_permissions = stat.S_IMODE(os.stat(self.path).st_mode)
os.chmod(self.path, 0o700)
with io.open(self.relative_path(self.cmd_file), 'at') as cmds:
cmds.write("%s\n" % (pretty_print_cmdline(command)))
p = subprocess.Popen(command, cwd=self.path,
stdin=stdin, stdout=stdout, stderr=stderr,
close_fds=close_fds)
os.chmod(self.path, prev_permissions)
# For secure commands, we clear the output so that it
# is not forwarded to the contestants. Secure commands
# are "setup" commands, which should not fail or
# provide information for the contestants.
open(os.path.join(self.path, self.stdout_file), "w").close()
open(os.path.join(self.path, self.stderr_file), "w").close()
self._write_empty_run_log(self.exec_num)
except OSError:
logger.critical(
"Failed to execute program in sandbox with command: %s",
pretty_print_cmdline(command), exc_info=True)
raise
return p
args = [self.box_exec] + self.build_box_options() + ["--"] + command
logger.debug("Executing program in sandbox with command: `%s'.",
pretty_print_cmdline(args))
# Temporarily allow writing new files.
prev_permissions = stat.S_IMODE(os.stat(self.path).st_mode)
os.chmod(self.path, 0o770)
with io.open(self.relative_path(self.cmd_file), 'at') as commands:
commands.write("%s\n" % (pretty_print_cmdline(args)))
os.chmod(self.path, prev_permissions)
try:
p = subprocess.Popen(args,
stdin=stdin, stdout=stdout, stderr=stderr,
close_fds=close_fds)
except OSError:
logger.critical("Failed to execute program in sandbox "
"with command: %s", pretty_print_cmdline(args),
exc_info=True)
raise
return p
def execute(self, entry):
"""Print a print job.
This is the core of PrintingService.
entry (QueueEntry): the entry containing the operation to
perform.
"""
# TODO: automatically re-enqueue in case of a recoverable
# error.
printjob_id = entry.item.printjob_id
with SessionGen() as session:
# Obtain print job.
printjob = PrintJob.get_from_id(printjob_id, session)
if printjob is None:
raise ValueError("Print job %d not found in the database." %
printjob_id)
user = printjob.participation.user
contest = printjob.participation.contest
timezone = get_timezone(user, contest)
timestr = format_datetime(printjob.timestamp, timezone)
filename = printjob.filename
# Check if it's ready to be printed.
if printjob.done:
logger.info("Print job %d was already sent to the printer.",
printjob_id)
directory = tempfile.mkdtemp(dir=config.temp_dir)
logger.info("Preparing print job in directory %s", directory)
# Take the base name just to be sure.
relname = "source_" + os.path.basename(filename)
source = os.path.join(directory, relname)
with open(source, "wb") as file_:
self.file_cacher.get_file_to_fobj(printjob.digest, file_)
if filename.endswith(".pdf") and config.pdf_printing_allowed:
source_pdf = source
else:
# Convert text to ps.
source_ps = os.path.join(directory, "source.ps")
cmd = ["a2ps",
source,
"--delegate=no",
"--output=" + source_ps,
"--medium=%s" % config.paper_size.capitalize(),
"--portrait",
"--columns=1",
"--rows=1",
"--pages=1-%d" % (config.max_pages_per_job),
"--header=",
"--footer=",
"--left-footer=",
"--right-footer=",
"--center-title=" + filename,
"--left-title=" + timestr]
ret = subprocess.call(cmd, cwd=directory)
if ret != 0:
raise Exception(
"Failed to convert text file to ps with command: %s"
"(error %d)" % (pretty_print_cmdline(cmd), ret))
if not os.path.exists(source_ps):
logger.warning("Unable to convert from text to ps.")
printjob.done = True
printjob.status = json.dumps([
N_("Invalid file")])
session.commit()
rmtree(directory)
return
# Convert ps to pdf
source_pdf = os.path.join(directory, "source.pdf")
cmd = ["ps2pdf",
"-sPAPERSIZE=%s" % config.paper_size.lower(),
source_ps]
ret = subprocess.call(cmd, cwd=directory)
if ret != 0:
raise Exception(
"Failed to convert ps file to pdf with command: %s"
"(error %d)" % (pretty_print_cmdline(cmd), ret))
# Find out number of pages
with open(source_pdf, "rb") as file_:
pdfreader = PdfFileReader(file_)
page_count = pdfreader.getNumPages()
logger.info("Preparing %d page(s) (plus the title page)",
page_count)
if page_count > config.max_pages_per_job:
logger.info("Too many pages.")
printjob.done = True
printjob.status = json.dumps([
N_("Print job has too many pages")])
session.commit()
rmtree(directory)
return
# Add the title page
title_tex = os.path.join(directory, "title_page.tex")
title_pdf = os.path.join(directory, "title_page.pdf")
with open(title_tex, "w") as f:
f.write(self.template_loader.load("title_page.tex")
.generate(user=user, filename=filename,
timestr=timestr,
page_count=page_count,
paper_size=config.paper_size))
cmd = ["pdflatex",
"-interaction",
"nonstopmode",
title_tex]
ret = subprocess.call(cmd, cwd=directory)
if ret != 0:
raise Exception(
"Failed to create title page with command: %s"
"(error %d)" % (pretty_print_cmdline(cmd), ret))
pdfmerger = PdfFileMerger()
with open(title_pdf, "rb") as file_:
pdfmerger.append(file_)
with open(source_pdf, "rb") as file_:
pdfmerger.append(file_)
result = os.path.join(directory, "document.pdf")
with open(result, "wb") as file_:
pdfmerger.write(file_)
try:
printer_connection = cups.Connection()
printer_connection.printFile(
config.printer, result,
"Printout %d" % printjob_id, {})
except cups.IPPError as error:
logger.error("Unable to print: `%s'.", error)
else:
printjob.done = True
printjob.status = json.dumps([N_("Sent to printer")])
session.commit()
finally:
rmtree(directory)
def __init__(self, multithreaded, file_cacher, name=None, temp_dir=None):
"""Initialization.
For arguments documentation, see SandboxBase.__init__.
"""
SandboxBase.__init__(self, multithreaded, file_cacher, name, temp_dir)
# Isolate only accepts ids between 0 and 99. We assign the
# range [(shard+1)*10, (shard+2)*10) to each Worker and keep
# the range [0, 10) for other uses (command-line scripts like
# cmsMake or direct console users of isolate). Inside each
# range ids are assigned sequentially, with a wrap-around.
# FIXME This is the only use of FileCacher.service, and it's an
# improper use! Avoid it!
if file_cacher is not None and file_cacher.service is not None:
box_id = ((file_cacher.service.shard + 1) * 10
+ (IsolateSandbox.next_id % 10)) % 100
else:
box_id = IsolateSandbox.next_id % 10
IsolateSandbox.next_id += 1
# We create a directory "tmp" inside the outer temporary directory,
# because the sandbox will bind-mount the inner one. The sandbox also
# runs code as a different user, and so we need to ensure that they can
# read and write to the directory. But we don't want everybody on the
# system to, which is why the outer directory exists with no read
# permissions.
self.inner_temp_dir = "/tmp"
self.outer_temp_dir = tempfile.mkdtemp(
dir=self.temp_dir,
prefix="cms-%s-" % (self.name))
# Don't use os.path.join here, because the absoluteness of /tmp will
# bite you.
self.path = self.outer_temp_dir + self.inner_temp_dir
os.mkdir(self.path)
self.allow_writing_all()
self.exec_name = 'isolate'
self.box_exec = self.detect_box_executable()
self.info_basename = "run.log" # Used for -M
self.log = None
self.exec_num = -1
logger.debug("Sandbox in `%s' created, using box `%s'.",
self.path, self.box_exec)
# Default parameters for isolate
self.box_id = box_id # -b
self.cgroup = config.use_cgroups # --cg
self.chdir = self.inner_temp_dir # -c
self.dirs = [] # -d
self.dirs += [(self.inner_temp_dir, self.path, "rw")]
self.preserve_env = False # -e
self.inherit_env = [] # -E
self.set_env = {} # -E
self.fsize = None # -f
self.stdin_file = None # -i
self.stack_space = None # -k
self.address_space = None # -m
self.stdout_file = None # -o
self.stderr_file = None # -r
self.timeout = None # -t
self.verbosity = 0 # -v
self.wallclock_timeout = None # -w
self.extra_timeout = None # -x
# Set common environment variables.
# Specifically needed by Python, that searches the home for
# packages.
self.set_env["HOME"] = "./"
# Needed on Ubuntu by PHP (and more, ) that
# have in /usr/bin only a symlink to one out of many
# alternatives.
if os.path.isdir("/etc/alternatives"):
self.add_mapped_directories(["/etc/alternatives"])
# Tell isolate to get the sandbox ready. We do our best to
# cleanup after ourselves, but we might have missed something
# if the worker was interrupted in the middle of an execution.
self.cleanup()
init_cmd = [self.box_exec] + (["--cg"] if self.cgroup else []) \
+ ["--box-id=%d" % self.box_id] + ["--init"]
ret = subprocess.call(init_cmd)
if ret != 0:
raise SandboxInterfaceException(
"Failed to initialize sandbox with command: %s "
"(error %d)" % (pretty_print_cmdline(init_cmd), ret))
def _popen(self, command,
stdin=None, stdout=None, stderr=None,
close_fds=True):
"""Execute the given command in the sandbox using
subprocess.Popen, assigning the corresponding standard file
descriptors.
command ([string]): executable filename and arguments of the
command.
stdin (int|None): a file descriptor.
stdout (int|None): a file descriptor.
stderr (int|None): a file descriptor.
close_fds (bool): close all file descriptor before executing.
return (Popen): popen object.
"""
self.log = None
self.exec_num += 1
# We run a selection of commands without isolate, as they need
# to create new files. This is safe because these commands do
# not depend on the user input.
if command[0] in IsolateSandbox.SECURE_COMMANDS:
logger.debug("Executing non-securely: %s at %s",
pretty_print_cmdline(command), self.path)
try:
prev_permissions = stat.S_IMODE(os.stat(self.path).st_mode)
os.chmod(self.path, 0o700)
with io.open(self.relative_path(self.cmd_file), 'at') as cmds:
cmds.write("%s\n" % (pretty_print_cmdline(command)))
p = subprocess.Popen(command, cwd=self.path,
stdin=stdin, stdout=stdout, stderr=stderr,
close_fds=close_fds)
os.chmod(self.path, prev_permissions)
# For secure commands, we clear the output so that it
# is not forwarded to the contestants. Secure commands
# are "setup" commands, which should not fail or
# provide information for the contestants.
open(os.path.join(self.path, self.stdout_file), "w").close()
open(os.path.join(self.path, self.stderr_file), "w").close()
self._write_empty_run_log(self.exec_num)
except OSError:
logger.critical(
"Failed to execute program in sandbox with command: %s",
pretty_print_cmdline(command), exc_info=True)
raise
return p
args = [self.box_exec] + self.build_box_options() + ["--"] + command
logger.debug("Executing program in sandbox with command: `%s'.",
pretty_print_cmdline(args))
# Temporarily allow writing new files.
prev_permissions = stat.S_IMODE(os.stat(self.path).st_mode)
os.chmod(self.path, 0o770)
with io.open(self.relative_path(self.cmd_file), 'at') as commands:
commands.write("%s\n" % (pretty_print_cmdline(args)))
os.chmod(self.path, prev_permissions)
try:
p = subprocess.Popen(args,
stdin=stdin, stdout=stdout, stderr=stderr,
close_fds=close_fds)
except OSError:
logger.critical("Failed to execute program in sandbox "
"with command: %s", pretty_print_cmdline(args),
exc_info=True)
raise
return p
def test_success(self):
self.assertEqual(
pretty_print_cmdline(["ls", "-al", "file"]),
"ls -al file")
def test_spaces(self):
self.assertEqual(
pretty_print_cmdline(["ls", "-al", "file with spaces"]),
"ls -al 'file with spaces'")
def test_quotes(self):
self.assertEqual(
pretty_print_cmdline(["ls", "-al", "file'with'quotes"]),
"""ls -al 'file'"'"'with'"'"'quotes'""")
def test_double_quotes(self):
self.assertEqual(
pretty_print_cmdline(["ls", "-al", "file\"with\"dblquotes"]),
"""ls -al 'file"with"dblquotes'""")
