def _popen(self, command, stdin=None, stdout=None, stderr=None, close_fds=True): """Execute the given command in the sandbox using subprocess.Popen, assigning the corresponding standard file descriptors. command ([string]): executable filename and arguments of the command. stdin (int|None): a file descriptor. stdout (int|None): a file descriptor. stderr (int|None): a file descriptor. close_fds (bool): close all file descriptor before executing. return (Popen): popen object. """ self.exec_num += 1 self.log = None args = [self.box_exec] + self.build_box_options() + ["--"] + command logger.debug("Executing program in sandbox with command: `%s'.", pretty_print_cmdline(args)) with io.open(self.relative_path(self.cmd_file), 'at') as commands: commands.write("%s\n" % (pretty_print_cmdline(args))) try: p = subprocess.Popen(args, stdin=stdin, stdout=stdout, stderr=stderr, close_fds=close_fds) except OSError: logger.critical("Failed to execute program in sandbox " "with command: %s", pretty_print_cmdline(args), exc_info=True) raise return p
def _execute(self, command): """Execute the given command in the sandbox. command (list): executable filename and arguments of the command. return (bool): True if the sandbox didn't report errors (caused by the sandbox itself), False otherwise """ self.exec_num += 1 self.log = None args = [self.box_exec] + self.build_box_options() + ["--"] + command logger.debug("Executing program in sandbox with command: `%s'.", pretty_print_cmdline(args)) with io.open(self.relative_path(self.cmd_file), 'at') as commands: commands.write("%s\n" % (pretty_print_cmdline(args))) return self.translate_box_exitcode(subprocess.call(args))
def initialize_isolate(self): """Initialize isolate's box.""" init_cmd = ([self.box_exec] + (["--cg"] if self.cgroup else []) + ["--box-id=%d" % self.box_id, "--init"]) ret = subprocess.call(init_cmd) if ret != 0: raise SandboxInterfaceException( "Failed to initialize sandbox with command: %s " "(error %d)" % (pretty_print_cmdline(init_cmd), ret))
def sh(cmdline, ignore_failure=False): """Execute a simple command. cmd ([str]): the (unescaped) command to execute. ignore_failure (bool): whether to suppress failures. raise (TestException): if the command failed and ignore_failure was False. """ if CONFIG["VERBOSITY"] >= 1: logger.info('$ %s', pretty_print_cmdline(cmdline)) kwargs = dict() if CONFIG["VERBOSITY"] >= 3: kwargs["stdout"] = subprocess.DEVNULL kwargs["stderr"] = subprocess.STDOUT ret = subprocess.call(cmdline, **kwargs) if not ignore_failure and ret != 0: raise TestException( "Execution failed with %d/%d. Tried to execute:\n%s\n" % (ret & 0xff, ret >> 8, pretty_print_cmdline(cmdline)))
def _popen(self, command, stdin=None, stdout=None, stderr=None, preexec_fn=None, close_fds=True): """Execute the given command in the sandbox using subprocess.Popen, assigning the corresponding standard file descriptors. command ([string]): executable filename and arguments of the command. stdin (file|None): a file descriptor/object or None. stdout (file|None): a file descriptor/object or None. stderr (file|None): a file descriptor/object or None. preexec_fn (function|None): to be called just before execve() or None. close_fds (bool): close all file descriptor before executing. return (object): popen object. """ self.exec_time = None self.exec_num += 1 logger.debug("Executing program in sandbox with command: `%s'.", " ".join(command)) with io.open(self.relative_path(self.cmd_file), 'at') as commands: commands.write("%s\n" % (pretty_print_cmdline(command))) try: p = subprocess.Popen(command, stdin=stdin, stdout=stdout, stderr=stderr, preexec_fn=preexec_fn, close_fds=close_fds) except OSError: logger.critical( "Failed to execute program in sandbox " "with command: `%s'.", " ".join(command), exc_info=True) raise return p
def sh(cmdline, ignore_failure=False): """Execute a simple command. cmd ([str]): the (unescaped) command to execute. ignore_failure (bool): whether to suppress failures. raise (TestException): if the command failed and ignore_failure was False. """ if CONFIG["VERBOSITY"] >= 1: logger.info('$ %s', pretty_print_cmdline(cmdline)) kwargs = dict() if CONFIG["VERBOSITY"] >= 3: kwargs["stdout"] = subprocess.DEVNULL kwargs["stderr"] = subprocess.STDOUT kwargs["check"] = not ignore_failure try: subprocess.run(cmdline, **kwargs) except subprocess.CalledProcessError as e: raise TestException("Execution failed") from e
def _popen(self, command, stdin=None, stdout=None, stderr=None, preexec_fn=None, close_fds=True): """Execute the given command in the sandbox using subprocess.Popen, assigning the corresponding standard file descriptors. command ([string]): executable filename and arguments of the command. stdin (file|None): a file descriptor/object or None. stdout (file|None): a file descriptor/object or None. stderr (file|None): a file descriptor/object or None. preexec_fn (function|None): to be called just before execve() or None. close_fds (bool): close all file descriptor before executing. return (object): popen object. """ self.exec_time = None self.exec_num += 1 logger.debug("Executing program in sandbox with command: `%s'.", " ".join(command)) with io.open(self.relative_path(self.cmd_file), 'at') as commands: commands.write("%s\n" % (pretty_print_cmdline(command))) try: p = subprocess.Popen(command, stdin=stdin, stdout=stdout, stderr=stderr, preexec_fn=preexec_fn, close_fds=close_fds) except OSError: logger.critical("Failed to execute program in sandbox " "with command: `%s'.", " ".join(command), exc_info=True) raise return p
def test_double_quotes(self): self.assertEqual( pretty_print_cmdline(["ls", "-al", "file\"with\"dblquotes"]), """ls -al 'file"with"dblquotes'""")
def test_quotes(self): self.assertEqual( pretty_print_cmdline(["ls", "-al", "file'with'quotes"]), """ls -al 'file'"'"'with'"'"'quotes'""")
def test_spaces(self): self.assertEqual( pretty_print_cmdline(["ls", "-al", "file with spaces"]), "ls -al 'file with spaces'")
def test_success(self): self.assertEqual( pretty_print_cmdline(["ls", "-al", "file"]), "ls -al file")
def __init__(self, multithreaded, file_cacher, temp_dir=None): """Initialization. For arguments documentation, see SandboxBase.__init__. """ SandboxBase.__init__(self, multithreaded, file_cacher, temp_dir) # Isolate only accepts ids between 0 and 99. We assign the # range [(shard+1)*10, (shard+2)*10) to each Worker and keep # the range [0, 10) for other uses (command-line scripts like # cmsMake or direct console users of isolate). Inside each # range ids are assigned sequentially, with a wrap-around. # FIXME This is the only use of FileCacher.service, and it's an # improper use! Avoid it! if file_cacher is not None and file_cacher.service is not None: box_id = ((file_cacher.service.shard + 1) * 10 + (IsolateSandbox.next_id % 10)) % 100 else: box_id = IsolateSandbox.next_id % 10 IsolateSandbox.next_id += 1 # We create a directory "tmp" inside the outer temporary directory, # because the sandbox will bind-mount the inner one. The sandbox also # runs code as a different user, and so we need to ensure that they can # read and write to the directory. But we don't want everybody on the # system to, which is why the outer directory exists with no read # permissions. self.inner_temp_dir = "/tmp" self.outer_temp_dir = tempfile.mkdtemp(dir=self.temp_dir) # Don't use os.path.join here, because the absoluteness of /tmp will # bite you. self.path = self.outer_temp_dir + self.inner_temp_dir os.mkdir(self.path) self.allow_writing_all() self.exec_name = 'isolate' self.box_exec = self.detect_box_executable() self.info_basename = "run.log" # Used for -M self.log = None self.exec_num = -1 logger.debug("Sandbox in `%s' created, using box `%s'.", self.path, self.box_exec) # Default parameters for isolate self.box_id = box_id # -b self.cgroup = config.use_cgroups # --cg self.chdir = self.inner_temp_dir # -c self.dirs = [] # -d self.dirs += [(self.inner_temp_dir, self.path, "rw")] self.preserve_env = False # -e self.inherit_env = [] # -E self.set_env = {} # -E self.fsize = None # -f self.stdin_file = None # -i self.stack_space = None # -k self.address_space = None # -m self.stdout_file = None # -o self.stderr_file = None # -r self.timeout = None # -t self.verbosity = 0 # -v self.wallclock_timeout = None # -w self.extra_timeout = None # -x # Set common environment variables. # Specifically needed by Python, that searches the home for # packages. self.set_env["HOME"] = "./" # Needed on Ubuntu by PHP (and more, ) that # have in /usr/bin only a symlink to one out of many # alternatives. if os.path.isdir("/etc/alternatives"): self.add_mapped_directories(["/etc/alternatives"]) # Tell isolate to get the sandbox ready. box_cmd = [self.box_exec] + (["--cg"] if self.cgroup else []) \ + ["--box-id=%d" % self.box_id] + ["--init"] ret = subprocess.call(box_cmd) if ret != 0: raise SandboxInterfaceException( "Failed to initialize sandbox with command: %s " "(error %d)" % (pretty_print_cmdline(box_cmd), ret)) self._has_cleanedup = False
def _popen(self, command, stdin=None, stdout=None, stderr=None, close_fds=True): """Execute the given command in the sandbox using subprocess.Popen, assigning the corresponding standard file descriptors. command ([string]): executable filename and arguments of the command. stdin (int|None): a file descriptor. stdout (int|None): a file descriptor. stderr (int|None): a file descriptor. close_fds (bool): close all file descriptor before executing. return (Popen): popen object. """ self.log = None self.exec_num += 1 # We run a selection of commands without isolate, as they need # to create new files. This is safe because these commands do # not depend on the user input. if command[0] in IsolateSandbox.SECURE_COMMANDS: logger.debug("Executing non-securely: %s at %s", pretty_print_cmdline(command), self.path) try: prev_permissions = stat.S_IMODE(os.stat(self.path).st_mode) os.chmod(self.path, 0o700) with io.open(self.relative_path(self.cmd_file), 'at') as cmds: cmds.write("%s\n" % (pretty_print_cmdline(command))) p = subprocess.Popen(command, cwd=self.path, stdin=stdin, stdout=stdout, stderr=stderr, close_fds=close_fds) os.chmod(self.path, prev_permissions) # For secure commands, we clear the output so that it # is not forwarded to the contestants. Secure commands # are "setup" commands, which should not fail or # provide information for the contestants. open(os.path.join(self.path, self.stdout_file), "w").close() open(os.path.join(self.path, self.stderr_file), "w").close() self._write_empty_run_log(self.exec_num) except OSError: logger.critical( "Failed to execute program in sandbox with command: %s", pretty_print_cmdline(command), exc_info=True) raise return p args = [self.box_exec] + self.build_box_options() + ["--"] + command logger.debug("Executing program in sandbox with command: `%s'.", pretty_print_cmdline(args)) # Temporarily allow writing new files. prev_permissions = stat.S_IMODE(os.stat(self.path).st_mode) os.chmod(self.path, 0o770) with io.open(self.relative_path(self.cmd_file), 'at') as commands: commands.write("%s\n" % (pretty_print_cmdline(args))) os.chmod(self.path, prev_permissions) try: p = subprocess.Popen(args, stdin=stdin, stdout=stdout, stderr=stderr, close_fds=close_fds) except OSError: logger.critical("Failed to execute program in sandbox " "with command: %s", pretty_print_cmdline(args), exc_info=True) raise return p
def execute(self, entry): """Print a print job. This is the core of PrintingService. entry (QueueEntry): the entry containing the operation to perform. """ # TODO: automatically re-enqueue in case of a recoverable # error. printjob_id = entry.item.printjob_id with SessionGen() as session: # Obtain print job. printjob = PrintJob.get_from_id(printjob_id, session) if printjob is None: raise ValueError("Print job %d not found in the database." % printjob_id) user = printjob.participation.user contest = printjob.participation.contest timezone = get_timezone(user, contest) timestr = format_datetime(printjob.timestamp, timezone) filename = printjob.filename # Check if it's ready to be printed. if printjob.done: logger.info("Print job %d was already sent to the printer.", printjob_id) directory = tempfile.mkdtemp(dir=config.temp_dir) logger.info("Preparing print job in directory %s", directory) # Take the base name just to be sure. relname = "source_" + os.path.basename(filename) source = os.path.join(directory, relname) with open(source, "wb") as file_: self.file_cacher.get_file_to_fobj(printjob.digest, file_) if filename.endswith(".pdf") and config.pdf_printing_allowed: source_pdf = source else: # Convert text to ps. source_ps = os.path.join(directory, "source.ps") cmd = ["a2ps", source, "--delegate=no", "--output=" + source_ps, "--medium=%s" % config.paper_size.capitalize(), "--portrait", "--columns=1", "--rows=1", "--pages=1-%d" % (config.max_pages_per_job), "--header=", "--footer=", "--left-footer=", "--right-footer=", "--center-title=" + filename, "--left-title=" + timestr] ret = subprocess.call(cmd, cwd=directory) if ret != 0: raise Exception( "Failed to convert text file to ps with command: %s" "(error %d)" % (pretty_print_cmdline(cmd), ret)) if not os.path.exists(source_ps): logger.warning("Unable to convert from text to ps.") printjob.done = True printjob.status = json.dumps([ N_("Invalid file")]) session.commit() rmtree(directory) return # Convert ps to pdf source_pdf = os.path.join(directory, "source.pdf") cmd = ["ps2pdf", "-sPAPERSIZE=%s" % config.paper_size.lower(), source_ps] ret = subprocess.call(cmd, cwd=directory) if ret != 0: raise Exception( "Failed to convert ps file to pdf with command: %s" "(error %d)" % (pretty_print_cmdline(cmd), ret)) # Find out number of pages with open(source_pdf, "rb") as file_: pdfreader = PdfFileReader(file_) page_count = pdfreader.getNumPages() logger.info("Preparing %d page(s) (plus the title page)", page_count) if page_count > config.max_pages_per_job: logger.info("Too many pages.") printjob.done = True printjob.status = json.dumps([ N_("Print job has too many pages")]) session.commit() rmtree(directory) return # Add the title page title_tex = os.path.join(directory, "title_page.tex") title_pdf = os.path.join(directory, "title_page.pdf") with open(title_tex, "w") as f: f.write(self.template_loader.load("title_page.tex") .generate(user=user, filename=filename, timestr=timestr, page_count=page_count, paper_size=config.paper_size)) cmd = ["pdflatex", "-interaction", "nonstopmode", title_tex] ret = subprocess.call(cmd, cwd=directory) if ret != 0: raise Exception( "Failed to create title page with command: %s" "(error %d)" % (pretty_print_cmdline(cmd), ret)) pdfmerger = PdfFileMerger() with open(title_pdf, "rb") as file_: pdfmerger.append(file_) with open(source_pdf, "rb") as file_: pdfmerger.append(file_) result = os.path.join(directory, "document.pdf") with open(result, "wb") as file_: pdfmerger.write(file_) try: printer_connection = cups.Connection() printer_connection.printFile( config.printer, result, "Printout %d" % printjob_id, {}) except cups.IPPError as error: logger.error("Unable to print: `%s'.", error) else: printjob.done = True printjob.status = json.dumps([N_("Sent to printer")]) session.commit() finally: rmtree(directory)
def __init__(self, multithreaded, file_cacher, name=None, temp_dir=None): """Initialization. For arguments documentation, see SandboxBase.__init__. """ SandboxBase.__init__(self, multithreaded, file_cacher, name, temp_dir) # Isolate only accepts ids between 0 and 99. We assign the # range [(shard+1)*10, (shard+2)*10) to each Worker and keep # the range [0, 10) for other uses (command-line scripts like # cmsMake or direct console users of isolate). Inside each # range ids are assigned sequentially, with a wrap-around. # FIXME This is the only use of FileCacher.service, and it's an # improper use! Avoid it! if file_cacher is not None and file_cacher.service is not None: box_id = ((file_cacher.service.shard + 1) * 10 + (IsolateSandbox.next_id % 10)) % 100 else: box_id = IsolateSandbox.next_id % 10 IsolateSandbox.next_id += 1 # We create a directory "tmp" inside the outer temporary directory, # because the sandbox will bind-mount the inner one. The sandbox also # runs code as a different user, and so we need to ensure that they can # read and write to the directory. But we don't want everybody on the # system to, which is why the outer directory exists with no read # permissions. self.inner_temp_dir = "/tmp" self.outer_temp_dir = tempfile.mkdtemp( dir=self.temp_dir, prefix="cms-%s-" % (self.name)) # Don't use os.path.join here, because the absoluteness of /tmp will # bite you. self.path = self.outer_temp_dir + self.inner_temp_dir os.mkdir(self.path) self.allow_writing_all() self.exec_name = 'isolate' self.box_exec = self.detect_box_executable() self.info_basename = "run.log" # Used for -M self.log = None self.exec_num = -1 logger.debug("Sandbox in `%s' created, using box `%s'.", self.path, self.box_exec) # Default parameters for isolate self.box_id = box_id # -b self.cgroup = config.use_cgroups # --cg self.chdir = self.inner_temp_dir # -c self.dirs = [] # -d self.dirs += [(self.inner_temp_dir, self.path, "rw")] self.preserve_env = False # -e self.inherit_env = [] # -E self.set_env = {} # -E self.fsize = None # -f self.stdin_file = None # -i self.stack_space = None # -k self.address_space = None # -m self.stdout_file = None # -o self.stderr_file = None # -r self.timeout = None # -t self.verbosity = 0 # -v self.wallclock_timeout = None # -w self.extra_timeout = None # -x # Set common environment variables. # Specifically needed by Python, that searches the home for # packages. self.set_env["HOME"] = "./" # Needed on Ubuntu by PHP (and more, ) that # have in /usr/bin only a symlink to one out of many # alternatives. if os.path.isdir("/etc/alternatives"): self.add_mapped_directories(["/etc/alternatives"]) # Tell isolate to get the sandbox ready. We do our best to # cleanup after ourselves, but we might have missed something # if the worker was interrupted in the middle of an execution. self.cleanup() init_cmd = [self.box_exec] + (["--cg"] if self.cgroup else []) \ + ["--box-id=%d" % self.box_id] + ["--init"] ret = subprocess.call(init_cmd) if ret != 0: raise SandboxInterfaceException( "Failed to initialize sandbox with command: %s " "(error %d)" % (pretty_print_cmdline(init_cmd), ret))