Beispiel #1
0
 def _make_cmdline(self, executor):
     cmdline = [
         noquote(tempcwd(self.source_file))
     ]  #'/usr/bin/python3compile.py',  tempcwd(self.source_file)]
     os.chmod(tempcwd(os.path.dirname(self.source_file)), 0777)
     os.chmod(tempcwd(self.source_file), 0777)
     return cmdline
Beispiel #2
0
    def _make_cmdline(self, executor):
        cmdline = [self.compiler, tempcwd(self.source_file),
                    '-o', tempcwd(self.output_file)] + \
                    self.options + list(self.extra_compilation_args)

        cmdline.extend(tempcwd(os.path.basename(source))
            for source in self.additional_sources)
        return cmdline
Beispiel #3
0
    def _make_cmdline(self, executor):
        cmdline = [self.compiler, tempcwd(self.source_file),
                    '-o', tempcwd(self.output_file)] + \
                    self.options + list(self.extra_compilation_args)

        cmdline.extend(
            tempcwd(os.path.basename(source))
            for source in self.additional_sources)
        return cmdline
Beispiel #4
0
def test_zip():
    with TemporaryCwd():
        upload_files()
        compile_and_run(
            "/echo.c", {
                'in_file': '/input.zip',
                'out_file': '/output',
                'exec_mem_limit': 102400
            }, DetailedUnprotectedExecutor())
        ft.download({'in_file': '/input'}, 'in_file', 'out.expected')
        ft.download({'out_file': '/output'}, 'out_file', 'out.real')
        ok_(filecmp.cmp(tempcwd('out.expected'), tempcwd('out.real')))
Beispiel #5
0
def test_zip():
    with TemporaryCwd():
        upload_files()
        compile_and_run("/echo.c", {
            'in_file': '/input.zip',
            'out_file': '/output',
            'exec_mem_limit': 102400
            }, DetailedUnprotectedExecutor())
        ft.download({'in_file': '/input'}, 'in_file', 'out.expected')
        ft.download({'out_file': '/output'}, 'out_file', 'out.real')
        ok_(filecmp.cmp(tempcwd('out.expected'),
                        tempcwd('out.real')))
Beispiel #6
0
def test_execute():
    with TemporaryCwd():
        rc, out = execute(['echo', '2'])
        eq_(rc, 0)
        eq_(out, '2\n')
        rc, out = execute(['exit', '1'], ignore_errors=True)
        eq_(rc, 1)

        assert_raises(ExecError, execute, ['exit', '1'])

        rc, out = execute(['mkdir', tempcwd('spam')])
        eq_(rc, 0)
        rc, out = execute(['ls', tempcwd()])
        in_('spam', out)
Beispiel #7
0
def test_execute():
    with TemporaryCwd():
        rc, out = execute(['echo', '2'])
        eq_(rc, 0)
        eq_(out, '2\n')
        rc, out = execute(['exit', '1'], ignore_errors=True)
        eq_(rc, 1)

        assert_raises(ExecError, execute, ['exit', '1'])

        rc, out = execute(['mkdir', tempcwd('spam')])
        eq_(rc, 0)
        rc, out = execute(['ls', tempcwd()])
        in_('spam', out)
Beispiel #8
0
def test_output_compilation_and_running(source):
    with TemporaryCwd():
        upload_files()
        result_env = run(
            {
                'source_file': source,
                'compiler': 'output-only',
            }
        )
        eq_(result_env['result_code'], 'OK')
        eq_(result_env['exec_info'], {'mode': 'output-only'})

        ft.download(result_env, 'out_file', tempcwd('out.txt'))
        ft.download({'source_file': source}, 'source_file', tempcwd('source.txt'))
        with open(tempcwd('out.txt'), 'r') as outfile:
            with open(tempcwd('source.txt'), 'r') as sourcefile:
                eq_(outfile.read(), sourcefile.read())

        post_run_env = run_from_executors(
            {
                'exec_info': result_env['exec_info'],
                'exe_file': result_env['out_file'],
                'check_output': True,
                'hint_file': source,
            },
            executor=None,
        )
        eq_(post_run_env['result_code'], 'OK')

        ft.download(post_run_env, 'out_file', tempcwd('out.txt'))
        ft.download({'source_file': source}, 'source_file', tempcwd('source.txt'))
        with open(tempcwd('out.txt'), 'r') as outfile:
            with open(tempcwd('source.txt'), 'r') as sourcefile:
                eq_(outfile.read(), sourcefile.read())
Beispiel #9
0
def compile_and_execute(source, executor, **exec_args):
    cenv = compile(source,
                   use_sandboxes=isinstance(executor, _SIOSupervisedExecutor))
    frunner = get_file_runner(executor, cenv)

    ft.download({'exe_file': cenv['out_file']}, 'exe_file',
                frunner.preferred_filename())
    os.chmod(tempcwd('exe'), 0700)
    ft.download({'in_file': '/input'}, 'in_file', 'in')

    with frunner:
        with open(tempcwd('in'), 'rb') as inf:
            renv = frunner(tempcwd('exe'), [], stdin=inf, **exec_args)

    return renv
Beispiel #10
0
def _extract_all(archive_path):
    target_path = tempcwd()
    with ZipFile(tempcwd(archive_path), 'r') as zipf:
        for name in zipf.namelist():
            filename = name.rstrip('/')
            extract_path = os.path.join(target_path, filename)
            extract_path = os.path.normpath(os.path.realpath(extract_path))
            if os.path.exists(extract_path):
                logger.warning("Cannot extract %s, file already exists.",
                        extract_path)
            elif not extract_path.startswith(target_path):
                logger.warning("Cannot extract %s, target path outside "
                        "working directory.", extract_path)
            else:
                zipf.extract(name, target_path)
Beispiel #11
0
 def _postprocess(self, renv):
     self.environ['compiler_output'] = replace_invalid_UTF(renv['stdout'])
     if renv['return_code']:
         self.environ['result_code'] = 'CE'
     elif 'compilation_result_size_limit' in self.environ and \
             os.path.getsize(tempcwd(self.output_file)) > \
             self.environ['compilation_result_size_limit']:
         self.environ['result_code'] = 'CE'
         self.environ['compiler_output'] = \
                 'Compiled file size limit exceeded.'
     else:
         self.environ['result_code'] = 'OK'
         self.environ['exec_info'] = {'mode': 'executable'}
         ft.upload(self.environ, 'out_file', tempcwd(self.output_file))
     return self.environ
Beispiel #12
0
def compile_and_execute(source, executor, **exec_args):
    cenv = compile(source,
                   use_sandboxes=isinstance(executor, _SIOSupervisedExecutor))
    frunner = get_file_runner(executor, cenv)

    ft.download({'exe_file': cenv['out_file']}, 'exe_file',
                frunner.preferred_filename())
    os.chmod(tempcwd('exe'), 0700)
    ft.download({'in_file': '/input'}, 'in_file', 'in')

    with frunner:
        with open(tempcwd('in'), 'rb') as inf:
            renv = frunner(tempcwd('exe'), [], stdin=inf, **exec_args)

    return renv
Beispiel #13
0
def download(environ, key, dest=None, skip_if_exists=False, **kwargs):
    """Downloads the file from ``environ[key]`` and saves it to ``dest``.

       ``dest``
         A filename, directory name or ``None``. In the two latter cases,
         the file is named the same as in ``environ[key]``.

       ``skip_if_exists``
         If ``True`` and ``dest`` points to an existing file (not a directory
         or ``None``), then the file is not downloaded.

       ``**kwargs``
         Passed directly to :meth:`filetracker.Client.get_file`.

       The value under ``environ['use_filetracker']`` affects downloading
       in the followins way:

       * if ``True``, nothing special happens

       * if ``False``, the file is not downloaded from filetracker, but the
         passed path is assumed to be a regular filesystem path

       * if ``'auto'``, the file is assumed to be a local filename only if
         it is a relative path (this is usually the case when developers play).

       Returns the path to the saved file.
    """

    if dest and skip_if_exists and os.path.exists(util.tempcwd(dest)):
        return dest
    source = environ[key]
    if dest is None:
        dest = os.path.split(source)[1]
    elif dest.endswith(os.sep):
        dest = os.path.join(dest, os.path.split(source)[1])

    dest = util.tempcwd(dest)
    if not _use_filetracker(source, environ):
        source = os.path.join(_original_cwd, source)
        if not os.path.exists(dest) or not os.path.samefile(source, dest):
            shutil.copy(source, dest)
    else:
        kwargs.setdefault('add_to_cache', False)
        logger.debug("Downloading %s", source)
        perf_timer = util.PerfTimer()
        instance().get_file(source, dest, **kwargs)
        logger.debug(" completed in %.2fs", perf_timer.elapsed)
    return dest
Beispiel #14
0
def download(environ, key, dest=None, skip_if_exists=False, **kwargs):
    """Downloads the file from ``environ[key]`` and saves it to ``dest``.

    ``dest``
      A filename, directory name or ``None``. In the two latter cases,
      the file is named the same as in ``environ[key]``.

    ``skip_if_exists``
      If ``True`` and ``dest`` points to an existing file (not a directory
      or ``None``), then the file is not downloaded.

    ``**kwargs``
      Passed directly to :meth:`filetracker.client.Client.get_file`.

    The value under ``environ['use_filetracker']`` affects downloading
    in the followins way:

    * if ``True``, nothing special happens

    * if ``False``, the file is not downloaded from filetracker, but the
      passed path is assumed to be a regular filesystem path

    * if ``'auto'``, the file is assumed to be a local filename only if
      it is a relative path (this is usually the case when developers play).

    Returns the path to the saved file.
    """

    if dest and skip_if_exists and os.path.exists(util.tempcwd(dest)):
        return dest
    source = environ[key]
    if dest is None:
        dest = os.path.split(source)[1]
    elif dest.endswith(os.sep):
        dest = os.path.join(dest, os.path.split(source)[1])

    dest = util.tempcwd(dest)
    if not _use_filetracker(source, environ):
        source = os.path.join(_original_cwd, source)
        if not os.path.exists(dest) or not os.path.samefile(source, dest):
            shutil.copy(source, dest)
    else:
        kwargs.setdefault('add_to_cache', False)
        logger.debug("Downloading %s", source)
        perf_timer = util.PerfTimer()
        instance().get_file(source, dest, **kwargs)
        logger.debug(" completed in %.2fs", perf_timer.elapsed)
    return dest
Beispiel #15
0
def _extract_all(archive_path):
    target_path = tempcwd()
    with ZipFile(tempcwd(archive_path), 'r') as zipf:
        for name in zipf.namelist():
            filename = name.rstrip('/')
            extract_path = os.path.join(target_path, filename)
            extract_path = os.path.normpath(os.path.realpath(extract_path))
            if os.path.exists(extract_path):
                logger.warning("Cannot extract %s, file already exists.",
                               extract_path)
            elif not extract_path.startswith(target_path):
                logger.warning(
                    "Cannot extract %s, target path outside "
                    "working directory.", extract_path)
            else:
                zipf.extract(name, target_path)
Beispiel #16
0
def run(environ, use_sandboxes=True):
    ft.download(environ, 'out_file', 'out', skip_if_exists=True)
    ft.download(environ, 'hint_file', 'hint', add_to_cache=True)

    try:
        if environ.get('chk_file'):
            ft.download(environ, 'in_file', 'in', skip_if_exists=True,
                    add_to_cache=True)
            ft.download(environ, 'chk_file', 'chk', add_to_cache=True)
            os.chmod(tempcwd('chk'), 0o700)

            output = _run_checker(environ, use_sandboxes)
        elif use_sandboxes:
            output = _run_compare(environ)
        else:
            output = _run_diff(environ)
    except (CheckerError, ExecError) as e:
        logger.error('Checker failed! %s', e)
        logger.error('Environ dump: %s', environ)
        raise SystemError(e)

    while len(output) < 3:
        output.append('')
    if output[0] == b'OK':
        environ['result_code'] = 'OK'
        if output[1]:
            environ['result_string'] = _limit_length(output[1])
        environ['result_percentage'] = float(output[2] or 100)
    else:
        environ['result_code'] = 'WA'
        environ['result_string'] = _limit_length(output[1])
        environ['result_percentage'] = 0
    return environ
Beispiel #17
0
    def _run_in_executor(self, executor):
        javac = ['javac', '-J-Xss32M'] + list(self.extra_compilation_args) \
                + [tempcwd(self.source_file)]
        javac.extend(tempcwd(os.path.basename(source))
            for source in self.additional_sources)

        renv = self._execute(executor, javac)
        if renv['return_code']:
            return renv

        classes = [os.path.basename(x)
                   for x in glob.glob(tempcwd() + '/*.class')]
        jar = ['jar', 'cf', self.output_file] + classes
        renv2 = self._execute(executor, jar)
        renv2['stdout'] = renv['stdout'] + renv2['stdout']
        return renv2
Beispiel #18
0
def _run_ingen(environ, use_sandboxes=False):
    command = [tempcwd("ingen")]
    if use_sandboxes:
        executor = PRootExecutor("null-sandbox")
    else:
        executor = UnprotectedExecutor()
    return _run_in_executor(environ, command, executor, ignore_errors=True)
Beispiel #19
0
    def _postprocess(self, renv):
        self.environ['compiler_output'] = replace_invalid_UTF(renv['stdout'])
        if renv['return_code']:
            self.environ['result_code'] = 'CE'
        elif 'compilation_result_size_limit' in self.environ and \
                os.path.getsize(tempcwd(self.output_file)) > \
                self.environ['compilation_result_size_limit']:
            self.environ['result_code'] = 'CE'
            self.environ['compiler_output'] = \
                    'Compiled file size limit exceeded.'
        else:
            self.environ['result_code'] = 'OK'
            self.environ['exec_info'] = {'mode': 'executable'}
            ft.upload(self.environ, 'out_file', tempcwd(self.output_file))

        return self.environ
Beispiel #20
0
def _run_ingen(environ, use_sandboxes=False):
    command = [tempcwd('ingen')]
    if use_sandboxes:
        executor = PRootExecutor('null-sandbox')
    else:
        executor = UnprotectedExecutor()
    return _run_in_executor(environ, command, executor, ignore_errors=True)
Beispiel #21
0
def _run_inwer(environ, use_sandboxes=False):
    command = [tempcwd('inwer')]
    if use_sandboxes:
        executor = SupervisedExecutor()
    else:
        executor = DetailedUnprotectedExecutor()
    return _run_in_executor(environ, command, executor, ignore_errors=True)
Beispiel #22
0
def _run_inwer(environ, use_sandboxes=False):
    command = [tempcwd('inwer')]
    if use_sandboxes:
        executor = SupervisedExecutor()
    else:
        executor = DetailedUnprotectedExecutor()
    return _run_in_executor(environ, command, executor, ignore_errors=True)
Beispiel #23
0
def _test_transparent_exec(source, executor, callback, kwargs):
    with TemporaryCwd():
        upload_files()
        ft.download({'path': '/somefile'}, 'path', tempcwd('somefile'))
        result_env = compile_and_execute(source, executor, **kwargs)
        print_env(result_env)
        if callback:
            callback(result_env)
Beispiel #24
0
def _test_transparent_exec(source, executor, callback, kwargs):
    with TemporaryCwd():
        upload_files()
        ft.download({'path': '/somefile'}, 'path', tempcwd('somefile'))
        result_env = compile_and_execute(source, executor, **kwargs)
        print_env(result_env)
        if callback:
            callback(result_env)
Beispiel #25
0
def run(environ):
    """Runs a verifying program and returns its output.

       Used ``environ`` keys:

       ``exe_file``: the filetracker path to the program

       ``in_file``: the file redirected to the program's stdin

       ``use_sandboxes``: if this key equals ``True``, the program is executed
                        in the SupervisedExecutor, otherwise the UnsafeExecutor
                        is used

       ``inwer_time_limit``: time limit in ms
                           (optional, the default is 30 s)

       ``inwer_mem_limit``: memory limit in KiB
                           (optional, the default is 256 MiB)

       ``inwer_output_limit``: output limit in B
                           (optional, the default is 10 KiB)

       Returns a new environ, whose ``stdout`` key contains the program's
       output.

       The verifying program is expected to return 0, its first line of output
       should begin with "OK". If this does not happen, an appropriate message
       is logged.

       Program's output is returned under the ``stdout`` key. If the output has
       more than ``inwer_output_limit`` bytes and ``use_sandboxes`` is
       set to ``True``, the execution of the program fails with ``OLE`` result
       code.
    """

    use_sandboxes = environ.get('use_sandboxes', False)
    ft.download(environ,
                'exe_file',
                'inwer',
                skip_if_exists=True,
                add_to_cache=True)
    ft.download(environ,
                'in_file',
                'in',
                skip_if_exists=True,
                add_to_cache=True)
    os.chmod(tempcwd('inwer'), 0500)

    renv = _run_inwer(environ, use_sandboxes)
    if renv['result_code'] != "OK":
        logger.error("Inwer failed!\nEnviron dump: %s\nExecution environ: %s",
                     environ, renv)
    elif not renv['stdout'][0].startswith("OK"):
        logger.error(
            "Bad inwer output!\nEnviron dump: %s\n"
            "Execution environ: %s", environ, renv)

    return renv
Beispiel #26
0
    def _run_in_executor(self, executor):
        # Generate FPC configuration
        with open(os.path.join(executor.path, 'fpc.cfg.in')) as f:
            fpc_cfg = f.read()
        fpc_cfg = fpc_cfg.replace('__DIR__', executor.rpath.rstrip(os.sep))
        with open(tempcwd('fpc.cfg'), 'w') as f:
            f.write(fpc_cfg)

        return super(FPCCompiler, self)._run_in_executor(executor)
Beispiel #27
0
    def _run_in_executor(self, executor):
        javac = (['javac', '-J-Xss32M'] + list(self.extra_compilation_args) +
                 [tempcwd(self.source_file)])
        javac.extend(
            tempcwd(os.path.basename(source))
            for source in self.additional_sources)

        renv = self._execute(executor, javac)
        if renv['return_code']:
            return renv

        classes = [
            os.path.basename(x) for x in glob.glob(tempcwd() + '/*.class')
        ]
        jar = ['jar', 'cf', self.output_file] + classes
        renv2 = self._execute(executor, jar)
        renv2['stdout'] = renv['stdout'] + renv2['stdout']
        return renv2
Beispiel #28
0
    def _run_in_executor(self, executor):
        # Generate FPC configuration
        with open(os.path.join(executor.path, 'fpc.cfg.in')) as f:
            fpc_cfg = f.read()
        fpc_cfg = fpc_cfg.replace('__DIR__', executor.rpath.rstrip(os.sep))
        with open(tempcwd('fpc.cfg'), 'w') as f:
            f.write(fpc_cfg)

        return super(FPCCompiler, self)._run_in_executor(executor)
Beispiel #29
0
def _run_in_executor(environ, command, executor, **kwargs):
    with executor:
        with open(tempcwd('in'), 'rb') as inf:
            return executor(command, stdin=inf,
                capture_output=True, split_lines=True, forward_stderr=True,
                mem_limit=DEFAULT_INWER_MEM_LIMIT,
                time_limit=DEFAULT_INWER_TIME_LIMIT,
                output_limit=DEFAULT_INWER_OUTPUT_LIMIT,
                environ=environ, environ_prefix='inwer_', **kwargs)
Beispiel #30
0
def run(environ, executor, use_sandboxes=True):
    """
    Common code for executors.

    :param: environ Recipe to pass to `filetracker` and `sio.workers.executors`
                    For all supported options, see the global documentation for
                    `sio.workers.executors` and prefix them with ``exec_``.
    :param: executor Executor instance used for executing commands.
    :param: use_sandboxes Enables safe checking output correctness.
                       See `sio.executors.checkers`. True by default.
    """
    input_name = tempcwd('in')

    file_executor = get_file_runner(executor, environ)
    exe_filename = file_executor.preferred_filename()

    ft.download(environ, 'exe_file', exe_filename, add_to_cache=True)
    os.chmod(tempcwd(exe_filename), 0700)
    ft.download(environ, 'in_file', input_name, add_to_cache=True)

    zipdir = tempcwd('in_dir')
    os.mkdir(zipdir)
    try:
        if is_zipfile(input_name):
            try:
                # If not a zip file, will pass it directly to exe
                with ZipFile(tempcwd('in'), 'r') as f:
                    if len(f.namelist()) != 1:
                        raise Exception("Archive should have only one file.")

                    f.extract(f.namelist()[0], zipdir)
                    input_name = os.path.join(zipdir, f.namelist()[0])
            # zipfile throws some undocumented exceptions
            except Exception as e:
                raise StandardError("Failed to open archive: " + unicode(e))

        with file_executor as fe:
            with open(input_name, 'rb') as inf:
                with open(tempcwd('out'), 'wb') as outf:
                    renv = fe(tempcwd(exe_filename), [],
                              stdin=inf, stdout=outf, ignore_errors=True,
                              environ=environ, environ_prefix='exec_')

        _populate_environ(renv, environ)

        if renv['result_code'] == 'OK' and environ.get('check_output'):
            environ = checker.run(environ, use_sandboxes=use_sandboxes)

        for key in ('result_code', 'result_string'):
            environ[key] = replace_invalid_UTF(environ[key])

        if 'out_file' in environ:
            ft.upload(environ, 'out_file', tempcwd('out'),
                to_remote_store=environ.get('upload_out', False))
    finally:
        rmtree(zipdir)

    return environ
Beispiel #31
0
    def _proot_options(self):
        self._verbosity(-1)
        self._chroot(self.chroot.path)

        sh_target = path.join(os.sep, 'bin', 'sh')
        if not path.exists(path_join_abs(self.chroot.path, sh_target)):
            self._bind(path_join_abs(self.proot.path, sh_target), sh_target)
        else:
            # If /bin/sh exists, then bind unpatched version to it
            sh_patched = elf_loader_patch._get_unpatched_name(
                path.realpath(path_join_abs(self.chroot.path, sh_target)))
            if path.exists(sh_patched):
                self._bind(sh_patched, sh_target, force=True)

        self._bind(os.path.join(self.proot.path, 'lib'), 'lib')
        self._bind(tempcwd(), 'tmp', force=True)

        # Make absolute `outside paths' visible in sandbox
        self._bind(self.chroot.path, force=True)
        self._bind(tempcwd(), force=True)
Beispiel #32
0
    def _proot_options(self):
        self._verbosity(-1)
        self._chroot(self.chroot.path)

        sh_target = path.join(os.sep, 'bin', 'sh')
        if not path.exists(path_join_abs(self.chroot.path, sh_target)):
            self._bind(path_join_abs(self.proot.path, sh_target), sh_target)
        else:
            # If /bin/sh exists, then bind unpatched version to it
            sh_patched = elf_loader_patch._get_unpatched_name(
                path.realpath(path_join_abs(self.chroot.path, sh_target)))
            if path.exists(sh_patched):
                self._bind(sh_patched, sh_target, force=True)

        self._bind(os.path.join(self.proot.path, 'lib'), 'lib')
        self._bind(tempcwd(), 'tmp', force=True)

        # Make absolute `outside paths' visible in sandbox
        self._bind(self.chroot.path, force=True)
        self._bind(tempcwd(), force=True)
Beispiel #33
0
def _run_in_executor(environ, command, executor, **kwargs):
    with executor:
        renv = executor(command,
            capture_output=True, split_lines=True, forward_stderr=True,
            mem_limit=DEFAULT_INGEN_MEM_LIMIT,
            time_limit=DEFAULT_INGEN_TIME_LIMIT,
            output_limit=DEFAULT_INGEN_OUTPUT_LIMIT,
            environ=environ, environ_prefix='ingen_', **kwargs)
        if renv['return_code'] == 0:
            _collect_and_upload(renv, tempcwd(),
                    environ['collected_files_path'], environ['re_string'])
        return renv
Beispiel #34
0
def test_uploading_out():
    with TemporaryCwd():
        upload_files()
        renv = compile_and_run('/add_print.c', {
            'in_file': '/input',
            'out_file': '/output',
            'upload_out': True,
        }, DetailedUnprotectedExecutor())
        print_env(renv)

        ft.download({'path': '/output'}, 'path', 'd_out')
        in_('84', open(tempcwd('d_out')).read())
Beispiel #35
0
def test_uploading_out():
    with TemporaryCwd():
        upload_files()
        renv = compile_and_run('/add_print.c', {
            'in_file': '/input',
            'out_file': '/output',
            'upload_out': True,
        }, DetailedUnprotectedExecutor())
        print_env(renv)

        ft.download({'path': '/output'}, 'path', 'd_out')
        in_('84', open(tempcwd('d_out')).read())
Beispiel #36
0
def run(environ):
    """Runs a verifying program and returns its output.

       Used ``environ`` keys:

       ``exe_file``: the filetracker path to the program

       ``in_file``: the file redirected to the program's stdin

       ``use_sandboxes``: if this key equals ``True``, the program is executed
                        in the SupervisedExecutor, otherwise the UnsafeExecutor
                        is used

       ``inwer_time_limit``: time limit in ms
                           (optional, the default is 30 s)

       ``inwer_mem_limit``: memory limit in KiB
                           (optional, the default is 256 MiB)

       ``inwer_output_limit``: output limit in B
                           (optional, the default is 10 KiB)

       Returns a new environ, whose ``stdout`` key contains the program's
       output.

       The verifying program is expected to return 0, its first line of output
       should begin with "OK". If this does not happen, an appropriate message
       is logged.

       Program's output is returned under the ``stdout`` key. If the output has
       more than ``inwer_output_limit`` bytes and ``use_sandboxes`` is
       set to ``True``, the execution of the program fails with ``OLE`` result
       code.
    """

    use_sandboxes = environ.get('use_sandboxes', False)
    ft.download(environ, 'exe_file', 'inwer', skip_if_exists=True,
            add_to_cache=True)
    ft.download(environ, 'in_file', 'in', skip_if_exists=True,
            add_to_cache=True)
    os.chmod(tempcwd('inwer'), 0o500)

    renv = _run_inwer(environ, use_sandboxes)
    if renv['result_code'] != "OK":
        logger.error("Inwer failed!\nEnviron dump: %s\nExecution environ: %s",
                environ, renv)
    elif not renv['stdout'][0].startswith(b"OK"):
        logger.error("Bad inwer output!\nEnviron dump: %s\n"
                "Execution environ: %s", environ, renv)

    return renv
Beispiel #37
0
def _run_in_executor(environ, command, executor, **kwargs):
    with executor:
        with open(tempcwd('in'), 'rb') as inf:
            return executor(command,
                            stdin=inf,
                            capture_output=True,
                            split_lines=True,
                            forward_stderr=True,
                            mem_limit=DEFAULT_INWER_MEM_LIMIT,
                            time_limit=DEFAULT_INWER_TIME_LIMIT,
                            output_limit=DEFAULT_INWER_OUTPUT_LIMIT,
                            environ=environ,
                            environ_prefix='inwer_',
                            **kwargs)
Beispiel #38
0
        def inner(env):
            eq_(env['return_code'], 0)
            eq_(env['stdout'], expected_output)
            collected = env['collected_files']
            eq_(len(expected_files), len(collected))
            for filename, path in collected.iteritems():
                in_(filename, expected_files)
                unversioned_path = '/%s/%s' % (upload_dir, filename)
                upload_re_str = '%s@\d+' % (unversioned_path)
                upload_re = re.compile(upload_re_str)
                ok_(upload_re.match(path), 'Unexpected filetracker path')

                ft.download({'in': unversioned_path}, 'in', filename)
                eq_(expected_files[filename], open(tempcwd(filename)).read())
Beispiel #39
0
        def inner(env):
            eq_(env['return_code'], 0)
            eq_(env['stdout'], expected_output)
            collected = env['collected_files']
            eq_(len(expected_files), len(collected))
            for filename, path in collected.iteritems():
                in_(filename, expected_files)
                unversioned_path = '/%s/%s' % (upload_dir, filename)
                upload_re_str = '%s@\d+' % (unversioned_path)
                upload_re = re.compile(upload_re_str)
                ok_(upload_re.match(path), 'Unexpected filetracker path')

                ft.download({'in': unversioned_path}, 'in', filename)
                eq_(expected_files[filename], open(tempcwd(filename)).read())
Beispiel #40
0
def run(environ):
    """Runs a program, collects the files produced by it and uploads them
       to filetracker.

       Used ``environ`` keys:

       ``exe_file``: the filetracker path to the program

       ``re_string``: a regular expression string used to identify the files
                    which should be uploaded

       ``collected_files_path``: a directory into which the collected files
                               should be uploaded in filetracker

       ``use_sandboxes``: if this key equals ``True``, the program is executed
                        in the PRootExecutor, otherwise the UnsafeExecutor is
                        used

       ``ingen_time_limit``: time limit in ms
                           (optional, the default is 10 mins)

       ``ingen_mem_limit``: memory limit in KiB
                           (optional, the default is 256 MiB)

       ``ingen_output_limit``: output limit in B
                           (optional, the default is 10 KiB)

       On success returns a new environ with a dictionary mapping collected
       files' names to their filetracker paths under ``collected_files``.
       Program's output is returned under the ``stdout`` key. The output is
       trimmed to the first ``ingen_output_limit`` bytes.
    """

    use_sandboxes = environ.get('use_sandboxes', False)
    ft.download(environ,
                'exe_file',
                'ingen',
                skip_if_exists=True,
                add_to_cache=True)
    os.chmod(tempcwd('ingen'), 0500)
    renv = _run_ingen(environ, use_sandboxes)
    if renv['return_code'] != 0:
        logger.error("Ingen failed!\nEnviron dump: %s\nExecution environ: %s",
                     environ, renv)

    return renv
Beispiel #41
0
def run(environ):
    """Runs a program, collects the files produced by it and uploads them
       to filetracker.

       Used ``environ`` keys:

       ``exe_file``: the filetracker path to the program

       ``re_string``: a regular expression string used to identify the files
                    which should be uploaded

       ``collected_files_path``: a directory into which the collected files
                               should be uploaded in filetracker

       ``use_sandboxes``: if this key equals ``True``, the program is executed
                        in the PRootExecutor, otherwise the UnsafeExecutor is
                        used

       ``ingen_time_limit``: time limit in ms
                           (optional, the default is 10 mins)

       ``ingen_mem_limit``: memory limit in KiB
                           (optional, the default is 256 MiB)

       ``ingen_output_limit``: output limit in B
                           (optional, the default is 10 KiB)

       On success returns a new environ with a dictionary mapping collected
       files' names to their filetracker paths under ``collected_files``.
       Program's output is returned under the ``stdout`` key. The output is
       trimmed to the first ``ingen_output_limit`` bytes.
    """

    use_sandboxes = environ.get('use_sandboxes', False)
    ft.download(environ, 'exe_file', 'ingen', skip_if_exists=True,
            add_to_cache=True)
    os.chmod(tempcwd('ingen'), 0o500)
    renv = _run_ingen(environ, use_sandboxes)
    if renv['return_code'] != 0:
        logger.error("Ingen failed!\nEnviron dump: %s\nExecution environ: %s",
                environ, renv)

    return renv
Beispiel #42
0
def run(environ, use_sandboxes=True):
    ft.download(environ, 'out_file', 'out', skip_if_exists=True)
    ft.download(environ, 'hint_file', 'hint', add_to_cache=True)

    try:
        if environ.get('chk_file'):
            ft.download(environ,
                        'in_file',
                        'in',
                        skip_if_exists=True,
                        add_to_cache=True)
            ft.download(environ, 'chk_file', 'chk', add_to_cache=True)
            os.chmod(tempcwd('chk'), 0o700)

            output = _run_checker(environ, use_sandboxes)
        elif use_sandboxes:
            output = _run_compare(environ)
        else:
            output = _run_diff(environ)
    except (CheckerError, ExecError) as e:
        logger.error('Checker failed! %s', e)
        logger.error('Environ dump: %s', environ)
        raise SystemError(e)

    while len(output) < 3:
        output.append('')

    if six.ensure_binary(output[0]) == b'OK':
        environ['result_code'] = 'OK'
        if output[1]:
            environ['result_string'] = _limit_length(output[1])
        environ['result_percentage'] = float(output[2] or 100)
    else:
        environ['result_code'] = 'WA'
        environ['result_string'] = _limit_length(output[1])
        environ['result_percentage'] = 0
    return environ
Beispiel #43
0
def run(environ, executor, use_sandboxes=True):
    """
    Common code for executors.

    :param: environ Recipe to pass to `filetracker` and `sio.workers.executors`
                    For all supported options, see the global documentation for
                    `sio.workers.executors` and prefix them with ``exec_``.
    :param: executor Executor instance used for executing commands.
    :param: use_sandboxes Enables safe checking output correctness.
                       See `sio.executors.checkers`. True by default.
    """
    input_name = tempcwd('in')

    file_executor = get_file_runner(executor, environ)
    exe_filename = file_executor.preferred_filename()

    ft.download(environ, 'exe_file', exe_filename, add_to_cache=True)
    os.chmod(tempcwd(exe_filename), 0o700)
    ft.download(environ, 'in_file', input_name, add_to_cache=True)

    zipdir = tempcwd('in_dir')
    os.mkdir(zipdir)
    try:
        if is_zipfile(input_name):
            try:
                # If not a zip file, will pass it directly to exe
                with ZipFile(tempcwd('in'), 'r') as f:
                    if len(f.namelist()) != 1:
                        raise Exception("Archive should have only one file.")

                    f.extract(f.namelist()[0], zipdir)
                    input_name = os.path.join(zipdir, f.namelist()[0])
            # zipfile throws some undocumented exceptions
            except Exception as e:
                raise Exception("Failed to open archive: " + six.text_type(e))

        with file_executor as fe:
            with open(input_name, 'rb') as inf:
                # Open output file in append mode to allow appending
                # only to the end of the output file. Otherwise,
                # a contestant's program could modify the middle of the file.
                with open(tempcwd('out'), 'ab') as outf:
                    renv = fe(tempcwd(exe_filename), [],
                              stdin=inf,
                              stdout=outf,
                              ignore_errors=True,
                              environ=environ,
                              environ_prefix='exec_')

        _populate_environ(renv, environ)

        if renv['result_code'] == 'OK' and environ.get('check_output'):
            environ = checker.run(environ, use_sandboxes=use_sandboxes)

        for key in ('result_code', 'result_string'):
            environ[key] = replace_invalid_UTF(environ[key])

        if 'out_file' in environ:
            ft.upload(environ,
                      'out_file',
                      tempcwd('out'),
                      to_remote_store=environ.get('upload_out', False))
    finally:
        rmtree(zipdir)

    return environ
Beispiel #44
0
 def _make_cmdline(self, executor):
     # Addinational sources are automatically included
     return ['fpc', tempcwd('a.pas')] + self.options + \
             list(self.extra_compilation_args)
Beispiel #45
0
    def _execute(self, command, **kwargs):
        env = kwargs.get('env')
        env.update({
            'MEM_LIMIT': kwargs['mem_limit'] or 64 * 2**10,
            'TIME_LIMIT': kwargs['time_limit'] or 30000,
            'OUT_LIMIT': kwargs['output_limit'] or 50 * 2**20,
        })

        if kwargs['real_time_limit']:
            env['HARD_LIMIT'] = 1 + ceil_ms2s(kwargs['real_time_limit'])
        elif kwargs['time_limit'] and kwargs['real_time_limit'] is None:
            env['HARD_LIMIT'] = 1 + ceil_ms2s(64 * kwargs['time_limit'])

        if 'HARD_LIMIT' in env:
            # Limiting outside supervisor
            kwargs['real_time_limit'] = 2 * s2ms(env['HARD_LIMIT'])

        ignore_errors = kwargs.pop('ignore_errors')
        extra_ignore_errors = kwargs.pop('extra_ignore_errors')
        renv = {}
        try:
            result_file = tempfile.NamedTemporaryFile(dir=tempcwd())
            kwargs['ignore_errors'] = True
            renv = execute_command(command + [noquote('3>'), result_file.name],
                                   **kwargs)

            if 'real_time_killed' in renv:
                raise ExecError('Supervisor exceeded realtime limit')
            elif renv['return_code'] and renv[
                    'return_code'] not in extra_ignore_errors:
                raise ExecError('Supervisor returned code %s' %
                                renv['return_code'])

            result_file.seek(0)
            status_line = result_file.readline().strip().split()[1:]
            renv['result_string'] = result_file.readline().strip()
            result_file.close()
            for num, key in enumerate(('result_code', 'time_used', None,
                                       'mem_used', 'num_syscalls')):
                if key:
                    renv[key] = int(status_line[num])

            result_code = self._supervisor_result_to_code(renv['result_code'])

        except Exception as e:
            logger.error('SupervisedExecutor error: %s',
                         traceback.format_exc())
            logger.error(
                'SupervisedExecutor error dirlist: %s: %s',
                tempcwd(),
                str(os.listdir(tempcwd())),
            )

            result_code = 'SE'
            for i in ('time_used', 'mem_used', 'num_syscalls'):
                renv.setdefault(i, 0)
            renv['result_string'] = str(e)

        renv['result_code'] = result_code

        if (result_code != 'OK' and not ignore_errors
                and not (result_code != 'RV'
                         and renv['return_code'] in extra_ignore_errors)):
            raise ExecError('Failed to execute command: %s. Reason: %s' %
                            (command, renv['result_string']))
        return renv
Beispiel #46
0
    def _execute(self, command, **kwargs):
        options = []
        options += ['-b', os.path.join(self.rpath, 'boxes/minimal') + ':/:ro']
        options += [
            '--memory-limit',
            str(kwargs['mem_limit'] or self.DEFAULT_MEMORY_LIMIT) + 'K',
        ]
        options += [
            '--instruction-count-limit',
            str((kwargs['time_limit'] or self.DEFAULT_TIME_LIMIT) *
                self.INSTRUCTIONS_PER_VIRTUAL_SECOND / 1000),
        ]
        options += [
            '--rtimelimit',
            str((kwargs['time_limit'] or self.DEFAULT_TIME_LIMIT) *
                self.REAL_TIME_LIMIT_MULTIPLIER + self.REAL_TIME_LIMIT_ADDEND)
            + 'ms',
        ]
        options += [
            '--output-limit',
            str(kwargs['output_limit'] or self.DEFAULT_OUTPUT_LIMIT) + 'K',
        ]
        command = [os.path.join(self.rpath, 'sio2jail')
                   ] + options + ['--'] + command

        renv = {}
        try:
            result_file = tempfile.NamedTemporaryFile(dir=tempcwd())
            kwargs['ignore_errors'] = True
            renv = execute_command(command + [noquote('2>'), result_file.name],
                                   **kwargs)

            if renv['return_code'] != 0:
                raise ExecError('Sio2Jail returned code %s, stderr: %s' %
                                (renv['return_code'], result_file.read(10240)))

            result_file.seek(0)
            status_line = result_file.readline().strip().split()[1:]
            renv['result_string'] = result_file.readline().strip()
            result_file.close()
            for num, key in enumerate(
                ('result_code', 'time_used', None, 'mem_used', None)):
                if key:
                    renv[key] = int(status_line[num])

            if renv['result_string'] == 'ok':
                renv['result_code'] = 'OK'
            elif renv['result_string'] == 'time limit exceeded':
                renv['result_code'] = 'TLE'
            elif renv['result_string'] == 'real time limit exceeded':
                renv['result_code'] = 'TLE'
            elif renv['result_string'] == 'memory limit exceeded':
                renv['result_code'] = 'MLE'
            elif renv['result_string'].startswith(
                    'intercepted forbidden syscall'):
                renv['result_code'] = 'RV'
            elif renv['result_string'].startswith(
                    'process exited due to signal'):
                renv['result_code'] = 'RE'
            else:
                raise ExecError('Unrecognized Sio2Jail result string: %s' %
                                renv['result_string'])

        except (EnvironmentError, EOFError, RuntimeError) as e:
            logger.error('Sio2JailExecutor error: %s', traceback.format_exc())
            logger.error(
                'Sio2JailExecutor error dirlist: %s: %s',
                tempcwd(),
                str(os.listdir(tempcwd())),
            )

            renv['result_code'] = 'SE'
            for i in ('time_used', 'mem_used'):
                renv.setdefault(i, 0)
            renv['result_string'] = str(e)

            if not kwargs.get('ignore_errors', False):
                raise ExecError('Failed to execute command: %s. Reason: %s' %
                                (command, renv['result_string']))

        return renv
Beispiel #47
0
def execute_command(command,
                    env=None,
                    split_lines=False,
                    stdin=None,
                    stdout=None,
                    stderr=None,
                    forward_stderr=False,
                    capture_output=False,
                    output_limit=None,
                    real_time_limit=None,
                    ignore_errors=False,
                    extra_ignore_errors=(),
                    **kwargs):
    """Utility function to run arbitrary command.
    ``stdin``
      Could be either file opened with ``open(fname, 'r')``
      or None (then it is inherited from current process).

    ``stdout``, ``stderr``
      Could be files opened with ``open(fname, 'w')``, sys.std*
      or None - then it's suppressed.

    ``forward_stderr``
      Forwards stderr to stdout.

    ``capture_output``
      Returns program output in renv key ``stdout``.

    ``output_limit``
      Limits returned output when ``capture_output=True`` (in bytes).

    Returns renv: dictionary containing:
    ``real_time_used``
      Wall clock time it took to execute the command (in ms).

    ``return_code``
      Status code that program returned.

    ``real_time_killed``
      Only when process was killed due to exceeding real time limit.

    ``stdout``
      Only when ``capture_output=True``: output of the command
    """
    # Using temporary file is way faster than using subproces.PIPE
    # and it prevents deadlocks.
    command = shellquote(command)

    logger.debug('Executing: %s', command)

    stdout = capture_output and tempfile.TemporaryFile() or stdout
    # redirect output to /dev/null if None given
    devnull = open(os.devnull, 'wb')
    stdout = stdout or devnull
    stderr = stderr or devnull

    ret_env = {}
    if env is not None:
        for key, value in six.iteritems(env):
            env[key] = str(value)

    perf_timer = util.PerfTimer()
    p = subprocess.Popen(
        command,
        stdin=stdin,
        stdout=stdout,
        stderr=forward_stderr and subprocess.STDOUT or stderr,
        shell=True,
        close_fds=True,
        universal_newlines=True,
        env=env,
        cwd=tempcwd(),
        preexec_fn=os.setpgrp,
    )

    kill_timer = None
    if real_time_limit:

        def oot_killer():
            ret_env['real_time_killed'] = True
            os.killpg(p.pid, signal.SIGKILL)

        kill_timer = Timer(ms2s(real_time_limit), oot_killer)
        kill_timer.start()

    rc = p.wait()
    ret_env['return_code'] = rc

    if kill_timer:
        kill_timer.cancel()

    ret_env['real_time_used'] = s2ms(perf_timer.elapsed)

    logger.debug(
        'Command "%s" exited with code %d, took %.2fs',
        str(command),
        rc,
        perf_timer.elapsed,
    )

    devnull.close()
    if capture_output:
        stdout.seek(0)
        ret_env['stdout'] = stdout.read(output_limit or -1)
        stdout.close()
        if split_lines:
            ret_env['stdout'] = ret_env['stdout'].split(b'\n')

    if rc and not ignore_errors and rc not in extra_ignore_errors:
        raise ExecError(
            'Failed to execute command: %s. Returned with code %s\n' %
            (command, rc))

    return ret_env
Beispiel #48
0
 def change(env):
     res_ok(env)
     eq_('13', open(tempcwd('somefile')).read().strip())
     ok_(os.path.exists(tempcwd('./not_existing')))
Beispiel #49
0
 def nochange(env):
     res_re(1)(env)
     eq_('42', open(tempcwd('somefile')).read().strip())
     ok_(not os.path.exists(tempcwd('./not_existing')))
Beispiel #50
0
 def change(env):
     res_ok(env)
     eq_('13', open(tempcwd('somefile')).read().strip())
     ok_(os.path.exists(tempcwd('./not_existing')))
Beispiel #51
0
    def _execute(self, command, **kwargs):
        env = kwargs.get('env')
        env.update({
                    'MEM_LIMIT': kwargs['mem_limit'] or 64 * 2**10,
                    'TIME_LIMIT': kwargs['time_limit'] or 30000,
                    'OUT_LIMIT': kwargs['output_limit'] or 50 * 2**20,
                    })

        if kwargs['real_time_limit']:
            env['HARD_LIMIT'] = 1 + ceil_ms2s(kwargs['real_time_limit'])
        elif kwargs['time_limit'] and kwargs['real_time_limit'] is None:
            env['HARD_LIMIT'] = 1 + ceil_ms2s(64 * kwargs['time_limit'])

        if 'HARD_LIMIT' in env:
            # Limiting outside supervisor
            kwargs['real_time_limit'] = 2 * s2ms(env['HARD_LIMIT'])

        ignore_errors = kwargs.pop('ignore_errors')
        extra_ignore_errors = kwargs.pop('extra_ignore_errors')
        renv = {}
        try:
            result_file = tempfile.NamedTemporaryFile(dir=tempcwd())
            kwargs['ignore_errors'] = True
            renv = execute_command(
                        command + [noquote('3>'), result_file.name],
                         **kwargs
                        )

            if 'real_time_killed' in renv:
                raise ExecError('Supervisor exceeded realtime limit')
            elif renv['return_code'] and \
                    renv['return_code'] not in extra_ignore_errors:
                raise ExecError('Supervisor returned code %s'
                                % renv['return_code'])

            result_file.seek(0)
            status_line = result_file.readline().strip().split()[1:]
            renv['result_string'] = result_file.readline().strip()
            result_file.close()
            for num, key in enumerate(('result_code', 'time_used',
                        None, 'mem_used', 'num_syscalls')):
                    if key:
                        renv[key] = int(status_line[num])

            result_code = self._supervisor_result_to_code(renv['result_code'])

        except Exception as e:
            logger.error('SupervisedExecutor error: %s', traceback.format_exc())
            logger.error('SupervisedExecutor error dirlist: %s: %s',
                         tempcwd(), str(os.listdir(tempcwd())))

            result_code = 'SE'
            for i in ('time_used', 'mem_used', 'num_syscalls'):
                renv.setdefault(i, 0)
            renv['result_string'] = str(e)

        renv['result_code'] = result_code

        if result_code != 'OK' and not ignore_errors and not \
                (result_code != 'RV' and renv['return_code'] in \
                        extra_ignore_errors):
            raise ExecError('Failed to execute command: %s. Reason: %s'
                        % (command, renv['result_string']))
        return renv
Beispiel #52
0
 def _make_cmdline(self, executor):
     # Addinational sources are automatically included
     return (['fpc', tempcwd('a.pas')] + self.options +
             list(self.extra_compilation_args))
Beispiel #53
0
 def nochange(env):
     res_re(1)(env)
     eq_('42', open(tempcwd('somefile')).read().strip())
     ok_(not os.path.exists(tempcwd('./not_existing')))
Beispiel #54
0
def execute_command(command, env=None, split_lines=False, stdin=None,
                    stdout=None, stderr=None, forward_stderr=False,
                    capture_output=False, output_limit=None,
                    real_time_limit=None,
                    ignore_errors=False, extra_ignore_errors=(), **kwargs):
    """Utility function to run arbitrary command.
       ``stdin``
         Could be either file opened with ``open(fname, 'r')``
         or None (then it is inherited from current process).

       ``stdout``, ``stderr``
         Could be files opened with ``open(fname, 'w')``, sys.std*
         or None - then it's suppressed.

       ``forward_stderr``
         Forwards stderr to stdout.

       ``capture_output``
         Returns program output in renv key ``stdout``.

       ``output_limit``
         Limits returned output when ``capture_output=True`` (in bytes).

       Returns renv: dictionary containing:
       ``real_time_used``
         Wall clock time it took to execute the command (in ms).

       ``return_code``
         Status code that program returned.

       ``real_time_killed``
         Only when process was killed due to exceeding real time limit.

       ``stdout``
         Only when ``capture_output=True``: output of the command
    """
    # Using temporary file is way faster than using subproces.PIPE
    # and it prevents deadlocks.
    command = shellquote(command)

    logger.debug('Executing: %s', command)

    stdout = capture_output and tempfile.TemporaryFile() or stdout
    # redirect output to /dev/null if None given
    devnull = open(os.devnull, 'wb')
    stdout = stdout or devnull
    stderr = stderr or devnull

    ret_env = {}
    if env is not None:
        for key, value in env.iteritems():
            env[key] = str(value)

    perf_timer = util.PerfTimer()
    p = subprocess.Popen(command,
                         stdin=stdin,
                         stdout=stdout,
                         stderr=forward_stderr and subprocess.STDOUT
                                                or stderr,
                         shell=True,
                         close_fds=True,
                         universal_newlines=True,
                         env=env,
                         cwd=tempcwd(),
                         preexec_fn=os.setpgrp)

    kill_timer = None
    if real_time_limit:
        def oot_killer():
            ret_env['real_time_killed'] = True
            os.killpg(p.pid, signal.SIGKILL)
        kill_timer = Timer(ms2s(real_time_limit), oot_killer)
        kill_timer.start()

    rc = p.wait()
    ret_env['return_code'] = rc

    if kill_timer:
        kill_timer.cancel()

    ret_env['real_time_used'] = s2ms(perf_timer.elapsed)

    logger.debug('Command "%s" exited with code %d, took %.2fs',
            str(command), rc, perf_timer.elapsed)

    devnull.close()
    if capture_output:
        stdout.seek(0)
        ret_env['stdout'] = stdout.read(output_limit or -1)
        stdout.close()
        if split_lines:
            ret_env['stdout'] = ret_env['stdout'].split('\n')

    if rc and not ignore_errors and rc not in extra_ignore_errors:
        raise ExecError('Failed to execute command: %s. Returned with code %s\n'
                        % (command, rc))

    return ret_env
Beispiel #55
0
def run(environ, executor, use_sandboxes=True):
    """
    Common code for executors.

    :param: environ Recipe to pass to `filetracker` and `sio.workers.executors`
                    For all supported options, see the global documentation for
                    `sio.workers.executors` and prefix them with ``exec_``.
    :param: executor Executor instance used for executing commands.
    :param: use_sandboxes Enables safe checking output correctness.
                       See `sio.executors.checkers`. True by default.
    """

    if feedback.judge_prepare(environ).force_not_judge:
        environ['time_used'] = 0
        environ['result_string'] = 'not judged'
        environ['result_code'] = 'NJ'
        environ['mem_used'] = 0
        environ['num_syscalls'] = 0
        environ['stderr'] = ''
        return environ

    input_name = tempcwd('in')

    file_executor = get_file_runner(executor, environ)
    exe_filename = file_executor.preferred_filename()

    ft.download(environ, 'exe_file', exe_filename, add_to_cache=True)
    os.chmod(tempcwd(exe_filename), 0700)
    ft.download(environ, 'in_file', input_name, add_to_cache=True)

    zipdir = tempcwd('in_dir')
    os.mkdir(zipdir)
    try:
        if is_zipfile(input_name):
            try:
                # If not a zip file, will pass it directly to exe
                with ZipFile(tempcwd('in'), 'r') as f:
                    if len(f.namelist()) != 1:
                        raise Exception("Archive should have only one file.")

                    f.extract(f.namelist()[0], zipdir)
                    input_name = os.path.join(zipdir, f.namelist()[0])
            # zipfile throws some undocumented exceptions
            except Exception as e:
                raise StandardError("Failed to open archive: " + unicode(e))

        with file_executor as fe:
            with open(input_name, 'rb') as inf:
                with open(tempcwd('out'), 'wb') as outf:
                    feedback.judge_started(environ)
                    renv = fe(tempcwd(exe_filename), [],
                              stdin=inf,
                              stdout=outf,
                              ignore_errors=True,
                              environ=environ,
                              environ_prefix='exec_')

        _populate_environ(renv, environ)

        if renv['result_code'] == 'OK' and environ.get('check_output'):
            environ = checker.run(environ, use_sandboxes=use_sandboxes)

        for key in ('result_code', 'result_string'):
            environ[key] = replace_invalid_UTF(environ[key])
        feedback.judge_finished(environ)

        if 'out_file' in environ:
            ft.upload(environ,
                      'out_file',
                      tempcwd('out'),
                      to_remote_store=environ.get('upload_out', False))
    finally:
        rmtree(zipdir)

    return environ