def compile(self, environ): """ Compile the file specified in the `environ` dictionary. :param environ: Recipe to pass to `filetracker` and `sio.workers.execute` For all supported options, see the global documentation for `sio.compilers`. """ self.environ = environ compilation_started(environ) # using a copy of the environment in order to avoid polluting it with # temoporary elements self.tmp_environ = environ.copy() self.source_file = self._make_filename() ft.download(environ, 'source_file', self.source_file) self._process_extra_files() self.extra_compilation_args = \ _lang_option(environ, 'extra_compilation_args', self.lang) with self.executor as executor: renv = self._run_in_executor(executor) result = self._postprocess(renv) compilation_finished(result) return result
def run(environ): exe_file = ft.download(environ, 'exe_file', 'exe', add_to_cache=True) os.chmod(exe_file, 0700) in_file = ft.download(environ, 'in_file', 'in', add_to_cache=True) env = os.environ.copy() for key, default in _options: value = environ.get('exec_' + key.lower(), default) env[key] = value with get_sandbox('vcpu_exec-sandbox') as sandbox: retcode, output = execute( [os.path.join(sandbox.path, 'pin-supervisor/supervisor-bin/', 'supervisor'), '-f', '3', '--', './exe', noquote('<'), 'in', noquote('3>'), 'supervisor_result', noquote('>'), 'out'], env=env) result_file = open('supervisor_result') status_line = result_file.readline().strip().split() environ['result_string'] = result_file.readline().strip() result_file.close() try: for num, key in enumerate((None, 'result_code', 'time_used', 'exectime_used', 'mem_used', 'num_syscalls')): if key: environ[key] = int(status_line[num]) result_code = _supervisor_result_to_code(environ['result_code']) except Exception, e: result_code = 'SE' for i in ('time_used', 'exectime_used', 'mem_used', 'num_syscalls'): environ.setdefault(i, 0) environ['result_string'] = str(e)
def run(environ, executor, safe_check=True): """ Common code for executors. :param: environ Recipe to pass to `filetracker` and `sio.workers.executors` For all supported options, see the global documentation for `sio.workers.executors`. :param: executor Executor instance used for executing commands. :param: safe_check Enables safe checking output corectness. See `sio.executors.checkers`. True by default. """ ft.download(environ, 'exe_file', 'exe', add_to_cache=True) os.chmod('exe', 0700) ft.download(environ, 'in_file', 'in', add_to_cache=True) with executor as e: with open('in', 'rb') as inf: with open('out', 'wb') as outf: renv = e(['./exe'], stdin=inf, stdout=outf, ignore_errors=True, environ=environ) _populate_environ(renv, environ) if renv['result_code'] == 'OK' and environ.get('check_output'): environ = checker.run(environ, no_sandbox=not safe_check) if 'out_file' in environ: ft.upload(environ, 'out_file', 'out', to_remote_store=environ.get('upload_out', False)) return environ
def _test_transparent_exec(source, executor, callback, kwargs): with TemporaryCwd(): upload_files() ft.download({'path': '/somefile'}, 'path', tempcwd('somefile')) result_env = compile_and_execute(source, executor, **kwargs) print_env(result_env) if callback: callback(result_env)
def run(environ): """Runs a verifying program and returns its output. Used ``environ`` keys: ``exe_file``: the filetracker path to the program ``in_file``: the file redirected to the program's stdin ``use_sandboxes``: if this key equals ``True``, the program is executed in the SupervisedExecutor, otherwise the UnsafeExecutor is used ``inwer_time_limit``: time limit in ms (optional, the default is 30 s) ``inwer_mem_limit``: memory limit in KiB (optional, the default is 256 MiB) ``inwer_output_limit``: output limit in B (optional, the default is 10 KiB) Returns a new environ, whose ``stdout`` key contains the program's output. The verifying program is expected to return 0, its first line of output should begin with "OK". If this does not happen, an appropriate message is logged. Program's output is returned under the ``stdout`` key. If the output has more than ``inwer_output_limit`` bytes and ``use_sandboxes`` is set to ``True``, the execution of the program fails with ``OLE`` result code. """ use_sandboxes = environ.get('use_sandboxes', False) ft.download(environ, 'exe_file', 'inwer', skip_if_exists=True, add_to_cache=True) ft.download(environ, 'in_file', 'in', skip_if_exists=True, add_to_cache=True) os.chmod(tempcwd('inwer'), 0500) renv = _run_inwer(environ, use_sandboxes) if renv['result_code'] != "OK": logger.error("Inwer failed!\nEnviron dump: %s\nExecution environ: %s", environ, renv) elif not renv['stdout'][0].startswith("OK"): logger.error( "Bad inwer output!\nEnviron dump: %s\n" "Execution environ: %s", environ, renv) return renv
def run(environ, executor, use_sandboxes=True): """ Common code for executors. :param: environ Recipe to pass to `filetracker` and `sio.workers.executors` For all supported options, see the global documentation for `sio.workers.executors` and prefix them with ``exec_``. :param: executor Executor instance used for executing commands. :param: use_sandboxes Enables safe checking output correctness. See `sio.executors.checkers`. True by default. """ input_name = tempcwd('in') file_executor = get_file_runner(executor, environ) exe_filename = file_executor.preferred_filename() ft.download(environ, 'exe_file', exe_filename, add_to_cache=True) os.chmod(tempcwd(exe_filename), 0700) ft.download(environ, 'in_file', input_name, add_to_cache=True) zipdir = tempcwd('in_dir') os.mkdir(zipdir) try: if is_zipfile(input_name): try: # If not a zip file, will pass it directly to exe with ZipFile(tempcwd('in'), 'r') as f: if len(f.namelist()) != 1: raise Exception("Archive should have only one file.") f.extract(f.namelist()[0], zipdir) input_name = os.path.join(zipdir, f.namelist()[0]) # zipfile throws some undocumented exceptions except Exception as e: raise StandardError("Failed to open archive: " + unicode(e)) with file_executor as fe: with open(input_name, 'rb') as inf: with open(tempcwd('out'), 'wb') as outf: renv = fe(tempcwd(exe_filename), [], stdin=inf, stdout=outf, ignore_errors=True, environ=environ, environ_prefix='exec_') _populate_environ(renv, environ) if renv['result_code'] == 'OK' and environ.get('check_output'): environ = checker.run(environ, use_sandboxes=use_sandboxes) for key in ('result_code', 'result_string'): environ[key] = replace_invalid_UTF(environ[key]) if 'out_file' in environ: ft.upload(environ, 'out_file', tempcwd('out'), to_remote_store=environ.get('upload_out', False)) finally: rmtree(zipdir) return environ
def compile_and_execute(source, executor, **exec_args): exe_file = compile(source) ft.download({'exe_file': exe_file}, 'exe_file', 'exe') os.chmod('exe', 0700) ft.download({'in_file': '/input'}, 'in_file', 'in') with executor as e: with open('in', 'rb') as inf: renv = e(['./exe'], stdin=inf, **exec_args) return renv
def test_zip(): with TemporaryCwd(): upload_files() compile_and_run( "/echo.c", { 'in_file': '/input.zip', 'out_file': '/output', 'exec_mem_limit': 102400 }, DetailedUnprotectedExecutor()) ft.download({'in_file': '/input'}, 'in_file', 'out.expected') ft.download({'out_file': '/output'}, 'out_file', 'out.real') ok_(filecmp.cmp(tempcwd('out.expected'), tempcwd('out.real')))
def test_uploading_out(): with TemporaryCwd(): upload_files() renv = compile_and_run('/add_print.c', { 'in_file': '/input', 'out_file': '/output', 'upload_out': True, }, DetailedUnprotectedExecutor()) print_env(renv) ft.download({'path': '/output'}, 'path', 'd_out') in_('84', open(tempcwd('d_out')).read())
def test_zip(): with TemporaryCwd(): upload_files() compile_and_run("/echo.c", { 'in_file': '/input.zip', 'out_file': '/output', 'exec_mem_limit': 102400 }, DetailedUnprotectedExecutor()) ft.download({'in_file': '/input'}, 'in_file', 'out.expected') ft.download({'out_file': '/output'}, 'out_file', 'out.real') ok_(filecmp.cmp(tempcwd('out.expected'), tempcwd('out.real')))
def run(environ): """Runs a verifying program and returns its output. Used ``environ`` keys: ``exe_file``: the filetracker path to the program ``in_file``: the file redirected to the program's stdin ``use_sandboxes``: if this key equals ``True``, the program is executed in the SupervisedExecutor, otherwise the UnsafeExecutor is used ``inwer_time_limit``: time limit in ms (optional, the default is 30 s) ``inwer_mem_limit``: memory limit in KiB (optional, the default is 256 MiB) ``inwer_output_limit``: output limit in B (optional, the default is 10 KiB) Returns a new environ, whose ``stdout`` key contains the program's output. The verifying program is expected to return 0, its first line of output should begin with "OK". If this does not happen, an appropriate message is logged. Program's output is returned under the ``stdout`` key. If the output has more than ``inwer_output_limit`` bytes and ``use_sandboxes`` is set to ``True``, the execution of the program fails with ``OLE`` result code. """ use_sandboxes = environ.get('use_sandboxes', False) ft.download(environ, 'exe_file', 'inwer', skip_if_exists=True, add_to_cache=True) ft.download(environ, 'in_file', 'in', skip_if_exists=True, add_to_cache=True) os.chmod(tempcwd('inwer'), 0o500) renv = _run_inwer(environ, use_sandboxes) if renv['result_code'] != "OK": logger.error("Inwer failed!\nEnviron dump: %s\nExecution environ: %s", environ, renv) elif not renv['stdout'][0].startswith(b"OK"): logger.error("Bad inwer output!\nEnviron dump: %s\n" "Execution environ: %s", environ, renv) return renv
def inner(env): eq_(env['return_code'], 0) eq_(env['stdout'], expected_output) collected = env['collected_files'] eq_(len(expected_files), len(collected)) for filename, path in collected.iteritems(): in_(filename, expected_files) unversioned_path = '/%s/%s' % (upload_dir, filename) upload_re_str = '%s@\d+' % (unversioned_path) upload_re = re.compile(upload_re_str) ok_(upload_re.match(path), 'Unexpected filetracker path') ft.download({'in': unversioned_path}, 'in', filename) eq_(expected_files[filename], open(tempcwd(filename)).read())
def _process_extra_files(self): self.additional_includes = _lang_option(self.environ, 'additional_includes', self.lang) self.additional_sources = _lang_option(self.environ, 'additional_sources', self.lang) for include in self.additional_includes: self.tmp_environ['additional_include'] = include ft.download(self.tmp_environ, 'additional_include', os.path.basename(include)) for source in self.additional_sources: self.tmp_environ['additional_source'] = source ft.download(self.tmp_environ, 'additional_source', os.path.basename(source)) extra_files = self.environ.get('extra_files', {}) for name, ft_path in extra_files.iteritems(): self.tmp_environ['extra_file'] = ft_path ft.download(self.tmp_environ, 'extra_file', os.path.basename(name)) archive = self.environ.get('additional_archive', '') if archive: self.tmp_environ['additional_archive'] = archive archive_path = os.path.basename(archive) ft.download(self.tmp_environ, 'additional_archive', archive_path) _extract_all(archive_path)
def run(environ, no_sandbox=False): ft.download(environ, 'out_file', 'out', skip_if_exists=True) ft.download(environ, 'hint_file', 'hint', add_to_cache=True) if environ.get('chk_file'): ft.download(environ, 'in_file', 'in', skip_if_exists=True, add_to_cache=True) ft.download(environ, 'chk_file', 'chk', add_to_cache=True) os.chmod('chk', 0700) with UnprotectedExecutor() as e: renv = e(['./chk', 'in', 'out', 'hint'], capture_output=True, ignore_errors=True, split_lines=True) output = renv['stdout'] elif no_sandbox: output = [_run_diff()] else: with SandboxExecutor('exec-sandbox') as e: renv = e([os.path.join('bin', 'compare'), 'hint', 'out'], capture_output=True, split_lines=True, ignore_errors=True) output = renv['stdout'] while len(output) < 3: output.append('') if output[0] == 'OK': environ['result_code'] = 'OK' if output[1]: environ['result_string'] = output[1] environ['result_percentage'] = float(output[2] or 100) else: environ['result_code'] = 'WA' environ['result_string'] = output[1] environ['result_percentage'] = 0 return environ
def run(environ, no_sandbox=False): ft.download(environ, 'out_file', 'out', skip_if_exists=True) ft.download(environ, 'hint_file', 'hint', add_to_cache=True) if environ.get('chk_file'): sandbox = NullSandbox() ft.download(environ, 'in_file', 'in', skip_if_exists=True, add_to_cache=True) ft.download(environ, 'chk_file', 'chk', add_to_cache=True) os.chmod('chk', 0700) cmd = ['./chk', 'in', 'out', 'hint', noquote('2>'), '/dev/null'] elif no_sandbox: sandbox = NullSandbox() cmd = ['diff', '-b', '-q', 'out', 'hint', noquote('>'), '/dev/null', noquote('2>&1'), noquote('&&'), 'echo', 'OK'] else: sandbox = get_sandbox('exec-sandbox') cmd = [os.path.join(sandbox.path, 'bin', 'compare'), 'hint', 'out', noquote('2>'), '/dev/null'] with sandbox: retcode, output = execute(cmd, ignore_errors=True, split_lines=True) while len(output) < 3: output.append('') if output[0] == 'OK': environ['result_code'] = 'OK' if output[1]: environ['result_string'] = output[1] environ['result_percentage'] = float(output[2] or 100) else: environ['result_code'] = 'WA' environ['result_string'] = output[1] environ['result_percentage'] = 0 return environ
def test_output_compilation_and_running(source): with TemporaryCwd(): upload_files() result_env = run( { 'source_file': source, 'compiler': 'output-only', } ) eq_(result_env['result_code'], 'OK') eq_(result_env['exec_info'], {'mode': 'output-only'}) ft.download(result_env, 'out_file', tempcwd('out.txt')) ft.download({'source_file': source}, 'source_file', tempcwd('source.txt')) with open(tempcwd('out.txt'), 'r') as outfile: with open(tempcwd('source.txt'), 'r') as sourcefile: eq_(outfile.read(), sourcefile.read()) post_run_env = run_from_executors( { 'exec_info': result_env['exec_info'], 'exe_file': result_env['out_file'], 'check_output': True, 'hint_file': source, }, executor=None, ) eq_(post_run_env['result_code'], 'OK') ft.download(post_run_env, 'out_file', tempcwd('out.txt')) ft.download({'source_file': source}, 'source_file', tempcwd('source.txt')) with open(tempcwd('out.txt'), 'r') as outfile: with open(tempcwd('source.txt'), 'r') as sourcefile: eq_(outfile.read(), sourcefile.read())
def run(environ, use_sandboxes=True): ft.download(environ, 'out_file', 'out', skip_if_exists=True) ft.download(environ, 'hint_file', 'hint', add_to_cache=True) try: if environ.get('chk_file'): ft.download(environ, 'in_file', 'in', skip_if_exists=True, add_to_cache=True) ft.download(environ, 'chk_file', 'chk', add_to_cache=True) os.chmod('chk', 0700) output = _run_checker(environ, use_sandboxes) elif use_sandboxes: output = _run_compare(environ) else: output = _run_diff(environ) except ExecError as e: logger.error('Checker failed! %s', e) logger.error('Environ dump: %s', environ) environ['result_code'] = 'SE' environ['result_string'] = 'checker failure' return environ while len(output) < 3: output.append('') if output[0] == 'OK': environ['result_code'] = 'OK' if output[1]: environ['result_string'] = output[1] environ['result_percentage'] = float(output[2] or 100) else: environ['result_code'] = 'WA' environ['result_string'] = output[1] environ['result_percentage'] = 0 return environ
def compile_and_execute(source, executor, **exec_args): cenv = compile(source, use_sandboxes=isinstance(executor, _SIOSupervisedExecutor)) frunner = get_file_runner(executor, cenv) ft.download({'exe_file': cenv['out_file']}, 'exe_file', frunner.preferred_filename()) os.chmod(tempcwd('exe'), 0700) ft.download({'in_file': '/input'}, 'in_file', 'in') with frunner: with open(tempcwd('in'), 'rb') as inf: renv = frunner(tempcwd('exe'), [], stdin=inf, **exec_args) return renv
def run(environ): exe_file = ft.download(environ, 'exe_file', 'exe', add_to_cache=True) os.chmod(exe_file, 0700) in_file = ft.download(environ, 'in_file', 'in', add_to_cache=True) if 'exec_time_limit' in environ: time_limit = environ['exec_time_limit'] time_ulimit = (time_limit + 999) / 1000 else: time_limit = time_ulimit = None if 'exec_mem_limit' in environ: mem_limit = environ['exec_mem_limit'] / 1024 else: mem_limit = None retcode, output = execute(['bash', '-c', 'time ./exe < in > out'], time_limit=time_ulimit, mem_limit=mem_limit, ignore_errors=True) time_output_matches = TIME_OUTPUT_RE.findall(output) if time_output_matches: mins, secs = time_output_matches[-1] time_used = int((int(mins) * 60 + float(secs)) * 1000) else: raise RuntimeError('Could not find output of time program. ' 'Captured output: %s' % output) if time_limit is not None and time_used >= 0.99 * time_limit: environ['result_string'] = 'time limit exceeded' environ['result_code'] = 'TLE' elif retcode == 0: environ['result_string'] = 'ok' environ['result_code'] = 'OK' else: environ['result_string'] = 'program exited with code %d' % retcode environ['result_code'] = 'RE' environ['time_used'] = time_used environ['exectime_used'] = 0 environ['mem_used'] = 0 environ['num_syscalls'] = 0 if environ['result_code'] == 'OK' and environ.get('check_output'): environ = checker.run(environ, no_sandbox=True) if 'out_file' in environ: ft.upload(environ, 'out_file', 'out', to_remote_store=environ.get('upload_out', False)) return environ
def run(environ): """Runs a program, collects the files produced by it and uploads them to filetracker. Used ``environ`` keys: ``exe_file``: the filetracker path to the program ``re_string``: a regular expression string used to identify the files which should be uploaded ``collected_files_path``: a directory into which the collected files should be uploaded in filetracker ``use_sandboxes``: if this key equals ``True``, the program is executed in the PRootExecutor, otherwise the UnsafeExecutor is used ``ingen_time_limit``: time limit in ms (optional, the default is 10 mins) ``ingen_mem_limit``: memory limit in KiB (optional, the default is 256 MiB) ``ingen_output_limit``: output limit in B (optional, the default is 10 KiB) On success returns a new environ with a dictionary mapping collected files' names to their filetracker paths under ``collected_files``. Program's output is returned under the ``stdout`` key. The output is trimmed to the first ``ingen_output_limit`` bytes. """ use_sandboxes = environ.get('use_sandboxes', False) ft.download(environ, 'exe_file', 'ingen', skip_if_exists=True, add_to_cache=True) os.chmod(tempcwd('ingen'), 0500) renv = _run_ingen(environ, use_sandboxes) if renv['return_code'] != 0: logger.error("Ingen failed!\nEnviron dump: %s\nExecution environ: %s", environ, renv) return renv
def run(environ): """Runs a program, collects the files produced by it and uploads them to filetracker. Used ``environ`` keys: ``exe_file``: the filetracker path to the program ``re_string``: a regular expression string used to identify the files which should be uploaded ``collected_files_path``: a directory into which the collected files should be uploaded in filetracker ``use_sandboxes``: if this key equals ``True``, the program is executed in the PRootExecutor, otherwise the UnsafeExecutor is used ``ingen_time_limit``: time limit in ms (optional, the default is 10 mins) ``ingen_mem_limit``: memory limit in KiB (optional, the default is 256 MiB) ``ingen_output_limit``: output limit in B (optional, the default is 10 KiB) On success returns a new environ with a dictionary mapping collected files' names to their filetracker paths under ``collected_files``. Program's output is returned under the ``stdout`` key. The output is trimmed to the first ``ingen_output_limit`` bytes. """ use_sandboxes = environ.get('use_sandboxes', False) ft.download(environ, 'exe_file', 'ingen', skip_if_exists=True, add_to_cache=True) os.chmod('ingen', 0500) renv = _run_ingen(environ, use_sandboxes) if renv['return_code'] != 0: logger.error("Ingen failed!\nEnviron dump: %s\nExecution environ: %s", environ, renv) return renv
def run(environ): if environ['compiler'] not in ('foo.1_0', 'foo.2_0'): raise RuntimeError("Compiler '%s' not found.", environ['compiler']) input_file = ft.download(environ, 'source_file', 'a.foo') print input_file size = os.path.getsize(input_file) out = open('compiled', 'w') out.write("#!/bin/sh\n") out.write("# Compiled using Foo compiler named %s\n" % environ['compiler']) out.write("echo %d" % size) out.close() ft.upload(environ, 'out_file', 'compiled') return environ
def run(environ): if environ['compiler'] not in ('foo.1_0', 'foo.2_0'): raise RuntimeError("Compiler '%s' not found.", environ['compiler']) input_file = ft.download(environ, 'source_file', 'a.foo') print(input_file) size = os.path.getsize(input_file) out = open('compiled', 'w') out.write("#!/bin/sh\n") out.write("# Compiled using Foo compiler named %s\n" % environ['compiler']) out.write("echo %d" % size) out.close() ft.upload(environ, 'out_file', 'compiled') return environ
def compile(self, environ): """ Compile the file specified in the `environ` dictionary. :param environ: Recipe to pass to `filetracker` and `sio.workers.execute` For all supported options, see the global documentation for `sio.compilers`. """ self.environ = environ # using a copy of the environment in order to avoid polluting it with # temoporary elements self.tmp_environ = environ.copy() self.source_file = self._make_filename() ft.download(environ, 'source_file', self.source_file) self._process_extra_files() self.extra_compilation_args = \ _lang_option(environ, 'extra_compilation_args', self.lang) with self.executor as executor: renv = self._run_in_executor(executor) return self._postprocess(renv)
def compile_and_run(compiler_env, expected_output): """Helper function for compiling, launching and testing the result of a program. """ run(compiler_env) binary = ft.download({'path': '/out'}, 'path') os.chmod(binary, stat.S_IXUSR) retcode, output = execute(['./' + binary]) eq_(retcode, 0) eq_(output.strip(), expected_output) os.remove(binary)
def compile_and_run(compiler_env, expected_output): """Helper function for compiling, launching and testing the result of a program. """ # Dummy sandbox doesn't support asking for versioned filename out_file = compiler_env['out_file'] result_env = run(compiler_env) eq_(result_env['result_code'], 'OK') binary = ft.download({'path': out_file}, 'path') os.chmod(binary, stat.S_IXUSR) retcode, output = execute(['./' + binary]) eq_(retcode, 0) eq_(output.strip(), expected_output) os.remove(binary)
def compile_and_run(compiler_env, expected_output, program_args=None): """Helper function for compiling, launching and testing the result of a program. """ # Dummy sandbox doesn't support asking for versioned filename out_file = compiler_env['out_file'] if compiler_env.get('compiler', '').endswith('-java'): compiler_env['compilation_time_limit'] = 180000 with TemporaryCwd('compile'): result_env = run(compiler_env) print_env(result_env) eq_(result_env['result_code'], 'OK') binary = ft.download({'path': out_file}, 'path') os.chmod(binary, stat.S_IXUSR | os.stat(binary).st_mode) if not program_args: program_args = [] executor = UnprotectedExecutor() frkwargs = {} # Hack for java if compiler_env.get('compiler') == 'default-java': executor = PRootExecutor('compiler-java.1_8') frkwargs['proot_options'] = ['-b', '/proc'] frunner = get_file_runner(executor, result_env) with frunner: renv = frunner(binary, program_args, stderr=sys.__stderr__, capture_output=True, **frkwargs) eq_(renv['return_code'], 0) eq_(renv['stdout'].decode().strip(), expected_output) os.remove(binary)
def run(environ, use_sandboxes=True): ft.download(environ, 'out_file', 'out', skip_if_exists=True) ft.download(environ, 'hint_file', 'hint', add_to_cache=True) try: if environ.get('chk_file'): ft.download(environ, 'in_file', 'in', skip_if_exists=True, add_to_cache=True) ft.download(environ, 'chk_file', 'chk', add_to_cache=True) os.chmod(tempcwd('chk'), 0o700) output = _run_checker(environ, use_sandboxes) elif use_sandboxes: output = _run_compare(environ) else: output = _run_diff(environ) except (CheckerError, ExecError) as e: logger.error('Checker failed! %s', e) logger.error('Environ dump: %s', environ) raise SystemError(e) while len(output) < 3: output.append('') if six.ensure_binary(output[0]) == b'OK': environ['result_code'] = 'OK' if output[1]: environ['result_string'] = _limit_length(output[1]) environ['result_percentage'] = float(output[2] or 100) else: environ['result_code'] = 'WA' environ['result_string'] = _limit_length(output[1]) environ['result_percentage'] = 0 return environ
def compile_and_run(compiler_env, expected_output, program_args=None): """Helper function for compiling, launching and testing the result of a program. """ # Dummy sandbox doesn't support asking for versioned filename out_file = compiler_env['out_file'] if compiler_env.get('compiler', '').endswith('-java'): compiler_env['compilation_time_limit'] = 180000 with TemporaryCwd('compile'): result_env = run(compiler_env) print_env(result_env) eq_(result_env['result_code'], 'OK') binary = ft.download({'path': out_file}, 'path') os.chmod(binary, stat.S_IXUSR | os.stat(binary).st_mode) if not program_args: program_args = [] executor = UnprotectedExecutor() frkwargs = {} # Hack for java if compiler_env.get('compiler') == 'default-java': executor = PRootExecutor('compiler-java.1_8') frkwargs['proot_options'] = ['-b', '/proc'] frunner = get_file_runner(executor, result_env) with frunner: renv = frunner(binary, program_args, stderr=sys.__stderr__, capture_output=True, **frkwargs) eq_(renv['return_code'], 0) eq_(renv['stdout'].strip(), expected_output) os.remove(binary)
def run(environ, lang, compiler, extension, output_file, compiler_options=(), compile_additional_sources=True, sandbox=False, sandbox_callback=None): """ Common code for compiler handlers: :param environ: Recipe to pass to `filetracker` and `sio.workers.execute` For all supported options, see the global documentation for `sio.compilers`. :param lang: Language code (for example: `c`, `cpp`, `pas`) :param compiler: Compiler binary name :param extension: Usual extension for source files of the given language. :param output_file: Default output binary file, assuming the input is named `a.<extension>` :param compiler_options: Optional tuple of command line parameters to the compiler. :param compile_additional_sources: Enables passing additional source files to the compiler - used as a hack to support FPC. Defaults to True. :param sandbox: Enables sandboxing (using compiler name as a sandbox). Defaults to False. :param sandbox_callback: Optional callback called immediately after creating the executor, with the said executor and the command argument list as arguments. Should return new command if modified. """ if sandbox is False: executor = UnprotectedExecutor() else: executor = PRootExecutor('compiler-' + environ['compiler']) extra_compilation_args = \ _lang_option(environ, 'extra_compilation_args', lang) ft.download(environ, 'source_file', 'a.' + extension) cmdline = [compiler, ] + list(compiler_options) + \ list(extra_compilation_args) + ['a.' + extension, ] # this cmdline may be later extended # using a copy of the environment in order to avoid polluting it with # temoporary elements tmp_environ = environ.copy() additional_includes = _lang_option(environ, 'additional_includes', lang) additional_sources = _lang_option(environ, 'additional_sources', lang) for include in additional_includes: tmp_environ['additional_include'] = include ft.download(tmp_environ, 'additional_include', os.path.basename(include)) for source in additional_sources: tmp_environ['additional_source'] = source ft.download(tmp_environ, 'additional_source', os.path.basename(source)) if compile_additional_sources: cmdline += [os.path.basename(source), ] extra_files = environ.get('extra_files', {}) for name, ft_path in extra_files.iteritems(): tmp_environ['extra_file'] = ft_path ft.download(tmp_environ, 'extra_file', os.path.basename(name)) with executor: if sandbox_callback: cmdline = sandbox_callback(executor, cmdline) or cmdline renv = executor(cmdline, time_limit=DEFAULT_COMPILER_TIME_LIMIT, mem_limit=DEFAULT_COMPILER_MEM_LIMIT, output_limit=DEFAULT_COMPILER_OUTPUT_LIMIT, ignore_errors=True, environ=tmp_environ, environ_prefix='compilation_', capture_output=True, forward_stderr=True) environ['compiler_output'] = replace_invalid_UTF(renv['stdout']) if renv['return_code']: environ['result_code'] = 'CE' elif 'compilation_result_size_limit' in environ and \ os.path.getsize(output_file) > \ environ['compilation_result_size_limit']: environ['result_code'] = 'CE' environ['compiler_output'] = 'Compiled file size limit exceeded.' else: environ['result_code'] = 'OK' ft.upload(environ, 'out_file', output_file) return environ
def run(environ, lang, compiler, extension, output_file, compiler_options=(), compile_additional_sources=True, sandbox=False, sandbox_callback=None): """ Common code for compiler handlers: :param environ: Recipe to pass to `filetracker` and `sio.workers.execute` For all supported options, see the global documentation for `sio.compilers`. :param lang: Language code (for example: `c`, `cpp`, `pas`) :param compiler: Compiler binary name :param extension: Usual extension for source files of the given language. :param output_file: Default output binary file, assuming the input is named `a.<extension>` :param compiler_options: Optional tuple of command line parameters to the compiler. :param compile_additional_sources: Disables passing additional source files to the compiler - used as a hack to support FPC. Defaults to False. :param sandbox: Enables sandboxing (using compiler name as a sandbox). Defaults to False. :param sandbox_callback: Optional callback called immediately after creating the sandbox, with the said sandbox as its sole parameter. """ if sandbox is True: sandbox = get_sandbox('compiler-' + environ['compiler']) elif not sandbox: sandbox = NullSandbox() extra_compilation_args = \ _lang_option(environ, 'extra_compilation_args', lang) ft.download(environ, 'source_file', 'a.' + extension) cmdline = (compiler, ) + tuple(compiler_options) + \ tuple(extra_compilation_args) + ('a.' + extension, ) # this cmdline may be later extended # using a copy of the environment in order to avoid polluting it with # temoporary elements tmp_environ = environ.copy() additional_includes = _lang_option(environ, 'additional_includes', lang) additional_sources = _lang_option(environ, 'additional_sources', lang) for include in additional_includes: tmp_environ['additional_include'] = include ft.download(tmp_environ, 'additional_include', os.path.basename(include)) for source in additional_sources: tmp_environ['additional_source'] = source ft.download(tmp_environ, 'additional_source', os.path.basename(source)) if compile_additional_sources: cmdline += (os.path.basename(source), ) extra_files = environ.get('extra_files', {}) for name, ft_path in extra_files.iteritems(): tmp_environ['extra_file'] = ft_path ft.download(tmp_environ, 'extra_file', os.path.basename(name)) shell_environ = os.environ.copy() if sandbox: shell_environ['LD_LIBRARY_PATH'] = (lang == 'pas') \ and os.path.join(sandbox.path, 'lib') \ or os.path.join(sandbox.path, 'usr', 'lib') shell_environ['PATH'] = (lang == 'pas') \ and os.path.join(sandbox.path, 'bin') \ or os.path.join(sandbox.path, 'usr', 'bin') shell_environ['PATH'] += ':' + os.environ['PATH'] with sandbox: if sandbox_callback: sandbox_callback(sandbox) retcode, output = execute(list(cmdline), env=shell_environ, time_limit=30, mem_limit=256, ignore_errors=True, environ=tmp_environ, environ_prefix='compilation_') environ['compiler_output'] = output if retcode: environ['result_code'] = 'CE' elif 'compilation_result_size_limit' in environ and \ os.path.getsize(output_file) > \ environ['compilation_result_size_limit']: environ['result_code'] = 'CE' environ['compiler_output'] = 'Compiled file size limit exceeded.' else: environ['result_code'] = 'OK' ft.upload(environ, 'out_file', output_file) return environ
def run(environ, executor, use_sandboxes=True): """ Common code for executors. :param: environ Recipe to pass to `filetracker` and `sio.workers.executors` For all supported options, see the global documentation for `sio.workers.executors` and prefix them with ``exec_``. :param: executor Executor instance used for executing commands. :param: use_sandboxes Enables safe checking output correctness. See `sio.executors.checkers`. True by default. """ if feedback.judge_prepare(environ).force_not_judge: environ['time_used'] = 0 environ['result_string'] = 'not judged' environ['result_code'] = 'NJ' environ['mem_used'] = 0 environ['num_syscalls'] = 0 environ['stderr'] = '' return environ input_name = tempcwd('in') file_executor = get_file_runner(executor, environ) exe_filename = file_executor.preferred_filename() ft.download(environ, 'exe_file', exe_filename, add_to_cache=True) os.chmod(tempcwd(exe_filename), 0700) ft.download(environ, 'in_file', input_name, add_to_cache=True) zipdir = tempcwd('in_dir') os.mkdir(zipdir) try: if is_zipfile(input_name): try: # If not a zip file, will pass it directly to exe with ZipFile(tempcwd('in'), 'r') as f: if len(f.namelist()) != 1: raise Exception("Archive should have only one file.") f.extract(f.namelist()[0], zipdir) input_name = os.path.join(zipdir, f.namelist()[0]) # zipfile throws some undocumented exceptions except Exception as e: raise StandardError("Failed to open archive: " + unicode(e)) with file_executor as fe: with open(input_name, 'rb') as inf: with open(tempcwd('out'), 'wb') as outf: feedback.judge_started(environ) renv = fe(tempcwd(exe_filename), [], stdin=inf, stdout=outf, ignore_errors=True, environ=environ, environ_prefix='exec_') _populate_environ(renv, environ) if renv['result_code'] == 'OK' and environ.get('check_output'): environ = checker.run(environ, use_sandboxes=use_sandboxes) for key in ('result_code', 'result_string'): environ[key] = replace_invalid_UTF(environ[key]) feedback.judge_finished(environ) if 'out_file' in environ: ft.upload(environ, 'out_file', tempcwd('out'), to_remote_store=environ.get('upload_out', False)) finally: rmtree(zipdir) return environ
def run(environ, executor, use_sandboxes=True): """ Common code for executors. :param: environ Recipe to pass to `filetracker` and `sio.workers.executors` For all supported options, see the global documentation for `sio.workers.executors` and prefix them with ``exec_``. :param: executor Executor instance used for executing commands. :param: use_sandboxes Enables safe checking output correctness. See `sio.executors.checkers`. True by default. """ input_name = tempcwd('in') file_executor = get_file_runner(executor, environ) exe_filename = file_executor.preferred_filename() ft.download(environ, 'exe_file', exe_filename, add_to_cache=True) os.chmod(tempcwd(exe_filename), 0o700) ft.download(environ, 'in_file', input_name, add_to_cache=True) zipdir = tempcwd('in_dir') os.mkdir(zipdir) try: if is_zipfile(input_name): try: # If not a zip file, will pass it directly to exe with ZipFile(tempcwd('in'), 'r') as f: if len(f.namelist()) != 1: raise Exception("Archive should have only one file.") f.extract(f.namelist()[0], zipdir) input_name = os.path.join(zipdir, f.namelist()[0]) # zipfile throws some undocumented exceptions except Exception as e: raise Exception("Failed to open archive: " + six.text_type(e)) with file_executor as fe: with open(input_name, 'rb') as inf: # Open output file in append mode to allow appending # only to the end of the output file. Otherwise, # a contestant's program could modify the middle of the file. with open(tempcwd('out'), 'ab') as outf: renv = fe(tempcwd(exe_filename), [], stdin=inf, stdout=outf, ignore_errors=True, environ=environ, environ_prefix='exec_') _populate_environ(renv, environ) if renv['result_code'] == 'OK' and environ.get('check_output'): environ = checker.run(environ, use_sandboxes=use_sandboxes) for key in ('result_code', 'result_string'): environ[key] = replace_invalid_UTF(environ[key]) if 'out_file' in environ: ft.upload(environ, 'out_file', tempcwd('out'), to_remote_store=environ.get('upload_out', False)) finally: rmtree(zipdir) return environ