def upload_files(): "Uploads all files from SOURCES to a newly created dummy filetracker" # DummyClient operates in the working directory. ft.set_instance(DummyClient()) for path in glob.glob(os.path.join(SOURCES, '*')): ft.upload({'path': '/' + os.path.basename(path)}, 'path', path)
def _collect_and_upload(env, path, upload_path, re_string): names_re = re.compile(re_string) env['collected_files'] = dict() for out_file in os.listdir(path): if names_re.match(out_file): ft.upload(env['collected_files'], out_file, out_file, '%s/%s' % (upload_path, out_file))
def run(environ, executor, safe_check=True): """ Common code for executors. :param: environ Recipe to pass to `filetracker` and `sio.workers.executors` For all supported options, see the global documentation for `sio.workers.executors`. :param: executor Executor instance used for executing commands. :param: safe_check Enables safe checking output corectness. See `sio.executors.checkers`. True by default. """ ft.download(environ, 'exe_file', 'exe', add_to_cache=True) os.chmod('exe', 0700) ft.download(environ, 'in_file', 'in', add_to_cache=True) with executor as e: with open('in', 'rb') as inf: with open('out', 'wb') as outf: renv = e(['./exe'], stdin=inf, stdout=outf, ignore_errors=True, environ=environ) _populate_environ(renv, environ) if renv['result_code'] == 'OK' and environ.get('check_output'): environ = checker.run(environ, no_sandbox=not safe_check) if 'out_file' in environ: ft.upload(environ, 'out_file', 'out', to_remote_store=environ.get('upload_out', False)) return environ
def _collect_and_upload(env, path, upload_path, re_string): names_re = re.compile(re_string) env['collected_files'] = dict() for out_file in os.listdir(path): if names_re.match(out_file): ft.upload(env['collected_files'], out_file, os.path.join(path, out_file), '%s/%s' % (upload_path, out_file))
def run(environ, executor, use_sandboxes=True): """ Common code for executors. :param: environ Recipe to pass to `filetracker` and `sio.workers.executors` For all supported options, see the global documentation for `sio.workers.executors` and prefix them with ``exec_``. :param: executor Executor instance used for executing commands. :param: use_sandboxes Enables safe checking output correctness. See `sio.executors.checkers`. True by default. """ input_name = tempcwd('in') file_executor = get_file_runner(executor, environ) exe_filename = file_executor.preferred_filename() ft.download(environ, 'exe_file', exe_filename, add_to_cache=True) os.chmod(tempcwd(exe_filename), 0700) ft.download(environ, 'in_file', input_name, add_to_cache=True) zipdir = tempcwd('in_dir') os.mkdir(zipdir) try: if is_zipfile(input_name): try: # If not a zip file, will pass it directly to exe with ZipFile(tempcwd('in'), 'r') as f: if len(f.namelist()) != 1: raise Exception("Archive should have only one file.") f.extract(f.namelist()[0], zipdir) input_name = os.path.join(zipdir, f.namelist()[0]) # zipfile throws some undocumented exceptions except Exception as e: raise StandardError("Failed to open archive: " + unicode(e)) with file_executor as fe: with open(input_name, 'rb') as inf: with open(tempcwd('out'), 'wb') as outf: renv = fe(tempcwd(exe_filename), [], stdin=inf, stdout=outf, ignore_errors=True, environ=environ, environ_prefix='exec_') _populate_environ(renv, environ) if renv['result_code'] == 'OK' and environ.get('check_output'): environ = checker.run(environ, use_sandboxes=use_sandboxes) for key in ('result_code', 'result_string'): environ[key] = replace_invalid_UTF(environ[key]) if 'out_file' in environ: ft.upload(environ, 'out_file', tempcwd('out'), to_remote_store=environ.get('upload_out', False)) finally: rmtree(zipdir) return environ
def run(environ): if environ['compiler'] not in ('foo.1_0', 'foo.2_0'): raise RuntimeError("Compiler '%s' not found.", environ['compiler']) input_file = ft.download(environ, 'source_file', 'a.foo') print input_file size = os.path.getsize(input_file) out = open('compiled', 'w') out.write("#!/bin/sh\n") out.write("# Compiled using Foo compiler named %s\n" % environ['compiler']) out.write("echo %d" % size) out.close() ft.upload(environ, 'out_file', 'compiled') return environ
def run(environ): if environ['compiler'] not in ('foo.1_0', 'foo.2_0'): raise RuntimeError("Compiler '%s' not found.", environ['compiler']) input_file = ft.download(environ, 'source_file', 'a.foo') print(input_file) size = os.path.getsize(input_file) out = open('compiled', 'w') out.write("#!/bin/sh\n") out.write("# Compiled using Foo compiler named %s\n" % environ['compiler']) out.write("echo %d" % size) out.close() ft.upload(environ, 'out_file', 'compiled') return environ
def _postprocess(self, renv): self.environ['compiler_output'] = replace_invalid_UTF(renv['stdout']) if renv['return_code']: self.environ['result_code'] = 'CE' elif 'compilation_result_size_limit' in self.environ and \ os.path.getsize(tempcwd(self.output_file)) > \ self.environ['compilation_result_size_limit']: self.environ['result_code'] = 'CE' self.environ['compiler_output'] = \ 'Compiled file size limit exceeded.' else: self.environ['result_code'] = 'OK' self.environ['exec_info'] = {'mode': 'executable'} ft.upload(self.environ, 'out_file', tempcwd(self.output_file)) return self.environ
def run(environ): exe_file = ft.download(environ, 'exe_file', 'exe', add_to_cache=True) os.chmod(exe_file, 0700) in_file = ft.download(environ, 'in_file', 'in', add_to_cache=True) if 'exec_time_limit' in environ: time_limit = environ['exec_time_limit'] time_ulimit = (time_limit + 999) / 1000 else: time_limit = time_ulimit = None if 'exec_mem_limit' in environ: mem_limit = environ['exec_mem_limit'] / 1024 else: mem_limit = None retcode, output = execute(['bash', '-c', 'time ./exe < in > out'], time_limit=time_ulimit, mem_limit=mem_limit, ignore_errors=True) time_output_matches = TIME_OUTPUT_RE.findall(output) if time_output_matches: mins, secs = time_output_matches[-1] time_used = int((int(mins) * 60 + float(secs)) * 1000) else: raise RuntimeError('Could not find output of time program. ' 'Captured output: %s' % output) if time_limit is not None and time_used >= 0.99 * time_limit: environ['result_string'] = 'time limit exceeded' environ['result_code'] = 'TLE' elif retcode == 0: environ['result_string'] = 'ok' environ['result_code'] = 'OK' else: environ['result_string'] = 'program exited with code %d' % retcode environ['result_code'] = 'RE' environ['time_used'] = time_used environ['exectime_used'] = 0 environ['mem_used'] = 0 environ['num_syscalls'] = 0 if environ['result_code'] == 'OK' and environ.get('check_output'): environ = checker.run(environ, no_sandbox=True) if 'out_file' in environ: ft.upload(environ, 'out_file', 'out', to_remote_store=environ.get('upload_out', False)) return environ
def run(environ): exe_file = ft.download(environ, 'exe_file', 'exe', add_to_cache=True) os.chmod(exe_file, 0700) in_file = ft.download(environ, 'in_file', 'in', add_to_cache=True) env = os.environ.copy() for key, default in _options: value = environ.get('exec_' + key.lower(), default) env[key] = value with get_sandbox('vcpu_exec-sandbox') as sandbox: retcode, output = execute( [os.path.join(sandbox.path, 'pin-supervisor/supervisor-bin/', 'supervisor'), '-f', '3', '--', './exe', noquote('<'), 'in', noquote('3>'), 'supervisor_result', noquote('>'), 'out'], env=env) result_file = open('supervisor_result') status_line = result_file.readline().strip().split() environ['result_string'] = result_file.readline().strip() result_file.close() for num, key in enumerate((None, 'result_code', 'time_used', 'exectime_used', 'mem_used', 'num_syscalls')): if key: environ[key] = int(status_line[num]) result_code = _supervisor_result_to_code(environ['result_code']) environ['result_code'] = result_code if result_code == 'OK' and environ.get('check_output'): environ = checker.run(environ) if 'out_file' in environ: ft.upload(environ, 'out_file', 'out', to_remote_store=environ.get('upload_out', False)) return environ
def run(environ, executor, use_sandboxes=True): """ Common code for executors. :param: environ Recipe to pass to `filetracker` and `sio.workers.executors` For all supported options, see the global documentation for `sio.workers.executors` and prefix them with ``exec_``. :param: executor Executor instance used for executing commands. :param: use_sandboxes Enables safe checking output correctness. See `sio.executors.checkers`. True by default. """ if feedback.judge_prepare(environ).force_not_judge: environ['time_used'] = 0 environ['result_string'] = 'not judged' environ['result_code'] = 'NJ' environ['mem_used'] = 0 environ['num_syscalls'] = 0 environ['stderr'] = '' return environ input_name = tempcwd('in') file_executor = get_file_runner(executor, environ) exe_filename = file_executor.preferred_filename() ft.download(environ, 'exe_file', exe_filename, add_to_cache=True) os.chmod(tempcwd(exe_filename), 0700) ft.download(environ, 'in_file', input_name, add_to_cache=True) zipdir = tempcwd('in_dir') os.mkdir(zipdir) try: if is_zipfile(input_name): try: # If not a zip file, will pass it directly to exe with ZipFile(tempcwd('in'), 'r') as f: if len(f.namelist()) != 1: raise Exception("Archive should have only one file.") f.extract(f.namelist()[0], zipdir) input_name = os.path.join(zipdir, f.namelist()[0]) # zipfile throws some undocumented exceptions except Exception as e: raise StandardError("Failed to open archive: " + unicode(e)) with file_executor as fe: with open(input_name, 'rb') as inf: with open(tempcwd('out'), 'wb') as outf: feedback.judge_started(environ) renv = fe(tempcwd(exe_filename), [], stdin=inf, stdout=outf, ignore_errors=True, environ=environ, environ_prefix='exec_') _populate_environ(renv, environ) if renv['result_code'] == 'OK' and environ.get('check_output'): environ = checker.run(environ, use_sandboxes=use_sandboxes) for key in ('result_code', 'result_string'): environ[key] = replace_invalid_UTF(environ[key]) feedback.judge_finished(environ) if 'out_file' in environ: ft.upload(environ, 'out_file', tempcwd('out'), to_remote_store=environ.get('upload_out', False)) finally: rmtree(zipdir) return environ
def _collect_and_upload(env, path, upload_path, re_string): names_re = re.compile(re_string) env["collected_files"] = dict() for out_file in os.listdir(path): if names_re.match(out_file): ft.upload(env["collected_files"], out_file, os.path.join(path, out_file), "%s/%s" % (upload_path, out_file))
'-f', '3', '--', './exe', noquote('<'), 'in', noquote('3>'), 'supervisor_result', noquote('>'), 'out'], env=env) result_file = open('supervisor_result') status_line = result_file.readline().strip().split() environ['result_string'] = result_file.readline().strip() result_file.close() try: for num, key in enumerate((None, 'result_code', 'time_used', 'exectime_used', 'mem_used', 'num_syscalls')): if key: environ[key] = int(status_line[num]) result_code = _supervisor_result_to_code(environ['result_code']) except Exception, e: result_code = 'SE' for i in ('time_used', 'exectime_used', 'mem_used', 'num_syscalls'): environ.setdefault(i, 0) environ['result_string'] = str(e) environ['result_code'] = result_code if result_code == 'OK' and environ.get('check_output'): environ = checker.run(environ) if 'out_file' in environ: ft.upload(environ, 'out_file', 'out', to_remote_store=environ.get('upload_out', False)) return environ
def run(environ, lang, compiler, extension, output_file, compiler_options=(), compile_additional_sources=True, sandbox=False, sandbox_callback=None): """ Common code for compiler handlers: :param environ: Recipe to pass to `filetracker` and `sio.workers.execute` For all supported options, see the global documentation for `sio.compilers`. :param lang: Language code (for example: `c`, `cpp`, `pas`) :param compiler: Compiler binary name :param extension: Usual extension for source files of the given language. :param output_file: Default output binary file, assuming the input is named `a.<extension>` :param compiler_options: Optional tuple of command line parameters to the compiler. :param compile_additional_sources: Disables passing additional source files to the compiler - used as a hack to support FPC. Defaults to False. :param sandbox: Enables sandboxing (using compiler name as a sandbox). Defaults to False. :param sandbox_callback: Optional callback called immediately after creating the sandbox, with the said sandbox as its sole parameter. """ if sandbox is True: sandbox = get_sandbox('compiler-' + environ['compiler']) elif not sandbox: sandbox = NullSandbox() extra_compilation_args = \ _lang_option(environ, 'extra_compilation_args', lang) ft.download(environ, 'source_file', 'a.' + extension) cmdline = (compiler, ) + tuple(compiler_options) + \ tuple(extra_compilation_args) + ('a.' + extension, ) # this cmdline may be later extended # using a copy of the environment in order to avoid polluting it with # temoporary elements tmp_environ = environ.copy() additional_includes = _lang_option(environ, 'additional_includes', lang) additional_sources = _lang_option(environ, 'additional_sources', lang) for include in additional_includes: tmp_environ['additional_include'] = include ft.download(tmp_environ, 'additional_include', os.path.basename(include)) for source in additional_sources: tmp_environ['additional_source'] = source ft.download(tmp_environ, 'additional_source', os.path.basename(source)) if compile_additional_sources: cmdline += (os.path.basename(source), ) extra_files = environ.get('extra_files', {}) for name, ft_path in extra_files.iteritems(): tmp_environ['extra_file'] = ft_path ft.download(tmp_environ, 'extra_file', os.path.basename(name)) shell_environ = os.environ.copy() if sandbox: shell_environ['LD_LIBRARY_PATH'] = (lang == 'pas') \ and os.path.join(sandbox.path, 'lib') \ or os.path.join(sandbox.path, 'usr', 'lib') shell_environ['PATH'] = (lang == 'pas') \ and os.path.join(sandbox.path, 'bin') \ or os.path.join(sandbox.path, 'usr', 'bin') shell_environ['PATH'] += ':' + os.environ['PATH'] with sandbox: if sandbox_callback: sandbox_callback(sandbox) retcode, output = execute(list(cmdline), env=shell_environ, time_limit=30, mem_limit=256, ignore_errors=True, environ=tmp_environ, environ_prefix='compilation_') environ['compiler_output'] = output if retcode: environ['result_code'] = 'CE' elif 'compilation_result_size_limit' in environ and \ os.path.getsize(output_file) > \ environ['compilation_result_size_limit']: environ['result_code'] = 'CE' environ['compiler_output'] = 'Compiled file size limit exceeded.' else: environ['result_code'] = 'OK' ft.upload(environ, 'out_file', output_file) return environ
def run(environ, executor, use_sandboxes=True): """ Common code for executors. :param: environ Recipe to pass to `filetracker` and `sio.workers.executors` For all supported options, see the global documentation for `sio.workers.executors` and prefix them with ``exec_``. :param: executor Executor instance used for executing commands. :param: use_sandboxes Enables safe checking output correctness. See `sio.executors.checkers`. True by default. """ input_name = tempcwd('in') file_executor = get_file_runner(executor, environ) exe_filename = file_executor.preferred_filename() ft.download(environ, 'exe_file', exe_filename, add_to_cache=True) os.chmod(tempcwd(exe_filename), 0o700) ft.download(environ, 'in_file', input_name, add_to_cache=True) zipdir = tempcwd('in_dir') os.mkdir(zipdir) try: if is_zipfile(input_name): try: # If not a zip file, will pass it directly to exe with ZipFile(tempcwd('in'), 'r') as f: if len(f.namelist()) != 1: raise Exception("Archive should have only one file.") f.extract(f.namelist()[0], zipdir) input_name = os.path.join(zipdir, f.namelist()[0]) # zipfile throws some undocumented exceptions except Exception as e: raise Exception("Failed to open archive: " + six.text_type(e)) with file_executor as fe: with open(input_name, 'rb') as inf: # Open output file in append mode to allow appending # only to the end of the output file. Otherwise, # a contestant's program could modify the middle of the file. with open(tempcwd('out'), 'ab') as outf: renv = fe(tempcwd(exe_filename), [], stdin=inf, stdout=outf, ignore_errors=True, environ=environ, environ_prefix='exec_') _populate_environ(renv, environ) if renv['result_code'] == 'OK' and environ.get('check_output'): environ = checker.run(environ, use_sandboxes=use_sandboxes) for key in ('result_code', 'result_string'): environ[key] = replace_invalid_UTF(environ[key]) if 'out_file' in environ: ft.upload(environ, 'out_file', tempcwd('out'), to_remote_store=environ.get('upload_out', False)) finally: rmtree(zipdir) return environ
def run(environ, lang, compiler, extension, output_file, compiler_options=(), compile_additional_sources=True, sandbox=False, sandbox_callback=None): """ Common code for compiler handlers: :param environ: Recipe to pass to `filetracker` and `sio.workers.execute` For all supported options, see the global documentation for `sio.compilers`. :param lang: Language code (for example: `c`, `cpp`, `pas`) :param compiler: Compiler binary name :param extension: Usual extension for source files of the given language. :param output_file: Default output binary file, assuming the input is named `a.<extension>` :param compiler_options: Optional tuple of command line parameters to the compiler. :param compile_additional_sources: Enables passing additional source files to the compiler - used as a hack to support FPC. Defaults to True. :param sandbox: Enables sandboxing (using compiler name as a sandbox). Defaults to False. :param sandbox_callback: Optional callback called immediately after creating the executor, with the said executor and the command argument list as arguments. Should return new command if modified. """ if sandbox is False: executor = UnprotectedExecutor() else: executor = PRootExecutor('compiler-' + environ['compiler']) extra_compilation_args = \ _lang_option(environ, 'extra_compilation_args', lang) ft.download(environ, 'source_file', 'a.' + extension) cmdline = [compiler, ] + list(compiler_options) + \ list(extra_compilation_args) + ['a.' + extension, ] # this cmdline may be later extended # using a copy of the environment in order to avoid polluting it with # temoporary elements tmp_environ = environ.copy() additional_includes = _lang_option(environ, 'additional_includes', lang) additional_sources = _lang_option(environ, 'additional_sources', lang) for include in additional_includes: tmp_environ['additional_include'] = include ft.download(tmp_environ, 'additional_include', os.path.basename(include)) for source in additional_sources: tmp_environ['additional_source'] = source ft.download(tmp_environ, 'additional_source', os.path.basename(source)) if compile_additional_sources: cmdline += [os.path.basename(source), ] extra_files = environ.get('extra_files', {}) for name, ft_path in extra_files.iteritems(): tmp_environ['extra_file'] = ft_path ft.download(tmp_environ, 'extra_file', os.path.basename(name)) with executor: if sandbox_callback: cmdline = sandbox_callback(executor, cmdline) or cmdline renv = executor(cmdline, time_limit=DEFAULT_COMPILER_TIME_LIMIT, mem_limit=DEFAULT_COMPILER_MEM_LIMIT, output_limit=DEFAULT_COMPILER_OUTPUT_LIMIT, ignore_errors=True, environ=tmp_environ, environ_prefix='compilation_', capture_output=True, forward_stderr=True) environ['compiler_output'] = replace_invalid_UTF(renv['stdout']) if renv['return_code']: environ['result_code'] = 'CE' elif 'compilation_result_size_limit' in environ and \ os.path.getsize(output_file) > \ environ['compilation_result_size_limit']: environ['result_code'] = 'CE' environ['compiler_output'] = 'Compiled file size limit exceeded.' else: environ['result_code'] = 'OK' ft.upload(environ, 'out_file', output_file) return environ