def compile(self, environ):
"""
Compile the file specified in the `environ` dictionary.
:param environ: Recipe to pass to `filetracker` and
`sio.workers.execute` For all supported options,
see the global documentation for `sio.compilers`.
"""
self.environ = environ
compilation_started(environ)
# using a copy of the environment in order to avoid polluting it with
# temoporary elements
self.tmp_environ = environ.copy()
self.source_file = self._make_filename()
ft.download(environ, 'source_file', self.source_file)
self._process_extra_files()
self.extra_compilation_args = \
_lang_option(environ, 'extra_compilation_args', self.lang)
with self.executor as executor:
renv = self._run_in_executor(executor)
result = self._postprocess(renv)
compilation_finished(result)
return result
def run(environ):
exe_file = ft.download(environ, 'exe_file', 'exe', add_to_cache=True)
os.chmod(exe_file, 0700)
in_file = ft.download(environ, 'in_file', 'in', add_to_cache=True)
env = os.environ.copy()
for key, default in _options:
value = environ.get('exec_' + key.lower(), default)
env[key] = value
with get_sandbox('vcpu_exec-sandbox') as sandbox:
retcode, output = execute(
[os.path.join(sandbox.path, 'pin-supervisor/supervisor-bin/',
'supervisor'),
'-f', '3', '--', './exe',
noquote('<'), 'in', noquote('3>'), 'supervisor_result',
noquote('>'), 'out'], env=env)
result_file = open('supervisor_result')
status_line = result_file.readline().strip().split()
environ['result_string'] = result_file.readline().strip()
result_file.close()
try:
for num, key in enumerate((None, 'result_code', 'time_used',
'exectime_used', 'mem_used', 'num_syscalls')):
if key:
environ[key] = int(status_line[num])
result_code = _supervisor_result_to_code(environ['result_code'])
except Exception, e:
result_code = 'SE'
for i in ('time_used', 'exectime_used', 'mem_used', 'num_syscalls'):
environ.setdefault(i, 0)
environ['result_string'] = str(e)
def run(environ, executor, safe_check=True):
"""
Common code for executors.
:param: environ Recipe to pass to `filetracker` and `sio.workers.executors`
For all supported options, see the global documentation for
`sio.workers.executors`.
:param: executor Executor instance used for executing commands.
:param: safe_check Enables safe checking output corectness.
See `sio.executors.checkers`. True by default.
"""
ft.download(environ, 'exe_file', 'exe', add_to_cache=True)
os.chmod('exe', 0700)
ft.download(environ, 'in_file', 'in', add_to_cache=True)
with executor as e:
with open('in', 'rb') as inf:
with open('out', 'wb') as outf:
renv = e(['./exe'], stdin=inf, stdout=outf, ignore_errors=True,
environ=environ)
_populate_environ(renv, environ)
if renv['result_code'] == 'OK' and environ.get('check_output'):
environ = checker.run(environ, no_sandbox=not safe_check)
if 'out_file' in environ:
ft.upload(environ, 'out_file', 'out',
to_remote_store=environ.get('upload_out', False))
return environ
def _test_transparent_exec(source, executor, callback, kwargs):
with TemporaryCwd():
upload_files()
ft.download({'path': '/somefile'}, 'path', tempcwd('somefile'))
result_env = compile_and_execute(source, executor, **kwargs)
print_env(result_env)
if callback:
callback(result_env)
def _test_transparent_exec(source, executor, callback, kwargs):
with TemporaryCwd():
upload_files()
ft.download({'path': '/somefile'}, 'path', tempcwd('somefile'))
result_env = compile_and_execute(source, executor, **kwargs)
print_env(result_env)
if callback:
callback(result_env)
def run(environ):
"""Runs a verifying program and returns its output.
Used ``environ`` keys:
``exe_file``: the filetracker path to the program
``in_file``: the file redirected to the program's stdin
``use_sandboxes``: if this key equals ``True``, the program is executed
in the SupervisedExecutor, otherwise the UnsafeExecutor
is used
``inwer_time_limit``: time limit in ms
(optional, the default is 30 s)
``inwer_mem_limit``: memory limit in KiB
(optional, the default is 256 MiB)
``inwer_output_limit``: output limit in B
(optional, the default is 10 KiB)
Returns a new environ, whose ``stdout`` key contains the program's
output.
The verifying program is expected to return 0, its first line of output
should begin with "OK". If this does not happen, an appropriate message
is logged.
Program's output is returned under the ``stdout`` key. If the output has
more than ``inwer_output_limit`` bytes and ``use_sandboxes`` is
set to ``True``, the execution of the program fails with ``OLE`` result
code.
"""
use_sandboxes = environ.get('use_sandboxes', False)
ft.download(environ,
'exe_file',
'inwer',
skip_if_exists=True,
add_to_cache=True)
ft.download(environ,
'in_file',
'in',
skip_if_exists=True,
add_to_cache=True)
os.chmod(tempcwd('inwer'), 0500)
renv = _run_inwer(environ, use_sandboxes)
if renv['result_code'] != "OK":
logger.error("Inwer failed!\nEnviron dump: %s\nExecution environ: %s",
environ, renv)
elif not renv['stdout'][0].startswith("OK"):
logger.error(
"Bad inwer output!\nEnviron dump: %s\n"
"Execution environ: %s", environ, renv)
return renv
def run(environ, executor, use_sandboxes=True):
"""
Common code for executors.
:param: environ Recipe to pass to `filetracker` and `sio.workers.executors`
For all supported options, see the global documentation for
`sio.workers.executors` and prefix them with ``exec_``.
:param: executor Executor instance used for executing commands.
:param: use_sandboxes Enables safe checking output correctness.
See `sio.executors.checkers`. True by default.
"""
input_name = tempcwd('in')
file_executor = get_file_runner(executor, environ)
exe_filename = file_executor.preferred_filename()
ft.download(environ, 'exe_file', exe_filename, add_to_cache=True)
os.chmod(tempcwd(exe_filename), 0700)
ft.download(environ, 'in_file', input_name, add_to_cache=True)
zipdir = tempcwd('in_dir')
os.mkdir(zipdir)
try:
if is_zipfile(input_name):
try:
# If not a zip file, will pass it directly to exe
with ZipFile(tempcwd('in'), 'r') as f:
if len(f.namelist()) != 1:
raise Exception("Archive should have only one file.")
f.extract(f.namelist()[0], zipdir)
input_name = os.path.join(zipdir, f.namelist()[0])
# zipfile throws some undocumented exceptions
except Exception as e:
raise StandardError("Failed to open archive: " + unicode(e))
with file_executor as fe:
with open(input_name, 'rb') as inf:
with open(tempcwd('out'), 'wb') as outf:
renv = fe(tempcwd(exe_filename), [],
stdin=inf, stdout=outf, ignore_errors=True,
environ=environ, environ_prefix='exec_')
_populate_environ(renv, environ)
if renv['result_code'] == 'OK' and environ.get('check_output'):
environ = checker.run(environ, use_sandboxes=use_sandboxes)
for key in ('result_code', 'result_string'):
environ[key] = replace_invalid_UTF(environ[key])
if 'out_file' in environ:
ft.upload(environ, 'out_file', tempcwd('out'),
to_remote_store=environ.get('upload_out', False))
finally:
rmtree(zipdir)
return environ
def compile_and_execute(source, executor, **exec_args):
exe_file = compile(source)
ft.download({'exe_file': exe_file}, 'exe_file', 'exe')
os.chmod('exe', 0700)
ft.download({'in_file': '/input'}, 'in_file', 'in')
with executor as e:
with open('in', 'rb') as inf:
renv = e(['./exe'], stdin=inf, **exec_args)
return renv
def test_zip():
with TemporaryCwd():
upload_files()
compile_and_run(
"/echo.c", {
'in_file': '/input.zip',
'out_file': '/output',
'exec_mem_limit': 102400
}, DetailedUnprotectedExecutor())
ft.download({'in_file': '/input'}, 'in_file', 'out.expected')
ft.download({'out_file': '/output'}, 'out_file', 'out.real')
ok_(filecmp.cmp(tempcwd('out.expected'), tempcwd('out.real')))
def test_uploading_out():
with TemporaryCwd():
upload_files()
renv = compile_and_run('/add_print.c', {
'in_file': '/input',
'out_file': '/output',
'upload_out': True,
}, DetailedUnprotectedExecutor())
print_env(renv)
ft.download({'path': '/output'}, 'path', 'd_out')
in_('84', open(tempcwd('d_out')).read())
def test_uploading_out():
with TemporaryCwd():
upload_files()
renv = compile_and_run('/add_print.c', {
'in_file': '/input',
'out_file': '/output',
'upload_out': True,
}, DetailedUnprotectedExecutor())
print_env(renv)
ft.download({'path': '/output'}, 'path', 'd_out')
in_('84', open(tempcwd('d_out')).read())
def test_zip():
with TemporaryCwd():
upload_files()
compile_and_run("/echo.c", {
'in_file': '/input.zip',
'out_file': '/output',
'exec_mem_limit': 102400
}, DetailedUnprotectedExecutor())
ft.download({'in_file': '/input'}, 'in_file', 'out.expected')
ft.download({'out_file': '/output'}, 'out_file', 'out.real')
ok_(filecmp.cmp(tempcwd('out.expected'),
tempcwd('out.real')))
def run(environ):
"""Runs a verifying program and returns its output.
Used ``environ`` keys:
``exe_file``: the filetracker path to the program
``in_file``: the file redirected to the program's stdin
``use_sandboxes``: if this key equals ``True``, the program is executed
in the SupervisedExecutor, otherwise the UnsafeExecutor
is used
``inwer_time_limit``: time limit in ms
(optional, the default is 30 s)
``inwer_mem_limit``: memory limit in KiB
(optional, the default is 256 MiB)
``inwer_output_limit``: output limit in B
(optional, the default is 10 KiB)
Returns a new environ, whose ``stdout`` key contains the program's
output.
The verifying program is expected to return 0, its first line of output
should begin with "OK". If this does not happen, an appropriate message
is logged.
Program's output is returned under the ``stdout`` key. If the output has
more than ``inwer_output_limit`` bytes and ``use_sandboxes`` is
set to ``True``, the execution of the program fails with ``OLE`` result
code.
"""
use_sandboxes = environ.get('use_sandboxes', False)
ft.download(environ, 'exe_file', 'inwer', skip_if_exists=True,
add_to_cache=True)
ft.download(environ, 'in_file', 'in', skip_if_exists=True,
add_to_cache=True)
os.chmod(tempcwd('inwer'), 0o500)
renv = _run_inwer(environ, use_sandboxes)
if renv['result_code'] != "OK":
logger.error("Inwer failed!\nEnviron dump: %s\nExecution environ: %s",
environ, renv)
elif not renv['stdout'][0].startswith(b"OK"):
logger.error("Bad inwer output!\nEnviron dump: %s\n"
"Execution environ: %s", environ, renv)
return renv
def inner(env):
eq_(env['return_code'], 0)
eq_(env['stdout'], expected_output)
collected = env['collected_files']
eq_(len(expected_files), len(collected))
for filename, path in collected.iteritems():
in_(filename, expected_files)
unversioned_path = '/%s/%s' % (upload_dir, filename)
upload_re_str = '%s@\d+' % (unversioned_path)
upload_re = re.compile(upload_re_str)
ok_(upload_re.match(path), 'Unexpected filetracker path')
ft.download({'in': unversioned_path}, 'in', filename)
eq_(expected_files[filename], open(tempcwd(filename)).read())
def _process_extra_files(self):
self.additional_includes = _lang_option(self.environ,
'additional_includes',
self.lang)
self.additional_sources = _lang_option(self.environ,
'additional_sources', self.lang)
for include in self.additional_includes:
self.tmp_environ['additional_include'] = include
ft.download(self.tmp_environ, 'additional_include',
os.path.basename(include))
for source in self.additional_sources:
self.tmp_environ['additional_source'] = source
ft.download(self.tmp_environ, 'additional_source',
os.path.basename(source))
extra_files = self.environ.get('extra_files', {})
for name, ft_path in extra_files.iteritems():
self.tmp_environ['extra_file'] = ft_path
ft.download(self.tmp_environ, 'extra_file', os.path.basename(name))
archive = self.environ.get('additional_archive', '')
if archive:
self.tmp_environ['additional_archive'] = archive
archive_path = os.path.basename(archive)
ft.download(self.tmp_environ, 'additional_archive', archive_path)
_extract_all(archive_path)
def inner(env):
eq_(env['return_code'], 0)
eq_(env['stdout'], expected_output)
collected = env['collected_files']
eq_(len(expected_files), len(collected))
for filename, path in collected.iteritems():
in_(filename, expected_files)
unversioned_path = '/%s/%s' % (upload_dir, filename)
upload_re_str = '%s@\d+' % (unversioned_path)
upload_re = re.compile(upload_re_str)
ok_(upload_re.match(path), 'Unexpected filetracker path')
ft.download({'in': unversioned_path}, 'in', filename)
eq_(expected_files[filename], open(tempcwd(filename)).read())
def run(environ, no_sandbox=False):
ft.download(environ, 'out_file', 'out', skip_if_exists=True)
ft.download(environ, 'hint_file', 'hint', add_to_cache=True)
if environ.get('chk_file'):
ft.download(environ, 'in_file', 'in', skip_if_exists=True,
add_to_cache=True)
ft.download(environ, 'chk_file', 'chk', add_to_cache=True)
os.chmod('chk', 0700)
with UnprotectedExecutor() as e:
renv = e(['./chk', 'in', 'out', 'hint'], capture_output=True,
ignore_errors=True, split_lines=True)
output = renv['stdout']
elif no_sandbox:
output = [_run_diff()]
else:
with SandboxExecutor('exec-sandbox') as e:
renv = e([os.path.join('bin', 'compare'), 'hint', 'out'],
capture_output=True, split_lines=True, ignore_errors=True)
output = renv['stdout']
while len(output) < 3:
output.append('')
if output[0] == 'OK':
environ['result_code'] = 'OK'
if output[1]:
environ['result_string'] = output[1]
environ['result_percentage'] = float(output[2] or 100)
else:
environ['result_code'] = 'WA'
environ['result_string'] = output[1]
environ['result_percentage'] = 0
return environ
def _process_extra_files(self):
self.additional_includes = _lang_option(self.environ,
'additional_includes',
self.lang)
self.additional_sources = _lang_option(self.environ,
'additional_sources', self.lang)
for include in self.additional_includes:
self.tmp_environ['additional_include'] = include
ft.download(self.tmp_environ, 'additional_include',
os.path.basename(include))
for source in self.additional_sources:
self.tmp_environ['additional_source'] = source
ft.download(self.tmp_environ, 'additional_source',
os.path.basename(source))
extra_files = self.environ.get('extra_files', {})
for name, ft_path in extra_files.iteritems():
self.tmp_environ['extra_file'] = ft_path
ft.download(self.tmp_environ, 'extra_file', os.path.basename(name))
archive = self.environ.get('additional_archive', '')
if archive:
self.tmp_environ['additional_archive'] = archive
archive_path = os.path.basename(archive)
ft.download(self.tmp_environ, 'additional_archive', archive_path)
_extract_all(archive_path)
def run(environ, no_sandbox=False):
ft.download(environ, 'out_file', 'out', skip_if_exists=True)
ft.download(environ, 'hint_file', 'hint', add_to_cache=True)
if environ.get('chk_file'):
sandbox = NullSandbox()
ft.download(environ, 'in_file', 'in', skip_if_exists=True,
add_to_cache=True)
ft.download(environ, 'chk_file', 'chk', add_to_cache=True)
os.chmod('chk', 0700)
cmd = ['./chk', 'in', 'out', 'hint', noquote('2>'), '/dev/null']
elif no_sandbox:
sandbox = NullSandbox()
cmd = ['diff', '-b', '-q', 'out', 'hint', noquote('>'), '/dev/null',
noquote('2>&1'), noquote('&&'), 'echo', 'OK']
else:
sandbox = get_sandbox('exec-sandbox')
cmd = [os.path.join(sandbox.path, 'bin', 'compare'), 'hint', 'out',
noquote('2>'), '/dev/null']
with sandbox:
retcode, output = execute(cmd, ignore_errors=True, split_lines=True)
while len(output) < 3:
output.append('')
if output[0] == 'OK':
environ['result_code'] = 'OK'
if output[1]:
environ['result_string'] = output[1]
environ['result_percentage'] = float(output[2] or 100)
else:
environ['result_code'] = 'WA'
environ['result_string'] = output[1]
environ['result_percentage'] = 0
return environ
def test_output_compilation_and_running(source):
with TemporaryCwd():
upload_files()
result_env = run(
{
'source_file': source,
'compiler': 'output-only',
}
)
eq_(result_env['result_code'], 'OK')
eq_(result_env['exec_info'], {'mode': 'output-only'})
ft.download(result_env, 'out_file', tempcwd('out.txt'))
ft.download({'source_file': source}, 'source_file', tempcwd('source.txt'))
with open(tempcwd('out.txt'), 'r') as outfile:
with open(tempcwd('source.txt'), 'r') as sourcefile:
eq_(outfile.read(), sourcefile.read())
post_run_env = run_from_executors(
{
'exec_info': result_env['exec_info'],
'exe_file': result_env['out_file'],
'check_output': True,
'hint_file': source,
},
executor=None,
)
eq_(post_run_env['result_code'], 'OK')
ft.download(post_run_env, 'out_file', tempcwd('out.txt'))
ft.download({'source_file': source}, 'source_file', tempcwd('source.txt'))
with open(tempcwd('out.txt'), 'r') as outfile:
with open(tempcwd('source.txt'), 'r') as sourcefile:
eq_(outfile.read(), sourcefile.read())
def run(environ, use_sandboxes=True):
ft.download(environ, 'out_file', 'out', skip_if_exists=True)
ft.download(environ, 'hint_file', 'hint', add_to_cache=True)
try:
if environ.get('chk_file'):
ft.download(environ, 'in_file', 'in', skip_if_exists=True,
add_to_cache=True)
ft.download(environ, 'chk_file', 'chk', add_to_cache=True)
os.chmod('chk', 0700)
output = _run_checker(environ, use_sandboxes)
elif use_sandboxes:
output = _run_compare(environ)
else:
output = _run_diff(environ)
except ExecError as e:
logger.error('Checker failed! %s', e)
logger.error('Environ dump: %s', environ)
environ['result_code'] = 'SE'
environ['result_string'] = 'checker failure'
return environ
while len(output) < 3:
output.append('')
if output[0] == 'OK':
environ['result_code'] = 'OK'
if output[1]:
environ['result_string'] = output[1]
environ['result_percentage'] = float(output[2] or 100)
else:
environ['result_code'] = 'WA'
environ['result_string'] = output[1]
environ['result_percentage'] = 0
return environ
def compile_and_execute(source, executor, **exec_args):
cenv = compile(source,
use_sandboxes=isinstance(executor, _SIOSupervisedExecutor))
frunner = get_file_runner(executor, cenv)
ft.download({'exe_file': cenv['out_file']}, 'exe_file',
frunner.preferred_filename())
os.chmod(tempcwd('exe'), 0700)
ft.download({'in_file': '/input'}, 'in_file', 'in')
with frunner:
with open(tempcwd('in'), 'rb') as inf:
renv = frunner(tempcwd('exe'), [], stdin=inf, **exec_args)
return renv
def compile_and_execute(source, executor, **exec_args):
cenv = compile(source,
use_sandboxes=isinstance(executor, _SIOSupervisedExecutor))
frunner = get_file_runner(executor, cenv)
ft.download({'exe_file': cenv['out_file']}, 'exe_file',
frunner.preferred_filename())
os.chmod(tempcwd('exe'), 0700)
ft.download({'in_file': '/input'}, 'in_file', 'in')
with frunner:
with open(tempcwd('in'), 'rb') as inf:
renv = frunner(tempcwd('exe'), [], stdin=inf, **exec_args)
return renv
def run(environ):
exe_file = ft.download(environ, 'exe_file', 'exe', add_to_cache=True)
os.chmod(exe_file, 0700)
in_file = ft.download(environ, 'in_file', 'in', add_to_cache=True)
if 'exec_time_limit' in environ:
time_limit = environ['exec_time_limit']
time_ulimit = (time_limit + 999) / 1000
else:
time_limit = time_ulimit = None
if 'exec_mem_limit' in environ:
mem_limit = environ['exec_mem_limit'] / 1024
else:
mem_limit = None
retcode, output = execute(['bash', '-c', 'time ./exe < in > out'],
time_limit=time_ulimit, mem_limit=mem_limit, ignore_errors=True)
time_output_matches = TIME_OUTPUT_RE.findall(output)
if time_output_matches:
mins, secs = time_output_matches[-1]
time_used = int((int(mins) * 60 + float(secs)) * 1000)
else:
raise RuntimeError('Could not find output of time program. '
'Captured output: %s' % output)
if time_limit is not None and time_used >= 0.99 * time_limit:
environ['result_string'] = 'time limit exceeded'
environ['result_code'] = 'TLE'
elif retcode == 0:
environ['result_string'] = 'ok'
environ['result_code'] = 'OK'
else:
environ['result_string'] = 'program exited with code %d' % retcode
environ['result_code'] = 'RE'
environ['time_used'] = time_used
environ['exectime_used'] = 0
environ['mem_used'] = 0
environ['num_syscalls'] = 0
if environ['result_code'] == 'OK' and environ.get('check_output'):
environ = checker.run(environ, no_sandbox=True)
if 'out_file' in environ:
ft.upload(environ, 'out_file', 'out',
to_remote_store=environ.get('upload_out', False))
return environ
def run(environ):
"""Runs a program, collects the files produced by it and uploads them
to filetracker.
Used ``environ`` keys:
``exe_file``: the filetracker path to the program
``re_string``: a regular expression string used to identify the files
which should be uploaded
``collected_files_path``: a directory into which the collected files
should be uploaded in filetracker
``use_sandboxes``: if this key equals ``True``, the program is executed
in the PRootExecutor, otherwise the UnsafeExecutor is
used
``ingen_time_limit``: time limit in ms
(optional, the default is 10 mins)
``ingen_mem_limit``: memory limit in KiB
(optional, the default is 256 MiB)
``ingen_output_limit``: output limit in B
(optional, the default is 10 KiB)
On success returns a new environ with a dictionary mapping collected
files' names to their filetracker paths under ``collected_files``.
Program's output is returned under the ``stdout`` key. The output is
trimmed to the first ``ingen_output_limit`` bytes.
"""
use_sandboxes = environ.get('use_sandboxes', False)
ft.download(environ,
'exe_file',
'ingen',
skip_if_exists=True,
add_to_cache=True)
os.chmod(tempcwd('ingen'), 0500)
renv = _run_ingen(environ, use_sandboxes)
if renv['return_code'] != 0:
logger.error("Ingen failed!\nEnviron dump: %s\nExecution environ: %s",
environ, renv)
return renv
def run(environ):
"""Runs a program, collects the files produced by it and uploads them
to filetracker.
Used ``environ`` keys:
``exe_file``: the filetracker path to the program
``re_string``: a regular expression string used to identify the files
which should be uploaded
``collected_files_path``: a directory into which the collected files
should be uploaded in filetracker
``use_sandboxes``: if this key equals ``True``, the program is executed
in the PRootExecutor, otherwise the UnsafeExecutor is
used
``ingen_time_limit``: time limit in ms
(optional, the default is 10 mins)
``ingen_mem_limit``: memory limit in KiB
(optional, the default is 256 MiB)
``ingen_output_limit``: output limit in B
(optional, the default is 10 KiB)
On success returns a new environ with a dictionary mapping collected
files' names to their filetracker paths under ``collected_files``.
Program's output is returned under the ``stdout`` key. The output is
trimmed to the first ``ingen_output_limit`` bytes.
"""
use_sandboxes = environ.get('use_sandboxes', False)
ft.download(environ, 'exe_file', 'ingen', skip_if_exists=True,
add_to_cache=True)
os.chmod('ingen', 0500)
renv = _run_ingen(environ, use_sandboxes)
if renv['return_code'] != 0:
logger.error("Ingen failed!\nEnviron dump: %s\nExecution environ: %s",
environ, renv)
return renv
def run(environ):
if environ['compiler'] not in ('foo.1_0', 'foo.2_0'):
raise RuntimeError("Compiler '%s' not found.", environ['compiler'])
input_file = ft.download(environ, 'source_file', 'a.foo')
print input_file
size = os.path.getsize(input_file)
out = open('compiled', 'w')
out.write("#!/bin/sh\n")
out.write("# Compiled using Foo compiler named %s\n" % environ['compiler'])
out.write("echo %d" % size)
out.close()
ft.upload(environ, 'out_file', 'compiled')
return environ
def run(environ):
if environ['compiler'] not in ('foo.1_0', 'foo.2_0'):
raise RuntimeError("Compiler '%s' not found.", environ['compiler'])
input_file = ft.download(environ, 'source_file', 'a.foo')
print(input_file)
size = os.path.getsize(input_file)
out = open('compiled', 'w')
out.write("#!/bin/sh\n")
out.write("# Compiled using Foo compiler named %s\n" % environ['compiler'])
out.write("echo %d" % size)
out.close()
ft.upload(environ, 'out_file', 'compiled')
return environ
def compile(self, environ):
"""
Compile the file specified in the `environ` dictionary.
:param environ: Recipe to pass to `filetracker` and
`sio.workers.execute` For all supported options,
see the global documentation for `sio.compilers`.
"""
self.environ = environ
# using a copy of the environment in order to avoid polluting it with
# temoporary elements
self.tmp_environ = environ.copy()
self.source_file = self._make_filename()
ft.download(environ, 'source_file', self.source_file)
self._process_extra_files()
self.extra_compilation_args = \
_lang_option(environ, 'extra_compilation_args', self.lang)
with self.executor as executor:
renv = self._run_in_executor(executor)
return self._postprocess(renv)
def compile_and_run(compiler_env, expected_output):
"""Helper function for compiling, launching and
testing the result of a program.
"""
run(compiler_env)
binary = ft.download({'path': '/out'}, 'path')
os.chmod(binary, stat.S_IXUSR)
retcode, output = execute(['./' + binary])
eq_(retcode, 0)
eq_(output.strip(), expected_output)
os.remove(binary)
def compile_and_run(compiler_env, expected_output):
"""Helper function for compiling, launching and
testing the result of a program.
"""
# Dummy sandbox doesn't support asking for versioned filename
out_file = compiler_env['out_file']
result_env = run(compiler_env)
eq_(result_env['result_code'], 'OK')
binary = ft.download({'path': out_file}, 'path')
os.chmod(binary, stat.S_IXUSR)
retcode, output = execute(['./' + binary])
eq_(retcode, 0)
eq_(output.strip(), expected_output)
os.remove(binary)
def compile_and_run(compiler_env, expected_output, program_args=None):
"""Helper function for compiling, launching and
testing the result of a program.
"""
# Dummy sandbox doesn't support asking for versioned filename
out_file = compiler_env['out_file']
if compiler_env.get('compiler', '').endswith('-java'):
compiler_env['compilation_time_limit'] = 180000
with TemporaryCwd('compile'):
result_env = run(compiler_env)
print_env(result_env)
eq_(result_env['result_code'], 'OK')
binary = ft.download({'path': out_file}, 'path')
os.chmod(binary, stat.S_IXUSR | os.stat(binary).st_mode)
if not program_args:
program_args = []
executor = UnprotectedExecutor()
frkwargs = {}
# Hack for java
if compiler_env.get('compiler') == 'default-java':
executor = PRootExecutor('compiler-java.1_8')
frkwargs['proot_options'] = ['-b', '/proc']
frunner = get_file_runner(executor, result_env)
with frunner:
renv = frunner(binary,
program_args,
stderr=sys.__stderr__,
capture_output=True,
**frkwargs)
eq_(renv['return_code'], 0)
eq_(renv['stdout'].decode().strip(), expected_output)
os.remove(binary)
def run(environ, use_sandboxes=True):
ft.download(environ, 'out_file', 'out', skip_if_exists=True)
ft.download(environ, 'hint_file', 'hint', add_to_cache=True)
try:
if environ.get('chk_file'):
ft.download(environ,
'in_file',
'in',
skip_if_exists=True,
add_to_cache=True)
ft.download(environ, 'chk_file', 'chk', add_to_cache=True)
os.chmod(tempcwd('chk'), 0o700)
output = _run_checker(environ, use_sandboxes)
elif use_sandboxes:
output = _run_compare(environ)
else:
output = _run_diff(environ)
except (CheckerError, ExecError) as e:
logger.error('Checker failed! %s', e)
logger.error('Environ dump: %s', environ)
raise SystemError(e)
while len(output) < 3:
output.append('')
if six.ensure_binary(output[0]) == b'OK':
environ['result_code'] = 'OK'
if output[1]:
environ['result_string'] = _limit_length(output[1])
environ['result_percentage'] = float(output[2] or 100)
else:
environ['result_code'] = 'WA'
environ['result_string'] = _limit_length(output[1])
environ['result_percentage'] = 0
return environ
def compile_and_run(compiler_env, expected_output, program_args=None):
"""Helper function for compiling, launching and
testing the result of a program.
"""
# Dummy sandbox doesn't support asking for versioned filename
out_file = compiler_env['out_file']
if compiler_env.get('compiler', '').endswith('-java'):
compiler_env['compilation_time_limit'] = 180000
with TemporaryCwd('compile'):
result_env = run(compiler_env)
print_env(result_env)
eq_(result_env['result_code'], 'OK')
binary = ft.download({'path': out_file}, 'path')
os.chmod(binary, stat.S_IXUSR | os.stat(binary).st_mode)
if not program_args:
program_args = []
executor = UnprotectedExecutor()
frkwargs = {}
# Hack for java
if compiler_env.get('compiler') == 'default-java':
executor = PRootExecutor('compiler-java.1_8')
frkwargs['proot_options'] = ['-b', '/proc']
frunner = get_file_runner(executor, result_env)
with frunner:
renv = frunner(binary, program_args,
stderr=sys.__stderr__, capture_output=True, **frkwargs)
eq_(renv['return_code'], 0)
eq_(renv['stdout'].strip(), expected_output)
os.remove(binary)
def run(environ, lang, compiler, extension, output_file, compiler_options=(),
compile_additional_sources=True, sandbox=False,
sandbox_callback=None):
"""
Common code for compiler handlers:
:param environ: Recipe to pass to `filetracker` and `sio.workers.execute`
For all supported options, see the global documentation for
`sio.compilers`.
:param lang: Language code (for example: `c`, `cpp`, `pas`)
:param compiler: Compiler binary name
:param extension: Usual extension for source files of the given language.
:param output_file: Default output binary file, assuming the input is named
`a.<extension>`
:param compiler_options: Optional tuple of command line parameters to the
compiler.
:param compile_additional_sources: Enables passing additional
source files to the compiler - used
as a hack to support FPC.
Defaults to True.
:param sandbox: Enables sandboxing (using compiler name
as a sandbox). Defaults to False.
:param sandbox_callback: Optional callback called immediately after
creating the executor, with the said executor
and the command argument list as arguments.
Should return new command if modified.
"""
if sandbox is False:
executor = UnprotectedExecutor()
else:
executor = PRootExecutor('compiler-' + environ['compiler'])
extra_compilation_args = \
_lang_option(environ, 'extra_compilation_args', lang)
ft.download(environ, 'source_file', 'a.' + extension)
cmdline = [compiler, ] + list(compiler_options) + \
list(extra_compilation_args) + ['a.' + extension, ]
# this cmdline may be later extended
# using a copy of the environment in order to avoid polluting it with
# temoporary elements
tmp_environ = environ.copy()
additional_includes = _lang_option(environ, 'additional_includes', lang)
additional_sources = _lang_option(environ, 'additional_sources', lang)
for include in additional_includes:
tmp_environ['additional_include'] = include
ft.download(tmp_environ, 'additional_include',
os.path.basename(include))
for source in additional_sources:
tmp_environ['additional_source'] = source
ft.download(tmp_environ, 'additional_source',
os.path.basename(source))
if compile_additional_sources:
cmdline += [os.path.basename(source), ]
extra_files = environ.get('extra_files', {})
for name, ft_path in extra_files.iteritems():
tmp_environ['extra_file'] = ft_path
ft.download(tmp_environ, 'extra_file', os.path.basename(name))
with executor:
if sandbox_callback:
cmdline = sandbox_callback(executor, cmdline) or cmdline
renv = executor(cmdline,
time_limit=DEFAULT_COMPILER_TIME_LIMIT,
mem_limit=DEFAULT_COMPILER_MEM_LIMIT,
output_limit=DEFAULT_COMPILER_OUTPUT_LIMIT,
ignore_errors=True,
environ=tmp_environ,
environ_prefix='compilation_',
capture_output=True,
forward_stderr=True)
environ['compiler_output'] = replace_invalid_UTF(renv['stdout'])
if renv['return_code']:
environ['result_code'] = 'CE'
elif 'compilation_result_size_limit' in environ and \
os.path.getsize(output_file) > \
environ['compilation_result_size_limit']:
environ['result_code'] = 'CE'
environ['compiler_output'] = 'Compiled file size limit exceeded.'
else:
environ['result_code'] = 'OK'
ft.upload(environ, 'out_file', output_file)
return environ
def run(environ, lang, compiler, extension, output_file, compiler_options=(),
compile_additional_sources=True, sandbox=False,
sandbox_callback=None):
"""
Common code for compiler handlers:
:param environ: Recipe to pass to `filetracker` and `sio.workers.execute`
For all supported options, see the global documentation for
`sio.compilers`.
:param lang: Language code (for example: `c`, `cpp`, `pas`)
:param compiler: Compiler binary name
:param extension: Usual extension for source files of the given language.
:param output_file: Default output binary file, assuming the input is named
`a.<extension>`
:param compiler_options: Optional tuple of command line parameters to the
compiler.
:param compile_additional_sources: Disables passing additional
source files to the compiler - used
as a hack to support FPC.
Defaults to False.
:param sandbox: Enables sandboxing (using compiler name
as a sandbox). Defaults to False.
:param sandbox_callback: Optional callback called immediately after
creating the sandbox, with the said sandbox
as its sole parameter.
"""
if sandbox is True:
sandbox = get_sandbox('compiler-' + environ['compiler'])
elif not sandbox:
sandbox = NullSandbox()
extra_compilation_args = \
_lang_option(environ, 'extra_compilation_args', lang)
ft.download(environ, 'source_file', 'a.' + extension)
cmdline = (compiler, ) + tuple(compiler_options) + \
tuple(extra_compilation_args) + ('a.' + extension, )
# this cmdline may be later extended
# using a copy of the environment in order to avoid polluting it with
# temoporary elements
tmp_environ = environ.copy()
additional_includes = _lang_option(environ, 'additional_includes', lang)
additional_sources = _lang_option(environ, 'additional_sources', lang)
for include in additional_includes:
tmp_environ['additional_include'] = include
ft.download(tmp_environ, 'additional_include',
os.path.basename(include))
for source in additional_sources:
tmp_environ['additional_source'] = source
ft.download(tmp_environ, 'additional_source',
os.path.basename(source))
if compile_additional_sources:
cmdline += (os.path.basename(source), )
extra_files = environ.get('extra_files', {})
for name, ft_path in extra_files.iteritems():
tmp_environ['extra_file'] = ft_path
ft.download(tmp_environ, 'extra_file', os.path.basename(name))
shell_environ = os.environ.copy()
if sandbox:
shell_environ['LD_LIBRARY_PATH'] = (lang == 'pas') \
and os.path.join(sandbox.path, 'lib') \
or os.path.join(sandbox.path, 'usr', 'lib')
shell_environ['PATH'] = (lang == 'pas') \
and os.path.join(sandbox.path, 'bin') \
or os.path.join(sandbox.path, 'usr', 'bin')
shell_environ['PATH'] += ':' + os.environ['PATH']
with sandbox:
if sandbox_callback:
sandbox_callback(sandbox)
retcode, output = execute(list(cmdline),
env=shell_environ,
time_limit=30,
mem_limit=256,
ignore_errors=True,
environ=tmp_environ,
environ_prefix='compilation_')
environ['compiler_output'] = output
if retcode:
environ['result_code'] = 'CE'
elif 'compilation_result_size_limit' in environ and \
os.path.getsize(output_file) > \
environ['compilation_result_size_limit']:
environ['result_code'] = 'CE'
environ['compiler_output'] = 'Compiled file size limit exceeded.'
else:
environ['result_code'] = 'OK'
ft.upload(environ, 'out_file', output_file)
return environ
def run(environ, executor, use_sandboxes=True):
"""
Common code for executors.
:param: environ Recipe to pass to `filetracker` and `sio.workers.executors`
For all supported options, see the global documentation for
`sio.workers.executors` and prefix them with ``exec_``.
:param: executor Executor instance used for executing commands.
:param: use_sandboxes Enables safe checking output correctness.
See `sio.executors.checkers`. True by default.
"""
if feedback.judge_prepare(environ).force_not_judge:
environ['time_used'] = 0
environ['result_string'] = 'not judged'
environ['result_code'] = 'NJ'
environ['mem_used'] = 0
environ['num_syscalls'] = 0
environ['stderr'] = ''
return environ
input_name = tempcwd('in')
file_executor = get_file_runner(executor, environ)
exe_filename = file_executor.preferred_filename()
ft.download(environ, 'exe_file', exe_filename, add_to_cache=True)
os.chmod(tempcwd(exe_filename), 0700)
ft.download(environ, 'in_file', input_name, add_to_cache=True)
zipdir = tempcwd('in_dir')
os.mkdir(zipdir)
try:
if is_zipfile(input_name):
try:
# If not a zip file, will pass it directly to exe
with ZipFile(tempcwd('in'), 'r') as f:
if len(f.namelist()) != 1:
raise Exception("Archive should have only one file.")
f.extract(f.namelist()[0], zipdir)
input_name = os.path.join(zipdir, f.namelist()[0])
# zipfile throws some undocumented exceptions
except Exception as e:
raise StandardError("Failed to open archive: " + unicode(e))
with file_executor as fe:
with open(input_name, 'rb') as inf:
with open(tempcwd('out'), 'wb') as outf:
feedback.judge_started(environ)
renv = fe(tempcwd(exe_filename), [],
stdin=inf,
stdout=outf,
ignore_errors=True,
environ=environ,
environ_prefix='exec_')
_populate_environ(renv, environ)
if renv['result_code'] == 'OK' and environ.get('check_output'):
environ = checker.run(environ, use_sandboxes=use_sandboxes)
for key in ('result_code', 'result_string'):
environ[key] = replace_invalid_UTF(environ[key])
feedback.judge_finished(environ)
if 'out_file' in environ:
ft.upload(environ,
'out_file',
tempcwd('out'),
to_remote_store=environ.get('upload_out', False))
finally:
rmtree(zipdir)
return environ
def run(environ, executor, use_sandboxes=True):
"""
Common code for executors.
:param: environ Recipe to pass to `filetracker` and `sio.workers.executors`
For all supported options, see the global documentation for
`sio.workers.executors` and prefix them with ``exec_``.
:param: executor Executor instance used for executing commands.
:param: use_sandboxes Enables safe checking output correctness.
See `sio.executors.checkers`. True by default.
"""
input_name = tempcwd('in')
file_executor = get_file_runner(executor, environ)
exe_filename = file_executor.preferred_filename()
ft.download(environ, 'exe_file', exe_filename, add_to_cache=True)
os.chmod(tempcwd(exe_filename), 0o700)
ft.download(environ, 'in_file', input_name, add_to_cache=True)
zipdir = tempcwd('in_dir')
os.mkdir(zipdir)
try:
if is_zipfile(input_name):
try:
# If not a zip file, will pass it directly to exe
with ZipFile(tempcwd('in'), 'r') as f:
if len(f.namelist()) != 1:
raise Exception("Archive should have only one file.")
f.extract(f.namelist()[0], zipdir)
input_name = os.path.join(zipdir, f.namelist()[0])
# zipfile throws some undocumented exceptions
except Exception as e:
raise Exception("Failed to open archive: " + six.text_type(e))
with file_executor as fe:
with open(input_name, 'rb') as inf:
# Open output file in append mode to allow appending
# only to the end of the output file. Otherwise,
# a contestant's program could modify the middle of the file.
with open(tempcwd('out'), 'ab') as outf:
renv = fe(tempcwd(exe_filename), [],
stdin=inf,
stdout=outf,
ignore_errors=True,
environ=environ,
environ_prefix='exec_')
_populate_environ(renv, environ)
if renv['result_code'] == 'OK' and environ.get('check_output'):
environ = checker.run(environ, use_sandboxes=use_sandboxes)
for key in ('result_code', 'result_string'):
environ[key] = replace_invalid_UTF(environ[key])
if 'out_file' in environ:
ft.upload(environ,
'out_file',
tempcwd('out'),
to_remote_store=environ.get('upload_out', False))
finally:
rmtree(zipdir)
return environ
