def _make_cmdline(self, executor):
cmdline = [
noquote(tempcwd(self.source_file))
] #'/usr/bin/python3compile.py', tempcwd(self.source_file)]
os.chmod(tempcwd(os.path.dirname(self.source_file)), 0777)
os.chmod(tempcwd(self.source_file), 0777)
return cmdline
def _make_cmdline(self, executor):
cmdline = [self.compiler, tempcwd(self.source_file),
'-o', tempcwd(self.output_file)] + \
self.options + list(self.extra_compilation_args)
cmdline.extend(tempcwd(os.path.basename(source))
for source in self.additional_sources)
return cmdline
def _make_cmdline(self, executor):
cmdline = [self.compiler, tempcwd(self.source_file),
'-o', tempcwd(self.output_file)] + \
self.options + list(self.extra_compilation_args)
cmdline.extend(
tempcwd(os.path.basename(source))
for source in self.additional_sources)
return cmdline
def test_zip():
with TemporaryCwd():
upload_files()
compile_and_run(
"/echo.c", {
'in_file': '/input.zip',
'out_file': '/output',
'exec_mem_limit': 102400
}, DetailedUnprotectedExecutor())
ft.download({'in_file': '/input'}, 'in_file', 'out.expected')
ft.download({'out_file': '/output'}, 'out_file', 'out.real')
ok_(filecmp.cmp(tempcwd('out.expected'), tempcwd('out.real')))
def test_zip():
with TemporaryCwd():
upload_files()
compile_and_run("/echo.c", {
'in_file': '/input.zip',
'out_file': '/output',
'exec_mem_limit': 102400
}, DetailedUnprotectedExecutor())
ft.download({'in_file': '/input'}, 'in_file', 'out.expected')
ft.download({'out_file': '/output'}, 'out_file', 'out.real')
ok_(filecmp.cmp(tempcwd('out.expected'),
tempcwd('out.real')))
def test_execute():
with TemporaryCwd():
rc, out = execute(['echo', '2'])
eq_(rc, 0)
eq_(out, '2\n')
rc, out = execute(['exit', '1'], ignore_errors=True)
eq_(rc, 1)
assert_raises(ExecError, execute, ['exit', '1'])
rc, out = execute(['mkdir', tempcwd('spam')])
eq_(rc, 0)
rc, out = execute(['ls', tempcwd()])
in_('spam', out)
def test_execute():
with TemporaryCwd():
rc, out = execute(['echo', '2'])
eq_(rc, 0)
eq_(out, '2\n')
rc, out = execute(['exit', '1'], ignore_errors=True)
eq_(rc, 1)
assert_raises(ExecError, execute, ['exit', '1'])
rc, out = execute(['mkdir', tempcwd('spam')])
eq_(rc, 0)
rc, out = execute(['ls', tempcwd()])
in_('spam', out)
def test_output_compilation_and_running(source):
with TemporaryCwd():
upload_files()
result_env = run(
{
'source_file': source,
'compiler': 'output-only',
}
)
eq_(result_env['result_code'], 'OK')
eq_(result_env['exec_info'], {'mode': 'output-only'})
ft.download(result_env, 'out_file', tempcwd('out.txt'))
ft.download({'source_file': source}, 'source_file', tempcwd('source.txt'))
with open(tempcwd('out.txt'), 'r') as outfile:
with open(tempcwd('source.txt'), 'r') as sourcefile:
eq_(outfile.read(), sourcefile.read())
post_run_env = run_from_executors(
{
'exec_info': result_env['exec_info'],
'exe_file': result_env['out_file'],
'check_output': True,
'hint_file': source,
},
executor=None,
)
eq_(post_run_env['result_code'], 'OK')
ft.download(post_run_env, 'out_file', tempcwd('out.txt'))
ft.download({'source_file': source}, 'source_file', tempcwd('source.txt'))
with open(tempcwd('out.txt'), 'r') as outfile:
with open(tempcwd('source.txt'), 'r') as sourcefile:
eq_(outfile.read(), sourcefile.read())
def compile_and_execute(source, executor, **exec_args):
cenv = compile(source,
use_sandboxes=isinstance(executor, _SIOSupervisedExecutor))
frunner = get_file_runner(executor, cenv)
ft.download({'exe_file': cenv['out_file']}, 'exe_file',
frunner.preferred_filename())
os.chmod(tempcwd('exe'), 0700)
ft.download({'in_file': '/input'}, 'in_file', 'in')
with frunner:
with open(tempcwd('in'), 'rb') as inf:
renv = frunner(tempcwd('exe'), [], stdin=inf, **exec_args)
return renv
def _extract_all(archive_path):
target_path = tempcwd()
with ZipFile(tempcwd(archive_path), 'r') as zipf:
for name in zipf.namelist():
filename = name.rstrip('/')
extract_path = os.path.join(target_path, filename)
extract_path = os.path.normpath(os.path.realpath(extract_path))
if os.path.exists(extract_path):
logger.warning("Cannot extract %s, file already exists.",
extract_path)
elif not extract_path.startswith(target_path):
logger.warning("Cannot extract %s, target path outside "
"working directory.", extract_path)
else:
zipf.extract(name, target_path)
def _postprocess(self, renv):
self.environ['compiler_output'] = replace_invalid_UTF(renv['stdout'])
if renv['return_code']:
self.environ['result_code'] = 'CE'
elif 'compilation_result_size_limit' in self.environ and \
os.path.getsize(tempcwd(self.output_file)) > \
self.environ['compilation_result_size_limit']:
self.environ['result_code'] = 'CE'
self.environ['compiler_output'] = \
'Compiled file size limit exceeded.'
else:
self.environ['result_code'] = 'OK'
self.environ['exec_info'] = {'mode': 'executable'}
ft.upload(self.environ, 'out_file', tempcwd(self.output_file))
return self.environ
def compile_and_execute(source, executor, **exec_args):
cenv = compile(source,
use_sandboxes=isinstance(executor, _SIOSupervisedExecutor))
frunner = get_file_runner(executor, cenv)
ft.download({'exe_file': cenv['out_file']}, 'exe_file',
frunner.preferred_filename())
os.chmod(tempcwd('exe'), 0700)
ft.download({'in_file': '/input'}, 'in_file', 'in')
with frunner:
with open(tempcwd('in'), 'rb') as inf:
renv = frunner(tempcwd('exe'), [], stdin=inf, **exec_args)
return renv
def download(environ, key, dest=None, skip_if_exists=False, **kwargs):
"""Downloads the file from ``environ[key]`` and saves it to ``dest``.
``dest``
A filename, directory name or ``None``. In the two latter cases,
the file is named the same as in ``environ[key]``.
``skip_if_exists``
If ``True`` and ``dest`` points to an existing file (not a directory
or ``None``), then the file is not downloaded.
``**kwargs``
Passed directly to :meth:`filetracker.Client.get_file`.
The value under ``environ['use_filetracker']`` affects downloading
in the followins way:
* if ``True``, nothing special happens
* if ``False``, the file is not downloaded from filetracker, but the
passed path is assumed to be a regular filesystem path
* if ``'auto'``, the file is assumed to be a local filename only if
it is a relative path (this is usually the case when developers play).
Returns the path to the saved file.
"""
if dest and skip_if_exists and os.path.exists(util.tempcwd(dest)):
return dest
source = environ[key]
if dest is None:
dest = os.path.split(source)[1]
elif dest.endswith(os.sep):
dest = os.path.join(dest, os.path.split(source)[1])
dest = util.tempcwd(dest)
if not _use_filetracker(source, environ):
source = os.path.join(_original_cwd, source)
if not os.path.exists(dest) or not os.path.samefile(source, dest):
shutil.copy(source, dest)
else:
kwargs.setdefault('add_to_cache', False)
logger.debug("Downloading %s", source)
perf_timer = util.PerfTimer()
instance().get_file(source, dest, **kwargs)
logger.debug(" completed in %.2fs", perf_timer.elapsed)
return dest
def download(environ, key, dest=None, skip_if_exists=False, **kwargs):
"""Downloads the file from ``environ[key]`` and saves it to ``dest``.
``dest``
A filename, directory name or ``None``. In the two latter cases,
the file is named the same as in ``environ[key]``.
``skip_if_exists``
If ``True`` and ``dest`` points to an existing file (not a directory
or ``None``), then the file is not downloaded.
``**kwargs``
Passed directly to :meth:`filetracker.client.Client.get_file`.
The value under ``environ['use_filetracker']`` affects downloading
in the followins way:
* if ``True``, nothing special happens
* if ``False``, the file is not downloaded from filetracker, but the
passed path is assumed to be a regular filesystem path
* if ``'auto'``, the file is assumed to be a local filename only if
it is a relative path (this is usually the case when developers play).
Returns the path to the saved file.
"""
if dest and skip_if_exists and os.path.exists(util.tempcwd(dest)):
return dest
source = environ[key]
if dest is None:
dest = os.path.split(source)[1]
elif dest.endswith(os.sep):
dest = os.path.join(dest, os.path.split(source)[1])
dest = util.tempcwd(dest)
if not _use_filetracker(source, environ):
source = os.path.join(_original_cwd, source)
if not os.path.exists(dest) or not os.path.samefile(source, dest):
shutil.copy(source, dest)
else:
kwargs.setdefault('add_to_cache', False)
logger.debug("Downloading %s", source)
perf_timer = util.PerfTimer()
instance().get_file(source, dest, **kwargs)
logger.debug(" completed in %.2fs", perf_timer.elapsed)
return dest
def _extract_all(archive_path):
target_path = tempcwd()
with ZipFile(tempcwd(archive_path), 'r') as zipf:
for name in zipf.namelist():
filename = name.rstrip('/')
extract_path = os.path.join(target_path, filename)
extract_path = os.path.normpath(os.path.realpath(extract_path))
if os.path.exists(extract_path):
logger.warning("Cannot extract %s, file already exists.",
extract_path)
elif not extract_path.startswith(target_path):
logger.warning(
"Cannot extract %s, target path outside "
"working directory.", extract_path)
else:
zipf.extract(name, target_path)
def run(environ, use_sandboxes=True):
ft.download(environ, 'out_file', 'out', skip_if_exists=True)
ft.download(environ, 'hint_file', 'hint', add_to_cache=True)
try:
if environ.get('chk_file'):
ft.download(environ, 'in_file', 'in', skip_if_exists=True,
add_to_cache=True)
ft.download(environ, 'chk_file', 'chk', add_to_cache=True)
os.chmod(tempcwd('chk'), 0o700)
output = _run_checker(environ, use_sandboxes)
elif use_sandboxes:
output = _run_compare(environ)
else:
output = _run_diff(environ)
except (CheckerError, ExecError) as e:
logger.error('Checker failed! %s', e)
logger.error('Environ dump: %s', environ)
raise SystemError(e)
while len(output) < 3:
output.append('')
if output[0] == b'OK':
environ['result_code'] = 'OK'
if output[1]:
environ['result_string'] = _limit_length(output[1])
environ['result_percentage'] = float(output[2] or 100)
else:
environ['result_code'] = 'WA'
environ['result_string'] = _limit_length(output[1])
environ['result_percentage'] = 0
return environ
def _run_in_executor(self, executor):
javac = ['javac', '-J-Xss32M'] + list(self.extra_compilation_args) \
+ [tempcwd(self.source_file)]
javac.extend(tempcwd(os.path.basename(source))
for source in self.additional_sources)
renv = self._execute(executor, javac)
if renv['return_code']:
return renv
classes = [os.path.basename(x)
for x in glob.glob(tempcwd() + '/*.class')]
jar = ['jar', 'cf', self.output_file] + classes
renv2 = self._execute(executor, jar)
renv2['stdout'] = renv['stdout'] + renv2['stdout']
return renv2
def _run_ingen(environ, use_sandboxes=False):
command = [tempcwd("ingen")]
if use_sandboxes:
executor = PRootExecutor("null-sandbox")
else:
executor = UnprotectedExecutor()
return _run_in_executor(environ, command, executor, ignore_errors=True)
def _postprocess(self, renv):
self.environ['compiler_output'] = replace_invalid_UTF(renv['stdout'])
if renv['return_code']:
self.environ['result_code'] = 'CE'
elif 'compilation_result_size_limit' in self.environ and \
os.path.getsize(tempcwd(self.output_file)) > \
self.environ['compilation_result_size_limit']:
self.environ['result_code'] = 'CE'
self.environ['compiler_output'] = \
'Compiled file size limit exceeded.'
else:
self.environ['result_code'] = 'OK'
self.environ['exec_info'] = {'mode': 'executable'}
ft.upload(self.environ, 'out_file', tempcwd(self.output_file))
return self.environ
def _run_ingen(environ, use_sandboxes=False):
command = [tempcwd('ingen')]
if use_sandboxes:
executor = PRootExecutor('null-sandbox')
else:
executor = UnprotectedExecutor()
return _run_in_executor(environ, command, executor, ignore_errors=True)
def _run_inwer(environ, use_sandboxes=False):
command = [tempcwd('inwer')]
if use_sandboxes:
executor = SupervisedExecutor()
else:
executor = DetailedUnprotectedExecutor()
return _run_in_executor(environ, command, executor, ignore_errors=True)
def _run_inwer(environ, use_sandboxes=False):
command = [tempcwd('inwer')]
if use_sandboxes:
executor = SupervisedExecutor()
else:
executor = DetailedUnprotectedExecutor()
return _run_in_executor(environ, command, executor, ignore_errors=True)
def _test_transparent_exec(source, executor, callback, kwargs):
with TemporaryCwd():
upload_files()
ft.download({'path': '/somefile'}, 'path', tempcwd('somefile'))
result_env = compile_and_execute(source, executor, **kwargs)
print_env(result_env)
if callback:
callback(result_env)
def _test_transparent_exec(source, executor, callback, kwargs):
with TemporaryCwd():
upload_files()
ft.download({'path': '/somefile'}, 'path', tempcwd('somefile'))
result_env = compile_and_execute(source, executor, **kwargs)
print_env(result_env)
if callback:
callback(result_env)
def run(environ):
"""Runs a verifying program and returns its output.
Used ``environ`` keys:
``exe_file``: the filetracker path to the program
``in_file``: the file redirected to the program's stdin
``use_sandboxes``: if this key equals ``True``, the program is executed
in the SupervisedExecutor, otherwise the UnsafeExecutor
is used
``inwer_time_limit``: time limit in ms
(optional, the default is 30 s)
``inwer_mem_limit``: memory limit in KiB
(optional, the default is 256 MiB)
``inwer_output_limit``: output limit in B
(optional, the default is 10 KiB)
Returns a new environ, whose ``stdout`` key contains the program's
output.
The verifying program is expected to return 0, its first line of output
should begin with "OK". If this does not happen, an appropriate message
is logged.
Program's output is returned under the ``stdout`` key. If the output has
more than ``inwer_output_limit`` bytes and ``use_sandboxes`` is
set to ``True``, the execution of the program fails with ``OLE`` result
code.
"""
use_sandboxes = environ.get('use_sandboxes', False)
ft.download(environ,
'exe_file',
'inwer',
skip_if_exists=True,
add_to_cache=True)
ft.download(environ,
'in_file',
'in',
skip_if_exists=True,
add_to_cache=True)
os.chmod(tempcwd('inwer'), 0500)
renv = _run_inwer(environ, use_sandboxes)
if renv['result_code'] != "OK":
logger.error("Inwer failed!\nEnviron dump: %s\nExecution environ: %s",
environ, renv)
elif not renv['stdout'][0].startswith("OK"):
logger.error(
"Bad inwer output!\nEnviron dump: %s\n"
"Execution environ: %s", environ, renv)
return renv
def _run_in_executor(self, executor):
# Generate FPC configuration
with open(os.path.join(executor.path, 'fpc.cfg.in')) as f:
fpc_cfg = f.read()
fpc_cfg = fpc_cfg.replace('__DIR__', executor.rpath.rstrip(os.sep))
with open(tempcwd('fpc.cfg'), 'w') as f:
f.write(fpc_cfg)
return super(FPCCompiler, self)._run_in_executor(executor)
def _run_in_executor(self, executor):
javac = (['javac', '-J-Xss32M'] + list(self.extra_compilation_args) +
[tempcwd(self.source_file)])
javac.extend(
tempcwd(os.path.basename(source))
for source in self.additional_sources)
renv = self._execute(executor, javac)
if renv['return_code']:
return renv
classes = [
os.path.basename(x) for x in glob.glob(tempcwd() + '/*.class')
]
jar = ['jar', 'cf', self.output_file] + classes
renv2 = self._execute(executor, jar)
renv2['stdout'] = renv['stdout'] + renv2['stdout']
return renv2
def _run_in_executor(self, executor):
# Generate FPC configuration
with open(os.path.join(executor.path, 'fpc.cfg.in')) as f:
fpc_cfg = f.read()
fpc_cfg = fpc_cfg.replace('__DIR__', executor.rpath.rstrip(os.sep))
with open(tempcwd('fpc.cfg'), 'w') as f:
f.write(fpc_cfg)
return super(FPCCompiler, self)._run_in_executor(executor)
def _run_in_executor(environ, command, executor, **kwargs):
with executor:
with open(tempcwd('in'), 'rb') as inf:
return executor(command, stdin=inf,
capture_output=True, split_lines=True, forward_stderr=True,
mem_limit=DEFAULT_INWER_MEM_LIMIT,
time_limit=DEFAULT_INWER_TIME_LIMIT,
output_limit=DEFAULT_INWER_OUTPUT_LIMIT,
environ=environ, environ_prefix='inwer_', **kwargs)
def run(environ, executor, use_sandboxes=True):
"""
Common code for executors.
:param: environ Recipe to pass to `filetracker` and `sio.workers.executors`
For all supported options, see the global documentation for
`sio.workers.executors` and prefix them with ``exec_``.
:param: executor Executor instance used for executing commands.
:param: use_sandboxes Enables safe checking output correctness.
See `sio.executors.checkers`. True by default.
"""
input_name = tempcwd('in')
file_executor = get_file_runner(executor, environ)
exe_filename = file_executor.preferred_filename()
ft.download(environ, 'exe_file', exe_filename, add_to_cache=True)
os.chmod(tempcwd(exe_filename), 0700)
ft.download(environ, 'in_file', input_name, add_to_cache=True)
zipdir = tempcwd('in_dir')
os.mkdir(zipdir)
try:
if is_zipfile(input_name):
try:
# If not a zip file, will pass it directly to exe
with ZipFile(tempcwd('in'), 'r') as f:
if len(f.namelist()) != 1:
raise Exception("Archive should have only one file.")
f.extract(f.namelist()[0], zipdir)
input_name = os.path.join(zipdir, f.namelist()[0])
# zipfile throws some undocumented exceptions
except Exception as e:
raise StandardError("Failed to open archive: " + unicode(e))
with file_executor as fe:
with open(input_name, 'rb') as inf:
with open(tempcwd('out'), 'wb') as outf:
renv = fe(tempcwd(exe_filename), [],
stdin=inf, stdout=outf, ignore_errors=True,
environ=environ, environ_prefix='exec_')
_populate_environ(renv, environ)
if renv['result_code'] == 'OK' and environ.get('check_output'):
environ = checker.run(environ, use_sandboxes=use_sandboxes)
for key in ('result_code', 'result_string'):
environ[key] = replace_invalid_UTF(environ[key])
if 'out_file' in environ:
ft.upload(environ, 'out_file', tempcwd('out'),
to_remote_store=environ.get('upload_out', False))
finally:
rmtree(zipdir)
return environ
def _proot_options(self):
self._verbosity(-1)
self._chroot(self.chroot.path)
sh_target = path.join(os.sep, 'bin', 'sh')
if not path.exists(path_join_abs(self.chroot.path, sh_target)):
self._bind(path_join_abs(self.proot.path, sh_target), sh_target)
else:
# If /bin/sh exists, then bind unpatched version to it
sh_patched = elf_loader_patch._get_unpatched_name(
path.realpath(path_join_abs(self.chroot.path, sh_target)))
if path.exists(sh_patched):
self._bind(sh_patched, sh_target, force=True)
self._bind(os.path.join(self.proot.path, 'lib'), 'lib')
self._bind(tempcwd(), 'tmp', force=True)
# Make absolute `outside paths' visible in sandbox
self._bind(self.chroot.path, force=True)
self._bind(tempcwd(), force=True)
def _proot_options(self):
self._verbosity(-1)
self._chroot(self.chroot.path)
sh_target = path.join(os.sep, 'bin', 'sh')
if not path.exists(path_join_abs(self.chroot.path, sh_target)):
self._bind(path_join_abs(self.proot.path, sh_target), sh_target)
else:
# If /bin/sh exists, then bind unpatched version to it
sh_patched = elf_loader_patch._get_unpatched_name(
path.realpath(path_join_abs(self.chroot.path, sh_target)))
if path.exists(sh_patched):
self._bind(sh_patched, sh_target, force=True)
self._bind(os.path.join(self.proot.path, 'lib'), 'lib')
self._bind(tempcwd(), 'tmp', force=True)
# Make absolute `outside paths' visible in sandbox
self._bind(self.chroot.path, force=True)
self._bind(tempcwd(), force=True)
def _run_in_executor(environ, command, executor, **kwargs):
with executor:
renv = executor(command,
capture_output=True, split_lines=True, forward_stderr=True,
mem_limit=DEFAULT_INGEN_MEM_LIMIT,
time_limit=DEFAULT_INGEN_TIME_LIMIT,
output_limit=DEFAULT_INGEN_OUTPUT_LIMIT,
environ=environ, environ_prefix='ingen_', **kwargs)
if renv['return_code'] == 0:
_collect_and_upload(renv, tempcwd(),
environ['collected_files_path'], environ['re_string'])
return renv
def test_uploading_out():
with TemporaryCwd():
upload_files()
renv = compile_and_run('/add_print.c', {
'in_file': '/input',
'out_file': '/output',
'upload_out': True,
}, DetailedUnprotectedExecutor())
print_env(renv)
ft.download({'path': '/output'}, 'path', 'd_out')
in_('84', open(tempcwd('d_out')).read())
def test_uploading_out():
with TemporaryCwd():
upload_files()
renv = compile_and_run('/add_print.c', {
'in_file': '/input',
'out_file': '/output',
'upload_out': True,
}, DetailedUnprotectedExecutor())
print_env(renv)
ft.download({'path': '/output'}, 'path', 'd_out')
in_('84', open(tempcwd('d_out')).read())
def run(environ):
"""Runs a verifying program and returns its output.
Used ``environ`` keys:
``exe_file``: the filetracker path to the program
``in_file``: the file redirected to the program's stdin
``use_sandboxes``: if this key equals ``True``, the program is executed
in the SupervisedExecutor, otherwise the UnsafeExecutor
is used
``inwer_time_limit``: time limit in ms
(optional, the default is 30 s)
``inwer_mem_limit``: memory limit in KiB
(optional, the default is 256 MiB)
``inwer_output_limit``: output limit in B
(optional, the default is 10 KiB)
Returns a new environ, whose ``stdout`` key contains the program's
output.
The verifying program is expected to return 0, its first line of output
should begin with "OK". If this does not happen, an appropriate message
is logged.
Program's output is returned under the ``stdout`` key. If the output has
more than ``inwer_output_limit`` bytes and ``use_sandboxes`` is
set to ``True``, the execution of the program fails with ``OLE`` result
code.
"""
use_sandboxes = environ.get('use_sandboxes', False)
ft.download(environ, 'exe_file', 'inwer', skip_if_exists=True,
add_to_cache=True)
ft.download(environ, 'in_file', 'in', skip_if_exists=True,
add_to_cache=True)
os.chmod(tempcwd('inwer'), 0o500)
renv = _run_inwer(environ, use_sandboxes)
if renv['result_code'] != "OK":
logger.error("Inwer failed!\nEnviron dump: %s\nExecution environ: %s",
environ, renv)
elif not renv['stdout'][0].startswith(b"OK"):
logger.error("Bad inwer output!\nEnviron dump: %s\n"
"Execution environ: %s", environ, renv)
return renv
def _run_in_executor(environ, command, executor, **kwargs):
with executor:
with open(tempcwd('in'), 'rb') as inf:
return executor(command,
stdin=inf,
capture_output=True,
split_lines=True,
forward_stderr=True,
mem_limit=DEFAULT_INWER_MEM_LIMIT,
time_limit=DEFAULT_INWER_TIME_LIMIT,
output_limit=DEFAULT_INWER_OUTPUT_LIMIT,
environ=environ,
environ_prefix='inwer_',
**kwargs)
def inner(env):
eq_(env['return_code'], 0)
eq_(env['stdout'], expected_output)
collected = env['collected_files']
eq_(len(expected_files), len(collected))
for filename, path in collected.iteritems():
in_(filename, expected_files)
unversioned_path = '/%s/%s' % (upload_dir, filename)
upload_re_str = '%s@\d+' % (unversioned_path)
upload_re = re.compile(upload_re_str)
ok_(upload_re.match(path), 'Unexpected filetracker path')
ft.download({'in': unversioned_path}, 'in', filename)
eq_(expected_files[filename], open(tempcwd(filename)).read())
def inner(env):
eq_(env['return_code'], 0)
eq_(env['stdout'], expected_output)
collected = env['collected_files']
eq_(len(expected_files), len(collected))
for filename, path in collected.iteritems():
in_(filename, expected_files)
unversioned_path = '/%s/%s' % (upload_dir, filename)
upload_re_str = '%s@\d+' % (unversioned_path)
upload_re = re.compile(upload_re_str)
ok_(upload_re.match(path), 'Unexpected filetracker path')
ft.download({'in': unversioned_path}, 'in', filename)
eq_(expected_files[filename], open(tempcwd(filename)).read())
def run(environ):
"""Runs a program, collects the files produced by it and uploads them
to filetracker.
Used ``environ`` keys:
``exe_file``: the filetracker path to the program
``re_string``: a regular expression string used to identify the files
which should be uploaded
``collected_files_path``: a directory into which the collected files
should be uploaded in filetracker
``use_sandboxes``: if this key equals ``True``, the program is executed
in the PRootExecutor, otherwise the UnsafeExecutor is
used
``ingen_time_limit``: time limit in ms
(optional, the default is 10 mins)
``ingen_mem_limit``: memory limit in KiB
(optional, the default is 256 MiB)
``ingen_output_limit``: output limit in B
(optional, the default is 10 KiB)
On success returns a new environ with a dictionary mapping collected
files' names to their filetracker paths under ``collected_files``.
Program's output is returned under the ``stdout`` key. The output is
trimmed to the first ``ingen_output_limit`` bytes.
"""
use_sandboxes = environ.get('use_sandboxes', False)
ft.download(environ,
'exe_file',
'ingen',
skip_if_exists=True,
add_to_cache=True)
os.chmod(tempcwd('ingen'), 0500)
renv = _run_ingen(environ, use_sandboxes)
if renv['return_code'] != 0:
logger.error("Ingen failed!\nEnviron dump: %s\nExecution environ: %s",
environ, renv)
return renv
def run(environ):
"""Runs a program, collects the files produced by it and uploads them
to filetracker.
Used ``environ`` keys:
``exe_file``: the filetracker path to the program
``re_string``: a regular expression string used to identify the files
which should be uploaded
``collected_files_path``: a directory into which the collected files
should be uploaded in filetracker
``use_sandboxes``: if this key equals ``True``, the program is executed
in the PRootExecutor, otherwise the UnsafeExecutor is
used
``ingen_time_limit``: time limit in ms
(optional, the default is 10 mins)
``ingen_mem_limit``: memory limit in KiB
(optional, the default is 256 MiB)
``ingen_output_limit``: output limit in B
(optional, the default is 10 KiB)
On success returns a new environ with a dictionary mapping collected
files' names to their filetracker paths under ``collected_files``.
Program's output is returned under the ``stdout`` key. The output is
trimmed to the first ``ingen_output_limit`` bytes.
"""
use_sandboxes = environ.get('use_sandboxes', False)
ft.download(environ, 'exe_file', 'ingen', skip_if_exists=True,
add_to_cache=True)
os.chmod(tempcwd('ingen'), 0o500)
renv = _run_ingen(environ, use_sandboxes)
if renv['return_code'] != 0:
logger.error("Ingen failed!\nEnviron dump: %s\nExecution environ: %s",
environ, renv)
return renv
def run(environ, use_sandboxes=True):
ft.download(environ, 'out_file', 'out', skip_if_exists=True)
ft.download(environ, 'hint_file', 'hint', add_to_cache=True)
try:
if environ.get('chk_file'):
ft.download(environ,
'in_file',
'in',
skip_if_exists=True,
add_to_cache=True)
ft.download(environ, 'chk_file', 'chk', add_to_cache=True)
os.chmod(tempcwd('chk'), 0o700)
output = _run_checker(environ, use_sandboxes)
elif use_sandboxes:
output = _run_compare(environ)
else:
output = _run_diff(environ)
except (CheckerError, ExecError) as e:
logger.error('Checker failed! %s', e)
logger.error('Environ dump: %s', environ)
raise SystemError(e)
while len(output) < 3:
output.append('')
if six.ensure_binary(output[0]) == b'OK':
environ['result_code'] = 'OK'
if output[1]:
environ['result_string'] = _limit_length(output[1])
environ['result_percentage'] = float(output[2] or 100)
else:
environ['result_code'] = 'WA'
environ['result_string'] = _limit_length(output[1])
environ['result_percentage'] = 0
return environ
def run(environ, executor, use_sandboxes=True):
"""
Common code for executors.
:param: environ Recipe to pass to `filetracker` and `sio.workers.executors`
For all supported options, see the global documentation for
`sio.workers.executors` and prefix them with ``exec_``.
:param: executor Executor instance used for executing commands.
:param: use_sandboxes Enables safe checking output correctness.
See `sio.executors.checkers`. True by default.
"""
input_name = tempcwd('in')
file_executor = get_file_runner(executor, environ)
exe_filename = file_executor.preferred_filename()
ft.download(environ, 'exe_file', exe_filename, add_to_cache=True)
os.chmod(tempcwd(exe_filename), 0o700)
ft.download(environ, 'in_file', input_name, add_to_cache=True)
zipdir = tempcwd('in_dir')
os.mkdir(zipdir)
try:
if is_zipfile(input_name):
try:
# If not a zip file, will pass it directly to exe
with ZipFile(tempcwd('in'), 'r') as f:
if len(f.namelist()) != 1:
raise Exception("Archive should have only one file.")
f.extract(f.namelist()[0], zipdir)
input_name = os.path.join(zipdir, f.namelist()[0])
# zipfile throws some undocumented exceptions
except Exception as e:
raise Exception("Failed to open archive: " + six.text_type(e))
with file_executor as fe:
with open(input_name, 'rb') as inf:
# Open output file in append mode to allow appending
# only to the end of the output file. Otherwise,
# a contestant's program could modify the middle of the file.
with open(tempcwd('out'), 'ab') as outf:
renv = fe(tempcwd(exe_filename), [],
stdin=inf,
stdout=outf,
ignore_errors=True,
environ=environ,
environ_prefix='exec_')
_populate_environ(renv, environ)
if renv['result_code'] == 'OK' and environ.get('check_output'):
environ = checker.run(environ, use_sandboxes=use_sandboxes)
for key in ('result_code', 'result_string'):
environ[key] = replace_invalid_UTF(environ[key])
if 'out_file' in environ:
ft.upload(environ,
'out_file',
tempcwd('out'),
to_remote_store=environ.get('upload_out', False))
finally:
rmtree(zipdir)
return environ
def _make_cmdline(self, executor):
# Addinational sources are automatically included
return ['fpc', tempcwd('a.pas')] + self.options + \
list(self.extra_compilation_args)
def _execute(self, command, **kwargs):
env = kwargs.get('env')
env.update({
'MEM_LIMIT': kwargs['mem_limit'] or 64 * 2**10,
'TIME_LIMIT': kwargs['time_limit'] or 30000,
'OUT_LIMIT': kwargs['output_limit'] or 50 * 2**20,
})
if kwargs['real_time_limit']:
env['HARD_LIMIT'] = 1 + ceil_ms2s(kwargs['real_time_limit'])
elif kwargs['time_limit'] and kwargs['real_time_limit'] is None:
env['HARD_LIMIT'] = 1 + ceil_ms2s(64 * kwargs['time_limit'])
if 'HARD_LIMIT' in env:
# Limiting outside supervisor
kwargs['real_time_limit'] = 2 * s2ms(env['HARD_LIMIT'])
ignore_errors = kwargs.pop('ignore_errors')
extra_ignore_errors = kwargs.pop('extra_ignore_errors')
renv = {}
try:
result_file = tempfile.NamedTemporaryFile(dir=tempcwd())
kwargs['ignore_errors'] = True
renv = execute_command(command + [noquote('3>'), result_file.name],
**kwargs)
if 'real_time_killed' in renv:
raise ExecError('Supervisor exceeded realtime limit')
elif renv['return_code'] and renv[
'return_code'] not in extra_ignore_errors:
raise ExecError('Supervisor returned code %s' %
renv['return_code'])
result_file.seek(0)
status_line = result_file.readline().strip().split()[1:]
renv['result_string'] = result_file.readline().strip()
result_file.close()
for num, key in enumerate(('result_code', 'time_used', None,
'mem_used', 'num_syscalls')):
if key:
renv[key] = int(status_line[num])
result_code = self._supervisor_result_to_code(renv['result_code'])
except Exception as e:
logger.error('SupervisedExecutor error: %s',
traceback.format_exc())
logger.error(
'SupervisedExecutor error dirlist: %s: %s',
tempcwd(),
str(os.listdir(tempcwd())),
)
result_code = 'SE'
for i in ('time_used', 'mem_used', 'num_syscalls'):
renv.setdefault(i, 0)
renv['result_string'] = str(e)
renv['result_code'] = result_code
if (result_code != 'OK' and not ignore_errors
and not (result_code != 'RV'
and renv['return_code'] in extra_ignore_errors)):
raise ExecError('Failed to execute command: %s. Reason: %s' %
(command, renv['result_string']))
return renv
def _execute(self, command, **kwargs):
options = []
options += ['-b', os.path.join(self.rpath, 'boxes/minimal') + ':/:ro']
options += [
'--memory-limit',
str(kwargs['mem_limit'] or self.DEFAULT_MEMORY_LIMIT) + 'K',
]
options += [
'--instruction-count-limit',
str((kwargs['time_limit'] or self.DEFAULT_TIME_LIMIT) *
self.INSTRUCTIONS_PER_VIRTUAL_SECOND / 1000),
]
options += [
'--rtimelimit',
str((kwargs['time_limit'] or self.DEFAULT_TIME_LIMIT) *
self.REAL_TIME_LIMIT_MULTIPLIER + self.REAL_TIME_LIMIT_ADDEND)
+ 'ms',
]
options += [
'--output-limit',
str(kwargs['output_limit'] or self.DEFAULT_OUTPUT_LIMIT) + 'K',
]
command = [os.path.join(self.rpath, 'sio2jail')
] + options + ['--'] + command
renv = {}
try:
result_file = tempfile.NamedTemporaryFile(dir=tempcwd())
kwargs['ignore_errors'] = True
renv = execute_command(command + [noquote('2>'), result_file.name],
**kwargs)
if renv['return_code'] != 0:
raise ExecError('Sio2Jail returned code %s, stderr: %s' %
(renv['return_code'], result_file.read(10240)))
result_file.seek(0)
status_line = result_file.readline().strip().split()[1:]
renv['result_string'] = result_file.readline().strip()
result_file.close()
for num, key in enumerate(
('result_code', 'time_used', None, 'mem_used', None)):
if key:
renv[key] = int(status_line[num])
if renv['result_string'] == 'ok':
renv['result_code'] = 'OK'
elif renv['result_string'] == 'time limit exceeded':
renv['result_code'] = 'TLE'
elif renv['result_string'] == 'real time limit exceeded':
renv['result_code'] = 'TLE'
elif renv['result_string'] == 'memory limit exceeded':
renv['result_code'] = 'MLE'
elif renv['result_string'].startswith(
'intercepted forbidden syscall'):
renv['result_code'] = 'RV'
elif renv['result_string'].startswith(
'process exited due to signal'):
renv['result_code'] = 'RE'
else:
raise ExecError('Unrecognized Sio2Jail result string: %s' %
renv['result_string'])
except (EnvironmentError, EOFError, RuntimeError) as e:
logger.error('Sio2JailExecutor error: %s', traceback.format_exc())
logger.error(
'Sio2JailExecutor error dirlist: %s: %s',
tempcwd(),
str(os.listdir(tempcwd())),
)
renv['result_code'] = 'SE'
for i in ('time_used', 'mem_used'):
renv.setdefault(i, 0)
renv['result_string'] = str(e)
if not kwargs.get('ignore_errors', False):
raise ExecError('Failed to execute command: %s. Reason: %s' %
(command, renv['result_string']))
return renv
def execute_command(command,
env=None,
split_lines=False,
stdin=None,
stdout=None,
stderr=None,
forward_stderr=False,
capture_output=False,
output_limit=None,
real_time_limit=None,
ignore_errors=False,
extra_ignore_errors=(),
**kwargs):
"""Utility function to run arbitrary command.
``stdin``
Could be either file opened with ``open(fname, 'r')``
or None (then it is inherited from current process).
``stdout``, ``stderr``
Could be files opened with ``open(fname, 'w')``, sys.std*
or None - then it's suppressed.
``forward_stderr``
Forwards stderr to stdout.
``capture_output``
Returns program output in renv key ``stdout``.
``output_limit``
Limits returned output when ``capture_output=True`` (in bytes).
Returns renv: dictionary containing:
``real_time_used``
Wall clock time it took to execute the command (in ms).
``return_code``
Status code that program returned.
``real_time_killed``
Only when process was killed due to exceeding real time limit.
``stdout``
Only when ``capture_output=True``: output of the command
"""
# Using temporary file is way faster than using subproces.PIPE
# and it prevents deadlocks.
command = shellquote(command)
logger.debug('Executing: %s', command)
stdout = capture_output and tempfile.TemporaryFile() or stdout
# redirect output to /dev/null if None given
devnull = open(os.devnull, 'wb')
stdout = stdout or devnull
stderr = stderr or devnull
ret_env = {}
if env is not None:
for key, value in six.iteritems(env):
env[key] = str(value)
perf_timer = util.PerfTimer()
p = subprocess.Popen(
command,
stdin=stdin,
stdout=stdout,
stderr=forward_stderr and subprocess.STDOUT or stderr,
shell=True,
close_fds=True,
universal_newlines=True,
env=env,
cwd=tempcwd(),
preexec_fn=os.setpgrp,
)
kill_timer = None
if real_time_limit:
def oot_killer():
ret_env['real_time_killed'] = True
os.killpg(p.pid, signal.SIGKILL)
kill_timer = Timer(ms2s(real_time_limit), oot_killer)
kill_timer.start()
rc = p.wait()
ret_env['return_code'] = rc
if kill_timer:
kill_timer.cancel()
ret_env['real_time_used'] = s2ms(perf_timer.elapsed)
logger.debug(
'Command "%s" exited with code %d, took %.2fs',
str(command),
rc,
perf_timer.elapsed,
)
devnull.close()
if capture_output:
stdout.seek(0)
ret_env['stdout'] = stdout.read(output_limit or -1)
stdout.close()
if split_lines:
ret_env['stdout'] = ret_env['stdout'].split(b'\n')
if rc and not ignore_errors and rc not in extra_ignore_errors:
raise ExecError(
'Failed to execute command: %s. Returned with code %s\n' %
(command, rc))
return ret_env
def change(env):
res_ok(env)
eq_('13', open(tempcwd('somefile')).read().strip())
ok_(os.path.exists(tempcwd('./not_existing')))
def nochange(env):
res_re(1)(env)
eq_('42', open(tempcwd('somefile')).read().strip())
ok_(not os.path.exists(tempcwd('./not_existing')))
def change(env):
res_ok(env)
eq_('13', open(tempcwd('somefile')).read().strip())
ok_(os.path.exists(tempcwd('./not_existing')))
def _execute(self, command, **kwargs):
env = kwargs.get('env')
env.update({
'MEM_LIMIT': kwargs['mem_limit'] or 64 * 2**10,
'TIME_LIMIT': kwargs['time_limit'] or 30000,
'OUT_LIMIT': kwargs['output_limit'] or 50 * 2**20,
})
if kwargs['real_time_limit']:
env['HARD_LIMIT'] = 1 + ceil_ms2s(kwargs['real_time_limit'])
elif kwargs['time_limit'] and kwargs['real_time_limit'] is None:
env['HARD_LIMIT'] = 1 + ceil_ms2s(64 * kwargs['time_limit'])
if 'HARD_LIMIT' in env:
# Limiting outside supervisor
kwargs['real_time_limit'] = 2 * s2ms(env['HARD_LIMIT'])
ignore_errors = kwargs.pop('ignore_errors')
extra_ignore_errors = kwargs.pop('extra_ignore_errors')
renv = {}
try:
result_file = tempfile.NamedTemporaryFile(dir=tempcwd())
kwargs['ignore_errors'] = True
renv = execute_command(
command + [noquote('3>'), result_file.name],
**kwargs
)
if 'real_time_killed' in renv:
raise ExecError('Supervisor exceeded realtime limit')
elif renv['return_code'] and \
renv['return_code'] not in extra_ignore_errors:
raise ExecError('Supervisor returned code %s'
% renv['return_code'])
result_file.seek(0)
status_line = result_file.readline().strip().split()[1:]
renv['result_string'] = result_file.readline().strip()
result_file.close()
for num, key in enumerate(('result_code', 'time_used',
None, 'mem_used', 'num_syscalls')):
if key:
renv[key] = int(status_line[num])
result_code = self._supervisor_result_to_code(renv['result_code'])
except Exception as e:
logger.error('SupervisedExecutor error: %s', traceback.format_exc())
logger.error('SupervisedExecutor error dirlist: %s: %s',
tempcwd(), str(os.listdir(tempcwd())))
result_code = 'SE'
for i in ('time_used', 'mem_used', 'num_syscalls'):
renv.setdefault(i, 0)
renv['result_string'] = str(e)
renv['result_code'] = result_code
if result_code != 'OK' and not ignore_errors and not \
(result_code != 'RV' and renv['return_code'] in \
extra_ignore_errors):
raise ExecError('Failed to execute command: %s. Reason: %s'
% (command, renv['result_string']))
return renv
def _make_cmdline(self, executor):
# Addinational sources are automatically included
return (['fpc', tempcwd('a.pas')] + self.options +
list(self.extra_compilation_args))
def nochange(env):
res_re(1)(env)
eq_('42', open(tempcwd('somefile')).read().strip())
ok_(not os.path.exists(tempcwd('./not_existing')))
def execute_command(command, env=None, split_lines=False, stdin=None,
stdout=None, stderr=None, forward_stderr=False,
capture_output=False, output_limit=None,
real_time_limit=None,
ignore_errors=False, extra_ignore_errors=(), **kwargs):
"""Utility function to run arbitrary command.
``stdin``
Could be either file opened with ``open(fname, 'r')``
or None (then it is inherited from current process).
``stdout``, ``stderr``
Could be files opened with ``open(fname, 'w')``, sys.std*
or None - then it's suppressed.
``forward_stderr``
Forwards stderr to stdout.
``capture_output``
Returns program output in renv key ``stdout``.
``output_limit``
Limits returned output when ``capture_output=True`` (in bytes).
Returns renv: dictionary containing:
``real_time_used``
Wall clock time it took to execute the command (in ms).
``return_code``
Status code that program returned.
``real_time_killed``
Only when process was killed due to exceeding real time limit.
``stdout``
Only when ``capture_output=True``: output of the command
"""
# Using temporary file is way faster than using subproces.PIPE
# and it prevents deadlocks.
command = shellquote(command)
logger.debug('Executing: %s', command)
stdout = capture_output and tempfile.TemporaryFile() or stdout
# redirect output to /dev/null if None given
devnull = open(os.devnull, 'wb')
stdout = stdout or devnull
stderr = stderr or devnull
ret_env = {}
if env is not None:
for key, value in env.iteritems():
env[key] = str(value)
perf_timer = util.PerfTimer()
p = subprocess.Popen(command,
stdin=stdin,
stdout=stdout,
stderr=forward_stderr and subprocess.STDOUT
or stderr,
shell=True,
close_fds=True,
universal_newlines=True,
env=env,
cwd=tempcwd(),
preexec_fn=os.setpgrp)
kill_timer = None
if real_time_limit:
def oot_killer():
ret_env['real_time_killed'] = True
os.killpg(p.pid, signal.SIGKILL)
kill_timer = Timer(ms2s(real_time_limit), oot_killer)
kill_timer.start()
rc = p.wait()
ret_env['return_code'] = rc
if kill_timer:
kill_timer.cancel()
ret_env['real_time_used'] = s2ms(perf_timer.elapsed)
logger.debug('Command "%s" exited with code %d, took %.2fs',
str(command), rc, perf_timer.elapsed)
devnull.close()
if capture_output:
stdout.seek(0)
ret_env['stdout'] = stdout.read(output_limit or -1)
stdout.close()
if split_lines:
ret_env['stdout'] = ret_env['stdout'].split('\n')
if rc and not ignore_errors and rc not in extra_ignore_errors:
raise ExecError('Failed to execute command: %s. Returned with code %s\n'
% (command, rc))
return ret_env
def run(environ, executor, use_sandboxes=True):
"""
Common code for executors.
:param: environ Recipe to pass to `filetracker` and `sio.workers.executors`
For all supported options, see the global documentation for
`sio.workers.executors` and prefix them with ``exec_``.
:param: executor Executor instance used for executing commands.
:param: use_sandboxes Enables safe checking output correctness.
See `sio.executors.checkers`. True by default.
"""
if feedback.judge_prepare(environ).force_not_judge:
environ['time_used'] = 0
environ['result_string'] = 'not judged'
environ['result_code'] = 'NJ'
environ['mem_used'] = 0
environ['num_syscalls'] = 0
environ['stderr'] = ''
return environ
input_name = tempcwd('in')
file_executor = get_file_runner(executor, environ)
exe_filename = file_executor.preferred_filename()
ft.download(environ, 'exe_file', exe_filename, add_to_cache=True)
os.chmod(tempcwd(exe_filename), 0700)
ft.download(environ, 'in_file', input_name, add_to_cache=True)
zipdir = tempcwd('in_dir')
os.mkdir(zipdir)
try:
if is_zipfile(input_name):
try:
# If not a zip file, will pass it directly to exe
with ZipFile(tempcwd('in'), 'r') as f:
if len(f.namelist()) != 1:
raise Exception("Archive should have only one file.")
f.extract(f.namelist()[0], zipdir)
input_name = os.path.join(zipdir, f.namelist()[0])
# zipfile throws some undocumented exceptions
except Exception as e:
raise StandardError("Failed to open archive: " + unicode(e))
with file_executor as fe:
with open(input_name, 'rb') as inf:
with open(tempcwd('out'), 'wb') as outf:
feedback.judge_started(environ)
renv = fe(tempcwd(exe_filename), [],
stdin=inf,
stdout=outf,
ignore_errors=True,
environ=environ,
environ_prefix='exec_')
_populate_environ(renv, environ)
if renv['result_code'] == 'OK' and environ.get('check_output'):
environ = checker.run(environ, use_sandboxes=use_sandboxes)
for key in ('result_code', 'result_string'):
environ[key] = replace_invalid_UTF(environ[key])
feedback.judge_finished(environ)
if 'out_file' in environ:
ft.upload(environ,
'out_file',
tempcwd('out'),
to_remote_store=environ.get('upload_out', False))
finally:
rmtree(zipdir)
return environ