def upload_files():
"Uploads all files from SOURCES to a newly created dummy filetracker"
# DummyClient operates in the working directory.
ft.set_instance(DummyClient())
for path in glob.glob(os.path.join(SOURCES, '*')):
ft.upload({'path': '/' + os.path.basename(path)}, 'path', path)
def _collect_and_upload(env, path, upload_path, re_string):
names_re = re.compile(re_string)
env['collected_files'] = dict()
for out_file in os.listdir(path):
if names_re.match(out_file):
ft.upload(env['collected_files'], out_file, out_file,
'%s/%s' % (upload_path, out_file))
def upload_files():
"Uploads all files from SOURCES to a newly created dummy filetracker"
# DummyClient operates in the working directory.
ft.set_instance(DummyClient())
for path in glob.glob(os.path.join(SOURCES, '*')):
ft.upload({'path': '/' + os.path.basename(path)}, 'path', path)
def run(environ, executor, safe_check=True):
"""
Common code for executors.
:param: environ Recipe to pass to `filetracker` and `sio.workers.executors`
For all supported options, see the global documentation for
`sio.workers.executors`.
:param: executor Executor instance used for executing commands.
:param: safe_check Enables safe checking output corectness.
See `sio.executors.checkers`. True by default.
"""
ft.download(environ, 'exe_file', 'exe', add_to_cache=True)
os.chmod('exe', 0700)
ft.download(environ, 'in_file', 'in', add_to_cache=True)
with executor as e:
with open('in', 'rb') as inf:
with open('out', 'wb') as outf:
renv = e(['./exe'], stdin=inf, stdout=outf, ignore_errors=True,
environ=environ)
_populate_environ(renv, environ)
if renv['result_code'] == 'OK' and environ.get('check_output'):
environ = checker.run(environ, no_sandbox=not safe_check)
if 'out_file' in environ:
ft.upload(environ, 'out_file', 'out',
to_remote_store=environ.get('upload_out', False))
return environ
def _collect_and_upload(env, path, upload_path, re_string):
names_re = re.compile(re_string)
env['collected_files'] = dict()
for out_file in os.listdir(path):
if names_re.match(out_file):
ft.upload(env['collected_files'], out_file,
os.path.join(path, out_file),
'%s/%s' % (upload_path, out_file))
def run(environ, executor, use_sandboxes=True):
"""
Common code for executors.
:param: environ Recipe to pass to `filetracker` and `sio.workers.executors`
For all supported options, see the global documentation for
`sio.workers.executors` and prefix them with ``exec_``.
:param: executor Executor instance used for executing commands.
:param: use_sandboxes Enables safe checking output correctness.
See `sio.executors.checkers`. True by default.
"""
input_name = tempcwd('in')
file_executor = get_file_runner(executor, environ)
exe_filename = file_executor.preferred_filename()
ft.download(environ, 'exe_file', exe_filename, add_to_cache=True)
os.chmod(tempcwd(exe_filename), 0700)
ft.download(environ, 'in_file', input_name, add_to_cache=True)
zipdir = tempcwd('in_dir')
os.mkdir(zipdir)
try:
if is_zipfile(input_name):
try:
# If not a zip file, will pass it directly to exe
with ZipFile(tempcwd('in'), 'r') as f:
if len(f.namelist()) != 1:
raise Exception("Archive should have only one file.")
f.extract(f.namelist()[0], zipdir)
input_name = os.path.join(zipdir, f.namelist()[0])
# zipfile throws some undocumented exceptions
except Exception as e:
raise StandardError("Failed to open archive: " + unicode(e))
with file_executor as fe:
with open(input_name, 'rb') as inf:
with open(tempcwd('out'), 'wb') as outf:
renv = fe(tempcwd(exe_filename), [],
stdin=inf, stdout=outf, ignore_errors=True,
environ=environ, environ_prefix='exec_')
_populate_environ(renv, environ)
if renv['result_code'] == 'OK' and environ.get('check_output'):
environ = checker.run(environ, use_sandboxes=use_sandboxes)
for key in ('result_code', 'result_string'):
environ[key] = replace_invalid_UTF(environ[key])
if 'out_file' in environ:
ft.upload(environ, 'out_file', tempcwd('out'),
to_remote_store=environ.get('upload_out', False))
finally:
rmtree(zipdir)
return environ
def run(environ):
if environ['compiler'] not in ('foo.1_0', 'foo.2_0'):
raise RuntimeError("Compiler '%s' not found.", environ['compiler'])
input_file = ft.download(environ, 'source_file', 'a.foo')
print input_file
size = os.path.getsize(input_file)
out = open('compiled', 'w')
out.write("#!/bin/sh\n")
out.write("# Compiled using Foo compiler named %s\n" % environ['compiler'])
out.write("echo %d" % size)
out.close()
ft.upload(environ, 'out_file', 'compiled')
return environ
def run(environ):
if environ['compiler'] not in ('foo.1_0', 'foo.2_0'):
raise RuntimeError("Compiler '%s' not found.", environ['compiler'])
input_file = ft.download(environ, 'source_file', 'a.foo')
print(input_file)
size = os.path.getsize(input_file)
out = open('compiled', 'w')
out.write("#!/bin/sh\n")
out.write("# Compiled using Foo compiler named %s\n" % environ['compiler'])
out.write("echo %d" % size)
out.close()
ft.upload(environ, 'out_file', 'compiled')
return environ
def _postprocess(self, renv):
self.environ['compiler_output'] = replace_invalid_UTF(renv['stdout'])
if renv['return_code']:
self.environ['result_code'] = 'CE'
elif 'compilation_result_size_limit' in self.environ and \
os.path.getsize(tempcwd(self.output_file)) > \
self.environ['compilation_result_size_limit']:
self.environ['result_code'] = 'CE'
self.environ['compiler_output'] = \
'Compiled file size limit exceeded.'
else:
self.environ['result_code'] = 'OK'
self.environ['exec_info'] = {'mode': 'executable'}
ft.upload(self.environ, 'out_file', tempcwd(self.output_file))
return self.environ
def run(environ):
exe_file = ft.download(environ, 'exe_file', 'exe', add_to_cache=True)
os.chmod(exe_file, 0700)
in_file = ft.download(environ, 'in_file', 'in', add_to_cache=True)
if 'exec_time_limit' in environ:
time_limit = environ['exec_time_limit']
time_ulimit = (time_limit + 999) / 1000
else:
time_limit = time_ulimit = None
if 'exec_mem_limit' in environ:
mem_limit = environ['exec_mem_limit'] / 1024
else:
mem_limit = None
retcode, output = execute(['bash', '-c', 'time ./exe < in > out'],
time_limit=time_ulimit, mem_limit=mem_limit, ignore_errors=True)
time_output_matches = TIME_OUTPUT_RE.findall(output)
if time_output_matches:
mins, secs = time_output_matches[-1]
time_used = int((int(mins) * 60 + float(secs)) * 1000)
else:
raise RuntimeError('Could not find output of time program. '
'Captured output: %s' % output)
if time_limit is not None and time_used >= 0.99 * time_limit:
environ['result_string'] = 'time limit exceeded'
environ['result_code'] = 'TLE'
elif retcode == 0:
environ['result_string'] = 'ok'
environ['result_code'] = 'OK'
else:
environ['result_string'] = 'program exited with code %d' % retcode
environ['result_code'] = 'RE'
environ['time_used'] = time_used
environ['exectime_used'] = 0
environ['mem_used'] = 0
environ['num_syscalls'] = 0
if environ['result_code'] == 'OK' and environ.get('check_output'):
environ = checker.run(environ, no_sandbox=True)
if 'out_file' in environ:
ft.upload(environ, 'out_file', 'out',
to_remote_store=environ.get('upload_out', False))
return environ
def _postprocess(self, renv):
self.environ['compiler_output'] = replace_invalid_UTF(renv['stdout'])
if renv['return_code']:
self.environ['result_code'] = 'CE'
elif 'compilation_result_size_limit' in self.environ and \
os.path.getsize(tempcwd(self.output_file)) > \
self.environ['compilation_result_size_limit']:
self.environ['result_code'] = 'CE'
self.environ['compiler_output'] = \
'Compiled file size limit exceeded.'
else:
self.environ['result_code'] = 'OK'
self.environ['exec_info'] = {'mode': 'executable'}
ft.upload(self.environ, 'out_file', tempcwd(self.output_file))
return self.environ
def run(environ):
exe_file = ft.download(environ, 'exe_file', 'exe', add_to_cache=True)
os.chmod(exe_file, 0700)
in_file = ft.download(environ, 'in_file', 'in', add_to_cache=True)
env = os.environ.copy()
for key, default in _options:
value = environ.get('exec_' + key.lower(), default)
env[key] = value
with get_sandbox('vcpu_exec-sandbox') as sandbox:
retcode, output = execute(
[os.path.join(sandbox.path, 'pin-supervisor/supervisor-bin/',
'supervisor'), '-f', '3',
'--', './exe',
noquote('<'), 'in', noquote('3>'), 'supervisor_result',
noquote('>'), 'out'], env=env)
result_file = open('supervisor_result')
status_line = result_file.readline().strip().split()
environ['result_string'] = result_file.readline().strip()
result_file.close()
for num, key in enumerate((None, 'result_code', 'time_used',
'exectime_used', 'mem_used', 'num_syscalls')):
if key:
environ[key] = int(status_line[num])
result_code = _supervisor_result_to_code(environ['result_code'])
environ['result_code'] = result_code
if result_code == 'OK' and environ.get('check_output'):
environ = checker.run(environ)
if 'out_file' in environ:
ft.upload(environ, 'out_file', 'out',
to_remote_store=environ.get('upload_out', False))
return environ
def run(environ, executor, use_sandboxes=True):
"""
Common code for executors.
:param: environ Recipe to pass to `filetracker` and `sio.workers.executors`
For all supported options, see the global documentation for
`sio.workers.executors` and prefix them with ``exec_``.
:param: executor Executor instance used for executing commands.
:param: use_sandboxes Enables safe checking output correctness.
See `sio.executors.checkers`. True by default.
"""
if feedback.judge_prepare(environ).force_not_judge:
environ['time_used'] = 0
environ['result_string'] = 'not judged'
environ['result_code'] = 'NJ'
environ['mem_used'] = 0
environ['num_syscalls'] = 0
environ['stderr'] = ''
return environ
input_name = tempcwd('in')
file_executor = get_file_runner(executor, environ)
exe_filename = file_executor.preferred_filename()
ft.download(environ, 'exe_file', exe_filename, add_to_cache=True)
os.chmod(tempcwd(exe_filename), 0700)
ft.download(environ, 'in_file', input_name, add_to_cache=True)
zipdir = tempcwd('in_dir')
os.mkdir(zipdir)
try:
if is_zipfile(input_name):
try:
# If not a zip file, will pass it directly to exe
with ZipFile(tempcwd('in'), 'r') as f:
if len(f.namelist()) != 1:
raise Exception("Archive should have only one file.")
f.extract(f.namelist()[0], zipdir)
input_name = os.path.join(zipdir, f.namelist()[0])
# zipfile throws some undocumented exceptions
except Exception as e:
raise StandardError("Failed to open archive: " + unicode(e))
with file_executor as fe:
with open(input_name, 'rb') as inf:
with open(tempcwd('out'), 'wb') as outf:
feedback.judge_started(environ)
renv = fe(tempcwd(exe_filename), [],
stdin=inf,
stdout=outf,
ignore_errors=True,
environ=environ,
environ_prefix='exec_')
_populate_environ(renv, environ)
if renv['result_code'] == 'OK' and environ.get('check_output'):
environ = checker.run(environ, use_sandboxes=use_sandboxes)
for key in ('result_code', 'result_string'):
environ[key] = replace_invalid_UTF(environ[key])
feedback.judge_finished(environ)
if 'out_file' in environ:
ft.upload(environ,
'out_file',
tempcwd('out'),
to_remote_store=environ.get('upload_out', False))
finally:
rmtree(zipdir)
return environ
def _collect_and_upload(env, path, upload_path, re_string):
names_re = re.compile(re_string)
env["collected_files"] = dict()
for out_file in os.listdir(path):
if names_re.match(out_file):
ft.upload(env["collected_files"], out_file, os.path.join(path, out_file), "%s/%s" % (upload_path, out_file))
'-f', '3', '--', './exe',
noquote('<'), 'in', noquote('3>'), 'supervisor_result',
noquote('>'), 'out'], env=env)
result_file = open('supervisor_result')
status_line = result_file.readline().strip().split()
environ['result_string'] = result_file.readline().strip()
result_file.close()
try:
for num, key in enumerate((None, 'result_code', 'time_used',
'exectime_used', 'mem_used', 'num_syscalls')):
if key:
environ[key] = int(status_line[num])
result_code = _supervisor_result_to_code(environ['result_code'])
except Exception, e:
result_code = 'SE'
for i in ('time_used', 'exectime_used', 'mem_used', 'num_syscalls'):
environ.setdefault(i, 0)
environ['result_string'] = str(e)
environ['result_code'] = result_code
if result_code == 'OK' and environ.get('check_output'):
environ = checker.run(environ)
if 'out_file' in environ:
ft.upload(environ, 'out_file', 'out',
to_remote_store=environ.get('upload_out', False))
return environ
def run(environ, lang, compiler, extension, output_file, compiler_options=(),
compile_additional_sources=True, sandbox=False,
sandbox_callback=None):
"""
Common code for compiler handlers:
:param environ: Recipe to pass to `filetracker` and `sio.workers.execute`
For all supported options, see the global documentation for
`sio.compilers`.
:param lang: Language code (for example: `c`, `cpp`, `pas`)
:param compiler: Compiler binary name
:param extension: Usual extension for source files of the given language.
:param output_file: Default output binary file, assuming the input is named
`a.<extension>`
:param compiler_options: Optional tuple of command line parameters to the
compiler.
:param compile_additional_sources: Disables passing additional
source files to the compiler - used
as a hack to support FPC.
Defaults to False.
:param sandbox: Enables sandboxing (using compiler name
as a sandbox). Defaults to False.
:param sandbox_callback: Optional callback called immediately after
creating the sandbox, with the said sandbox
as its sole parameter.
"""
if sandbox is True:
sandbox = get_sandbox('compiler-' + environ['compiler'])
elif not sandbox:
sandbox = NullSandbox()
extra_compilation_args = \
_lang_option(environ, 'extra_compilation_args', lang)
ft.download(environ, 'source_file', 'a.' + extension)
cmdline = (compiler, ) + tuple(compiler_options) + \
tuple(extra_compilation_args) + ('a.' + extension, )
# this cmdline may be later extended
# using a copy of the environment in order to avoid polluting it with
# temoporary elements
tmp_environ = environ.copy()
additional_includes = _lang_option(environ, 'additional_includes', lang)
additional_sources = _lang_option(environ, 'additional_sources', lang)
for include in additional_includes:
tmp_environ['additional_include'] = include
ft.download(tmp_environ, 'additional_include',
os.path.basename(include))
for source in additional_sources:
tmp_environ['additional_source'] = source
ft.download(tmp_environ, 'additional_source',
os.path.basename(source))
if compile_additional_sources:
cmdline += (os.path.basename(source), )
extra_files = environ.get('extra_files', {})
for name, ft_path in extra_files.iteritems():
tmp_environ['extra_file'] = ft_path
ft.download(tmp_environ, 'extra_file', os.path.basename(name))
shell_environ = os.environ.copy()
if sandbox:
shell_environ['LD_LIBRARY_PATH'] = (lang == 'pas') \
and os.path.join(sandbox.path, 'lib') \
or os.path.join(sandbox.path, 'usr', 'lib')
shell_environ['PATH'] = (lang == 'pas') \
and os.path.join(sandbox.path, 'bin') \
or os.path.join(sandbox.path, 'usr', 'bin')
shell_environ['PATH'] += ':' + os.environ['PATH']
with sandbox:
if sandbox_callback:
sandbox_callback(sandbox)
retcode, output = execute(list(cmdline),
env=shell_environ,
time_limit=30,
mem_limit=256,
ignore_errors=True,
environ=tmp_environ,
environ_prefix='compilation_')
environ['compiler_output'] = output
if retcode:
environ['result_code'] = 'CE'
elif 'compilation_result_size_limit' in environ and \
os.path.getsize(output_file) > \
environ['compilation_result_size_limit']:
environ['result_code'] = 'CE'
environ['compiler_output'] = 'Compiled file size limit exceeded.'
else:
environ['result_code'] = 'OK'
ft.upload(environ, 'out_file', output_file)
return environ
def run(environ, executor, use_sandboxes=True):
"""
Common code for executors.
:param: environ Recipe to pass to `filetracker` and `sio.workers.executors`
For all supported options, see the global documentation for
`sio.workers.executors` and prefix them with ``exec_``.
:param: executor Executor instance used for executing commands.
:param: use_sandboxes Enables safe checking output correctness.
See `sio.executors.checkers`. True by default.
"""
input_name = tempcwd('in')
file_executor = get_file_runner(executor, environ)
exe_filename = file_executor.preferred_filename()
ft.download(environ, 'exe_file', exe_filename, add_to_cache=True)
os.chmod(tempcwd(exe_filename), 0o700)
ft.download(environ, 'in_file', input_name, add_to_cache=True)
zipdir = tempcwd('in_dir')
os.mkdir(zipdir)
try:
if is_zipfile(input_name):
try:
# If not a zip file, will pass it directly to exe
with ZipFile(tempcwd('in'), 'r') as f:
if len(f.namelist()) != 1:
raise Exception("Archive should have only one file.")
f.extract(f.namelist()[0], zipdir)
input_name = os.path.join(zipdir, f.namelist()[0])
# zipfile throws some undocumented exceptions
except Exception as e:
raise Exception("Failed to open archive: " + six.text_type(e))
with file_executor as fe:
with open(input_name, 'rb') as inf:
# Open output file in append mode to allow appending
# only to the end of the output file. Otherwise,
# a contestant's program could modify the middle of the file.
with open(tempcwd('out'), 'ab') as outf:
renv = fe(tempcwd(exe_filename), [],
stdin=inf,
stdout=outf,
ignore_errors=True,
environ=environ,
environ_prefix='exec_')
_populate_environ(renv, environ)
if renv['result_code'] == 'OK' and environ.get('check_output'):
environ = checker.run(environ, use_sandboxes=use_sandboxes)
for key in ('result_code', 'result_string'):
environ[key] = replace_invalid_UTF(environ[key])
if 'out_file' in environ:
ft.upload(environ,
'out_file',
tempcwd('out'),
to_remote_store=environ.get('upload_out', False))
finally:
rmtree(zipdir)
return environ
def run(environ, lang, compiler, extension, output_file, compiler_options=(),
compile_additional_sources=True, sandbox=False,
sandbox_callback=None):
"""
Common code for compiler handlers:
:param environ: Recipe to pass to `filetracker` and `sio.workers.execute`
For all supported options, see the global documentation for
`sio.compilers`.
:param lang: Language code (for example: `c`, `cpp`, `pas`)
:param compiler: Compiler binary name
:param extension: Usual extension for source files of the given language.
:param output_file: Default output binary file, assuming the input is named
`a.<extension>`
:param compiler_options: Optional tuple of command line parameters to the
compiler.
:param compile_additional_sources: Enables passing additional
source files to the compiler - used
as a hack to support FPC.
Defaults to True.
:param sandbox: Enables sandboxing (using compiler name
as a sandbox). Defaults to False.
:param sandbox_callback: Optional callback called immediately after
creating the executor, with the said executor
and the command argument list as arguments.
Should return new command if modified.
"""
if sandbox is False:
executor = UnprotectedExecutor()
else:
executor = PRootExecutor('compiler-' + environ['compiler'])
extra_compilation_args = \
_lang_option(environ, 'extra_compilation_args', lang)
ft.download(environ, 'source_file', 'a.' + extension)
cmdline = [compiler, ] + list(compiler_options) + \
list(extra_compilation_args) + ['a.' + extension, ]
# this cmdline may be later extended
# using a copy of the environment in order to avoid polluting it with
# temoporary elements
tmp_environ = environ.copy()
additional_includes = _lang_option(environ, 'additional_includes', lang)
additional_sources = _lang_option(environ, 'additional_sources', lang)
for include in additional_includes:
tmp_environ['additional_include'] = include
ft.download(tmp_environ, 'additional_include',
os.path.basename(include))
for source in additional_sources:
tmp_environ['additional_source'] = source
ft.download(tmp_environ, 'additional_source',
os.path.basename(source))
if compile_additional_sources:
cmdline += [os.path.basename(source), ]
extra_files = environ.get('extra_files', {})
for name, ft_path in extra_files.iteritems():
tmp_environ['extra_file'] = ft_path
ft.download(tmp_environ, 'extra_file', os.path.basename(name))
with executor:
if sandbox_callback:
cmdline = sandbox_callback(executor, cmdline) or cmdline
renv = executor(cmdline,
time_limit=DEFAULT_COMPILER_TIME_LIMIT,
mem_limit=DEFAULT_COMPILER_MEM_LIMIT,
output_limit=DEFAULT_COMPILER_OUTPUT_LIMIT,
ignore_errors=True,
environ=tmp_environ,
environ_prefix='compilation_',
capture_output=True,
forward_stderr=True)
environ['compiler_output'] = replace_invalid_UTF(renv['stdout'])
if renv['return_code']:
environ['result_code'] = 'CE'
elif 'compilation_result_size_limit' in environ and \
os.path.getsize(output_file) > \
environ['compilation_result_size_limit']:
environ['result_code'] = 'CE'
environ['compiler_output'] = 'Compiled file size limit exceeded.'
else:
environ['result_code'] = 'OK'
ft.upload(environ, 'out_file', output_file)
return environ
