def _execute(self, command, **kwargs):
env = kwargs.get("env")
env.update(
{
"MEM_LIMIT": kwargs["mem_limit"] or 64 * 2 ** 10,
"TIME_LIMIT": kwargs["time_limit"] or 30000,
"OUT_LIMIT": kwargs["output_limit"] or 50 * 2 ** 20,
}
)
if kwargs["real_time_limit"]:
env["HARD_LIMIT"] = 1 + ceil_ms2s(kwargs["real_time_limit"])
elif kwargs["time_limit"] and kwargs["real_time_limit"] is None:
env["HARD_LIMIT"] = 1 + ceil_ms2s(64 * kwargs["time_limit"])
if "HARD_LIMIT" in env:
# Limiting outside supervisor
kwargs["real_time_limit"] = 2 * s2ms(env["HARD_LIMIT"])
ignore_errors = kwargs.pop("ignore_errors")
extra_ignore_errors = kwargs.pop("extra_ignore_errors")
renv = {}
try:
result_file = tempfile.NamedTemporaryFile(dir=os.getcwd())
kwargs["ignore_errors"] = True
renv = execute_command(command + [noquote("3>"), result_file.name], **kwargs)
if "real_time_killed" in renv:
raise ExecError("Supervisor exceeded realtime limit")
elif renv["return_code"]:
raise ExecError("Supervisor returned code %s" % renv["return_code"])
result_file.seek(0)
status_line = result_file.readline().strip().split()[1:]
renv["result_string"] = result_file.readline().strip()
result_file.close()
for num, key in enumerate(("result_code", "time_used", None, "mem_used", "num_syscalls")):
if key:
renv[key] = int(status_line[num])
result_code = self._supervisor_result_to_code(renv["result_code"])
except Exception as e:
logger.error("SupervisedExecutor error: %s", traceback.format_exc())
logger.error("SupervisedExecutor error dirlist: %s: %s", os.getcwd(), str(os.listdir(".")))
result_code = "SE"
for i in ("time_used", "mem_used", "num_syscalls"):
renv.setdefault(i, 0)
renv["result_string"] = str(e)
renv["result_code"] = result_code
if result_code != "OK" and not ignore_errors and result_code not in extra_ignore_errors:
raise ExecError("Failed to execute command: %s. Reason: %s" % (command, renv["result_string"]))
return renv
def execute_command(command, env=None, split_lines=False, stdin=None,
stdout=None, stderr=None, forward_stderr=False,
capture_output=False, output_limit=None,
real_time_limit=None,
ignore_errors=False, extra_ignore_errors=(), **kwargs):
"""Utility function to run arbitrary command.
``stdin``
Could be either file opened with ``open(fname, 'r')``
or None (then it is inherited from current process).
``stdout``, ``stderr``
Could be files opened with ``open(fname, 'w')``, sys.std*
or None - then it's suppressed.
``forward_stderr``
Forwards stderr to stdout.
``capture_output``
Returns program output in renv key ``stdout``.
``output_limit``
Limits returned output when ``capture_output=True`` (in bytes).
Returns renv: dictionary containing:
``real_time_used``
Wall clock time it took to execute the command (in ms).
``return_code``
Status code that program returned.
``real_time_killed``
Only when process was killed due to exceeding real time limit.
``stdout``
Only when ``capture_output=True``: output of the command
"""
# Using temporary file is way faster than using subproces.PIPE
# and it prevents deadlocks.
command = shellquote(command)
logger.debug('Executing: %s', command)
stdout = capture_output and tempfile.TemporaryFile() or stdout
# redirect output to /dev/null if None given
devnull = open(os.devnull, 'wb')
stdout = stdout or devnull
stderr = stderr or devnull
ret_env = {}
if env is not None:
for key, value in env.iteritems():
env[key] = str(value)
perf_timer = util.PerfTimer()
p = subprocess.Popen(command,
stdin=stdin,
stdout=stdout,
stderr=forward_stderr and subprocess.STDOUT
or stderr,
shell=True,
close_fds=True,
universal_newlines=True,
env=env,
preexec_fn=os.setpgrp)
kill_timer = None
if real_time_limit:
def oot_killer():
ret_env['real_time_killed'] = True
os.killpg(p.pid, signal.SIGKILL)
kill_timer = Timer(ms2s(real_time_limit), oot_killer)
kill_timer.start()
rc = p.wait()
ret_env['return_code'] = rc
if kill_timer:
kill_timer.cancel()
ret_env['real_time_used'] = s2ms(perf_timer.elapsed)
logger.debug('Command "%s" exited with code %d, took %.2fs',
str(command), rc, perf_timer.elapsed)
devnull.close()
if capture_output:
stdout.seek(0)
ret_env['stdout'] = stdout.read(output_limit or -1)
stdout.close()
if split_lines:
ret_env['stdout'] = ret_env['stdout'].split('\n')
if rc and not ignore_errors and rc not in extra_ignore_errors:
raise ExecError('Failed to execute command: %s. Returned with code %s\n'
% (command, rc))
return ret_env
def _execute(self, command, **kwargs):
env = kwargs.get('env')
env.update({
'MEM_LIMIT': kwargs['mem_limit'] or 64 * 2**10,
'TIME_LIMIT': kwargs['time_limit'] or 30000,
'OUT_LIMIT': kwargs['output_limit'] or 50 * 2**20,
})
if kwargs['real_time_limit']:
env['HARD_LIMIT'] = 1 + ceil_ms2s(kwargs['real_time_limit'])
elif kwargs['time_limit'] and kwargs['real_time_limit'] is None:
env['HARD_LIMIT'] = 1 + ceil_ms2s(64 * kwargs['time_limit'])
if 'HARD_LIMIT' in env:
# Limiting outside supervisor
kwargs['real_time_limit'] = 2 * s2ms(env['HARD_LIMIT'])
ignore_errors = kwargs.pop('ignore_errors')
extra_ignore_errors = kwargs.pop('extra_ignore_errors')
renv = {}
try:
result_file = tempfile.NamedTemporaryFile(dir=os.getcwd())
kwargs['ignore_errors'] = True
renv = execute_command(
command + [noquote('3>'), result_file.name],
**kwargs
)
if 'real_time_killed' in renv:
raise ExecError('Supervisor exceeded realtime limit')
elif renv['return_code'] and \
renv['return_code'] not in extra_ignore_errors:
raise ExecError('Supervisor returned code %s'
% renv['return_code'])
result_file.seek(0)
status_line = result_file.readline().strip().split()[1:]
renv['result_string'] = result_file.readline().strip()
result_file.close()
for num, key in enumerate(('result_code', 'time_used',
None, 'mem_used', 'num_syscalls')):
if key:
renv[key] = int(status_line[num])
result_code = self._supervisor_result_to_code(renv['result_code'])
except Exception as e:
logger.error('SupervisedExecutor error: %s', traceback.format_exc())
logger.error('SupervisedExecutor error dirlist: %s: %s',
os.getcwd(), str(os.listdir('.')))
result_code = 'SE'
for i in ('time_used', 'mem_used', 'num_syscalls'):
renv.setdefault(i, 0)
renv['result_string'] = str(e)
renv['result_code'] = result_code
if result_code != 'OK' and not ignore_errors and not \
(result_code != 'RV' and renv['return_code'] in \
extra_ignore_errors):
raise ExecError('Failed to execute command: %s. Reason: %s'
% (command, renv['result_string']))
return renv
def execute_command(command,
env=None,
split_lines=False,
stdin=None,
stdout=None,
stderr=None,
forward_stderr=False,
capture_output=False,
output_limit=None,
real_time_limit=None,
ignore_errors=False,
extra_ignore_errors=(),
**kwargs):
"""Utility function to run arbitrary command.
``stdin``
Could be either file opened with ``open(fname, 'r')``
or None (then it is inherited from current process).
``stdout``, ``stderr``
Could be files opened with ``open(fname, 'w')``, sys.std*
or None - then it's suppressed.
``forward_stderr``
Forwards stderr to stdout.
``capture_output``
Returns program output in renv key ``stdout``.
``output_limit``
Limits returned output when ``capture_output=True`` (in bytes).
Returns renv: dictionary containing:
``real_time_used``
Wall clock time it took to execute the command (in ms).
``return_code``
Status code that program returned.
``real_time_killed``
Only when process was killed due to exceeding real time limit.
``stdout``
Only when ``capture_output=True``: output of the command
"""
# Using temporary file is way faster than using subproces.PIPE
# and it prevents deadlocks.
command = shellquote(command)
logger.debug('Executing: %s', command)
stdout = capture_output and tempfile.TemporaryFile() or stdout
# redirect output to /dev/null if None given
devnull = open(os.devnull, 'wb')
stdout = stdout or devnull
stderr = stderr or devnull
ret_env = {}
if env is not None:
for key, value in six.iteritems(env):
env[key] = str(value)
perf_timer = util.PerfTimer()
p = subprocess.Popen(
command,
stdin=stdin,
stdout=stdout,
stderr=forward_stderr and subprocess.STDOUT or stderr,
shell=True,
close_fds=True,
universal_newlines=True,
env=env,
cwd=tempcwd(),
preexec_fn=os.setpgrp,
)
kill_timer = None
if real_time_limit:
def oot_killer():
ret_env['real_time_killed'] = True
os.killpg(p.pid, signal.SIGKILL)
kill_timer = Timer(ms2s(real_time_limit), oot_killer)
kill_timer.start()
rc = p.wait()
ret_env['return_code'] = rc
if kill_timer:
kill_timer.cancel()
ret_env['real_time_used'] = s2ms(perf_timer.elapsed)
logger.debug(
'Command "%s" exited with code %d, took %.2fs',
str(command),
rc,
perf_timer.elapsed,
)
devnull.close()
if capture_output:
stdout.seek(0)
ret_env['stdout'] = stdout.read(output_limit or -1)
stdout.close()
if split_lines:
ret_env['stdout'] = ret_env['stdout'].split(b'\n')
if rc and not ignore_errors and rc not in extra_ignore_errors:
raise ExecError(
'Failed to execute command: %s. Returned with code %s\n' %
(command, rc))
return ret_env
def _execute(self, command, **kwargs):
env = kwargs.get('env')
env.update({
'MEM_LIMIT': kwargs['mem_limit'] or 64 * 2**10,
'TIME_LIMIT': kwargs['time_limit'] or 30000,
'OUT_LIMIT': kwargs['output_limit'] or 50 * 2**20,
})
if kwargs['real_time_limit']:
env['HARD_LIMIT'] = 1 + ceil_ms2s(kwargs['real_time_limit'])
elif kwargs['time_limit'] and kwargs['real_time_limit'] is None:
env['HARD_LIMIT'] = 1 + ceil_ms2s(64 * kwargs['time_limit'])
if 'HARD_LIMIT' in env:
# Limiting outside supervisor
kwargs['real_time_limit'] = 2 * s2ms(env['HARD_LIMIT'])
ignore_errors = kwargs.pop('ignore_errors')
extra_ignore_errors = kwargs.pop('extra_ignore_errors')
renv = {}
try:
result_file = tempfile.NamedTemporaryFile(dir=tempcwd())
kwargs['ignore_errors'] = True
renv = execute_command(command + [noquote('3>'), result_file.name],
**kwargs)
if 'real_time_killed' in renv:
raise ExecError('Supervisor exceeded realtime limit')
elif renv['return_code'] and renv[
'return_code'] not in extra_ignore_errors:
raise ExecError('Supervisor returned code %s' %
renv['return_code'])
result_file.seek(0)
status_line = result_file.readline().strip().split()[1:]
renv['result_string'] = result_file.readline().strip()
result_file.close()
for num, key in enumerate(('result_code', 'time_used', None,
'mem_used', 'num_syscalls')):
if key:
renv[key] = int(status_line[num])
result_code = self._supervisor_result_to_code(renv['result_code'])
except Exception as e:
logger.error('SupervisedExecutor error: %s',
traceback.format_exc())
logger.error(
'SupervisedExecutor error dirlist: %s: %s',
tempcwd(),
str(os.listdir(tempcwd())),
)
result_code = 'SE'
for i in ('time_used', 'mem_used', 'num_syscalls'):
renv.setdefault(i, 0)
renv['result_string'] = str(e)
renv['result_code'] = result_code
if (result_code != 'OK' and not ignore_errors
and not (result_code != 'RV'
and renv['return_code'] in extra_ignore_errors)):
raise ExecError('Failed to execute command: %s. Reason: %s' %
(command, renv['result_string']))
return renv
