def _execute(self, command, **kwargs): env = kwargs.get("env") env.update( { "MEM_LIMIT": kwargs["mem_limit"] or 64 * 2 ** 10, "TIME_LIMIT": kwargs["time_limit"] or 30000, "OUT_LIMIT": kwargs["output_limit"] or 50 * 2 ** 20, } ) if kwargs["real_time_limit"]: env["HARD_LIMIT"] = 1 + ceil_ms2s(kwargs["real_time_limit"]) elif kwargs["time_limit"] and kwargs["real_time_limit"] is None: env["HARD_LIMIT"] = 1 + ceil_ms2s(64 * kwargs["time_limit"]) if "HARD_LIMIT" in env: # Limiting outside supervisor kwargs["real_time_limit"] = 2 * s2ms(env["HARD_LIMIT"]) ignore_errors = kwargs.pop("ignore_errors") extra_ignore_errors = kwargs.pop("extra_ignore_errors") renv = {} try: result_file = tempfile.NamedTemporaryFile(dir=os.getcwd()) kwargs["ignore_errors"] = True renv = execute_command(command + [noquote("3>"), result_file.name], **kwargs) if "real_time_killed" in renv: raise ExecError("Supervisor exceeded realtime limit") elif renv["return_code"]: raise ExecError("Supervisor returned code %s" % renv["return_code"]) result_file.seek(0) status_line = result_file.readline().strip().split()[1:] renv["result_string"] = result_file.readline().strip() result_file.close() for num, key in enumerate(("result_code", "time_used", None, "mem_used", "num_syscalls")): if key: renv[key] = int(status_line[num]) result_code = self._supervisor_result_to_code(renv["result_code"]) except Exception as e: logger.error("SupervisedExecutor error: %s", traceback.format_exc()) logger.error("SupervisedExecutor error dirlist: %s: %s", os.getcwd(), str(os.listdir("."))) result_code = "SE" for i in ("time_used", "mem_used", "num_syscalls"): renv.setdefault(i, 0) renv["result_string"] = str(e) renv["result_code"] = result_code if result_code != "OK" and not ignore_errors and result_code not in extra_ignore_errors: raise ExecError("Failed to execute command: %s. Reason: %s" % (command, renv["result_string"])) return renv
def execute_command(command, env=None, split_lines=False, stdin=None, stdout=None, stderr=None, forward_stderr=False, capture_output=False, output_limit=None, real_time_limit=None, ignore_errors=False, extra_ignore_errors=(), **kwargs): """Utility function to run arbitrary command. ``stdin`` Could be either file opened with ``open(fname, 'r')`` or None (then it is inherited from current process). ``stdout``, ``stderr`` Could be files opened with ``open(fname, 'w')``, sys.std* or None - then it's suppressed. ``forward_stderr`` Forwards stderr to stdout. ``capture_output`` Returns program output in renv key ``stdout``. ``output_limit`` Limits returned output when ``capture_output=True`` (in bytes). Returns renv: dictionary containing: ``real_time_used`` Wall clock time it took to execute the command (in ms). ``return_code`` Status code that program returned. ``real_time_killed`` Only when process was killed due to exceeding real time limit. ``stdout`` Only when ``capture_output=True``: output of the command """ # Using temporary file is way faster than using subproces.PIPE # and it prevents deadlocks. command = shellquote(command) logger.debug('Executing: %s', command) stdout = capture_output and tempfile.TemporaryFile() or stdout # redirect output to /dev/null if None given devnull = open(os.devnull, 'wb') stdout = stdout or devnull stderr = stderr or devnull ret_env = {} if env is not None: for key, value in env.iteritems(): env[key] = str(value) perf_timer = util.PerfTimer() p = subprocess.Popen(command, stdin=stdin, stdout=stdout, stderr=forward_stderr and subprocess.STDOUT or stderr, shell=True, close_fds=True, universal_newlines=True, env=env, preexec_fn=os.setpgrp) kill_timer = None if real_time_limit: def oot_killer(): ret_env['real_time_killed'] = True os.killpg(p.pid, signal.SIGKILL) kill_timer = Timer(ms2s(real_time_limit), oot_killer) kill_timer.start() rc = p.wait() ret_env['return_code'] = rc if kill_timer: kill_timer.cancel() ret_env['real_time_used'] = s2ms(perf_timer.elapsed) logger.debug('Command "%s" exited with code %d, took %.2fs', str(command), rc, perf_timer.elapsed) devnull.close() if capture_output: stdout.seek(0) ret_env['stdout'] = stdout.read(output_limit or -1) stdout.close() if split_lines: ret_env['stdout'] = ret_env['stdout'].split('\n') if rc and not ignore_errors and rc not in extra_ignore_errors: raise ExecError('Failed to execute command: %s. Returned with code %s\n' % (command, rc)) return ret_env
def _execute(self, command, **kwargs): env = kwargs.get('env') env.update({ 'MEM_LIMIT': kwargs['mem_limit'] or 64 * 2**10, 'TIME_LIMIT': kwargs['time_limit'] or 30000, 'OUT_LIMIT': kwargs['output_limit'] or 50 * 2**20, }) if kwargs['real_time_limit']: env['HARD_LIMIT'] = 1 + ceil_ms2s(kwargs['real_time_limit']) elif kwargs['time_limit'] and kwargs['real_time_limit'] is None: env['HARD_LIMIT'] = 1 + ceil_ms2s(64 * kwargs['time_limit']) if 'HARD_LIMIT' in env: # Limiting outside supervisor kwargs['real_time_limit'] = 2 * s2ms(env['HARD_LIMIT']) ignore_errors = kwargs.pop('ignore_errors') extra_ignore_errors = kwargs.pop('extra_ignore_errors') renv = {} try: result_file = tempfile.NamedTemporaryFile(dir=os.getcwd()) kwargs['ignore_errors'] = True renv = execute_command( command + [noquote('3>'), result_file.name], **kwargs ) if 'real_time_killed' in renv: raise ExecError('Supervisor exceeded realtime limit') elif renv['return_code'] and \ renv['return_code'] not in extra_ignore_errors: raise ExecError('Supervisor returned code %s' % renv['return_code']) result_file.seek(0) status_line = result_file.readline().strip().split()[1:] renv['result_string'] = result_file.readline().strip() result_file.close() for num, key in enumerate(('result_code', 'time_used', None, 'mem_used', 'num_syscalls')): if key: renv[key] = int(status_line[num]) result_code = self._supervisor_result_to_code(renv['result_code']) except Exception as e: logger.error('SupervisedExecutor error: %s', traceback.format_exc()) logger.error('SupervisedExecutor error dirlist: %s: %s', os.getcwd(), str(os.listdir('.'))) result_code = 'SE' for i in ('time_used', 'mem_used', 'num_syscalls'): renv.setdefault(i, 0) renv['result_string'] = str(e) renv['result_code'] = result_code if result_code != 'OK' and not ignore_errors and not \ (result_code != 'RV' and renv['return_code'] in \ extra_ignore_errors): raise ExecError('Failed to execute command: %s. Reason: %s' % (command, renv['result_string'])) return renv
def execute_command(command, env=None, split_lines=False, stdin=None, stdout=None, stderr=None, forward_stderr=False, capture_output=False, output_limit=None, real_time_limit=None, ignore_errors=False, extra_ignore_errors=(), **kwargs): """Utility function to run arbitrary command. ``stdin`` Could be either file opened with ``open(fname, 'r')`` or None (then it is inherited from current process). ``stdout``, ``stderr`` Could be files opened with ``open(fname, 'w')``, sys.std* or None - then it's suppressed. ``forward_stderr`` Forwards stderr to stdout. ``capture_output`` Returns program output in renv key ``stdout``. ``output_limit`` Limits returned output when ``capture_output=True`` (in bytes). Returns renv: dictionary containing: ``real_time_used`` Wall clock time it took to execute the command (in ms). ``return_code`` Status code that program returned. ``real_time_killed`` Only when process was killed due to exceeding real time limit. ``stdout`` Only when ``capture_output=True``: output of the command """ # Using temporary file is way faster than using subproces.PIPE # and it prevents deadlocks. command = shellquote(command) logger.debug('Executing: %s', command) stdout = capture_output and tempfile.TemporaryFile() or stdout # redirect output to /dev/null if None given devnull = open(os.devnull, 'wb') stdout = stdout or devnull stderr = stderr or devnull ret_env = {} if env is not None: for key, value in six.iteritems(env): env[key] = str(value) perf_timer = util.PerfTimer() p = subprocess.Popen( command, stdin=stdin, stdout=stdout, stderr=forward_stderr and subprocess.STDOUT or stderr, shell=True, close_fds=True, universal_newlines=True, env=env, cwd=tempcwd(), preexec_fn=os.setpgrp, ) kill_timer = None if real_time_limit: def oot_killer(): ret_env['real_time_killed'] = True os.killpg(p.pid, signal.SIGKILL) kill_timer = Timer(ms2s(real_time_limit), oot_killer) kill_timer.start() rc = p.wait() ret_env['return_code'] = rc if kill_timer: kill_timer.cancel() ret_env['real_time_used'] = s2ms(perf_timer.elapsed) logger.debug( 'Command "%s" exited with code %d, took %.2fs', str(command), rc, perf_timer.elapsed, ) devnull.close() if capture_output: stdout.seek(0) ret_env['stdout'] = stdout.read(output_limit or -1) stdout.close() if split_lines: ret_env['stdout'] = ret_env['stdout'].split(b'\n') if rc and not ignore_errors and rc not in extra_ignore_errors: raise ExecError( 'Failed to execute command: %s. Returned with code %s\n' % (command, rc)) return ret_env
def _execute(self, command, **kwargs): env = kwargs.get('env') env.update({ 'MEM_LIMIT': kwargs['mem_limit'] or 64 * 2**10, 'TIME_LIMIT': kwargs['time_limit'] or 30000, 'OUT_LIMIT': kwargs['output_limit'] or 50 * 2**20, }) if kwargs['real_time_limit']: env['HARD_LIMIT'] = 1 + ceil_ms2s(kwargs['real_time_limit']) elif kwargs['time_limit'] and kwargs['real_time_limit'] is None: env['HARD_LIMIT'] = 1 + ceil_ms2s(64 * kwargs['time_limit']) if 'HARD_LIMIT' in env: # Limiting outside supervisor kwargs['real_time_limit'] = 2 * s2ms(env['HARD_LIMIT']) ignore_errors = kwargs.pop('ignore_errors') extra_ignore_errors = kwargs.pop('extra_ignore_errors') renv = {} try: result_file = tempfile.NamedTemporaryFile(dir=tempcwd()) kwargs['ignore_errors'] = True renv = execute_command(command + [noquote('3>'), result_file.name], **kwargs) if 'real_time_killed' in renv: raise ExecError('Supervisor exceeded realtime limit') elif renv['return_code'] and renv[ 'return_code'] not in extra_ignore_errors: raise ExecError('Supervisor returned code %s' % renv['return_code']) result_file.seek(0) status_line = result_file.readline().strip().split()[1:] renv['result_string'] = result_file.readline().strip() result_file.close() for num, key in enumerate(('result_code', 'time_used', None, 'mem_used', 'num_syscalls')): if key: renv[key] = int(status_line[num]) result_code = self._supervisor_result_to_code(renv['result_code']) except Exception as e: logger.error('SupervisedExecutor error: %s', traceback.format_exc()) logger.error( 'SupervisedExecutor error dirlist: %s: %s', tempcwd(), str(os.listdir(tempcwd())), ) result_code = 'SE' for i in ('time_used', 'mem_used', 'num_syscalls'): renv.setdefault(i, 0) renv['result_string'] = str(e) renv['result_code'] = result_code if (result_code != 'OK' and not ignore_errors and not (result_code != 'RV' and renv['return_code'] in extra_ignore_errors)): raise ExecError('Failed to execute command: %s. Reason: %s' % (command, renv['result_string'])) return renv