def run_process(*popenargs, **kwargs): """ A straight copy-paste of subprocess.run() from the CPython source to support Python versions earlier than 3.5. """ # Can't put these directly in function signature because PEP-3102 is # not a thing in Python 2 input = kwargs.pop('input', None) timeout = kwargs.pop('timeout', None) check = kwargs.pop('check', False) if input is not None: if 'stdin' in kwargs: raise ValueError('stdin and input arguments may not both be used.') kwargs['stdin'] = subprocess.PIPE with subprocess.Popen(*popenargs, **kwargs) as process: try: stdout, stderr = process.communicate(input, timeout=timeout) except subprocess.TimeoutExpired: process.kill() stdout, stderr = process.communicate() raise subprocess.TimeoutExpired(process.args, timeout, output=stdout, stderr=stderr) except: # noqa process.kill() process.wait() raise retcode = process.poll() if check and retcode: raise subprocess.CalledProcessError(retcode, process.args, output=stdout, stderr=stderr) return CompletedProcess(process.args, retcode, stdout, stderr)
def test_execute_job_run_pb_process_timeout_expired(self): # create job template job_template = JobTemplate( job_template_type='device', job_template_job_runtime='ansible', job_template_multi_device_job=False, job_template_playbooks=TestJobManagerUtils.playbooks_list, name='run_pb_prc_rc_timeout') job_template_uuid = self._vnc_lib.job_template_create(job_template) TestJobManagerUtils.mock_sandesh_check() job_input_json, log_utils = TestJobManagerUtils.get_min_details( job_template_uuid) wm = WFManager(log_utils.get_config_logger(), self._vnc_lib, job_input_json, log_utils) loop_var = 0 def mock_readline(): global loop_var loop_var += 1 if loop_var == 1: with open("tests/test.txt", 'r') as f: line = f.readline() return line if loop_var == 2: fake_process.should_receive("poll").and_return(123) loop_var = 0 return "" stdout = flexmock(readline=mock_readline) flexmock(json).should_receive("loads") flexmock(os.path).should_receive('exists').and_return(True) flexmock(os).should_receive('kill') # mock the call to raise exception, using return code = 1 fake_process = flexmock(returncode=1, pid=1234, stdout=stdout) exc = subprocess32.TimeoutExpired(cmd='Mock timeout exc cmd', timeout=3600) fake_process.should_receive('poll').and_return(123) fake_process.should_receive('wait').and_raise(exc) flexmock(subprocess32).should_receive('Popen').and_return( fake_process) sys_exit_msg = self.assertRaises(SystemExit, wm.start_job) self.assertEqual( sys_exit_msg.code, MsgBundle.getMessage(MsgBundle.JOB_SUMMARY_MESSAGE_HDR) + MsgBundle.getMessage(MsgBundle. JOB_SINGLE_DEVICE_FAILED_MESSAGE_HDR) + MsgBundle.getMessage(MsgBundle.RUN_PLAYBOOK_PROCESS_TIMEOUT, playbook_uri=TestJobManagerUtils.play_info. playbook_uri, exc_msg=repr(exc)))
def test_health_check_timeout(self, mock_popen): # Fail due to command returning a non-0 exit status. mock_popen.side_effect = subprocess.TimeoutExpired('failed', timeout=30) shell = ShellHealthCheck('cmd', timeout_secs=30) success, msg = shell() mock_popen.assert_called_once_with('cmd', shell=True, timeout=30, preexec_fn=mock.ANY) self.assertFalse(success) self.assertEqual(msg, 'Health check timed out.')
def test_stop_timeout_and_terminate(self): self._create_server() self.server.run_command = mock.MagicMock() pipe = mock.MagicMock() self.server.pipe = pipe self.server.pipe.wait = mock.MagicMock(side_effect=subprocess.TimeoutExpired('cmd', 1)) self.server.stop() assert pipe.terminate.call_count == 1
def test_stop_timeout_and_terminate_with_connection(self): self._create_server() self.server.run_command = mock.MagicMock() pipe = mock.MagicMock() self.server.pipe = pipe self.server.pipe.wait = mock.MagicMock(side_effect=subprocess.TimeoutExpired('cmd', 1)) mock_connection = mock.MagicMock() self.server.stop(mock_connection) assert mock_connection.send_message.call_count == 2 assert mock_connection.send_message.call_args_list[0][0] == ( 'Stopping Minecraft server "name"...', ) assert mock_connection.send_message.call_args_list[1][0] == ( 'Server "name" did not stop within %s seconds. Killing...' % SERVER_STOP_TIMEOUT_SEC, ) assert pipe.terminate.call_count == 1
def run_in_subprocess(cmd, check_output=False, **kwargs): """ Execute command using default subprocess configuration. Parameters ---------- cmd : string Command to be executed in subprocess. kwargs : keywords Options to pass to subprocess.Popen. Returns ------- proc : Popen subprocess Subprocess used to run command. """ logger.debug('Executing command: {0}'.format(cmd)) config = DEFAULT_SUBPROCESS_CONFIG.copy() config.update(kwargs) if not check_output: if omniduct_config.logging_level < 20: config['stdout'] = None config['stderr'] = None else: config['stdout'] = open(os.devnull, 'w') config['stderr'] = open(os.devnull, 'w') timeout = config.pop('timeout', None) process = subprocess.Popen(cmd, **config) try: stdout, stderr = process.communicate(None, timeout=timeout) except subprocess.TimeoutExpired: os.killpg( os.getpgid(process.pid), signal.SIGINT ) # send signal to the process group, recurively killing all children output, unused_err = process.communicate() raise subprocess.TimeoutExpired(process.args, timeout, output=output) return SubprocessResults(returncode=process.returncode, stdout=stdout or '', stderr=stderr or '')
def handle_subprocess_pipe_with_timeout(pipe, timeout, input=None): if (not input is None) and not pipe.stdin: raise Exception('pipe.stdin must be a stream if you pass in input') stdout = '' stderr = '' kwargs = dict(input=input) if timeout: kwargs['timeout'] = timeout try: stdout, stderr = pipe.communicate(**kwargs) except TimeoutExpired: pipe.kill() try: # Other solution to handle timeout for shell=True: http://stackoverflow.com/a/4791612/633961 # related: http://stackoverflow.com/questions/36592068/subprocess-with-timeout-what-to-do-after-timeoutexpired-exception stdout, stderr = pipe.communicate(timeout=0.1) except subprocess.TimeoutExpired: pass raise subprocess.TimeoutExpired( pipe.args, timeout, 'stdout: %r stderr: %r' % (stdout, stderr)) return (stdout, stderr)
def test_execute_job_run_pb_process_timeout_expired(self): # create job template job_template = JobTemplate( job_template_type='device', job_template_job_runtime='ansible', job_template_multi_device_job=False, job_template_playbooks=TestJobManagerUtils. playbooks_list, name='run_pb_prc_rc_timeout') job_template_uuid = self._vnc_lib.job_template_create(job_template) TestJobManagerUtils.mock_sandesh_check() job_input_json, log_utils = TestJobManagerUtils.get_min_details( job_template_uuid) jm = JobManager(log_utils.get_config_logger(), self._vnc_lib, job_input_json, log_utils) flexmock(os.path).should_receive('exists').and_return(True) flexmock(os).should_receive('kill') # mock the call to raise exception fake_process = flexmock(returncode=1, pid=1234) e = subprocess32.TimeoutExpired(cmd='Mock timeout exc cmd', timeout=3600) fake_process.should_receive('wait').and_raise(e) flexmock(subprocess32).should_receive('Popen').and_return(fake_process) sys_exit_msg = self.assertRaises(SystemExit, jm.start_job) self.assertEqual( sys_exit_msg.code, MsgBundle.getMessage(MsgBundle.JOB_SUMMARY_MESSAGE_HDR) + MsgBundle.getMessage(MsgBundle. JOB_SINGLE_DEVICE_FAILED_MESSAGE_HDR) + MsgBundle.getMessage(MsgBundle.RUN_PLAYBOOK_PROCESS_TIMEOUT, playbook_uri=TestJobManagerUtils.play_info. playbook_uri, exc_msg=repr(e)))
def local_execute(command, timeout=60, ignore_return_codes=None): """ Execute a command on local machine. Returns combined stdout and stderr if return code is 0 or included in the list 'ignore_return_codes'. Otherwise raises a subprocess32 error. """ process = subprocess32.Popen(command, universal_newlines=True, stdout=subprocess32.PIPE, stderr=subprocess32.STDOUT) # Loop until process returns or timeout expires. start = time.time() output = "" return_code = None while time.time() < start + timeout and return_code == None: return_code = process.poll() if return_code == None: try: output += process.communicate(timeout=1)[0] except subprocess32.TimeoutExpired: pass if return_code == None: # Time ran out but the process didn't end. raise subprocess32.TimeoutExpired(cmd=command, output=output, timeout=timeout) if ignore_return_codes == None: ignore_return_codes = [] if return_code in ignore_return_codes or return_code == 0: return output else: raise subprocess32.CalledProcessError(returncode=return_code, cmd=command, output=output)
def run_duplicate_streams(cmd, timeout=_default_timeout()): """ <Purpose> Provide a function that executes a command in a subprocess and, upon termination, returns its exit code and the contents of what was printed to its standard streams. * Might behave unexpectedly with interactive commands. * Might not duplicate output in real time, if the command buffers it (see e.g. `print("foo")` vs. `print("foo", flush=True)` in Python 3). <Arguments> cmd: The command and its arguments. (list of str, or str) Splits a string specifying a command and its argument into a list of substrings, if necessary. timeout: (default see settings.SUBPROCESS_TIMEOUT) If the timeout expires, the child process will be killed and waited for and then subprocess.TimeoutExpired will be raised. <Exceptions> securesystemslib.exceptions.FormatError: If the `cmd` is a list and does not match securesystemslib.formats.LIST_OF_ANY_STRING_SCHEMA. OSError: If the given command is not present or non-executable. subprocess.TimeoutExpired: If the process does not terminate after timeout seconds. Default is `settings.SUBPROCESS_TIMEOUT` <Side Effects> The side effects of executing the given command in this environment. <Returns> A tuple of command's exit code, standard output and standard error contents. """ if isinstance(cmd, six.string_types): cmd = shlex.split(cmd) else: formats.LIST_OF_ANY_STRING_SCHEMA.check_match(cmd) # Use temporary files as targets for child process standard stream redirects # They seem to work better (i.e. do not hang) than pipes, when using # interactive commands like `vi`. stdout_fd, stdout_name = tempfile.mkstemp() stderr_fd, stderr_name = tempfile.mkstemp() try: with io.open(stdout_name, "r") as stdout_reader, \ os.fdopen(stdout_fd, "w") as stdout_writer, \ io.open(stderr_name, "r") as stderr_reader, \ os.fdopen(stderr_fd, "w") as stderr_writer: # Store stream results in mutable dict to update it inside nested helper _std = {"out": "", "err": ""} def _duplicate_streams(): """Helper to read from child process standard streams, write their contents to parent process standard streams, and build up return values for outer function. """ # Read until EOF but at most `io.DEFAULT_BUFFER_SIZE` bytes per call. # Reading and writing in reasonably sized chunks prevents us from # subverting a timeout, due to being busy for too long or indefinitely. stdout_part = stdout_reader.read(io.DEFAULT_BUFFER_SIZE) stderr_part = stderr_reader.read(io.DEFAULT_BUFFER_SIZE) sys.stdout.write(stdout_part) sys.stderr.write(stderr_part) sys.stdout.flush() sys.stderr.flush() _std["out"] += stdout_part _std["err"] += stderr_part # Start child process, writing its standard streams to temporary files proc = subprocess.Popen(cmd, stdout=stdout_writer, stderr=stderr_writer, universal_newlines=True) proc_start_time = time.time() # Duplicate streams until the process exits (or times out) while proc.poll() is None: # Time out as Python's `subprocess` would do it if (timeout is not None and time.time() > proc_start_time + timeout): proc.kill() proc.wait() raise subprocess.TimeoutExpired(cmd, timeout) _duplicate_streams() # Read/write once more to grab everything that the process wrote between # our last read in the loop and exiting, i.e. breaking the loop. _duplicate_streams() finally: # The work is done or was interrupted, the temp files can be removed os.remove(stdout_name) os.remove(stderr_name) # Return process exit code and captured streams return proc.poll(), _std["out"], _std["err"]
def judge_func(self, user): while self.judging: result = 'JE' elapsed = 0.0 user_id = None try: subm_id, log = self.queue.get(timeout=1) self.logger.info("%s: judging submission %d (%s, %s, %s, %s)" % (user, subm_id, log['prob_id'], log['source_name'], log['lang'], log['user_id'])) self.logger.debug("%s: log=%s" % (user, str(log))) self.contest.change_submission(subm_id, result='CJ') user_id = log['user_id'] subprocess.call('chdir "%s"; rm -f *.class; rm -f a.out; rm -f *.pyc' % log['path'], shell=True) error_log = None if log['lang'] in Judge.lang_compile: compile_cmd = Judge.lang_compile[log['lang']] + [log['source_name']] self.logger.debug("%s: %s" % (user, ' '.join(compile_cmd))) compiler = subprocess.Popen('cd "%s"; %s' % (log['path'], ' '.join(compile_cmd)), shell=True, stderr=subprocess.PIPE) try: stderr_data = compiler.communicate(timeout=60)[1] if compiler.returncode != 0: result = 'CE' self.logger.debug("%s: compile returned non-zero exit status" % user) error_log = stderr_data with open(os.path.join(log['path'], 'compile_errors.txt'), 'w') as out_file: out_file.write(stderr_data) raise AssertionError() except subprocess.TimeoutExpired: compiler.kill() compiler.communicate() elapsed = 60 result = 'CE' self.logger.debug("%s: compile took longer than 60 seconds" % user) error_log = 'Exceeded max time allowed (60 seconds) for compiling.' with open(os.path.join(log['path'], 'compile_errors.txt'), 'w') as out_file: out_file.write("Exceeded max time allowed (60 seconds) for compiling.") raise AssertionError() prob = self.problems[log['prob_id']] run_cmd = ['time', '--portability'] + Judge.lang_run[log['lang']] if 'Java' in log['lang']: run_cmd.append(log['source_name'][:-5]) elif 'Python' in log['lang']: run_cmd.append(log['source_name']) docker_name = str(uuid.uuid4()) docker_cmd = ['docker', 'run', '-i', '--name="%s"' % (docker_name,), '--rm=true', '--net="none"', '--cpu-shares=128', '-m="%dm"' % (prob.mem_limit,), '--read-only', '-v', '"%s":/judging_dir:ro' % (os.path.abspath(log['path']),), '-w', '/judging_dir', '-u', user, 'chenclee/sandbox', ' '.join(run_cmd)] self.logger.debug("%s: %s: " % (user, ' '.join(docker_cmd))) runner = subprocess.Popen(shlex.split(' '.join(docker_cmd)), stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE) try: ran_to_completion = [True] def timeout_func(): self.logger.debug("%s: time limit exceeded; killing docker container" % (user,)) ran_to_completion[0] = False try: subprocess.call('docker rm -f %s' % (docker_name,), shell=True) except: pass timer = threading.Timer(prob.time_limit * 4, timeout_func) timer.start() stdout_data, stderr_data = runner.communicate(input=prob.input_text) timer.cancel() ran_to_completion = ran_to_completion[0] with open(os.path.join(log['path'], 'runtime_output.txt'), 'w') as out_file: out_file.write(stdout_data) with open(os.path.join(log['path'], 'runtime_errors.txt'), 'w') as out_file: out_file.write(stderr_data) if ran_to_completion: self.logger.debug("%s: program ran to completion" % (user,)) stderr_lines = stderr_data.splitlines() if runner.returncode != 0: if 'read unix /var/run/docker.sock' in stderr_lines[-1]: stderr_lines = stderr_lines[:-1] result = 'RE' if stderr_lines[0].strip() != 'Command terminated by signal 9' else 'ML' raise AssertionError() time_matches = [re.search('(\d+\.\d{2})', s) for s in stderr_lines[-2:]] elapsed = sum([float(time_match.group(0)) for time_match in time_matches]) if elapsed > prob.time_limit: self.logger.debug("%s: user+sys time exceeds time limit" % (user,)) raise subprocess.TimeoutExpired( cmd=' '.join(docker_cmd), timeout=prob.time_limit, output=None) actual = [line.strip() for line in stdout_data.splitlines() if line.strip() != ''] expected = [line.strip() for line in prob.output_text.splitlines() if line.strip() != ''] self.logger.debug("%s: actual=%s" % (user, str(actual[:10]))) self.logger.debug("%s: expected=%s" % (user, str(expected[:10]))) if actual == expected: result = 'AC' else: result = 'WA' else: self.logger.debug("%s: program exceeded time limit and was terminated" % (user,)) raise subprocess.TimeoutExpired( cmd=' '.join(docker_cmd), timeout=prob.time_limit, output=None) except subprocess.TimeoutExpired: elapsed = self.problems[log['prob_id']].time_limit result = 'TL' raise AssertionError() except Queue.Empty: continue except AssertionError: pass except Exception as e: self.logger.error("%s: %s" % (user, traceback.format_exception(*sys.exc_info()))) result = 'JE' self.logger.info("%s: result for submission %d is %s" % (user, subm_id, Contest.verdicts[result])) self.contest.change_submission(subm_id, result=result, run_time=elapsed, error_log=error_log) if user_id: self.in_queue[user_id] -= 1 self.queue.task_done() subprocess.call('chdir "%s"; rm -f *.class; rm -f a.out' % log['path'], shell=True) self.logger.info(user + " halted")
def communicate(self, timeout=None): if self._will_timeout: raise subprocess.TimeoutExpired( -1, 'Timed out according to test logic') return self._stdout, self._stderr
def do_capture(status_code, the_record, base_url, model='capture', phantomjs_timeout=app.config['PHANTOMJS_TIMEOUT']): """ Create a screenshot, text scrape, from a provided html file. This depends on phantomjs and an associated javascript file to perform the captures. In the event an error occurs, an exception is raised and handled by the celery task or the controller that called this method. """ # Make sure the the_record db.session.add(the_record) # If the capture is for static content, use a differnet PhantomJS config file if model == 'static': capture_name = the_record.filename service_args = [ app.config['PHANTOMJS'], '--ssl-protocol=any', '--ignore-ssl-errors=yes', os.path.dirname(os.path.dirname(os.path.abspath(__file__))) + '/assets/static.js', app.config['LOCAL_STORAGE_FOLDER'], capture_name ] content_to_parse = os.path.join(app.config['LOCAL_STORAGE_FOLDER'], capture_name) else: capture_name = grab_domain(the_record.url) + '_' + str(the_record.id) service_args = [ app.config['PHANTOMJS'], '--ssl-protocol=any', '--ignore-ssl-errors=yes', os.path.dirname(os.path.dirname(os.path.abspath(__file__))) + '/assets/capture.js', the_record.url, os.path.join(app.config['LOCAL_STORAGE_FOLDER'], capture_name) ] content_to_parse = os.path.join(app.config['LOCAL_STORAGE_FOLDER'], capture_name + '.html') # Using subprocess32 backport, call phantom and if process hangs kill it pid = subprocess32.Popen(service_args, stdout=PIPE, stderr=PIPE) try: stdout, stderr = pid.communicate(timeout=phantomjs_timeout) except subprocess32.TimeoutExpired: pid.kill() stdout, stderr = pid.communicate() app.logger.error('PhantomJS Capture timeout at {} seconds'.format( phantomjs_timeout)) raise subprocess32.TimeoutExpired('phantomjs capture', phantomjs_timeout) # If the subprocess has an error, raise an exception if stderr or stdout: raise Exception("{}{}".format(stdout, stderr)) # Strip tags and parse out all text ignore_tags = ('script', 'noscript', 'style') with open(content_to_parse, 'r') as content_file: content = content_file.read() cleaner = clean.Cleaner() content = cleaner.clean_html(content) doc = LH.fromstring(content) output = "" for elt in doc.iterdescendants(): if elt.tag in ignore_tags: continue text = elt.text or '' tail = elt.tail or '' wordz = " ".join((text, tail)).strip('\t') if wordz and len(wordz) >= 2 and not re.match("^[ \t\n]*$", wordz): output += wordz.encode('utf-8') # Since the filename format is different for static captures, update the filename # This will ensure the URLs are pointing to the correct resources if model == 'static': capture_name = capture_name.split('.')[0] # Wite our html text that was parsed into our capture folder parsed_text = open( os.path.join(app.config['LOCAL_STORAGE_FOLDER'], capture_name + '.txt'), 'wb') parsed_text.write(output) # Update the sketch record with the local URLs for the sketch, scrape, and html captures the_record.sketch_url = base_url + '/files/' + capture_name + '.png' the_record.scrape_url = base_url + '/files/' + capture_name + '.txt' the_record.html_url = base_url + '/files/' + capture_name + '.html' # Create a dict that contains what files may need to be written to S3 files_to_write = defaultdict(list) files_to_write['sketch'] = capture_name + '.png' files_to_write['scrape'] = capture_name + '.txt' files_to_write['html'] = capture_name + '.html' # If we are not writing to S3, update the capture_status that we are completed. if not app.config['USE_S3']: the_record.job_status = "COMPLETED" the_record.capture_status = "LOCAL_CAPTURES_CREATED" else: the_record.capture_status = "LOCAL_CAPTURES_CREATED" db.session.commit() return files_to_write
def test_test_connection__time_out__return_false(self, mock_popen): mock_popen.side_effect = subprocess.TimeoutExpired("echo hi", 1) res = self.con.test_connection() self.assertFalse(res)
def run_duplicate_streams(cmd, timeout=SUBPROCESS_TIMEOUT): """ <Purpose> Provide a function that executes a command in a subprocess and returns its exit code and the contents of what it printed to its standard streams upon termination. NOTE: The function might behave unexpectedly with interactive commands. <Arguments> cmd: The command and its arguments. (list of str, or str) Splits a string specifying a command and its argument into a list of substrings, if necessary. timeout: (default see settings.SUBPROCESS_TIMEOUT) If the timeout expires, the child process will be killed and waited for and then subprocess.TimeoutExpired will be raised. <Exceptions> securesystemslib.exceptions.FormatError: If the `cmd` is a list and does not match securesystemslib.formats.LIST_OF_ANY_STRING_SCHEMA. OSError: If the given command is not present or non-executable. subprocess.TimeoutExpired: If the process does not terminate after timeout seconds. Default is `settings.SUBPROCESS_TIMEOUT` <Side Effects> The side effects of executing the given command in this environment. <Returns> A tuple of command's exit code, standard output and standard error contents. """ if isinstance(cmd, six.string_types): cmd = shlex.split(cmd) else: securesystemslib.formats.LIST_OF_ANY_STRING_SCHEMA.check_match(cmd) # Use temporary files as targets for child process standard stream redirects # They seem to work better (i.e. do not hang) than pipes, when using # interactive commands like `vi`. stdout_fd, stdout_name = tempfile.mkstemp() stderr_fd, stderr_name = tempfile.mkstemp() try: with io.open(stdout_name, "r") as stdout_reader, \ os.fdopen(stdout_fd, "w") as stdout_writer, \ io.open(stderr_name, "r") as stderr_reader, \ os.fdopen(stderr_fd, "w") as stderr_writer: # Start child , writing standard streams to temporary files proc = subprocess.Popen(cmd, stdout=stdout_writer, stderr=stderr_writer, universal_newlines=True) proc_start_time = time.time() stdout_str = stderr_str = "" stdout_part = stderr_part = "" # Read as long as the process runs or there is data on one of the streams while proc.poll() is None or stdout_part or stderr_part: # Raise timeout error in they same manner as `subprocess` would do it if (timeout is not None and time.time() > proc_start_time + timeout): proc.kill() proc.wait() raise subprocess.TimeoutExpired(cmd, timeout) # Read from child process's redirected streams, write to parent # process's standard streams and construct retuirn values stdout_part = stdout_reader.read() stderr_part = stderr_reader.read() sys.stdout.write(stdout_part) sys.stderr.write(stderr_part) sys.stdout.flush() sys.stderr.flush() stdout_str += stdout_part stderr_str += stderr_part finally: # The work is done or was interrupted, the temp files can be removed os.remove(stdout_name) os.remove(stderr_name) # Return process exit code and captured stream return proc.poll(), stdout_str, stderr_str