def test_Popen_defaults(self, mockPopen): with mock.patch('sys.platform', 'win32'): subprocess2.Popen(['foo'], a=True) mockPopen.assert_called_with(['foo'], a=True, shell=True) with mock.patch('sys.platform', 'non-win32'): subprocess2.Popen(['foo'], a=True) mockPopen.assert_called_with(['foo'], a=True, shell=False)
def testCheckCallAndFilter_RetryOnce(self): cwd = 'bleh' args = ['boo', 'foo', 'bar'] test_string = 'ahah\naccb\nallo\naddb\n✔' # pylint: disable=no-member subprocess2.Popen( args, cwd=cwd, stdout=subprocess2.PIPE, stderr=subprocess2.STDOUT, bufsize=0).AndReturn(self.ProcessIdMock(test_string, 1)) os.getcwd() # pylint: disable=no-member subprocess2.Popen( args, cwd=cwd, stdout=subprocess2.PIPE, stderr=subprocess2.STDOUT, bufsize=0).AndReturn(self.ProcessIdMock(test_string, 0)) self.mox.ReplayAll() line_list = [] result = gclient_utils.CheckCallAndFilter( args, cwd=cwd, show_header=True, always_show_header=True, filter_fn=line_list.append, retry=True) self.assertEqual(result, test_string.encode('utf-8')) self.assertEqual(line_list, [ '________ running \'boo foo bar\' in \'bleh\'\n', 'ahah', 'accb', 'allo', 'addb', '✔', '________ running \'boo foo bar\' in \'bleh\' attempt 2 / 4\n', 'ahah', 'accb', 'allo', 'addb', '✔', ]) self.checkstdout( 'WARNING: subprocess \'"boo" "foo" "bar"\' in bleh failed; will retry ' 'after a short nap...\n')
def test_communicate_returns(self): # communicate() should return None if no redirection is active p = subprocess.Popen([sys.executable, "-c", "import sys; sys.exit(47)"]) (stdout, stderr) = p.communicate() self.assertEqual(stdout, None) self.assertEqual(stderr, None)
def test_communicate_stdout(self): p = subprocess.Popen([sys.executable, "-c", 'import sys; sys.stdout.write("pineapple")'], stdout=subprocess.PIPE) (stdout, stderr) = p.communicate() self.assertEqual(stdout, "pineapple") self.assertEqual(stderr, None)
def test_stderr_pipe(self): # stderr redirection p = subprocess.Popen([sys.executable, "-c", 'import sys; sys.stderr.write("strawberry")'], stderr=subprocess.PIPE) self.assertEqual(remove_stderr_debug_decorations(p.stderr.read()), "strawberry")
def test_universal_newlines_communicate(self): # universal newlines through communicate() p = subprocess.Popen([sys.executable, "-c", 'import sys,os;' + SETBINARY + 'sys.stdout.write("line1\\n");' 'sys.stdout.flush();' 'sys.stdout.write("line2\\r");' 'sys.stdout.flush();' 'sys.stdout.write("line3\\r\\n");' 'sys.stdout.flush();' 'sys.stdout.write("line4\\r");' 'sys.stdout.flush();' 'sys.stdout.write("\\nline5");' 'sys.stdout.flush();' 'sys.stdout.write("\\nline6");'], stdout=subprocess.PIPE, stderr=subprocess.PIPE, universal_newlines=1) (stdout, stderr) = p.communicate() if hasattr(file, 'newlines'): # Interpreter with universal newline support self.assertEqual(stdout, "line1\nline2\nline3\nline4\nline5\nline6") else: # Interpreter without universal newline support self.assertEqual(stdout, "line1\nline2\rline3\r\nline4\r\nline5\nline6")
def testCheckCallAndFilter(self): cwd = 'bleh' args = ['boo', 'foo', 'bar'] test_string = 'ahah\naccb\nallo\naddb\n✔' # pylint: disable=no-member subprocess2.Popen( args, cwd=cwd, stdout=subprocess2.PIPE, stderr=subprocess2.STDOUT, bufsize=0).AndReturn(self.ProcessIdMock(test_string)) os.getcwd() self.mox.ReplayAll() line_list = [] result = gclient_utils.CheckCallAndFilter( args, cwd=cwd, show_header=True, always_show_header=True, filter_fn=line_list.append) self.assertEqual(result, test_string.encode('utf-8')) self.assertEqual(line_list, [ '________ running \'boo foo bar\' in \'bleh\'\n', 'ahah', 'accb', 'allo', 'addb', '✔'])
def run_with_stderr(*cmd, **kwargs): """Runs a git command. Returns (stdout, stderr) as a pair of strings. kwargs autostrip (bool) - Strip the output. Defaults to True. indata (str) - Specifies stdin data for the process. """ kwargs.setdefault('stdin', subprocess2.PIPE) kwargs.setdefault('stdout', subprocess2.PIPE) kwargs.setdefault('stderr', subprocess2.PIPE) kwargs.setdefault('shell', False) autostrip = kwargs.pop('autostrip', True) indata = kwargs.pop('indata', None) cmd = (GIT_EXE, '-c', 'color.ui=never') + cmd proc = subprocess2.Popen(cmd, **kwargs) ret, err = proc.communicate(indata) retcode = proc.wait() if retcode != 0: raise subprocess2.CalledProcessError(retcode, cmd, os.getcwd(), ret, err) if autostrip: ret = (ret or '').strip() err = (err or '').strip() return ret, err
def RetryFailedTests(self): """Reruns any failed tests serially and prints another summary of the results if no more than retry_percent failed. """ num_tests_run = self.test_counter.next() if len(self.failed_tests) > self.retry_percent * num_tests_run: sys.stderr.write("\nNOT RETRYING FAILED TESTS (too many failed)\n") return 1 self.WriteText(sys.stderr, "\nRETRYING FAILED TESTS:\n", "\x1b[1;5;33m") sharded_description = re.compile(r": (?:\d+>)?(.*)") gtest_filters = [ sharded_description.search(line).group(1) for line in self.failed_tests ] failed_retries = [] for test_filter in gtest_filters: args = [self.test, "--gtest_filter=" + test_filter] args.extend(self.gtest_args) rerun = subprocess.Popen(args) rerun.wait() if rerun.returncode != 0: failed_retries.append(test_filter) self.WriteText(sys.stderr, "RETRY RESULTS:\n", "\x1b[1;5;33m") self.PrintSummary(failed_retries) return len(failed_retries) > 0
def _inner(self, args, test_string): cwd = 'bleh' gclient_utils.sys.stdout.write( '________ running \'boo foo bar\' in \'bleh\'\n') for i in test_string: gclient_utils.sys.stdout.write(i) # pylint: disable=no-member subprocess2.Popen(args, cwd=cwd, stdout=subprocess2.PIPE, stderr=subprocess2.STDOUT, bufsize=0).AndReturn(self.ProcessIdMock(test_string)) os.getcwd() self.mox.ReplayAll() compiled_pattern = gclient_utils.re.compile(r'a(.*)b') line_list = [] capture_list = [] def FilterLines(line): line_list.append(line) assert isinstance(line, str), type(line) match = compiled_pattern.search(line) if match: capture_list.append(match.group(1)) gclient_utils.CheckCallAndFilterAndHeader(args, cwd=cwd, always=True, filter_fn=FilterLines) self.assertEquals(line_list, ['ahah', 'accb', 'allo', 'addb']) self.assertEquals(capture_list, ['cc', 'dd'])
def RunShard(test, total_shards, index, gtest_args, stdout, stderr): """Runs a single test shard in a subprocess. Returns: The Popen object representing the subprocess handle. """ args = [test] # If there is a gtest_output test_args = AppendToGTestOutput(gtest_args, str(index)) args.extend(test_args) env = os.environ.copy() env["GTEST_TOTAL_SHARDS"] = str(total_shards) env["GTEST_SHARD_INDEX"] = str(index) # Use a unique log file for each shard # Allows ui_tests to be run in parallel on the same machine env["CHROME_LOG_FILE"] = "chrome_log_%d" % index return subprocess.Popen(args, stdout=stdout, stderr=stderr, env=env, bufsize=0, universal_newlines=True)
def less(): # pragma: no cover """Runs 'less' as context manager yielding its stdin as a PIPE. Automatically checks if sys.stdout is a non-TTY stream. If so, it avoids running less and just yields sys.stdout. The returned PIPE is opened on binary mode. """ if not setup_color.IS_TTY: # On Python 3, sys.stdout doesn't accept bytes, and sys.stdout.buffer must # be used. yield getattr(sys.stdout, 'buffer', sys.stdout) return # Run with the same options that git uses (see setup_pager in git repo). # -F: Automatically quit if the output is less than one screen. # -R: Don't escape ANSI color codes. # -X: Don't clear the screen before starting. cmd = ('less', '-FRX') try: proc = subprocess2.Popen(cmd, stdin=subprocess2.PIPE) yield proc.stdin finally: try: proc.stdin.close() except BrokenPipeError: # BrokenPipeError is raised if proc has already completed, pass proc.wait()
def start_server(self, verbose=False): self.install_prerequisites() self.port = find_free_port() if verbose: pipe = None else: pipe = subprocess2.VOID cmd = [ sys.executable, self.dev_app, '--skip_sdk_update_check', '.', '--port=%d' % self.port, '--datastore_path=' + os.path.join(self.rietveld, 'tmp.db'), '-c' ] # CHEAP TRICK # By default you only want to bind on loopback but I'm testing over a # headless computer so it's useful to be able to access the test instance # remotely. if os.environ.get('GAE_LISTEN_ALL', '') == 'true': cmd.extend(('-a', '0.0.0.0')) self.test_server = subprocess2.Popen(cmd, stdout=pipe, stderr=pipe, cwd=self.rietveld) # Loop until port 127.0.0.1:port opens or the process dies. while not test_port(self.port): self.test_server.poll() if self.test_server.returncode is not None: raise Failure( 'Test rietveld instance failed early on port %s' % self.port) time.sleep(0.01)
def _CheckCommitSubjectLength(input_api, output_api): """Ensures commit's subject length is no longer than 65 chars.""" name = "git-commit subject" cmd = ["git", "log", "-1", "--pretty=%s"] start = input_api.time.time() proc = subprocess2.Popen(cmd, stderr=subprocess2.PIPE, stdout=subprocess2.PIPE, universal_newlines=True) stdout, _ = proc.communicate() duration = input_api.time.time() - start if not re.match(r"^Revert", stdout) and (len(stdout) - 1) > _GIT_COMMIT_SUBJECT_LENGTH: failure_msg = ( "The commit subject: %s is too long (%d chars)\n" "Try to keep this to 50 or less (up to 65 is permitted for " "non-reverts).\n" "https://www.git-scm.com/book/en/v2/Distributed-Git-Contributing-to-a-" "Project#_commit_guidelines") % (stdout, len(stdout) - 1) return output_api.PresubmitError("%s\n (%4.2fs) failed\n%s" % (name, duration, failure_msg)) return output_api.PresubmitResult("%s\n (%4.2fs) success" % (name, duration))
def RetryFailedTests(self): """Reruns any failed tests serially and prints another summary of the results if no more than retry_percent failed. """ num_tests_run = self.test_counter.next() if len(self.failed_tests) > self.retry_percent * num_tests_run: sys.stdout.write("\nNOT RETRYING FAILED TESTS (too many failed)\n") return 1 self.WriteText(sys.stdout, "\nRETRYING FAILED TESTS:\n", "\x1b[1;5;33m") sharded_description = re.compile(r": (?:\d+>)?(.*)") gtest_filters = [ sharded_description.search(line).group(1) for line in self.failed_tests ] sys.stdout.write("\nRETRY GTEST FILTERS: %r\n" % gtest_filters) failed_retries = [] for test_filter in gtest_filters: args = [self.test, "--gtest_filter=" + test_filter] # Don't update the xml output files during retry. stripped_gtests_args = RemoveGTestOutput(self.gtest_args) args.extend(stripped_gtests_args) sys.stdout.write("\nRETRY COMMAND: %r\n" % args) rerun = subprocess.Popen(args, stdout=sys.stdout, stderr=sys.stderr) rerun.wait() if rerun.returncode != 0: failed_retries.append(test_filter) self.WriteText(sys.stdout, "RETRY RESULTS:\n", "\x1b[1;5;33m") self.PrintSummary(failed_retries) return len(failed_retries) > 0
def get_repo_timestamp(path_to_repo): """Get an approximate timestamp for the upstream of |path_to_repo|. Returns the top two bits of the timestamp of the HEAD for the upstream of the branch path_to_repo is checked out at. """ # Get the upstream for the current branch. If we're not in a branch, fallback # to HEAD. try: upstream = scm.GIT.GetUpstreamBranch(path_to_repo) or 'HEAD' except subprocess2.CalledProcessError: upstream = 'HEAD' # Get the timestamp of the HEAD for the upstream of the current branch. p = subprocess2.Popen( ['git', '-C', path_to_repo, 'log', '-n1', upstream, '--format=%at'], stdout=subprocess2.PIPE, stderr=subprocess2.PIPE) stdout, _ = p.communicate() # If there was an error, give up. if p.returncode != 0: return None return stdout.strip()
def v0_4(client, options, test_name): """Handles swarm_client/swarming.py starting b39e8cf08c.""" swarming = os.path.join(client, 'swarming.py') cmd = [ sys.executable, swarming, 'collect', '--swarming', options.swarming, '--decorate', test_name, ] print('Running: %s' % ' '.join(cmd)) sys.stdout.flush() proc = subprocess2.Popen(cmd, bufsize=0, stdout=subprocess2.PIPE) gtest_parser = gtest_utils.GTestLogParser() for line in proc.stdout.readlines(): line = line.rstrip() print line gtest_parser.ProcessLine(line) proc.wait() annotation_utils.annotate(test_name, proc.returncode, gtest_parser) print('') return proc.returncode
def set_up_git(self): """Creates git repositories and start the servers.""" self.set_up() if self.gitdaemon: return True assert self.git_pid_file == None try: subprocess2.check_output(['git', '--version']) except (OSError, subprocess2.CalledProcessError): return False for repo in ['repo_%d' % r for r in range(1, self.NB_GIT_REPOS + 1)]: subprocess2.check_call(['git', 'init', '-q', join(self.git_root, repo)]) self.git_hashes[repo] = [None] self.git_port = find_free_port(self.host, 20000) self.git_base = 'git://%s:%d/git/' % (self.host, self.git_port) # Start the daemon. self.git_pid_file = tempfile.NamedTemporaryFile() cmd = ['git', 'daemon', '--export-all', '--reuseaddr', '--base-path=' + self.root_dir, '--pid-file=' + self.git_pid_file.name, '--port=%d' % self.git_port] if self.host == '127.0.0.1': cmd.append('--listen=' + self.host) self.check_port_is_free(self.git_port) self.gitdaemon = subprocess2.Popen( cmd, cwd=self.root_dir, stdout=subprocess2.PIPE, stderr=subprocess2.PIPE) wait_for_port_to_bind(self.host, self.git_port, self.gitdaemon) self.populateGit() self.git_dirty = False return True
def GetSVNBranch(cwd): """Returns the svn branch name if found.""" # Try to figure out which remote branch we're based on. # Strategy: # 1) iterate through our branch history and find the svn URL. # 2) find the svn-remote that fetches from the URL. # regexp matching the git-svn line that contains the URL. git_svn_re = re.compile(r'^\s*git-svn-id: (\S+)@', re.MULTILINE) # We don't want to go through all of history, so read a line from the # pipe at a time. # The -100 is an arbitrary limit so we don't search forever. cmd = ['git', 'log', '-100', '--pretty=medium'] proc = subprocess2.Popen(cmd, cwd=cwd, stdout=subprocess2.PIPE) url = None for line in proc.stdout: match = git_svn_re.match(line) if match: url = match.group(1) proc.stdout.close() # Cut pipe. break if url: svn_remote_re = re.compile(r'^svn-remote\.([^.]+)\.url (.*)$') remotes = GIT.Capture( ['config', '--get-regexp', r'^svn-remote\..*\.url'], cwd=cwd).splitlines() for remote in remotes: match = svn_remote_re.match(remote) if match: remote = match.group(1) base_url = match.group(2) try: fetch_spec = GIT.Capture( ['config', 'svn-remote.%s.fetch' % remote], cwd=cwd).strip() branch = GIT.MatchSvnGlob(url, base_url, fetch_spec, False) except subprocess2.CalledProcessError: branch = None if branch: return branch try: branch_spec = GIT.Capture( ['config', 'svn-remote.%s.branches' % remote], cwd=cwd).strip() branch = GIT.MatchSvnGlob(url, base_url, branch_spec, True) except subprocess2.CalledProcessError: branch = None if branch: return branch try: tag_spec = GIT.Capture( ['config', 'svn-remote.%s.tags' % remote], cwd=cwd).strip() branch = GIT.MatchSvnGlob(url, base_url, tag_spec, True) except subprocess2.CalledProcessError: branch = None if branch: return branch
def _upload_metrics_data(self): """Upload the metrics data to the AppEngine app.""" # We invoke a subprocess, and use stdin.write instead of communicate(), # so that we are able to return immediately, leaving the upload running in # the background. p = subprocess2.Popen(['vpython3', UPLOAD_SCRIPT], stdin=subprocess2.PIPE) p.stdin.write(json.dumps(self._reported_metrics).encode('utf-8'))
def test_shell_string(self): # Run command through the shell (string) newenv = os.environ.copy() newenv["FRUIT"] = "physalis" p = subprocess.Popen("set", shell=1, stdout=subprocess.PIPE, env=newenv) self.assertNotEqual(p.stdout.read().find("physalis"), -1)
def test_shell_string(self): # Run command through the shell (string) newenv = os.environ.copy() newenv["FRUIT"] = "apple" p = subprocess.Popen("echo $FRUIT", shell=1, stdout=subprocess.PIPE, env=newenv) self.assertEqual(p.stdout.read().strip(), "apple")
def test_preexec(self): # preexec function p = subprocess.Popen([sys.executable, "-c", 'import sys,os;' \ 'sys.stdout.write(os.getenv("FRUIT"))'], stdout=subprocess.PIPE, preexec_fn=lambda: os.putenv("FRUIT", "apple")) self.assertEqual(p.stdout.read(), "apple")
def main(argv, outbuf): if '-h' in argv or '--help' in argv: _print_help(outbuf) return 0 map_extra = git_common.get_config_list('depot_tools.map_extra') cmd = [ git_common.GIT_EXE, 'log', git_common.root(), '--graph', '--branches', '--tags', '--color=always', '--date=short', '--pretty=format:%H%x00%D%x00%cd%x00%s' ] + map_extra + argv log_proc = subprocess2.Popen(cmd, stdout=subprocess2.PIPE, shell=False) current = git_common.current_branch() all_tags = set(git_common.tags()) all_branches = set(git_common.branches()) if current in all_branches: all_branches.remove(current) merge_base_map = {} for branch in all_branches: merge_base = git_common.get_or_create_merge_base(branch) if merge_base: merge_base_map.setdefault(merge_base, set()).add(branch) for merge_base, branches in merge_base_map.items(): merge_base_map[merge_base] = ', '.join(branches) try: for line in log_proc.stdout: if b'\x00' not in line: outbuf.write(line) continue graph, commit, branch_list, commit_date, subject = _parse_log_line( line) if 'HEAD' in branch_list: graph = graph.replace('*', BLUE_BACK + '*') line = '{graph}{commit}\t{branches}{date} ~ {subject}'.format( graph=graph, commit=BRIGHT_RED + commit[:10] + RESET, branches=_color_branch_list(branch_list, all_branches, all_tags, current), date=YELLOW + commit_date + RESET, subject=subject) if commit in merge_base_map: line += ' <({})'.format(WHITE + merge_base_map[commit] + RESET) line += os.linesep outbuf.write(line.encode('utf-8', 'replace')) except (BrokenPipeError, KeyboardInterrupt): pass return 0
def test_invalid_bufsize(self): # an invalid type of the bufsize argument should raise # TypeError. try: subprocess.Popen([sys.executable, "-c", "pass"], "orange") except TypeError: pass else: self.fail("Expected TypeError")
def test_stdout_none(self): # .stdout is None when not redirected p = subprocess.Popen([sys.executable, "-c", 'print " this bit of output is from a ' 'test of stdout in a different ' 'process ..."'], stdin=subprocess.PIPE, stderr=subprocess.PIPE) p.wait() self.assertEqual(p.stdout, None)
def test_communicate_stderr(self): p = subprocess.Popen([sys.executable, "-c", 'import sys; sys.stderr.write("pineapple")'], stderr=subprocess.PIPE) (stdout, stderr) = p.communicate() self.assertEqual(stdout, None) # When running with a pydebug build, the # of references is outputted # to stderr, so just check if stderr at least started with "pinapple" self.assert_(stderr.startswith("pineapple"))
def test_exceptions(self): # catched & re-raised exceptions try: p = subprocess.Popen([sys.executable, "-c", ""], cwd="/this/path/does/not/exist") except OSError, e: # The attribute child_traceback should contain "os.chdir" # somewhere. self.assertNotEqual(e.child_traceback.find("os.chdir"), -1)
def test_stderr(self, c, cmd, un, subp): cmd = ['expr', '1', '/', '0'] if sys.platform == 'win32': cmd = ['cmd.exe', '/c', 'exit', '1'] p1 = subprocess.Popen(cmd, stderr=subprocess.PIPE, shell=False) p2 = subprocess2.Popen(cmd, stderr=subprocess.PIPE, shell=False) r1 = p1.communicate() r2 = p2.communicate() self.assertEqual(r1, r2)
def test_communicate_stdin(self): cmd = [ sys.executable, "-c", 'import sys; sys.exit(sys.stdin.read() == "pear")', ] p = subprocess.Popen(cmd, stdin=subprocess.PIPE) p.communicate("pear") self.assertEqual(p.returncode, 1)