def test_Popen_defaults(self, mockPopen):
with mock.patch('sys.platform', 'win32'):
subprocess2.Popen(['foo'], a=True)
mockPopen.assert_called_with(['foo'], a=True, shell=True)
with mock.patch('sys.platform', 'non-win32'):
subprocess2.Popen(['foo'], a=True)
mockPopen.assert_called_with(['foo'], a=True, shell=False)
def testCheckCallAndFilter_RetryOnce(self):
cwd = 'bleh'
args = ['boo', 'foo', 'bar']
test_string = 'ahah\naccb\nallo\naddb\n✔'
# pylint: disable=no-member
subprocess2.Popen(
args,
cwd=cwd,
stdout=subprocess2.PIPE,
stderr=subprocess2.STDOUT,
bufsize=0).AndReturn(self.ProcessIdMock(test_string, 1))
os.getcwd()
# pylint: disable=no-member
subprocess2.Popen(
args,
cwd=cwd,
stdout=subprocess2.PIPE,
stderr=subprocess2.STDOUT,
bufsize=0).AndReturn(self.ProcessIdMock(test_string, 0))
self.mox.ReplayAll()
line_list = []
result = gclient_utils.CheckCallAndFilter(
args, cwd=cwd, show_header=True, always_show_header=True,
filter_fn=line_list.append, retry=True)
self.assertEqual(result, test_string.encode('utf-8'))
self.assertEqual(line_list, [
'________ running \'boo foo bar\' in \'bleh\'\n',
'ahah',
'accb',
'allo',
'addb',
'✔',
'________ running \'boo foo bar\' in \'bleh\' attempt 2 / 4\n',
'ahah',
'accb',
'allo',
'addb',
'✔',
])
self.checkstdout(
'WARNING: subprocess \'"boo" "foo" "bar"\' in bleh failed; will retry '
'after a short nap...\n')
def test_communicate_returns(self):
# communicate() should return None if no redirection is active
p = subprocess.Popen([sys.executable, "-c",
"import sys; sys.exit(47)"])
(stdout, stderr) = p.communicate()
self.assertEqual(stdout, None)
self.assertEqual(stderr, None)
def test_communicate_stdout(self):
p = subprocess.Popen([sys.executable, "-c",
'import sys; sys.stdout.write("pineapple")'],
stdout=subprocess.PIPE)
(stdout, stderr) = p.communicate()
self.assertEqual(stdout, "pineapple")
self.assertEqual(stderr, None)
def test_stderr_pipe(self):
# stderr redirection
p = subprocess.Popen([sys.executable, "-c",
'import sys; sys.stderr.write("strawberry")'],
stderr=subprocess.PIPE)
self.assertEqual(remove_stderr_debug_decorations(p.stderr.read()),
"strawberry")
def test_universal_newlines_communicate(self):
# universal newlines through communicate()
p = subprocess.Popen([sys.executable, "-c",
'import sys,os;' + SETBINARY +
'sys.stdout.write("line1\\n");'
'sys.stdout.flush();'
'sys.stdout.write("line2\\r");'
'sys.stdout.flush();'
'sys.stdout.write("line3\\r\\n");'
'sys.stdout.flush();'
'sys.stdout.write("line4\\r");'
'sys.stdout.flush();'
'sys.stdout.write("\\nline5");'
'sys.stdout.flush();'
'sys.stdout.write("\\nline6");'],
stdout=subprocess.PIPE, stderr=subprocess.PIPE,
universal_newlines=1)
(stdout, stderr) = p.communicate()
if hasattr(file, 'newlines'):
# Interpreter with universal newline support
self.assertEqual(stdout,
"line1\nline2\nline3\nline4\nline5\nline6")
else:
# Interpreter without universal newline support
self.assertEqual(stdout,
"line1\nline2\rline3\r\nline4\r\nline5\nline6")
def testCheckCallAndFilter(self):
cwd = 'bleh'
args = ['boo', 'foo', 'bar']
test_string = 'ahah\naccb\nallo\naddb\n✔'
# pylint: disable=no-member
subprocess2.Popen(
args,
cwd=cwd,
stdout=subprocess2.PIPE,
stderr=subprocess2.STDOUT,
bufsize=0).AndReturn(self.ProcessIdMock(test_string))
os.getcwd()
self.mox.ReplayAll()
line_list = []
result = gclient_utils.CheckCallAndFilter(
args, cwd=cwd, show_header=True, always_show_header=True,
filter_fn=line_list.append)
self.assertEqual(result, test_string.encode('utf-8'))
self.assertEqual(line_list, [
'________ running \'boo foo bar\' in \'bleh\'\n',
'ahah',
'accb',
'allo',
'addb',
'✔'])
def run_with_stderr(*cmd, **kwargs):
"""Runs a git command.
Returns (stdout, stderr) as a pair of strings.
kwargs
autostrip (bool) - Strip the output. Defaults to True.
indata (str) - Specifies stdin data for the process.
"""
kwargs.setdefault('stdin', subprocess2.PIPE)
kwargs.setdefault('stdout', subprocess2.PIPE)
kwargs.setdefault('stderr', subprocess2.PIPE)
kwargs.setdefault('shell', False)
autostrip = kwargs.pop('autostrip', True)
indata = kwargs.pop('indata', None)
cmd = (GIT_EXE, '-c', 'color.ui=never') + cmd
proc = subprocess2.Popen(cmd, **kwargs)
ret, err = proc.communicate(indata)
retcode = proc.wait()
if retcode != 0:
raise subprocess2.CalledProcessError(retcode, cmd, os.getcwd(), ret,
err)
if autostrip:
ret = (ret or '').strip()
err = (err or '').strip()
return ret, err
def RetryFailedTests(self):
"""Reruns any failed tests serially and prints another summary of the
results if no more than retry_percent failed.
"""
num_tests_run = self.test_counter.next()
if len(self.failed_tests) > self.retry_percent * num_tests_run:
sys.stderr.write("\nNOT RETRYING FAILED TESTS (too many failed)\n")
return 1
self.WriteText(sys.stderr, "\nRETRYING FAILED TESTS:\n",
"\x1b[1;5;33m")
sharded_description = re.compile(r": (?:\d+>)?(.*)")
gtest_filters = [
sharded_description.search(line).group(1)
for line in self.failed_tests
]
failed_retries = []
for test_filter in gtest_filters:
args = [self.test, "--gtest_filter=" + test_filter]
args.extend(self.gtest_args)
rerun = subprocess.Popen(args)
rerun.wait()
if rerun.returncode != 0:
failed_retries.append(test_filter)
self.WriteText(sys.stderr, "RETRY RESULTS:\n", "\x1b[1;5;33m")
self.PrintSummary(failed_retries)
return len(failed_retries) > 0
def _inner(self, args, test_string):
cwd = 'bleh'
gclient_utils.sys.stdout.write(
'________ running \'boo foo bar\' in \'bleh\'\n')
for i in test_string:
gclient_utils.sys.stdout.write(i)
# pylint: disable=no-member
subprocess2.Popen(args,
cwd=cwd,
stdout=subprocess2.PIPE,
stderr=subprocess2.STDOUT,
bufsize=0).AndReturn(self.ProcessIdMock(test_string))
os.getcwd()
self.mox.ReplayAll()
compiled_pattern = gclient_utils.re.compile(r'a(.*)b')
line_list = []
capture_list = []
def FilterLines(line):
line_list.append(line)
assert isinstance(line, str), type(line)
match = compiled_pattern.search(line)
if match:
capture_list.append(match.group(1))
gclient_utils.CheckCallAndFilterAndHeader(args,
cwd=cwd,
always=True,
filter_fn=FilterLines)
self.assertEquals(line_list, ['ahah', 'accb', 'allo', 'addb'])
self.assertEquals(capture_list, ['cc', 'dd'])
def RunShard(test, total_shards, index, gtest_args, stdout, stderr):
"""Runs a single test shard in a subprocess.
Returns:
The Popen object representing the subprocess handle.
"""
args = [test]
# If there is a gtest_output
test_args = AppendToGTestOutput(gtest_args, str(index))
args.extend(test_args)
env = os.environ.copy()
env["GTEST_TOTAL_SHARDS"] = str(total_shards)
env["GTEST_SHARD_INDEX"] = str(index)
# Use a unique log file for each shard
# Allows ui_tests to be run in parallel on the same machine
env["CHROME_LOG_FILE"] = "chrome_log_%d" % index
return subprocess.Popen(args,
stdout=stdout,
stderr=stderr,
env=env,
bufsize=0,
universal_newlines=True)
def less(): # pragma: no cover
"""Runs 'less' as context manager yielding its stdin as a PIPE.
Automatically checks if sys.stdout is a non-TTY stream. If so, it avoids
running less and just yields sys.stdout.
The returned PIPE is opened on binary mode.
"""
if not setup_color.IS_TTY:
# On Python 3, sys.stdout doesn't accept bytes, and sys.stdout.buffer must
# be used.
yield getattr(sys.stdout, 'buffer', sys.stdout)
return
# Run with the same options that git uses (see setup_pager in git repo).
# -F: Automatically quit if the output is less than one screen.
# -R: Don't escape ANSI color codes.
# -X: Don't clear the screen before starting.
cmd = ('less', '-FRX')
try:
proc = subprocess2.Popen(cmd, stdin=subprocess2.PIPE)
yield proc.stdin
finally:
try:
proc.stdin.close()
except BrokenPipeError:
# BrokenPipeError is raised if proc has already completed,
pass
proc.wait()
def start_server(self, verbose=False):
self.install_prerequisites()
self.port = find_free_port()
if verbose:
pipe = None
else:
pipe = subprocess2.VOID
cmd = [
sys.executable, self.dev_app, '--skip_sdk_update_check', '.',
'--port=%d' % self.port,
'--datastore_path=' + os.path.join(self.rietveld, 'tmp.db'), '-c'
]
# CHEAP TRICK
# By default you only want to bind on loopback but I'm testing over a
# headless computer so it's useful to be able to access the test instance
# remotely.
if os.environ.get('GAE_LISTEN_ALL', '') == 'true':
cmd.extend(('-a', '0.0.0.0'))
self.test_server = subprocess2.Popen(cmd,
stdout=pipe,
stderr=pipe,
cwd=self.rietveld)
# Loop until port 127.0.0.1:port opens or the process dies.
while not test_port(self.port):
self.test_server.poll()
if self.test_server.returncode is not None:
raise Failure(
'Test rietveld instance failed early on port %s' %
self.port)
time.sleep(0.01)
def _CheckCommitSubjectLength(input_api, output_api):
"""Ensures commit's subject length is no longer than 65 chars."""
name = "git-commit subject"
cmd = ["git", "log", "-1", "--pretty=%s"]
start = input_api.time.time()
proc = subprocess2.Popen(cmd,
stderr=subprocess2.PIPE,
stdout=subprocess2.PIPE,
universal_newlines=True)
stdout, _ = proc.communicate()
duration = input_api.time.time() - start
if not re.match(r"^Revert",
stdout) and (len(stdout) - 1) > _GIT_COMMIT_SUBJECT_LENGTH:
failure_msg = (
"The commit subject: %s is too long (%d chars)\n"
"Try to keep this to 50 or less (up to 65 is permitted for "
"non-reverts).\n"
"https://www.git-scm.com/book/en/v2/Distributed-Git-Contributing-to-a-"
"Project#_commit_guidelines") % (stdout, len(stdout) - 1)
return output_api.PresubmitError("%s\n (%4.2fs) failed\n%s" %
(name, duration, failure_msg))
return output_api.PresubmitResult("%s\n (%4.2fs) success" %
(name, duration))
def RetryFailedTests(self):
"""Reruns any failed tests serially and prints another summary of the
results if no more than retry_percent failed.
"""
num_tests_run = self.test_counter.next()
if len(self.failed_tests) > self.retry_percent * num_tests_run:
sys.stdout.write("\nNOT RETRYING FAILED TESTS (too many failed)\n")
return 1
self.WriteText(sys.stdout, "\nRETRYING FAILED TESTS:\n",
"\x1b[1;5;33m")
sharded_description = re.compile(r": (?:\d+>)?(.*)")
gtest_filters = [
sharded_description.search(line).group(1)
for line in self.failed_tests
]
sys.stdout.write("\nRETRY GTEST FILTERS: %r\n" % gtest_filters)
failed_retries = []
for test_filter in gtest_filters:
args = [self.test, "--gtest_filter=" + test_filter]
# Don't update the xml output files during retry.
stripped_gtests_args = RemoveGTestOutput(self.gtest_args)
args.extend(stripped_gtests_args)
sys.stdout.write("\nRETRY COMMAND: %r\n" % args)
rerun = subprocess.Popen(args,
stdout=sys.stdout,
stderr=sys.stderr)
rerun.wait()
if rerun.returncode != 0:
failed_retries.append(test_filter)
self.WriteText(sys.stdout, "RETRY RESULTS:\n", "\x1b[1;5;33m")
self.PrintSummary(failed_retries)
return len(failed_retries) > 0
def get_repo_timestamp(path_to_repo):
"""Get an approximate timestamp for the upstream of |path_to_repo|.
Returns the top two bits of the timestamp of the HEAD for the upstream of the
branch path_to_repo is checked out at.
"""
# Get the upstream for the current branch. If we're not in a branch, fallback
# to HEAD.
try:
upstream = scm.GIT.GetUpstreamBranch(path_to_repo) or 'HEAD'
except subprocess2.CalledProcessError:
upstream = 'HEAD'
# Get the timestamp of the HEAD for the upstream of the current branch.
p = subprocess2.Popen(
['git', '-C', path_to_repo, 'log', '-n1', upstream, '--format=%at'],
stdout=subprocess2.PIPE,
stderr=subprocess2.PIPE)
stdout, _ = p.communicate()
# If there was an error, give up.
if p.returncode != 0:
return None
return stdout.strip()
def v0_4(client, options, test_name):
"""Handles swarm_client/swarming.py starting b39e8cf08c."""
swarming = os.path.join(client, 'swarming.py')
cmd = [
sys.executable,
swarming,
'collect',
'--swarming',
options.swarming,
'--decorate',
test_name,
]
print('Running: %s' % ' '.join(cmd))
sys.stdout.flush()
proc = subprocess2.Popen(cmd, bufsize=0, stdout=subprocess2.PIPE)
gtest_parser = gtest_utils.GTestLogParser()
for line in proc.stdout.readlines():
line = line.rstrip()
print line
gtest_parser.ProcessLine(line)
proc.wait()
annotation_utils.annotate(test_name, proc.returncode, gtest_parser)
print('')
return proc.returncode
def set_up_git(self):
"""Creates git repositories and start the servers."""
self.set_up()
if self.gitdaemon:
return True
assert self.git_pid_file == None
try:
subprocess2.check_output(['git', '--version'])
except (OSError, subprocess2.CalledProcessError):
return False
for repo in ['repo_%d' % r for r in range(1, self.NB_GIT_REPOS + 1)]:
subprocess2.check_call(['git', 'init', '-q', join(self.git_root, repo)])
self.git_hashes[repo] = [None]
self.git_port = find_free_port(self.host, 20000)
self.git_base = 'git://%s:%d/git/' % (self.host, self.git_port)
# Start the daemon.
self.git_pid_file = tempfile.NamedTemporaryFile()
cmd = ['git', 'daemon',
'--export-all',
'--reuseaddr',
'--base-path=' + self.root_dir,
'--pid-file=' + self.git_pid_file.name,
'--port=%d' % self.git_port]
if self.host == '127.0.0.1':
cmd.append('--listen=' + self.host)
self.check_port_is_free(self.git_port)
self.gitdaemon = subprocess2.Popen(
cmd,
cwd=self.root_dir,
stdout=subprocess2.PIPE,
stderr=subprocess2.PIPE)
wait_for_port_to_bind(self.host, self.git_port, self.gitdaemon)
self.populateGit()
self.git_dirty = False
return True
def GetSVNBranch(cwd):
"""Returns the svn branch name if found."""
# Try to figure out which remote branch we're based on.
# Strategy:
# 1) iterate through our branch history and find the svn URL.
# 2) find the svn-remote that fetches from the URL.
# regexp matching the git-svn line that contains the URL.
git_svn_re = re.compile(r'^\s*git-svn-id: (\S+)@', re.MULTILINE)
# We don't want to go through all of history, so read a line from the
# pipe at a time.
# The -100 is an arbitrary limit so we don't search forever.
cmd = ['git', 'log', '-100', '--pretty=medium']
proc = subprocess2.Popen(cmd, cwd=cwd, stdout=subprocess2.PIPE)
url = None
for line in proc.stdout:
match = git_svn_re.match(line)
if match:
url = match.group(1)
proc.stdout.close() # Cut pipe.
break
if url:
svn_remote_re = re.compile(r'^svn-remote\.([^.]+)\.url (.*)$')
remotes = GIT.Capture(
['config', '--get-regexp', r'^svn-remote\..*\.url'],
cwd=cwd).splitlines()
for remote in remotes:
match = svn_remote_re.match(remote)
if match:
remote = match.group(1)
base_url = match.group(2)
try:
fetch_spec = GIT.Capture(
['config', 'svn-remote.%s.fetch' % remote],
cwd=cwd).strip()
branch = GIT.MatchSvnGlob(url, base_url, fetch_spec, False)
except subprocess2.CalledProcessError:
branch = None
if branch:
return branch
try:
branch_spec = GIT.Capture(
['config', 'svn-remote.%s.branches' % remote],
cwd=cwd).strip()
branch = GIT.MatchSvnGlob(url, base_url, branch_spec, True)
except subprocess2.CalledProcessError:
branch = None
if branch:
return branch
try:
tag_spec = GIT.Capture(
['config', 'svn-remote.%s.tags' % remote],
cwd=cwd).strip()
branch = GIT.MatchSvnGlob(url, base_url, tag_spec, True)
except subprocess2.CalledProcessError:
branch = None
if branch:
return branch
def _upload_metrics_data(self):
"""Upload the metrics data to the AppEngine app."""
# We invoke a subprocess, and use stdin.write instead of communicate(),
# so that we are able to return immediately, leaving the upload running in
# the background.
p = subprocess2.Popen(['vpython3', UPLOAD_SCRIPT],
stdin=subprocess2.PIPE)
p.stdin.write(json.dumps(self._reported_metrics).encode('utf-8'))
def test_shell_string(self):
# Run command through the shell (string)
newenv = os.environ.copy()
newenv["FRUIT"] = "physalis"
p = subprocess.Popen("set", shell=1,
stdout=subprocess.PIPE,
env=newenv)
self.assertNotEqual(p.stdout.read().find("physalis"), -1)
def test_shell_string(self):
# Run command through the shell (string)
newenv = os.environ.copy()
newenv["FRUIT"] = "apple"
p = subprocess.Popen("echo $FRUIT", shell=1,
stdout=subprocess.PIPE,
env=newenv)
self.assertEqual(p.stdout.read().strip(), "apple")
def test_preexec(self):
# preexec function
p = subprocess.Popen([sys.executable, "-c",
'import sys,os;' \
'sys.stdout.write(os.getenv("FRUIT"))'],
stdout=subprocess.PIPE,
preexec_fn=lambda: os.putenv("FRUIT", "apple"))
self.assertEqual(p.stdout.read(), "apple")
def main(argv, outbuf):
if '-h' in argv or '--help' in argv:
_print_help(outbuf)
return 0
map_extra = git_common.get_config_list('depot_tools.map_extra')
cmd = [
git_common.GIT_EXE, 'log',
git_common.root(), '--graph', '--branches', '--tags', '--color=always',
'--date=short', '--pretty=format:%H%x00%D%x00%cd%x00%s'
] + map_extra + argv
log_proc = subprocess2.Popen(cmd, stdout=subprocess2.PIPE, shell=False)
current = git_common.current_branch()
all_tags = set(git_common.tags())
all_branches = set(git_common.branches())
if current in all_branches:
all_branches.remove(current)
merge_base_map = {}
for branch in all_branches:
merge_base = git_common.get_or_create_merge_base(branch)
if merge_base:
merge_base_map.setdefault(merge_base, set()).add(branch)
for merge_base, branches in merge_base_map.items():
merge_base_map[merge_base] = ', '.join(branches)
try:
for line in log_proc.stdout:
if b'\x00' not in line:
outbuf.write(line)
continue
graph, commit, branch_list, commit_date, subject = _parse_log_line(
line)
if 'HEAD' in branch_list:
graph = graph.replace('*', BLUE_BACK + '*')
line = '{graph}{commit}\t{branches}{date} ~ {subject}'.format(
graph=graph,
commit=BRIGHT_RED + commit[:10] + RESET,
branches=_color_branch_list(branch_list, all_branches,
all_tags, current),
date=YELLOW + commit_date + RESET,
subject=subject)
if commit in merge_base_map:
line += ' <({})'.format(WHITE + merge_base_map[commit] +
RESET)
line += os.linesep
outbuf.write(line.encode('utf-8', 'replace'))
except (BrokenPipeError, KeyboardInterrupt):
pass
return 0
def test_invalid_bufsize(self):
# an invalid type of the bufsize argument should raise
# TypeError.
try:
subprocess.Popen([sys.executable, "-c", "pass"], "orange")
except TypeError:
pass
else:
self.fail("Expected TypeError")
def test_stdout_none(self):
# .stdout is None when not redirected
p = subprocess.Popen([sys.executable, "-c",
'print " this bit of output is from a '
'test of stdout in a different '
'process ..."'],
stdin=subprocess.PIPE, stderr=subprocess.PIPE)
p.wait()
self.assertEqual(p.stdout, None)
def test_communicate_stderr(self):
p = subprocess.Popen([sys.executable, "-c",
'import sys; sys.stderr.write("pineapple")'],
stderr=subprocess.PIPE)
(stdout, stderr) = p.communicate()
self.assertEqual(stdout, None)
# When running with a pydebug build, the # of references is outputted
# to stderr, so just check if stderr at least started with "pinapple"
self.assert_(stderr.startswith("pineapple"))
def test_exceptions(self):
# catched & re-raised exceptions
try:
p = subprocess.Popen([sys.executable, "-c", ""],
cwd="/this/path/does/not/exist")
except OSError, e:
# The attribute child_traceback should contain "os.chdir"
# somewhere.
self.assertNotEqual(e.child_traceback.find("os.chdir"), -1)
def test_stderr(self, c, cmd, un, subp):
cmd = ['expr', '1', '/', '0']
if sys.platform == 'win32':
cmd = ['cmd.exe', '/c', 'exit', '1']
p1 = subprocess.Popen(cmd, stderr=subprocess.PIPE, shell=False)
p2 = subprocess2.Popen(cmd, stderr=subprocess.PIPE, shell=False)
r1 = p1.communicate()
r2 = p2.communicate()
self.assertEqual(r1, r2)
def test_communicate_stdin(self):
cmd = [
sys.executable,
"-c",
'import sys; sys.exit(sys.stdin.read() == "pear")',
]
p = subprocess.Popen(cmd, stdin=subprocess.PIPE)
p.communicate("pear")
self.assertEqual(p.returncode, 1)
