def setUp(self):
mock.patch('gclient_utils.FileRead',
return_value=self._GITCOOKIES).start()
mock.patch('os.getenv', return_value={}).start()
mock.patch('os.environ', {'HOME': '$HOME'}).start()
mock.patch('os.path.exists', return_value=True).start()
mock.patch(
'subprocess2.check_output',
side_effect=[
subprocess2.CalledProcessError(1, ['cmd'], 'cwd', 'out', 'err')
],
).start()
self.addCleanup(mock.patch.stopall)
self.maxDiff = None
def run_stream_with_retcode(*cmd, **kwargs):
"""Runs a git command as context manager yielding stdout as a PIPE.
stderr is dropped to avoid races if the process outputs to both stdout and
stderr.
Raises subprocess2.CalledProcessError on nonzero return code.
"""
kwargs.setdefault('stderr', subprocess2.VOID)
kwargs.setdefault('stdout', subprocess2.PIPE)
cmd = (GIT_EXE, '-c', 'color.ui=never') + cmd
try:
proc = subprocess2.Popen(cmd, **kwargs)
yield proc.stdout
finally:
retcode = proc.wait()
if retcode != 0:
raise subprocess2.CalledProcessError(retcode, cmd, os.getcwd(),
None, None)
def run_svn(*cmd, **kwargs):
"""Runs an svn command.
Returns (stdout, stderr) as a pair of strings.
Raises subprocess2.CalledProcessError on nonzero return code.
"""
kwargs.setdefault('stdin', subprocess2.PIPE)
kwargs.setdefault('stdout', subprocess2.PIPE)
kwargs.setdefault('stderr', subprocess2.PIPE)
cmd = (SVN_EXE,) + cmd
proc = subprocess2.Popen(cmd, **kwargs)
ret, err = proc.communicate()
retcode = proc.wait()
if retcode != 0:
raise subprocess2.CalledProcessError(retcode, cmd, os.getcwd(), ret, err)
return ret, err
def test_gerrit_patch_conflict(self):
self._patch_common(is_gerrit=True)
self.mock(git_cl, 'DieWithError',
lambda msg: self._mocked_call(['DieWithError', msg]))
class SystemExitMock(Exception):
pass
self.calls += [
(([
'git', 'fetch', 'https://chromium.googlesource.com/my/repo',
'refs/changes/56/123456/1'
], ), ''),
((['git', 'cherry-pick', 'FETCH_HEAD'], ), '',
subprocess2.CalledProcessError(1, '', '', '', '')),
((['DieWithError', 'git cherry-pick FETCH_HEAD" failed.\n'], ), '',
SystemExitMock()),
]
with self.assertRaises(SystemExitMock):
git_cl.main([
'patch',
'https://chromium-review.googlesource.com/#/c/123456/1'
])
def CheckCallAndFilter(args,
print_stdout=False,
filter_fn=None,
show_header=False,
always_show_header=False,
retry=False,
**kwargs):
"""Runs a command and calls back a filter function if needed.
Accepts all subprocess2.Popen() parameters plus:
print_stdout: If True, the command's stdout is forwarded to stdout.
filter_fn: A function taking a single string argument called with each line
of the subprocess2's output. Each line has the trailing newline
character trimmed.
show_header: Whether to display a header before the command output.
always_show_header: Show header even when the command produced no output.
retry: If the process exits non-zero, sleep for a brief interval and try
again, up to RETRY_MAX times.
stderr is always redirected to stdout.
Returns the output of the command as a binary string.
"""
def show_header_if_necessary(needs_header, attempt):
"""Show the header at most once."""
if not needs_header[0]:
return
needs_header[0] = False
# Automatically generated header. We only prepend a newline if
# always_show_header is false, since it usually indicates there's an
# external progress display, and it's better not to clobber it in that case.
header = '' if always_show_header else '\n'
header += '________ running \'%s\' in \'%s\'' % (
' '.join(args), kwargs.get('cwd', '.'))
if attempt:
header += ' attempt %s / %s' % (attempt + 1, RETRY_MAX + 1)
header += '\n'
if print_stdout:
stdout_write = getattr(sys.stdout, 'buffer', sys.stdout).write
stdout_write(header.encode())
if filter_fn:
filter_fn(header)
def filter_line(command_output, line_start):
"""Extract the last line from command output and filter it."""
if not filter_fn or line_start is None:
return
command_output.seek(line_start)
filter_fn(command_output.read().decode('utf-8'))
# Initialize stdout writer if needed. On Python 3, sys.stdout does not accept
# byte inputs and sys.stdout.buffer must be used instead.
if print_stdout:
sys.stdout.flush()
stdout_write = getattr(sys.stdout, 'buffer', sys.stdout).write
else:
stdout_write = lambda _: None
sleep_interval = RETRY_INITIAL_SLEEP
run_cwd = kwargs.get('cwd', os.getcwd())
for attempt in range(RETRY_MAX + 1):
kid = subprocess2.Popen(args,
bufsize=0,
stdout=subprocess2.PIPE,
stderr=subprocess2.STDOUT,
**kwargs)
GClientChildren.add(kid)
# Store the output of the command regardless of the value of print_stdout or
# filter_fn.
command_output = io.BytesIO()
# Passed as a list for "by ref" semantics.
needs_header = [show_header]
if always_show_header:
show_header_if_necessary(needs_header, attempt)
# Also, we need to forward stdout to prevent weird re-ordering of output.
# This has to be done on a per byte basis to make sure it is not buffered:
# normally buffering is done for each line, but if the process requests
# input, no end-of-line character is output after the prompt and it would
# not show up.
try:
line_start = None
while True:
in_byte = kid.stdout.read(1)
is_newline = in_byte in (b'\n', b'\r')
if not in_byte:
break
show_header_if_necessary(needs_header, attempt)
if is_newline:
filter_line(command_output, line_start)
line_start = None
elif line_start is None:
line_start = command_output.tell()
stdout_write(in_byte)
command_output.write(in_byte)
# Flush the rest of buffered output.
sys.stdout.flush()
if line_start is not None:
filter_line(command_output, line_start)
rv = kid.wait()
kid.stdout.close()
# Don't put this in a 'finally,' since the child may still run if we get
# an exception.
GClientChildren.remove(kid)
except KeyboardInterrupt:
print('Failed while running "%s"' % ' '.join(args),
file=sys.stderr)
raise
if rv == 0:
return command_output.getvalue()
if not retry:
break
print(
"WARNING: subprocess '%s' in %s failed; will retry after a short "
'nap...' % (' '.join('"%s"' % x for x in args), run_cwd))
time.sleep(sleep_interval)
sleep_interval *= 2
raise subprocess2.CalledProcessError(rv, args, kwargs.get('cwd', None),
None, None)
def CheckCallAndFilter(args, stdout=None, filter_fn=None,
print_stdout=None, call_filter_on_first_line=False,
retry=False, **kwargs):
"""Runs a command and calls back a filter function if needed.
Accepts all subprocess2.Popen() parameters plus:
print_stdout: If True, the command's stdout is forwarded to stdout.
filter_fn: A function taking a single string argument called with each line
of the subprocess2's output. Each line has the trailing newline
character trimmed.
stdout: Can be any bufferable output.
retry: If the process exits non-zero, sleep for a brief interval and try
again, up to RETRY_MAX times.
stderr is always redirected to stdout.
"""
assert print_stdout or filter_fn
stdout = stdout or sys.stdout
output = cStringIO.StringIO()
filter_fn = filter_fn or (lambda x: None)
sleep_interval = RETRY_INITIAL_SLEEP
run_cwd = kwargs.get('cwd', os.getcwd())
for _ in xrange(RETRY_MAX + 1):
kid = subprocess2.Popen(
args, bufsize=0, stdout=subprocess2.PIPE, stderr=subprocess2.STDOUT,
**kwargs)
GClientChildren.add(kid)
# Do a flush of stdout before we begin reading from the subprocess2's stdout
stdout.flush()
# Also, we need to forward stdout to prevent weird re-ordering of output.
# This has to be done on a per byte basis to make sure it is not buffered:
# normally buffering is done for each line, but if svn requests input, no
# end-of-line character is output after the prompt and it would not show up.
try:
in_byte = kid.stdout.read(1)
if in_byte:
if call_filter_on_first_line:
filter_fn(None)
in_line = ''
while in_byte:
output.write(in_byte)
if print_stdout:
stdout.write(in_byte)
if in_byte not in ['\r', '\n']:
in_line += in_byte
else:
filter_fn(in_line)
in_line = ''
in_byte = kid.stdout.read(1)
# Flush the rest of buffered output. This is only an issue with
# stdout/stderr not ending with a \n.
if len(in_line):
filter_fn(in_line)
rv = kid.wait()
# Don't put this in a 'finally,' since the child may still run if we get
# an exception.
GClientChildren.remove(kid)
except KeyboardInterrupt:
print >> sys.stderr, 'Failed while running "%s"' % ' '.join(args)
raise
if rv == 0:
return output.getvalue()
if not retry:
break
print ("WARNING: subprocess '%s' in %s failed; will retry after a short "
'nap...' % (' '.join('"%s"' % x for x in args), run_cwd))
time.sleep(sleep_interval)
sleep_interval *= 2
raise subprocess2.CalledProcessError(
rv, args, kwargs.get('cwd', None), None, None)
def callError(code=1, cmd='', cwd='', stdout=b'', stderr=b''):
return subprocess2.CalledProcessError(code, cmd, cwd, stdout, stderr)
def testGetUsableRevGitSvn(self):
# pylint: disable=E1101
options = self.Options()
too_big = str(1e7)
# Pretend like the git-svn repo's HEAD is at r2.
self.mox.StubOutWithMock(gclient_scm.scm.GIT, 'GetGitSvnHeadRev', True)
gclient_scm.scm.GIT.GetGitSvnHeadRev(cwd=self.base_path).MultipleTimes(
).AndReturn(2)
self.mox.StubOutWithMock(
gclient_scm.scm.GIT, 'GetBlessedSha1ForSvnRev', True)
# r1 -> first fake hash, r3 -> second fake hash.
gclient_scm.scm.GIT.GetBlessedSha1ForSvnRev(cwd=self.base_path, rev='1'
).AndReturn(self.fake_hash_1)
gclient_scm.scm.GIT.GetBlessedSha1ForSvnRev(cwd=self.base_path, rev='3'
).MultipleTimes().AndReturn(self.fake_hash_2)
# Ensure that we call git svn fetch if our LKGR is > the git-svn HEAD rev.
self.mox.StubOutWithMock(gclient_scm.scm.GIT, 'Capture', True)
gclient_scm.scm.GIT.Capture(['config', '--get', 'svn-remote.svn.fetch'],
cwd=self.base_path).AndReturn('blah')
gclient_scm.scm.GIT.Capture(['fetch'], cwd=self.base_path)
gclient_scm.scm.GIT.Capture(['svn', 'fetch'], cwd=self.base_path)
error = subprocess2.CalledProcessError(1, 'cmd', '/cwd', 'stdout', 'stderr')
gclient_scm.scm.GIT.Capture(['config', '--get', 'svn-remote.svn.fetch'],
cwd=self.base_path).AndRaise(error)
gclient_scm.scm.GIT.Capture(['svn', 'fetch'], cwd=self.base_path)
gclient_scm.scm.GIT.Capture(['fetch', 'origin'], cwd=self.base_path)
self.mox.StubOutWithMock(gclient_scm.scm.GIT, 'IsGitSvn', True)
gclient_scm.scm.GIT.IsGitSvn(cwd=self.base_path).MultipleTimes(
).AndReturn(True)
self.mox.StubOutWithMock(gclient_scm.scm.GIT, 'IsValidRevision', True)
gclient_scm.scm.GIT.IsValidRevision(cwd=self.base_path, rev=self.fake_hash_1
).AndReturn(True)
gclient_scm.scm.GIT.IsValidRevision(cwd=self.base_path, rev=too_big
).MultipleTimes(2).AndReturn(False)
gclient_scm.os.path.isdir(self.base_path).AndReturn(False)
gclient_scm.os.path.isdir(self.base_path).MultipleTimes().AndReturn(True)
self.mox.ReplayAll()
git_svn_scm = self._scm_wrapper(url=self.url, root_dir=self.root_dir,
relpath=self.relpath)
# Without an existing checkout, this should fail.
# TODO(dbeam) Fix this. http://crbug.com/109184
self.assertRaises(gclient_scm.gclient_utils.Error,
git_svn_scm.GetUsableRev, '1', options)
# Given an SVN revision with a git-svn checkout, it should be translated to
# a git sha1 and be usable.
self.assertEquals(git_svn_scm.GetUsableRev('1', options),
self.fake_hash_1)
# Our fake HEAD rev is r2, so this should call git fetch and git svn fetch
# to get more revs (pymox will complain if this doesn't happen). We mock an
# optimized checkout the first time, so this run should call git fetch.
self.assertEquals(git_svn_scm.GetUsableRev('3', options),
self.fake_hash_2)
# The time we pretend we're not optimized, so no git fetch should fire.
self.assertEquals(git_svn_scm.GetUsableRev('3', options),
self.fake_hash_2)
# Given a git sha1 with a git-svn checkout, it should be used as is.
self.assertEquals(git_svn_scm.GetUsableRev(self.fake_hash_1, options),
self.fake_hash_1)
# We currently check for seemingly valid SVN revisions by assuming 6 digit
# numbers, so assure that numeric revs >= 1000000 don't work.
self.assertRaises(gclient_scm.gclient_utils.Error,
git_svn_scm.GetUsableRev, too_big, options)
def CheckCallAndFilter(args,
stdout=None,
filter_fn=None,
print_stdout=None,
call_filter_on_first_line=False,
**kwargs):
"""Runs a command and calls back a filter function if needed.
Accepts all subprocess2.Popen() parameters plus:
print_stdout: If True, the command's stdout is forwarded to stdout.
filter_fn: A function taking a single string argument called with each line
of the subprocess2's output. Each line has the trailing newline
character trimmed.
stdout: Can be any bufferable output.
stderr is always redirected to stdout.
"""
assert print_stdout or filter_fn
stdout = stdout or sys.stdout
filter_fn = filter_fn or (lambda x: None)
kid = subprocess2.Popen(args,
bufsize=0,
stdout=subprocess2.PIPE,
stderr=subprocess2.STDOUT,
**kwargs)
# Do a flush of stdout before we begin reading from the subprocess2's stdout
stdout.flush()
# Also, we need to forward stdout to prevent weird re-ordering of output.
# This has to be done on a per byte basis to make sure it is not buffered:
# normally buffering is done for each line, but if svn requests input, no
# end-of-line character is output after the prompt and it would not show up.
try:
in_byte = kid.stdout.read(1)
if in_byte:
if call_filter_on_first_line:
filter_fn(None)
in_line = ''
while in_byte:
if in_byte != '\r':
if print_stdout:
stdout.write(in_byte)
if in_byte != '\n':
in_line += in_byte
else:
filter_fn(in_line)
in_line = ''
else:
filter_fn(in_line)
in_line = ''
in_byte = kid.stdout.read(1)
# Flush the rest of buffered output. This is only an issue with
# stdout/stderr not ending with a \n.
if len(in_line):
filter_fn(in_line)
rv = kid.wait()
except KeyboardInterrupt:
print >> sys.stderr, 'Failed while running "%s"' % ' '.join(args)
raise
if rv:
raise subprocess2.CalledProcessError(rv, args, kwargs.get('cwd', None),
None, None)
return 0
def testHasCachedCredentials_NotLoggedIn(self):
subprocess2.check_call_out.side_effect = [
subprocess2.CalledProcessError(1, ['cmd'], 'cwd', 'stdout',
'stderr')
]
self.assertFalse(auth.Authenticator().has_cached_credentials())
def CheckCallAndFilter(args,
stdout=None,
filter_fn=None,
print_stdout=None,
call_filter_on_first_line=False,
nag_timer=None,
nag_max=None,
**kwargs):
"""Runs a command and calls back a filter function if needed.
Accepts all subprocess2.Popen() parameters plus:
print_stdout: If True, the command's stdout is forwarded to stdout.
filter_fn: A function taking a single string argument called with each line
of the subprocess2's output. Each line has the trailing newline
character trimmed.
stdout: Can be any bufferable output.
stderr is always redirected to stdout.
"""
assert print_stdout or filter_fn
stdout = stdout or sys.stdout
filter_fn = filter_fn or (lambda x: None)
kid = subprocess2.Popen(args,
bufsize=0,
stdout=subprocess2.PIPE,
stderr=subprocess2.STDOUT,
**kwargs)
GClientChildren.add(kid)
# Do a flush of stdout before we begin reading from the subprocess2's stdout
stdout.flush()
nag = None
if nag_timer:
# Hack thread.index to force correct annotation.
index = getattr(threading.currentThread(), 'index', 0)
def _nag_cb(elapsed):
setattr(threading.currentThread(), 'index', index)
stdout.write(' No output for %.0f seconds from command:\n' %
elapsed)
stdout.write(' %s\n' % kid.cmd_str)
if (nag_max and int('%.0f' % (elapsed / nag_timer)) >= nag_max):
stdout.write(' ... killing it!\n')
kid.kill()
nag = subprocess2.NagTimer(nag_timer, _nag_cb)
nag.start()
# Also, we need to forward stdout to prevent weird re-ordering of output.
# This has to be done on a per byte basis to make sure it is not buffered:
# normally buffering is done for each line, but if svn requests input, no
# end-of-line character is output after the prompt and it would not show up.
try:
in_byte = kid.stdout.read(1)
if in_byte:
if nag:
nag.event()
if call_filter_on_first_line:
filter_fn(None)
in_line = ''
while in_byte:
if in_byte != '\r':
if print_stdout:
stdout.write(in_byte)
if in_byte != '\n':
in_line += in_byte
else:
filter_fn(in_line)
in_line = ''
else:
filter_fn(in_line)
in_line = ''
in_byte = kid.stdout.read(1)
if in_byte and nag:
nag.event()
# Flush the rest of buffered output. This is only an issue with
# stdout/stderr not ending with a \n.
if len(in_line):
filter_fn(in_line)
rv = kid.wait()
# Don't put this in a 'finally,' since the child may still run if we get an
# exception.
GClientChildren.remove(kid)
except KeyboardInterrupt:
print >> sys.stderr, 'Failed while running "%s"' % ' '.join(args)
raise
finally:
if nag:
nag.cancel()
if rv:
raise subprocess2.CalledProcessError(rv, args, kwargs.get('cwd', None),
None, None)
return 0
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
import gclient_paths
import gclient_utils
import subprocess2
if sys.version_info.major == 2:
from StringIO import StringIO
import mock
else:
from io import StringIO
from unittest import mock
EXCEPTION = subprocess2.CalledProcessError(
128, ['cmd'], 'cwd', 'stdout', 'stderr')
class TestBase(unittest.TestCase):
def setUp(self):
super(TestBase, self).setUp()
self.file_tree = {}
self.root = 'C:\\' if sys.platform == 'win32' else '/'
self.cwd = self.root
mock.patch('gclient_utils.FileRead', self.read).start()
mock.patch('os.environ', {}).start()
mock.patch('os.getcwd', self.getcwd).start()
mock.patch('os.path.exists', self.exists).start()
mock.patch('os.path.realpath', side_effect=lambda path: path).start()
mock.patch('subprocess2.check_output').start()
mock.patch('sys.platform', '').start()
