def testCheckCallAndFilter(self, mockPopen):
cwd = 'bleh'
args = ['boo', 'foo', 'bar']
test_string = 'ahah\naccb\nallo\naddb\n✔'
mockPopen.return_value = self.ProcessIdMock(test_string)
line_list = []
result = gclient_utils.CheckCallAndFilter(args,
cwd=cwd,
show_header=True,
always_show_header=True,
filter_fn=line_list.append)
self.assertEqual(result, test_string.encode('utf-8'))
self.assertEqual(line_list, [
'________ running \'boo foo bar\' in \'bleh\'\n', 'ahah', 'accb',
'allo', 'addb', '✔'
])
mockPopen.assert_called_with(args,
cwd=cwd,
stdout=subprocess2.PIPE,
stderr=subprocess2.STDOUT,
bufsize=0)
def GetPrimarySolutionPath():
"""Returns the full path to the primary solution. (gclient_root + src)"""
gclient_root = FindGclientRoot(os.getcwd())
if not gclient_root:
# Some projects might not use .gclient. Try to see whether we're in a git
# checkout.
top_dir = [os.getcwd()]
def filter_fn(line):
repo_root_path = os.path.normpath(line.rstrip('\n'))
if os.path.exists(repo_root_path):
top_dir[0] = repo_root_path
try:
gclient_utils.CheckCallAndFilter(["git", "rev-parse", "--show-toplevel"],
print_stdout=False,
filter_fn=filter_fn)
except Exception:
pass
top_dir = top_dir[0]
if os.path.exists(os.path.join(top_dir, 'buildtools')):
return top_dir
return None
# Some projects' top directory is not named 'src'.
source_dir_name = GetGClientPrimarySolutionName(gclient_root) or 'src'
return os.path.join(gclient_root, source_dir_name)
def testCheckCallAndFilter(self):
cwd = 'bleh'
args = ['boo', 'foo', 'bar']
test_string = 'ahah\naccb\nallo\naddb\n✔'
# pylint: disable=no-member
subprocess2.Popen(
args,
cwd=cwd,
stdout=subprocess2.PIPE,
stderr=subprocess2.STDOUT,
bufsize=0).AndReturn(self.ProcessIdMock(test_string))
os.getcwd()
self.mox.ReplayAll()
line_list = []
result = gclient_utils.CheckCallAndFilter(
args, cwd=cwd, show_header=True, always_show_header=True,
filter_fn=line_list.append)
self.assertEqual(result, test_string.encode('utf-8'))
self.assertEqual(line_list, [
'________ running \'boo foo bar\' in \'bleh\'\n',
'ahah',
'accb',
'allo',
'addb',
'✔'])
def main(directories):
if not directories:
directories = [SCRIPT_DIR]
for path in directories:
cmd = [
sys.executable,
os.path.join(find_depot_tools.DEPOT_TOOLS_PATH,
'download_from_google_storage.py'),
'--directory',
'--num_threads=10',
'--bucket',
'chrome-webrtc-resources',
'--auto_platform',
'--recursive',
path,
]
print 'Downloading precompiled tools...'
# Perform download similar to how gclient hooks execute.
try:
gclient_utils.CheckCallAndFilter(cmd,
cwd=SRC_DIR,
always_show_header=True)
except (gclient_utils.Error, subprocess2.CalledProcessError) as e:
print 'Error: %s' % str(e)
return 2
return 0
def testCheckCallAndFilter_RetryOnce(self, mockPopen, mockTime):
cwd = 'bleh'
args = ['boo', 'foo', 'bar']
test_string = 'ahah\naccb\nallo\naddb\n✔'
self.kids = [
self.ProcessIdMock(test_string, 1),
self.ProcessIdMock(test_string, 0)
]
mockPopen.side_effect = self.PopenMock
line_list = []
result = gclient_utils.CheckCallAndFilter(args,
cwd=cwd,
show_header=True,
always_show_header=True,
filter_fn=line_list.append,
retry=True)
self.assertEqual(result, test_string.encode('utf-8'))
self.assertEqual(line_list, [
'________ running \'boo foo bar\' in \'bleh\'\n',
'ahah',
'accb',
'allo',
'addb',
'✔',
'________ running \'boo foo bar\' in \'bleh\' attempt 2 / 4\n',
'ahah',
'accb',
'allo',
'addb',
'✔',
])
mockTime.assert_called_with(gclient_utils.RETRY_INITIAL_SLEEP)
self.assertEqual(mockPopen.mock_calls, [
mock.call(args,
cwd=cwd,
stdout=mock.ANY,
stderr=subprocess2.STDOUT,
bufsize=0),
mock.call(args,
cwd=cwd,
stdout=mock.ANY,
stderr=subprocess2.STDOUT,
bufsize=0),
])
self.assertEqual(self.stdout.getvalue(), b'')
self.assertEqual(
self.printfn.getvalue(),
'WARNING: subprocess \'"boo" "foo" "bar"\' in bleh failed; will retry '
'after a short nap...')
def RunGit(self, cmd, **kwargs):
"""Run git in a subprocess."""
cwd = kwargs.setdefault('cwd', self.mirror_path)
kwargs.setdefault('print_stdout', False)
kwargs.setdefault('filter_fn', self.print)
env = kwargs.get('env') or kwargs.setdefault('env', os.environ.copy())
env.setdefault('GIT_ASKPASS', 'true')
env.setdefault('SSH_ASKPASS', 'true')
self.print('running "git %s" in "%s"' % (' '.join(cmd), cwd))
gclient_utils.CheckCallAndFilter([self.git_exe] + cmd, **kwargs)
def pack(self, options, args, file_list):
"""Generates a patch file which can be applied to the root of the
repository."""
if not os.path.isdir(self.checkout_path):
raise gclient_utils.Error('Directory %s is not present.' %
self.checkout_path)
gclient_utils.CheckCallAndFilter(
['svn', 'diff', '-x', '--ignore-eol-style'] + args,
cwd=self.checkout_path,
print_stdout=False,
filter_fn=DiffFilterer(self.relpath).Filter)
def pack(self, options, args, file_list):
"""Generates a patch file which can be applied to the root of the
repository.
The patch file is generated from a diff of the merge base of HEAD and
its upstream branch.
"""
merge_base = self._Capture(['merge-base', 'HEAD', 'origin'])
gclient_utils.CheckCallAndFilter(['git', 'diff', merge_base],
cwd=self.checkout_path,
filter_fn=DiffFilterer(
self.relpath).Filter)
def _Run(self, args, options, show_header=True, **kwargs):
# Disable 'unused options' warning | pylint: disable=unused-argument
kwargs.setdefault('cwd', self.checkout_path)
kwargs.setdefault('stdout', self.out_fh)
kwargs['filter_fn'] = self.filter
kwargs.setdefault('print_stdout', False)
env = scm.GIT.ApplyEnvVars(kwargs)
cmd = ['git'] + args
if show_header:
gclient_utils.CheckCallAndFilterAndHeader(cmd, env=env, **kwargs)
else:
gclient_utils.CheckCallAndFilter(cmd, env=env, **kwargs)
def RunAction(dir, command):
"""Runs the action."""
try:
gclient_utils.CheckCallAndFilter(command,
cwd=dir,
always_show_header=True,
print_stdout=True)
except gclient_utils.Error as e:
# Use a discrete exit status code of 2 to indicate that a hook action
# failed. Users of this script may wish to treat hook action failures
# differently from VC failures.
print('Error: %s' % str(e), file=sys.stderr)
sys.exit(2)
def testCheckCallAndFilter_RetryOnce(self):
cwd = 'bleh'
args = ['boo', 'foo', 'bar']
test_string = 'ahah\naccb\nallo\naddb\n✔'
# pylint: disable=no-member
subprocess2.Popen(
args,
cwd=cwd,
stdout=subprocess2.PIPE,
stderr=subprocess2.STDOUT,
bufsize=0).AndReturn(self.ProcessIdMock(test_string, 1))
os.getcwd()
# pylint: disable=no-member
subprocess2.Popen(
args,
cwd=cwd,
stdout=subprocess2.PIPE,
stderr=subprocess2.STDOUT,
bufsize=0).AndReturn(self.ProcessIdMock(test_string, 0))
self.mox.ReplayAll()
line_list = []
result = gclient_utils.CheckCallAndFilter(
args, cwd=cwd, show_header=True, always_show_header=True,
filter_fn=line_list.append, retry=True)
self.assertEqual(result, test_string.encode('utf-8'))
self.assertEqual(line_list, [
'________ running \'boo foo bar\' in \'bleh\'\n',
'ahah',
'accb',
'allo',
'addb',
'✔',
'________ running \'boo foo bar\' in \'bleh\' attempt 2 / 4\n',
'ahah',
'accb',
'allo',
'addb',
'✔',
])
self.checkstdout(
'WARNING: subprocess \'"boo" "foo" "bar"\' in bleh failed; will retry '
'after a short nap...\n')
def pack(self, _options, _args, _file_list):
"""Generates a patch file which can be applied to the root of the
repository.
The patch file is generated from a diff of the merge base of HEAD and
its upstream branch.
"""
try:
merge_base = [self._Capture(['merge-base', 'HEAD', self.remote])]
except subprocess2.CalledProcessError:
merge_base = []
gclient_utils.CheckCallAndFilter(
['git', 'diff'] + merge_base,
cwd=self.checkout_path,
filter_fn=GitDiffFilterer(self.relpath, print_func=self.Print).Filter)
def RunAction(dir, command):
"""Runs the action."""
if command[0] == 'python':
# If the hook specified "python" as the first item, the action is a
# Python script. Run it by starting a new copy of the same
# interpreter.
command[0] = sys.executable
try:
gclient_utils.CheckCallAndFilter(
command, cwd=dir, always_show_header=True, print_stdout=True)
except gclient_utils.Error, e:
# Use a discrete exit status code of 2 to indicate that a hook action
# failed. Users of this script may wish to treat hook action failures
# differently from VC failures.
print >> sys.stderr, 'Error: %s' % str(e)
sys.exit(2)
def testCheckCallAndFilter_PrintStdout(self, mockPopen):
cwd = 'bleh'
args = ['boo', 'foo', 'bar']
test_string = 'ahah\naccb\nallo\naddb\n✔'
mockPopen.return_value = self.ProcessIdMock(test_string)
result = gclient_utils.CheckCallAndFilter(
args, cwd=cwd, show_header=True, always_show_header=True,
print_stdout=True)
self.assertEqual(result, test_string.encode('utf-8'))
self.assertEqual(self.stdout.getvalue().splitlines(), [
b"________ running 'boo foo bar' in 'bleh'",
b'ahah',
b'accb',
b'allo',
b'addb',
b'\xe2\x9c\x94',
])
def GetCachedFile(filename, max_age=60*60*24*3, use_root=False):
"""Retrieves a file from the repository and caches it in GetCacheDir() for
max_age seconds.
use_root: If False, look up the arborescence for the first match, otherwise go
directory to the root repository.
Note: The cache will be inconsistent if the same file is retrieved with both
use_root=True and use_root=False. Don't be stupid.
"""
if filename not in FILES_CACHE:
# Don't try to look up twice.
FILES_CACHE[filename] = None
# First we check if we have a cached version.
try:
cached_file = os.path.join(GetCacheDir(), filename)
except gclient_utils.Error:
return None
if (not os.path.exists(cached_file) or
(time.time() - os.stat(cached_file).st_mtime) > max_age):
dir_info = SVN.CaptureInfo('.')
repo_root = dir_info['Repository Root']
if use_root:
url_path = repo_root
else:
url_path = dir_info['URL']
while True:
# Look in the repository at the current level for the file.
for _ in range(5):
content = None
try:
# Take advantage of the fact that svn won't output to stderr in case
# of success but will do in case of failure so don't mind putting
# stderr into content_array.
content_array = []
svn_path = url_path + '/' + filename
args = ['svn', 'cat', svn_path]
if sys.platform != 'darwin':
# MacOSX 10.5.2 has a bug with svn 1.4.4 that will trigger the
# 'Can\'t get username or password' and can be fixed easily.
# The fix doesn't work if the user upgraded to svn 1.6.x. Bleh.
# I don't have time to fix their broken stuff.
args.append('--non-interactive')
gclient_utils.CheckCallAndFilter(
args, cwd='.', filter_fn=content_array.append)
# Exit the loop if the file was found. Override content.
content = '\n'.join(content_array)
break
except gclient_utils.Error:
if content_array[0].startswith(
'svn: Can\'t get username or password'):
ErrorExit('Your svn credentials expired. Please run svn update '
'to fix the cached credentials')
if content_array[0].startswith('svn: Can\'t get password'):
ErrorExit('If are using a Mac and svn --version shows 1.4.x, '
'please hack gcl.py to remove --non-interactive usage, it\'s'
'a bug on your installed copy')
if not content_array[0].startswith('svn: File not found:'):
# Try again.
continue
if content:
break
if url_path == repo_root:
# Reached the root. Abandoning search.
break
# Go up one level to try again.
url_path = os.path.dirname(url_path)
if content is not None or filename != CODEREVIEW_SETTINGS_FILE:
# Write a cached version even if there isn't a file, so we don't try to
# fetch it each time. codereview.settings must always be present so do
# not cache negative.
gclient_utils.FileWrite(cached_file, content or '')
else:
content = gclient_utils.FileRead(cached_file, 'r')
# Keep the content cached in memory.
FILES_CACHE[filename] = content
return FILES_CACHE[filename]
