def main(directories):
if not directories:
directories = [SCRIPT_DIR]
for path in directories:
cmd = [
sys.executable,
os.path.join(find_depot_tools.DEPOT_TOOLS_PATH,
'download_from_google_storage.py'),
'--directory',
'--num_threads=10',
'--bucket',
'chrome-webrtc-resources',
'--auto_platform',
'--recursive',
path,
]
print 'Downloading precompiled tools...'
# Perform download similar to how gclient hooks execute.
try:
gclient_utils.CheckCallAndFilterAndHeader(cmd,
cwd=SRC_DIR,
always=True)
except (gclient_utils.Error, subprocess2.CalledProcessError) as e:
print 'Error: %s' % str(e)
return 2
return 0
def _inner(self, args, test_string):
cwd = 'bleh'
gclient_utils.sys.stdout.write(
'________ running \'boo foo bar\' in \'bleh\'\n')
for i in test_string:
gclient_utils.sys.stdout.write(i)
# pylint: disable=no-member
subprocess2.Popen(args,
cwd=cwd,
stdout=subprocess2.PIPE,
stderr=subprocess2.STDOUT,
bufsize=0).AndReturn(self.ProcessIdMock(test_string))
os.getcwd()
self.mox.ReplayAll()
compiled_pattern = gclient_utils.re.compile(r'a(.*)b')
line_list = []
capture_list = []
def FilterLines(line):
line_list.append(line)
assert isinstance(line, str), type(line)
match = compiled_pattern.search(line)
if match:
capture_list.append(match.group(1))
gclient_utils.CheckCallAndFilterAndHeader(args,
cwd=cwd,
always=True,
filter_fn=FilterLines)
self.assertEquals(line_list, ['ahah', 'accb', 'allo', 'addb'])
self.assertEquals(capture_list, ['cc', 'dd'])
def DoGclientSyncForChromium(self):
gclient_cmd = ['gclient', 'sync', '--verbose', '--reset', '--with_branch_heads', '--ignore_locks', '--delete_unversioned_trees']
gclient_cmd.append('--gclientfile=%s' % os.path.basename(self._new_gclient_file))
# As of revision 454e53abae6e4d68ee992b0a93a4174b75519393,
# src/xwalk/buildtools does not pass gclient's syntax validation.
# gclient_cmd.append('--disable-syntax-validation')
gclient_utils.CheckCallAndFilterAndHeader(gclient_cmd,
always=self._options.verbose, cwd=self._root_dir)
def DoGclientSyncForChromium(self):
gclient_cmd = ['gclient', 'sync', '--verbose', '--reset',
'--force', '--with_branch_heads']
gclient_cmd.append('--gclientfile=%s' % self._new_gclient_file)
gclient_utils.CheckCallAndFilterAndHeader(gclient_cmd,
always=self._options.verbose, cwd=self._root_dir)
# CheckCallAndFilterAndHeader will raise exception if return
# value is not 0. So we can easily return 0 here.
return 0
def DoGclientSyncForChromium(self):
gclient_cmd = [
'gclient', 'sync', '--verbose', '--reset', '--force',
'--with_branch_heads', '--delete_unversioned_trees'
]
gclient_cmd.append('--gclientfile=%s' %
os.path.basename(self._new_gclient_file))
gclient_utils.CheckCallAndFilterAndHeader(gclient_cmd,
always=self._options.verbose,
cwd=self._root_dir)
def _Run(self, args, options, show_header=True, **kwargs):
# Disable 'unused options' warning | pylint: disable=unused-argument
kwargs.setdefault('cwd', self.checkout_path)
kwargs.setdefault('stdout', self.out_fh)
kwargs['filter_fn'] = self.filter
kwargs.setdefault('print_stdout', False)
env = scm.GIT.ApplyEnvVars(kwargs)
cmd = ['git'] + args
if show_header:
gclient_utils.CheckCallAndFilterAndHeader(cmd, env=env, **kwargs)
else:
gclient_utils.CheckCallAndFilter(cmd, env=env, **kwargs)
def RunAction(dir, command):
"""Runs the action."""
if command[0] == 'python':
# If the hook specified "python" as the first item, the action is a
# Python script. Run it by starting a new copy of the same
# interpreter.
command[0] = sys.executable
try:
gclient_utils.CheckCallAndFilterAndHeader(
command, cwd=dir, always=True)
except gclient_utils.Error, e:
# Use a discrete exit status code of 2 to indicate that a hook action
# failed. Users of this script may wish to treat hook action failures
# differently from VC failures.
print >> sys.stderr, 'Error: %s' % str(e)
sys.exit(2)
def _checkKillTimeout(self, output_block_for=0, kill_raises=False):
cv = threading.Condition()
order = []
output = list(reversed('output'))
def kid_stdout_read(_):
if output:
return output.pop()
else:
with cv:
cv.wait(timeout=output_block_for)
order.append('unblock')
return ''
def kid_kill():
with cv:
order.append('killed')
cv.notify()
if kill_raises:
raise OSError('something went wrong')
kid = self.ProcessIdMock('')
kid.stdout.read = kid_stdout_read
kid.kill = kid_kill # pylint: disable=W0201
cwd = 'bleh'
args = ['ar', 'gs']
gclient_utils.sys.stdout.write(
'\n________ running \'ar gs\' in \'bleh\'\noutput')
os.getcwd()
subprocess2.Popen(
args, cwd=cwd,
stdout=subprocess2.PIPE,
stderr=subprocess2.STDOUT,
bufsize=0).AndReturn(kid)
self.mox.ReplayAll()
# This test relies on the testing machine's ability to process 1 char
# of output in <0.01 seconds set in kill_timeout.
gclient_utils.CheckCallAndFilterAndHeader(
args, cwd=cwd, always=True, kill_timeout=0.01)
self.checkstdout(
'\n________ running \'ar gs\' in \'bleh\'\noutput\n'
'________ running \'ar gs\' in \'bleh\'\noutput')
return order
def Run(self, command, args):
if command != 'runtasks':
print "No run tasks, only show task info"
for task in self.tasks:
print "Task '%s':\n action: %s" % (
task['name'],
gclient_utils.CommandToStr(self.GetTaskAction(task)))
return
for task in self.tasks:
try:
start_time = time.time()
action = self.GetTaskAction(task)
gclient_utils.CheckCallAndFilterAndHeader(action,
cwd=self.root_dir,
always=True)
except (gclient_utils.Error, subprocess2.CalledProcessError), e:
# Use a discrete exit status code of 2 to indicate that a task action
# failed. Users of this script may wish to treat task action failures
# differently from VC failures.
print >> sys.stderr, 'Error: %s' % str(e)
sys.exit(2)
finally:
def RemoveOldSCMCheckouts(self):
gclient_entries = os.path.join(self._root_dir, '.gclient_entries')
if not os.path.isfile(gclient_entries):
# This is the first time gclient sync is being called (ie. this is a new
# checkout), so skip this step since `gclient recurse' requires
# .gclient_entries to be present, even if it does not use it.
return
cmd = ('gclient', 'recurse', '--no-progress', '-j1',
'--gclientfile=%s' % os.path.basename(self._new_gclient_file),
os.path.join(self._tools_dir, 'scm-remove-wrong-checkout.py'))
def _FilterSkippedDependencyMessage(line):
if line.startswith('Skipped omitted dependency'):
return
print line
gclient_utils.CheckCallAndFilterAndHeader(
cmd,
always=self._options.verbose,
print_stdout=False,
cwd=self._root_dir,
filter_fn=_FilterSkippedDependencyMessage)
def RunAndGetFileList(verbose, args, cwd, file_list, stdout=None):
"""Runs svn checkout, update, or status, output to stdout.
The first item in args must be either "checkout", "update", or "status".
svn's stdout is parsed to collect a list of files checked out or updated.
These files are appended to file_list. svn's stdout is also printed to
sys.stdout as in Run.
Args:
verbose: If True, uses verbose output
args: A sequence of command line parameters to be passed to svn.
cwd: The directory where svn is to be run.
Raises:
Error: An error occurred while running the svn command.
"""
stdout = stdout or sys.stdout
# svn update and svn checkout use the same pattern: the first three columns
# are for file status, property status, and lock status. This is followed
# by two spaces, and then the path to the file.
update_pattern = '^... (.*)$'
# The first three columns of svn status are the same as for svn update and
# svn checkout. The next three columns indicate addition-with-history,
# switch, and remote lock status. This is followed by one space, and then
# the path to the file.
status_pattern = '^...... (.*)$'
# args[0] must be a supported command. This will blow up if it's something
# else, which is good. Note that the patterns are only effective when
# these commands are used in their ordinary forms, the patterns are invalid
# for "svn status --show-updates", for example.
pattern = {
'checkout': update_pattern,
'status': status_pattern,
'update': update_pattern,
}[args[0]]
compiled_pattern = re.compile(pattern)
# Place an upper limit.
backoff_time = 5
retries = 0
while True:
retries += 1
previous_list_len = len(file_list)
failure = []
def CaptureMatchingLines(line):
match = compiled_pattern.search(line)
if match:
file_list.append(match.group(1))
if line.startswith('svn: '):
failure.append(line)
try:
gclient_utils.CheckCallAndFilterAndHeader(
['svn'] + args,
cwd=cwd,
always=verbose,
filter_fn=CaptureMatchingLines,
stdout=stdout)
except subprocess2.CalledProcessError:
def IsKnownFailure():
for x in failure:
if (x.startswith('svn: OPTIONS of')
or x.startswith('svn: PROPFIND of')
or x.startswith('svn: REPORT of')
or x.startswith('svn: Unknown hostname')
or x.startswith(
'svn: Server sent unexpected return value')
):
return True
return False
# Subversion client is really misbehaving with Google Code.
if args[0] == 'checkout':
# Ensure at least one file was checked out, otherwise *delete* the
# directory.
if len(file_list) == previous_list_len:
if not IsKnownFailure():
# No known svn error was found, bail out.
raise
# No file were checked out, so make sure the directory is
# deleted in case it's messed up and try again.
# Warning: It's bad, it assumes args[2] is the directory
# argument.
if os.path.isdir(args[2]):
gclient_utils.RemoveDirectory(args[2])
else:
# Progress was made, convert to update since an aborted checkout
# is now an update.
args = ['update'] + args[1:]
else:
# It was an update or export.
# We enforce that some progress has been made or a known failure.
if len(file_list
) == previous_list_len and not IsKnownFailure():
# No known svn error was found and no progress, bail out.
raise
if retries == 10:
raise
print "Sleeping %.1f seconds and retrying...." % backoff_time
time.sleep(backoff_time)
backoff_time *= 1.3
continue
break
def _Run(self, args, options, **kwargs):
kwargs.setdefault('cwd', self.checkout_path)
gclient_utils.CheckCallAndFilterAndHeader(['git'] + args,
always=options.verbose,
**kwargs)
def _Run(self, args, options, **kwargs):
"""Runs a commands that goes to stdout."""
kwargs.setdefault('cwd', self.checkout_path)
gclient_utils.CheckCallAndFilterAndHeader(['svn'] + args,
always=options.verbose,
**kwargs)
