def main(directories): if not directories: directories = [SCRIPT_DIR] for path in directories: cmd = [ sys.executable, os.path.join(find_depot_tools.DEPOT_TOOLS_PATH, 'download_from_google_storage.py'), '--directory', '--num_threads=10', '--bucket', 'chrome-webrtc-resources', '--auto_platform', '--recursive', path, ] print 'Downloading precompiled tools...' # Perform download similar to how gclient hooks execute. try: gclient_utils.CheckCallAndFilterAndHeader(cmd, cwd=SRC_DIR, always=True) except (gclient_utils.Error, subprocess2.CalledProcessError) as e: print 'Error: %s' % str(e) return 2 return 0
def _inner(self, args, test_string): cwd = 'bleh' gclient_utils.sys.stdout.write( '________ running \'boo foo bar\' in \'bleh\'\n') for i in test_string: gclient_utils.sys.stdout.write(i) # pylint: disable=no-member subprocess2.Popen(args, cwd=cwd, stdout=subprocess2.PIPE, stderr=subprocess2.STDOUT, bufsize=0).AndReturn(self.ProcessIdMock(test_string)) os.getcwd() self.mox.ReplayAll() compiled_pattern = gclient_utils.re.compile(r'a(.*)b') line_list = [] capture_list = [] def FilterLines(line): line_list.append(line) assert isinstance(line, str), type(line) match = compiled_pattern.search(line) if match: capture_list.append(match.group(1)) gclient_utils.CheckCallAndFilterAndHeader(args, cwd=cwd, always=True, filter_fn=FilterLines) self.assertEquals(line_list, ['ahah', 'accb', 'allo', 'addb']) self.assertEquals(capture_list, ['cc', 'dd'])
def DoGclientSyncForChromium(self): gclient_cmd = ['gclient', 'sync', '--verbose', '--reset', '--with_branch_heads', '--ignore_locks', '--delete_unversioned_trees'] gclient_cmd.append('--gclientfile=%s' % os.path.basename(self._new_gclient_file)) # As of revision 454e53abae6e4d68ee992b0a93a4174b75519393, # src/xwalk/buildtools does not pass gclient's syntax validation. # gclient_cmd.append('--disable-syntax-validation') gclient_utils.CheckCallAndFilterAndHeader(gclient_cmd, always=self._options.verbose, cwd=self._root_dir)
def DoGclientSyncForChromium(self): gclient_cmd = ['gclient', 'sync', '--verbose', '--reset', '--force', '--with_branch_heads'] gclient_cmd.append('--gclientfile=%s' % self._new_gclient_file) gclient_utils.CheckCallAndFilterAndHeader(gclient_cmd, always=self._options.verbose, cwd=self._root_dir) # CheckCallAndFilterAndHeader will raise exception if return # value is not 0. So we can easily return 0 here. return 0
def DoGclientSyncForChromium(self): gclient_cmd = [ 'gclient', 'sync', '--verbose', '--reset', '--force', '--with_branch_heads', '--delete_unversioned_trees' ] gclient_cmd.append('--gclientfile=%s' % os.path.basename(self._new_gclient_file)) gclient_utils.CheckCallAndFilterAndHeader(gclient_cmd, always=self._options.verbose, cwd=self._root_dir)
def _Run(self, args, options, show_header=True, **kwargs): # Disable 'unused options' warning | pylint: disable=unused-argument kwargs.setdefault('cwd', self.checkout_path) kwargs.setdefault('stdout', self.out_fh) kwargs['filter_fn'] = self.filter kwargs.setdefault('print_stdout', False) env = scm.GIT.ApplyEnvVars(kwargs) cmd = ['git'] + args if show_header: gclient_utils.CheckCallAndFilterAndHeader(cmd, env=env, **kwargs) else: gclient_utils.CheckCallAndFilter(cmd, env=env, **kwargs)
def RunAction(dir, command): """Runs the action.""" if command[0] == 'python': # If the hook specified "python" as the first item, the action is a # Python script. Run it by starting a new copy of the same # interpreter. command[0] = sys.executable try: gclient_utils.CheckCallAndFilterAndHeader( command, cwd=dir, always=True) except gclient_utils.Error, e: # Use a discrete exit status code of 2 to indicate that a hook action # failed. Users of this script may wish to treat hook action failures # differently from VC failures. print >> sys.stderr, 'Error: %s' % str(e) sys.exit(2)
def _checkKillTimeout(self, output_block_for=0, kill_raises=False): cv = threading.Condition() order = [] output = list(reversed('output')) def kid_stdout_read(_): if output: return output.pop() else: with cv: cv.wait(timeout=output_block_for) order.append('unblock') return '' def kid_kill(): with cv: order.append('killed') cv.notify() if kill_raises: raise OSError('something went wrong') kid = self.ProcessIdMock('') kid.stdout.read = kid_stdout_read kid.kill = kid_kill # pylint: disable=W0201 cwd = 'bleh' args = ['ar', 'gs'] gclient_utils.sys.stdout.write( '\n________ running \'ar gs\' in \'bleh\'\noutput') os.getcwd() subprocess2.Popen( args, cwd=cwd, stdout=subprocess2.PIPE, stderr=subprocess2.STDOUT, bufsize=0).AndReturn(kid) self.mox.ReplayAll() # This test relies on the testing machine's ability to process 1 char # of output in <0.01 seconds set in kill_timeout. gclient_utils.CheckCallAndFilterAndHeader( args, cwd=cwd, always=True, kill_timeout=0.01) self.checkstdout( '\n________ running \'ar gs\' in \'bleh\'\noutput\n' '________ running \'ar gs\' in \'bleh\'\noutput') return order
def Run(self, command, args): if command != 'runtasks': print "No run tasks, only show task info" for task in self.tasks: print "Task '%s':\n action: %s" % ( task['name'], gclient_utils.CommandToStr(self.GetTaskAction(task))) return for task in self.tasks: try: start_time = time.time() action = self.GetTaskAction(task) gclient_utils.CheckCallAndFilterAndHeader(action, cwd=self.root_dir, always=True) except (gclient_utils.Error, subprocess2.CalledProcessError), e: # Use a discrete exit status code of 2 to indicate that a task action # failed. Users of this script may wish to treat task action failures # differently from VC failures. print >> sys.stderr, 'Error: %s' % str(e) sys.exit(2) finally:
def RemoveOldSCMCheckouts(self): gclient_entries = os.path.join(self._root_dir, '.gclient_entries') if not os.path.isfile(gclient_entries): # This is the first time gclient sync is being called (ie. this is a new # checkout), so skip this step since `gclient recurse' requires # .gclient_entries to be present, even if it does not use it. return cmd = ('gclient', 'recurse', '--no-progress', '-j1', '--gclientfile=%s' % os.path.basename(self._new_gclient_file), os.path.join(self._tools_dir, 'scm-remove-wrong-checkout.py')) def _FilterSkippedDependencyMessage(line): if line.startswith('Skipped omitted dependency'): return print line gclient_utils.CheckCallAndFilterAndHeader( cmd, always=self._options.verbose, print_stdout=False, cwd=self._root_dir, filter_fn=_FilterSkippedDependencyMessage)
def RunAndGetFileList(verbose, args, cwd, file_list, stdout=None): """Runs svn checkout, update, or status, output to stdout. The first item in args must be either "checkout", "update", or "status". svn's stdout is parsed to collect a list of files checked out or updated. These files are appended to file_list. svn's stdout is also printed to sys.stdout as in Run. Args: verbose: If True, uses verbose output args: A sequence of command line parameters to be passed to svn. cwd: The directory where svn is to be run. Raises: Error: An error occurred while running the svn command. """ stdout = stdout or sys.stdout # svn update and svn checkout use the same pattern: the first three columns # are for file status, property status, and lock status. This is followed # by two spaces, and then the path to the file. update_pattern = '^... (.*)$' # The first three columns of svn status are the same as for svn update and # svn checkout. The next three columns indicate addition-with-history, # switch, and remote lock status. This is followed by one space, and then # the path to the file. status_pattern = '^...... (.*)$' # args[0] must be a supported command. This will blow up if it's something # else, which is good. Note that the patterns are only effective when # these commands are used in their ordinary forms, the patterns are invalid # for "svn status --show-updates", for example. pattern = { 'checkout': update_pattern, 'status': status_pattern, 'update': update_pattern, }[args[0]] compiled_pattern = re.compile(pattern) # Place an upper limit. backoff_time = 5 retries = 0 while True: retries += 1 previous_list_len = len(file_list) failure = [] def CaptureMatchingLines(line): match = compiled_pattern.search(line) if match: file_list.append(match.group(1)) if line.startswith('svn: '): failure.append(line) try: gclient_utils.CheckCallAndFilterAndHeader( ['svn'] + args, cwd=cwd, always=verbose, filter_fn=CaptureMatchingLines, stdout=stdout) except subprocess2.CalledProcessError: def IsKnownFailure(): for x in failure: if (x.startswith('svn: OPTIONS of') or x.startswith('svn: PROPFIND of') or x.startswith('svn: REPORT of') or x.startswith('svn: Unknown hostname') or x.startswith( 'svn: Server sent unexpected return value') ): return True return False # Subversion client is really misbehaving with Google Code. if args[0] == 'checkout': # Ensure at least one file was checked out, otherwise *delete* the # directory. if len(file_list) == previous_list_len: if not IsKnownFailure(): # No known svn error was found, bail out. raise # No file were checked out, so make sure the directory is # deleted in case it's messed up and try again. # Warning: It's bad, it assumes args[2] is the directory # argument. if os.path.isdir(args[2]): gclient_utils.RemoveDirectory(args[2]) else: # Progress was made, convert to update since an aborted checkout # is now an update. args = ['update'] + args[1:] else: # It was an update or export. # We enforce that some progress has been made or a known failure. if len(file_list ) == previous_list_len and not IsKnownFailure(): # No known svn error was found and no progress, bail out. raise if retries == 10: raise print "Sleeping %.1f seconds and retrying...." % backoff_time time.sleep(backoff_time) backoff_time *= 1.3 continue break
def _Run(self, args, options, **kwargs): kwargs.setdefault('cwd', self.checkout_path) gclient_utils.CheckCallAndFilterAndHeader(['git'] + args, always=options.verbose, **kwargs)
def _Run(self, args, options, **kwargs): """Runs a commands that goes to stdout.""" kwargs.setdefault('cwd', self.checkout_path) gclient_utils.CheckCallAndFilterAndHeader(['svn'] + args, always=options.verbose, **kwargs)