def test_check_call_nonzero(self): # check_call() function with non-zero return code try: subprocess.check_call([sys.executable, "-c", "import sys; sys.exit(47)"]) except subprocess.CalledProcessError, e: self.assertEqual(e.returncode, 47)
def main(args): default_args = git.config_list('depot-tools.upstream-diff.default-args') args = default_args + args parser = argparse.ArgumentParser() parser.add_argument('--wordwise', action='store_true', default=False, help=( 'Print a colorized wordwise diff ' 'instead of line-wise diff')) opts, extra_args = parser.parse_known_args(args) cur = git.current_branch() if not cur or cur == 'HEAD': print 'fatal: Cannot perform git-upstream-diff while not on a branch' return 1 par = git.upstream(cur) if not par: print 'fatal: No upstream configured for branch \'%s\'' % cur return 1 cmd = [git.GIT_EXE, 'diff', '--patience', '-C', '-C'] if opts.wordwise: cmd += ['--word-diff=color', r'--word-diff-regex=(\w+|[^[:space:]])'] cmd += [git.get_or_create_merge_base(cur, par)] cmd += extra_args subprocess2.check_call(cmd)
def set_up_svn(self): """Creates subversion repositories and start the servers.""" self.set_up() if self.svnserve: return True try: subprocess2.check_call(["svnadmin", "create", self.svn_repo]) except (OSError, subprocess2.CalledProcessError): return False write( join(self.svn_repo, "conf", "svnserve.conf"), "[general]\n" "anon-access = read\n" "auth-access = write\n" "password-db = passwd\n", ) text = "[users]\n" text += "".join("%s = %s\n" % (usr, pwd) for usr, pwd in self.USERS) write(join(self.svn_repo, "conf", "passwd"), text) # Mac 10.6 ships with a buggy subversion build and we need this line # to work around the bug. write(join(self.svn_repo, "db", "fsfs.conf"), "[rep-sharing]\n" "enable-rep-sharing = false\n") # Start the daemon. self.svn_port = find_free_port(self.host, 10000) logging.debug("Using port %d" % self.svn_port) cmd = ["svnserve", "-d", "--foreground", "-r", self.root_dir, "--listen-port=%d" % self.svn_port] if self.host == "127.0.0.1": cmd.append("--listen-host=" + self.host) self.check_port_is_free(self.svn_port) self.svnserve = subprocess2.Popen(cmd, cwd=self.svn_repo, stdout=subprocess2.PIPE, stderr=subprocess2.PIPE) wait_for_port_to_bind(self.host, self.svn_port, self.svnserve) self.svn_base = "svn://%s:%d/svn/" % (self.host, self.svn_port) self.populateSvn() self.svn_dirty = False return True
def populateSvn(self): """Creates a few revisions of changes files.""" subprocess2.check_call( ['svn', 'checkout', self.svn_base, self.svn_checkout, '-q', '--non-interactive', '--no-auth-cache', '--username', self.USERS[0][0], '--password', self.USERS[0][1]]) assert os.path.isdir(os.path.join(self.svn_checkout, '.svn')) fs = {} fs['trunk/origin'] = 'svn@1' fs['trunk/codereview.settings'] = ( '# Test data\n' 'bar: pouet\n') fs['trunk/svn_utils_test.txt'] = ( 'a\n' 'bb\n' 'ccc\n' 'dd\n' 'e\n' 'ff\n' 'ggg\n' 'hh\n' 'i\n' 'jj\n' 'kkk\n' 'll\n' 'm\n' 'nn\n' 'ooo\n' 'pp\n' 'q\n') self._commit_svn(fs) fs['trunk/origin'] = 'svn@2\n' fs['trunk/extra'] = 'dummy\n' fs['trunk/bin_file'] = '\x00' self._commit_svn(fs)
def RunEditor(content, git, git_editor=None): """Opens up the default editor in the system to get the CL description.""" file_handle, filename = tempfile.mkstemp(text=True, prefix='cl_description') # Make sure CRLF is handled properly by requiring none. if '\r' in content: print('!! Please remove \\r from your change description !!', file=sys.stderr) fileobj = os.fdopen(file_handle, 'w') # Still remove \r if present. content = re.sub('\r?\n', '\n', content) # Some editors complain when the file doesn't end in \n. if not content.endswith('\n'): content += '\n' fileobj.write(content) fileobj.close() try: editor = GetEditor(git_editor=git_editor) if not editor: return None cmd = '%s %s' % (editor, filename) if sys.platform == 'win32' and os.environ.get('TERM') == 'msys': # Msysgit requires the usage of 'env' to be present. cmd = 'env ' + cmd try: # shell=True to allow the shell to handle all forms of quotes in # $EDITOR. subprocess2.check_call(cmd, shell=True) except subprocess2.CalledProcessError: return None return FileRead(filename) finally: os.remove(filename)
def populateSvn(self): """Create revisions which simulate the Skia DEPS transition in Chrome.""" subprocess2.check_call( ['svn', 'checkout', self.svn_base, self.svn_checkout, '-q', '--non-interactive', '--no-auth-cache', '--username', self.USERS[0][0], '--password', self.USERS[0][1]]) assert os.path.isdir(join(self.svn_checkout, '.svn')) # Skia repo. self._commit_svn({ 'skia/skia_base_file': 'root-level file.', 'skia/gyp/gyp_file': 'file in the gyp directory', 'skia/include/include_file': 'file in the include directory', 'skia/src/src_file': 'file in the src directory', }) # Chrome repo. self._commit_svn({ 'trunk/src/DEPS': self.DEPS_svn_pre % {'svn_base': self.svn_base}, 'trunk/src/myfile': 'svn/trunk/src@1' }) self._commit_svn({ 'trunk/src/DEPS': self.DEPS_post % {'git_base': self.git_base}, 'trunk/src/myfile': 'svn/trunk/src@2' })
def testBlinkLocalBranchesArePreserved(self): """Checks that the state of local git branches are effectively preserved when going back and forth.""" if not self.enabled: return self.gclient(['config', '--spec', 'solutions=[' '{"name": "src",' ' "url": "' + self.git_base + 'repo_1",' '}]']) # Initialize to pre-merge point. self.gclient(['sync', '--revision', 'src@%s' % self.pre_merge_sha]) self.CheckStatusPreMergePoint() # Create a branch named "foo". subprocess2.check_call(['git', 'checkout', '-qB', 'foo'], cwd=self.blink) # Cross the pre-merge point. self.gclient(['sync', '--revision', 'src@%s' % self.post_merge_sha]) self.CheckStatusPostMergePoint() # Go backwards and check that we still have the foo branch. self.gclient(['sync', '--revision', 'src@%s' % self.pre_merge_sha]) self.CheckStatusPreMergePoint() subprocess2.check_call( ['git', 'show-ref', '-q', '--verify', 'refs/heads/foo'], cwd=self.blink)
def set_up_git(self): """Creates git repositories and start the servers.""" self.set_up() if self.gitdaemon: return True assert self.git_pid_file == None try: subprocess2.check_output(["git", "--version"]) except (OSError, subprocess2.CalledProcessError): return False for repo in ["repo_%d" % r for r in range(1, self.NB_GIT_REPOS + 1)]: subprocess2.check_call(["git", "init", "-q", join(self.git_root, repo)]) self.git_hashes[repo] = [None] self.git_port = find_free_port(self.host, 20000) self.git_base = "git://%s:%d/git/" % (self.host, self.git_port) # Start the daemon. self.git_pid_file = tempfile.NamedTemporaryFile() cmd = [ "git", "daemon", "--export-all", "--reuseaddr", "--base-path=" + self.root_dir, "--pid-file=" + self.git_pid_file.name, "--port=%d" % self.git_port, ] if self.host == "127.0.0.1": cmd.append("--listen=" + self.host) self.check_port_is_free(self.git_port) self.gitdaemon = subprocess2.Popen(cmd, cwd=self.root_dir, stdout=subprocess2.PIPE, stderr=subprocess2.PIPE) wait_for_port_to_bind(self.host, self.git_port, self.gitdaemon) self.populateGit() self.git_dirty = False return True
def testBlinkDEPSChangeUsingGit(self): """Like testBlinkDEPSChangeUsingGclient, but move the main project using directly git and not gclient sync.""" if not self.enabled: return self.gclient([ 'config', '--spec', 'solutions=[' '{"name": "src",' ' "url": "' + self.git_base + 'repo_1",' ' "managed": False,' '}]' ]) # Perform an initial sync to bootstrap the repo. res = self.gclient(['sync', '--jobs', '1']) self.assertEqual(res[2], 0, 'Initial gclient sync failed.') # Go back and forth two times. for _ in xrange(2): subprocess2.check_call( ['git', 'checkout', '-q', self.pre_merge_sha], cwd=self.checkout_path) res = self.gclient(['sync', '--jobs', '1']) self.assertEqual(res[2], 0, 'gclient sync failed.') self.CheckStatusPreMergePoint() subprocess2.check_call( ['git', 'checkout', '-q', self.post_merge_sha], cwd=self.checkout_path) res = self.gclient(['sync', '--jobs', '1']) self.assertEqual(res[2], 0, 'DEPS change sync failed.') self.CheckStatusPostMergePoint()
def testBlinkLocalBranchesArePreserved(self): """Checks that the state of local git branches are effectively preserved when going back and forth.""" if not self.enabled: return self.gclient([ 'config', '--spec', 'solutions=[' '{"name": "src",' ' "url": "' + self.git_base + 'repo_1",' '}]' ]) # Initialize to pre-merge point. self.gclient(['sync', '--revision', 'src@%s' % self.pre_merge_sha]) self.CheckStatusPreMergePoint() # Create a branch named "foo". subprocess2.check_call(['git', 'checkout', '-qB', 'foo'], cwd=self.blink) # Cross the pre-merge point. self.gclient(['sync', '--revision', 'src@%s' % self.post_merge_sha]) self.CheckStatusPostMergePoint() # Go backwards and check that we still have the foo branch. self.gclient(['sync', '--revision', 'src@%s' % self.pre_merge_sha]) self.CheckStatusPreMergePoint() subprocess2.check_call( ['git', 'show-ref', '-q', '--verify', 'refs/heads/foo'], cwd=self.blink)
def RunEditor(content, git, git_editor=None): """Opens up the default editor in the system to get the CL description.""" file_handle, filename = tempfile.mkstemp(text=True, prefix='cl_description') # Make sure CRLF is handled properly by requiring none. if '\r' in content: print >> sys.stderr, ( '!! Please remove \\r from your change description !!') fileobj = os.fdopen(file_handle, 'w') # Still remove \r if present. content = re.sub('\r?\n', '\n', content) # Some editors complain when the file doesn't end in \n. if not content.endswith('\n'): content += '\n' fileobj.write(content) fileobj.close() try: editor = GetEditor(git, git_editor=git_editor) if not editor: return None cmd = '%s %s' % (editor, filename) if sys.platform == 'win32' and os.environ.get('TERM') == 'msys': # Msysgit requires the usage of 'env' to be present. cmd = 'env ' + cmd try: # shell=True to allow the shell to handle all forms of quotes in # $EDITOR. subprocess2.check_call(cmd, shell=True) except subprocess2.CalledProcessError: return None return FileRead(filename) finally: os.remove(filename)
def _compare_revisions(self, last_roll_revision, new_roll_revision): """Ensure that new_roll_revision is newer than last_roll_revision. Raises: AutoRollException if new_roll_revision is not newer than than last_roll_revision. """ git_dir = self._path_from_chromium_root(self._path_to_project, '.git') if (scm.GIT.IsValidRevision(git_dir, last_roll_revision) and scm.GIT.IsValidRevision(git_dir, new_roll_revision)): # Ensure that new_roll_revision is not an ancestor of old_roll_revision. try: subprocess2.check_call([ 'git', 'merge-base', '--is-ancestor', new_roll_revision, last_roll_revision ]) raise AutoRollException( 'Already at %s refusing to roll backwards to ' '%s.' % (last_roll_revision, new_roll_revision)) except subprocess2.CalledProcessError: pass else: # Fall back on svn revisions. if int(new_roll_revision) <= int(last_roll_revision): raise AutoRollException( 'Already at %s refusing to roll backwards to %s.' % (last_roll_revision, new_roll_revision))
def testBlinkDEPSChangeUsingGit(self): """Like testBlinkDEPSChangeUsingGclient, but move the main project using directly git and not gclient sync.""" if not self.enabled: return self.gclient(['config', '--spec', 'solutions=[' '{"name": "src",' ' "url": "' + self.git_base + 'repo_1",' ' "managed": False,' '}]']) # Perform an initial sync to bootstrap the repo. res = self.gclient(['sync', '--jobs', '1']) self.assertEqual(res[2], 0, 'Initial gclient sync failed.') # Go back and forth two times. for _ in xrange(2): subprocess2.check_call(['git', 'checkout', '-q', self.pre_merge_sha], cwd=self.checkout_path) res = self.gclient(['sync', '--jobs', '1']) self.assertEqual(res[2], 0, 'gclient sync failed.') self.CheckStatusPreMergePoint() subprocess2.check_call(['git', 'checkout', '-q', self.post_merge_sha], cwd=self.checkout_path) res = self.gclient(['sync', '--jobs', '1']) self.assertEqual(res[2], 0, 'DEPS change sync failed.') self.CheckStatusPostMergePoint()
def populateSvn(self): """Create revisions which simulate the Skia DEPS transition in Chrome.""" subprocess2.check_call([ 'svn', 'checkout', self.svn_base, self.svn_checkout, '-q', '--non-interactive', '--no-auth-cache', '--username', self.USERS[0][0], '--password', self.USERS[0][1] ]) assert os.path.isdir(join(self.svn_checkout, '.svn')) # Skia repo. self._commit_svn({ 'skia/skia_base_file': 'root-level file.', 'skia/gyp/gyp_file': 'file in the gyp directory', 'skia/include/include_file': 'file in the include directory', 'skia/src/src_file': 'file in the src directory', }) # Chrome repo. self._commit_svn({ 'trunk/src/DEPS': self.DEPS_svn_pre % { 'svn_base': self.svn_base }, 'trunk/src/myfile': 'svn/trunk/src@1' }) self._commit_svn({ 'trunk/src/DEPS': self.DEPS_post % { 'git_base': self.git_base }, 'trunk/src/myfile': 'svn/trunk/src@2' })
def main(): tool_dir = os.path.dirname(os.path.abspath(__file__)) parser = optparse.OptionParser(usage='<new webkit rev>') parser.add_option('-v', '--verbose', action='count', default=0) parser.add_option('--commit', action='store_true', default=True, help='(default) Put change in commit queue on upload.') parser.add_option('--no-commit', action='store_false', dest='commit', help='Don\'t put change in commit queue on upload.') parser.add_option('-r', '--reviewers', default='', help='Add given users as either reviewers or TBR as' ' appropriate.') parser.add_option('--upstream', default='origin/master', help='(default "%default") Use given start point for change' ' to upload. For instance, if you use the old git workflow,' ' you might set it to "origin/trunk".') parser.add_option('--cc', help='CC email addresses for issue.') options, args = parser.parse_args() logging.basicConfig( level= [logging.WARNING, logging.INFO, logging.DEBUG][ min(2, options.verbose)]) if len(args) != 1: parser.error('Need only one arg: new webkit revision to roll to.') root_dir = os.path.dirname(tool_dir) os.chdir(root_dir) new_rev = int(args[0]) print 'Roll webkit revision to %s' % new_rev # Silence the editor. os.environ['EDITOR'] = 'true' old_branch = scm.GIT.GetBranch(root_dir) if old_branch == 'webkit_roll': parser.error( 'Please delete the branch webkit_roll and move to a different branch') subprocess2.check_output( ['git', 'checkout', '-b', 'webkit_roll', options.upstream]) try: old_rev = process_deps(os.path.join(root_dir, 'DEPS'), new_rev) review_field = 'TBR' if options.commit else 'R' commit_msg = 'Webkit roll %s:%s\n\n%s=%s\n' % (old_rev, new_rev, review_field, options.reviewers) subprocess2.check_output(['git', 'commit', '-m', commit_msg, 'DEPS']) subprocess2.check_call(['git', 'diff', options.upstream]) upload_cmd = ['git', 'cl', 'upload'] if options.commit: upload_cmd.append('--use-commit-queue') if options.reviewers: upload_cmd.append('--send-mail') if options.cc: upload_cmd.extend(['--cc', options.cc]) subprocess2.check_call(upload_cmd) finally: subprocess2.check_output(['git', 'checkout', old_branch]) subprocess2.check_output(['git', 'branch', '-D', 'webkit_roll']) return 0
def main(args): default_args = git.config_list('depot-tools.upstream-diff.default-args') args = default_args + args parser = argparse.ArgumentParser() parser.add_argument('--wordwise', action='store_true', default=False, help=('Print a colorized wordwise diff ' 'instead of line-wise diff')) opts, extra_args = parser.parse_known_args(args) cur = git.current_branch() if not cur or cur == 'HEAD': print 'fatal: Cannot perform git-upstream-diff while not on a branch' return 1 par = git.upstream(cur) if not par: print 'fatal: No upstream configured for branch \'%s\'' % cur return 1 cmd = [git.GIT_EXE, 'diff', '--patience', '-C', '-C'] if opts.wordwise: cmd += ['--word-diff=color', r'--word-diff-regex=(\w+|[^[:space:]])'] cmd += [git.get_or_create_merge_base(cur, par)] cmd += extra_args subprocess2.check_call(cmd)
def main(): tool_dir = os.path.dirname(os.path.abspath(__file__)) parser = optparse.OptionParser(usage="<new webkit rev>") parser.add_option("-v", "--verbose", action="count", default=0) options, args = parser.parse_args() logging.basicConfig(level=[logging.WARNING, logging.INFO, logging.DEBUG][min(2, options.verbose)]) if len(args) != 1: parser.error("Need only one arg: new webkit revision to roll to.") root_dir = os.path.dirname(tool_dir) os.chdir(root_dir) new_rev = int(args[0]) print "Roll webkit revision to %s" % new_rev # Silence the editor. os.environ["EDITOR"] = "true" old_branch = scm.GIT.GetBranch(root_dir) if old_branch == "webkit_roll": parser.error("Please delete the branch webkit_roll and move to a different branch") subprocess2.check_output(["git", "checkout", "-b", "webkit_roll", "origin/master"]) try: old_rev = process_deps(os.path.join(root_dir, "DEPS"), new_rev) commit_msg = "Webkit roll %s:%s\n\nTBR=\n" % (old_rev, new_rev) subprocess2.check_output(["git", "commit", "-m", commit_msg, "DEPS"]) subprocess2.check_call(["git", "diff", "origin/master"]) subprocess2.check_call(["git", "cl", "upload", "--use-commit-queue"]) finally: subprocess2.check_output(["git", "checkout", old_branch]) subprocess2.check_output(["git", "branch", "-D", "webkit_roll"]) return 0
def get_dllexports(dll): """ Get exports of DLL. """ dll2def = os.path.join(env.bin_dir, 'dll2def.exe') if not os.path.isfile(dll2def): build_script = os.path.join(env.src_dir, 'dll2def', 'build.bat') try: subproc.check_call([ build_script ], stderr=subproc.VOID, stdout=subproc.VOID, stdin=subproc.VOID) except subproc.CalledProcessError: import traceback traceback.print_exc() fatal_message( 'Failed to build dll2def', ''.join(( 'There was an error while trying to build dll2def. To resolve this issue, you ', 'may want to check the following files to make sure they are correct:\n\n', ' /repository/util/src/dll2def/build.bat\n' ' /repository/util/bin/msvcc.bat\n' ' /repository/util/bin/_buildenv.bat\n\n' 'If you continue having issue, please report this error at this project\'s GitHub. ', '(https://github.com/Juntalis/dypywin32)' )) ) return split_lines_clean(subproc.check_output( [ dll2def, env.data_file(dll) ], stderr=subproc.VOID ))
def RunEditor(content, git): """Opens up the default editor in the system to get the CL description.""" file_handle, filename = tempfile.mkstemp(text=True) # Make sure CRLF is handled properly by requiring none. if '\r' in content: print >> sys.stderr, ( '!! Please remove \\r from your change description !!') fileobj = os.fdopen(file_handle, 'w') # Still remove \r if present. fileobj.write(re.sub('\r?\n', '\n', content)) fileobj.close() try: cmd = '%s %s' % (GetEditor(git), filename) if sys.platform == 'win32' and os.environ.get('TERM') == 'msys': # Msysgit requires the usage of 'env' to be present. cmd = 'env ' + cmd try: # shell=True to allow the shell to handle all forms of quotes in # $EDITOR. subprocess2.check_call(cmd, shell=True) except subprocess2.CalledProcessError: return None return FileRead(filename) finally: os.remove(filename)
def set_up_git(self): """Creates git repositories and start the servers.""" self.set_up() if self.gitdaemon: return True assert self.git_pid_file == None try: subprocess2.check_output(['git', '--version']) except (OSError, subprocess2.CalledProcessError): return False for repo in ['repo_%d' % r for r in range(1, self.NB_GIT_REPOS + 1)]: subprocess2.check_call(['git', 'init', '-q', join(self.git_root, repo)]) self.git_hashes[repo] = [None] self.git_port = find_free_port(self.host, 20000) self.git_base = 'git://%s:%d/git/' % (self.host, self.git_port) # Start the daemon. self.git_pid_file = tempfile.NamedTemporaryFile() cmd = ['git', 'daemon', '--export-all', '--reuseaddr', '--base-path=' + self.root_dir, '--pid-file=' + self.git_pid_file.name, '--port=%d' % self.git_port] if self.host == '127.0.0.1': cmd.append('--listen=' + self.host) self.check_port_is_free(self.git_port) self.gitdaemon = subprocess2.Popen( cmd, cwd=self.root_dir, stdout=subprocess2.PIPE, stderr=subprocess2.PIPE) wait_for_port_to_bind(self.host, self.git_port, self.gitdaemon) self.populateGit() self.git_dirty = False return True
def commit_git(repo): """Commits the changes and returns the new hash.""" subprocess2.check_call(["git", "add", "-A", "-f"], cwd=repo) subprocess2.check_call(["git", "commit", "-q", "--message", "foo"], cwd=repo) rev = subprocess2.check_output(["git", "show-ref", "--head", "HEAD"], cwd=repo).split(" ", 1)[0] logging.debug("At revision %s" % rev) return rev
def RunEditor(content, git, git_editor=None): """Opens up the default editor in the system to get the CL description.""" file_handle, filename = tempfile.mkstemp(text=True) # Make sure CRLF is handled properly by requiring none. if "\r" in content: print >> sys.stderr, ("!! Please remove \\r from your change description !!") fileobj = os.fdopen(file_handle, "w") # Still remove \r if present. fileobj.write(re.sub("\r?\n", "\n", content)) fileobj.close() try: editor = GetEditor(git, git_editor=git_editor) if not editor: return None cmd = "%s %s" % (editor, filename) if sys.platform == "win32" and os.environ.get("TERM") == "msys": # Msysgit requires the usage of 'env' to be present. cmd = "env " + cmd try: # shell=True to allow the shell to handle all forms of quotes in # $EDITOR. subprocess2.check_call(cmd, shell=True) except subprocess2.CalledProcessError: return None return FileRead(filename) finally: os.remove(filename)
def test_check_call_throw(self): try: subprocess2.check_call(self.exe + ['--fail', '--stderr']) self.fail() except subprocess2.CalledProcessError, e: self.assertEquals(None, e.stdout) self.assertEquals(None, e.stderr) self.assertEquals(64, e.returncode)
def install_prerequisites(self): # First, install the Google AppEngine SDK. cmd = [os.path.join(self.base_dir, 'get_appengine.py'), '--dest=%s' % self.base_dir] try: subprocess2.check_call(cmd) except (OSError, subprocess2.CalledProcessError), e: raise Failure('Failed to run %s\n%s' % (cmd, e))
def commit_git(repo): """Commits the changes and returns the new hash.""" subprocess2.check_call(['git', 'add', '-A', '-f'], cwd=repo) subprocess2.check_call(['git', 'commit', '-q', '--message', 'foo'], cwd=repo) rev = subprocess2.check_output( ['git', 'show-ref', '--head', 'HEAD'], cwd=repo).split(' ', 1)[0] logging.debug('At revision %s' % rev) return rev
def system3(*parms, **kwargs): print subprocess.list2cmdline(parms) try: subprocess.check_call(parms, **kwargs) except (OSError, subprocess.CalledProcessError), err: print "execution failure, aborting:", parms print err sys.exit(2)
def _run_oauth_dance(scopes): """Perform full 3-legged OAuth2 flow with the browser. Returns: oauth2client.Credentials. """ subprocess2.check_call(['luci-auth', 'login', '-scopes', scopes]) return _get_luci_auth_credentials(scopes)
def CMDrebase(parser, args): """rebase current branch on top of svn repo""" # Provide a wrapper for git svn rebase to help avoid accidental # git svn dcommit. # It's the only command that doesn't use parser at all since we just defer # execution to git-svn. subprocess2.check_call(['git', 'svn', 'rebase'] + args) return 0
def _SendChangeSVN(options): """Send a change to the try server by committing a diff file on a subversion server.""" if not options.svn_repo: raise NoTryServerAccess( 'Please use the --svn_repo option to specify the' ' try server svn repository to connect to.') values = _ParseSendChangeOptions(options) description = ''.join("%s=%s\n" % (k, v) for k, v in values) logging.info('Sending by SVN') logging.info(description) logging.info(options.svn_repo) logging.info(options.diff) if options.dry_run: return # Create a temporary directory, put a uniquely named file in it with the diff # content and svn import that. temp_dir = tempfile.mkdtemp() temp_file = tempfile.NamedTemporaryFile() try: try: # Description temp_file.write(description) temp_file.flush() # Diff file current_time = str(datetime.datetime.now()).replace(':', '.') file_name = (Escape(options.user) + '.' + Escape(options.name) + '.%s.diff' % current_time) full_path = os.path.join(temp_dir, file_name) with open(full_path, 'wb') as f: f.write(options.diff) # Committing it will trigger a try job. if sys.platform == "cygwin": # Small chromium-specific issue here: # git-try uses /usr/bin/python on cygwin but svn.bat will be used # instead of /usr/bin/svn by default. That causes bad things(tm) since # Windows' svn.exe has no clue about cygwin paths. Hence force to use # the cygwin version in this particular context. exe = "/usr/bin/svn" else: exe = "svn" command = [ exe, 'import', '-q', temp_dir, options.svn_repo, '--file', temp_file.name ] if scm.SVN.AssertVersion("1.5")[0]: command.append('--no-ignore') subprocess2.check_call(command) except subprocess2.CalledProcessError, e: raise NoTryServerAccess(str(e)) finally: temp_file.close() shutil.rmtree(temp_dir, True)
def populateSvn(self): """Creates a few revisions of changes files.""" subprocess2.check_call( ['svn', 'checkout', self.svn_base, self.svn_checkout, '-q', '--non-interactive', '--no-auth-cache', '--username', self.USERS[0][0], '--password', self.USERS[0][1]]) assert os.path.isdir(os.path.join(self.svn_checkout, '.svn')) self._commit_svn(self._svn_tree_1()) self._commit_svn(self._svn_tree_2())
def _run_luci_auth_login(self): """Run luci-auth login. Returns: AccessToken with credentials. """ logging.debug('Running luci-auth login') subprocess2.check_call(['luci-auth', 'login', '-scopes', self._scopes]) return self._get_luci_auth_token()
def populateSvn(self): """Creates a few revisions of changes files.""" subprocess2.check_call( ['svn', 'checkout', self.svn_base, self.svn_checkout, '-q', '--non-interactive', '--no-auth-cache', '--username', self.USERS[0][0], '--password', self.USERS[0][1]]) assert os.path.isdir(os.path.join(self.svn_checkout, '.svn')) self._commit_svn(self._tree_1()) self._commit_svn(self._tree_2())
def _last_roll_revision(self): if not self._cached_last_roll_revision: git_dir = self._path_from_chromium_root('.git') subprocess2.check_call(['git', '--git-dir', git_dir, 'fetch']) git_show_cmd = ['git', '--git-dir', git_dir, 'show', 'origin/master:DEPS'] deps_contents = subprocess2.check_output(git_show_cmd) pattern = self.REVISION_REGEXP % self._project_alias match = re.search(pattern, deps_contents, re.MULTILINE) self._cached_last_roll_revision = match.group('revision') return self._cached_last_roll_revision
def _SendChangeSVN(options): """Send a change to the try server by committing a diff file on a subversion server.""" if not options.svn_repo: raise NoTryServerAccess('Please use the --svn_repo option to specify the' ' try server svn repository to connect to.') values = _ParseSendChangeOptions(options) description = ''.join("%s=%s\n" % (k, v) for k, v in values) logging.info('Sending by SVN') logging.info(description) logging.info(options.svn_repo) logging.info(options.diff) if options.dry_run: return # Create a temporary directory, put a uniquely named file in it with the diff # content and svn import that. temp_dir = tempfile.mkdtemp() temp_file = tempfile.NamedTemporaryFile() try: try: # Description temp_file.write(description) temp_file.flush() # Diff file current_time = str(datetime.datetime.now()).replace(':', '.') file_name = (Escape(options.user) + '.' + Escape(options.name) + '.%s.diff' % current_time) full_path = os.path.join(temp_dir, file_name) with open(full_path, 'wb') as f: f.write(options.diff) # Committing it will trigger a try job. if sys.platform == "cygwin": # Small chromium-specific issue here: # git-try uses /usr/bin/python on cygwin but svn.bat will be used # instead of /usr/bin/svn by default. That causes bad things(tm) since # Windows' svn.exe has no clue about cygwin paths. Hence force to use # the cygwin version in this particular context. exe = "/usr/bin/svn" else: exe = "svn" command = [exe, 'import', '-q', temp_dir, options.svn_repo, '--file', temp_file.name] if scm.SVN.AssertVersion("1.5")[0]: command.append('--no-ignore') subprocess2.check_call(command) except subprocess2.CalledProcessError, e: raise NoTryServerAccess(str(e)) finally: temp_file.close() shutil.rmtree(temp_dir, True)
def logout(self): """Revokes the refresh token and deletes it from the cache. Returns True if had some credentials cached. """ with self._lock: self._access_token = None had_creds = bool(_get_luci_auth_credentials(self._scopes)) subprocess2.check_call( ['luci-auth', 'logout', '-scopes', self._scopes]) return had_creds
class LocalRietveld(object): """Downloads everything needed to run a local instance of Rietveld.""" def __init__(self, base_dir=None): # Paths self.base_dir = base_dir if not self.base_dir: self.base_dir = os.path.dirname(os.path.abspath(__file__)) # TODO(maruel): This should be in /tmp but that would mean having to fetch # everytime. This test is already annoyingly slow. self.rietveld = os.path.join(self.base_dir, '_rietveld') self.rietveld_app = os.path.join( self.rietveld, 'appengine', 'chromium_rietveld') self.test_server = None self.port = None self.tempdir = None self.dev_app = None def install_prerequisites(self): # First, install the Google AppEngine SDK. cmd = [os.path.join(self.base_dir, 'get_appengine.py'), '--dest=%s' % self.base_dir] try: subprocess2.check_call(cmd) except (OSError, subprocess2.CalledProcessError), e: raise Failure('Failed to run %s\n%s' % (cmd, e)) sdk_path = os.path.join(self.base_dir, 'google_appengine') self.dev_app = os.path.join(sdk_path, 'dev_appserver.py') if os.path.isdir(os.path.join(self.rietveld, '.hg')): # Left over from mercurial. Delete it. print('Deleting deprecated mercurial rietveld files...') shutil.rmtree(self.rietveld) # Second, checkout rietveld if not available. if not os.path.isdir(self.rietveld): print('Checking out rietveld...') try: subprocess2.check_call(['git', 'init', self.rietveld]) subprocess2.check_call( ['git', 'remote', 'add', '-f', 'origin', 'https://chromium.googlesource.com/infra/infra.git'], cwd=self.rietveld) subprocess2.check_call( ['git', 'config', 'core.sparseCheckout', 'true'], cwd=self.rietveld) with file(os.path.join(self.rietveld, '.git/info/sparse-checkout'), 'w') as sparse_file: sparse_file.write('appengine/chromium_rietveld') subprocess2.check_call( ['git', 'pull', 'origin', 'master'], cwd=self.rietveld) except (OSError, subprocess2.CalledProcessError), e: raise Failure('Failed to clone rietveld. \n%s' % e)
def setUp(self): super(GClientSmokeFromCheckout, self).setUp() self.enabled = self.FAKE_REPOS.set_up_svn() os.rmdir(self.root_dir) if self.enabled: usr, pwd = self.FAKE_REPOS.USERS[0] subprocess2.check_call( ['svn', 'checkout', self.svn_base + '/trunk/webkit', self.root_dir, '-q', '--non-interactive', '--no-auth-cache', '--username', usr, '--password', pwd])
def _last_roll_revision(self): if not self._cached_last_roll_revision: git_dir = self._path_from_chromium_root('.git') subprocess2.check_call(['git', '--git-dir', git_dir, 'fetch']) git_show_cmd = [ 'git', '--git-dir', git_dir, 'show', 'origin/master:DEPS' ] deps_contents = subprocess2.check_output(git_show_cmd) pattern = self.REVISION_REGEXP % self._project_alias match = re.search(pattern, deps_contents, re.MULTILINE) self._cached_last_roll_revision = match.group('revision') return self._cached_last_roll_revision
def set_up_svn(self): """Creates subversion repositories and start the servers.""" self.set_up() if self.svnserve: return True try: subprocess2.check_call(['svnadmin', 'create', self.svn_repo]) except (OSError, subprocess2.CalledProcessError): return False write(join(self.svn_repo, 'conf', 'svnserve.conf'), '[general]\n' 'anon-access = read\n' 'auth-access = write\n' 'password-db = passwd\n') text = '[users]\n' text += ''.join('%s = %s\n' % (usr, pwd) for usr, pwd in self.USERS) write(join(self.svn_repo, 'conf', 'passwd'), text) # Necessary to be able to change revision properties revprop_hook_filename = join(self.svn_repo, 'hooks', 'pre-revprop-change') if sys.platform == 'win32': # TODO(kustermann): Test on Windows one day. write("%s.bat" % revprop_hook_filename, "") else: write(revprop_hook_filename, '#!/bin/sh\n' 'exit 0\n') os.chmod(revprop_hook_filename, 0755) # Mac 10.6 ships with a buggy subversion build and we need this line # to work around the bug. write(join(self.svn_repo, 'db', 'fsfs.conf'), '[rep-sharing]\n' 'enable-rep-sharing = false\n') # Start the daemon. self.svn_port = find_free_port(self.host, 10000) logging.debug('Using port %d' % self.svn_port) cmd = ['svnserve', '-d', '--foreground', '-r', self.root_dir, '--listen-port=%d' % self.svn_port] if self.host == '127.0.0.1': cmd.append('--listen-host=' + self.host) self.check_port_is_free(self.svn_port) self.svnserve = subprocess2.Popen( cmd, cwd=self.svn_repo, stdout=subprocess2.PIPE, stderr=subprocess2.PIPE) wait_for_port_to_bind(self.host, self.svn_port, self.svnserve) self.svn_base = 'svn://%s:%d/svn/' % (self.host, self.svn_port) self.populateSvn() self.svn_dirty = False return True
def build_protopy(): out = proc.check_output('cl.exe -DNDEBUG -DPROTOPY -D_NDEBUG -D_WIN32 -DWIN32 -MD -EP -I..\\include -I. ..\\include\\pydynload.h'.split(' '), stderr=proc.STDOUT) lines = [] for line in out.splitlines(): line = line.strip(' \r\n\t') if re.match(r"^(printf\(.+)$", line): lines.append('\t'+line) fout = open('protopy.c', 'w') fout.write(base_template % '\n'.join(lines)) fout.close() clcmd = 'cl.exe -nologo -W3 -WX- -O2 -Ob2 -Oi -Oy -GL -GF -Gm- -GS- -Gy -fp:precise protopy.c -link -OUT:protopy.exe -LTCG -OPT:REF -OPT:ICF' proc.check_call(clcmd.split(' '), stderr=os.devnull, stdout=os.devnull)
def _compare_revisions(self, last_roll_revision, new_roll_revision): """Ensure that new_roll_revision is newer than last_roll_revision.""" # Ensure that new_roll_revision is not an ancestor of old_roll_revision. try: subprocess2.check_call(['git', '--git-dir', self._project_git_dir, 'merge-base', '--is-ancestor', new_roll_revision, last_roll_revision]) print ('Already at %s refusing to roll backwards to %s.' % ( last_roll_revision, new_roll_revision)) return False except subprocess2.CalledProcessError: pass return True
def populateSvn(self): """Creates a few revisions of changes including a DEPS file.""" # Repos subprocess2.check_call( [ "svn", "checkout", self.svn_base, self.svn_checkout, "-q", "--non-interactive", "--no-auth-cache", "--username", self.USERS[0][0], "--password", self.USERS[0][1], ] ) assert os.path.isdir(join(self.svn_checkout, ".svn")) def file_system(rev): DEPS = """deps = { 'src/different_repo': '%(svn_base)strunk/third_party', 'src/different_repo_fixed': '%(svn_base)strunk/third_party@1', 'src/same_repo': '/trunk/third_party', 'src/same_repo_fixed': '/trunk/third_party@1', }""" % { "svn_base": self.svn_base } return { "trunk/src/DEPS": DEPS, "trunk/src/origin": "svn/trunk/src@%(rev)d" % {"rev": rev}, "trunk/third_party/origin": "svn/trunk/third_party@%(rev)d" % {"rev": rev}, } # We make three commits. We use always the same DEPS contents but # - 'trunk/src/origin' contains 'svn/trunk/src/origin@rX' # - 'trunk/third_party/origin' contains 'svn/trunk/third_party/origin@rX' # where 'X' is the revision number. # So the 'origin' files will change in every commit. self._commit_svn(file_system(1)) self._commit_svn(file_system(2)) self._commit_svn(file_system(3)) # We rewrite the timestamps so we can test that '--transitive' will take the # parent timestamp on different repositories and the parent revision # otherwise. self._set_svn_commit_date("1", "2011-10-01T03:00:00.000000Z") self._set_svn_commit_date("2", "2011-10-09T03:00:00.000000Z") self._set_svn_commit_date("3", "2011-10-02T03:00:00.000000Z")
def install_prerequisites(self): # First, verify the Google AppEngine SDK is available. if not os.path.isfile(self.dev_app): raise Failure( 'Install google_appengine sdk in %s or higher up' % self.base_dir) # Second, checkout rietveld if not available. if not os.path.isdir(self.rietveld): print('Checking out rietveld...') try: subprocess2.check_call( ['svn', 'co', '-q', 'http://rietveld.googlecode.com/svn/trunk@681', self.rietveld]) except (OSError, subprocess2.CalledProcessError), e: raise Failure('Failed to checkout rietveld\n%s' % e)
def _commit_git(self, repo, tree, base=None): repo_root = join(self.git_base, repo) if base: base_commit = self.git_hashes[repo][base][0] subprocess2.check_call( ['git', 'checkout', base_commit], cwd=repo_root) self._genTree(repo_root, tree) commit_hash = commit_git(repo_root) base = base or -1 if self.git_hashes[repo][base][1]: new_tree = self.git_hashes[repo][base][1].copy() new_tree.update(tree) else: new_tree = tree.copy() self.git_hashes[repo].append((commit_hash, new_tree))
def set_up_git(self): """Creates git repositories and start the servers.""" self.set_up() if self.initialized: return True try: subprocess2.check_output(['git', '--version']) except (OSError, subprocess2.CalledProcessError): return False for repo in ['repo_%d' % r for r in range(1, self.NB_GIT_REPOS + 1)]: subprocess2.check_call(['git', 'init', '-q', join(self.git_base, repo)]) self.git_hashes[repo] = [(None, None)] self.populateGit() self.initialized = True return True
def _current_revision(self): git_dir = self._path_from_chromium_root(self._path_to_project, '.git') subprocess2.check_call(['git', '--git-dir', git_dir, 'fetch']) git_show_cmd = ['git', '--git-dir', git_dir, 'show', '-s', 'origin/master'] git_log = subprocess2.check_output(git_show_cmd) match = re.search('^\s*git-svn-id:.*@(?P<svn_revision>\d+)\ ', git_log, re.MULTILINE) if match: return int(match.group('svn_revision')) else: # If it's not git-svn, fall back on git. git_revparse_cmd = ['git', '--git-dir', git_dir, 'rev-parse', 'origin/master'] return subprocess2.check_output(git_revparse_cmd).rstrip()
def _commit_git(self, repo, tree, base=None): repo_root = join(self.git_root, repo) if base: base_commit = self.git_hashes[repo][base][0] subprocess2.check_call( ['git', 'checkout', base_commit], cwd=repo_root) self._genTree(repo_root, tree) commit_hash = commit_git(repo_root) base = base or -1 if self.git_hashes[repo][base][1]: new_tree = self.git_hashes[repo][base][1].copy() new_tree.update(tree) else: new_tree = tree.copy() self.git_hashes[repo].append((commit_hash, new_tree))
def determine_scm(root): """Similar to upload.py's version but much simpler. Returns 'git' or None. """ if os.path.isdir(os.path.join(root, '.git')): return 'git' else: try: subprocess2.check_call(['git', 'rev-parse', '--show-cdup'], stdout=subprocess2.VOID, stderr=subprocess2.VOID, cwd=root) return 'git' except (OSError, subprocess2.CalledProcessError): return None
def _current_revision(self): git_dir = self._path_from_chromium_root(self._path_to_project, '.git') subprocess2.check_call(['git', '--git-dir', git_dir, 'fetch']) git_show_cmd = [ 'git', '--git-dir', git_dir, 'show', '-s', 'origin/master' ] git_log = subprocess2.check_output(git_show_cmd) match = re.search('^\s*git-svn-id:.*@(?P<svn_revision>\d+)\ ', git_log, re.MULTILINE) if match: return int(match.group('svn_revision')) else: # If it's not git-svn, fall back on git. git_revparse_cmd = [ 'git', '--git-dir', git_dir, 'rev-parse', 'origin/master' ] return subprocess2.check_output(git_revparse_cmd).rstrip()
def testCheckoutUpdatedBranchHeads(self): # Travel back in time, and set refs/branch-heads/5 to its parent. subprocess2.check_call( ['git', 'update-ref', 'refs/branch-heads/5', self.githash('repo_1', 4)], cwd=self.url) # Sync to refs/branch-heads/5 scm = gclient_scm.GitWrapper(self.url, self.root_dir, '.') self.options.revision = 'refs/branch-heads/5' scm.update(self.options, None, []) # Set refs/branch-heads/5 back to its original value. subprocess2.check_call( ['git', 'update-ref', 'refs/branch-heads/5', self.githash('repo_1', 5)], cwd=self.url) # Attempt to sync to refs/branch-heads/5 again. self.testCheckoutBranchHeads()
def populateSvn(self): """Creates a few revisions of changes including a DEPS file.""" # Repos subprocess2.check_call([ 'svn', 'checkout', self.svn_base, self.svn_checkout, '-q', '--non-interactive', '--no-auth-cache', '--username', self.USERS[0][0], '--password', self.USERS[0][1] ]) assert os.path.isdir(join(self.svn_checkout, '.svn')) def file_system(rev): DEPS = """deps = { 'src/different_repo': '%(svn_base)strunk/third_party', 'src/different_repo_fixed': '%(svn_base)strunk/third_party@1', 'src/same_repo': '/trunk/third_party', 'src/same_repo_fixed': '/trunk/third_party@1', }""" % { 'svn_base': self.svn_base } return { 'trunk/src/DEPS': DEPS, 'trunk/src/origin': 'svn/trunk/src@%(rev)d' % { 'rev': rev }, 'trunk/third_party/origin': 'svn/trunk/third_party@%(rev)d' % { 'rev': rev }, } # We make three commits. We use always the same DEPS contents but # - 'trunk/src/origin' contains 'svn/trunk/src/origin@rX' # - 'trunk/third_party/origin' contains 'svn/trunk/third_party/origin@rX' # where 'X' is the revision number. # So the 'origin' files will change in every commit. self._commit_svn(file_system(1)) self._commit_svn(file_system(2)) self._commit_svn(file_system(3)) # We rewrite the timestamps so we can test that '--transitive' will take the # parent timestamp on different repositories and the parent revision # otherwise. self._set_svn_commit_date('1', '2011-10-01T03:00:00.000000Z') self._set_svn_commit_date('2', '2011-10-09T03:00:00.000000Z') self._set_svn_commit_date('3', '2011-10-02T03:00:00.000000Z')