def _set_svn_commit_date(self, revision, date):
subprocess2.check_output(
['svn', 'propset', 'svn:date', '--revprop', '-r', revision, date,
self.svn_base,
'--username', self.USERS[0][0],
'--password', self.USERS[0][1],
'--non-interactive'])
def testMove(self):
co = self._get_co(None)
self._check_move(co)
out = subprocess2.check_output(
['svn', 'status'], cwd=co.project_path)
out = sorted(out.splitlines())
expected = sorted(
[
'A + chromeos/views/webui_menu_widget.h',
'D chromeos/views/DOMui_menu_widget.h',
])
self.assertEquals(expected, out)
# Make sure ancestry is what is expected;
env = os.environ.copy()
env['LANGUAGE'] = 'en_US.UTF-8'
out = subprocess2.check_output(
['svn', 'info', 'chromeos/views/webui_menu_widget.h'],
cwd=co.project_path,
env=env)
values = dict(l.split(': ', 1) for l in out.splitlines() if l)
expected = {
'Checksum': '65837bb3da662c8fa88a4a50940ea7c6',
'Copied From Rev': '2',
'Copied From URL':
'%strunk/chromeos/views/DOMui_menu_widget.h' % self.svn_base,
'Name': 'webui_menu_widget.h',
'Node Kind': 'file',
'Path': 'chromeos/views/webui_menu_widget.h',
'Repository Root': '%s' % self.svn_base.rstrip('/'),
'Revision': '2',
'Schedule': 'add',
'URL': '%strunk/chromeos/views/webui_menu_widget.h' % self.svn_base,
}
self.assertEquals(expected, values)
def GuessVCS(options, path):
"""Helper to guess the version control system.
NOTE: Very similar to upload.GuessVCS. Doesn't look for hg since we don't
support it yet.
This examines the path directory, guesses which SCM we're using, and
returns an instance of the appropriate class. Exit with an error if we can't
figure it out.
Returns:
A SCM instance. Exits if the SCM can't be guessed.
"""
__pychecker__ = 'no-returnvalues'
real_path = path.split('@')[0]
logging.info("GuessVCS(%s)" % path)
# Subversion has a .svn in all working directories.
if os.path.isdir(os.path.join(real_path, '.svn')):
return SVN(options, path)
# Git has a command to test if you're in a git tree.
# Try running it, but don't die if we don't have git installed.
try:
subprocess2.check_output(
['git', 'rev-parse', '--is-inside-work-tree'], cwd=real_path,
stderr=subprocess2.VOID)
return GIT(options, path)
except subprocess2.CalledProcessError, e:
if e.returncode != errno.ENOENT and e.returncode != 128:
# ENOENT == 2 = they don't have git installed.
# 128 = git error code when not in a repo.
logging.warning('Unexpected error code: %s' % e.returncode)
raise
def set_up_git(self):
"""Creates git repositories and start the servers."""
self.set_up()
if self.gitdaemon:
return True
assert self.git_pid_file == None
try:
subprocess2.check_output(["git", "--version"])
except (OSError, subprocess2.CalledProcessError):
return False
for repo in ["repo_%d" % r for r in range(1, self.NB_GIT_REPOS + 1)]:
subprocess2.check_call(["git", "init", "-q", join(self.git_root, repo)])
self.git_hashes[repo] = [None]
self.git_port = find_free_port(self.host, 20000)
self.git_base = "git://%s:%d/git/" % (self.host, self.git_port)
# Start the daemon.
self.git_pid_file = tempfile.NamedTemporaryFile()
cmd = [
"git",
"daemon",
"--export-all",
"--reuseaddr",
"--base-path=" + self.root_dir,
"--pid-file=" + self.git_pid_file.name,
"--port=%d" % self.git_port,
]
if self.host == "127.0.0.1":
cmd.append("--listen=" + self.host)
self.check_port_is_free(self.git_port)
self.gitdaemon = subprocess2.Popen(cmd, cwd=self.root_dir, stdout=subprocess2.PIPE, stderr=subprocess2.PIPE)
wait_for_port_to_bind(self.host, self.git_port, self.gitdaemon)
self.populateGit()
self.git_dirty = False
return True
def testMove(self):
co = self._get_co(None)
self._check_move(co)
out = subprocess2.check_output(["svn", "status"], cwd=co.project_path)
out = sorted(out.splitlines())
expected = sorted(["A + chromeos/views/webui_menu_widget.h", "D chromeos/views/DOMui_menu_widget.h"])
self.assertEquals(expected, out)
# Make sure ancestry is what is expected;
env = os.environ.copy()
env["LANGUAGE"] = "en_US.UTF-8"
out = subprocess2.check_output(
["svn", "info", "chromeos/views/webui_menu_widget.h"], cwd=co.project_path, env=env
)
values = dict(l.split(": ", 1) for l in out.splitlines() if l)
expected = {
"Checksum": "65837bb3da662c8fa88a4a50940ea7c6",
"Copied From Rev": "2",
"Copied From URL": "%strunk/chromeos/views/DOMui_menu_widget.h" % self.svn_base,
"Name": "webui_menu_widget.h",
"Node Kind": "file",
"Path": "chromeos/views/webui_menu_widget.h",
"Repository Root": "%s" % self.svn_base.rstrip("/"),
"Revision": "2",
"Schedule": "add",
"URL": "%strunk/chromeos/views/webui_menu_widget.h" % self.svn_base,
}
self.assertEquals(expected, values)
def start_master(master, path):
try:
subprocess2.check_output(
['make', 'start'], timeout=120, cwd=path,
stderr=subprocess2.STDOUT)
except subprocess2.CalledProcessError, e:
logging.error('Error: cannot start %s' % master)
print e
return False
def _start_roll(self, last_roll_revision, new_roll_revision):
roll_branch = '%s_roll' % self._project
cwd_kwargs = {'cwd': self._path_to_chrome}
subprocess2.check_call(['git', 'clean', '-d', '-f'], **cwd_kwargs)
subprocess2.call(['git', 'rebase', '--abort'], **cwd_kwargs)
subprocess2.call(['git', 'branch', '-D', roll_branch], **cwd_kwargs)
subprocess2.check_call(['git', 'checkout', 'origin/master', '-f'],
**cwd_kwargs)
subprocess2.check_call(['git', 'checkout', '-b', roll_branch,
'-t', 'origin/master', '-f'], **cwd_kwargs)
try:
subprocess2.check_call(['roll-dep-svn', self._path_to_project,
new_roll_revision], **cwd_kwargs)
subprocess2.check_call(['git', 'add', 'DEPS'], **cwd_kwargs)
subprocess2.check_call(['git', 'commit', '--no-edit'], **cwd_kwargs)
commit_msg = subprocess2.check_output(
['git', 'log', '-n1', '--format=%B', 'HEAD'],
**cwd_kwargs).decode('utf-8')
if self._notry:
commit_msg += NO_TRY_STR % { 'project': self._project }
upload_cmd = ['git', 'cl', 'upload', '--bypass-hooks', '-f']
if self._cq_dry_run:
upload_cmd.append('--cq-dry-run')
else:
upload_cmd.append('--use-commit-queue')
if self._cq_extra_trybots:
commit_msg += ('\n' + CQ_INCLUDE_TRYBOTS +
','.join(self._cq_extra_trybots))
tbr = '\nTBR='
emails = self._emails_to_cc_on_rolls()
if emails:
emails_str = ','.join(emails)
tbr += emails_str
upload_cmd.extend(['--cc', emails_str, '--send-mail'])
commit_msg += tbr
if self._include_commit_log:
log_cmd = ['git', 'log', '--format=%h %ae %s',
'%s..%s' % (last_roll_revision, new_roll_revision)]
git_log = subprocess2.check_output(log_cmd, cwd=self._project_git_dir)
commit_msg += '\n\nCommits in this roll:\n' + git_log.decode('utf-8')
upload_cmd.extend(['-m', commit_msg])
subprocess2.check_call(upload_cmd, **cwd_kwargs)
finally:
subprocess2.check_call(['git', 'checkout', 'origin/master', '-f'],
**cwd_kwargs)
subprocess2.check_call(
['git', 'branch', '-D', roll_branch], **cwd_kwargs)
# FIXME: It's easier to pull the issue id from rietveld rather than
# parse it from the safely-roll-deps output. Once we inline
# safely-roll-deps into this script this can go away.
search_result = self._search_for_active_roll()
if search_result:
self._rietveld.add_comment(search_result['issue'],
self.ROLL_BOT_INSTRUCTIONS)
def test_check_output_throw_no_stderr(self):
try:
subprocess2.check_output(
self.exe + ['--fail', '--stderr'], universal_newlines=True)
self.fail()
except subprocess2.CalledProcessError, e:
self.assertEquals('', e.stdout)
self.assertEquals(None, e.stderr)
self.assertEquals(64, e.returncode)
def populateGit(self):
"""Creates a few revisions of changes files."""
self._commit_git(self.TEST_GIT_REPO, self._git_tree())
# Fix for the remote rejected error. For more details see:
# http://stackoverflow.com/questions/2816369/git-push-error-remote
subprocess2.check_output(
['git', '--git-dir',
os.path.join(self.git_root, self.TEST_GIT_REPO, '.git'),
'config', '--bool', 'core.bare', 'true'])
assert os.path.isdir(
os.path.join(self.git_root, self.TEST_GIT_REPO, '.git'))
def fn(c, e, un):
# Make sure failure still returns stderr completely.
stderr = []
try:
subprocess2.check_output(
e + ['--stderr', '--fail'],
stderr=stderr.append,
universal_newlines=un)
self.fail()
except subprocess2.CalledProcessError, e:
self._check_exception(e, '', None, 64)
self.assertEquals(c('a\nbb\nccc\n'), ''.join(stderr))
def main():
tool_dir = os.path.dirname(os.path.abspath(__file__))
parser = optparse.OptionParser(usage="<new webkit rev>")
parser.add_option("-v", "--verbose", action="count", default=0)
options, args = parser.parse_args()
logging.basicConfig(level=[logging.WARNING, logging.INFO, logging.DEBUG][min(2, options.verbose)])
if len(args) != 1:
parser.error("Need only one arg: new webkit revision to roll to.")
root_dir = os.path.dirname(tool_dir)
os.chdir(root_dir)
new_rev = int(args[0])
print "Roll webkit revision to %s" % new_rev
# Silence the editor.
os.environ["EDITOR"] = "true"
old_branch = scm.GIT.GetBranch(root_dir)
if old_branch == "webkit_roll":
parser.error("Please delete the branch webkit_roll and move to a different branch")
subprocess2.check_output(["git", "checkout", "-b", "webkit_roll", "origin/master"])
try:
old_rev = process_deps(os.path.join(root_dir, "DEPS"), new_rev)
commit_msg = "Webkit roll %s:%s\n\nTBR=\n" % (old_rev, new_rev)
subprocess2.check_output(["git", "commit", "-m", commit_msg, "DEPS"])
subprocess2.check_call(["git", "diff", "origin/master"])
subprocess2.check_call(["git", "cl", "upload", "--use-commit-queue"])
finally:
subprocess2.check_output(["git", "checkout", old_branch])
subprocess2.check_output(["git", "branch", "-D", "webkit_roll"])
return 0
def main():
tool_dir = os.path.dirname(os.path.abspath(__file__))
parser = optparse.OptionParser(usage='<new webkit rev>')
parser.add_option('-v', '--verbose', action='count', default=0)
parser.add_option('--commit', action='store_true', default=True,
help='(default) Put change in commit queue on upload.')
parser.add_option('--no-commit', action='store_false', dest='commit',
help='Don\'t put change in commit queue on upload.')
parser.add_option('-r', '--reviewers', default='',
help='Add given users as either reviewers or TBR as'
' appropriate.')
parser.add_option('--upstream', default='origin/master',
help='(default "%default") Use given start point for change'
' to upload. For instance, if you use the old git workflow,'
' you might set it to "origin/trunk".')
parser.add_option('--cc', help='CC email addresses for issue.')
options, args = parser.parse_args()
logging.basicConfig(
level=
[logging.WARNING, logging.INFO, logging.DEBUG][
min(2, options.verbose)])
if len(args) != 1:
parser.error('Need only one arg: new webkit revision to roll to.')
root_dir = os.path.dirname(tool_dir)
os.chdir(root_dir)
new_rev = int(args[0])
print 'Roll webkit revision to %s' % new_rev
# Silence the editor.
os.environ['EDITOR'] = 'true'
old_branch = scm.GIT.GetBranch(root_dir)
if old_branch == 'webkit_roll':
parser.error(
'Please delete the branch webkit_roll and move to a different branch')
subprocess2.check_output(
['git', 'checkout', '-b', 'webkit_roll', options.upstream])
try:
old_rev = process_deps(os.path.join(root_dir, 'DEPS'), new_rev)
review_field = 'TBR' if options.commit else 'R'
commit_msg = 'Webkit roll %s:%s\n\n%s=%s\n' % (old_rev, new_rev,
review_field,
options.reviewers)
subprocess2.check_output(['git', 'commit', '-m', commit_msg, 'DEPS'])
subprocess2.check_call(['git', 'diff', options.upstream])
upload_cmd = ['git', 'cl', 'upload']
if options.commit:
upload_cmd.append('--use-commit-queue')
if options.reviewers:
upload_cmd.append('--send-mail')
if options.cc:
upload_cmd.extend(['--cc', options.cc])
subprocess2.check_call(upload_cmd)
finally:
subprocess2.check_output(['git', 'checkout', old_branch])
subprocess2.check_output(['git', 'branch', '-D', 'webkit_roll'])
return 0
def test_check_output_no_stdout(self):
try:
subprocess2.check_output(self.exe, stdout=subprocess2.PIPE)
self.fail()
except ValueError:
pass
if (sys.version_info[0] * 10 + sys.version_info[1]) >= 27:
# python 2.7+
try:
# pylint: disable=E1101
subprocess.check_output(self.exe, stdout=subprocess.PIPE)
self.fail()
except ValueError:
pass
def _current_revision(self):
git_dir = self._path_from_chromium_root(self._path_to_project, '.git')
subprocess2.check_call(['git', '--git-dir', git_dir, 'fetch'])
git_show_cmd = ['git', '--git-dir', git_dir, 'show', '-s',
'origin/master']
git_log = subprocess2.check_output(git_show_cmd)
match = re.search('^\s*git-svn-id:.*@(?P<svn_revision>\d+)\ ',
git_log, re.MULTILINE)
if match:
return int(match.group('svn_revision'))
else:
# If it's not git-svn, fall back on git.
git_revparse_cmd = ['git', '--git-dir', git_dir, 'rev-parse',
'origin/master']
return subprocess2.check_output(git_revparse_cmd).rstrip()
def getRevisionLog(url, revision):
"""Takes an svn url and gets the associated revision."""
svn_log = subprocess2.check_output(
['svn', 'log', url, '-r', str(revision)],
universal_newlines=True).splitlines(True)
# Don't include the header lines and the trailing "---..." line.
return ''.join(svn_log[3:-1])
def _log(self):
# Don't use the local checkout in case of caching incorrency.
out = subprocess2.check_output(
['svn', 'log', self.svn_url,
'--non-interactive', '--no-auth-cache',
'--username', self.usr, '--password', self.pwd,
'--with-all-revprops', '--xml',
'--limit', '1'])
logentry = ElementTree.XML(out).find('logentry')
if logentry == None:
return {'revision': 0}
data = {
'revision': int(logentry.attrib['revision']),
}
def set_item(name):
item = logentry.find(name)
if item != None:
data[name] = item.text
set_item('author')
set_item('msg')
revprops = logentry.find('revprops')
if revprops != None:
data['revprops'] = []
for prop in revprops.getiterator('property'):
data['revprops'].append((prop.attrib['name'], prop.text))
return data
def test_check_output_defaults(self, mockCommunicate):
mockCommunicate.return_value = (('stdout', 'stderr'), 0)
self.assertEqual('stdout', subprocess2.check_output(['foo'], a=True))
mockCommunicate.assert_called_with(['foo'],
a=True,
stdin=subprocess2.VOID_INPUT,
stdout=subprocess2.PIPE)
def getRevisionLog(url, revision):
"""Takes an svn url and gets the associated revision."""
svn_log = subprocess2.check_output(
['svn', 'log', url, '-r', str(revision)],
universal_newlines=True).splitlines(True)
# Don't include the header lines and the trailing "---..." line.
return ''.join(svn_log[3:-1])
def testMove(self):
co = self._get_co(None)
self._check_move(co)
out = subprocess2.check_output(["git", "diff", "--staged", "--name-status"], cwd=co.project_path)
out = sorted(out.splitlines())
expected = sorted(["A\tchromeos/views/webui_menu_widget.h", "D\tchromeos/views/DOMui_menu_widget.h"])
self.assertEquals(expected, out)
def getFileInfo(url, revision):
global files_info_
if (files_info_ != None):
return files_info_
svn_log = subprocess2.check_output(
['svn', 'log', url, '-r', str(revision), '-v']).splitlines()
info = []
for line in svn_log:
# A workaround to dump the (from .*) stuff, regex not so friendly in the 2nd
# pass...
match = re.search(r"(.*) \(from.*\)", line)
if match:
line = match.group(1)
match = re.search(file_pattern_, line)
if match:
info.append([
match.group(1).strip(),
match.group(2).strip(),
match.group(3).strip(),
match.group(4).strip()
])
files_info_ = info
return info
def commit_git(repo):
"""Commits the changes and returns the new hash."""
subprocess2.check_call(["git", "add", "-A", "-f"], cwd=repo)
subprocess2.check_call(["git", "commit", "-q", "--message", "foo"], cwd=repo)
rev = subprocess2.check_output(["git", "show-ref", "--head", "HEAD"], cwd=repo).split(" ", 1)[0]
logging.debug("At revision %s" % rev)
return rev
def get_dllexports(dll): """ Get exports of DLL. """ dll2def = os.path.join(env.bin_dir, 'dll2def.exe') if not os.path.isfile(dll2def): build_script = os.path.join(env.src_dir, 'dll2def', 'build.bat') try: subproc.check_call([ build_script ], stderr=subproc.VOID, stdout=subproc.VOID, stdin=subproc.VOID) except subproc.CalledProcessError: import traceback traceback.print_exc() fatal_message( 'Failed to build dll2def', ''.join(( 'There was an error while trying to build dll2def. To resolve this issue, you ', 'may want to check the following files to make sure they are correct:\n\n', ' /repository/util/src/dll2def/build.bat\n' ' /repository/util/bin/msvcc.bat\n' ' /repository/util/bin/_buildenv.bat\n\n' 'If you continue having issue, please report this error at this project\'s GitHub. ', '(https://github.com/Juntalis/dypywin32)' )) ) return split_lines_clean(subproc.check_output( [ dll2def, env.data_file(dll) ], stderr=subproc.VOID ))
def _log(self):
# Don't use the local checkout in case of caching incorrency.
out = subprocess2.check_output([
'svn', 'log', self.svn_url, '--non-interactive', '--no-auth-cache',
'--username', self.usr, '--password', self.pwd,
'--with-all-revprops', '--xml', '--limit', '1'
])
logentry = ElementTree.XML(out).find('logentry')
if logentry == None:
return {'revision': 0}
data = {
'revision': int(logentry.attrib['revision']),
}
def set_item(name):
item = logentry.find(name)
if item != None:
data[name] = item.text
set_item('author')
set_item('msg')
revprops = logentry.find('revprops')
if revprops != None:
data['revprops'] = []
for prop in revprops.getiterator('property'):
data['revprops'].append((prop.attrib['name'], prop.text))
return data
def apply_patch(self, patches):
"""Ignores svn properties."""
for p in patches:
try:
stdout = ''
filename = os.path.join(self.project_path, p.filename)
if p.is_delete:
os.remove(filename)
else:
dirname = os.path.dirname(p.filename)
full_dir = os.path.join(self.project_path, dirname)
if dirname and not os.path.isdir(full_dir):
os.makedirs(full_dir)
filepath = os.path.join(self.project_path, p.filename)
if p.is_binary:
with open(filepath, 'wb') as f:
f.write(p.get())
else:
if p.diff_hunks:
stdout = subprocess2.check_output(
['patch', '-p%s' % p.patchlevel],
stdin=p.get(),
stderr=subprocess2.STDOUT,
cwd=self.project_path)
elif p.is_new and not os.path.exists(filepath):
# There is only a header. Just create the file.
open(filepath, 'w').close()
for post in (self.post_processors or []):
post(self, p)
except OSError, e:
raise PatchApplicationFailed(p.filename, '%s%s' % (stdout, e))
except subprocess.CalledProcessError, e:
raise PatchApplicationFailed(
p.filename, '%s%s' % (stdout, getattr(e, 'stdout', None)))
def Capture(args, cwd, **kwargs):
"""Always redirect stderr.
Throws an exception if non-0 is returned.
"""
return subprocess2.check_output(
['svn'] + args, stderr=subprocess2.PIPE, cwd=cwd, **kwargs)
def _current_revision(self):
git_dir = self._path_from_chromium_root(self._path_to_project, '.git')
subprocess2.check_call(['git', '--git-dir', git_dir, 'fetch'])
git_show_cmd = [
'git', '--git-dir', git_dir, 'show', '-s', 'origin/master'
]
git_log = subprocess2.check_output(git_show_cmd)
match = re.search('^\s*git-svn-id:.*@(?P<svn_revision>\d+)\ ', git_log,
re.MULTILINE)
if match:
return int(match.group('svn_revision'))
else:
# If it's not git-svn, fall back on git.
git_revparse_cmd = [
'git', '--git-dir', git_dir, 'rev-parse', 'origin/master'
]
return subprocess2.check_output(git_revparse_cmd).rstrip()
def set_up_git(self):
"""Creates git repositories and start the servers."""
self.set_up()
if self.initialized:
return True
try:
subprocess2.check_output(['git', '--version'])
except (OSError, subprocess2.CalledProcessError):
return False
for repo in ['repo_%d' % r for r in range(1, self.NB_GIT_REPOS + 1)]:
subprocess2.check_call(
['git', 'init', '-q',
join(self.git_base, repo)])
self.git_hashes[repo] = [(None, None)]
self.populateGit()
self.initialized = True
return True
def Capture(args, cwd, strip_out=True, **kwargs):
env = os.environ.copy()
# 'cat' is a magical git string that disables pagers on all platforms.
env['GIT_PAGER'] = 'cat'
output = subprocess2.check_output(
['git'] + args,
cwd=cwd, stderr=subprocess2.PIPE, env=env, **kwargs)
return output.strip() if strip_out else output
def get_gitcookies_path(cls):
if os.getenv('GIT_COOKIES_PATH'):
return os.getenv('GIT_COOKIES_PATH')
try:
return subprocess2.check_output(
['git', 'config', '--path', 'http.cookiefile']).strip()
except subprocess2.CalledProcessError:
return os.path.join(os.environ['HOME'], '.gitcookies')
def commit_git(repo):
"""Commits the changes and returns the new hash."""
subprocess2.check_call(['git', 'add', '-A', '-f'], cwd=repo)
subprocess2.check_call(['git', 'commit', '-q', '--message', 'foo'], cwd=repo)
rev = subprocess2.check_output(
['git', 'show-ref', '--head', 'HEAD'], cwd=repo).split(' ', 1)[0]
logging.debug('At revision %s' % rev)
return rev
def Capture(args, **kwargs):
"""Always redirect stderr.
Throws an exception if non-0 is returned.
"""
return subprocess2.check_output(['svn'] + args,
stderr=subprocess2.PIPE,
**kwargs)
def commit_git(repo):
"""Commits the changes and returns the new hash."""
subprocess2.check_call(['git', 'add', '-A', '-f'], cwd=repo)
subprocess2.check_call(['git', 'commit', '-q', '--message', 'foo'], cwd=repo)
rev = subprocess2.check_output(
['git', 'show-ref', '--head', 'HEAD'], cwd=repo).split(' ', 1)[0]
logging.debug('At revision %s' % rev)
return rev
def RunCommand(args, error_ok=False, error_message=None, **kwargs):
try:
return subprocess2.check_output(args, shell=False, **kwargs)
except subprocess2.CalledProcessError, e:
if not error_ok:
DieWithError('Command "%s" failed.\n%s' %
(' '.join(args), error_message or e.stdout or ''))
return e.stdout
def get_gitcookies_path(cls):
if os.getenv('GIT_COOKIES_PATH'):
return os.getenv('GIT_COOKIES_PATH')
try:
return subprocess2.check_output(
['git', 'config', '--path', 'http.cookiefile']).strip()
except subprocess2.CalledProcessError:
return os.path.join(os.environ['HOME'], '.gitcookies')
def isSVNDirty():
svn_status = subprocess2.check_output(['svn', 'status']).splitlines()
for line in svn_status:
match = re.search(r"^[^X?]", line)
if match:
return True
return False
def Capture(args, cwd, strip_out=True, **kwargs):
env = GIT.ApplyEnvVars(kwargs)
output = subprocess2.check_output(['git'] + args,
cwd=cwd,
stderr=subprocess2.PIPE,
env=env,
**kwargs)
return output.strip() if strip_out else output
def isSVNDirty():
svn_status = subprocess2.check_output(['svn', 'status']).splitlines()
for line in svn_status:
match = re.search(r"^[^X?]", line)
if match:
return True
return False
def get_gitcookies_path(cls):
if os.getenv('GIT_COOKIES_PATH'):
return os.getenv('GIT_COOKIES_PATH')
try:
path = subprocess2.check_output(
['git', 'config', '--path', 'http.cookiefile'])
return path.decode('utf-8', 'ignore').strip()
except subprocess2.CalledProcessError:
return os.path.expanduser(os.path.join('~', '.gitcookies'))
def _Capture(self, args, **kwargs):
kwargs.setdefault('cwd', self.checkout_path)
kwargs.setdefault('stderr', subprocess2.PIPE)
strip = kwargs.pop('strip', True)
env = scm.GIT.ApplyEnvVars(kwargs)
ret = subprocess2.check_output(['git'] + args, env=env, **kwargs)
if strip:
ret = ret.strip()
return ret
def RunCommand(args, error_ok=False, error_message=None, **kwargs):
try:
return subprocess2.check_output(args, shell=False, **kwargs)
except subprocess2.CalledProcessError, e:
if not error_ok:
DieWithError(
'Command "%s" failed.\n%s' % (
' '.join(args), error_message or e.stdout or ''))
return e.stdout
def stop_master(master, path):
if not os.path.isfile(os.path.join(path, 'twistd.pid')):
return True
try:
subprocess2.check_output(
['make', 'stop'], timeout=60, cwd=path,
stderr=subprocess2.STDOUT)
for _ in range(100):
if not os.path.isfile(os.path.join(path, 'twistd.pid')):
return True
time.sleep(0.1)
return False
except subprocess2.CalledProcessError, e:
if 'No such process' in e.stdout:
logging.warning('Flushed ghost twistd.pid for %s' % master)
os.remove(os.path.join(path, 'twistd.pid'))
return True
return False
def _set_svn_commit_date(self, revision, date):
subprocess2.check_output(
[
"svn",
"propset",
"svn:date",
"--revprop",
"-r",
revision,
date,
self.svn_base,
"--username",
self.USERS[0][0],
"--password",
self.USERS[0][1],
"--non-interactive",
]
)
def _check_output_svn(self, args, credentials=True, **kwargs):
"""Runs svn and throws an exception if the command failed.
Returns the output.
"""
kwargs.setdefault('cwd', self.project_path)
return subprocess2.check_output(
self._add_svn_flags(args, True, credentials),
stderr=subprocess2.STDOUT,
**kwargs)
def stop_master(master, path):
if not os.path.isfile(os.path.join(path, 'twistd.pid')):
return True
try:
subprocess2.check_output(['make', 'stop'],
timeout=60,
cwd=path,
stderr=subprocess2.STDOUT)
for _ in range(100):
if not os.path.isfile(os.path.join(path, 'twistd.pid')):
return True
time.sleep(0.1)
return False
except subprocess2.CalledProcessError, e:
if 'No such process' in e.stdout:
logging.warning('Flushed ghost twistd.pid for %s' % master)
os.remove(os.path.join(path, 'twistd.pid'))
return True
return False
def _check_output_svn(self, args, credentials=True, **kwargs):
"""Runs svn and throws an exception if the command failed.
Returns the output.
"""
kwargs.setdefault('cwd', self.project_path)
return subprocess2.check_output(
self._add_svn_flags(args, True, credentials),
stderr=subprocess2.STDOUT,
**kwargs)
def set_up_git(self):
"""Creates git repositories and start the servers."""
self.set_up()
if self.initialized:
return True
try:
subprocess2.check_output(['git', '--version'])
except (OSError, subprocess2.CalledProcessError):
return False
for repo in ['repo_%d' % r for r in range(1, self.NB_GIT_REPOS + 1)]:
# TODO(crbug.com/114712) use git.init -b and remove 'checkout' once git is
# upgraded to 2.28 on all builders.
subprocess2.check_call(['git', 'init', '-q', join(self.git_base, repo)])
subprocess2.check_call(['git', 'checkout', '-q', '-b', DEFAULT_BRANCH],
cwd=join(self.git_base, repo))
self.git_hashes[repo] = [(None, None)]
self.populateGit()
self.initialized = True
return True
def Capture(args, cwd, strip_out=True, **kwargs):
env = os.environ.copy()
# 'cat' is a magical git string that disables pagers on all platforms.
env['GIT_PAGER'] = 'cat'
output = subprocess2.check_output(['git'] + args,
cwd=cwd,
stderr=subprocess2.PIPE,
env=env,
**kwargs)
return output.strip() if strip_out else output
def testAutoProps(self):
co = self._get_co(None)
co.svn_config = checkout.SvnConfig(os.path.join(ROOT_DIR, "subversion_config"))
co.prepare(None)
patches = self.get_patches()
co.apply_patch(patches)
self.assertEquals(["bin_file", "chrome/file.cc", "new_dir/subdir/new_file", "extra"], patches.filenames)
# *.txt = svn:eol-style=LF in subversion_config/config.
out = subprocess2.check_output(["svn", "pget", "svn:eol-style", "chrome/file.cc"], cwd=co.project_path)
self.assertEquals("LF\n", out)
def apply_patch(self, patches, post_processors=None):
"""Ignores svn properties."""
post_processors = post_processors or self.post_processors or []
for p in patches:
try:
stdout = ''
filename = os.path.join(self.project_path, p.filename)
if p.is_delete:
os.remove(filename)
else:
dirname = os.path.dirname(p.filename)
full_dir = os.path.join(self.project_path, dirname)
if dirname and not os.path.isdir(full_dir):
os.makedirs(full_dir)
filepath = os.path.join(self.project_path, p.filename)
if p.is_binary:
with open(filepath, 'wb') as f:
f.write(p.get())
else:
if p.source_filename:
if not p.is_new:
raise PatchApplicationFailed(
p,
'File has a source filename specified but is not new'
)
# Copy the file first.
if os.path.isfile(filepath):
raise PatchApplicationFailed(
p,
'File exist but was about to be overwriten'
)
shutil.copy2(
os.path.join(self.project_path,
p.source_filename), filepath)
if p.diff_hunks:
stdout = subprocess2.check_output(
[
'patch', '-u', '--binary',
'-p%s' % p.patchlevel
],
stdin=p.get(False),
stderr=subprocess2.STDOUT,
cwd=self.project_path)
elif p.is_new and not os.path.exists(filepath):
# There is only a header. Just create the file.
open(filepath, 'w').close()
for post in post_processors:
post(self, p)
except OSError, e:
raise PatchApplicationFailed(p, '%s%s' % (stdout, e))
except subprocess.CalledProcessError, e:
raise PatchApplicationFailed(
p, '%s%s' % (stdout, getattr(e, 'stdout', None)))
def testMove(self):
co = self._get_co(None)
self._check_move(co)
out = subprocess2.check_output(
['git', 'diff', 'HEAD~', '--name-status'], cwd=co.project_path)
out = sorted(out.splitlines())
expected = sorted([
'A\tchromeos/views/webui_menu_widget.h',
'D\tchromeos/views/DOMui_menu_widget.h',
])
self.assertEquals(expected, out)
def testMove(self):
co = self._get_co(None)
self._check_move(co)
out = subprocess2.check_output(['svn', 'status'], cwd=co.project_path)
out = sorted(out.splitlines())
expected = sorted([
'A + chromeos/views/webui_menu_widget.h',
'D chromeos/views/DOMui_menu_widget.h',
])
self.assertEquals(expected, out)
# Make sure ancestry is what is expected;
env = os.environ.copy()
env['LANGUAGE'] = 'en_US.UTF-8'
out = subprocess2.check_output(
['svn', 'info', 'chromeos/views/webui_menu_widget.h'],
cwd=co.project_path,
env=env)
values = dict(l.split(': ', 1) for l in out.splitlines() if l)
expected = {
'Checksum':
'65837bb3da662c8fa88a4a50940ea7c6',
'Copied From Rev':
'2',
'Copied From URL':
'%strunk/chromeos/views/DOMui_menu_widget.h' % self.svn_base,
'Name':
'webui_menu_widget.h',
'Node Kind':
'file',
'Path':
'chromeos/views/webui_menu_widget.h',
'Repository Root':
'%s' % self.svn_base.rstrip('/'),
'Revision':
'2',
'Schedule':
'add',
'URL':
'%strunk/chromeos/views/webui_menu_widget.h' % self.svn_base,
}
self.assertEquals(expected, values)
def _last_roll_revision(self):
if not self._cached_last_roll_revision:
git_dir = self._path_from_chromium_root('.git')
subprocess2.check_call(['git', '--git-dir', git_dir, 'fetch'])
git_show_cmd = [
'git', '--git-dir', git_dir, 'show', 'origin/master:DEPS'
]
deps_contents = subprocess2.check_output(git_show_cmd)
pattern = self.REVISION_REGEXP % self._project_alias
match = re.search(pattern, deps_contents, re.MULTILINE)
self._cached_last_roll_revision = match.group('revision')
return self._cached_last_roll_revision
def call_with_timeout(cmd, cwd, timeout):
"""Runs an executable with an optional timeout."""
if timeout:
# This code path is much slower so only use it if necessary.
try:
output = subprocess2.check_output(cmd,
cwd=cwd,
timeout=timeout,
stderr=subprocess2.STDOUT)
return output, 0
except subprocess2.CalledProcessError, e:
return e.stdout, e.returncode
def getSVNInfo(url, revision):
info = {}
try:
svn_info = subprocess2.check_output(
['svn', 'info', '%s@%s' % (url, revision)]).splitlines()
for line in svn_info:
match = re.search(r"(.*?):(.*)", line)
if match:
info[match.group(1).strip()] = match.group(2).strip()
except subprocess2.CalledProcessError:
pass
return info
def test_check_output_defaults(self):
results = self._fake_communicate()
# It's discarding 'stderr' because it assumes stderr=subprocess2.STDOUT but
# fake_communicate() doesn't 'implement' that.
self.assertEquals('stdout', subprocess2.check_output(['foo'], a=True))
expected = {
'args': ['foo'],
'a': True,
'stdin': subprocess2.VOID,
'stdout': subprocess2.PIPE,
}
self.assertEquals(expected, results)
def set_up_git(self):
"""Creates git repositories and start the servers."""
self.set_up()
if self.gitdaemon:
return True
assert self.git_pid_file_name == None, self.git_pid_file_name
try:
subprocess2.check_output(['git', '--version'])
except (OSError, subprocess2.CalledProcessError):
return False
for repo in ['repo_%d' % r for r in range(1, self.NB_GIT_REPOS + 1)]:
subprocess2.check_call(
['git', 'init', '-q',
join(self.git_root, repo)])
self.git_hashes[repo] = [(None, None)]
git_pid_file = tempfile.NamedTemporaryFile(delete=False)
self.git_pid_file_name = git_pid_file.name
git_pid_file.close()
self.git_port = find_free_port(self.host)
self.git_base = 'git://%s:%d/git/' % (self.host, self.git_port)
cmd = [
'git', 'daemon', '--export-all', '--reuseaddr',
'--base-path=' + self.root_dir,
'--pid-file=' + self.git_pid_file_name,
'--port=%d' % self.git_port
]
if self.host == '127.0.0.1':
cmd.append('--listen=' + self.host)
# Verify that the port is free.
if not port_is_free(self.host, self.git_port):
return False
# Start the daemon.
self.gitdaemon = subprocess2.Popen(cmd,
cwd=self.root_dir,
stdout=subprocess2.PIPE,
stderr=subprocess2.PIPE)
wait_for_port_to_bind(self.host, self.git_port, self.gitdaemon)
self.populateGit()
self.git_dirty = False
return True
def commit_svn(repo, usr, pwd):
"""Commits the changes and returns the new revision number."""
to_add = []
to_remove = []
for status, filepath in scm.SVN.CaptureStatus(None, repo):
if status[0] == '?':
to_add.append(filepath)
elif status[0] == '!':
to_remove.append(filepath)
if to_add:
subprocess2.check_output(['svn', 'add', '--no-auto-props', '-q'] +
to_add,
cwd=repo)
if to_remove:
subprocess2.check_output(['svn', 'remove', '-q'] + to_remove, cwd=repo)
out = subprocess2.check_output([
'svn', 'commit', repo, '-m', 'foo', '--non-interactive',
'--no-auth-cache', '--username', usr, '--password', pwd
],
cwd=repo)
match = re.search(r'(\d+)', out)
if not match:
raise Exception('Commit failed', out)
rev = match.group(1)
status = subprocess2.check_output(['svn', 'status'], cwd=repo)
assert len(status) == 0, status
logging.debug('At revision %s' % rev)
return rev
def test_2011_04_09(self):
# Verifies commits done on that day.
# TODO(maruel): Import directly and do not use live data.
expected = (
"Getting data from 2011-04-09 for 1 days\n"
"Top users: 3 out of 3 total users 100.00%\n"
" Committed 3 out of 3 CQ'ed commits 100.00%\n"
"\n"
"Total commits: 26\n"
"Total commits by commit bot: 3 ( 11.5%)\n")
exe_path = os.path.join(PROJECT_DIR, 'tools', 'count.py')
args = [sys.executable, exe_path, '-s', '2011-04-09', '-d', '1', '-o']
self.assertEqual(expected, subprocess2.check_output(args))
def get_version(client):
"""Returns the version of swarming.py client tool as a tuple, if available."""
try:
version = subprocess2.check_output([
sys.executable,
os.path.join(client, 'swarming.py'),
'--version',
])
except (subprocess2.CalledProcessError, OSError):
return None
version = tuple(map(int, version.split('.')))
print('Detected swarming.py version %s' % '.'.join(map(str, version)))
return version