def Sync(revisions=None, force=False, delete_unversioned_trees=False,
verbose=False, jobs=None, no_hooks=False, extra_args=None):
""" Update the local checkout using gclient.
Args:
revisions: optional list of (branch, revision) tuples indicating which
projects to sync to which revisions.
force: whether to run with --force.
delete_unversioned_trees: whether to run with --delete-unversioned-trees.
verbose: whether to run with --verbose.
jobs: optional argument for the --jobs flag.
no_hooks: whether to run with --nohooks.
extra_args: optional list; any additional arguments.
"""
for branch, _ in (revisions or []):
# Do whatever it takes to get up-to-date with origin/master.
if os.path.exists(branch):
with misc.ChDir(branch):
# First, fix the git identity if needed.
maybe_fix_identity()
# If there are local changes, "git checkout" will fail.
shell_utils.run([GIT, 'reset', '--hard', 'HEAD'])
# In case HEAD is detached...
shell_utils.run([GIT, 'checkout', 'master'])
# Always fetch, in case we're unmanaged.
shell_utils.run([GIT, 'fetch'])
# This updates us to origin/master even if master has diverged.
shell_utils.run([GIT, 'reset', '--hard', 'origin/master'])
cmd = ['sync', '--no-nag-max']
if verbose:
cmd.append('--verbose')
if force:
cmd.append('--force')
if delete_unversioned_trees:
cmd.append('--delete_unversioned_trees')
if jobs:
cmd.append('-j%d' % jobs)
if no_hooks:
cmd.append('--nohooks')
for branch, revision in (revisions or []):
if revision:
cmd.extend(['--revision', '%s@%s' % (branch, revision)])
if extra_args:
cmd.extend(extra_args)
output = _RunCmd(cmd)
# "gclient sync" just downloads all of the commits. In order to actually sync
# to the desired commit, we have to "git reset" to that commit.
for branch, revision in (revisions or []):
with misc.ChDir(branch):
if revision:
shell_utils.run([GIT, 'reset', '--hard', revision])
else:
shell_utils.run([GIT, 'reset', '--hard', 'origin/master'])
return output
def Main():
parser = OptionParser()
parser.add_option('--skia_revision',
help=('Desired revision of Skia. Defaults to the most '
'recent revision.'))
parser.add_option('--chrome_revision',
help=('Desired revision of Chrome. Defaults to the most '
'recent revision.'))
parser.add_option('--destination',
help=('Where to sync the code. Defaults to the current '
'directory.'),
default=os.curdir)
parser.add_option('--fetch_target',
help=('Calls the fetch tool in depot_tools with the '
'specified target.'),
default=DEFAULT_FETCH_TARGET)
(options, _) = parser.parse_args()
dest_dir = os.path.abspath(options.destination)
with misc.ChDir(dest_dir):
actual_skia_rev, actual_chrome_rev = Sync(
skia_revision=options.skia_revision or SKIA_REV_MASTER,
chrome_revision=options.chrome_revision or CHROME_REV_MASTER,
fetch_target=options.fetch_target)
print 'Chrome synced to %s' % actual_chrome_rev
print 'Skia synced to %s' % actual_skia_rev
def _Run(self):
chrome_path = os.path.join(os.pardir, 'src')
with misc.ChDir(chrome_path):
shell_utils.run(
['git', 'config', '--local', 'user.name', DEPS_ROLL_NAME])
shell_utils.run(
['git', 'config', '--local', 'user.email', DEPS_ROLL_AUTHOR])
auto_roll = os.path.join(misc.BUILDBOT_PATH, 'third_party',
'chromium_buildbot_tot', 'scripts',
'tools', 'blink_roller', 'auto_roll.py')
# python auto_roll.py <project> <author> <path to chromium/src>
cmd = ['python', auto_roll, 'skia', DEPS_ROLL_AUTHOR, chrome_path]
exception = None
try:
output = shell_utils.run(cmd)
except shell_utils.CommandFailedException as e:
output = e.output
# Suppress failure for "refusing to roll backwards."
if not re.search(REGEXP_ROLL_TOO_OLD, output):
exception = e
bucket_url = gs_utils.GSUtils.with_gs_prefix(
skia_vars.GetGlobalVariable('googlestorage_bucket'))
match = re.search(REGEXP_ISSUE_CREATED, output)
if match:
issue = match.group('issue')
print 'Found issue #', issue
with open(FILENAME_CURRENT_ATTEMPT, 'w') as f:
f.write(HTML_CONTENT % (ISSUE_URL_TEMPLATE % {'issue': issue}))
slave_utils.GSUtilCopyFile(filename=FILENAME_CURRENT_ATTEMPT,
gs_base=bucket_url,
subdir=None,
gs_acl='public-read')
roll_status = None
for regexp, status_msg in ROLL_STATUSES:
match = re.search(regexp, output)
if match:
roll_status = status_msg % match.groupdict()
break
if roll_status:
with open(FILENAME_ROLL_STATUS, 'w') as f:
f.write(roll_status)
slave_utils.GSUtilCopyFile(filename=FILENAME_ROLL_STATUS,
gs_base=bucket_url,
subdir=None,
gs_acl='public-read')
#pylint: disable=E0702
if exception:
raise exception
def _Run(self):
try:
os.makedirs(ANDROID_CHECKOUT_PATH)
except OSError:
pass
with misc.ChDir(ANDROID_CHECKOUT_PATH):
if not os.path.exists(REPO):
# Download repo.
shell_utils.run(['curl', REPO_URL, '>', REPO])
shell_utils.run(['chmod', 'a+x', REPO])
with GitAuthenticate():
shell_utils.run([REPO, 'init', '-u', ANDROID_REPO_URL, '-g',
'all,-notdefault,-darwin', '-b', 'master-skia'])
shell_utils.run([REPO, 'sync', '-j32'])
def GetCheckedOutHash():
""" Determine what commit we actually got. If there are local modifications,
raise an exception. """
checkout_root, config_dict = _GetLocalConfig()
# Get the checked-out commit hash for the first gclient solution.
with misc.ChDir(os.path.join(checkout_root, config_dict[0]['name'])):
# First, print out the remote from which we synced, just for debugging.
cmd = [GIT, 'remote', '-v']
try:
shell_utils.run(cmd)
except shell_utils.CommandFailedException as e:
print e
# "git rev-parse HEAD" returns the commit hash for HEAD.
return shell_utils.run([GIT, 'rev-parse', 'HEAD'],
log_in_real_time=False).rstrip('\n')
def _Run(self):
with misc.ChDir(PATH_TO_SKIA):
shell_utils.run([GIT, 'config', '--local', 'user.name',
SKIA_COMMITTER_NAME])
shell_utils.run([GIT, 'config', '--local', 'user.email',
SKIA_COMMITTER_EMAIL])
if CHROMIUM_SKIA in shell_utils.run([GIT, 'remote', '-v']):
shell_utils.run([GIT, 'remote', 'set-url', 'origin', SKIA_GIT_URL,
CHROMIUM_SKIA])
version_file = 'SKP_VERSION'
skp_version = self._args.get('skp_version')
with git_utils.GitBranch(branch_name='update_skp_version',
commit_msg=COMMIT_MSG % skp_version,
commit_queue=not self._is_try):
# First, upload a version of the CL with just the SKP version changed.
with open(version_file, 'w') as f:
f.write(skp_version)
def force_update():
with misc.ChDir(os.path.join(misc.BUILDBOT_PATH, os.pardir)):
# Run "gclient" before doing anything else to ensure that we get the
# necessary stuff installed.
gclient_utils.GClient()
# Be sure that we sync to the most recent commit.
buildbot_revision = None
try:
output = git_utils.GetRemoteMasterHash(BUILDBOT_GIT_URL)
if output:
buildbot_revision = shlex.split(output)[0]
except shell_utils.CommandFailedException:
pass
if not buildbot_revision:
buildbot_revision = 'origin/master'
gclient_utils.Sync(revisions=[('buildbot', buildbot_revision)],
verbose=True,
force=True)
got_revision = gclient_utils.GetCheckedOutHash()
print GOT_REVISION_PATTERN % got_revision
return gclient_utils.GetCheckedOutHash()
def Sync(skia_revision=SKIA_REV_MASTER,
chrome_revision=CHROME_REV_LKGR,
fetch_target=DEFAULT_FETCH_TARGET,
gyp_defines=None,
gyp_generators=None):
""" Create and sync a checkout of Skia inside a checkout of Chrome. Returns
a tuple containing the actually-obtained revision of Skia and the actually-
obtained revision of Chrome.
skia_revision: revision of Skia to sync. Should be a commit hash or one of
(SKIA_REV_DEPS, SKIA_REV_MASTER).
chrome_revision: revision of Chrome to sync. Should be a commit hash or one
of (CHROME_REV_LKGR, CHROME_REV_MASTER).
fetch_target: string; Calls the fetch tool in depot_tools with the specified
argument. Default is DEFAULT_FETCH_TARGET.
gyp_defines: optional string; GYP_DEFINES to be passed to Gyp.
gyp_generators: optional string; which GYP_GENERATORS to use.
"""
# Figure out what revision of Skia we should use.
if skia_revision == SKIA_REV_MASTER:
output = git_utils.GetRemoteMasterHash(
skia_vars.GetGlobalVariable('skia_git_url'))
if output:
skia_revision = shlex.split(output)[0]
if not skia_revision:
raise Exception('Could not determine current Skia revision!')
skia_revision = str(skia_revision)
# Use Chrome LKGR, since gclient_utils will force a sync to origin/master.
if chrome_revision == CHROME_REV_LKGR:
chrome_revision = urllib2.urlopen(CHROME_LKGR_URL).read()
elif chrome_revision == CHROME_REV_MASTER:
chrome_revision = shlex.split(
git_utils.GetRemoteMasterHash(CHROME_GIT_URL))[0]
# Run "fetch chromium". The initial run is allowed to fail after it does some
# work. At the least, we expect the .gclient file to be present when it
# finishes.
if not os.path.isfile(GCLIENT_FILE):
try:
shell_utils.run([FETCH, fetch_target, '--nosvn=True'])
except shell_utils.CommandFailedException:
pass
if not os.path.isfile(GCLIENT_FILE):
raise Exception('Could not fetch %s!' % fetch_target)
# Run "gclient sync"
revisions = [('src', chrome_revision)]
if skia_revision != SKIA_REV_DEPS:
revisions.append(('src/third_party/skia', skia_revision))
try:
# Hack: We have to set some GYP_DEFINES, or upstream scripts will complain.
os.environ['GYP_DEFINES'] = os.environ.get('GYP_DEFINES') or ''
gclient_utils.Sync(revisions=revisions,
jobs=1,
no_hooks=True,
force=True)
except shell_utils.CommandFailedException as e:
# We frequently see sync failures because a lock file wasn't deleted. In
# that case, delete the lock file and try again.
pattern = r".*fatal: Unable to create '(\S+)': File exists\..*"
match = re.search(pattern, e.output)
if not match:
raise e
file_to_delete = match.groups()[0]
try:
print 'Attempting to remove %s' % file_to_delete
os.remove(file_to_delete)
except OSError:
# If the file no longer exists, just try again.
pass
gclient_utils.Sync(revisions=revisions,
jobs=1,
no_hooks=True,
force=True)
# Find the actually-obtained Chrome revision.
os.chdir('src')
actual_chrome_rev = shell_utils.run([GIT, 'rev-parse', 'HEAD'],
log_in_real_time=False).rstrip()
# Find the actually-obtained Skia revision.
with misc.ChDir(os.path.join('third_party', 'skia')):
actual_skia_rev = shell_utils.run([GIT, 'rev-parse', 'HEAD'],
log_in_real_time=False).rstrip()
# Run gclient hooks
gclient_utils.RunHooks(gyp_defines=gyp_defines,
gyp_generators=gyp_generators)
# Fix the submodules so that they don't show up in "git status"
# This fails on Windows...
if os.name != 'nt':
submodule_cmd = ('\'git config -f '
'$toplevel/.git/config submodule.$name.ignore all\'')
shell_utils.run(' '.join([GIT, 'submodule', 'foreach', submodule_cmd]),
shell=True)
# Verify that we got the requested revisions of Chrome and Skia.
if skia_revision != actual_skia_rev and skia_revision != SKIA_REV_DEPS:
raise Exception('Requested Skia revision %s but got %s!' %
(repr(skia_revision), repr(actual_skia_rev)))
if chrome_revision and chrome_revision != actual_chrome_rev:
raise Exception('Requested Chrome revision %s but got %s!' %
(repr(chrome_revision), repr(actual_chrome_rev)))
return (actual_skia_rev, actual_chrome_rev)
def _Run(self):
with misc.ChDir(misc.BUILDBOT_PATH):
shell_utils.run(['python', 'run_unittests'])
def _Run(self):
# Run "gclient" before doing anything else to ensure that we get the
# necessary stuff installed.
gclient_utils.GClient()
_MaybeUseSkiaLabMirror(self._revision)
# We receive gclient_solutions as a list of dictionaries flattened into a
# double-quoted string. This invocation of literal_eval converts that string
# into a list of strings.
solutions = ast.literal_eval(self._args['gclient_solutions'][1:-1])
# TODO(borenet): Move the gclient solutions parsing logic into a function.
# Parse each solution dictionary from a string and add it to a list, while
# building a string to pass as a spec to gclient, to indicate which
# branches should be downloaded.
solution_dicts = []
gclient_spec = 'solutions = ['
for solution in solutions:
gclient_spec += solution
solution_dicts += ast.literal_eval(solution)
gclient_spec += ']'
# Set the DEPS target_os if necessary.
if self._deps_target_os:
gclient_spec += '\ntarget_os = ["%s"]' % self._deps_target_os
# Run "gclient config" with the spec we just built.
gclient_utils.Config(spec=gclient_spec)
revisions = []
for solution in solution_dicts:
if solution['name'] == gclient_utils.SKIA_TRUNK:
revisions.append((solution['name'], self._revision))
else:
url_split = solution['url'].split('@')
if len(url_split) > 1:
revision = url_split[1]
revisions.append((solution['name'], revision))
try:
if self._is_try:
# Clean our checkout to make sure we don't have a patch left over.
if (os.path.isdir('skia')
and os.path.isdir(os.path.join('skia', '.git'))):
with misc.ChDir('skia'):
gclient_utils.Revert()
# Run "gclient sync"
gclient_utils.Sync(revisions=revisions,
verbose=True,
force=True,
delete_unversioned_trees=True)
got_revision = gclient_utils.GetCheckedOutHash()
except Exception:
# If the sync fails, clear the checkout and try again.
print 'Initial sync failed.'
# Attempt to remove the skia directory first.
if os.path.isdir('skia'):
print 'Removing "skia"'
chromium_utils.RemoveDirectory('skia')
# Now, remove *everything* in the build directory.
build_dir = os.path.abspath(os.curdir)
with misc.ChDir(os.pardir):
print 'Attempting to clear %s' % build_dir
file_utils.clear_directory(build_dir)
# Try to sync again.
print 'Attempting to sync again.'
gclient_utils.Config(spec=gclient_spec)
gclient_utils.Sync(revisions=revisions,
verbose=True,
force=True,
delete_unversioned_trees=True,
jobs=1)
got_revision = gclient_utils.GetCheckedOutHash()
# If the revision we actually got differs from what was requested, raise an
# exception.
if self._revision and got_revision != self._revision:
raise BuildStepFailure(
'Actually-synced revision "%s" is different from '
'the requested revision "%s".' %
(repr(got_revision), repr(self._revision)))
# Print the obtained revision number so that the master can parse it.
print 'Skia updated to %s' % got_revision
def _Run(self):
with misc.ChDir(EXTERNAL_SKIA):
# Check to see whether there is an upstream yet.
if not UPSTREAM_REMOTE_NAME in shell_utils.run(
[GIT, 'remote', 'show']):
try:
shell_utils.run([
GIT, 'remote', 'add', UPSTREAM_REMOTE_NAME,
SKIA_REPO_URL
])
except shell_utils.CommandFailedException as e:
if 'remote %s already exists' % UPSTREAM_REMOTE_NAME in e.output:
# Accept this error. The upstream remote name should have been in
# the output of git remote show, which would have made us skip this
# redundant command anyway.
print(
'%s was already added. Why did it not show in git remote'
' show?' % UPSTREAM_REMOTE_NAME)
else:
raise e
# Update the upstream remote.
shell_utils.run([GIT, 'fetch', UPSTREAM_REMOTE_NAME])
# Create a stack of commits to submit, one at a time, until we reach a
# commit that has already been merged.
commit_stack = []
head = git_utils.ShortHash('HEAD')
print 'HEAD is at %s' % head
if self._got_revision:
# Merge the revision that started this build.
commit = git_utils.ShortHash(self._got_revision)
else:
raise Exception('This build has no _got_revision to merge!')
print(
'Starting with %s, look for commits that have not been merged to '
'HEAD' % commit)
while not git_utils.AIsAncestorOfB(commit, head):
print 'Adding %s to list of commits to merge.' % commit
commit_stack.append(commit)
if git_utils.IsMerge(commit):
# Skia's commit history is not linear. There is no obvious way to
# merge each branch in, one commit at a time. So just start with the
# merge commit.
print '%s is a merge. Skipping merge of its parents.' % commit
break
commit = git_utils.ShortHash(commit + '~1')
else:
print '%s has already been merged.' % commit
if len(commit_stack) == 0:
raise BuildStepWarning(
'Nothing to merge; did someone already merge %s?'
' Exiting.' % commit)
print 'Merging %s commit(s):\n%s' % (len(commit_stack), '\n'.join(
reversed(commit_stack)))
# Now we have a list of commits to merge.
while len(commit_stack) > 0:
commit_to_merge = commit_stack.pop()
print 'Attempting to merge ' + commit_to_merge
# Start the merge.
try:
shell_utils.run(
[GIT, 'merge', commit_to_merge, '--no-commit'])
except shell_utils.CommandFailedException:
# Merge conflict. There may be a more elegant solution, but for now,
# undo the merge, and allow (/make) a human to do it.
git_utils.MergeAbort()
raise Exception(
'Failed to merge %s. Fall back to manual human '
'merge.' % commit_to_merge)
# Grab the upstream version of SkUserConfig, which will be used to
# generate Android's version.
shell_utils.run([
GIT, 'checkout', commit_to_merge, '--',
UPSTREAM_USER_CONFIG
])
# We don't want to commit the upstream version, so remove it from the
# index.
shell_utils.run([GIT, 'reset', 'HEAD', UPSTREAM_USER_CONFIG])
# Now generate Android.mk and SkUserConfig.h
gyp_failed = False
try:
gyp_to_android.main()
except AssertionError as e:
print e
# Failed to generate the makefiles. Make a human fix the problem.
git_utils.MergeAbort()
raise Exception(
'Failed to generate makefiles for %s. Fall back to '
'manual human merge.' % commit_to_merge)
except SystemExit as e:
gyp_failed = True
if not gyp_failed:
git_utils.Add('Android.mk')
git_utils.Add(ANDROID_USER_CONFIG)
git_utils.Add(os.path.join('tests', 'Android.mk'))
git_utils.Add(os.path.join('tools', 'Android.mk'))
git_utils.Add(os.path.join('bench', 'Android.mk'))
git_utils.Add(os.path.join('gm', 'Android.mk'))
git_utils.Add(os.path.join('dm', 'Android.mk'))
# Remove upstream user config, which is no longer needed.
os.remove(UPSTREAM_USER_CONFIG)
# Create a new branch.
shell_utils.run([REPO, 'start', LOCAL_BRANCH_NAME, '.'])
try:
orig_msg = shell_utils.run(
[GIT, 'show', commit_to_merge, '--format="%s"',
'-s']).rstrip()
message = 'Merge %s into master-skia\n\n' + SKIA_REV_URL
if gyp_failed:
message += '\n\nFIXME: Failed to generate makefiles!'
shell_utils.run([
GIT, 'commit', '-m',
message % (orig_msg, commit_to_merge)
])
except shell_utils.CommandFailedException:
# It is possible that someone else already did the merge (for example,
# if they are testing a build slave). Clean up and exit.
RepoAbandon(LOCAL_BRANCH_NAME)
raise BuildStepWarning(
'Nothing to merge; did someone already merge '
'%s?' % commit_to_merge)
# For some reason, sometimes the bot's authentication from sync_android
# does not carry over to this step. Authenticate again.
with GitAuthenticate():
# Now push to master-skia branch
try:
shell_utils.run(
[GIT, 'push', MASTER_SKIA_URL, MASTER_SKIA_REFS])
except shell_utils.CommandFailedException:
# It's possible someone submitted in between our sync and push or
# push failed for some other reason. Abandon and let the next
# attempt try again.
RepoAbandon(LOCAL_BRANCH_NAME)
raise BuildStepFailure('git push failed!')
# Our branch is no longer needed. Remove it.
shell_utils.run([REPO, 'sync', '-j32', '.'])
shell_utils.run([REPO, 'prune', '.'])
# If gyp failed, this probably means there was an error in the gyp
# files. We still want to push the commit. This way, when it gets
# fixed with a future commit, we don't remain hung up on this one.
if gyp_failed:
raise BuildStepFailure(
'Merged %s, but failed to generate makefiles.'
' Is there a mistake in the gyp files?' %
commit_to_merge)