def RevGitFile(filename, data, retries=5, dryrun=False):
"""Update and push the git file.
Args:
filename: file to modify that is in a git repo already
data: A dict of key/values to update in |filename|
retries: The number of times to retry before giving up, default: 5
"""
prebuilt_branch = 'prebuilt_branch'
cwd = os.path.abspath(os.path.dirname(filename))
commit = git.RunGit(cwd, ['rev-parse', 'HEAD']).output.rstrip()
description = '%s: updating %s' % (os.path.basename(filename),
', '.join(data.keys()))
# UpdateLocalFile will print out the keys/values for us.
print 'Revving git file %s' % filename
try:
git.CreatePushBranch(prebuilt_branch, cwd)
for key, value in data.iteritems():
UpdateLocalFile(filename, value, key)
git.RunGit(cwd, ['add', filename])
git.RunGit(cwd, ['commit', '-m', description])
git.PushWithRetry(prebuilt_branch, cwd, dryrun=dryrun, retries=retries)
finally:
cros_build_lib.RunCommand(['git', 'checkout', commit], cwd=cwd)
def testPushChange(self):
git_log = 'Marking test_one as stable\nMarking test_two as stable\n'
fake_description = 'Marking set of ebuilds as stable\n\n%s' % git_log
self.mox.StubOutWithMock(cros_mark_as_stable, '_DoWeHaveLocalCommits')
self.mox.StubOutWithMock(cros_mark_as_stable.GitBranch, 'CreateBranch')
self.mox.StubOutWithMock(cros_mark_as_stable.GitBranch, 'Exists')
self.mox.StubOutWithMock(git, 'PushWithRetry')
self.mox.StubOutWithMock(git, 'GetTrackingBranch')
self.mox.StubOutWithMock(git, 'SyncPushBranch')
self.mox.StubOutWithMock(git, 'CreatePushBranch')
self.mox.StubOutWithMock(git, 'RunGit')
cros_mark_as_stable._DoWeHaveLocalCommits(
self._branch, self._target_manifest_branch, '.').AndReturn(True)
git.GetTrackingBranch('.', for_push=True).AndReturn(
['gerrit', 'refs/remotes/gerrit/master'])
git.SyncPushBranch('.', 'gerrit', 'refs/remotes/gerrit/master')
cros_mark_as_stable._DoWeHaveLocalCommits(
self._branch, 'refs/remotes/gerrit/master', '.').AndReturn(True)
result = cros_build_lib.CommandResult(output=git_log)
cros_build_lib.RunCommandCaptureOutput(
['git', 'log', '--format=format:%s%n%n%b',
'refs/remotes/gerrit/master..%s' % self._branch],
cwd='.').AndReturn(result)
git.CreatePushBranch('merge_branch', '.')
git.RunGit('.', ['merge', '--squash', self._branch])
git.RunGit('.', ['commit', '-m', fake_description])
git.RunGit('.', ['config', 'push.default', 'tracking'])
git.PushWithRetry('merge_branch', '.', dryrun=False)
self.mox.ReplayAll()
cros_mark_as_stable.PushChange(self._branch, self._target_manifest_branch,
False, '.')
self.mox.VerifyAll()
def _PushCommits(self, candidate_files):
"""Commit and push changes to current branch.
Args:
candidate_files: candidate files to commit and update.
"""
git.RunGit(self.chromite_dir, ['add'] + candidate_files)
commit_msg = "Update config settings by config-updater."
git.RunGit(self.chromite_dir, ['commit', '-m', commit_msg],
print_cmd=True)
git.RunGit(self.chromite_dir, ['config', 'push.default', 'tracking'])
git.PushWithRetry(self.branch, self.chromite_dir, dryrun=self.dryrun)
def _PushConfig(self, workdir, testjob, dryrun, current_time):
"""Pushes the tryjob config to Git as a file.
Args:
workdir: see Submit()
testjob: see Submit()
dryrun: see Submit()
current_time: the current time as a string represention of the time since
unix epoch.
"""
push_branch = manifest_version.PUSH_BRANCH
remote_branch = None
if testjob:
remote_branch = git.RemoteRef('origin', 'refs/remotes/origin/test')
git.CreatePushBranch(push_branch,
workdir,
sync=False,
remote_push_branch=remote_branch)
file_name = '%s.%s' % (self.user, current_time)
user_dir = os.path.join(workdir, self.user)
if not os.path.isdir(user_dir):
os.mkdir(user_dir)
fullpath = os.path.join(user_dir, file_name)
with open(fullpath, 'w+') as job_desc_file:
json.dump(self.values, job_desc_file)
git.RunGit(workdir, ['add', fullpath])
extra_env = {
# The committer field makes sure the creds match what the remote
# gerrit instance expects while the author field allows lookup
# on the console to work. http://crosbug.com/27939
'GIT_COMMITTER_EMAIL': self.user_email,
'GIT_AUTHOR_EMAIL': self.user_email,
}
git.RunGit(workdir, ['commit', '-m', self.description],
extra_env=extra_env)
try:
git.PushWithRetry(push_branch, workdir, retries=3, dryrun=dryrun)
except cros_build_lib.RunCommandError:
logging.error(
'Failed to submit tryjob. This could be due to too many '
'submission requests by users. Please try again.')
raise
def PushChange(stable_branch, tracking_branch, dryrun, cwd):
"""Pushes commits in the stable_branch to the remote git repository.
Pushes local commits from calls to CommitChange to the remote git
repository specified by current working directory. If changes are
found to commit, they will be merged to the merge branch and pushed.
In that case, the local repository will be left on the merge branch.
Args:
stable_branch: The local branch with commits we want to push.
tracking_branch: The tracking branch of the local branch.
dryrun: Use git push --dryrun to emulate a push.
cwd: The directory to run commands in.
Raises:
OSError: Error occurred while pushing.
"""
if not _DoWeHaveLocalCommits(stable_branch, tracking_branch, cwd):
cros_build_lib.Info('No work found to push in %s. Exiting', cwd)
return
# For the commit queue, our local branch may contain commits that were
# just tested and pushed during the CommitQueueCompletion stage. Sync
# and rebase our local branch on top of the remote commits.
remote, push_branch = git.GetTrackingBranch(cwd, for_push=True)
git.SyncPushBranch(cwd, remote, push_branch)
# Check whether any local changes remain after the sync.
if not _DoWeHaveLocalCommits(stable_branch, push_branch, cwd):
cros_build_lib.Info('All changes already pushed for %s. Exiting', cwd)
return
description = cros_build_lib.RunCommandCaptureOutput([
'git', 'log', '--format=format:%s%n%n%b',
'%s..%s' % (push_branch, stable_branch)
],
cwd=cwd).output
description = 'Marking set of ebuilds as stable\n\n%s' % description
cros_build_lib.Info('For %s, using description %s', cwd, description)
git.CreatePushBranch(constants.MERGE_BRANCH, cwd)
git.RunGit(cwd, ['merge', '--squash', stable_branch])
git.RunGit(cwd, ['commit', '-m', description])
git.RunGit(cwd, ['config', 'push.default', 'tracking'])
git.PushWithRetry(constants.MERGE_BRANCH, cwd, dryrun=dryrun)
def PushChange(stable_branch, tracking_branch, dryrun, cwd,
staging_branch=None):
"""Pushes commits in the stable_branch to the remote git repository.
Pushes local commits from calls to CommitChange to the remote git
repository specified by current working directory. If changes are
found to commit, they will be merged to the merge branch and pushed.
In that case, the local repository will be left on the merge branch.
Args:
stable_branch: The local branch with commits we want to push.
tracking_branch: The tracking branch of the local branch.
dryrun: Use git push --dryrun to emulate a push.
cwd: The directory to run commands in.
staging_branch: The staging branch to push for a failed PFQ run
Raises:
OSError: Error occurred while pushing.
"""
if not git.DoesCommitExistInRepo(cwd, stable_branch):
logging.debug('No branch created for %s. Exiting', cwd)
return
if not _DoWeHaveLocalCommits(stable_branch, tracking_branch, cwd):
logging.debug('No work found to push in %s. Exiting', cwd)
return
# For the commit queue, our local branch may contain commits that were
# just tested and pushed during the CommitQueueCompletion stage. Sync
# and rebase our local branch on top of the remote commits.
remote_ref = git.GetTrackingBranch(cwd,
branch=stable_branch,
for_push=True)
# SyncPushBranch rebases HEAD onto the updated remote. We need to checkout
# stable_branch here in order to update it.
git.RunGit(cwd, ['checkout', stable_branch])
git.SyncPushBranch(cwd, remote_ref.remote, remote_ref.ref)
# Check whether any local changes remain after the sync.
if not _DoWeHaveLocalCommits(stable_branch, remote_ref.ref, cwd):
logging.info('All changes already pushed for %s. Exiting', cwd)
return
# Add a failsafe check here. Only CLs from the 'chrome-bot' user should
# be involved here. If any other CLs are found then complain.
# In dryruns extra CLs are normal, though, and can be ignored.
bad_cl_cmd = ['log', '--format=short', '--perl-regexp',
'--author', '^(?!chrome-bot)', '%s..%s' % (
remote_ref.ref, stable_branch)]
bad_cls = git.RunGit(cwd, bad_cl_cmd).output
if bad_cls.strip() and not dryrun:
logging.error('The Uprev stage found changes from users other than '
'chrome-bot:\n\n%s', bad_cls)
raise AssertionError('Unexpected CLs found during uprev stage.')
if staging_branch is not None:
logging.info('PFQ FAILED. Pushing uprev change to staging branch %s',
staging_branch)
description = git.RunGit(
cwd,
['log', '--format=format:%s%n%n%b',
'%s..%s' % (remote_ref.ref, stable_branch)]).output
description = '%s\n\n%s' % (GIT_COMMIT_SUBJECT, description)
logging.info('For %s, using description %s', cwd, description)
git.CreatePushBranch(constants.MERGE_BRANCH, cwd,
remote_push_branch=remote_ref)
git.RunGit(cwd, ['merge', '--squash', stable_branch])
git.RunGit(cwd, ['commit', '-m', description])
git.RunGit(cwd, ['config', 'push.default', 'tracking'])
git.PushWithRetry(constants.MERGE_BRANCH, cwd, dryrun=dryrun,
staging_branch=staging_branch)
def _Submit(self, testjob, dryrun):
"""Internal submission function. See Submit() for arg description."""
# TODO(rcui): convert to shallow clone when that's available.
current_time = str(int(time.time()))
ref_base = os.path.join('refs/tryjobs', self.user, current_time)
for patch in self.local_patches:
# Isolate the name; if it's a tag or a remote, let through.
# Else if it's a branch, get the full branch name minus refs/heads.
local_branch = git.StripRefsHeads(patch.ref, False)
ref_final = os.path.join(ref_base, local_branch, patch.sha1)
self.manifest.AssertProjectIsPushable(patch.project)
data = self.manifest.projects[patch.project]
print 'Uploading patch %s' % patch
patch.Upload(data['push_url'], ref_final, dryrun=dryrun)
# TODO(rcui): Pass in the remote instead of tag. http://crosbug.com/33937.
tag = constants.EXTERNAL_PATCH_TAG
if data['remote'] == constants.INTERNAL_REMOTE:
tag = constants.INTERNAL_PATCH_TAG
self.extra_args.append('--remote-patches=%s:%s:%s:%s:%s'
% (patch.project, local_branch, ref_final,
patch.tracking_branch, tag))
self._VerifyForBuildbot()
repository.CloneGitRepo(self.tryjob_repo, self.ssh_url)
version_path = os.path.join(self.tryjob_repo,
self.TRYJOB_FORMAT_FILE)
with open(version_path, 'r') as f:
try:
val = int(f.read().strip())
except ValueError:
raise ChromiteUpgradeNeeded()
if val > self.TRYJOB_FORMAT_VERSION:
raise ChromiteUpgradeNeeded(val)
push_branch = manifest_version.PUSH_BRANCH
remote_branch = ('origin', 'refs/remotes/origin/test') if testjob else None
git.CreatePushBranch(push_branch, self.tryjob_repo, sync=False,
remote_push_branch=remote_branch)
file_name = '%s.%s' % (self.user,
current_time)
user_dir = os.path.join(self.tryjob_repo, self.user)
if not os.path.isdir(user_dir):
os.mkdir(user_dir)
fullpath = os.path.join(user_dir, file_name)
with open(fullpath, 'w+') as job_desc_file:
json.dump(self.values, job_desc_file)
cros_build_lib.RunCommand(['git', 'add', fullpath], cwd=self.tryjob_repo)
extra_env = {
# The committer field makes sure the creds match what the remote
# gerrit instance expects while the author field allows lookup
# on the console to work. http://crosbug.com/27939
'GIT_COMMITTER_EMAIL' : self.user_email,
'GIT_AUTHOR_EMAIL' : self.user_email,
}
cros_build_lib.RunCommand(['git', 'commit', '-m', self.description],
cwd=self.tryjob_repo, extra_env=extra_env)
try:
git.PushWithRetry(
push_branch, self.tryjob_repo, retries=3, dryrun=dryrun)
except cros_build_lib.RunCommandError:
cros_build_lib.Error(
'Failed to submit tryjob. This could be due to too many '
'submission requests by users. Please try again.')
raise
def PushChanges(self):
"""Push changes to manifest live."""
git.PushWithRetry(TEST_BRANCH, self.manifest_dir, dryrun=self.dryrun)
