def setUp(self):
self._manifest = 'manifest'
self._parser = cros_mark_as_stable.GetParser()
self._package_list = ['pkg1']
self._overlays = [
os.path.join(self.tempdir, 'overlay_%s' % i) for i in range(0, 3)
]
self._overlay_remote_ref = {
self._overlays[0]: git.RemoteRef('remote', 'ref', 'project_1'),
self._overlays[1]: git.RemoteRef('remote', 'ref', 'project_1'),
self._overlays[2]: git.RemoteRef('remote', 'ref', 'project_2'),
}
self._git_project_overlays = {}
self._overlay_tracking_branch = {}
for overlay in self._overlays:
self._git_project_overlays.setdefault(
self._overlay_remote_ref[overlay], []).append(overlay)
self._overlay_tracking_branch[overlay] = (
self._overlay_remote_ref[overlay].ref)
self.PatchObject(git, 'GetTrackingBranchViaManifest')
# TODO: Remove explicit --buildroot after --srcroot remove and a normal
# default is set.
self._commit_options = self._parser.parse_args(
['commit', '--buildroot', constants.SOURCE_ROOT])
self._push_options = self._parser.parse_args(
['push', '--buildroot', constants.SOURCE_ROOT])
def setUp(self):
self._manifest = 'manifest'
self._parser = cros_mark_as_stable.GetParser()
self._package_list = ['pkg1']
self._overlays = [
os.path.join(self.tempdir, 'overlay_%s' % i) for i in range(0, 3)
]
self._overlay_remote_ref = {
self._overlays[0]: git.RemoteRef('remote', 'ref', 'project_1'),
self._overlays[1]: git.RemoteRef('remote', 'ref', 'project_1'),
self._overlays[2]: git.RemoteRef('remote', 'ref', 'project_2'),
}
self._git_project_overlays = {}
self._overlay_tracking_branch = {}
for overlay in self._overlays:
self._git_project_overlays.setdefault(
self._overlay_remote_ref[overlay], []).append(overlay)
self._overlay_tracking_branch[overlay] = (
self._overlay_remote_ref[overlay].ref)
self.PatchObject(git, 'GetTrackingBranchViaManifest')
self._commit_options = self._parser.parse_args(['commit'])
self._push_options = self._parser.parse_args(['push'])
def setUp(self):
self.PatchObject(git.ManifestCheckout,
'Cached',
return_value='manifest')
self.mock_work_on_push = self.PatchObject(cros_mark_as_stable,
'_WorkOnPush')
self.mock_work_on_commit = self.PatchObject(cros_mark_as_stable,
'_WorkOnCommit')
self._overlays = []
remote_refs = []
self._overlay_tracking_branch = {}
self._git_project_overlays = {}
for i in range(0, 3):
overlay = os.path.join(self.tempdir, 'overlay_%s' % i)
osutils.SafeMakedirs(overlay)
self._overlays.append(overlay)
remote_ref = git.RemoteRef('remote', 'ref', 'project_%s' % i)
remote_refs.append(remote_ref)
self._overlay_tracking_branch[overlay] = remote_ref.ref
self._git_project_overlays[remote_ref.project_name] = [overlay]
self.PatchObject(portage_util,
'FindOverlays',
return_value=self._overlays)
self.PatchObject(git,
'GetTrackingBranchViaManifest',
side_effect=remote_refs)
def BumpVersion(self, which, branch, message, dry_run=True, fetch=False):
"""Increment version in chromeos_version.sh and commit it.
Args:
which: Which version should be incremented. One of
'chrome_branch', 'build', 'branch, 'patch'.
branch: The branch to push to.
message: The commit message for the version bump.
dry_run: Whether to use git --dry-run.
fetch: Whether to fetch and checkout to the given branch.
"""
logging.notice(message)
chromiumos_overlay = self.manifest.GetUniqueProject(
'chromiumos/overlays/chromiumos-overlay')
remote = chromiumos_overlay.Remote().GitName()
ref = git.NormalizeRef(branch)
if fetch:
self.RunGit(chromiumos_overlay, ['fetch', remote, ref])
self.RunGit(chromiumos_overlay,
['checkout', '-B', branch, 'FETCH_HEAD'])
new_version = self.ReadVersion(incr_type=which)
new_version.IncrementVersion()
remote_ref = git.RemoteRef(remote, ref)
new_version.UpdateVersionFile(message,
dry_run=dry_run,
push_to=remote_ref)
def _IncrementVersionOnDiskForNewBranch(self, push_remote):
"""Bumps the version found in chromeos_version.sh on the new branch
When a new branch is created, the branch component of the new branch's
version needs to bumped.
For example, say 'stabilize-link' is created from a the 4230.0.0 manifest.
The new branch's version needs to be bumped to 4230.1.0.
Args:
push_remote: a git remote name where the new branch lives.
"""
# This needs to happen before the source branch version bumping above
# because we rely on the fact that since our current overlay checkout
# is what we just pushed to the new branch, we don't need to do another
# sync. This also makes it easier to implement skip_remote_push
# functionality (the new branch doesn't actually get created in
# skip_remote_push mode).
# Use local branch ref.
branch_ref = git.NormalizeRef(self.branch_name)
push_to = git.RemoteRef(push_remote, branch_ref)
version_info = manifest_version.VersionInfo(
version_string=self._run.options.force_version)
incr_type, incr_target = self.DetermineBranchIncrParams(version_info)
message = self.COMMIT_MESSAGE % {
'target': incr_target,
'branch': branch_ref,
}
self._IncrementVersionOnDisk(incr_type, push_to, message)
def _PushGitChanges(git_repo, message, dry_run=True, push_to=None):
"""Push the final commit into the git repo.
Args:
git_repo: git repo to push
message: Commit message
dry_run: If true, don't actually push changes to the server
push_to: A git.RemoteRef object specifying the remote branch to push to.
Defaults to the tracking branch of the current branch.
"""
push_branch = None
if push_to is None:
remote, push_branch = git.GetTrackingBranch(git_repo,
for_checkout=False,
for_push=True)
push_to = git.RemoteRef(remote, push_branch)
git.RunGit(git_repo, ['add', '-A'])
# It's possible that while we are running on dry_run, someone has already
# committed our change.
try:
git.RunGit(git_repo, ['commit', '-m', message])
except cros_build_lib.RunCommandError:
if dry_run:
return
raise
git.GitPush(git_repo, PUSH_BRANCH, push_to, dryrun=dry_run, force=dry_run)
def _FixUpManifests(self, repo_manifest):
"""Points the checkouts at the new branch in the manifests.
Within the branch, make sure all manifests with projects that are
"branchable" are checked out to "refs/heads/<new_branch>". Do this
by updating all manifests in the known manifest projects.
"""
assert not self._run.options.delete_branch, 'Cannot fix a deleted branch.'
# Use local branch ref.
branch_ref = git.NormalizeRef(self.branch_name)
logging.debug('Fixing manifest projects for new branch.')
for project in site_config.params.MANIFEST_PROJECTS:
manifest_checkout = repo_manifest.FindCheckout(project)
manifest_dir = manifest_checkout['local_path']
push_remote = manifest_checkout['push_remote']
# Checkout revision can be either a sha1 or a branch ref.
src_ref = manifest_checkout['revision']
if not git.IsSHA1(src_ref):
src_ref = git.NormalizeRemoteRef(push_remote, src_ref)
git.CreateBranch(manifest_dir, manifest_version.PUSH_BRANCH,
src_ref)
# We want to process default.xml and official.xml + their imports.
pending_manifests = [
constants.DEFAULT_MANIFEST, constants.OFFICIAL_MANIFEST
]
processed_manifests = []
while pending_manifests:
# Canonicalize the manifest name (resolve dir and symlinks).
manifest_path = os.path.join(manifest_dir,
pending_manifests.pop())
manifest_path = os.path.realpath(manifest_path)
# Don't process a manifest more than once.
if manifest_path in processed_manifests:
continue
processed_manifests.append(manifest_path)
if not os.path.exists(manifest_path):
logging.info('Manifest not found: %s', manifest_path)
continue
logging.debug('Fixing manifest at %s.', manifest_path)
included_manifests = self._UpdateManifest(manifest_path)
pending_manifests += included_manifests
git.RunGit(manifest_dir, ['add', '-A'], print_cmd=True)
message = 'Fix up manifest after branching %s.' % branch_ref
git.RunGit(manifest_dir, ['commit', '-m', message], print_cmd=True)
push_to = git.RemoteRef(push_remote, branch_ref)
git.GitPush(manifest_dir,
manifest_version.PUSH_BRANCH,
push_to,
skip=self.skip_remote_push)
def _RunGitPush():
"""Runs git.GitPush with some default arguments."""
git.GitPush('some_repo_path',
'local-ref',
git.RemoteRef('some-remote', 'remote-ref'),
retry=True,
skip=False)
def testGitPushProjectUpstreamDryRun(self):
"""Test _GitPushProjectUpstream with dry_run=True."""
create_manifest_snapshot._GitPushProjectUpstream(
'root', self.project_b, True)
self.mock_git_push.assert_called_with(
'root/project/b', 'cafe1234', git.RemoteRef('origin', 'short'),
dry_run=True)
def testGitPushComplix(self):
"""Test GitPush with some arguments."""
git.GitPush('git_path', 'HEAD', git.RemoteRef('origin', 'master'),
force=True, dry_run=True)
self.assertCommandCalled(['git', 'push', 'origin', 'HEAD:master',
'--force', '--dry-run'],
capture_output=True, print_cmd=False,
cwd='git_path', encoding='utf-8')
def testGitPushSimple(self):
"""Test GitPush with minimal arguments."""
git.GitPush('git_path', 'HEAD', git.RemoteRef('origin', 'master'))
self.assertCommandCalled(['git', 'push', 'origin', 'HEAD:master'],
capture_output=True,
print_cmd=False,
cwd='git_path',
encoding='utf-8')
def _IncrementVersionOnDiskForSourceBranch(self, overlay_dir, push_remote,
source_branch):
"""Bumps the version found in chromeos_version.sh on the source branch
The source branch refers to the branch that the manifest used for creating
the new branch came from. For release branches, we generally branch from a
'master' branch manifest.
To work around crbug.com/213075, for both non-release and release branches,
we need to bump the Chrome OS version on the source branch if the manifest
used for branch creation is the latest generated manifest for the source
branch.
When we are creating a release branch, the Chrome major version of the
'master' (source) branch needs to be bumped. For example, if we branch
'release-R29-4230.B' from the 4230.0.0 manifest (which is from the 'master'
branch), the 'master' branch's Chrome major version in chromeos_version.sh
(which is 29) needs to be bumped to 30.
Args:
overlay_dir: Absolute path to the chromiumos overlay repo.
push_remote: The remote to push to.
source_branch: The branch that the manifest we are using comes from.
"""
push_to = git.RemoteRef(push_remote, source_branch)
self._FetchAndCheckoutTo(overlay_dir, push_to)
# Use local branch ref.
branch_ref = git.NormalizeRef(self.branch_name)
tot_version_info = manifest_version.VersionInfo.from_repo(
self._build_root)
if (branch_ref.startswith('refs/heads/release-')
or tot_version_info.VersionString()
== self._run.options.force_version):
incr_type, incr_target = self.DetermineSourceIncrParams(
source_branch, branch_ref)
message = self.COMMIT_MESSAGE % {
'target': incr_target,
'branch': branch_ref,
}
try:
self._IncrementVersionOnDisk(incr_type, push_to, message)
except cros_build_lib.RunCommandError:
# There's a chance we are racing against the buildbots for this
# increment. We shouldn't quit the script because of this. Instead, we
# print a warning.
self._FetchAndCheckoutTo(overlay_dir, push_to)
new_version = manifest_version.VersionInfo.from_repo(
self._build_root)
if new_version.VersionString(
) != tot_version_info.VersionString():
logging.warning(
'Version number for branch %s was bumped by another '
'bot.', push_to.ref)
else:
raise
def testMainSnapshots(self):
"""Test main with projects that need snapshots."""
self.mock_is_reachable.return_value = False
args = self.main_args + ['--snapshot-ref', 'refs/snap']
with parallel_unittest.ParallelMock():
create_manifest_snapshot.main(args)
snapshot_xml = osutils.ReadFile(self.output_file)
self.mock_git_push.assert_has_calls([
mock.call(os.path.join(self.repo_root, 'project/a'),
'f01dab1e',
git.RemoteRef('origin', 'refs/snap'),
dry_run=False),
mock.call(os.path.join(self.repo_root, 'project/b'),
'cafe1234',
git.RemoteRef('origin', 'refs/snap'),
dry_run=False),
mock.call(os.path.join(self.repo_root, 'project/c'),
'deadbeef',
git.RemoteRef('origin', 'refs/snap'),
dry_run=False),
mock.call(os.path.join(self.repo_root, 'dupe'),
'd1',
git.RemoteRef('origin', 'refs/snap/dupe'),
dry_run=False),
mock.call(os.path.join(self.repo_root, 'dupe'),
'd2',
git.RemoteRef('origin', 'refs/snap/dupe/1'),
dry_run=False),
],
any_order=True)
expected = repo_manifest.Manifest.FromString(MANIFEST_XML)
for project in expected.Projects():
if project.name == 'dupe':
if project.revision == 'd1':
project.upstream = 'refs/snap/dupe'
else:
project.upstream = 'refs/snap/dupe/1'
else:
project.upstream = 'refs/snap'
expected_xml = repo_manifest_unittest.ManifestToString(expected)
self.AssertXMLAlmostEqual(snapshot_xml, expected_xml)
def testBumpVersionMinimal(self):
"""Test BumpVersion with minimal arguments."""
checkout = CrosCheckout('/root')
checkout.BumpVersion('patch', 'my-branch', 'My message.')
self.assertEqual(self.from_repo.call_args_list,
[mock.call('/root', incr_type='patch')])
self.assertEqual(self.increment_version.call_count, 1)
self.assertEqual(self.update_version.call_args_list, [
mock.call('My message.',
dry_run=True,
push_to=git.RemoteRef('cros', 'refs/heads/my-branch'))
])
def _RunPush(self, checkout, src_ref, dest_ref, force=False):
"""Perform a git push for a checkout.
Args:
checkout: A dictionary of checkout manifest attributes.
src_ref: The source local ref to push to the remote.
dest_ref: The local remote ref that correspond to destination ref name.
force: Whether to override non-fastforward checks.
"""
# Convert local tracking ref to refs/heads/* on a remote:
# refs/remotes/<remote name>/<branch> to refs/heads/<branch>.
# If dest_ref is already refs/heads/<branch> it's a noop.
dest_ref = git.NormalizeRef(git.StripRefs(dest_ref))
push_to = git.RemoteRef(checkout['push_remote'], dest_ref)
git.GitPush(checkout['local_path'], src_ref, push_to, force=force,
skip=self.skip_remote_push)
def _SetupWorkDirectoryForPatch(work_dir, patch, branch, manifest, email):
"""Set up local dir for uploading changes to the given patch's project."""
logging.notice('Setting up dir %s for uploading changes to %s', work_dir,
patch.project_url)
# Clone the git repo from reference if we have a pointer to a
# ManifestCheckout object.
reference = None
if manifest:
# Get the path to the first checkout associated with this change. Since
# all of the checkouts share git objects, it doesn't matter which checkout
# we pick.
path = manifest.FindCheckouts(patch.project,
only_patchable=True)[0]['path']
reference = os.path.join(constants.SOURCE_ROOT, path)
if not os.path.isdir(reference):
logging.error('Unable to locate git checkout: %s', reference)
logging.error('Did you mean to use --nomirror?')
# This will do an "raise OSError" with the right values.
os.open(reference, os.O_DIRECTORY)
# Use the email if email wasn't specified.
if not email:
email = git.GetProjectUserEmail(reference)
repository.CloneGitRepo(work_dir, patch.project_url, reference=reference)
# Set the git committer.
git.RunGit(work_dir, ['config', '--replace-all', 'user.email', email])
mbranch = git.MatchSingleBranchName(work_dir,
branch,
namespace='refs/remotes/origin/')
if branch != mbranch:
logging.notice('Auto resolved branch name "%s" to "%s"', branch,
mbranch)
branch = mbranch
# Finally, create a local branch for uploading changes to the given remote
# branch.
git.CreatePushBranch(constants.PATCH_BRANCH,
work_dir,
sync=False,
remote_push_branch=git.RemoteRef(
'ignore', 'origin/%s' % branch))
return branch
def _PushConfig(self, workdir, testjob, dryrun, current_time):
"""Pushes the tryjob config to Git as a file.
Args:
workdir: see Submit()
testjob: see Submit()
dryrun: see Submit()
current_time: the current time as a string represention of the time since
unix epoch.
"""
push_branch = manifest_version.PUSH_BRANCH
remote_branch = None
if testjob:
remote_branch = git.RemoteRef('origin', 'refs/remotes/origin/test')
git.CreatePushBranch(push_branch,
workdir,
sync=False,
remote_push_branch=remote_branch)
file_name = '%s.%s' % (self.user, current_time)
user_dir = os.path.join(workdir, self.user)
if not os.path.isdir(user_dir):
os.mkdir(user_dir)
fullpath = os.path.join(user_dir, file_name)
with open(fullpath, 'w+') as job_desc_file:
json.dump(self.values, job_desc_file)
git.RunGit(workdir, ['add', fullpath])
extra_env = {
# The committer field makes sure the creds match what the remote
# gerrit instance expects while the author field allows lookup
# on the console to work. http://crosbug.com/27939
'GIT_COMMITTER_EMAIL': self.user_email,
'GIT_AUTHOR_EMAIL': self.user_email,
}
git.RunGit(workdir, ['commit', '-m', self.description],
extra_env=extra_env)
try:
git.PushWithRetry(push_branch, workdir, retries=3, dryrun=dryrun)
except cros_build_lib.RunCommandError:
logging.error(
'Failed to submit tryjob. This could be due to too many '
'submission requests by users. Please try again.')
raise
def PushSpecChanges(self, commit_message):
"""Pushes any changes you have in the manifest directory.
Args:
commit_message: Message that the git commit will contain.
"""
# %submit enables Gerrit automerge feature to manage contention on the
# high traffic manifest_versions repository.
push_to_git = git.GetTrackingBranch(self.manifest_dir,
for_checkout=False,
for_push=False)
push_to = git.RemoteRef(push_to_git.remote, 'refs/for/master%submit',
push_to_git.project_name)
_PushGitChanges(self.manifest_dir,
commit_message,
dry_run=self.dry_run,
push_to=push_to)
def _UpdateLuciProject(self):
chromite_source_file = os.path.join(constants.CHROMITE_DIR, 'config',
'luci-scheduler.cfg')
generated_source_file = os.path.join(self.project_dir, 'generated',
'luci-scheduler.cfg')
target_file = os.path.join(self.project_dir, 'luci',
'luci-scheduler.cfg')
concatenated_content = (osutils.ReadFile(chromite_source_file) +
'\n\n' +
osutils.ReadFile(generated_source_file))
if concatenated_content == osutils.ReadFile(target_file):
logging.PrintBuildbotStepText(
'luci-scheduler.cfg current: No Update.')
return
chromite_rev = git.RunGit(
constants.CHROMITE_DIR,
['rev-parse', 'HEAD:config/luci-scheduler.cfg']).output.rstrip()
message = textwrap.dedent("""\
luci-scheduler.cfg: Chromite %s
Auto update to match generated file in chromite and luci config.
""" % chromite_rev)
with open(target_file, 'w') as f:
f.write(concatenated_content)
git.RunGit(self.project_dir, ['add', '-A'])
git.RunGit(self.project_dir, ['commit', '-m', message])
push_to = git.RemoteRef('origin', self.PROJECT_BRANCH)
logging.info('Pushing to branch (%s) with message: %s %s', push_to,
message, ' (dryrun)' if self._run.options.debug else '')
git.RunGit(self.project_dir, ['config', 'push.default', 'tracking'],
print_cmd=True)
git.PushBranch(self.PROJECT_BRANCH,
self.project_dir,
dryrun=self._run.options.debug)
logging.PrintBuildbotStepText('luci-scheduler.cfg: Updated.')
def testBumpVersionAllOptions(self):
"""Test BumpVersion properly defers to manifest_version functions."""
checkout = CrosCheckout('/root')
checkout.BumpVersion('patch',
'my-branch',
'My message.',
fetch=True,
dry_run=False)
self.rc_mock.assertCommandContains(
['git', 'fetch', 'cros', 'refs/heads/my-branch'],
cwd='/root/src/third_party/chromiumos-overlay')
self.rc_mock.assertCommandContains(
['git', 'checkout', '-B', 'my-branch', 'FETCH_HEAD'],
cwd='/root/src/third_party/chromiumos-overlay')
self.assertEqual(self.from_repo.call_args_list,
[mock.call('/root', incr_type='patch')])
self.assertEqual(self.increment_version.call_count, 1)
self.assertEqual(self.update_version.call_args_list, [
mock.call('My message.',
dry_run=False,
push_to=git.RemoteRef('cros', 'refs/heads/my-branch'))
])
def _TestPushChange(self, bad_cls):
side_effect = Exception('unittest says this should not be called')
git_log = 'Marking test_one as stable\nMarking test_two as stable\n'
fake_description = 'Marking set of ebuilds as stable\n\n%s' % git_log
self.PatchObject(git, 'DoesCommitExistInRepo', return_value=True)
self.PatchObject(cros_mark_as_stable,
'_DoWeHaveLocalCommits',
return_value=True)
self.PatchObject(cros_mark_as_stable.GitBranch,
'CreateBranch',
side_effect=side_effect)
self.PatchObject(cros_mark_as_stable.GitBranch,
'Exists',
side_effect=side_effect)
push_mock = self.PatchObject(git, 'PushWithRetry')
self.PatchObject(git,
'GetTrackingBranch',
return_value=git.RemoteRef(
'gerrit', 'refs/remotes/gerrit/master'))
sync_mock = self.PatchObject(git, 'SyncPushBranch')
create_mock = self.PatchObject(git, 'CreatePushBranch')
git_mock = self.StartPatcher(RunGitMock())
git_mock.AddCmdResult(['checkout', self._branch])
cmd = [
'log', '--format=short', '--perl-regexp', '--author',
'^(?!chrome-bot)',
'refs/remotes/gerrit/master..%s' % self._branch
]
if bad_cls:
push_mock.side_effect = side_effect
create_mock.side_effect = side_effect
git_mock.AddCmdResult(cmd, output='Found bad stuff')
else:
git_mock.AddCmdResult(cmd, output='\n')
cmd = [
'log', '--format=format:%s%n%n%b',
'refs/remotes/gerrit/master..%s' % self._branch
]
git_mock.AddCmdResult(cmd, output=git_log)
git_mock.AddCmdResult(['merge', '--squash', self._branch])
git_mock.AddCmdResult(['commit', '-m', fake_description])
git_mock.AddCmdResult(['config', 'push.default', 'tracking'])
cros_mark_as_stable.PushChange(self._branch,
self._target_manifest_branch, False,
'.')
sync_mock.assert_called_with('.', 'gerrit',
'refs/remotes/gerrit/master')
if not bad_cls:
push_mock.assert_called_with('merge_branch',
'.',
dryrun=False,
staging_branch=None)
create_mock.assert_called_with('merge_branch',
'.',
remote_push_branch=mock.ANY)
def _GitPushProjectUpstream(repo_root, project, dry_run):
"""Push the project revision to its remote upstream."""
git.GitPush(os.path.join(repo_root, project.Path()),
project.revision,
git.RemoteRef(project.Remote().GitName(), project.upstream),
dry_run=dry_run)
