def IsInternalRepoCheckout(root):
"""Returns whether root houses an internal 'repo' checkout."""
manifest_dir = os.path.join(root, '.repo', 'manifests')
manifest_url = git.RunGit(manifest_dir,
['config', 'remote.origin.url']).output.strip()
return (os.path.splitext(
os.path.basename(manifest_url))[0] == os.path.splitext(
os.path.basename(constants.MANIFEST_INT_URL))[0])
def ListRemoteBranches():
"""Get a list of all remote branches for the chromite repository.
Returns:
List of branch names as strings.
"""
ret = git.RunGit(constants.CHROMITE_DIR, ['branch', '-lr'])
return [l.strip() for l in ret.output.splitlines()]
def _DoWeHaveLocalCommits(stable_branch, tracking_branch, cwd):
"""Returns true if there are local commits."""
current_branch = git.GetCurrentBranch(cwd)
if current_branch != stable_branch:
return False
output = git.RunGit(cwd,
['rev-parse', 'HEAD', tracking_branch]).output.split()
return output[0] != output[1]
def Checkout(self, branch=None):
"""Function used to check out to another GitBranch."""
if not branch:
branch = self.branch_name
if branch == self.tracking_branch or self.Exists(branch):
git.RunGit(self.cwd, ['checkout', '-f', branch], quiet=True)
else:
repo = repo_util.Repository.MustFind(self.cwd)
repo.StartBranch(branch, projects=['.'], cwd=self.cwd)
def CommitIfChanged(ebuild_dir, message):
"""If there are changes to ebuild or Manifest, commit them.
Args:
ebuild_dir: the path to the directory of ebuild in the chroot
message: commit message
"""
# Check if anything changed compared to the previous version.
modifications = git.RunGit(
ebuild_dir, ['status', '--porcelain', '-uno'],
capture_output=True,
print_cmd=True).output
if not modifications:
logging.info('AFDO info for the ebuilds did not change. '
'Nothing to commit')
return
git.RunGit(ebuild_dir, ['commit', '-a', '-m', message], print_cmd=True)
def _PushConfig(self, workdir, testjob, dryrun, current_time):
"""Pushes the tryjob config to Git as a file.
Args:
workdir: see Submit()
testjob: see Submit()
dryrun: see Submit()
current_time: the current time as a string represention of the time since
unix epoch.
"""
push_branch = manifest_version.PUSH_BRANCH
remote_branch = None
if testjob:
remote_branch = git.RemoteRef('origin', 'refs/remotes/origin/test')
git.CreatePushBranch(push_branch, workdir, sync=False,
remote_push_branch=remote_branch)
file_name = '%s.%s' % (self.user, current_time)
user_dir = os.path.join(workdir, self.user)
if not os.path.isdir(user_dir):
os.mkdir(user_dir)
fullpath = os.path.join(user_dir, file_name)
with open(fullpath, 'w+') as job_desc_file:
json.dump(self.values, job_desc_file)
git.RunGit(workdir, ['add', fullpath])
extra_env = {
# The committer field makes sure the creds match what the remote
# gerrit instance expects while the author field allows lookup
# on the console to work. http://crosbug.com/27939
'GIT_COMMITTER_EMAIL' : self.user_email,
'GIT_AUTHOR_EMAIL' : self.user_email,
}
git.RunGit(workdir, ['commit', '-m', self.description],
extra_env=extra_env)
try:
git.PushWithRetry(push_branch, workdir, retries=3, dryrun=dryrun)
except cros_build_lib.RunCommandError:
logging.error(
'Failed to submit tryjob. This could be due to too many '
'submission requests by users. Please try again.')
raise
def main(argv):
parser = GetParser()
options = parser.parse_args(argv)
options.Freeze()
overlay_dir = os.path.abspath(_OVERLAY_DIR % {'srcroot': options.srcroot})
android_package_dir = os.path.join(overlay_dir, constants.ANDROID_CP)
version_to_uprev = None
(unstable_ebuild,
stable_ebuilds) = FindAndroidCandidates(android_package_dir)
acls = MakeAclDict(android_package_dir)
# Mirror artifacts, i.e., images and some sdk tools (e.g., adb, aapt).
version_to_uprev = MirrorArtifacts(options.android_bucket_url,
options.android_build_branch,
options.arc_bucket_url, acls,
constants.ANDROID_BUILD_TARGETS,
options.force_version)
# Mirror GTS.
MirrorArtifacts(options.android_bucket_url,
options.android_gts_build_branch, options.arc_bucket_url,
acls, constants.ANDROID_GTS_BUILD_TARGETS)
stable_candidate = portage_util.BestEBuild(stable_ebuilds)
if stable_candidate:
logging.info('Stable candidate found %s' % stable_candidate.version)
else:
logging.info('No stable candidate found.')
tracking_branch = 'remotes/m/%s' % os.path.basename(
options.tracking_branch)
existing_branch = git.GetCurrentBranch(android_package_dir)
work_branch = cros_mark_as_stable.GitBranch(constants.STABLE_EBUILD_BRANCH,
tracking_branch,
android_package_dir)
work_branch.CreateBranch()
# In the case of uprevving overlays that have patches applied to them,
# include the patched changes in the stabilizing branch.
if existing_branch:
git.RunGit(overlay_dir, ['rebase', existing_branch])
android_version_atom = MarkAndroidEBuildAsStable(
stable_candidate, unstable_ebuild, constants.ANDROID_PN,
version_to_uprev, android_package_dir, options.android_build_branch,
options.arc_bucket_url)
if android_version_atom:
if options.boards:
cros_mark_as_stable.CleanStalePackages(options.srcroot,
options.boards.split(':'),
[android_version_atom])
# Explicit print to communicate to caller.
print('ANDROID_VERSION_ATOM=%s' % android_version_atom)
def PushChange(stable_branch, tracking_branch, dryrun, cwd):
"""Pushes commits in the stable_branch to the remote git repository.
Pushes local commits from calls to CommitChange to the remote git
repository specified by current working directory. If changes are
found to commit, they will be merged to the merge branch and pushed.
In that case, the local repository will be left on the merge branch.
Args:
stable_branch: The local branch with commits we want to push.
tracking_branch: The tracking branch of the local branch.
dryrun: Use git push --dryrun to emulate a push.
cwd: The directory to run commands in.
Raises:
OSError: Error occurred while pushing.
"""
if not _DoWeHaveLocalCommits(stable_branch, tracking_branch, cwd):
cros_build_lib.Info('No work found to push in %s. Exiting', cwd)
return
# For the commit queue, our local branch may contain commits that were
# just tested and pushed during the CommitQueueCompletion stage. Sync
# and rebase our local branch on top of the remote commits.
remote, push_branch = git.GetTrackingBranch(cwd, for_push=True)
git.SyncPushBranch(cwd, remote, push_branch)
# Check whether any local changes remain after the sync.
if not _DoWeHaveLocalCommits(stable_branch, push_branch, cwd):
cros_build_lib.Info('All changes already pushed for %s. Exiting', cwd)
return
description = cros_build_lib.RunCommandCaptureOutput([
'git', 'log', '--format=format:%s%n%n%b',
'%s..%s' % (push_branch, stable_branch)
],
cwd=cwd).output
description = 'Marking set of ebuilds as stable\n\n%s' % description
cros_build_lib.Info('For %s, using description %s', cwd, description)
git.CreatePushBranch(constants.MERGE_BRANCH, cwd)
git.RunGit(cwd, ['merge', '--squash', stable_branch])
git.RunGit(cwd, ['commit', '-m', description])
git.RunGit(cwd, ['config', 'push.default', 'tracking'])
git.PushWithRetry(constants.MERGE_BRANCH, cwd, dryrun=dryrun)
def _UpdateLuciProject(self):
chromite_source_file = os.path.join(constants.CHROMITE_DIR, 'config',
'luci-scheduler.cfg')
generated_source_file = os.path.join(self.project_dir, 'generated',
'luci-scheduler.cfg')
target_file = os.path.join(self.project_dir, 'luci',
'luci-scheduler.cfg')
concatenated_content = (osutils.ReadFile(chromite_source_file) +
'\n\n' +
osutils.ReadFile(generated_source_file))
if concatenated_content == osutils.ReadFile(target_file):
logging.PrintBuildbotStepText(
'luci-scheduler.cfg current: No Update.')
return
chromite_rev = git.RunGit(
constants.CHROMITE_DIR,
['rev-parse', 'HEAD:config/luci-scheduler.cfg']).output.rstrip()
message = textwrap.dedent("""\
luci-scheduler.cfg: Chromite %s
Auto update to match generated file in chromite and luci config.
""" % chromite_rev)
with open(target_file, 'w') as f:
f.write(concatenated_content)
git.RunGit(self.project_dir, ['add', '-A'])
git.RunGit(self.project_dir, ['commit', '-m', message])
push_to = git.RemoteRef('origin', self.PROJECT_BRANCH)
logging.info('Pushing to branch (%s) with message: %s %s', push_to,
message, ' (dryrun)' if self._run.options.debug else '')
git.RunGit(self.project_dir, ['config', 'push.default', 'tracking'],
print_cmd=True)
git.PushBranch(self.PROJECT_BRANCH,
self.project_dir,
dryrun=self._run.options.debug)
logging.PrintBuildbotStepText('luci-scheduler.cfg: Updated.')
def _SetMirrorVersion(self, version, only_if_missing=False):
for path in (self.ext_mirror, self.int_mirror):
vpath = os.path.join(path,
remote_try.RemoteTryJob.TRYJOB_FORMAT_FILE)
if os.path.exists(vpath) and only_if_missing:
continue
# Get ourselves a working dir.
tmp_repo = os.path.join(self.tempdir, 'tmp-repo')
git.RunGit(self.tempdir, ['clone', path, tmp_repo])
vpath = os.path.join(tmp_repo,
remote_try.RemoteTryJob.TRYJOB_FORMAT_FILE)
with open(vpath, 'w') as f:
f.write(str(version))
git.RunGit(tmp_repo, ['add', vpath])
git.RunGit(tmp_repo,
['commit', '-m',
'setting version to %s' % version])
git.RunGit(tmp_repo, ['push', path, 'master:master'])
shutil.rmtree(tmp_repo)
def GetLatestSHA1ForBranch(self, project, branch):
"""Return the git hash at the tip of a branch."""
url = '%s://%s/%s' % (gob_util.GIT_PROTOCOL, self.host, project)
cmd = ['ls-remote', url, 'refs/heads/%s' % branch]
try:
result = git.RunGit('.', cmd, print_cmd=self.print_cmd)
if result:
return result.output.split()[0]
except cros_build_lib.RunCommandError:
logging.error('Command "%s" failed.', cros_build_lib.CmdToStr(cmd),
exc_info=True)
def _PerformStageInTempDir(self):
# The plan for the builders is to use master branch to bootstrap other
# branches. Now, if we wanted to test patches for both the bootstrap code
# (on master) and the branched chromite (say, R20), we need to filter the
# patches by branch.
filter_branch = self._run.manifest_branch
if self._run.options.test_bootstrap:
filter_branch = 'master'
# Filter all requested patches for the branch.
branch_pool = self.patch_pool.FilterBranch(filter_branch)
# Checkout the new version of chromite, and patch it.
chromite_dir = os.path.join(self.tempdir, 'chromite')
reference_repo = os.path.join(constants.CHROMITE_DIR, '.git')
git.Clone(chromite_dir, constants.CHROMITE_URL, reference=reference_repo)
git.RunGit(chromite_dir, ['checkout', filter_branch])
chromite_pool = branch_pool.Filter(project=constants.CHROMITE_PROJECT)
if chromite_pool:
patches = patch_series.PatchSeries.WorkOnSingleRepo(
chromite_dir, filter_branch)
self._ApplyPatchSeries(patches, chromite_pool)
# Re-exec into new instance of cbuildbot, with proper command line args.
cbuildbot_path = constants.PATH_TO_CBUILDBOT
if not os.path.exists(os.path.join(self.tempdir, cbuildbot_path)):
cbuildbot_path = 'chromite/cbuildbot/cbuildbot'
cmd = self.FilterArgsForTargetCbuildbot(self.tempdir, cbuildbot_path,
self._run.options)
extra_params = ['--sourceroot', self._run.options.sourceroot]
extra_params.extend(self._run.options.bootstrap_args)
if self._run.options.test_bootstrap:
# We don't want re-executed instance to see this.
cmd = [a for a in cmd if a != '--test-bootstrap']
else:
# If we've already done the desired number of bootstraps, disable
# bootstrapping for the next execution. Also pass in the patched manifest
# repository.
extra_params.append('--nobootstrap')
if self._run.config.internal:
manifest_pool = branch_pool.FilterIntManifest()
else:
manifest_pool = branch_pool.FilterExtManifest()
if manifest_pool:
manifest_dir = self._ApplyManifestPatches(manifest_pool)
extra_params.extend(['--manifest-repo-url', manifest_dir])
cmd += extra_params
result_obj = cros_build_lib.run(
cmd, cwd=self.tempdir, kill_timeout=30, error_code_ok=True)
self.returncode = result_obj.returncode
def _Transaction(self, commits):
"""ContextManager used to rollback changes to a build root if necessary.
Specifically, if an unhandled non system exception occurs, this context
manager will roll back all relevant modifications to the git repos
involved.
Args:
commits: A sequence of cros_patch.GitRepoPatch instances that compromise
this transaction- this is used to identify exactly what may be changed,
thus what needs to be tracked and rolled back if the transaction fails.
"""
# First, the book keeping code; gather required data so we know what
# to rollback to should this transaction fail. Specifically, we track
# what was checked out for each involved repo, and if it was a branch,
# the sha1 of the branch; that information is enough to rewind us back
# to the original repo state.
project_state = set(
map(functools.partial(self.GetGitRepoForChange, strict=True),
commits))
resets = []
for project_dir in project_state:
current_sha1 = git.RunGit(
project_dir, ['rev-list', '-n1', 'HEAD']).output.strip()
resets.append((project_dir, current_sha1))
assert current_sha1
committed_cache = self._committed_cache.copy()
try:
yield
except Exception:
logging.info("Rewinding transaction: failed changes: %s .",
', '.join(map(str, commits)),
exc_info=True)
for project_dir, sha1 in resets:
git.RunGit(project_dir, ['reset', '--hard', sha1])
self._committed_cache = committed_cache
raise
def testGitRepoHasChanges(self):
"""Tests that GitRepoHasChanges works correctly."""
git.RunGit(self.tempdir,
['clone', '--depth=1', constants.CHROMITE_DIR, self.tempdir])
# No changes yet as we just cloned the repo.
self.assertFalse(portage_util.EBuild.GitRepoHasChanges(self.tempdir))
# Update metadata but no real changes.
osutils.Touch(os.path.join(self.tempdir, 'LICENSE'))
self.assertFalse(portage_util.EBuild.GitRepoHasChanges(self.tempdir))
# A real change.
osutils.WriteFile(os.path.join(self.tempdir, 'LICENSE'), 'hi')
self.assertTrue(portage_util.EBuild.GitRepoHasChanges(self.tempdir))
def _Reset(checkout):
path = checkout.GetPath()
# There is no need to reset the branch if it doesn't exist.
if not git.DoesCommitExistInRepo(path, branch):
return
if fetch:
git.RunGit(path, ['fetch', '--all'])
def _LogBranch():
branches = git.RunGit(path,
['branch', '-vv']).output.splitlines()
branch_line = [b for b in branches if branch in b]
logging.info(branch_line)
_LogBranch()
git.RunGit(path, ['checkout', '-f', branch])
logging.info('Resetting to %s', checkout['tracking_branch'])
git.RunGit(path, ['reset', checkout['tracking_branch'], '--hard'])
_LogBranch()
def setUp(self):
self.manifest_dir = os.path.join(self.tempdir, '.repo', 'manifests')
# Initialize a repo instance here.
local_repo = os.path.join(constants.SOURCE_ROOT, '.repo/repo/.git')
# TODO(evanhernandez): This is a hack. Find a way to simplify this test.
# We used to use the current checkout's manifests.git, but that caused
# problems in production environemnts.
remote_manifests = os.path.join(self.tempdir, 'remote', 'manifests.git')
osutils.SafeMakedirs(remote_manifests)
git.Init(remote_manifests)
default_manifest = os.path.join(remote_manifests, 'default.xml')
osutils.WriteFile(
default_manifest,
'<?xml version="1.0" encoding="UTF-8"?><manifest></manifest>')
git.AddPath(default_manifest)
git.Commit(remote_manifests, 'dummy commit', allow_empty=True)
git.CreateBranch(remote_manifests, 'default')
git.CreateBranch(remote_manifests, 'release-R23-2913.B')
git.CreateBranch(remote_manifests, 'release-R23-2913.B-suffix')
git.CreateBranch(remote_manifests, 'firmware-link-')
# Create a copy of our existing manifests.git, but rewrite it so it
# looks like a remote manifests.git. This is to avoid hitting the
# network, and speeds things up in general.
local_manifests = 'file://%s' % remote_manifests
temp_manifests = os.path.join(self.tempdir, 'manifests.git')
git.RunGit(self.tempdir, ['clone', '-n', '--bare', local_manifests])
git.RunGit(temp_manifests,
['fetch', '-f', '-u', local_manifests,
'refs/remotes/origin/*:refs/heads/*'])
git.RunGit(temp_manifests, ['branch', '-D', 'default'])
repo = repository.RepoRepository(
temp_manifests, self.tempdir,
repo_url='file://%s' % local_repo, repo_branch='default')
repo.Initialize()
self.active_manifest = os.path.realpath(
os.path.join(self.tempdir, '.repo', 'manifest.xml'))
def Upload(self):
"""Uploads the change to gerrit."""
logging.info('Uploading commit.')
try:
# Run 'git cl upload' with --bypass-hooks to skip running scripts that are
# not part of the shallow checkout, -f to skip editing the CL message,
upload_args = self._git_committer_args + [
'cl', 'upload', '-v', '-m', self._commit_msg, '--bypass-hooks',
'-f', '--tbrs', constants.CHROME_GARDENER_REVIEW_EMAIL
]
# Marks CL as ready.
upload_args += ['--send-mail']
if self._dryrun:
upload_args += ['--dry-run']
git.RunGit(self._checkout_dir,
upload_args,
print_cmd=True,
redirect_stderr=True,
capture_output=False)
# Flip the CQ commit bit.
submit_args = ['cl', 'set-commit', '-v']
if self._dryrun:
submit_args += ['--dry-run']
git.RunGit(self._checkout_dir,
submit_args,
print_cmd=True,
redirect_stderr=True,
capture_output=False)
except cros_build_lib.RunCommandError as e:
# Log the change for debugging.
git.RunGit(self._checkout_dir,
['--no-pager', 'log', '--pretty=full'],
capture_output=False)
raise CommitError('Could not submit: %r' % e)
logging.info('Submitted to CQ.')
def Git(self, command):
"""Wrapper of git.RunGit.
It passes self.repo_dir as git_repo to git.RunGit. Also, it
sets error_code_ok=True so when git encounters an error, instead of
raising RunCommandError, it returns CommandResult with non-zero return code.
Args:
command: git command.
Returns:
A CommandResult object.
"""
return git.RunGit(self.repo_dir, command, error_code_ok=True)
def testGetDiffStatus(self):
git1, _, patch1 = self._CommonGitSetup()
# Ensure that it can work on the first commit, even if it
# doesn't report anything (no delta; it's the first files).
patch1 = self._MkPatch(git1, self._GetSha1(git1, self.DEFAULT_TRACKING))
self.assertEqual({}, patch1.GetDiffStatus(git1))
patch2 = self.CommitFile(git1, 'monkeys', 'blah')
self.assertEqual({'monkeys': 'M'}, patch2.GetDiffStatus(git1))
git.RunGit(git1, ['mv', 'monkeys', 'monkeys2'])
patch3 = self._MkPatch(git1, self._MakeCommit(git1, commit='mv'))
self.assertEqual({'monkeys': 'D', 'monkeys2': 'A'},
patch3.GetDiffStatus(git1))
patch4 = self.CommitFile(git1, 'monkey2', 'blah')
self.assertEqual({'monkey2': 'A'}, patch4.GetDiffStatus(git1))
def testBranchSpecifiedNoChanges(self):
"""Test when no changes on the branch specified by user."""
output_obj = self.mox.CreateMock(cros_build_lib.CommandResult)
output_obj.output = ''
self.manifest.GetProjectPath('my/project', True).AndReturn('mydir')
self.manifest.GetProjectsLocalRevision('my/project').AndReturn(
'm/master')
self.manifest.GetAttributeForProject('my/project',
'remote').AndReturn('cros')
git.RunGit('mydir', mox.In('m/master..mybranch')).AndReturn(output_obj)
self.mox.ReplayAll()
self.assertRaises(SystemExit, cros_patch.PrepareLocalPatches,
self.manifest, self.patches)
def testPushChange(self):
git_log = 'Marking test_one as stable\nMarking test_two as stable\n'
fake_description = 'Marking set of ebuilds as stable\n\n%s' % git_log
self.mox.StubOutWithMock(cros_mark_as_stable, '_DoWeHaveLocalCommits')
self.mox.StubOutWithMock(cros_mark_as_stable.GitBranch, 'CreateBranch')
self.mox.StubOutWithMock(cros_mark_as_stable.GitBranch, 'Exists')
self.mox.StubOutWithMock(git, 'PushWithRetry')
self.mox.StubOutWithMock(git, 'GetTrackingBranch')
self.mox.StubOutWithMock(git, 'SyncPushBranch')
self.mox.StubOutWithMock(git, 'CreatePushBranch')
self.mox.StubOutWithMock(git, 'RunGit')
cros_mark_as_stable._DoWeHaveLocalCommits(self._branch,
self._target_manifest_branch,
'.').AndReturn(True)
git.GetTrackingBranch('.', for_push=True).AndReturn(
['gerrit', 'refs/remotes/gerrit/master'])
git.SyncPushBranch('.', 'gerrit', 'refs/remotes/gerrit/master')
cros_mark_as_stable._DoWeHaveLocalCommits(
self._branch, 'refs/remotes/gerrit/master', '.').AndReturn(True)
result = cros_build_lib.CommandResult(output=git_log)
cros_build_lib.RunCommandCaptureOutput([
'git', 'log', '--format=format:%s%n%n%b',
'refs/remotes/gerrit/master..%s' % self._branch
],
cwd='.').AndReturn(result)
git.CreatePushBranch('merge_branch', '.')
git.RunGit('.', ['merge', '--squash', self._branch])
git.RunGit('.', ['commit', '-m', fake_description])
git.RunGit('.', ['config', 'push.default', 'tracking'])
git.PushWithRetry('merge_branch', '.', dryrun=False)
self.mox.ReplayAll()
cros_mark_as_stable.PushChange(self._branch,
self._target_manifest_branch, False,
'.')
self.mox.VerifyAll()
def GetActiveProjects():
"""Return the list of active projects."""
# Look at all the paths (files & dirs) in the top of the git repo. This way
# we ignore local directories devs created that aren't actually committed.
cmd = ['ls-tree', '--name-only', '-z', 'HEAD']
result = git.RunGit(TOP_DIR, cmd)
# Split the output on NULs to avoid whitespace/etc... issues.
paths = result.stdout.split('\0')
# ls-tree -z will include a trailing NUL on all entries, not just seperation,
# so filter it out if found (in case ls-tree behavior changes on us).
for path in [Path(x) for x in paths if x]:
if (TOP_DIR / path).is_dir():
yield path
def _CreateConfigPatch(self):
"""Create and return a diff patch file for config changes."""
config_change_patch = os.path.join(self.chromite_dir,
'config_change.patch')
try:
os.remove(config_change_patch)
except OSError as e:
if e.errno != errno.ENOENT:
raise
result = git.RunGit(self.chromite_dir, ['diff'] + self.config_paths,
print_cmd=True)
with open(config_change_patch, 'w') as f:
f.write(result.output)
return config_change_patch
def _ContainsConfigUpdates(self):
"""Check if updates exist and requires a push.
Returns:
True if updates exist; otherwise False.
"""
modifications = git.RunGit(self.chromite_dir,
['status', '--porcelain', '--'] +
self.config_paths,
capture_output=True,
print_cmd=True).output
if modifications:
logging.info('Changed files: %s ', modifications)
return True
else:
return False
def _SetupWorkDirectoryForPatch(work_dir, patch, branch, manifest, email):
"""Set up local dir for uploading changes to the given patch's project."""
logging.notice('Setting up dir %s for uploading changes to %s', work_dir,
patch.project_url)
# Clone the git repo from reference if we have a pointer to a
# ManifestCheckout object.
reference = None
if manifest:
# Get the path to the first checkout associated with this change. Since
# all of the checkouts share git objects, it doesn't matter which checkout
# we pick.
path = manifest.FindCheckouts(patch.project,
only_patchable=True)[0]['path']
reference = os.path.join(constants.SOURCE_ROOT, path)
if not os.path.isdir(reference):
logging.error('Unable to locate git checkout: %s', reference)
logging.error('Did you mean to use --nomirror?')
# This will do an "raise OSError" with the right values.
os.open(reference, os.O_DIRECTORY)
# Use the email if email wasn't specified.
if not email:
email = git.GetProjectUserEmail(reference)
repository.CloneGitRepo(work_dir, patch.project_url, reference=reference)
# Set the git committer.
git.RunGit(work_dir, ['config', '--replace-all', 'user.email', email])
mbranch = git.MatchSingleBranchName(work_dir,
branch,
namespace='refs/remotes/origin/')
if branch != mbranch:
logging.notice('Auto resolved branch name "%s" to "%s"', branch,
mbranch)
branch = mbranch
# Finally, create a local branch for uploading changes to the given remote
# branch.
git.CreatePushBranch(constants.PATCH_BRANCH,
work_dir,
sync=False,
remote_push_branch=git.RemoteRef(
'ignore', 'origin/%s' % branch))
return branch
def __init__(self, ebuild_dir, before=None):
"""Construct a Chrome uprev object
Args:
ebuild_dir: Path to the directory with the chrome ebuild in it.
before: CL to work backwards from.
"""
# Format includes the hash, commit body including subject, and author date.
cmd = [
'log', '-n', '1', '--author', 'chrome-bot', '--grep',
cros_mark_as_stable.GIT_COMMIT_SUBJECT,
'--format=format:%H%n%aD%n%B'
]
if before:
cmd.append(str(before) + '~')
cmd.append('.')
log = git.RunGit(ebuild_dir, cmd).output
if not log.strip():
raise UprevNotFound('No uprev CL was found')
self.sha, _, log = log.partition('\n')
self.date, _, message = log.partition('\n')
self.conf_files = [m.group('conf') for m in CONF_RE.finditer(message)]
entries = git.RawDiff(ebuild_dir, '%s^!' % self.sha)
for entry in entries:
if entry.status != 'R':
continue
from_path = entry.src_file
to_path = entry.dst_file
if (os.path.splitext(from_path)[1] != '.ebuild'
or os.path.splitext(to_path)[1] != '.ebuild'):
continue
self.from_parts = SplitPVPath(from_path)
self.to_parts = SplitPVPath(to_path)
if (self.from_parts.package != 'chromeos-chrome'
or self.to_parts.package != 'chromeos-chrome'):
continue
break
else:
raise Exception('Failed to find chromeos-chrome uprev in CL %s' %
self.sha)
def WalkReferences(repo_root, max_depth=5, suppress=()):
"""Given a repo checkout root, find the repo's it references up to max_depth.
Args:
repo_root: The root of a repo checkout to start from
max_depth: Git internally limits the max alternates depth to 5;
this option exists to adjust how deep we're willing to look.
suppress: List of repos already seen (and so to ignore).
Returns:
List of repository roots required for this repo_root.
"""
original_root = repo_root
seen = set(os.path.abspath(x) for x in suppress)
for _x in xrange(0, max_depth):
repo_root = os.path.abspath(repo_root)
if repo_root in seen:
# Cyclic reference graph; break out of it, if someone induced this the
# necessary objects should be in place. If they aren't, really isn't
# much that can be done.
return
yield repo_root
seen.add(repo_root)
base = os.path.join(repo_root, '.repo', 'manifests.git')
result = git.RunGit(base, ['config', 'repo.reference'],
error_code_ok=True)
if result.returncode not in (0, 1):
raise Failed('Unexpected returncode %i from examining %s git '
'repo.reference configuration' %
(result.returncode, base))
repo_root = result.output.strip()
if not repo_root:
break
else:
raise Failed(
'While tracing out the references of %s, we recursed more '
'than the allowed %i times ending at %s' %
(original_root, max_depth, repo_root))
def testDeleteEbuildTwice(self):
"""Test that double-deletes of ebuilds are flagged as conflicts."""
# Create monkeys.ebuild for testing.
git1 = self._MakeRepo('git1', self.source)
patch1 = self.CommitFile(git1, 'monkeys.ebuild', 'rule')
git.RunGit(git1, ['rm', 'monkeys.ebuild'])
patch2 = self._MkPatch(git1, self._MakeCommit(git1, commit='rm'))
# Delete an ebuild that does not exist in TOT.
check_attrs = {'inflight': False, 'files': ('monkeys.ebuild',)}
self.assertRaises2(cros_patch.EbuildConflict, patch2.Apply, git1,
self.DEFAULT_TRACKING, check_attrs=check_attrs)
# Delete an ebuild that exists in TOT, but does not exist in the current
# patch series.
check_attrs['inflight'] = True
self.assertRaises2(cros_patch.EbuildConflict, patch2.Apply, git1,
patch1.sha1, check_attrs=check_attrs)
def _FetchChangesForRepo(fetched_changes, by_repo, repo):
"""Fetch the changes for a given `repo`.
Args:
fetched_changes: A dict from change ids to changes which is updated by
this method.
by_repo: A mapping from repositories to changes.
repo: The repository we should fetch the changes for.
"""
changes = by_repo[repo]
refs = set(c.ref for c in changes if not c.HasBeenFetched(repo))
cmd = ['fetch', '-f', changes[0].project_url] + list(refs)
git.RunGit(repo, cmd, print_cmd=True)
for change in changes:
sha1 = change.HasBeenFetched(repo) or change.sha1
change.UpdateMetadataFromRepo(repo, sha1=sha1)
fetched_changes[change.id] = change
def _EnsureMirroring(self, post_sync=False):
"""Ensure git is usable from w/in the chroot if --references is enabled
repo init --references hardcodes the abspath to parent; this pathway
however isn't usable from the chroot (it doesn't exist). As such the
pathway is rewritten to use relative pathways pointing at the root of
the repo, which via I84988630 enter_chroot sets up a helper bind mount
allowing git/repo to access the actual referenced repo.
This has to be invoked prior to a repo sync of the target trybot to
fix any pathways that may have been broken by the parent repo moving
on disk, and needs to be invoked after the sync has completed to rewrite
any new project's abspath to relative.
"""
if not self._referenced_repo:
return
proj_root = os.path.join(self.directory, '.repo', 'project-objects')
if not os.path.exists(proj_root):
# Not yet synced, nothing to be done.
return
rewrite_git_alternates.RebuildRepoCheckout(self.directory,
self._referenced_repo)
if post_sync:
chroot_path = os.path.join(self._referenced_repo, '.repo',
'chroot', 'external')
chroot_path = path_util.ToChrootPath(chroot_path)
rewrite_git_alternates.RebuildRepoCheckout(self.directory,
self._referenced_repo,
chroot_path)
# Finally, force the git config marker that enter_chroot looks for
# to know when to do bind mounting trickery; this normally will exist,
# but if we're converting a pre-existing repo checkout, it's possible
# that it was invoked w/out the reference arg. Note this must be
# an absolute path to the source repo- enter_chroot uses that to know
# what to bind mount into the chroot.
cmd = [
'config', '--file', self._ManifestConfig, 'repo.reference',
self._referenced_repo
]
git.RunGit('.', cmd)
