def testGetGitRepoRevision(self):
git.GetGitRepoRevision(self.fake_git_dir)
self.assertCommandContains(['rev-parse', 'HEAD'])
git.GetGitRepoRevision(self.fake_git_dir, branch='branch')
self.assertCommandContains(['rev-parse', 'branch'])
git.GetGitRepoRevision(self.fake_git_dir, short=True)
self.assertCommandContains(['rev-parse', '--short', 'HEAD'])
git.GetGitRepoRevision(self.fake_git_dir, branch='branch', short=True)
self.assertCommandContains(['rev-parse', '--short', 'branch'])
def _UpdateManifest(self, manifest_path):
"""Rewrite |manifest_path| to point at the right branch.
Args:
manifest_path: The path to the manifest file.
"""
src_manifest = git.ManifestCheckout.Cached(self._build_root,
manifest_path=manifest_path)
doc = ElementTree.parse(manifest_path)
root = doc.getroot()
# Use the local branch ref.
new_branch_name = self.rename_to if self.rename_to else self.branch_name
new_branch_name = git.NormalizeRef(new_branch_name)
logging.info('Updating manifest for %s', new_branch_name)
default_nodes = root.findall('default')
for node in default_nodes:
node.attrib['revision'] = new_branch_name
for node in root.findall('project'):
path = node.attrib['path']
checkout = src_manifest.FindCheckoutFromPath(path)
if checkout.IsBranchableProject():
# Point at the new branch.
node.attrib.pop('revision', None)
node.attrib.pop('upstream', None)
suffix = self._GetBranchSuffix(src_manifest, checkout)
if suffix:
node.attrib['revision'] = '%s%s' % (new_branch_name,
suffix)
logging.info('Pointing project %s at: %s',
node.attrib['name'], node.attrib['revision'])
elif not default_nodes:
# If there isn't a default node we have to add the revision directly.
node.attrib['revision'] = new_branch_name
else:
if checkout.IsPinnableProject():
git_repo = checkout.GetPath(absolute=True)
repo_head = git.GetGitRepoRevision(git_repo)
node.attrib['revision'] = repo_head
logging.info('Pinning project %s at: %s',
node.attrib['name'], node.attrib['revision'])
else:
logging.info('Updating project %s', node.attrib['name'])
# We can't branch this repository. Leave it alone.
node.attrib['revision'] = checkout['revision']
logging.info('Project %s UNPINNED using: %s',
node.attrib['name'], node.attrib['revision'])
# Can not use the default version of get() here since
# 'upstream' can be a valid key with a None value.
upstream = checkout.get('upstream')
if upstream is not None:
node.attrib['upstream'] = upstream
doc.write(manifest_path)
return [node.attrib['name'] for node in root.findall('include')]
def _VerifyReposAndSymlink(self):
# Verify symlink was generated
test_file = 'chromium/src/third_party/cros/HELLO'
self.assertTrue(os.path.exists(os.path.join(self.repo_root,
test_file)))
test_file = 'chromium/src/third_party/cros_system_api/WORLD'
self.assertTrue(os.path.exists(os.path.join(self.repo_root,
test_file)))
# Verify repo's were reset properly
repos = {
'repo1':
'8d35063e1836c79c9ef97bf81eb43f450dc111ac',
'repo2':
'b2f03c74b48866eb3da5c4cab554c792a70aeda8',
'repo3':
'8d35063e1836c79c9ef97bf81eb43f450dc111ac',
'repo4':
'b2f03c74b48866eb3da5c4cab554c792a70aeda8',
'third_party/cros':
'8d35063e1836c79c9ef97bf81eb43f450dc111ac',
'third_party/cros_system_api':
'b2f03c74b48866eb3da5c4cab554c792a70aeda8'
}
for repo, revision in repos.iteritems():
repo_path = os.path.join(self.repo_root, 'chromium', 'src', repo)
self.assertTrue(git.GetGitRepoRevision(repo_path) == revision)
def testAutoSyncProject(self):
"""Test pulling down of projects not in manifest."""
chrome_set_ver.main(['-d',
os.path.join(self.test_base, 'test_6/DEPS.git')])
repo_path = os.path.join(self.repo_root, 'chromium/src/does_not_exist')
self.assertTrue(git.GetGitRepoRevision(repo_path) ==
'8d35063e1836c79c9ef97bf81eb43f450dc111ac')
def CreateHttpConn(host, path, reqtype='GET', headers=None, body=None):
"""Opens an https connection to a gerrit service, and sends a request."""
path = '/a/' + path.lstrip('/')
headers = headers or {}
if _InAppengine():
# TODO(phobbs) how can we choose to only run this on GCE / AppEngine?
credentials = _GetAppCredentials()
try:
headers.setdefault(
'Authorization',
'Bearer %s' % credentials.get_access_token().access_token)
except gce.HttpAccessTokenRefreshError as e:
logging.debug('Failed to retreive gce access token: %s', e)
# Not in an Appengine or GCE environment.
except httplib2.ServerNotFoundError as e:
pass
if 'Cookie' not in headers:
cookies = GetCookies(host, path)
headers['Cookie'] = '; '.join('%s=%s' % (n, v)
for n, v in cookies.items())
elif 'Authorization' not in headers:
logging.debug('No gitcookies file or Appengine credentials found.')
if 'User-Agent' not in headers:
# We may not be in a git repository.
try:
version = git.GetGitRepoRevision(
os.path.dirname(os.path.realpath(__file__)))
except cros_build_lib.RunCommandError:
version = 'unknown'
headers['User-Agent'] = ' '.join((
'chromite.lib.gob_util',
os.path.basename(sys.argv[0]),
version,
))
if body:
body = json.JSONEncoder().encode(body)
headers.setdefault('Content-Type', 'application/json')
if logging.getLogger().isEnabledFor(logging.DEBUG):
logging.debug('%s https://%s%s', reqtype, host, path)
for key, val in headers.items():
if key.lower() in ('authorization', 'cookie'):
val = 'HIDDEN'
logging.debug('%s: %s', key, val)
if body:
logging.debug(body)
conn = httplib.HTTPSConnection(host)
conn.req_host = host
conn.req_params = {
'url': path,
'method': reqtype,
'headers': headers,
'body': body,
}
conn.request(**conn.req_params)
return conn
def _GetNewestFile(self, dirname, basehash):
newhash = git.GetGitRepoRevision(dirname)
self.assertNotEqual(basehash, newhash)
cmd = ['log', '--format=%H', '%s..' % basehash]
# Make sure we have a single commit.
self._RunGitSingleOutput(dirname, cmd)
cmd = ['diff', '--name-only', 'HEAD^']
# Make sure only one file per commit.
return self._RunGitSingleOutput(dirname, cmd)
def _PrepareProject(self):
"""Make sure the project is synced properly and is ready for pinning."""
handler = git.ManifestCheckout.Cached(self.repo_root)
path_to_project_dict = dict(
([attrs['path'], project])
for project, attrs in handler.projects.iteritems())
# TODO(rcui): Handle case where a dependency never makes it to the manifest
# (i.e., dep path added as double checkout, and then gets deleted). We need
# to delete those. crosbug/22123.
if not git.IsGitRepo(self.abs_path):
if self.manifest_rel_path in path_to_project_dict:
raise ProjectException(
'%s in full layout manifest but not in working '
"tree. Please run 'repo sync %s'" %
(self.manifest_rel_path,
path_to_project_dict[self.manifest_rel_path]))
else:
cros_build_lib.Warning(
'Project %s is not in the manifest. Automatically checking out '
'to %s.\n' % (self.project_url, self.abs_path))
repository.CloneGitRepo(self.abs_path, self.project_url)
cros_build_lib.RunCommand(
['git', 'checkout',
git.GetGitRepoRevision(self.abs_path)],
cwd=self.abs_path)
elif not _IsGitStoreInRepo(self.abs_path):
if self.manifest_rel_path in path_to_project_dict:
# If path is now in the manifest, tell user to manually delete our
# managed checkout and re-sync.
raise ProjectException(
'%s needs to be replaced. Please remove the '
"directory and run 'repo sync %s'" %
(self.manifest_rel_path,
path_to_project_dict[self.manifest_rel_path]))
else:
# If not managed by Repo we need to perform sync.
cros_build_lib.RunCommand(
['git', 'pull', '--rebase', self.project_url],
cwd=self.abs_path)
elif not os.path.islink(self.abs_path):
# Skip symlinks - we don't want to error out for the cros.DEPS projects.
if self.manifest_rel_path not in path_to_project_dict:
# If it is 'managed by repo' but not in the manifest, repo tried
# deleting it but failed because of local changes.
raise ProjectException(
'%s is no longer in the manifest but has local '
'changes. Please remove and try again.' %
self.manifest_rel_path)
elif self.project_name != path_to_project_dict[
self.manifest_rel_path]:
cros_build_lib.Die(
'.DEPS.git for %s conflicts with manifest.xml! Running with '
'older .DEPS.git files are not yet supported. '
"Please run'repo sync --jobs=<jobs>' to sync everything up."
% self.manifest_rel_path)
def GitRevision(self, project):
"""Return the project's current git revision on disk.
Args:
project: The repo_manifest.Project in question.
Returns:
Git revision as a string.
"""
return git.GetGitRepoRevision(self.AbsoluteProjectPath(project))
def _SubmitJob(self, checkout_dir, job, version=None):
"""Returns the path to the tryjob description."""
self.assertTrue(isinstance(job, RemoteTryJobMock))
basehash = git.GetGitRepoRevision(job.repo_url)
if version is not None:
self._SetMirrorVersion(version)
job.Submit(workdir=checkout_dir, dryrun=True)
# Get the file that was just created.
created_file = self._GetNewestFile(checkout_dir, basehash)
return os.path.join(checkout_dir, created_file)
def testParseInternalDEPSFile(self):
"""Test that the internal DEPS file is found and parsed properly."""
shutil.copyfile(
os.path.join(self.test_base, 'test_internal/DEPS.git'),
os.path.join(self.repo_root, 'chromium/src-internal/.DEPS.git'))
chrome_set_ver.main(
['-d', os.path.join(self.test_base, 'test_1/DEPS.git')])
repos = {'repo_internal': '8d35063e1836c79c9ef97bf81eb43f450dc111ac'}
for repo, revision in repos.iteritems():
repo_path = os.path.join(self.repo_root, 'chromium', 'src', repo)
self.assertTrue(git.GetGitRepoRevision(repo_path) == revision)
def testCreateManifestRepo(self):
"""Test we can create a local git repository with a local manifest."""
CONTENTS = 'manifest contents'
src_manifest = os.path.join(self.tempdir, 'src_manifest')
git_repo = os.path.join(self.tempdir, 'git_repo')
dst_manifest = os.path.join(git_repo, 'default.xml')
osutils.WriteFile(src_manifest, CONTENTS)
repository.PrepManifestForRepo(git_repo, src_manifest)
self.assertEqual(CONTENTS, osutils.ReadFile(dst_manifest))
# This should fail if we don't have a valid Git repo. Not a perfect test.
git.GetGitRepoRevision(git_repo)
def CreateHttpConn(host, path, reqtype='GET', headers=None, body=None):
"""Opens an https connection to a gerrit service, and sends a request."""
headers = headers or {}
bare_host = host.partition(':')[0]
auth = NETRC.authenticators(bare_host)
if auth:
headers.setdefault(
'Authorization',
'Basic %s' % (base64.b64encode('%s:%s' % (auth[0], auth[2]))))
else:
logging.debug('No netrc file found')
if 'Cookie' not in headers:
cookies = GetCookies(host, '/a/%s' % path)
headers['Cookie'] = '; '.join('%s=%s' % (n, v)
for n, v in cookies.items())
if 'User-Agent' not in headers:
headers['User-Agent'] = ' '.join((
'chromite.lib.gob_util',
os.path.basename(sys.argv[0]),
git.GetGitRepoRevision(os.path.dirname(
os.path.realpath(__file__))),
))
if body:
body = json.JSONEncoder().encode(body)
headers.setdefault('Content-Type', 'application/json')
if logging.getLogger().isEnabledFor(logging.DEBUG):
logging.debug('%s https://%s/a/%s', reqtype, host, path)
for key, val in headers.iteritems():
if key.lower() in ('authorization', 'cookie'):
val = 'HIDDEN'
logging.debug('%s: %s', key, val)
if body:
logging.debug(body)
conn = httplib.HTTPSConnection(host)
conn.req_host = host
conn.req_params = {
'url': '/a/%s' % path,
'method': reqtype,
'headers': headers,
'body': body,
}
conn.request(**conn.req_params)
return conn
def Pin(self, commit_hash):
"""Attempt to pin the project to the specified commit hash.
Arguments:
commit_hash: The commit to pin the project to.
Raises:
ProjectException when an error occurs.
"""
self._PrepareProject()
if git.GetCurrentBranch(self.abs_path):
cros_build_lib.Warning(
"Not pinning project %s that's checked out to a "
'development branch.' % self.rel_path)
elif (commit_hash
and (commit_hash != git.GetGitRepoRevision(self.abs_path))):
print 'Pinning project %s' % self.rel_path
self._ResetProject(commit_hash)
else:
cros_build_lib.Debug('Skipping project %s, already pinned' %
self.rel_path)
def main(argv):
parser = GetParser()
options = parser.parse_args(argv)
options.Freeze()
if options.command == 'commit':
if not options.packages and not options.all:
parser.error('Please specify at least one package (--packages)')
if options.force and options.all:
parser.error('Cannot use --force with --all. You must specify a list of '
'packages you want to force uprev.')
if not os.path.isdir(options.srcroot):
parser.error('srcroot is not a valid path: %s' % options.srcroot)
portage_util.EBuild.VERBOSE = options.verbose
package_list = None
if options.packages:
package_list = options.packages.split(':')
if options.overlays:
overlays = {}
for path in options.overlays.split(':'):
if not os.path.isdir(path):
cros_build_lib.Die('Cannot find overlay: %s' % path)
overlays[path] = []
else:
logging.warning('Missing --overlays argument')
overlays = {
'%s/private-overlays/chromeos-overlay' % options.srcroot: [],
'%s/third_party/chromiumos-overlay' % options.srcroot: [],
}
manifest = git.ManifestCheckout.Cached(options.srcroot)
if options.command == 'commit':
portage_util.BuildEBuildDictionary(overlays, options.all, package_list,
allow_blacklisted=options.force)
# Contains the array of packages we actually revved.
revved_packages = []
new_package_atoms = []
for overlay in overlays:
ebuilds = overlays[overlay]
if not os.path.isdir(overlay):
logging.warning('Skipping %s' % overlay)
continue
# Note we intentionally work from the non push tracking branch;
# everything built thus far has been against it (meaning, http mirrors),
# thus we should honor that. During the actual push, the code switches
# to the correct urls, and does an appropriate rebasing.
tracking_branch = git.GetTrackingBranchViaManifest(
overlay, manifest=manifest).ref
if options.command == 'push':
PushChange(constants.STABLE_EBUILD_BRANCH, tracking_branch,
options.dryrun, cwd=overlay,
staging_branch=options.staging_branch)
elif options.command == 'commit':
existing_commit = git.GetGitRepoRevision(overlay)
work_branch = GitBranch(constants.STABLE_EBUILD_BRANCH, tracking_branch,
cwd=overlay)
work_branch.CreateBranch()
if not work_branch.Exists():
cros_build_lib.Die('Unable to create stabilizing branch in %s' %
overlay)
# In the case of uprevving overlays that have patches applied to them,
# include the patched changes in the stabilizing branch.
git.RunGit(overlay, ['rebase', existing_commit])
messages = []
for ebuild in ebuilds:
if options.verbose:
logging.info('Working on %s, info %s', ebuild.package,
ebuild.cros_workon_vars)
try:
new_package = ebuild.RevWorkOnEBuild(options.srcroot, manifest)
if new_package:
revved_packages.append(ebuild.package)
new_package_atoms.append('=%s' % new_package)
messages.append(_GIT_COMMIT_MESSAGE % ebuild.package)
except (OSError, IOError):
logging.warning(
'Cannot rev %s\n'
'Note you will have to go into %s '
'and reset the git repo yourself.' % (ebuild.package, overlay))
raise
if messages:
portage_util.EBuild.CommitChange('\n\n'.join(messages), overlay)
if options.command == 'commit':
chroot_path = os.path.join(options.srcroot, constants.DEFAULT_CHROOT_DIR)
if os.path.exists(chroot_path):
CleanStalePackages(options.srcroot, options.boards.split(':'),
new_package_atoms)
if options.drop_file:
osutils.WriteFile(options.drop_file, ' '.join(revved_packages))
def _UploadChangeToBranch(work_dir, patch, branch, draft, dryrun):
"""Creates a new change from GerritPatch |patch| to |branch| from |work_dir|.
Args:
patch: Instance of GerritPatch to upload.
branch: Branch to upload to.
work_dir: Local directory where repository is checked out in.
draft: If True, upload to refs/draft/|branch| rather than refs/for/|branch|.
dryrun: Don't actually upload a change but go through all the steps up to
and including git push --dry-run.
Returns:
A list of all the gerrit URLs found.
"""
upload_type = 'drafts' if draft else 'for'
# Download & setup the patch if need be.
patch.Fetch(work_dir)
# Apply the actual change.
patch.CherryPick(work_dir, inflight=True, leave_dirty=True)
# Get the new sha1 after apply.
new_sha1 = git.GetGitRepoRevision(work_dir)
reviewers = set()
# Filter out tags that are added by gerrit and chromite.
filter_re = re.compile(
r'((Commit|Trybot)-Ready|Commit-Queue|(Reviewed|Submitted|Tested)-by): '
)
# Rewrite the commit message all the time. Latest gerrit doesn't seem
# to like it when you use the same ChangeId on different branches.
msg = []
for line in patch.commit_message.splitlines():
if line.startswith('Reviewed-on: '):
line = 'Previous-' + line
elif filter_re.match(line):
# If the tag is malformed, or the person lacks a name,
# then that's just too bad -- throw it away.
ele = re.split(r'[<>@]+', line)
if len(ele) == 4:
reviewers.add('@'.join(ele[-3:-1]))
continue
msg.append(line)
msg += ['(cherry picked from commit %s)' % patch.sha1]
git.RunGit(work_dir, ['commit', '--amend', '-F', '-'],
input='\n'.join(msg).encode('utf8'))
# Get the new sha1 after rewriting the commit message.
new_sha1 = git.GetGitRepoRevision(work_dir)
# Create and use a LocalPatch to Upload the change to Gerrit.
local_patch = cros_patch.LocalPatch(work_dir, patch.project_url,
constants.PATCH_BRANCH,
patch.tracking_branch, patch.remote,
new_sha1)
for reviewers in (reviewers, ()):
try:
return local_patch.Upload(patch.project_url,
'refs/%s/%s' % (upload_type, branch),
carbon_copy=False,
dryrun=dryrun,
reviewers=reviewers)
except cros_build_lib.RunCommandError as e:
if (e.result.returncode == 128 and re.search(
r'fatal: user ".*?" not found', e.result.error)):
logging.warning(
'Some reviewers were not found (%s); '
'dropping them & retrying upload', ' '.join(reviewers))
continue
raise
def _CommitOverlays(options,
manifest,
overlays,
overlay_tracking_branch,
overlay_ebuilds,
revved_packages,
new_package_atoms,
reject_self_repo=True):
"""Commit uprevs for overlays in sequence.
Args:
options: The options object returned by the argument parser.
manifest: The manifest of the given source root.
overlays: A list over overlays to commit.
overlay_tracking_branch: A dict mapping from each overlay to its tracking
branch.
overlay_ebuilds: A dict mapping overlays to their ebuilds.
revved_packages: A shared list of revved packages.
new_package_atoms: A shared list of new package atoms.
reject_self_repo: Whether to abort if the ebuild lives in the same git
repo as it is tracking for uprevs.
"""
for overlay in overlays:
if not os.path.isdir(overlay):
logging.warning('Skipping %s, which is not a directory.', overlay)
continue
# Note we intentionally work from the non push tracking branch;
# everything built thus far has been against it (meaning, http mirrors),
# thus we should honor that. During the actual push, the code switches
# to the correct urls, and does an appropriate rebasing.
tracking_branch = overlay_tracking_branch[overlay]
existing_commit = git.GetGitRepoRevision(overlay)
# Make sure we run in the top-level git directory in case we are
# adding/removing an overlay in existing_commit.
git_root = git.FindGitTopLevel(overlay)
if git_root is None:
cros_build_lib.Die('No git repo at overlay directory %s.', overlay)
work_branch = GitBranch(constants.STABLE_EBUILD_BRANCH,
tracking_branch,
cwd=git_root)
work_branch.CreateBranch()
if not work_branch.Exists():
cros_build_lib.Die('Unable to create stabilizing branch in %s' %
overlay)
# In the case of uprevving overlays that have patches applied to them,
# include the patched changes in the stabilizing branch.
git.RunGit(git_root, ['rebase', existing_commit])
ebuilds = overlay_ebuilds.get(overlay, [])
if ebuilds:
with parallel.Manager() as manager:
# Contains the array of packages we actually revved.
messages = manager.list()
ebuild_paths_to_add = manager.list()
ebuild_paths_to_remove = manager.list()
inputs = [[
overlay, ebuild, manifest, options, ebuild_paths_to_add,
ebuild_paths_to_remove, messages, revved_packages,
new_package_atoms, reject_self_repo
] for ebuild in ebuilds]
parallel.RunTasksInProcessPool(_WorkOnEbuild, inputs)
if ebuild_paths_to_add:
logging.info(
'Adding new stable ebuild paths %s in overlay %s.',
ebuild_paths_to_add, overlay)
git.RunGit(overlay, ['add'] + list(ebuild_paths_to_add))
if ebuild_paths_to_remove:
logging.info('Removing old ebuild paths %s in overlay %s.',
ebuild_paths_to_remove, overlay)
git.RunGit(overlay,
['rm', '-f'] + list(ebuild_paths_to_remove))
if messages:
portage_util.EBuild.CommitChange('\n\n'.join(messages),
overlay)
