def testGetGitRepoRevision(self): git.GetGitRepoRevision(self.fake_git_dir) self.assertCommandContains(['rev-parse', 'HEAD']) git.GetGitRepoRevision(self.fake_git_dir, branch='branch') self.assertCommandContains(['rev-parse', 'branch']) git.GetGitRepoRevision(self.fake_git_dir, short=True) self.assertCommandContains(['rev-parse', '--short', 'HEAD']) git.GetGitRepoRevision(self.fake_git_dir, branch='branch', short=True) self.assertCommandContains(['rev-parse', '--short', 'branch'])
def _UpdateManifest(self, manifest_path): """Rewrite |manifest_path| to point at the right branch. Args: manifest_path: The path to the manifest file. """ src_manifest = git.ManifestCheckout.Cached(self._build_root, manifest_path=manifest_path) doc = ElementTree.parse(manifest_path) root = doc.getroot() # Use the local branch ref. new_branch_name = self.rename_to if self.rename_to else self.branch_name new_branch_name = git.NormalizeRef(new_branch_name) logging.info('Updating manifest for %s', new_branch_name) default_nodes = root.findall('default') for node in default_nodes: node.attrib['revision'] = new_branch_name for node in root.findall('project'): path = node.attrib['path'] checkout = src_manifest.FindCheckoutFromPath(path) if checkout.IsBranchableProject(): # Point at the new branch. node.attrib.pop('revision', None) node.attrib.pop('upstream', None) suffix = self._GetBranchSuffix(src_manifest, checkout) if suffix: node.attrib['revision'] = '%s%s' % (new_branch_name, suffix) logging.info('Pointing project %s at: %s', node.attrib['name'], node.attrib['revision']) elif not default_nodes: # If there isn't a default node we have to add the revision directly. node.attrib['revision'] = new_branch_name else: if checkout.IsPinnableProject(): git_repo = checkout.GetPath(absolute=True) repo_head = git.GetGitRepoRevision(git_repo) node.attrib['revision'] = repo_head logging.info('Pinning project %s at: %s', node.attrib['name'], node.attrib['revision']) else: logging.info('Updating project %s', node.attrib['name']) # We can't branch this repository. Leave it alone. node.attrib['revision'] = checkout['revision'] logging.info('Project %s UNPINNED using: %s', node.attrib['name'], node.attrib['revision']) # Can not use the default version of get() here since # 'upstream' can be a valid key with a None value. upstream = checkout.get('upstream') if upstream is not None: node.attrib['upstream'] = upstream doc.write(manifest_path) return [node.attrib['name'] for node in root.findall('include')]
def _VerifyReposAndSymlink(self): # Verify symlink was generated test_file = 'chromium/src/third_party/cros/HELLO' self.assertTrue(os.path.exists(os.path.join(self.repo_root, test_file))) test_file = 'chromium/src/third_party/cros_system_api/WORLD' self.assertTrue(os.path.exists(os.path.join(self.repo_root, test_file))) # Verify repo's were reset properly repos = { 'repo1': '8d35063e1836c79c9ef97bf81eb43f450dc111ac', 'repo2': 'b2f03c74b48866eb3da5c4cab554c792a70aeda8', 'repo3': '8d35063e1836c79c9ef97bf81eb43f450dc111ac', 'repo4': 'b2f03c74b48866eb3da5c4cab554c792a70aeda8', 'third_party/cros': '8d35063e1836c79c9ef97bf81eb43f450dc111ac', 'third_party/cros_system_api': 'b2f03c74b48866eb3da5c4cab554c792a70aeda8' } for repo, revision in repos.iteritems(): repo_path = os.path.join(self.repo_root, 'chromium', 'src', repo) self.assertTrue(git.GetGitRepoRevision(repo_path) == revision)
def testAutoSyncProject(self): """Test pulling down of projects not in manifest.""" chrome_set_ver.main(['-d', os.path.join(self.test_base, 'test_6/DEPS.git')]) repo_path = os.path.join(self.repo_root, 'chromium/src/does_not_exist') self.assertTrue(git.GetGitRepoRevision(repo_path) == '8d35063e1836c79c9ef97bf81eb43f450dc111ac')
def CreateHttpConn(host, path, reqtype='GET', headers=None, body=None): """Opens an https connection to a gerrit service, and sends a request.""" path = '/a/' + path.lstrip('/') headers = headers or {} if _InAppengine(): # TODO(phobbs) how can we choose to only run this on GCE / AppEngine? credentials = _GetAppCredentials() try: headers.setdefault( 'Authorization', 'Bearer %s' % credentials.get_access_token().access_token) except gce.HttpAccessTokenRefreshError as e: logging.debug('Failed to retreive gce access token: %s', e) # Not in an Appengine or GCE environment. except httplib2.ServerNotFoundError as e: pass if 'Cookie' not in headers: cookies = GetCookies(host, path) headers['Cookie'] = '; '.join('%s=%s' % (n, v) for n, v in cookies.items()) elif 'Authorization' not in headers: logging.debug('No gitcookies file or Appengine credentials found.') if 'User-Agent' not in headers: # We may not be in a git repository. try: version = git.GetGitRepoRevision( os.path.dirname(os.path.realpath(__file__))) except cros_build_lib.RunCommandError: version = 'unknown' headers['User-Agent'] = ' '.join(( 'chromite.lib.gob_util', os.path.basename(sys.argv[0]), version, )) if body: body = json.JSONEncoder().encode(body) headers.setdefault('Content-Type', 'application/json') if logging.getLogger().isEnabledFor(logging.DEBUG): logging.debug('%s https://%s%s', reqtype, host, path) for key, val in headers.items(): if key.lower() in ('authorization', 'cookie'): val = 'HIDDEN' logging.debug('%s: %s', key, val) if body: logging.debug(body) conn = httplib.HTTPSConnection(host) conn.req_host = host conn.req_params = { 'url': path, 'method': reqtype, 'headers': headers, 'body': body, } conn.request(**conn.req_params) return conn
def _GetNewestFile(self, dirname, basehash): newhash = git.GetGitRepoRevision(dirname) self.assertNotEqual(basehash, newhash) cmd = ['log', '--format=%H', '%s..' % basehash] # Make sure we have a single commit. self._RunGitSingleOutput(dirname, cmd) cmd = ['diff', '--name-only', 'HEAD^'] # Make sure only one file per commit. return self._RunGitSingleOutput(dirname, cmd)
def _PrepareProject(self): """Make sure the project is synced properly and is ready for pinning.""" handler = git.ManifestCheckout.Cached(self.repo_root) path_to_project_dict = dict( ([attrs['path'], project]) for project, attrs in handler.projects.iteritems()) # TODO(rcui): Handle case where a dependency never makes it to the manifest # (i.e., dep path added as double checkout, and then gets deleted). We need # to delete those. crosbug/22123. if not git.IsGitRepo(self.abs_path): if self.manifest_rel_path in path_to_project_dict: raise ProjectException( '%s in full layout manifest but not in working ' "tree. Please run 'repo sync %s'" % (self.manifest_rel_path, path_to_project_dict[self.manifest_rel_path])) else: cros_build_lib.Warning( 'Project %s is not in the manifest. Automatically checking out ' 'to %s.\n' % (self.project_url, self.abs_path)) repository.CloneGitRepo(self.abs_path, self.project_url) cros_build_lib.RunCommand( ['git', 'checkout', git.GetGitRepoRevision(self.abs_path)], cwd=self.abs_path) elif not _IsGitStoreInRepo(self.abs_path): if self.manifest_rel_path in path_to_project_dict: # If path is now in the manifest, tell user to manually delete our # managed checkout and re-sync. raise ProjectException( '%s needs to be replaced. Please remove the ' "directory and run 'repo sync %s'" % (self.manifest_rel_path, path_to_project_dict[self.manifest_rel_path])) else: # If not managed by Repo we need to perform sync. cros_build_lib.RunCommand( ['git', 'pull', '--rebase', self.project_url], cwd=self.abs_path) elif not os.path.islink(self.abs_path): # Skip symlinks - we don't want to error out for the cros.DEPS projects. if self.manifest_rel_path not in path_to_project_dict: # If it is 'managed by repo' but not in the manifest, repo tried # deleting it but failed because of local changes. raise ProjectException( '%s is no longer in the manifest but has local ' 'changes. Please remove and try again.' % self.manifest_rel_path) elif self.project_name != path_to_project_dict[ self.manifest_rel_path]: cros_build_lib.Die( '.DEPS.git for %s conflicts with manifest.xml! Running with ' 'older .DEPS.git files are not yet supported. ' "Please run'repo sync --jobs=<jobs>' to sync everything up." % self.manifest_rel_path)
def GitRevision(self, project): """Return the project's current git revision on disk. Args: project: The repo_manifest.Project in question. Returns: Git revision as a string. """ return git.GetGitRepoRevision(self.AbsoluteProjectPath(project))
def _SubmitJob(self, checkout_dir, job, version=None): """Returns the path to the tryjob description.""" self.assertTrue(isinstance(job, RemoteTryJobMock)) basehash = git.GetGitRepoRevision(job.repo_url) if version is not None: self._SetMirrorVersion(version) job.Submit(workdir=checkout_dir, dryrun=True) # Get the file that was just created. created_file = self._GetNewestFile(checkout_dir, basehash) return os.path.join(checkout_dir, created_file)
def testParseInternalDEPSFile(self): """Test that the internal DEPS file is found and parsed properly.""" shutil.copyfile( os.path.join(self.test_base, 'test_internal/DEPS.git'), os.path.join(self.repo_root, 'chromium/src-internal/.DEPS.git')) chrome_set_ver.main( ['-d', os.path.join(self.test_base, 'test_1/DEPS.git')]) repos = {'repo_internal': '8d35063e1836c79c9ef97bf81eb43f450dc111ac'} for repo, revision in repos.iteritems(): repo_path = os.path.join(self.repo_root, 'chromium', 'src', repo) self.assertTrue(git.GetGitRepoRevision(repo_path) == revision)
def testCreateManifestRepo(self): """Test we can create a local git repository with a local manifest.""" CONTENTS = 'manifest contents' src_manifest = os.path.join(self.tempdir, 'src_manifest') git_repo = os.path.join(self.tempdir, 'git_repo') dst_manifest = os.path.join(git_repo, 'default.xml') osutils.WriteFile(src_manifest, CONTENTS) repository.PrepManifestForRepo(git_repo, src_manifest) self.assertEqual(CONTENTS, osutils.ReadFile(dst_manifest)) # This should fail if we don't have a valid Git repo. Not a perfect test. git.GetGitRepoRevision(git_repo)
def CreateHttpConn(host, path, reqtype='GET', headers=None, body=None): """Opens an https connection to a gerrit service, and sends a request.""" headers = headers or {} bare_host = host.partition(':')[0] auth = NETRC.authenticators(bare_host) if auth: headers.setdefault( 'Authorization', 'Basic %s' % (base64.b64encode('%s:%s' % (auth[0], auth[2])))) else: logging.debug('No netrc file found') if 'Cookie' not in headers: cookies = GetCookies(host, '/a/%s' % path) headers['Cookie'] = '; '.join('%s=%s' % (n, v) for n, v in cookies.items()) if 'User-Agent' not in headers: headers['User-Agent'] = ' '.join(( 'chromite.lib.gob_util', os.path.basename(sys.argv[0]), git.GetGitRepoRevision(os.path.dirname( os.path.realpath(__file__))), )) if body: body = json.JSONEncoder().encode(body) headers.setdefault('Content-Type', 'application/json') if logging.getLogger().isEnabledFor(logging.DEBUG): logging.debug('%s https://%s/a/%s', reqtype, host, path) for key, val in headers.iteritems(): if key.lower() in ('authorization', 'cookie'): val = 'HIDDEN' logging.debug('%s: %s', key, val) if body: logging.debug(body) conn = httplib.HTTPSConnection(host) conn.req_host = host conn.req_params = { 'url': '/a/%s' % path, 'method': reqtype, 'headers': headers, 'body': body, } conn.request(**conn.req_params) return conn
def Pin(self, commit_hash): """Attempt to pin the project to the specified commit hash. Arguments: commit_hash: The commit to pin the project to. Raises: ProjectException when an error occurs. """ self._PrepareProject() if git.GetCurrentBranch(self.abs_path): cros_build_lib.Warning( "Not pinning project %s that's checked out to a " 'development branch.' % self.rel_path) elif (commit_hash and (commit_hash != git.GetGitRepoRevision(self.abs_path))): print 'Pinning project %s' % self.rel_path self._ResetProject(commit_hash) else: cros_build_lib.Debug('Skipping project %s, already pinned' % self.rel_path)
def main(argv): parser = GetParser() options = parser.parse_args(argv) options.Freeze() if options.command == 'commit': if not options.packages and not options.all: parser.error('Please specify at least one package (--packages)') if options.force and options.all: parser.error('Cannot use --force with --all. You must specify a list of ' 'packages you want to force uprev.') if not os.path.isdir(options.srcroot): parser.error('srcroot is not a valid path: %s' % options.srcroot) portage_util.EBuild.VERBOSE = options.verbose package_list = None if options.packages: package_list = options.packages.split(':') if options.overlays: overlays = {} for path in options.overlays.split(':'): if not os.path.isdir(path): cros_build_lib.Die('Cannot find overlay: %s' % path) overlays[path] = [] else: logging.warning('Missing --overlays argument') overlays = { '%s/private-overlays/chromeos-overlay' % options.srcroot: [], '%s/third_party/chromiumos-overlay' % options.srcroot: [], } manifest = git.ManifestCheckout.Cached(options.srcroot) if options.command == 'commit': portage_util.BuildEBuildDictionary(overlays, options.all, package_list, allow_blacklisted=options.force) # Contains the array of packages we actually revved. revved_packages = [] new_package_atoms = [] for overlay in overlays: ebuilds = overlays[overlay] if not os.path.isdir(overlay): logging.warning('Skipping %s' % overlay) continue # Note we intentionally work from the non push tracking branch; # everything built thus far has been against it (meaning, http mirrors), # thus we should honor that. During the actual push, the code switches # to the correct urls, and does an appropriate rebasing. tracking_branch = git.GetTrackingBranchViaManifest( overlay, manifest=manifest).ref if options.command == 'push': PushChange(constants.STABLE_EBUILD_BRANCH, tracking_branch, options.dryrun, cwd=overlay, staging_branch=options.staging_branch) elif options.command == 'commit': existing_commit = git.GetGitRepoRevision(overlay) work_branch = GitBranch(constants.STABLE_EBUILD_BRANCH, tracking_branch, cwd=overlay) work_branch.CreateBranch() if not work_branch.Exists(): cros_build_lib.Die('Unable to create stabilizing branch in %s' % overlay) # In the case of uprevving overlays that have patches applied to them, # include the patched changes in the stabilizing branch. git.RunGit(overlay, ['rebase', existing_commit]) messages = [] for ebuild in ebuilds: if options.verbose: logging.info('Working on %s, info %s', ebuild.package, ebuild.cros_workon_vars) try: new_package = ebuild.RevWorkOnEBuild(options.srcroot, manifest) if new_package: revved_packages.append(ebuild.package) new_package_atoms.append('=%s' % new_package) messages.append(_GIT_COMMIT_MESSAGE % ebuild.package) except (OSError, IOError): logging.warning( 'Cannot rev %s\n' 'Note you will have to go into %s ' 'and reset the git repo yourself.' % (ebuild.package, overlay)) raise if messages: portage_util.EBuild.CommitChange('\n\n'.join(messages), overlay) if options.command == 'commit': chroot_path = os.path.join(options.srcroot, constants.DEFAULT_CHROOT_DIR) if os.path.exists(chroot_path): CleanStalePackages(options.srcroot, options.boards.split(':'), new_package_atoms) if options.drop_file: osutils.WriteFile(options.drop_file, ' '.join(revved_packages))
def _UploadChangeToBranch(work_dir, patch, branch, draft, dryrun): """Creates a new change from GerritPatch |patch| to |branch| from |work_dir|. Args: patch: Instance of GerritPatch to upload. branch: Branch to upload to. work_dir: Local directory where repository is checked out in. draft: If True, upload to refs/draft/|branch| rather than refs/for/|branch|. dryrun: Don't actually upload a change but go through all the steps up to and including git push --dry-run. Returns: A list of all the gerrit URLs found. """ upload_type = 'drafts' if draft else 'for' # Download & setup the patch if need be. patch.Fetch(work_dir) # Apply the actual change. patch.CherryPick(work_dir, inflight=True, leave_dirty=True) # Get the new sha1 after apply. new_sha1 = git.GetGitRepoRevision(work_dir) reviewers = set() # Filter out tags that are added by gerrit and chromite. filter_re = re.compile( r'((Commit|Trybot)-Ready|Commit-Queue|(Reviewed|Submitted|Tested)-by): ' ) # Rewrite the commit message all the time. Latest gerrit doesn't seem # to like it when you use the same ChangeId on different branches. msg = [] for line in patch.commit_message.splitlines(): if line.startswith('Reviewed-on: '): line = 'Previous-' + line elif filter_re.match(line): # If the tag is malformed, or the person lacks a name, # then that's just too bad -- throw it away. ele = re.split(r'[<>@]+', line) if len(ele) == 4: reviewers.add('@'.join(ele[-3:-1])) continue msg.append(line) msg += ['(cherry picked from commit %s)' % patch.sha1] git.RunGit(work_dir, ['commit', '--amend', '-F', '-'], input='\n'.join(msg).encode('utf8')) # Get the new sha1 after rewriting the commit message. new_sha1 = git.GetGitRepoRevision(work_dir) # Create and use a LocalPatch to Upload the change to Gerrit. local_patch = cros_patch.LocalPatch(work_dir, patch.project_url, constants.PATCH_BRANCH, patch.tracking_branch, patch.remote, new_sha1) for reviewers in (reviewers, ()): try: return local_patch.Upload(patch.project_url, 'refs/%s/%s' % (upload_type, branch), carbon_copy=False, dryrun=dryrun, reviewers=reviewers) except cros_build_lib.RunCommandError as e: if (e.result.returncode == 128 and re.search( r'fatal: user ".*?" not found', e.result.error)): logging.warning( 'Some reviewers were not found (%s); ' 'dropping them & retrying upload', ' '.join(reviewers)) continue raise
def _CommitOverlays(options, manifest, overlays, overlay_tracking_branch, overlay_ebuilds, revved_packages, new_package_atoms, reject_self_repo=True): """Commit uprevs for overlays in sequence. Args: options: The options object returned by the argument parser. manifest: The manifest of the given source root. overlays: A list over overlays to commit. overlay_tracking_branch: A dict mapping from each overlay to its tracking branch. overlay_ebuilds: A dict mapping overlays to their ebuilds. revved_packages: A shared list of revved packages. new_package_atoms: A shared list of new package atoms. reject_self_repo: Whether to abort if the ebuild lives in the same git repo as it is tracking for uprevs. """ for overlay in overlays: if not os.path.isdir(overlay): logging.warning('Skipping %s, which is not a directory.', overlay) continue # Note we intentionally work from the non push tracking branch; # everything built thus far has been against it (meaning, http mirrors), # thus we should honor that. During the actual push, the code switches # to the correct urls, and does an appropriate rebasing. tracking_branch = overlay_tracking_branch[overlay] existing_commit = git.GetGitRepoRevision(overlay) # Make sure we run in the top-level git directory in case we are # adding/removing an overlay in existing_commit. git_root = git.FindGitTopLevel(overlay) if git_root is None: cros_build_lib.Die('No git repo at overlay directory %s.', overlay) work_branch = GitBranch(constants.STABLE_EBUILD_BRANCH, tracking_branch, cwd=git_root) work_branch.CreateBranch() if not work_branch.Exists(): cros_build_lib.Die('Unable to create stabilizing branch in %s' % overlay) # In the case of uprevving overlays that have patches applied to them, # include the patched changes in the stabilizing branch. git.RunGit(git_root, ['rebase', existing_commit]) ebuilds = overlay_ebuilds.get(overlay, []) if ebuilds: with parallel.Manager() as manager: # Contains the array of packages we actually revved. messages = manager.list() ebuild_paths_to_add = manager.list() ebuild_paths_to_remove = manager.list() inputs = [[ overlay, ebuild, manifest, options, ebuild_paths_to_add, ebuild_paths_to_remove, messages, revved_packages, new_package_atoms, reject_self_repo ] for ebuild in ebuilds] parallel.RunTasksInProcessPool(_WorkOnEbuild, inputs) if ebuild_paths_to_add: logging.info( 'Adding new stable ebuild paths %s in overlay %s.', ebuild_paths_to_add, overlay) git.RunGit(overlay, ['add'] + list(ebuild_paths_to_add)) if ebuild_paths_to_remove: logging.info('Removing old ebuild paths %s in overlay %s.', ebuild_paths_to_remove, overlay) git.RunGit(overlay, ['rm', '-f'] + list(ebuild_paths_to_remove)) if messages: portage_util.EBuild.CommitChange('\n\n'.join(messages), overlay)