def RefreshManifestCheckout(manifest_dir, manifest_repo): """Checks out manifest-versions into the manifest directory. If a repository is already present, it will be cleansed of any local changes and restored to its pristine state, checking out the origin. """ reinitialize = True if os.path.exists(manifest_dir): result = cros_build_lib.RunCommand( ['git', 'config', 'remote.origin.url'], cwd=manifest_dir, print_cmd=False, redirect_stdout=True, error_code_ok=True) if (result.returncode == 0 and result.output.rstrip() == manifest_repo): logging.info('Updating manifest-versions checkout.') try: git.RunGit(manifest_dir, ['gc', '--auto']) git.CleanAndCheckoutUpstream(manifest_dir) except cros_build_lib.RunCommandError: logging.warning('Could not update manifest-versions checkout.') else: reinitialize = False else: logging.info('No manifest-versions checkout exists at %s', manifest_dir) if reinitialize: logging.info('Cloning fresh manifest-versions checkout.') _RemoveDirs(manifest_dir) repository.CloneGitRepo(manifest_dir, manifest_repo)
def testToTCompatibility(self): """Validate that ToT can use our pickles, and that we can use ToT's data.""" repo = os.path.join(self.tempdir, 'chromite') reference = os.path.abspath(__file__) reference = os.path.normpath(os.path.join(reference, '../../')) repository.CloneGitRepo(repo, '%s/chromiumos/chromite' % constants.GIT_HTTP_URL, reference=reference) code = """ import sys from chromite.buildbot import validation_pool_unittest if not hasattr(validation_pool_unittest, 'TestPickling'): sys.exit(0) sys.stdout.write(validation_pool_unittest.TestPickling.%s) """ # Verify ToT can take our pickle. cros_build_lib.RunCommandCaptureOutput( ['python', '-c', code % '_CheckTestData(sys.stdin.read())'], cwd=self.tempdir, print_cmd=False, input=self._GetTestData()) # Verify we can handle ToT's pickle. ret = cros_build_lib.RunCommandCaptureOutput( ['python', '-c', code % '_GetTestData()'], cwd=self.tempdir, print_cmd=False) self._CheckTestData(ret.output)
def _SetupMirrors(self): mirror = os.path.join(self.tempdir, 'tryjobs_mirror') os.mkdir(mirror) url = '%s/%s' % (constants.GIT_HTTP_URL, 'chromiumos/tryjobs') repository.CloneGitRepo(mirror, url, bare=True) self.ext_mirror = mirror mirror = os.path.join(self.tempdir, 'tryjobs_int_mirror') os.mkdir(mirror) repository.CloneGitRepo(mirror, self.ext_mirror, reference=self.ext_mirror, bare=True) self.int_mirror = mirror RemoteTryJobMock.EXT_SSH_URL = self.ext_mirror RemoteTryJobMock.INT_SSH_URL = self.int_mirror self._SetMirrorVersion(remote_try.RemoteTryJob.TRYJOB_FORMAT_VERSION, True)
def _PrepareProject(self): """Make sure the project is synced properly and is ready for pinning.""" handler = git.ManifestCheckout.Cached(self.repo_root) path_to_project_dict = dict( ([attrs['path'], project]) for project, attrs in handler.projects.iteritems()) # TODO(rcui): Handle case where a dependency never makes it to the manifest # (i.e., dep path added as double checkout, and then gets deleted). We need # to delete those. crosbug/22123. if not git.IsGitRepo(self.abs_path): if self.manifest_rel_path in path_to_project_dict: raise ProjectException( '%s in full layout manifest but not in working ' "tree. Please run 'repo sync %s'" % (self.manifest_rel_path, path_to_project_dict[self.manifest_rel_path])) else: cros_build_lib.Warning( 'Project %s is not in the manifest. Automatically checking out ' 'to %s.\n' % (self.project_url, self.abs_path)) repository.CloneGitRepo(self.abs_path, self.project_url) cros_build_lib.RunCommand( ['git', 'checkout', git.GetGitRepoRevision(self.abs_path)], cwd=self.abs_path) elif not _IsGitStoreInRepo(self.abs_path): if self.manifest_rel_path in path_to_project_dict: # If path is now in the manifest, tell user to manually delete our # managed checkout and re-sync. raise ProjectException( '%s needs to be replaced. Please remove the ' "directory and run 'repo sync %s'" % (self.manifest_rel_path, path_to_project_dict[self.manifest_rel_path])) else: # If not managed by Repo we need to perform sync. cros_build_lib.RunCommand( ['git', 'pull', '--rebase', self.project_url], cwd=self.abs_path) elif not os.path.islink(self.abs_path): # Skip symlinks - we don't want to error out for the cros.DEPS projects. if self.manifest_rel_path not in path_to_project_dict: # If it is 'managed by repo' but not in the manifest, repo tried # deleting it but failed because of local changes. raise ProjectException( '%s is no longer in the manifest but has local ' 'changes. Please remove and try again.' % self.manifest_rel_path) elif self.project_name != path_to_project_dict[ self.manifest_rel_path]: cros_build_lib.Die( '.DEPS.git for %s conflicts with manifest.xml! Running with ' 'older .DEPS.git files are not yet supported. ' "Please run'repo sync --jobs=<jobs>' to sync everything up." % self.manifest_rel_path)
def process_target(internal, manifest_url, reference): subdir = 'internal' if internal else 'external' root = os.path.join(options.testroot, subdir) repo_dir = os.path.join(root, 'repo') osutils.SafeMakedirs(repo_dir) manifest_dir = os.path.join(root, 'manifest') if os.path.exists(manifest_dir): git.CleanAndCheckoutUpstream(manifest_dir) git.RunGit(manifest_dir, ['checkout', '-B', 'master', '-t', 'origin/master']) else: repository.CloneGitRepo(manifest_dir, manifest_url) m = Manifest(repo_dir, manifest_dir, internal, reference=reference, dryrun=options.dryrun) m.PerformUpdate() if options.dryrun: print "%s manifest is now:" % subdir print osutils.ReadFile(os.path.join(manifest_dir, MANIFEST_FILE))
def _SetupWorkDirectoryForPatch(work_dir, patch, branch, manifest, email): """Set up local dir for uploading changes to the given patch's project.""" logging.info('Setting up dir %s for uploading changes to %s', work_dir, patch.project_url) # Clone the git repo from reference if we have a pointer to a # ManifestCheckout object. reference = None if manifest: reference = os.path.join(constants.SOURCE_ROOT, manifest.GetProjectPath(patch.project)) if not os.path.isdir(reference): logging.error('Unable to locate git checkout: %s', reference) logging.error('Did you mean to use --nomirror?') # This will do an "raise OSError" with the right values. os.open(reference, os.O_DIRECTORY) # Use the email if email wasn't specified. if not email: email = git.GetProjectUserEmail(reference) repository.CloneGitRepo(work_dir, patch.project_url, reference=reference) # Set the git committer. git.RunGit(work_dir, ['config', '--replace-all', 'user.email', email]) mbranch = git.MatchSingleBranchName(work_dir, branch, namespace='refs/remotes/origin/') if branch != mbranch: logging.info('Auto resolved branch name "%s" to "%s"', branch, mbranch) branch = mbranch # Finally, create a local branch for uploading changes to the given remote # branch. git.CreatePushBranch(constants.PATCH_BRANCH, work_dir, sync=False, remote_push_branch=('ignore', 'origin/%s' % branch)) return branch
def _Submit(self, testjob, dryrun): """Internal submission function. See Submit() for arg description.""" # TODO(rcui): convert to shallow clone when that's available. current_time = str(int(time.time())) ref_base = os.path.join('refs/tryjobs', self.user, current_time) for patch in self.local_patches: # Isolate the name; if it's a tag or a remote, let through. # Else if it's a branch, get the full branch name minus refs/heads. local_branch = git.StripRefsHeads(patch.ref, False) ref_final = os.path.join(ref_base, local_branch, patch.sha1) self.manifest.AssertProjectIsPushable(patch.project) data = self.manifest.projects[patch.project] print 'Uploading patch %s' % patch patch.Upload(data['push_url'], ref_final, dryrun=dryrun) # TODO(rcui): Pass in the remote instead of tag. http://crosbug.com/33937. tag = constants.EXTERNAL_PATCH_TAG if data['remote'] == constants.INTERNAL_REMOTE: tag = constants.INTERNAL_PATCH_TAG self.extra_args.append('--remote-patches=%s:%s:%s:%s:%s' % (patch.project, local_branch, ref_final, patch.tracking_branch, tag)) self._VerifyForBuildbot() repository.CloneGitRepo(self.tryjob_repo, self.ssh_url) version_path = os.path.join(self.tryjob_repo, self.TRYJOB_FORMAT_FILE) with open(version_path, 'r') as f: try: val = int(f.read().strip()) except ValueError: raise ChromiteUpgradeNeeded() if val > self.TRYJOB_FORMAT_VERSION: raise ChromiteUpgradeNeeded(val) push_branch = manifest_version.PUSH_BRANCH remote_branch = ('origin', 'refs/remotes/origin/test') if testjob else None git.CreatePushBranch(push_branch, self.tryjob_repo, sync=False, remote_push_branch=remote_branch) file_name = '%s.%s' % (self.user, current_time) user_dir = os.path.join(self.tryjob_repo, self.user) if not os.path.isdir(user_dir): os.mkdir(user_dir) fullpath = os.path.join(user_dir, file_name) with open(fullpath, 'w+') as job_desc_file: json.dump(self.values, job_desc_file) cros_build_lib.RunCommand(['git', 'add', fullpath], cwd=self.tryjob_repo) extra_env = { # The committer field makes sure the creds match what the remote # gerrit instance expects while the author field allows lookup # on the console to work. http://crosbug.com/27939 'GIT_COMMITTER_EMAIL' : self.user_email, 'GIT_AUTHOR_EMAIL' : self.user_email, } cros_build_lib.RunCommand(['git', 'commit', '-m', self.description], cwd=self.tryjob_repo, extra_env=extra_env) try: git.PushWithRetry( push_branch, self.tryjob_repo, retries=3, dryrun=dryrun) except cros_build_lib.RunCommandError: cros_build_lib.Error( 'Failed to submit tryjob. This could be due to too many ' 'submission requests by users. Please try again.') raise