def RefreshManifestCheckout(manifest_dir, manifest_repo):
"""Checks out manifest-versions into the manifest directory.
If a repository is already present, it will be cleansed of any local
changes and restored to its pristine state, checking out the origin.
"""
reinitialize = True
if os.path.exists(manifest_dir):
result = cros_build_lib.RunCommand(
['git', 'config', 'remote.origin.url'],
cwd=manifest_dir,
print_cmd=False,
redirect_stdout=True,
error_code_ok=True)
if (result.returncode == 0
and result.output.rstrip() == manifest_repo):
logging.info('Updating manifest-versions checkout.')
try:
git.RunGit(manifest_dir, ['gc', '--auto'])
git.CleanAndCheckoutUpstream(manifest_dir)
except cros_build_lib.RunCommandError:
logging.warning('Could not update manifest-versions checkout.')
else:
reinitialize = False
else:
logging.info('No manifest-versions checkout exists at %s',
manifest_dir)
if reinitialize:
logging.info('Cloning fresh manifest-versions checkout.')
_RemoveDirs(manifest_dir)
repository.CloneGitRepo(manifest_dir, manifest_repo)
def testToTCompatibility(self):
"""Validate that ToT can use our pickles, and that we can use ToT's data."""
repo = os.path.join(self.tempdir, 'chromite')
reference = os.path.abspath(__file__)
reference = os.path.normpath(os.path.join(reference, '../../'))
repository.CloneGitRepo(repo,
'%s/chromiumos/chromite' %
constants.GIT_HTTP_URL,
reference=reference)
code = """
import sys
from chromite.buildbot import validation_pool_unittest
if not hasattr(validation_pool_unittest, 'TestPickling'):
sys.exit(0)
sys.stdout.write(validation_pool_unittest.TestPickling.%s)
"""
# Verify ToT can take our pickle.
cros_build_lib.RunCommandCaptureOutput(
['python', '-c', code % '_CheckTestData(sys.stdin.read())'],
cwd=self.tempdir,
print_cmd=False,
input=self._GetTestData())
# Verify we can handle ToT's pickle.
ret = cros_build_lib.RunCommandCaptureOutput(
['python', '-c', code % '_GetTestData()'],
cwd=self.tempdir,
print_cmd=False)
self._CheckTestData(ret.output)
def _SetupMirrors(self):
mirror = os.path.join(self.tempdir, 'tryjobs_mirror')
os.mkdir(mirror)
url = '%s/%s' % (constants.GIT_HTTP_URL, 'chromiumos/tryjobs')
repository.CloneGitRepo(mirror, url, bare=True)
self.ext_mirror = mirror
mirror = os.path.join(self.tempdir, 'tryjobs_int_mirror')
os.mkdir(mirror)
repository.CloneGitRepo(mirror,
self.ext_mirror,
reference=self.ext_mirror,
bare=True)
self.int_mirror = mirror
RemoteTryJobMock.EXT_SSH_URL = self.ext_mirror
RemoteTryJobMock.INT_SSH_URL = self.int_mirror
self._SetMirrorVersion(remote_try.RemoteTryJob.TRYJOB_FORMAT_VERSION,
True)
def _PrepareProject(self):
"""Make sure the project is synced properly and is ready for pinning."""
handler = git.ManifestCheckout.Cached(self.repo_root)
path_to_project_dict = dict(
([attrs['path'], project])
for project, attrs in handler.projects.iteritems())
# TODO(rcui): Handle case where a dependency never makes it to the manifest
# (i.e., dep path added as double checkout, and then gets deleted). We need
# to delete those. crosbug/22123.
if not git.IsGitRepo(self.abs_path):
if self.manifest_rel_path in path_to_project_dict:
raise ProjectException(
'%s in full layout manifest but not in working '
"tree. Please run 'repo sync %s'" %
(self.manifest_rel_path,
path_to_project_dict[self.manifest_rel_path]))
else:
cros_build_lib.Warning(
'Project %s is not in the manifest. Automatically checking out '
'to %s.\n' % (self.project_url, self.abs_path))
repository.CloneGitRepo(self.abs_path, self.project_url)
cros_build_lib.RunCommand(
['git', 'checkout',
git.GetGitRepoRevision(self.abs_path)],
cwd=self.abs_path)
elif not _IsGitStoreInRepo(self.abs_path):
if self.manifest_rel_path in path_to_project_dict:
# If path is now in the manifest, tell user to manually delete our
# managed checkout and re-sync.
raise ProjectException(
'%s needs to be replaced. Please remove the '
"directory and run 'repo sync %s'" %
(self.manifest_rel_path,
path_to_project_dict[self.manifest_rel_path]))
else:
# If not managed by Repo we need to perform sync.
cros_build_lib.RunCommand(
['git', 'pull', '--rebase', self.project_url],
cwd=self.abs_path)
elif not os.path.islink(self.abs_path):
# Skip symlinks - we don't want to error out for the cros.DEPS projects.
if self.manifest_rel_path not in path_to_project_dict:
# If it is 'managed by repo' but not in the manifest, repo tried
# deleting it but failed because of local changes.
raise ProjectException(
'%s is no longer in the manifest but has local '
'changes. Please remove and try again.' %
self.manifest_rel_path)
elif self.project_name != path_to_project_dict[
self.manifest_rel_path]:
cros_build_lib.Die(
'.DEPS.git for %s conflicts with manifest.xml! Running with '
'older .DEPS.git files are not yet supported. '
"Please run'repo sync --jobs=<jobs>' to sync everything up."
% self.manifest_rel_path)
def process_target(internal, manifest_url, reference):
subdir = 'internal' if internal else 'external'
root = os.path.join(options.testroot, subdir)
repo_dir = os.path.join(root, 'repo')
osutils.SafeMakedirs(repo_dir)
manifest_dir = os.path.join(root, 'manifest')
if os.path.exists(manifest_dir):
git.CleanAndCheckoutUpstream(manifest_dir)
git.RunGit(manifest_dir,
['checkout', '-B', 'master', '-t', 'origin/master'])
else:
repository.CloneGitRepo(manifest_dir, manifest_url)
m = Manifest(repo_dir, manifest_dir, internal,
reference=reference,
dryrun=options.dryrun)
m.PerformUpdate()
if options.dryrun:
print "%s manifest is now:" % subdir
print osutils.ReadFile(os.path.join(manifest_dir, MANIFEST_FILE))
def _SetupWorkDirectoryForPatch(work_dir, patch, branch, manifest, email):
"""Set up local dir for uploading changes to the given patch's project."""
logging.info('Setting up dir %s for uploading changes to %s', work_dir,
patch.project_url)
# Clone the git repo from reference if we have a pointer to a
# ManifestCheckout object.
reference = None
if manifest:
reference = os.path.join(constants.SOURCE_ROOT,
manifest.GetProjectPath(patch.project))
if not os.path.isdir(reference):
logging.error('Unable to locate git checkout: %s', reference)
logging.error('Did you mean to use --nomirror?')
# This will do an "raise OSError" with the right values.
os.open(reference, os.O_DIRECTORY)
# Use the email if email wasn't specified.
if not email:
email = git.GetProjectUserEmail(reference)
repository.CloneGitRepo(work_dir, patch.project_url, reference=reference)
# Set the git committer.
git.RunGit(work_dir, ['config', '--replace-all', 'user.email', email])
mbranch = git.MatchSingleBranchName(work_dir,
branch,
namespace='refs/remotes/origin/')
if branch != mbranch:
logging.info('Auto resolved branch name "%s" to "%s"', branch, mbranch)
branch = mbranch
# Finally, create a local branch for uploading changes to the given remote
# branch.
git.CreatePushBranch(constants.PATCH_BRANCH,
work_dir,
sync=False,
remote_push_branch=('ignore', 'origin/%s' % branch))
return branch
def _Submit(self, testjob, dryrun):
"""Internal submission function. See Submit() for arg description."""
# TODO(rcui): convert to shallow clone when that's available.
current_time = str(int(time.time()))
ref_base = os.path.join('refs/tryjobs', self.user, current_time)
for patch in self.local_patches:
# Isolate the name; if it's a tag or a remote, let through.
# Else if it's a branch, get the full branch name minus refs/heads.
local_branch = git.StripRefsHeads(patch.ref, False)
ref_final = os.path.join(ref_base, local_branch, patch.sha1)
self.manifest.AssertProjectIsPushable(patch.project)
data = self.manifest.projects[patch.project]
print 'Uploading patch %s' % patch
patch.Upload(data['push_url'], ref_final, dryrun=dryrun)
# TODO(rcui): Pass in the remote instead of tag. http://crosbug.com/33937.
tag = constants.EXTERNAL_PATCH_TAG
if data['remote'] == constants.INTERNAL_REMOTE:
tag = constants.INTERNAL_PATCH_TAG
self.extra_args.append('--remote-patches=%s:%s:%s:%s:%s'
% (patch.project, local_branch, ref_final,
patch.tracking_branch, tag))
self._VerifyForBuildbot()
repository.CloneGitRepo(self.tryjob_repo, self.ssh_url)
version_path = os.path.join(self.tryjob_repo,
self.TRYJOB_FORMAT_FILE)
with open(version_path, 'r') as f:
try:
val = int(f.read().strip())
except ValueError:
raise ChromiteUpgradeNeeded()
if val > self.TRYJOB_FORMAT_VERSION:
raise ChromiteUpgradeNeeded(val)
push_branch = manifest_version.PUSH_BRANCH
remote_branch = ('origin', 'refs/remotes/origin/test') if testjob else None
git.CreatePushBranch(push_branch, self.tryjob_repo, sync=False,
remote_push_branch=remote_branch)
file_name = '%s.%s' % (self.user,
current_time)
user_dir = os.path.join(self.tryjob_repo, self.user)
if not os.path.isdir(user_dir):
os.mkdir(user_dir)
fullpath = os.path.join(user_dir, file_name)
with open(fullpath, 'w+') as job_desc_file:
json.dump(self.values, job_desc_file)
cros_build_lib.RunCommand(['git', 'add', fullpath], cwd=self.tryjob_repo)
extra_env = {
# The committer field makes sure the creds match what the remote
# gerrit instance expects while the author field allows lookup
# on the console to work. http://crosbug.com/27939
'GIT_COMMITTER_EMAIL' : self.user_email,
'GIT_AUTHOR_EMAIL' : self.user_email,
}
cros_build_lib.RunCommand(['git', 'commit', '-m', self.description],
cwd=self.tryjob_repo, extra_env=extra_env)
try:
git.PushWithRetry(
push_branch, self.tryjob_repo, retries=3, dryrun=dryrun)
except cros_build_lib.RunCommandError:
cros_build_lib.Error(
'Failed to submit tryjob. This could be due to too many '
'submission requests by users. Please try again.')
raise
