def _GetSHA1ForProject(manifest, project):
"""Get the latest SHA1 for a given project from Gerrit.
This function looks up the remote and branch for a given project in the
manifest, and uses this to lookup the SHA1 from Gerrit. This only makes
sense for unpinned manifests.
Args:
manifest: git.ManifestCheckout object.
project: Project to look up.
Raises:
Exception if the manifest is pinned.
"""
helper = gerrit.GerritHelper.FromManifestProject(manifest, project)
manifest_branch = manifest.GetAttributeForProject(project, 'revision')
branch = git.StripRefsHeads(manifest_branch)
return helper.GetLatestSHA1ForBranch(project, branch)
def _Submit(self, workdir, testjob, dryrun):
"""Internal submission function. See Submit() for arg description."""
# TODO(rcui): convert to shallow clone when that's available.
current_time = str(int(time.time()))
ref_base = os.path.join('refs/tryjobs', self.user_email, current_time)
for patch in self.local_patches:
# Isolate the name; if it's a tag or a remote, let through.
# Else if it's a branch, get the full branch name minus refs/heads.
local_branch = git.StripRefsHeads(patch.ref, False)
ref_final = os.path.join(ref_base, local_branch, patch.sha1)
checkout = patch.GetCheckout(self.manifest)
checkout.AssertPushable()
print('Uploading patch %s' % patch)
patch.Upload(checkout['push_url'], ref_final, dryrun=dryrun)
# TODO(rcui): Pass in the remote instead of tag. http://crosbug.com/33937.
tag = constants.EXTERNAL_PATCH_TAG
if checkout['remote'] == site_config.params.INTERNAL_REMOTE:
tag = constants.INTERNAL_PATCH_TAG
self.extra_args.append('--remote-patches=%s:%s:%s:%s:%s'
% (patch.project, local_branch, ref_final,
patch.tracking_branch, tag))
self._VerifyForBuildbot()
repository.UpdateGitRepo(workdir, self.repo_url)
version_path = os.path.join(workdir,
self.TRYJOB_FORMAT_FILE)
with open(version_path, 'r') as f:
try:
val = int(f.read().strip())
except ValueError:
raise ChromiteUpgradeNeeded()
if val > self.TRYJOB_FORMAT_VERSION:
raise ChromiteUpgradeNeeded(val)
if self.use_buildbucket:
self._PostConfigsToBuildBucket(testjob, dryrun)
else:
self._PushConfig(workdir, testjob, dryrun, current_time)
def PushLocalPatches(site_config, local_patches, user_email, dryrun=False):
"""Push local changes to a remote ref, and generate args to send.
Args:
site_config: config_lib.SiteConfig containing all config info.
local_patches: patch_pool.local_patches from verified patch_pool.
user_email: Unique id for user submitting this tryjob.
dryrun: Is this a dryrun? If so, don't really push.
Returns:
List of strings to pass to builder to include these patches.
"""
manifest = git.ManifestCheckout.Cached(constants.SOURCE_ROOT)
current_time = str(int(time.time()))
ref_base = os.path.join('refs/tryjobs', user_email, current_time)
extra_args = []
for patch in local_patches:
# Isolate the name; if it's a tag or a remote, let through.
# Else if it's a branch, get the full branch name minus refs/heads.
local_branch = git.StripRefsHeads(patch.ref, False)
ref_final = os.path.join(ref_base, local_branch, patch.sha1)
checkout = patch.GetCheckout(manifest)
checkout.AssertPushable()
print('Uploading patch %s' % patch)
patch.Upload(checkout['push_url'], ref_final, dryrun=dryrun)
# TODO(rcui): Pass in the remote instead of tag. http://crosbug.com/33937.
tag = constants.EXTERNAL_PATCH_TAG
if checkout['remote'] == site_config.params.INTERNAL_REMOTE:
tag = constants.INTERNAL_PATCH_TAG
extra_args.append('--remote-patches=%s:%s:%s:%s:%s' %
(patch.project, local_branch, ref_final,
patch.tracking_branch, tag))
return extra_args
def main(argv):
parser = GetParser()
options = parser.parse_args(argv)
repo_dir = git.FindRepoDir(os.getcwd())
if not repo_dir:
parser.error("This script must be invoked from within a repository "
"checkout.")
options.git_config = os.path.join(repo_dir, 'manifests.git', 'config')
options.local_manifest_path = os.path.join(repo_dir, 'local_manifest.xml')
manifest_sym_path = os.path.join(repo_dir, 'manifest.xml')
if os.path.basename(os.readlink(manifest_sym_path)) == 'minilayout.xml':
_AssertNotMiniLayout()
# For now, we only support the add command.
assert options.command == 'add'
if options.workon:
if options.path is not None:
parser.error('Adding workon projects do not set project.')
else:
if options.remote is None:
parser.error('Adding non-workon projects requires a remote.')
if options.path is None:
parser.error('Adding non-workon projects requires a path.')
name = options.project
path = options.path
revision = options.revision
if revision is not None:
if (not git.IsRefsTags(revision) and not git.IsSHA1(revision)):
revision = git.StripRefsHeads(revision, False)
main_manifest = git.ManifestCheckout(os.getcwd())
main_element = main_manifest.FindCheckouts(name)
if path is not None:
main_element_from_path = main_manifest.FindCheckoutFromPath(
path, strict=False)
if main_element_from_path is not None:
main_element.append(main_element_from_path)
local_manifest = LocalManifest.FromPath(options.local_manifest_path)
if options.workon:
if not main_element:
parser.error('No project named %r in the default manifest.' % name)
_AddProjectsToManifestGroups(
options, [checkout['name'] for checkout in main_element])
elif main_element:
if options.remote is not None:
# Likely this project wasn't meant to be remote, so workon main element
print(
"Project already exists in manifest. Using that as workon project."
)
_AddProjectsToManifestGroups(
options, [checkout['name'] for checkout in main_element])
else:
# Conflict will occur; complain.
parser.error(
"Requested project name=%r path=%r will conflict with "
"your current manifest %s" %
(name, path, main_manifest.manifest_path))
elif local_manifest.GetProject(name, path=path) is not None:
parser.error("Requested project name=%r path=%r conflicts with "
"your local_manifest.xml" % (name, path))
else:
element = local_manifest.AddNonWorkonProject(name=name,
path=path,
remote=options.remote,
revision=revision)
_AddProjectsToManifestGroups(options, [element.attrib['name']])
with open(options.local_manifest_path, 'w') as f:
f.write(local_manifest.ToString())
return 0
def _Submit(self, testjob, dryrun):
"""Internal submission function. See Submit() for arg description."""
# TODO(rcui): convert to shallow clone when that's available.
current_time = str(int(time.time()))
ref_base = os.path.join('refs/tryjobs', self.user, current_time)
for patch in self.local_patches:
# Isolate the name; if it's a tag or a remote, let through.
# Else if it's a branch, get the full branch name minus refs/heads.
local_branch = git.StripRefsHeads(patch.ref, False)
ref_final = os.path.join(ref_base, local_branch, patch.sha1)
self.manifest.AssertProjectIsPushable(patch.project)
data = self.manifest.projects[patch.project]
print 'Uploading patch %s' % patch
patch.Upload(data['push_url'], ref_final, dryrun=dryrun)
# TODO(rcui): Pass in the remote instead of tag. http://crosbug.com/33937.
tag = constants.EXTERNAL_PATCH_TAG
if data['remote'] == constants.INTERNAL_REMOTE:
tag = constants.INTERNAL_PATCH_TAG
self.extra_args.append('--remote-patches=%s:%s:%s:%s:%s'
% (patch.project, local_branch, ref_final,
patch.tracking_branch, tag))
self._VerifyForBuildbot()
repository.CloneGitRepo(self.tryjob_repo, self.ssh_url)
version_path = os.path.join(self.tryjob_repo,
self.TRYJOB_FORMAT_FILE)
with open(version_path, 'r') as f:
try:
val = int(f.read().strip())
except ValueError:
raise ChromiteUpgradeNeeded()
if val > self.TRYJOB_FORMAT_VERSION:
raise ChromiteUpgradeNeeded(val)
push_branch = manifest_version.PUSH_BRANCH
remote_branch = ('origin', 'refs/remotes/origin/test') if testjob else None
git.CreatePushBranch(push_branch, self.tryjob_repo, sync=False,
remote_push_branch=remote_branch)
file_name = '%s.%s' % (self.user,
current_time)
user_dir = os.path.join(self.tryjob_repo, self.user)
if not os.path.isdir(user_dir):
os.mkdir(user_dir)
fullpath = os.path.join(user_dir, file_name)
with open(fullpath, 'w+') as job_desc_file:
json.dump(self.values, job_desc_file)
cros_build_lib.RunCommand(['git', 'add', fullpath], cwd=self.tryjob_repo)
extra_env = {
# The committer field makes sure the creds match what the remote
# gerrit instance expects while the author field allows lookup
# on the console to work. http://crosbug.com/27939
'GIT_COMMITTER_EMAIL' : self.user_email,
'GIT_AUTHOR_EMAIL' : self.user_email,
}
cros_build_lib.RunCommand(['git', 'commit', '-m', self.description],
cwd=self.tryjob_repo, extra_env=extra_env)
try:
git.PushWithRetry(
push_branch, self.tryjob_repo, retries=3, dryrun=dryrun)
except cros_build_lib.RunCommandError:
cros_build_lib.Error(
'Failed to submit tryjob. This could be due to too many '
'submission requests by users. Please try again.')
raise
def main(argv):
parser = optparse.OptionParser(usage='usage: %prog add [options] <name> '
'<--workon | <path> --remote <remote> >')
parser.add_option('-w',
'--workon',
action='store_true',
dest='workon',
default=False,
help='Is this a workon package?')
parser.add_option('-r', '--remote', dest='remote', default=None)
parser.add_option('-v',
'--revision',
dest='revision',
default=None,
help="Use to override the manifest defined default "
"revision used for a given project.")
parser.add_option('--upgrade-minilayout',
default=False,
action='store_true',
help="Upgrade a minilayout checkout into a full.xml "
"checkout utilizing manifest groups.")
(options, args) = parser.parse_args(argv)
repo_dir = git.FindRepoDir(os.getcwd())
if not repo_dir:
parser.error("This script must be invoked from within a repository "
"checkout.")
options.git_config = os.path.join(repo_dir, 'manifests.git', 'config')
options.repo_dir = repo_dir
options.local_manifest_path = os.path.join(repo_dir, 'local_manifest.xml')
# This constant is used only when we're doing an upgrade away from
# minilayout.xml to default.xml.
options.default_manifest_path = os.path.join(repo_dir, 'manifests',
'default.xml')
options.manifest_sym_path = os.path.join(repo_dir, 'manifest.xml')
active_manifest = os.path.basename(os.readlink(options.manifest_sym_path))
upgrade_required = active_manifest == 'minilayout.xml'
if options.upgrade_minilayout:
if args:
parser.error("--upgrade-minilayout takes no arguments.")
if not upgrade_required:
print "This repository checkout isn't using minilayout.xml; nothing to do"
else:
_UpgradeMinilayout(options)
return 0
elif upgrade_required:
logging.warn(
"Your repository checkout is using the old minilayout.xml workflow; "
"auto-upgrading it.")
cros_build_lib.RunCommand([sys.argv[0], '--upgrade-minilayout'],
cwd=os.getcwd(),
print_cmd=False)
if not args:
parser.error("No command specified.")
elif args[0] != 'add':
parser.error("Only supported subcommand is add right now.")
elif options.workon:
if len(args) != 2:
parser.error(
"Argument count is wrong for --workon; must be add <project>")
name, path = args[1], None
else:
if options.remote is None:
parser.error('Adding non-workon projects requires a remote.')
elif len(args) != 3:
parser.error("Argument count is wrong for non-workon mode; "
"must be add <project> <path> --remote <remote-arg>")
name, path = args[1:]
revision = options.revision
if revision is not None:
if (not git.IsRefsTags(revision) and not git.IsSHA1(revision)):
revision = git.StripRefsHeads(revision, False)
main_manifest = Manifest.FromPath(options.manifest_sym_path,
empty_if_missing=False)
local_manifest = Manifest.FromPath(options.local_manifest_path)
main_element = main_manifest.GetProject(name, path=path)
if options.workon:
if main_element is None:
parser.error('No project named %r in the default manifest.' % name)
_AddProjectsToManifestGroups(options, main_element.attrib['name'])
elif main_element is not None:
if options.remote is not None:
# Likely this project wasn't meant to be remote, so workon main element
print "Project already exists in manifest. Using that as workon project."
_AddProjectsToManifestGroups(options, main_element.attrib['name'])
else:
# Conflict will occur; complain.
parser.error(
"Requested project name=%r path=%r will conflict with "
"your current manifest %s" % (name, path, active_manifest))
elif local_manifest.GetProject(name, path=path) is not None:
parser.error("Requested project name=%r path=%r conflicts with "
"your local_manifest.xml" % (name, path))
else:
element = local_manifest.AddNonWorkonProject(name=name,
path=path,
remote=options.remote,
revision=revision)
_AddProjectsToManifestGroups(options, element.attrib['name'])
with open(options.local_manifest_path, 'w') as f:
f.write(local_manifest.ToString())
return 0
def main(argv):
parser = GetParser()
options = parser.parse_args(argv)
repo_dir = git.FindRepoDir(os.getcwd())
if not repo_dir:
parser.error("This script must be invoked from within a repository "
"checkout.")
options.git_config = os.path.join(repo_dir, 'manifests.git', 'config')
options.repo_dir = repo_dir
options.local_manifest_path = os.path.join(repo_dir, 'local_manifest.xml')
# This constant is used only when we're doing an upgrade away from
# minilayout.xml to default.xml.
options.default_manifest_path = os.path.join(repo_dir, 'manifests',
'default.xml')
options.manifest_sym_path = os.path.join(repo_dir, 'manifest.xml')
active_manifest = os.path.basename(os.readlink(options.manifest_sym_path))
upgrade_required = active_manifest == 'minilayout.xml'
if options.command == 'upgrade-minilayout':
if not upgrade_required:
print("This repository checkout isn't using minilayout.xml; "
"nothing to do")
else:
_UpgradeMinilayout(options)
return 0
elif upgrade_required:
logging.warning(
"Your repository checkout is using the old minilayout.xml workflow; "
"auto-upgrading it.")
main(['upgrade-minilayout'])
# For now, we only support the add command.
assert options.command == 'add'
if options.workon:
if options.path is not None:
parser.error('Adding workon projects do not set project.')
else:
if options.remote is None:
parser.error('Adding non-workon projects requires a remote.')
if options.path is None:
parser.error('Adding non-workon projects requires a path.')
name = options.project
path = options.path
revision = options.revision
if revision is not None:
if (not git.IsRefsTags(revision) and not git.IsSHA1(revision)):
revision = git.StripRefsHeads(revision, False)
main_manifest = Manifest.FromPath(options.manifest_sym_path,
empty_if_missing=False)
local_manifest = Manifest.FromPath(options.local_manifest_path)
main_element = main_manifest.GetProject(name, path=path)
if options.workon:
if main_element is None:
parser.error('No project named %r in the default manifest.' % name)
_AddProjectsToManifestGroups(options, main_element.attrib['name'])
elif main_element is not None:
if options.remote is not None:
# Likely this project wasn't meant to be remote, so workon main element
print(
"Project already exists in manifest. Using that as workon project."
)
_AddProjectsToManifestGroups(options, main_element.attrib['name'])
else:
# Conflict will occur; complain.
parser.error(
"Requested project name=%r path=%r will conflict with "
"your current manifest %s" % (name, path, active_manifest))
elif local_manifest.GetProject(name, path=path) is not None:
parser.error("Requested project name=%r path=%r conflicts with "
"your local_manifest.xml" % (name, path))
else:
element = local_manifest.AddNonWorkonProject(name=name,
path=path,
remote=options.remote,
revision=revision)
_AddProjectsToManifestGroups(options, element.attrib['name'])
with open(options.local_manifest_path, 'w') as f:
f.write(local_manifest.ToString())
return 0