def _CommitAndPush(manifest_repo, git_url, buildspec, contents, dryrun):
"""Helper for committing and pushing buildspecs.
Will create/checkout/clean the manifest_repo checkout at the given
path with no assumptions about the previous state. It should NOT be
considered clean when this function exits.
Args:
manifest_repo: Path to root of git repo for manifest_versions (int or ext).
git_url: Git URL for remote git repository.
buildspec: Relative path to buildspec in repo.
contents: String constaining contents of buildspec manifest.
dryrun: Git push --dry-run if set to True.
Returns:
Full path to buildspec created.
"""
RefreshManifestCheckout(manifest_repo, git_url)
filename = os.path.join(manifest_repo, buildspec)
assert not os.path.exists(filename)
git.CreatePushBranch(PUSH_BRANCH, manifest_repo, sync=False)
osutils.WriteFile(filename, contents, makedirs=True)
git.RunGit(manifest_repo, ['add', '-A'])
message = 'Creating buildspec: %s' % buildspec
git.RunGit(manifest_repo, ['commit', '-m', message])
git.PushBranch(PUSH_BRANCH, manifest_repo, dryrun=dryrun)
logging.info('Created buildspec: %s as %s', buildspec, filename)
return filename
def CreateNewManifest(self):
"""Generates a new manifest with updated Chrome entries."""
# Prepare git repo for push
git.CreatePushBranch(
TEST_BRANCH, self.manifest_dir,
remote_push_branch=('origin', 'refs/heads/master'))
content = StringIO.StringIO()
content.write(_TYPE_MARKER % {'type': 'EXTERNAL'})
content.write(_MkProject(self.CHROMIUM_ROOT, self.CHROMIUM_PROJECT,
self.EXTERNAL_REMOTE, branch='git-svn'))
# Grab the repo manifest, and ensure that we're not adding a project that
# our inherit target already has.
include_target = os.path.join(self.manifest_dir, INCLUDE_TARGET)
existing_projects = frozenset(git.Manifest.Cached(
include_target, manifest_include_dir=self.manifest_dir).projects)
ConvertDepsToManifest(
os.path.join(self.repo_root, self.CHROMIUM_ROOT, '.DEPS.git'),
existing_projects, content, self.EXTERNAL_REMOTE)
if self.internal:
content.write(_TYPE_MARKER % {'type': 'INTERNAL'})
content.write(_MkProject(self.CHROME_ROOT, self.CHROME_PROJECT,
self.INTERNAL_REMOTE))
ConvertDepsToManifest(
os.path.join(self.repo_root, self.CHROME_ROOT,
'.DEPS.git'),
existing_projects, content, self.INTERNAL_REMOTE)
osutils.WriteFile(
self.manifest_path,
_MANIFEST_TEMPLATE % {'content':content.getvalue(),
'include_target':INCLUDE_TARGET})
def PromoteCandidate(self, retries=manifest_version.NUM_RETRIES):
"""Promotes the current LKGM candidate to be a real versioned LKGM."""
assert self.current_version, 'No current manifest exists.'
last_error = None
path_to_candidate = self.GetLocalManifest(self.current_version)
assert os.path.exists(
path_to_candidate), 'Candidate not found locally.'
# This may potentially fail for not being at TOT while pushing.
for attempt in range(0, retries + 1):
try:
if attempt > 0:
self.RefreshManifestCheckout()
git.CreatePushBranch(manifest_version.PUSH_BRANCH,
self.manifest_dir,
sync=False)
manifest_version.CreateSymlink(path_to_candidate,
self.lkgm_path)
cros_build_lib.RunCommand(['git', 'add', self.LKGM_PATH],
cwd=self.manifest_dir)
self.PushSpecChanges('Automatic: %s promoting %s to LKGM' %
(self.build_name, self.current_version))
return
except cros_build_lib.RunCommandError as e:
last_error = 'Failed to promote manifest. error: %s' % e
logging.error(last_error)
logging.error('Retrying to promote manifest: Retry %d/%d',
attempt + 1, retries)
else:
raise PromoteCandidateException(last_error)
def testPushChange(self):
git_log = 'Marking test_one as stable\nMarking test_two as stable\n'
fake_description = 'Marking set of ebuilds as stable\n\n%s' % git_log
self.mox.StubOutWithMock(cros_mark_as_stable, '_DoWeHaveLocalCommits')
self.mox.StubOutWithMock(cros_mark_as_stable.GitBranch, 'CreateBranch')
self.mox.StubOutWithMock(cros_mark_as_stable.GitBranch, 'Exists')
self.mox.StubOutWithMock(git, 'PushWithRetry')
self.mox.StubOutWithMock(git, 'GetTrackingBranch')
self.mox.StubOutWithMock(git, 'SyncPushBranch')
self.mox.StubOutWithMock(git, 'CreatePushBranch')
self.mox.StubOutWithMock(git, 'RunGit')
cros_mark_as_stable._DoWeHaveLocalCommits(
self._branch, self._target_manifest_branch, '.').AndReturn(True)
git.GetTrackingBranch('.', for_push=True).AndReturn(
['gerrit', 'refs/remotes/gerrit/master'])
git.SyncPushBranch('.', 'gerrit', 'refs/remotes/gerrit/master')
cros_mark_as_stable._DoWeHaveLocalCommits(
self._branch, 'refs/remotes/gerrit/master', '.').AndReturn(True)
result = cros_build_lib.CommandResult(output=git_log)
cros_build_lib.RunCommandCaptureOutput(
['git', 'log', '--format=format:%s%n%n%b',
'refs/remotes/gerrit/master..%s' % self._branch],
cwd='.').AndReturn(result)
git.CreatePushBranch('merge_branch', '.')
git.RunGit('.', ['merge', '--squash', self._branch])
git.RunGit('.', ['commit', '-m', fake_description])
git.RunGit('.', ['config', 'push.default', 'tracking'])
git.PushWithRetry('merge_branch', '.', dryrun=False)
self.mox.ReplayAll()
cros_mark_as_stable.PushChange(self._branch, self._target_manifest_branch,
False, '.')
self.mox.VerifyAll()
def RevGitFile(filename, data, dryrun=False):
"""Update and push the git file.
Args:
filename: file to modify that is in a git repo already
data: A dict of key/values to update in |filename|
dryrun: If True, do not actually commit the change.
"""
prebuilt_branch = 'prebuilt_branch'
cwd = os.path.abspath(os.path.dirname(filename))
commit = git.RunGit(cwd, ['rev-parse', 'HEAD']).output.rstrip()
description = '%s: updating %s' % (os.path.basename(filename),
', '.join(data.keys()))
# UpdateLocalFile will print out the keys/values for us.
print('Revving git file %s' % filename)
try:
git.CreatePushBranch(prebuilt_branch, cwd)
for key, value in data.items():
UpdateLocalFile(filename, value, key)
git.RunGit(cwd, ['add', filename])
git.RunGit(cwd, ['commit', '-m', description])
git.PushBranch(prebuilt_branch, cwd, dryrun=dryrun, auto_merge=True)
finally:
# We reset the index and the working tree state in case there are any
# uncommitted or pending changes, but we don't change any existing commits.
git.RunGit(cwd, ['reset', '--hard'])
# Check out the last good commit as a sanity fallback.
git.RunGit(cwd, ['checkout', commit])
def RevGitFile(filename, data, retries=5, dryrun=False):
"""Update and push the git file.
Args:
filename: file to modify that is in a git repo already
data: A dict of key/values to update in |filename|
retries: The number of times to retry before giving up, default: 5
"""
prebuilt_branch = 'prebuilt_branch'
cwd = os.path.abspath(os.path.dirname(filename))
commit = git.RunGit(cwd, ['rev-parse', 'HEAD']).output.rstrip()
description = '%s: updating %s' % (os.path.basename(filename),
', '.join(data.keys()))
# UpdateLocalFile will print out the keys/values for us.
print 'Revving git file %s' % filename
try:
git.CreatePushBranch(prebuilt_branch, cwd)
for key, value in data.iteritems():
UpdateLocalFile(filename, value, key)
git.RunGit(cwd, ['add', filename])
git.RunGit(cwd, ['commit', '-m', description])
git.PushWithRetry(prebuilt_branch, cwd, dryrun=dryrun, retries=retries)
finally:
cros_build_lib.RunCommand(['git', 'checkout', commit], cwd=cwd)
def GetNextBuildSpec(self, retries=NUM_RETRIES, build_id=None):
"""Returns a path to the next manifest to build.
Args:
retries: Number of retries for updating the status.
build_id: Optional integer cidb id of this build, which will be used to
annotate the manifest-version commit if one is created.
Raises:
GenerateBuildSpecException in case of failure to generate a buildspec
"""
last_error = None
for index in range(0, retries + 1):
try:
self.CheckoutSourceCode()
version_info = self.GetCurrentVersionInfo()
self.RefreshManifestCheckout()
self.InitializeManifestVariables(version_info)
if not self.force and self.HasCheckoutBeenBuilt():
logging.info(
'Build is not forced and this checkout already built')
return None
# If we're the master, always create a new build spec. Otherwise,
# only create a new build spec if we've already built the existing
# spec.
if self.master or not self.latest_unprocessed:
logging.info(
'Build is master or build latest unprocessed is None')
git.CreatePushBranch(PUSH_BRANCH,
self.manifest_dir,
sync=False)
version = self.GetNextVersion(version_info)
new_manifest = self.CreateManifest()
logging.info('Publishing the new manifest version')
self.PublishManifest(new_manifest,
version,
build_id=build_id)
else:
version = self.latest_unprocessed
self.current_version = version
logging.info('current_version: %s', self.current_version)
to_return = self.GetLocalManifest(version)
logging.info('Local manifest for version: %s', to_return)
return to_return
except cros_build_lib.RunCommandError as e:
last_error = 'Failed to generate buildspec. error: %s' % e
logging.error(last_error)
logging.error('Retrying to generate buildspec: Retry %d/%d',
index + 1, retries)
# Cleanse any failed local changes and throw an exception.
self.RefreshManifestCheckout()
raise GenerateBuildSpecException(last_error)
def testPushGitChangesWithRealPrep(self):
"""Another push test that tests push but on non-repo does it on a branch."""
manifest_versions_url = cbuildbot_config.GetManifestVersionsRepoUrl(
internal_build=False, read_only=False)
git_dir = os.path.join(constants.SOURCE_ROOT, 'manifest-versions')
manifest_version.RefreshManifestCheckout(git_dir,
manifest_versions_url)
git.CreatePushBranch(manifest_version.PUSH_BRANCH, git_dir, sync=False)
cros_build_lib.RunCommand(
('tee --append %s/AUTHORS' % git_dir).split(),
input='TEST USER <*****@*****.**>')
manifest_version._PushGitChanges(git_dir,
'Test appending user.',
dry_run=True)
def CreateFromManifest(self,
manifest,
retries=manifest_version.NUM_RETRIES,
build_id=None):
"""Sets up an lkgm_manager from the given manifest.
This method sets up an LKGM manager and publishes a new manifest to the
manifest versions repo based on the passed in manifest but filtering
internal repositories and changes out of it.
Args:
manifest: A manifest that possibly contains private changes/projects. It
is named with the given version we want to create a new manifest from
i.e R20-1920.0.1-rc7.xml where R20-1920.0.1-rc7 is the version.
retries: Number of retries for updating the status.
build_id: Optional integer cidb build id of the build publishing the
manifest.
Returns:
Path to the manifest version file to use.
Raises:
GenerateBuildSpecException in case of failure to check-in the new
manifest because of a git error or the manifest is already checked-in.
"""
last_error = None
new_manifest = manifest_version.FilterManifest(
manifest,
whitelisted_remotes=config_lib.GetSiteParams().EXTERNAL_REMOTES)
version_info = self.GetCurrentVersionInfo()
for _attempt in range(0, retries + 1):
try:
self.RefreshManifestCheckout()
self.InitializeManifestVariables(version_info)
git.CreatePushBranch(manifest_version.PUSH_BRANCH,
self.manifest_dir,
sync=False)
version = os.path.splitext(os.path.basename(manifest))[0]
logging.info('Publishing filtered build spec')
self.PublishManifest(new_manifest, version, build_id=build_id)
self.current_version = version
return self.GetLocalManifest(version)
except cros_build_lib.RunCommandError as e:
err_msg = 'Failed to generate LKGM Candidate. error: %s' % e
logging.error(err_msg)
last_error = err_msg
raise manifest_version.GenerateBuildSpecException(last_error)
def UpdateStatus(self,
success_map,
message=None,
retries=NUM_RETRIES,
dashboard_url=None):
"""Updates the status of the build for the current build spec.
Args:
success_map: Map of config names to whether they succeeded.
message: Message accompanied with change in status.
retries: Number of retries for updating the status
dashboard_url: Optional url linking to builder dashboard for this build.
"""
last_error = None
if message:
logging.info('Updating status with message %s', message)
for index in range(0, retries + 1):
try:
self.RefreshManifestCheckout()
git.CreatePushBranch(PUSH_BRANCH,
self.manifest_dir,
sync=False)
success = all(success_map.values())
commit_message = (
'Automatic checkin: status=%s build_version %s for '
'%s' % (BuilderStatus.GetCompletedStatus(success),
self.current_version, self.build_names[0]))
self._SetPassSymlinks(success_map)
self.PushSpecChanges(commit_message)
except cros_build_lib.RunCommandError as e:
last_error = ('Failed to update the status for %s with the '
'following error %s' %
(self.build_names[0], e.message))
logging.error(last_error)
logging.error('Retrying to generate buildspec: Retry %d/%d',
index + 1, retries)
else:
# Upload status to Google Storage as well.
self.UploadStatus(success,
message=message,
dashboard_url=dashboard_url)
return
# Cleanse any failed local changes and throw an exception.
self.RefreshManifestCheckout()
raise StatusUpdateException(last_error)
def _SetupWorkDirectoryForPatch(work_dir, patch, branch, manifest, email):
"""Set up local dir for uploading changes to the given patch's project."""
logging.notice('Setting up dir %s for uploading changes to %s', work_dir,
patch.project_url)
# Clone the git repo from reference if we have a pointer to a
# ManifestCheckout object.
reference = None
if manifest:
# Get the path to the first checkout associated with this change. Since
# all of the checkouts share git objects, it doesn't matter which checkout
# we pick.
path = manifest.FindCheckouts(patch.project,
only_patchable=True)[0]['path']
reference = os.path.join(constants.SOURCE_ROOT, path)
if not os.path.isdir(reference):
logging.error('Unable to locate git checkout: %s', reference)
logging.error('Did you mean to use --nomirror?')
# This will do an "raise OSError" with the right values.
os.open(reference, os.O_DIRECTORY)
# Use the email if email wasn't specified.
if not email:
email = git.GetProjectUserEmail(reference)
repository.CloneGitRepo(work_dir, patch.project_url, reference=reference)
# Set the git committer.
git.RunGit(work_dir, ['config', '--replace-all', 'user.email', email])
mbranch = git.MatchSingleBranchName(work_dir,
branch,
namespace='refs/remotes/origin/')
if branch != mbranch:
logging.notice('Auto resolved branch name "%s" to "%s"', branch,
mbranch)
branch = mbranch
# Finally, create a local branch for uploading changes to the given remote
# branch.
git.CreatePushBranch(constants.PATCH_BRANCH,
work_dir,
sync=False,
remote_push_branch=git.RemoteRef(
'ignore', 'origin/%s' % branch))
return branch
def _PushConfig(self, workdir, testjob, dryrun, current_time):
"""Pushes the tryjob config to Git as a file.
Args:
workdir: see Submit()
testjob: see Submit()
dryrun: see Submit()
current_time: the current time as a string represention of the time since
unix epoch.
"""
push_branch = manifest_version.PUSH_BRANCH
remote_branch = None
if testjob:
remote_branch = git.RemoteRef('origin', 'refs/remotes/origin/test')
git.CreatePushBranch(push_branch,
workdir,
sync=False,
remote_push_branch=remote_branch)
file_name = '%s.%s' % (self.user, current_time)
user_dir = os.path.join(workdir, self.user)
if not os.path.isdir(user_dir):
os.mkdir(user_dir)
fullpath = os.path.join(user_dir, file_name)
with open(fullpath, 'w+') as job_desc_file:
json.dump(self.values, job_desc_file)
git.RunGit(workdir, ['add', fullpath])
extra_env = {
# The committer field makes sure the creds match what the remote
# gerrit instance expects while the author field allows lookup
# on the console to work. http://crosbug.com/27939
'GIT_COMMITTER_EMAIL': self.user_email,
'GIT_AUTHOR_EMAIL': self.user_email,
}
git.RunGit(workdir, ['commit', '-m', self.description],
extra_env=extra_env)
try:
git.PushWithRetry(push_branch, workdir, retries=3, dryrun=dryrun)
except cros_build_lib.RunCommandError:
logging.error(
'Failed to submit tryjob. This could be due to too many '
'submission requests by users. Please try again.')
raise
def UpdateBinhostConfFile(path, key, value):
"""Update binhost config file file with key=value.
Args:
path: Filename to update.
key: Key to update.
value: New value for key.
"""
cwd, filename = os.path.split(os.path.abspath(path))
osutils.SafeMakedirs(cwd)
if not git.GetCurrentBranch(cwd):
git.CreatePushBranch(constants.STABLE_EBUILD_BRANCH, cwd, sync=False)
osutils.WriteFile(path, '', mode='a')
if UpdateLocalFile(path, value, key):
desc = '%s: %s %s' % (filename, 'updating' if value else 'clearing', key)
git.AddPath(path)
git.Commit(cwd, desc)
def GetNextBuildSpec(self, retries=NUM_RETRIES):
"""Returns a path to the next manifest to build.
Args:
retries: Number of retries for updating the status.
Raises:
GenerateBuildSpecException in case of failure to generate a buildspec
"""
last_error = None
for index in range(0, retries + 1):
try:
self.CheckoutSourceCode()
version_info = self.GetCurrentVersionInfo()
self.RefreshManifestCheckout()
self.InitializeManifestVariables(version_info)
if not self.force and self.HasCheckoutBeenBuilt():
return None
# If we're the master, always create a new build spec. Otherwise,
# only create a new build spec if we've already built the existing
# spec.
if self.master or not self.latest_unprocessed:
git.CreatePushBranch(PUSH_BRANCH,
self.manifest_dir,
sync=False)
version = self.GetNextVersion(version_info)
new_manifest = self.CreateManifest()
self.PublishManifest(new_manifest, version)
else:
version = self.latest_unprocessed
self.SetInFlight(version)
self.current_version = version
return self.GetLocalManifest(version)
except cros_build_lib.RunCommandError as e:
last_error = 'Failed to generate buildspec. error: %s' % e
logging.error(last_error)
logging.error('Retrying to generate buildspec: Retry %d/%d',
index + 1, retries)
else:
# Cleanse any failed local changes and throw an exception.
self.RefreshManifestCheckout()
raise GenerateBuildSpecException(last_error)
def UpdateBinhostConfFile(path, key, value):
"""Update binhost config file file with key=value.
Args:
path: Filename to update.
key: Key to update.
value: New value for key.
"""
cwd = os.path.dirname(os.path.abspath(path))
filename = os.path.basename(path)
osutils.SafeMakedirs(cwd)
if not git.GetCurrentBranch(cwd):
git.CreatePushBranch(constants.STABLE_EBUILD_BRANCH, cwd, sync=False)
osutils.WriteFile(path, '', mode='a')
UpdateLocalFile(path, value, key)
cros_build_lib.RunCommand(['git', 'add', filename], cwd=cwd)
description = '%s: updating %s' % (os.path.basename(filename), key)
cros_build_lib.RunCommand(['git', 'commit', '-m', description], cwd=cwd)
def PushChange(stable_branch, tracking_branch, dryrun, cwd):
"""Pushes commits in the stable_branch to the remote git repository.
Pushes local commits from calls to CommitChange to the remote git
repository specified by current working directory. If changes are
found to commit, they will be merged to the merge branch and pushed.
In that case, the local repository will be left on the merge branch.
Args:
stable_branch: The local branch with commits we want to push.
tracking_branch: The tracking branch of the local branch.
dryrun: Use git push --dryrun to emulate a push.
cwd: The directory to run commands in.
Raises:
OSError: Error occurred while pushing.
"""
if not _DoWeHaveLocalCommits(stable_branch, tracking_branch, cwd):
cros_build_lib.Info('No work found to push in %s. Exiting', cwd)
return
# For the commit queue, our local branch may contain commits that were
# just tested and pushed during the CommitQueueCompletion stage. Sync
# and rebase our local branch on top of the remote commits.
remote, push_branch = git.GetTrackingBranch(cwd, for_push=True)
git.SyncPushBranch(cwd, remote, push_branch)
# Check whether any local changes remain after the sync.
if not _DoWeHaveLocalCommits(stable_branch, push_branch, cwd):
cros_build_lib.Info('All changes already pushed for %s. Exiting', cwd)
return
description = cros_build_lib.RunCommandCaptureOutput([
'git', 'log', '--format=format:%s%n%n%b',
'%s..%s' % (push_branch, stable_branch)
],
cwd=cwd).output
description = 'Marking set of ebuilds as stable\n\n%s' % description
cros_build_lib.Info('For %s, using description %s', cwd, description)
git.CreatePushBranch(constants.MERGE_BRANCH, cwd)
git.RunGit(cwd, ['merge', '--squash', stable_branch])
git.RunGit(cwd, ['commit', '-m', description])
git.RunGit(cwd, ['config', 'push.default', 'tracking'])
git.PushWithRetry(constants.MERGE_BRANCH, cwd, dryrun=dryrun)
def testCreateNewCandidate(self):
"""Tests that we can create a new candidate and uprev an old rc."""
# Let's stub out other LKGMManager calls cause they're already
# unit tested.
self.mox.StubOutWithMock(lkgm_manager.LKGMManager,
'GetCurrentVersionInfo')
self.mox.StubOutWithMock(lkgm_manager.LKGMManager,
'CheckoutSourceCode')
self.mox.StubOutWithMock(lkgm_manager.LKGMManager,
'RefreshManifestCheckout')
self.mox.StubOutWithMock(lkgm_manager.LKGMManager,
'InitializeManifestVariables')
self.mox.StubOutWithMock(lkgm_manager.LKGMManager,
'HasCheckoutBeenBuilt')
self.mox.StubOutWithMock(lkgm_manager.LKGMManager, 'CreateManifest')
self.mox.StubOutWithMock(lkgm_manager.LKGMManager, 'PublishManifest')
my_info = lkgm_manager._LKGMCandidateInfo('1.2.3')
most_recent_candidate = lkgm_manager._LKGMCandidateInfo('1.2.3-rc12')
self.manager.latest = most_recent_candidate.VersionString()
new_candidate = lkgm_manager._LKGMCandidateInfo('1.2.3-rc13')
new_manifest = 'some_manifest'
lkgm_manager.LKGMManager.CheckoutSourceCode()
lkgm_manager.LKGMManager.CreateManifest().AndReturn(new_manifest)
lkgm_manager.LKGMManager.HasCheckoutBeenBuilt().AndReturn(False)
# Do manifest refresh work.
lkgm_manager.LKGMManager.RefreshManifestCheckout()
git.CreatePushBranch(mox.IgnoreArg(), mox.IgnoreArg(), sync=False)
lkgm_manager.LKGMManager.GetCurrentVersionInfo().AndReturn(my_info)
lkgm_manager.LKGMManager.InitializeManifestVariables(my_info)
# Publish new candidate.
lkgm_manager.LKGMManager.PublishManifest(new_manifest,
new_candidate.VersionString())
self.mox.ReplayAll()
candidate_path = self.manager.CreateNewCandidate()
self.assertEqual(candidate_path,
self._GetPathToManifest(new_candidate))
self.mox.VerifyAll()
def CommonTestIncrementVersion(self, incr_type, version):
"""Common test increment. Returns path to new incremented file."""
message = 'Incrementing cuz I sed so'
self.mox.StubOutWithMock(git, 'CreatePushBranch')
self.mox.StubOutWithMock(manifest_version, '_PushGitChanges')
self.mox.StubOutWithMock(git, 'CleanAndCheckoutUpstream')
git.CreatePushBranch(manifest_version.PUSH_BRANCH, self.tempdir)
version_file = self.CreateFakeVersionFile(self.tempdir, version)
manifest_version._PushGitChanges(self.tempdir, message, dry_run=False)
git.CleanAndCheckoutUpstream(self.tempdir)
self.mox.ReplayAll()
info = manifest_version.VersionInfo(version_file=version_file,
incr_type=incr_type)
info.IncrementVersion(message, dry_run=False)
self.mox.VerifyAll()
return version_file
def UpdateStatus(self, success, message=None, retries=NUM_RETRIES):
"""Updates the status of the build for the current build spec.
Args:
success: True for success, False for failure
message: Message accompanied with change in status.
retries: Number of retries for updating the status
"""
last_error = None
if message: logging.info('Updating status with message %s', message)
for index in range(0, retries + 1):
try:
self.RefreshManifestCheckout()
git.CreatePushBranch(PUSH_BRANCH,
self.manifest_dir,
sync=False)
commit_message = (
'Automatic checkin: status=%s build_version %s for '
'%s' % (BuilderStatus.GetCompletedStatus(success),
self.current_version, self.build_name))
if success:
self._SetPassed()
else:
self._SetFailed()
self.PushSpecChanges(commit_message)
except cros_build_lib.RunCommandError as e:
last_error = ('Failed to update the status for %s with the '
'following error %s' %
(self.build_name, e.message))
logging.error(last_error)
logging.error('Retrying to generate buildspec: Retry %d/%d',
index + 1, retries)
else:
# Upload status to Google Storage as well.
self.UploadStatus(success, message=message)
return
else:
# Cleanse any failed local changes and throw an exception.
self.RefreshManifestCheckout()
raise StatusUpdateException(last_error)
def testCreateFromManifest(self):
"""Tests that we can create a new candidate from another manifest."""
# Let's stub out other LKGMManager calls cause they're already
# unit tested.
self.mox.StubOutWithMock(lkgm_manager.LKGMManager,
'GetCurrentVersionInfo')
self.mox.StubOutWithMock(lkgm_manager.LKGMManager,
'RefreshManifestCheckout')
self.mox.StubOutWithMock(lkgm_manager.LKGMManager,
'InitializeManifestVariables')
self.mox.StubOutWithMock(lkgm_manager.LKGMManager,
'_FilterCrosInternalProjectsFromManifest')
self.mox.StubOutWithMock(lkgm_manager.LKGMManager, 'PublishManifest')
version = '2010.0.0-rc7'
my_info = lkgm_manager._LKGMCandidateInfo('2010.0.0')
new_candidate = lkgm_manager._LKGMCandidateInfo(version)
manifest = ('/tmp/manifest-versions-internal/paladin/buildspecs/'
'20/%s.xml' % version)
new_manifest = '/path/to/tmp/file.xml'
lkgm_manager.LKGMManager._FilterCrosInternalProjectsFromManifest(
manifest).AndReturn(new_manifest)
# Do manifest refresh work.
lkgm_manager.LKGMManager.GetCurrentVersionInfo().AndReturn(my_info)
lkgm_manager.LKGMManager.RefreshManifestCheckout()
lkgm_manager.LKGMManager.InitializeManifestVariables(my_info)
git.CreatePushBranch(mox.IgnoreArg(), mox.IgnoreArg(), sync=False)
# Publish new candidate.
lkgm_manager.LKGMManager.PublishManifest(new_manifest, version)
self.mox.ReplayAll()
candidate_path = self.manager.CreateFromManifest(manifest)
self.assertEqual(candidate_path,
self._GetPathToManifest(new_candidate))
self.assertEqual(self.manager.current_version, version)
self.mox.VerifyAll()
def UpdateStatus(self, success_map, message=None, retries=NUM_RETRIES):
"""Updates the status of the build for the current build spec.
Args:
success_map: Map of config names to whether they succeeded.
message: Message accompanied with change in status.
retries: Number of retries for updating the status
"""
last_error = None
if message:
logging.info('Updating status with message %s', message)
for index in range(0, retries + 1):
try:
self.RefreshManifestCheckout()
git.CreatePushBranch(PUSH_BRANCH,
self.manifest_dir,
sync=False)
success = all(success_map.values())
commit_message = (
'Automatic checkin: status=%s build_version %s for %s' %
(builder_status_lib.BuilderStatus.GetCompletedStatus(
success), self.current_version, self.build_names[0]))
self._SetPassSymlinks(success_map)
self.PushSpecChanges(commit_message)
return
except cros_build_lib.RunCommandError as e:
last_error = (
'Failed to update the status for %s during remote'
' command: %s' % (self.build_names[0], e.message))
logging.error(last_error)
logging.error('Retrying to update the status: Retry %d/%d',
index + 1, retries)
# Cleanse any failed local changes and throw an exception.
self.RefreshManifestCheckout()
raise StatusUpdateException(last_error)
def CreateNewCandidate(self,
validation_pool=None,
retries=manifest_version.NUM_RETRIES):
"""Creates, syncs to, and returns the next candidate manifest.
Args:
validation_pool: Validation pool to apply to the manifest before
publishing.
retries: Number of retries for updating the status.
Raises:
GenerateBuildSpecException in case of failure to generate a buildspec
"""
self.CheckoutSourceCode()
# Refresh manifest logic from manifest_versions repository to grab the
# LKGM to generate the blamelist.
version_info = self.GetCurrentVersionInfo()
self.RefreshManifestCheckout()
self.InitializeManifestVariables(version_info)
self._GenerateBlameListSinceLKGM()
new_manifest = self.CreateManifest()
# For the Commit Queue, apply the validation pool as part of checkout.
if validation_pool:
# If we have nothing that could apply from the validation pool and
# we're not also a pfq type, we got nothing to do.
assert self.cros_source.directory == validation_pool.build_root
if (not validation_pool.ApplyPoolIntoRepo()
and not cbuildbot_config.IsPFQType(self.build_type)):
return None
self._AddPatchesToManifest(new_manifest, validation_pool.changes)
last_error = None
for attempt in range(0, retries + 1):
try:
# Refresh manifest logic from manifest_versions repository.
# Note we don't need to do this on our first attempt as we needed to
# have done it to get the LKGM.
if attempt != 0:
self.RefreshManifestCheckout()
self.InitializeManifestVariables(version_info)
# If we don't have any valid changes to test, make sure the checkout
# is at least different.
if ((not validation_pool or not validation_pool.changes)
and not self.force and self.HasCheckoutBeenBuilt()):
return None
# Check whether the latest spec available in manifest-versions is
# newer than our current version number. If so, use it as the base
# version number. Otherwise, we default to 'rc1'.
if self.latest:
latest = max(self.latest,
version_info.VersionString(),
key=self.compare_versions_fn)
version_info = _LKGMCandidateInfo(
latest,
chrome_branch=version_info.chrome_branch,
incr_type=self.incr_type)
git.CreatePushBranch(manifest_version.PUSH_BRANCH,
self.manifest_dir,
sync=False)
version = self.GetNextVersion(version_info)
self.PublishManifest(new_manifest, version)
self.current_version = version
return self.GetLocalManifest(version)
except cros_build_lib.RunCommandError as e:
err_msg = 'Failed to generate LKGM Candidate. error: %s' % e
logging.error(err_msg)
last_error = err_msg
else:
raise manifest_version.GenerateBuildSpecException(last_error)
def PushChange(stable_branch, tracking_branch, dryrun, cwd,
staging_branch=None):
"""Pushes commits in the stable_branch to the remote git repository.
Pushes local commits from calls to CommitChange to the remote git
repository specified by current working directory. If changes are
found to commit, they will be merged to the merge branch and pushed.
In that case, the local repository will be left on the merge branch.
Args:
stable_branch: The local branch with commits we want to push.
tracking_branch: The tracking branch of the local branch.
dryrun: Use git push --dryrun to emulate a push.
cwd: The directory to run commands in.
staging_branch: The staging branch to push for a failed PFQ run
Raises:
OSError: Error occurred while pushing.
"""
if not git.DoesCommitExistInRepo(cwd, stable_branch):
logging.debug('No branch created for %s. Exiting', cwd)
return
if not _DoWeHaveLocalCommits(stable_branch, tracking_branch, cwd):
logging.debug('No work found to push in %s. Exiting', cwd)
return
# For the commit queue, our local branch may contain commits that were
# just tested and pushed during the CommitQueueCompletion stage. Sync
# and rebase our local branch on top of the remote commits.
remote_ref = git.GetTrackingBranch(cwd,
branch=stable_branch,
for_push=True)
# SyncPushBranch rebases HEAD onto the updated remote. We need to checkout
# stable_branch here in order to update it.
git.RunGit(cwd, ['checkout', stable_branch])
git.SyncPushBranch(cwd, remote_ref.remote, remote_ref.ref)
# Check whether any local changes remain after the sync.
if not _DoWeHaveLocalCommits(stable_branch, remote_ref.ref, cwd):
logging.info('All changes already pushed for %s. Exiting', cwd)
return
# Add a failsafe check here. Only CLs from the 'chrome-bot' user should
# be involved here. If any other CLs are found then complain.
# In dryruns extra CLs are normal, though, and can be ignored.
bad_cl_cmd = ['log', '--format=short', '--perl-regexp',
'--author', '^(?!chrome-bot)', '%s..%s' % (
remote_ref.ref, stable_branch)]
bad_cls = git.RunGit(cwd, bad_cl_cmd).output
if bad_cls.strip() and not dryrun:
logging.error('The Uprev stage found changes from users other than '
'chrome-bot:\n\n%s', bad_cls)
raise AssertionError('Unexpected CLs found during uprev stage.')
if staging_branch is not None:
logging.info('PFQ FAILED. Pushing uprev change to staging branch %s',
staging_branch)
description = git.RunGit(
cwd,
['log', '--format=format:%s%n%n%b',
'%s..%s' % (remote_ref.ref, stable_branch)]).output
description = '%s\n\n%s' % (GIT_COMMIT_SUBJECT, description)
logging.info('For %s, using description %s', cwd, description)
git.CreatePushBranch(constants.MERGE_BRANCH, cwd,
remote_push_branch=remote_ref)
git.RunGit(cwd, ['merge', '--squash', stable_branch])
git.RunGit(cwd, ['commit', '-m', description])
git.RunGit(cwd, ['config', 'push.default', 'tracking'])
git.PushWithRetry(constants.MERGE_BRANCH, cwd, dryrun=dryrun,
staging_branch=staging_branch)
def CreateNewCandidate(self,
android_version=None,
chrome_version=None,
retries=manifest_version.NUM_RETRIES,
build_id=None):
"""Creates, syncs to, and returns the next candidate manifest.
Args:
android_version: The Android version to write in the manifest. Defaults
to None, in which case no version is written.
chrome_version: The Chrome version to write in the manifest. Defaults
to None, in which case no version is written.
retries: Number of retries for updating the status. Defaults to
manifest_version.NUM_RETRIES.
build_id: Optional integer cidb id of the build that is creating
this candidate.
Raises:
GenerateBuildSpecException in case of failure to generate a buildspec
"""
self.CheckoutSourceCode()
# Refresh manifest logic from manifest_versions repository to grab the
# LKGM to generate the blamelist.
version_info = self.GetCurrentVersionInfo()
self.RefreshManifestCheckout()
self.InitializeManifestVariables(version_info)
new_manifest = self.CreateManifest()
# For Android PFQ, add the version of Android to use.
if android_version:
self._AddAndroidVersionToManifest(new_manifest, android_version)
# For Chrome PFQ, add the version of Chrome to use.
if chrome_version:
self._AddChromeVersionToManifest(new_manifest, chrome_version)
last_error = None
for attempt in range(0, retries + 1):
try:
# Refresh manifest logic from manifest_versions repository.
# Note we don't need to do this on our first attempt as we needed to
# have done it to get the LKGM.
if attempt != 0:
self.RefreshManifestCheckout()
self.InitializeManifestVariables(version_info)
# If we don't have any valid changes to test, make sure the checkout
# is at least different.
if not self.force and self.HasCheckoutBeenBuilt():
return None
# Check whether the latest spec available in manifest-versions is
# newer than our current version number. If so, use it as the base
# version number. Otherwise, we default to 'rc1'.
if self.latest:
latest = max(self.latest,
version_info.VersionString(),
key=self.compare_versions_fn)
version_info = _LKGMCandidateInfo(
latest,
chrome_branch=version_info.chrome_branch,
incr_type=self.incr_type)
git.CreatePushBranch(manifest_version.PUSH_BRANCH,
self.manifest_dir,
sync=False)
version = self.GetNextVersion(version_info)
self.PublishManifest(new_manifest, version, build_id=build_id)
self.current_version = version
return self.GetLocalManifest(version)
except cros_build_lib.RunCommandError as e:
err_msg = 'Failed to generate LKGM Candidate. error: %s' % e
logging.error(err_msg)
last_error = err_msg
raise manifest_version.GenerateBuildSpecException(last_error)
def CreateNewCandidate(self,
validation_pool=None,
android_version=None,
chrome_version=None,
retries=manifest_version.NUM_RETRIES,
build_id=None):
"""Creates, syncs to, and returns the next candidate manifest.
Args:
validation_pool: Validation pool to apply to the manifest before
publishing.
android_version: The Android version to write in the manifest. Defaults
to None, in which case no version is written.
chrome_version: The Chrome version to write in the manifest. Defaults
to None, in which case no version is written.
retries: Number of retries for updating the status. Defaults to
manifest_version.NUM_RETRIES.
build_id: Optional integer cidb id of the build that is creating
this candidate.
Raises:
GenerateBuildSpecException in case of failure to generate a buildspec
"""
self.CheckoutSourceCode()
# Refresh manifest logic from manifest_versions repository to grab the
# LKGM to generate the blamelist.
version_info = self.GetCurrentVersionInfo()
self.RefreshManifestCheckout()
self.InitializeManifestVariables(version_info)
self.GenerateBlameListSinceLKGM()
# Throw away CLs that might not be used this run.
if validation_pool:
validation_pool.FilterChangesForThrottledTree()
# Apply any manifest CLs (internal or exteral).
validation_pool.ApplyPoolIntoRepo(
filter_fn=trybot_patch_pool.ManifestFilter)
manifest_dir = os.path.join(validation_pool.build_root,
'manifest-internal')
if not os.path.exists(manifest_dir):
# Fall back to external manifest directory.
manifest_dir = os.path.join(validation_pool.build_root,
'manifest')
# This is only needed if there were internal manifest changes, but we
# always run it to make sure this logic works.
self._AdjustRepoCheckoutToLocalManifest(manifest_dir)
new_manifest = self.CreateManifest()
# For Android PFQ, add the version of Android to use.
if android_version:
self._AddAndroidVersionToManifest(new_manifest, android_version)
# For Chrome PFQ, add the version of Chrome to use.
if chrome_version:
self._AddChromeVersionToManifest(new_manifest, chrome_version)
# For the Commit Queue, apply the validation pool as part of checkout.
if validation_pool:
# If we have nothing that could apply from the validation pool and
# we're not also a pfq type, we got nothing to do.
assert self.cros_source.directory == validation_pool.build_root
if (not validation_pool.ApplyPoolIntoRepo()
and not config_lib.IsPFQType(self.build_type)):
return None
self._AddPatchesToManifest(new_manifest, validation_pool.applied)
# Add info about the last known good version to the manifest. This will
# be used by slaves to calculate what artifacts from old builds are safe
# to use.
self._AddLKGMToManifest(new_manifest)
last_error = None
for attempt in range(0, retries + 1):
try:
# Refresh manifest logic from manifest_versions repository.
# Note we don't need to do this on our first attempt as we needed to
# have done it to get the LKGM.
if attempt != 0:
self.RefreshManifestCheckout()
self.InitializeManifestVariables(version_info)
# If we don't have any valid changes to test, make sure the checkout
# is at least different.
if ((not validation_pool or not validation_pool.applied)
and not self.force and self.HasCheckoutBeenBuilt()):
return None
# Check whether the latest spec available in manifest-versions is
# newer than our current version number. If so, use it as the base
# version number. Otherwise, we default to 'rc1'.
if self.latest:
latest = max(self.latest,
version_info.VersionString(),
key=self.compare_versions_fn)
version_info = _LKGMCandidateInfo(
latest,
chrome_branch=version_info.chrome_branch,
incr_type=self.incr_type)
git.CreatePushBranch(manifest_version.PUSH_BRANCH,
self.manifest_dir,
sync=False)
version = self.GetNextVersion(version_info)
self.PublishManifest(new_manifest, version, build_id=build_id)
self.current_version = version
return self.GetLocalManifest(version)
except cros_build_lib.RunCommandError as e:
err_msg = 'Failed to generate LKGM Candidate. error: %s' % e
logging.error(err_msg)
last_error = err_msg
raise manifest_version.GenerateBuildSpecException(last_error)
def _Submit(self, testjob, dryrun):
"""Internal submission function. See Submit() for arg description."""
# TODO(rcui): convert to shallow clone when that's available.
current_time = str(int(time.time()))
ref_base = os.path.join('refs/tryjobs', self.user, current_time)
for patch in self.local_patches:
# Isolate the name; if it's a tag or a remote, let through.
# Else if it's a branch, get the full branch name minus refs/heads.
local_branch = git.StripRefsHeads(patch.ref, False)
ref_final = os.path.join(ref_base, local_branch, patch.sha1)
self.manifest.AssertProjectIsPushable(patch.project)
data = self.manifest.projects[patch.project]
print 'Uploading patch %s' % patch
patch.Upload(data['push_url'], ref_final, dryrun=dryrun)
# TODO(rcui): Pass in the remote instead of tag. http://crosbug.com/33937.
tag = constants.EXTERNAL_PATCH_TAG
if data['remote'] == constants.INTERNAL_REMOTE:
tag = constants.INTERNAL_PATCH_TAG
self.extra_args.append('--remote-patches=%s:%s:%s:%s:%s'
% (patch.project, local_branch, ref_final,
patch.tracking_branch, tag))
self._VerifyForBuildbot()
repository.CloneGitRepo(self.tryjob_repo, self.ssh_url)
version_path = os.path.join(self.tryjob_repo,
self.TRYJOB_FORMAT_FILE)
with open(version_path, 'r') as f:
try:
val = int(f.read().strip())
except ValueError:
raise ChromiteUpgradeNeeded()
if val > self.TRYJOB_FORMAT_VERSION:
raise ChromiteUpgradeNeeded(val)
push_branch = manifest_version.PUSH_BRANCH
remote_branch = ('origin', 'refs/remotes/origin/test') if testjob else None
git.CreatePushBranch(push_branch, self.tryjob_repo, sync=False,
remote_push_branch=remote_branch)
file_name = '%s.%s' % (self.user,
current_time)
user_dir = os.path.join(self.tryjob_repo, self.user)
if not os.path.isdir(user_dir):
os.mkdir(user_dir)
fullpath = os.path.join(user_dir, file_name)
with open(fullpath, 'w+') as job_desc_file:
json.dump(self.values, job_desc_file)
cros_build_lib.RunCommand(['git', 'add', fullpath], cwd=self.tryjob_repo)
extra_env = {
# The committer field makes sure the creds match what the remote
# gerrit instance expects while the author field allows lookup
# on the console to work. http://crosbug.com/27939
'GIT_COMMITTER_EMAIL' : self.user_email,
'GIT_AUTHOR_EMAIL' : self.user_email,
}
cros_build_lib.RunCommand(['git', 'commit', '-m', self.description],
cwd=self.tryjob_repo, extra_env=extra_env)
try:
git.PushWithRetry(
push_branch, self.tryjob_repo, retries=3, dryrun=dryrun)
except cros_build_lib.RunCommandError:
cros_build_lib.Error(
'Failed to submit tryjob. This could be due to too many '
'submission requests by users. Please try again.')
raise
def IncrementVersion(self, message, dry_run):
"""Updates the version file by incrementing the patch component.
Args:
message: Commit message to use when incrementing the version.
dry_run: Git dry_run.
"""
def IncrementOldValue(line, key, new_value):
"""Change key to new_value if found on line. Returns True if changed."""
old_value = self.FindValue(key, line)
if old_value:
temp_fh.write(line.replace(old_value, new_value, 1))
return True
else:
return False
if not self.version_file:
raise VersionUpdateException(
'Cannot call IncrementVersion without '
'an associated version_file')
if not self.incr_type or self.incr_type not in ('build', 'branch'):
raise VersionUpdateException(
'Need to specify the part of the version to'
' increment')
if self.incr_type == 'build':
self.build_number = str(int(self.build_number) + 1)
self.branch_build_number = '0'
self.patch_number = '0'
elif self.patch_number == '0':
self.branch_build_number = str(int(self.branch_build_number) + 1)
else:
self.patch_number = str(int(self.patch_number) + 1)
temp_file = tempfile.mkstemp(suffix='mvp',
prefix='tmp',
dir=None,
text=True)[1]
with open(self.version_file, 'r') as source_version_fh:
with open(temp_file, 'w') as temp_fh:
for line in source_version_fh:
if IncrementOldValue(line, 'CHROMEOS_BUILD',
self.build_number):
pass
elif IncrementOldValue(line, 'CHROMEOS_BRANCH',
self.branch_build_number):
pass
elif IncrementOldValue(line, 'CHROMEOS_PATCH',
self.patch_number):
pass
else:
temp_fh.write(line)
temp_fh.close()
source_version_fh.close()
repo_dir = os.path.dirname(self.version_file)
try:
git.CreatePushBranch(PUSH_BRANCH, repo_dir)
shutil.copyfile(temp_file, self.version_file)
os.unlink(temp_file)
_PushGitChanges(repo_dir, message, dry_run=dry_run)
finally:
# Update to the remote version that contains our changes. This is needed
# to ensure that we don't build a release using a local commit.
git.CleanAndCheckoutUpstream(repo_dir)
return self.VersionString()