def _GsUpload(local_file, remote_file, acl):
"""Upload to GS bucket.
Args:
args: a tuple of three arguments that contains local_file, remote_file, and
the acl used for uploading the file.
Returns:
Return the arg tuple of two if the upload failed
"""
CANNED_ACLS = ['public-read', 'private', 'bucket-owner-read',
'authenticated-read', 'bucket-owner-full-control',
'public-read-write']
if acl in CANNED_ACLS:
cmd = [gs.GSUTIL_BIN, 'cp', '-a', acl, local_file, remote_file]
acl_cmd = None
else:
# For private uploads we assume that the overlay board is set up properly
# and a googlestore_acl.xml is present. Otherwise, this script errors.
cmd = [gs.GSUTIL_BIN, 'cp', '-a', 'private', local_file, remote_file]
acl_cmd = [gs.GSUTIL_BIN, 'setacl', acl, remote_file]
cros_build_lib.RunCommandWithRetries(_RETRIES, cmd, print_cmd=True,
sleep=_SLEEP_TIME,
redirect_stdout=True,
redirect_stderr=True)
if acl_cmd:
# Apply the passed in ACL xml file to the uploaded object.
cros_build_lib.RunCommandWithRetries(_RETRIES, acl_cmd, print_cmd=False,
sleep=_SLEEP_TIME)
def GetLatestSHA1ForBranch(self, project, branch):
"""Finds the latest commit hash for a repository/branch.
Returns:
The latest commit hash for this patch's repo/branch.
Raises:
FailedToReachGerrit if we fail to contact gerrit.
"""
ssh_url_project = '%s/%s' % (self.ssh_url, project)
try:
result = cros_build_lib.RunCommandWithRetries(
3, [
'git', 'ls-remote', ssh_url_project,
'refs/heads/%s' % (branch, )
],
redirect_stdout=True,
print_cmd=self.print_cmd)
if result:
return result.output.split()[0]
except cros_build_lib.RunCommandError as e:
# Fall out to Gerrit error.
logging.error('Failed to contact git server with %s', e)
raise FailedToReachGerrit(
'Could not contact gerrit to get latest sha1')
def CloneGitRepo(working_dir, repo_url, reference=None, bare=False,
mirror=False, retries=constants.SYNC_RETRIES, depth=None):
"""Clone given git repo
Args:
repo_url: git repo to clone
repo_dir: location where it should be cloned to
reference: If given, pathway to a git repository to access git objects
from. Note that the reference must exist as long as the newly created
repo is to be usable.
bare: Clone a bare checkout.
mirror: Clone a mirror checkout.
retries: If error code 128 is encountered, how many times to retry. When
128 is returned from git, it's essentially a server error- specifically
common to manifest-versions and gerrit.
depth: If given, do a shallow clone limiting the objects pulled to just
that # of revs of history. This option is mutually exclusive to
reference.
"""
osutils.SafeMakedirs(working_dir)
cmd = ['git', 'clone', repo_url, working_dir]
if reference:
if depth:
raise ValueError("reference and depth are mutually exclusive "
"options; please pick one or the other.")
cmd += ['--reference', reference]
if bare:
cmd += ['--bare']
if mirror:
cmd += ['--mirror']
if depth:
cmd += ['--depth', str(int(depth))]
cros_build_lib.RunCommandWithRetries(
retries, cmd, cwd=working_dir, redirect_stdout=True, redirect_stderr=True,
retry_on=[128])
def _UploadStatus(self,
version,
status,
message=None,
fail_if_exists=False):
"""Upload build status to Google Storage.
Args:
version: Version number to use. Must be a string.
status: Status string.
message: Additional message explaining the status.
fail_if_exists: If set, fail if the status already exists.
"""
cmd = [gs.GSUTIL_BIN]
if fail_if_exists:
# This HTTP header tells Google Storage toreturn the PreconditionFailed
# error message if the file already exists.
cmd += ['-h', 'x-goog-if-generation-match: 0']
url = BuildSpecsManager._GetStatusUrl(self.build_name, version)
cmd += ['cp', '-', url]
# Create a BuilderStatus object and pickle it.
data = cPickle.dumps(dict(status=status, message=message))
if self.dry_run:
logging.info('Would have run: %s', ' '.join(cmd))
else:
# TODO(davidjames): Use chromite.lib.gs here.
cros_build_lib.RunCommandWithRetries(3,
cmd,
redirect_stdout=True,
redirect_stderr=True,
input=data)
def GetBuildStatus(builder, version, retries=3):
"""Returns a BuilderStatus instance for the given the builder.
Args:
builder: Builder to look at.
version: Version string.
retries: Number of retries for getting the status.
Returns:
A BuilderStatus instance containing the builder status and any optional
message associated with the status passed by the builder.
"""
url = BuildSpecsManager._GetStatusUrl(builder, version)
cmd = [gs.GSUTIL_BIN, 'cat', url]
try:
# TODO(davidjames): Use chromite.lib.gs here.
result = cros_build_lib.RunCommandWithRetries(
retries,
cmd,
redirect_stdout=True,
redirect_stderr=True,
debug_level=logging.DEBUG)
except cros_build_lib.RunCommandError as ex:
# If the file does not exist, InvalidUriError is returned.
if ex.result.error and ex.result.error.startswith(
'InvalidUriError:'):
return None
raise
return BuilderStatus(**cPickle.loads(result.output))
def GetLatestSHA1ForBranch(self, project, branch):
url = 'https://%s/a/%s' % (self.host, project)
cmd = ['git', 'ls-remote', url, 'refs/heads/%s' % branch]
try:
result = cros_build_lib.RunCommandWithRetries(
3, cmd, redirect_stdout=True, print_cmd=self.print_cmd)
if result:
return result.output.split()[0]
except cros_build_lib.RunCommandError:
logging.error('Command "%s" failed.',
' '.join(map(repr, cmd)),
exc_info=True)
def testGetLatestSHA1ForBranch(self):
"""Verifies we can return the correct sha1 from mock data."""
self.mox.StubOutWithMock(cros_build_lib, 'RunCommandWithRetries')
my_hash = 'sadfjaslfkj2135'
my_branch = 'master'
result = self.mox.CreateMock(cros_build_lib.CommandResult)
result.returncode = 0
result.output = ' '.join([my_hash, my_branch])
cros_build_lib.RunCommandWithRetries(
3, ['git', 'ls-remote',
'ssh://gerrit.chromium.org:29418/tacos/chromite',
'refs/heads/master'],
redirect_stdout=True, print_cmd=True).AndReturn(result)
self.mox.ReplayAll()
helper = self._GetHelper()
self.assertEqual(helper.GetLatestSHA1ForBranch('tacos/chromite',
my_branch), my_hash)
self.mox.VerifyAll()
def Sync(self, local_manifest=None, jobs=None, cleanup=True,
all_branches=False, network_only=False):
"""Sync/update the source. Changes manifest if specified.
Args:
local_manifest: If true, checks out source to manifest. DEFAULT_MANIFEST
may be used to set it back to the default manifest.
jobs: May be set to override the default sync parallelism defined by
the manifest.
cleanup: If true, repo referencing is rebuilt, insteadOf configuration is
wiped, and appropriate remotes are setup. Should only be turned off
by code that knows the repo is clean.
all_branches: If False (the default), a repo sync -c is performed; this
saves on sync'ing via grabbing only what is needed for the manifest
specified branch.
network_only: If true, perform only the network half of the sync; skip
the checkout. Primarily of use to validate a manifest (although
if the manifest has bad copyfile statements, via skipping checkout
the broken copyfile tag won't be spotted), or of use when the
invoking code is fine w/ operating on bare repos, ie .repo/projects/*.
"""
try:
# Always re-initialize to the current branch.
self.Initialize(local_manifest)
# Fix existing broken mirroring configurations.
self._EnsureMirroring()
if cleanup:
configure_repo.FixBrokenExistingRepos(self.directory)
cmd = ['repo', '--time', 'sync']
if jobs:
cmd += ['--jobs', str(jobs)]
if not all_branches:
cmd.append('-c')
# Do the network half of the sync; retry as necessary to get the content.
cros_build_lib.RunCommandWithRetries(constants.SYNC_RETRIES, cmd + ['-n'],
cwd=self.directory)
if network_only:
return
# Do the local sync; note that there is a couple of corner cases where
# the new manifest cannot transition from the old checkout cleanly-
# primarily involving git submodules. Thus we intercept, and do
# a forced wipe, then a retry.
try:
cros_build_lib.RunCommand(cmd + ['-l'], cwd=self.directory)
except cros_build_lib.RunCommandError:
manifest = git.ManifestCheckout.Cached(self.directory)
targets = set(project['path'].split('/', 1)[0]
for project in manifest.projects.itervalues())
if not targets:
# No directories to wipe, thus nothing we can fix.
raise
cros_build_lib.SudoRunCommand(['rm', '-rf'] + sorted(targets),
cwd=self.directory)
# Retry the sync now; if it fails, let the exception propagate.
cros_build_lib.RunCommand(cmd + ['-l'], cwd=self.directory)
# Setup gerrit remote for any new repositories.
configure_repo.SetupGerritRemote(self.directory)
# We do a second run to fix any new repositories created by repo to
# use relative object pathways. Note that cros_sdk also triggers the
# same cleanup- we however kick it erring on the side of caution.
self._EnsureMirroring(True)
self._DoCleanup()
except cros_build_lib.RunCommandError, e:
err_msg = e.Stringify(error=False, output=False)
logging.error(err_msg)
raise SrcCheckOutException(err_msg)
