def _DoCommand(self, gsutil_cmd, headers=(), retries=None, **kwargs):
"""Run a gsutil command, suppressing output, and setting retry/sleep.
Returns:
A RunCommandResult object.
"""
cmd = [self.gsutil_bin]
for header in headers:
cmd += ['-h', header]
cmd.extend(gsutil_cmd)
if retries is None:
retries = self._retries
extra_env = kwargs.pop('extra_env', {})
extra_env.setdefault('BOTO_CONFIG', self.boto_file)
if self.dry_run:
logging.debug("%s: would've ran %r", self.__class__.__name__, cmd)
else:
return cros_build_lib.RetryCommand(self._RunCommand,
retries,
cmd,
sleep=self._sleep_time,
extra_env=extra_env,
**kwargs)
def SyncPushBranch(git_repo, remote, rebase_target):
"""Sync and rebase a local push branch to the latest remote version.
Args:
git_repo: Git repository to rebase in.
remote: The remote returned by GetTrackingBranch(for_push=True)
rebase_target: The branch name returned by GetTrackingBranch(). Must
start with refs/remotes/ (specifically must be a proper remote
target rather than an ambiguous name).
"""
if not rebase_target.startswith("refs/remotes/"):
raise Exception(
"Was asked to rebase to a non branch target w/in the push pathways. "
"This is highly indicative of an internal bug. remote %s, rebase %s"
% (remote, rebase_target))
cmd = ['remote', 'update', remote]
cros_build_lib.RetryCommand(RunGit,
3,
git_repo,
cmd,
sleep=10,
retry_on=(1, ))
try:
RunGit(git_repo, ['rebase', rebase_target])
except cros_build_lib.RunCommandError:
# Looks like our change conflicts with upstream. Cleanup our failed
# rebase.
RunGit(git_repo, ['rebase', '--abort'], error_code_ok=True)
raise
def CreatePushBranch(branch, git_repo, sync=True, remote_push_branch=None):
"""Create a local branch for pushing changes inside a repo repository.
Args:
branch: Local branch to create.
git_repo: Git repository to create the branch in.
sync: Update remote before creating push branch.
remote_push_branch: A tuple of the (remote, branch) to push to. i.e.,
('cros', 'master'). By default it tries to
automatically determine which tracking branch to use
(see GetTrackingBranch()).
"""
if not remote_push_branch:
remote, push_branch = GetTrackingBranch(git_repo, for_push=True)
else:
remote, push_branch = remote_push_branch
if sync:
cmd = ['remote', 'update', remote]
cros_build_lib.RetryCommand(RunGit,
3,
git_repo,
cmd,
sleep=10,
retry_on=(1, ))
RunGit(git_repo, ['checkout', '-B', branch, '-t', push_branch])
def CleanAndCheckoutUpstream(git_repo, refresh_upstream=True):
"""Remove all local changes and checkout the latest origin.
All local changes in the supplied repo will be removed. The branch will
also be switched to a detached head pointing at the latest origin.
Args:
git_repo: Directory of git repository.
refresh_upstream: If True, run a remote update prior to checking it out.
"""
remote, local_upstream = GetTrackingBranch(git_repo,
for_push=refresh_upstream)
RunGit(git_repo, ['am', '--abort'], error_code_ok=True)
RunGit(git_repo, ['rebase', '--abort'], error_code_ok=True)
if refresh_upstream:
cmd = ['remote', 'update', remote]
cros_build_lib.RetryCommand(RunGit,
3,
git_repo,
cmd,
sleep=10,
retry_on=(1, ))
RunGit(git_repo, ['clean', '-dfx'])
RunGit(git_repo, ['reset', '--hard', 'HEAD'])
RunGit(git_repo, ['checkout', local_upstream])
def main(args):
parser = GetParser()
options = parser.parse_args(args)
ctx = gs.GSContext(retries=0)
try:
cros_build_lib.RetryCommand(Copy, ctx.DEFAULT_RETRIES, ctx, options.uri,
options.filename, sleep=ctx.DEFAULT_SLEEP_TIME)
except (gs.GSContextException, cros_build_lib.RunCommandError) as ex:
# Hide the stack trace using Die.
cros_build_lib.Die('%s', ex)
def UploadSymbol(sym_file,
upload_url,
file_limit=DEFAULT_FILE_LIMIT,
sleep=0,
num_errors=None):
"""Upload |sym_file| to |upload_url|
Args:
sym_file: The full path to the breakpad symbol to upload
upload_url: The crash server to upload things to
file_limit: The max file size of a symbol file before we try to strip it
sleep: Number of seconds to sleep before running
num_errors: An object to update with the error count (needs a .value member)
Returns:
The number of errors that were encountered.
"""
if num_errors is None:
num_errors = ctypes.c_int()
elif num_errors.value > MAX_TOTAL_ERRORS_FOR_RETRY:
# Abandon ship! It's on fire! NOoooooooooooOOOoooooo.
return 0
upload_file = sym_file
if sleep:
# Keeps us from DoS-ing the symbol server.
time.sleep(sleep)
cros_build_lib.Debug('uploading %s' % sym_file)
# Ideally there'd be a tempfile.SpooledNamedTemporaryFile that we could use.
with tempfile.NamedTemporaryFile(prefix='upload_symbols',
bufsize=0) as temp_sym_file:
if file_limit:
# If the symbols size is too big, strip out the call frame info. The CFI
# is unnecessary for 32bit x86 targets where the frame pointer is used (as
# all of ours have) and it accounts for over half the size of the symbols
# uploaded.
file_size = os.path.getsize(sym_file)
if file_size > file_limit:
cros_build_lib.Warning(
'stripping CFI from %s due to size %s > %s', sym_file,
file_size, file_limit)
temp_sym_file.writelines([
x for x in open(sym_file, 'rb').readlines()
if not x.startswith('STACK CFI')
])
upload_file = temp_sym_file.name
# Hopefully the crash server will let it through. But it probably won't.
# Not sure what the best answer is in this case.
file_size = os.path.getsize(upload_file)
if file_size > CRASH_SERVER_FILE_LIMIT:
cros_build_lib.PrintBuildbotStepWarnings()
cros_build_lib.Error(
'upload file %s is awfully large, risking rejection '
'by symbol server (%s > %s)', sym_file, file_size,
CRASH_SERVER_FILE_LIMIT)
num_errors.value += 1
# Upload the symbol file.
try:
cros_build_lib.RetryCommand(SymUpload,
MAX_RETRIES,
upload_file,
upload_url,
sleep=INITIAL_RETRY_DELAY)
cros_build_lib.Info('successfully uploaded %10i bytes: %s',
file_size, os.path.basename(sym_file))
except cros_build_lib.RunCommandError as e:
cros_build_lib.Warning(
'could not upload: %s:\n{stdout} %s\n{stderr} %s',
os.path.basename(sym_file), e.result.output, e.result.error)
num_errors.value += 1
return num_errors.value
