def PushWithRetry(branch, git_repo, dryrun=False, retries=5):
"""General method to push local git changes.
This method only works with branches created via the CreatePushBranch
function.
Args:
branch: Local branch to push. Branch should have already been created
with a local change committed ready to push to the remote branch. Must
also already be checked out to that branch.
git_repo: Git repository to push from.
dryrun: Git push --dry-run if set to True.
retries: The number of times to retry before giving up, default: 5
Raises:
GitPushFailed if push was unsuccessful after retries
"""
remote, ref = GetTrackingBranch(git_repo,
branch,
for_checkout=False,
for_push=True)
# Don't like invoking this twice, but there is a bit of API
# impedence here; cros_mark_as_stable
_, local_ref = GetTrackingBranch(git_repo, branch, for_push=True)
if not ref.startswith("refs/heads/"):
raise Exception("Was asked to push to a non branch namespace: %s" %
(ref, ))
push_command = ['push', remote, '%s:%s' % (branch, ref)]
cros_build_lib.Debug("Trying to push %s to %s:%s", git_repo, branch, ref)
if dryrun:
push_command.append('--dry-run')
for retry in range(1, retries + 1):
SyncPushBranch(git_repo, remote, local_ref)
try:
RunGit(git_repo, push_command)
break
except cros_build_lib.RunCommandError:
if retry < retries:
Warning('Error pushing changes trying again (%s/%s)', retry,
retries)
time.sleep(5 * retry)
continue
raise
cros_build_lib.Info("Successfully pushed %s to %s:%s", git_repo, branch,
ref)
def TestingSymUpload(sym_file, upload_url):
"""A stub version of SymUpload for --testing usage"""
cmd = ['sym_upload', sym_file, upload_url]
# Randomly fail 80% of the time (the retry logic makes this 80%/3 per file).
returncode = random.randint(1, 100) <= 80
cros_build_lib.Debug('would run (and return %i): %s', returncode,
' '.join(map(repr, cmd)))
if returncode:
output = 'Failed to send the symbol file.'
else:
output = 'Successfully sent the symbol file.'
result = cros_build_lib.CommandResult(cmd=cmd,
error=None,
output=output,
returncode=returncode)
if returncode:
raise cros_build_lib.RunCommandError('forced test fail', result)
else:
return result
def RunGit(git_repo, cmd, **kwds):
"""RunCommandCaptureOutput wrapper for git commands.
This suppresses print_cmd, and suppresses output by default. Git
functionality w/in this module should use this unless otherwise
warranted, to standardize git output (primarily, keeping it quiet
and being able to throw useful errors for it).
Args:
git_repo: Pathway to the git repo to operate on.
cmd: A sequence of the git subcommand to run. The 'git' prefix is
added automatically. If you wished to run 'git remote update',
this would be ['remote', 'update'] for example.
kwds: Any RunCommand options/overrides to use.
Returns:
A CommandResult object."""
kwds.setdefault('print_cmd', False)
cros_build_lib.Debug("RunGit(%r, %r, **%r)", git_repo, cmd, kwds)
return cros_build_lib.RunCommandCaptureOutput(['git'] + cmd,
cwd=git_repo,
**kwds)
def Pin(self, commit_hash):
"""Attempt to pin the project to the specified commit hash.
Arguments:
commit_hash: The commit to pin the project to.
Raises:
ProjectException when an error occurs.
"""
self._PrepareProject()
if git.GetCurrentBranch(self.abs_path):
cros_build_lib.Warning(
"Not pinning project %s that's checked out to a "
'development branch.' % self.rel_path)
elif (commit_hash
and (commit_hash != git.GetGitRepoRevision(self.abs_path))):
print 'Pinning project %s' % self.rel_path
self._ResetProject(commit_hash)
else:
cros_build_lib.Debug('Skipping project %s, already pinned' %
self.rel_path)
def UploadSymbol(sym_file,
upload_url,
file_limit=DEFAULT_FILE_LIMIT,
sleep=0,
num_errors=None):
"""Upload |sym_file| to |upload_url|
Args:
sym_file: The full path to the breakpad symbol to upload
upload_url: The crash server to upload things to
file_limit: The max file size of a symbol file before we try to strip it
sleep: Number of seconds to sleep before running
num_errors: An object to update with the error count (needs a .value member)
Returns:
The number of errors that were encountered.
"""
if num_errors is None:
num_errors = ctypes.c_int()
elif num_errors.value > MAX_TOTAL_ERRORS_FOR_RETRY:
# Abandon ship! It's on fire! NOoooooooooooOOOoooooo.
return 0
upload_file = sym_file
if sleep:
# Keeps us from DoS-ing the symbol server.
time.sleep(sleep)
cros_build_lib.Debug('uploading %s' % sym_file)
# Ideally there'd be a tempfile.SpooledNamedTemporaryFile that we could use.
with tempfile.NamedTemporaryFile(prefix='upload_symbols',
bufsize=0) as temp_sym_file:
if file_limit:
# If the symbols size is too big, strip out the call frame info. The CFI
# is unnecessary for 32bit x86 targets where the frame pointer is used (as
# all of ours have) and it accounts for over half the size of the symbols
# uploaded.
file_size = os.path.getsize(sym_file)
if file_size > file_limit:
cros_build_lib.Warning(
'stripping CFI from %s due to size %s > %s', sym_file,
file_size, file_limit)
temp_sym_file.writelines([
x for x in open(sym_file, 'rb').readlines()
if not x.startswith('STACK CFI')
])
upload_file = temp_sym_file.name
# Hopefully the crash server will let it through. But it probably won't.
# Not sure what the best answer is in this case.
file_size = os.path.getsize(upload_file)
if file_size > CRASH_SERVER_FILE_LIMIT:
cros_build_lib.PrintBuildbotStepWarnings()
cros_build_lib.Error(
'upload file %s is awfully large, risking rejection '
'by symbol server (%s > %s)', sym_file, file_size,
CRASH_SERVER_FILE_LIMIT)
num_errors.value += 1
# Upload the symbol file.
try:
cros_build_lib.RetryCommand(SymUpload,
MAX_RETRIES,
upload_file,
upload_url,
sleep=INITIAL_RETRY_DELAY)
cros_build_lib.Info('successfully uploaded %10i bytes: %s',
file_size, os.path.basename(sym_file))
except cros_build_lib.RunCommandError as e:
cros_build_lib.Warning(
'could not upload: %s:\n{stdout} %s\n{stderr} %s',
os.path.basename(sym_file), e.result.output, e.result.error)
num_errors.value += 1
return num_errors.value
def __init__(self, options):
cros.CrosCommand.__init__(self, options)
self.chroot_update = options.chroot_update and options.deps
if options.chroot_update and not options.deps:
cros_build_lib.Debug('Skipping chroot update due to --nodeps')
def GenerateBreakpadSymbols(board,
breakpad_dir=None,
strip_cfi=False,
generate_count=None,
sysroot=None):
"""Generate all the symbols for this board
TODO(build):
This should be merged with buildbot_commands.GenerateBreakpadSymbols()
once we rewrite cros_generate_breakpad_symbols in python.
Args:
board: The board whose symbols we wish to generate
breakpad_dir: The full path to the breakpad directory where symbols live
strip_cfi: Do not generate CFI data
generate_count: If set, only generate this many symbols (meant for testing)
sysroot: The root where to find the corresponding ELFs
Returns:
The number of errors that were encountered.
"""
if breakpad_dir is None:
breakpad_dir = FindBreakpadDir(board)
if sysroot is None:
sysroot = os.path.join('/build', board)
# Make sure non-root can write out symbols as needed.
osutils.SafeMakedirs(breakpad_dir, sudo=True)
if not os.access(breakpad_dir, os.W_OK):
cros_build_lib.SudoRunCommand(
['chown', '-R', str(os.getuid()), breakpad_dir])
debug_dir = FindDebugDir(board)
cros_build_lib.Info('generating all breakpad symbol files using %s',
debug_dir)
# Let's locate all the debug_files first and their size. This way we can
# start processing the largest files first in parallel with the small ones.
debug_files = []
for root, _, files in os.walk(debug_dir):
for debug_file in files:
debug_file = os.path.join(root, debug_file)
if debug_file.endswith('.ko.debug'):
cros_build_lib.Debug('Skipping kernel module %s', debug_file)
elif debug_file.endswith('.debug'):
if os.path.islink(debug_file):
# The build-id stuff is common enough to filter out by default.
if '/.build-id/' in debug_file:
msg = cros_build_lib.Debug
else:
msg = cros_build_lib.Warning
msg('Skipping symbolic link %s', debug_file)
else:
debug_files.append(
(os.path.getsize(debug_file), debug_file))
# Now start generating symbols for all the inputs.
bg_errors = multiprocessing.Value('i')
with parallel.BackgroundTaskRunner(GenerateBreakpadSymbol,
breakpad_dir=breakpad_dir,
board=board,
strip_cfi=strip_cfi,
num_errors=bg_errors) as queue:
for _, debug_file in sorted(debug_files, reverse=True):
if generate_count == 0:
break
# Turn /build/$BOARD/usr/lib/debug/sbin/foo.debug into
# /build/$BOARD/sbin/foo.
elf_file = os.path.join(sysroot, debug_file[len(debug_dir) + 1:-6])
if not os.path.exists(elf_file):
# Sometimes we filter out programs from /usr/bin but leave behind
# the .debug file.
cros_build_lib.Warning('Skipping missing %s', elf_file)
continue
queue.put([elf_file, debug_file])
if generate_count is not None:
generate_count -= 1
if generate_count == 0:
break
return bg_errors.value