def main(argv):
parser = commandline.ArgumentParser(description=__doc__)
# TODO: Make sym_paths, breakpad_root, and root exclusive.
parser.add_argument('sym_paths',
type='path_or_uri',
nargs='*',
default=None,
help='symbol file or directory or URL or tarball')
parser.add_argument('--board',
default=None,
help='Used to find default breakpad_root.')
parser.add_argument('--breakpad_root',
type='path',
default=None,
help='full path to the breakpad symbol directory')
parser.add_argument('--root',
type='path',
default=None,
help='full path to the chroot dir')
parser.add_argument('--official_build',
action='store_true',
default=False,
help='point to official symbol server')
parser.add_argument('--server',
type=str,
default=None,
help='URI for custom symbol server')
parser.add_argument('--regenerate',
action='store_true',
default=False,
help='regenerate all symbols')
parser.add_argument('--upload-limit',
type=int,
help='only upload # number of symbols')
parser.add_argument('--strip_cfi',
type=int,
default=DEFAULT_FILE_LIMIT,
help='strip CFI data for files above this size')
parser.add_argument('--failed-list',
type='path',
help='where to save a list of failed symbols')
parser.add_argument('--dedupe',
action='store_true',
default=False,
help='use the swarming service to avoid re-uploading')
parser.add_argument('--yes',
action='store_true',
default=False,
help='answer yes to all prompts')
parser.add_argument('--product_name',
type=str,
default='ChromeOS',
help='Produce Name for breakpad stats.')
opts = parser.parse_args(argv)
opts.Freeze()
# Figure out the symbol files/directories to upload.
if opts.sym_paths:
sym_paths = opts.sym_paths
elif opts.breakpad_root:
sym_paths = [opts.breakpad_root]
elif opts.root:
if not opts.board:
raise ValueError('--board must be set if --root is used.')
breakpad_dir = cros_generate_breakpad_symbols.FindBreakpadDir(
opts.board)
sym_paths = [os.path.join(opts.root, breakpad_dir.lstrip('/'))]
else:
raise ValueError(
'--sym_paths, --breakpad_root, or --root must be set.')
if opts.sym_paths or opts.breakpad_root:
if opts.regenerate:
cros_build_lib.Die(
'--regenerate may not be used with specific files, '
'or breakpad_root')
else:
if opts.board is None:
cros_build_lib.Die('--board is required')
# Figure out the dedupe namespace.
dedupe_namespace = None
if opts.dedupe:
if opts.official_build:
dedupe_namespace = OFFICIAL_DEDUPE_NAMESPACE_TMPL % opts.product_name
else:
dedupe_namespace = STAGING_DEDUPE_NAMESPACE_TMPL % opts.product_name
# Figure out which crash server to upload too.
upload_url = opts.server
if not upload_url:
if opts.official_build:
upload_url = OFFICIAL_UPLOAD_URL
else:
logging.warning('unofficial builds upload to the staging server')
upload_url = STAGING_UPLOAD_URL
# Confirm we really want the long upload.
if not opts.yes:
prolog = '\n'.join(
textwrap.wrap(
textwrap.dedent("""
Uploading symbols for an entire Chromium OS build is really only
necessary for release builds and in a few cases for developers
to debug problems. It will take considerable time to run. For
developer debugging purposes, consider instead passing specific
files to upload.
"""), 80)).strip()
if not cros_build_lib.BooleanPrompt(
prompt='Are you sure you want to upload all build symbols',
default=False,
prolog=prolog):
cros_build_lib.Die('better safe than sorry')
ret = 0
# Regenerate symbols from binaries.
if opts.regenerate:
ret += cros_generate_breakpad_symbols.GenerateBreakpadSymbols(
opts.board, breakpad_dir=opts.breakpad_root)
# Do the upload.
ret += UploadSymbols(sym_paths=sym_paths,
upload_url=upload_url,
product_name=opts.product_name,
dedupe_namespace=dedupe_namespace,
failed_list=opts.failed_list,
upload_limit=opts.upload_limit,
strip_cfi=opts.strip_cfi)
if ret:
logging.error('encountered %i problem(s)', ret)
# Since exit(status) gets masked, clamp it to 1 so we don't inadvertently
# return 0 in case we are a multiple of the mask.
return 1
def UploadSymbols(board=None,
official=False,
breakpad_dir=None,
file_limit=DEFAULT_FILE_LIMIT,
sleep=DEFAULT_SLEEP_DELAY,
upload_count=None,
sym_files=None):
"""Upload all the generated symbols for |board| to the crash server
You can use in a few ways:
* pass |board| to locate all of its symbols
* pass |breakpad_dir| to upload all the symbols in there
* pass |sym_files| to upload specific symbols
Args:
board: The board whose symbols we wish to upload
official: Use the official symbol server rather than the staging one
breakpad_dir: The full path to the breakpad directory where symbols live
file_limit: The max file size of a symbol file before we try to strip it
sleep: How long to sleep in between uploads
upload_count: If set, only upload this many symbols (meant for testing)
sym_files: Specific symbol files to upload, otherwise search |breakpad_dir|
Returns:
The number of errors that were encountered.
"""
num_errors = 0
if official:
upload_url = OFFICIAL_UPLOAD_URL
else:
cros_build_lib.Warning(
'unofficial builds upload to the staging server')
upload_url = STAGING_UPLOAD_URL
if sym_files:
cros_build_lib.Info('uploading specified symbol files to %s',
upload_url)
sym_file_sets = [('', '', sym_files)]
all_files = True
else:
if breakpad_dir is None:
breakpad_dir = cros_generate_breakpad_symbols.FindBreakpadDir(
board)
cros_build_lib.Info('uploading all symbols to %s from %s', upload_url,
breakpad_dir)
sym_file_sets = os.walk(breakpad_dir)
all_files = False
# We need to limit ourselves to one upload at a time to avoid the server
# kicking in DoS protection. See these bugs for more details:
# http://crbug.com/209442
# http://crbug.com/212496
bg_errors = multiprocessing.Value('i')
with parallel.BackgroundTaskRunner(UploadSymbol,
file_limit=file_limit,
sleep=sleep,
num_errors=bg_errors,
processes=1) as queue:
for root, _, files in sym_file_sets:
if upload_count == 0:
break
for sym_file in files:
if all_files or sym_file.endswith('.sym'):
sym_file = os.path.join(root, sym_file)
queue.put([sym_file, upload_url])
if upload_count is not None:
upload_count -= 1
if upload_count == 0:
break
num_errors += bg_errors.value
return num_errors
def testBreakpadDir(self):
"""Make sure board->breakpad path expansion works"""
expected = '/build/blah/usr/lib/debug/breakpad'
result = cros_generate_breakpad_symbols.FindBreakpadDir('blah')
self.assertEquals(expected, result)
def UploadSymbols(board=None,
official=False,
server=None,
breakpad_dir=None,
file_limit=DEFAULT_FILE_LIMIT,
sleep=DEFAULT_SLEEP_DELAY,
upload_limit=None,
sym_paths=None,
failed_list=None,
root=None,
retry=True,
dedupe_namespace=None,
product_name='ChromeOS'):
"""Upload all the generated symbols for |board| to the crash server
You can use in a few ways:
* pass |board| to locate all of its symbols
* pass |breakpad_dir| to upload all the symbols in there
* pass |sym_paths| to upload specific symbols (or dirs of symbols)
Args:
board: The board whose symbols we wish to upload
official: Use the official symbol server rather than the staging one
server: Explicit server to post symbols to
breakpad_dir: The full path to the breakpad directory where symbols live
file_limit: The max file size of a symbol file before we try to strip it
sleep: How long to sleep in between uploads
upload_limit: If set, only upload this many symbols (meant for testing)
sym_paths: Specific symbol files (or dirs of sym files) to upload,
otherwise search |breakpad_dir|
failed_list: Write the names of all sym files we did not upload; can be a
filename or file-like object.
root: The tree to prefix to |breakpad_dir| (if |breakpad_dir| is not set)
retry: Whether we should retry failures.
dedupe_namespace: The isolateserver namespace to dedupe uploaded symbols.
product_name: A string for stats purposes. Usually 'ChromeOS' or 'Android'.
Returns:
The number of errors that were encountered.
"""
if server is None:
if official:
upload_url = OFFICIAL_UPLOAD_URL
else:
logging.warning('unofficial builds upload to the staging server')
upload_url = STAGING_UPLOAD_URL
else:
upload_url = server
if sym_paths:
logging.info('uploading specified symbols to %s', upload_url)
else:
if breakpad_dir is None:
if root is None:
raise ValueError('breakpad_dir requires root to be set')
breakpad_dir = os.path.join(
root,
cros_generate_breakpad_symbols.FindBreakpadDir(board).lstrip(
'/'))
logging.info('uploading all symbols to %s from %s', upload_url,
breakpad_dir)
sym_paths = [breakpad_dir]
# We use storage_query to ask the server about existing symbols. The
# storage_notify_proc process is used to post updates to the server. We
# cannot safely share the storage object between threads/processes, but
# we also want to minimize creating new ones as each object has to init
# new state (like server connections).
storage_query = None
if dedupe_namespace:
dedupe_limit = DEDUPE_LIMIT
dedupe_queue = multiprocessing.Queue()
try:
with timeout_util.Timeout(DEDUPE_TIMEOUT):
storage_query = isolateserver.get_storage_api(
constants.ISOLATESERVER, dedupe_namespace)
except Exception:
logging.warning('initializing dedupe server connection failed',
exc_info=True)
else:
dedupe_limit = 1
dedupe_queue = None
# Can't use parallel.BackgroundTaskRunner because that'll create multiple
# processes and we want only one the whole time (see comment above).
storage_notify_proc = multiprocessing.Process(
target=SymbolDeduplicatorNotify, args=(dedupe_namespace, dedupe_queue))
bg_errors = multiprocessing.Value('i')
watermark_errors = multiprocessing.Value('f')
failed_queue = multiprocessing.Queue()
uploader = functools.partial(UploadSymbol,
upload_url,
product_name=product_name,
file_limit=file_limit,
sleep=sleep,
num_errors=bg_errors,
watermark_errors=watermark_errors,
failed_queue=failed_queue,
passed_queue=dedupe_queue)
start_time = datetime.datetime.now()
Counters = cros_build_lib.Collection('Counters',
upload_limit=upload_limit,
uploaded_count=0,
deduped_count=0)
counters = Counters()
def _Upload(queue, counters, files):
if not files:
return
missing_count = 0
for item in SymbolDeduplicator(storage_query, files):
missing_count += 1
if counters.upload_limit == 0:
continue
queue.put((item, ))
counters.uploaded_count += 1
if counters.upload_limit is not None:
counters.upload_limit -= 1
counters.deduped_count += (len(files) - missing_count)
try:
storage_notify_proc.start()
with osutils.TempDir(prefix='upload_symbols.') as tempdir:
# For the first run, we collect the symbols that failed. If the
# overall failure rate was low, we'll retry them on the second run.
for retry in (retry, False):
# We need to limit ourselves to one upload at a time to avoid the server
# kicking in DoS protection. See these bugs for more details:
# http://crbug.com/209442
# http://crbug.com/212496
with parallel.BackgroundTaskRunner(uploader,
processes=1) as queue:
dedupe_list = []
for sym_file in SymbolFinder(tempdir, sym_paths):
dedupe_list.append(sym_file)
dedupe_len = len(dedupe_list)
if dedupe_len < dedupe_limit:
if (counters.upload_limit is None
or dedupe_len < counters.upload_limit):
continue
# We check the counter before _Upload so that we don't keep talking
# to the dedupe server. Otherwise, we end up sending one symbol at
# a time to it and that slows things down a lot.
if counters.upload_limit == 0:
break
_Upload(queue, counters, dedupe_list)
dedupe_list = []
_Upload(queue, counters, dedupe_list)
# See if we need to retry, and if we haven't failed too many times yet.
if not retry or ErrorLimitHit(bg_errors, watermark_errors):
break
sym_paths = []
failed_queue.put(None)
while True:
sym_path = failed_queue.get()
if sym_path is None:
break
sym_paths.append(sym_path)
if sym_paths:
logging.warning('retrying %i symbols', len(sym_paths))
if counters.upload_limit is not None:
counters.upload_limit += len(sym_paths)
# Decrement the error count in case we recover in the second pass.
assert bg_errors.value >= len(sym_paths), \
'more failed files than errors?'
bg_errors.value -= len(sym_paths)
else:
# No failed symbols, so just return now.
break
# If the user has requested it, save all the symbol files that we failed to
# upload to a listing file. This should help with recovery efforts later.
failed_queue.put(None)
WriteQueueToFile(failed_list, failed_queue, breakpad_dir)
finally:
logging.info('finished uploading; joining background process')
if dedupe_queue:
dedupe_queue.put(None)
# The notification might be slow going, so give it some time to finish.
# We have to poll here as the process monitor is watching for output and
# will kill us if we go silent for too long.
wait_minutes = DEDUPE_NOTIFY_TIMEOUT
while storage_notify_proc.is_alive() and wait_minutes > 0:
if dedupe_queue:
qsize = str(dedupe_queue.qsize())
else:
qsize = '[None]'
logging.info('waiting up to %i minutes for ~%s notifications',
wait_minutes, qsize)
storage_notify_proc.join(60)
wait_minutes -= 1
# The process is taking too long, so kill it and complain.
if storage_notify_proc.is_alive():
logging.warning('notification process took too long')
logging.PrintBuildbotStepWarnings()
# Kill it gracefully first (traceback) before tacking it down harder.
pid = storage_notify_proc.pid
for sig in (signal.SIGINT, signal.SIGTERM, signal.SIGKILL):
logging.warning('sending %s to %i', signals.StrSignal(sig),
pid)
# The process might have exited between the last check and the
# actual kill below, so ignore ESRCH errors.
try:
os.kill(pid, sig)
except OSError as e:
if e.errno == errno.ESRCH:
break
else:
raise
time.sleep(5)
if not storage_notify_proc.is_alive():
break
# Drain the queue so we don't hang when we finish.
try:
logging.warning('draining the notify queue manually')
with timeout_util.Timeout(60):
try:
while dedupe_queue.get_nowait():
pass
except Queue.Empty:
pass
except timeout_util.TimeoutError:
logging.warning(
'draining the notify queue failed; trashing it')
dedupe_queue.cancel_join_thread()
logging.info('uploaded %i symbols (%i were deduped) which took: %s',
counters.uploaded_count, counters.deduped_count,
datetime.datetime.now() - start_time)
return bg_errors.value