def main(argv): parser = commandline.ArgumentParser(description=__doc__) # TODO: Make sym_paths, breakpad_root, and root exclusive. parser.add_argument('sym_paths', type='path_or_uri', nargs='*', default=None, help='symbol file or directory or URL or tarball') parser.add_argument('--board', default=None, help='Used to find default breakpad_root.') parser.add_argument('--breakpad_root', type='path', default=None, help='full path to the breakpad symbol directory') parser.add_argument('--root', type='path', default=None, help='full path to the chroot dir') parser.add_argument('--official_build', action='store_true', default=False, help='point to official symbol server') parser.add_argument('--server', type=str, default=None, help='URI for custom symbol server') parser.add_argument('--regenerate', action='store_true', default=False, help='regenerate all symbols') parser.add_argument('--upload-limit', type=int, help='only upload # number of symbols') parser.add_argument('--strip_cfi', type=int, default=DEFAULT_FILE_LIMIT, help='strip CFI data for files above this size') parser.add_argument('--failed-list', type='path', help='where to save a list of failed symbols') parser.add_argument('--dedupe', action='store_true', default=False, help='use the swarming service to avoid re-uploading') parser.add_argument('--yes', action='store_true', default=False, help='answer yes to all prompts') parser.add_argument('--product_name', type=str, default='ChromeOS', help='Produce Name for breakpad stats.') opts = parser.parse_args(argv) opts.Freeze() # Figure out the symbol files/directories to upload. if opts.sym_paths: sym_paths = opts.sym_paths elif opts.breakpad_root: sym_paths = [opts.breakpad_root] elif opts.root: if not opts.board: raise ValueError('--board must be set if --root is used.') breakpad_dir = cros_generate_breakpad_symbols.FindBreakpadDir( opts.board) sym_paths = [os.path.join(opts.root, breakpad_dir.lstrip('/'))] else: raise ValueError( '--sym_paths, --breakpad_root, or --root must be set.') if opts.sym_paths or opts.breakpad_root: if opts.regenerate: cros_build_lib.Die( '--regenerate may not be used with specific files, ' 'or breakpad_root') else: if opts.board is None: cros_build_lib.Die('--board is required') # Figure out the dedupe namespace. dedupe_namespace = None if opts.dedupe: if opts.official_build: dedupe_namespace = OFFICIAL_DEDUPE_NAMESPACE_TMPL % opts.product_name else: dedupe_namespace = STAGING_DEDUPE_NAMESPACE_TMPL % opts.product_name # Figure out which crash server to upload too. upload_url = opts.server if not upload_url: if opts.official_build: upload_url = OFFICIAL_UPLOAD_URL else: logging.warning('unofficial builds upload to the staging server') upload_url = STAGING_UPLOAD_URL # Confirm we really want the long upload. if not opts.yes: prolog = '\n'.join( textwrap.wrap( textwrap.dedent(""" Uploading symbols for an entire Chromium OS build is really only necessary for release builds and in a few cases for developers to debug problems. It will take considerable time to run. For developer debugging purposes, consider instead passing specific files to upload. """), 80)).strip() if not cros_build_lib.BooleanPrompt( prompt='Are you sure you want to upload all build symbols', default=False, prolog=prolog): cros_build_lib.Die('better safe than sorry') ret = 0 # Regenerate symbols from binaries. if opts.regenerate: ret += cros_generate_breakpad_symbols.GenerateBreakpadSymbols( opts.board, breakpad_dir=opts.breakpad_root) # Do the upload. ret += UploadSymbols(sym_paths=sym_paths, upload_url=upload_url, product_name=opts.product_name, dedupe_namespace=dedupe_namespace, failed_list=opts.failed_list, upload_limit=opts.upload_limit, strip_cfi=opts.strip_cfi) if ret: logging.error('encountered %i problem(s)', ret) # Since exit(status) gets masked, clamp it to 1 so we don't inadvertently # return 0 in case we are a multiple of the mask. return 1
def UploadSymbols(board=None, official=False, breakpad_dir=None, file_limit=DEFAULT_FILE_LIMIT, sleep=DEFAULT_SLEEP_DELAY, upload_count=None, sym_files=None): """Upload all the generated symbols for |board| to the crash server You can use in a few ways: * pass |board| to locate all of its symbols * pass |breakpad_dir| to upload all the symbols in there * pass |sym_files| to upload specific symbols Args: board: The board whose symbols we wish to upload official: Use the official symbol server rather than the staging one breakpad_dir: The full path to the breakpad directory where symbols live file_limit: The max file size of a symbol file before we try to strip it sleep: How long to sleep in between uploads upload_count: If set, only upload this many symbols (meant for testing) sym_files: Specific symbol files to upload, otherwise search |breakpad_dir| Returns: The number of errors that were encountered. """ num_errors = 0 if official: upload_url = OFFICIAL_UPLOAD_URL else: cros_build_lib.Warning( 'unofficial builds upload to the staging server') upload_url = STAGING_UPLOAD_URL if sym_files: cros_build_lib.Info('uploading specified symbol files to %s', upload_url) sym_file_sets = [('', '', sym_files)] all_files = True else: if breakpad_dir is None: breakpad_dir = cros_generate_breakpad_symbols.FindBreakpadDir( board) cros_build_lib.Info('uploading all symbols to %s from %s', upload_url, breakpad_dir) sym_file_sets = os.walk(breakpad_dir) all_files = False # We need to limit ourselves to one upload at a time to avoid the server # kicking in DoS protection. See these bugs for more details: # http://crbug.com/209442 # http://crbug.com/212496 bg_errors = multiprocessing.Value('i') with parallel.BackgroundTaskRunner(UploadSymbol, file_limit=file_limit, sleep=sleep, num_errors=bg_errors, processes=1) as queue: for root, _, files in sym_file_sets: if upload_count == 0: break for sym_file in files: if all_files or sym_file.endswith('.sym'): sym_file = os.path.join(root, sym_file) queue.put([sym_file, upload_url]) if upload_count is not None: upload_count -= 1 if upload_count == 0: break num_errors += bg_errors.value return num_errors
def testBreakpadDir(self): """Make sure board->breakpad path expansion works""" expected = '/build/blah/usr/lib/debug/breakpad' result = cros_generate_breakpad_symbols.FindBreakpadDir('blah') self.assertEquals(expected, result)
def UploadSymbols(board=None, official=False, server=None, breakpad_dir=None, file_limit=DEFAULT_FILE_LIMIT, sleep=DEFAULT_SLEEP_DELAY, upload_limit=None, sym_paths=None, failed_list=None, root=None, retry=True, dedupe_namespace=None, product_name='ChromeOS'): """Upload all the generated symbols for |board| to the crash server You can use in a few ways: * pass |board| to locate all of its symbols * pass |breakpad_dir| to upload all the symbols in there * pass |sym_paths| to upload specific symbols (or dirs of symbols) Args: board: The board whose symbols we wish to upload official: Use the official symbol server rather than the staging one server: Explicit server to post symbols to breakpad_dir: The full path to the breakpad directory where symbols live file_limit: The max file size of a symbol file before we try to strip it sleep: How long to sleep in between uploads upload_limit: If set, only upload this many symbols (meant for testing) sym_paths: Specific symbol files (or dirs of sym files) to upload, otherwise search |breakpad_dir| failed_list: Write the names of all sym files we did not upload; can be a filename or file-like object. root: The tree to prefix to |breakpad_dir| (if |breakpad_dir| is not set) retry: Whether we should retry failures. dedupe_namespace: The isolateserver namespace to dedupe uploaded symbols. product_name: A string for stats purposes. Usually 'ChromeOS' or 'Android'. Returns: The number of errors that were encountered. """ if server is None: if official: upload_url = OFFICIAL_UPLOAD_URL else: logging.warning('unofficial builds upload to the staging server') upload_url = STAGING_UPLOAD_URL else: upload_url = server if sym_paths: logging.info('uploading specified symbols to %s', upload_url) else: if breakpad_dir is None: if root is None: raise ValueError('breakpad_dir requires root to be set') breakpad_dir = os.path.join( root, cros_generate_breakpad_symbols.FindBreakpadDir(board).lstrip( '/')) logging.info('uploading all symbols to %s from %s', upload_url, breakpad_dir) sym_paths = [breakpad_dir] # We use storage_query to ask the server about existing symbols. The # storage_notify_proc process is used to post updates to the server. We # cannot safely share the storage object between threads/processes, but # we also want to minimize creating new ones as each object has to init # new state (like server connections). storage_query = None if dedupe_namespace: dedupe_limit = DEDUPE_LIMIT dedupe_queue = multiprocessing.Queue() try: with timeout_util.Timeout(DEDUPE_TIMEOUT): storage_query = isolateserver.get_storage_api( constants.ISOLATESERVER, dedupe_namespace) except Exception: logging.warning('initializing dedupe server connection failed', exc_info=True) else: dedupe_limit = 1 dedupe_queue = None # Can't use parallel.BackgroundTaskRunner because that'll create multiple # processes and we want only one the whole time (see comment above). storage_notify_proc = multiprocessing.Process( target=SymbolDeduplicatorNotify, args=(dedupe_namespace, dedupe_queue)) bg_errors = multiprocessing.Value('i') watermark_errors = multiprocessing.Value('f') failed_queue = multiprocessing.Queue() uploader = functools.partial(UploadSymbol, upload_url, product_name=product_name, file_limit=file_limit, sleep=sleep, num_errors=bg_errors, watermark_errors=watermark_errors, failed_queue=failed_queue, passed_queue=dedupe_queue) start_time = datetime.datetime.now() Counters = cros_build_lib.Collection('Counters', upload_limit=upload_limit, uploaded_count=0, deduped_count=0) counters = Counters() def _Upload(queue, counters, files): if not files: return missing_count = 0 for item in SymbolDeduplicator(storage_query, files): missing_count += 1 if counters.upload_limit == 0: continue queue.put((item, )) counters.uploaded_count += 1 if counters.upload_limit is not None: counters.upload_limit -= 1 counters.deduped_count += (len(files) - missing_count) try: storage_notify_proc.start() with osutils.TempDir(prefix='upload_symbols.') as tempdir: # For the first run, we collect the symbols that failed. If the # overall failure rate was low, we'll retry them on the second run. for retry in (retry, False): # We need to limit ourselves to one upload at a time to avoid the server # kicking in DoS protection. See these bugs for more details: # http://crbug.com/209442 # http://crbug.com/212496 with parallel.BackgroundTaskRunner(uploader, processes=1) as queue: dedupe_list = [] for sym_file in SymbolFinder(tempdir, sym_paths): dedupe_list.append(sym_file) dedupe_len = len(dedupe_list) if dedupe_len < dedupe_limit: if (counters.upload_limit is None or dedupe_len < counters.upload_limit): continue # We check the counter before _Upload so that we don't keep talking # to the dedupe server. Otherwise, we end up sending one symbol at # a time to it and that slows things down a lot. if counters.upload_limit == 0: break _Upload(queue, counters, dedupe_list) dedupe_list = [] _Upload(queue, counters, dedupe_list) # See if we need to retry, and if we haven't failed too many times yet. if not retry or ErrorLimitHit(bg_errors, watermark_errors): break sym_paths = [] failed_queue.put(None) while True: sym_path = failed_queue.get() if sym_path is None: break sym_paths.append(sym_path) if sym_paths: logging.warning('retrying %i symbols', len(sym_paths)) if counters.upload_limit is not None: counters.upload_limit += len(sym_paths) # Decrement the error count in case we recover in the second pass. assert bg_errors.value >= len(sym_paths), \ 'more failed files than errors?' bg_errors.value -= len(sym_paths) else: # No failed symbols, so just return now. break # If the user has requested it, save all the symbol files that we failed to # upload to a listing file. This should help with recovery efforts later. failed_queue.put(None) WriteQueueToFile(failed_list, failed_queue, breakpad_dir) finally: logging.info('finished uploading; joining background process') if dedupe_queue: dedupe_queue.put(None) # The notification might be slow going, so give it some time to finish. # We have to poll here as the process monitor is watching for output and # will kill us if we go silent for too long. wait_minutes = DEDUPE_NOTIFY_TIMEOUT while storage_notify_proc.is_alive() and wait_minutes > 0: if dedupe_queue: qsize = str(dedupe_queue.qsize()) else: qsize = '[None]' logging.info('waiting up to %i minutes for ~%s notifications', wait_minutes, qsize) storage_notify_proc.join(60) wait_minutes -= 1 # The process is taking too long, so kill it and complain. if storage_notify_proc.is_alive(): logging.warning('notification process took too long') logging.PrintBuildbotStepWarnings() # Kill it gracefully first (traceback) before tacking it down harder. pid = storage_notify_proc.pid for sig in (signal.SIGINT, signal.SIGTERM, signal.SIGKILL): logging.warning('sending %s to %i', signals.StrSignal(sig), pid) # The process might have exited between the last check and the # actual kill below, so ignore ESRCH errors. try: os.kill(pid, sig) except OSError as e: if e.errno == errno.ESRCH: break else: raise time.sleep(5) if not storage_notify_proc.is_alive(): break # Drain the queue so we don't hang when we finish. try: logging.warning('draining the notify queue manually') with timeout_util.Timeout(60): try: while dedupe_queue.get_nowait(): pass except Queue.Empty: pass except timeout_util.TimeoutError: logging.warning( 'draining the notify queue failed; trashing it') dedupe_queue.cancel_join_thread() logging.info('uploaded %i symbols (%i were deduped) which took: %s', counters.uploaded_count, counters.deduped_count, datetime.datetime.now() - start_time) return bg_errors.value