else:
return NormalizeLocalOrGSPath(value)
# A Device object holds information parsed from the command line input:
# scheme: DEVICE_SCHEME_SSH, DEVICE_SCHEME_USB, or DEVICE_SCHEME_FILE.
# username: String SSH username or None.
# hostname: String SSH hostname or None.
# port: Int SSH port or None.
# path: String USB/file path or None.
# raw: String raw input from the command line.
# For now this is a superset of all information for USB, SSH, or file devices.
# If functionality diverges based on type, it may be useful to split this into
# separate device classes instead.
Device = cros_build_lib.Collection(
'Device', scheme=None, username=None, hostname=None, port=None, path=None,
raw=None)
class DeviceParser(object):
"""Parses devices as an argparse argument type.
In addition to parsing user input, this class will also ensure that only
supported device schemes are accepted by the parser. For example,
`cros deploy` only makes sense with an SSH device, but `cros flash` can use
SSH, USB, or file device schemes.
If the device input is malformed or the scheme is wrong, an error message will
be printed and the program will exit.
Valid device inputs are:
os.path.basename(sym_file), e)
finally:
if success:
_UpdateCounter(watermark_errors, ERROR_ADJUST_PASS)
else:
_UpdateCounter(num_errors, 1)
_UpdateCounter(watermark_errors, ERROR_ADJUST_FAIL)
if failed_queue:
failed_queue.put(sym_file)
return num_errors.value
# A dummy class that allows for stubbing in tests and SymUpload.
FakeItem = cros_build_lib.Collection('FakeItem',
sym_file=None,
sym_header=None,
content=lambda x: '')
class SymbolItem(isolateserver.BufferItem):
"""Turn a sym_file into an isolateserver.Item"""
ALGO = hashlib.sha1
def __init__(self, sym_file):
sym_header = cros_generate_breakpad_symbols.ReadSymsHeader(sym_file)
super(SymbolItem, self).__init__(str(sym_header), self.ALGO)
self.sym_header = sym_header
self.sym_file = sym_file
def UploadSymbols(board=None,
official=False,
server=None,
breakpad_dir=None,
file_limit=DEFAULT_FILE_LIMIT,
sleep=DEFAULT_SLEEP_DELAY,
upload_limit=None,
sym_paths=None,
failed_list=None,
root=None,
retry=True,
dedupe_namespace=None,
product_name='ChromeOS'):
"""Upload all the generated symbols for |board| to the crash server
You can use in a few ways:
* pass |board| to locate all of its symbols
* pass |breakpad_dir| to upload all the symbols in there
* pass |sym_paths| to upload specific symbols (or dirs of symbols)
Args:
board: The board whose symbols we wish to upload
official: Use the official symbol server rather than the staging one
server: Explicit server to post symbols to
breakpad_dir: The full path to the breakpad directory where symbols live
file_limit: The max file size of a symbol file before we try to strip it
sleep: How long to sleep in between uploads
upload_limit: If set, only upload this many symbols (meant for testing)
sym_paths: Specific symbol files (or dirs of sym files) to upload,
otherwise search |breakpad_dir|
failed_list: Write the names of all sym files we did not upload; can be a
filename or file-like object.
root: The tree to prefix to |breakpad_dir| (if |breakpad_dir| is not set)
retry: Whether we should retry failures.
dedupe_namespace: The isolateserver namespace to dedupe uploaded symbols.
product_name: A string for stats purposes. Usually 'ChromeOS' or 'Android'.
Returns:
The number of errors that were encountered.
"""
if server is None:
if official:
upload_url = OFFICIAL_UPLOAD_URL
else:
logging.warning('unofficial builds upload to the staging server')
upload_url = STAGING_UPLOAD_URL
else:
upload_url = server
if sym_paths:
logging.info('uploading specified symbols to %s', upload_url)
else:
if breakpad_dir is None:
if root is None:
raise ValueError('breakpad_dir requires root to be set')
breakpad_dir = os.path.join(
root,
cros_generate_breakpad_symbols.FindBreakpadDir(board).lstrip(
'/'))
logging.info('uploading all symbols to %s from %s', upload_url,
breakpad_dir)
sym_paths = [breakpad_dir]
# We use storage_query to ask the server about existing symbols. The
# storage_notify_proc process is used to post updates to the server. We
# cannot safely share the storage object between threads/processes, but
# we also want to minimize creating new ones as each object has to init
# new state (like server connections).
storage_query = None
if dedupe_namespace:
dedupe_limit = DEDUPE_LIMIT
dedupe_queue = multiprocessing.Queue()
try:
with timeout_util.Timeout(DEDUPE_TIMEOUT):
storage_query = isolateserver.get_storage_api(
constants.ISOLATESERVER, dedupe_namespace)
except Exception:
logging.warning('initializing dedupe server connection failed',
exc_info=True)
else:
dedupe_limit = 1
dedupe_queue = None
# Can't use parallel.BackgroundTaskRunner because that'll create multiple
# processes and we want only one the whole time (see comment above).
storage_notify_proc = multiprocessing.Process(
target=SymbolDeduplicatorNotify, args=(dedupe_namespace, dedupe_queue))
bg_errors = multiprocessing.Value('i')
watermark_errors = multiprocessing.Value('f')
failed_queue = multiprocessing.Queue()
uploader = functools.partial(UploadSymbol,
upload_url,
product_name=product_name,
file_limit=file_limit,
sleep=sleep,
num_errors=bg_errors,
watermark_errors=watermark_errors,
failed_queue=failed_queue,
passed_queue=dedupe_queue)
start_time = datetime.datetime.now()
Counters = cros_build_lib.Collection('Counters',
upload_limit=upload_limit,
uploaded_count=0,
deduped_count=0)
counters = Counters()
def _Upload(queue, counters, files):
if not files:
return
missing_count = 0
for item in SymbolDeduplicator(storage_query, files):
missing_count += 1
if counters.upload_limit == 0:
continue
queue.put((item, ))
counters.uploaded_count += 1
if counters.upload_limit is not None:
counters.upload_limit -= 1
counters.deduped_count += (len(files) - missing_count)
try:
storage_notify_proc.start()
with osutils.TempDir(prefix='upload_symbols.') as tempdir:
# For the first run, we collect the symbols that failed. If the
# overall failure rate was low, we'll retry them on the second run.
for retry in (retry, False):
# We need to limit ourselves to one upload at a time to avoid the server
# kicking in DoS protection. See these bugs for more details:
# http://crbug.com/209442
# http://crbug.com/212496
with parallel.BackgroundTaskRunner(uploader,
processes=1) as queue:
dedupe_list = []
for sym_file in SymbolFinder(tempdir, sym_paths):
dedupe_list.append(sym_file)
dedupe_len = len(dedupe_list)
if dedupe_len < dedupe_limit:
if (counters.upload_limit is None
or dedupe_len < counters.upload_limit):
continue
# We check the counter before _Upload so that we don't keep talking
# to the dedupe server. Otherwise, we end up sending one symbol at
# a time to it and that slows things down a lot.
if counters.upload_limit == 0:
break
_Upload(queue, counters, dedupe_list)
dedupe_list = []
_Upload(queue, counters, dedupe_list)
# See if we need to retry, and if we haven't failed too many times yet.
if not retry or ErrorLimitHit(bg_errors, watermark_errors):
break
sym_paths = []
failed_queue.put(None)
while True:
sym_path = failed_queue.get()
if sym_path is None:
break
sym_paths.append(sym_path)
if sym_paths:
logging.warning('retrying %i symbols', len(sym_paths))
if counters.upload_limit is not None:
counters.upload_limit += len(sym_paths)
# Decrement the error count in case we recover in the second pass.
assert bg_errors.value >= len(sym_paths), \
'more failed files than errors?'
bg_errors.value -= len(sym_paths)
else:
# No failed symbols, so just return now.
break
# If the user has requested it, save all the symbol files that we failed to
# upload to a listing file. This should help with recovery efforts later.
failed_queue.put(None)
WriteQueueToFile(failed_list, failed_queue, breakpad_dir)
finally:
logging.info('finished uploading; joining background process')
if dedupe_queue:
dedupe_queue.put(None)
# The notification might be slow going, so give it some time to finish.
# We have to poll here as the process monitor is watching for output and
# will kill us if we go silent for too long.
wait_minutes = DEDUPE_NOTIFY_TIMEOUT
while storage_notify_proc.is_alive() and wait_minutes > 0:
if dedupe_queue:
qsize = str(dedupe_queue.qsize())
else:
qsize = '[None]'
logging.info('waiting up to %i minutes for ~%s notifications',
wait_minutes, qsize)
storage_notify_proc.join(60)
wait_minutes -= 1
# The process is taking too long, so kill it and complain.
if storage_notify_proc.is_alive():
logging.warning('notification process took too long')
logging.PrintBuildbotStepWarnings()
# Kill it gracefully first (traceback) before tacking it down harder.
pid = storage_notify_proc.pid
for sig in (signal.SIGINT, signal.SIGTERM, signal.SIGKILL):
logging.warning('sending %s to %i', signals.StrSignal(sig),
pid)
# The process might have exited between the last check and the
# actual kill below, so ignore ESRCH errors.
try:
os.kill(pid, sig)
except OSError as e:
if e.errno == errno.ESRCH:
break
else:
raise
time.sleep(5)
if not storage_notify_proc.is_alive():
break
# Drain the queue so we don't hang when we finish.
try:
logging.warning('draining the notify queue manually')
with timeout_util.Timeout(60):
try:
while dedupe_queue.get_nowait():
pass
except Queue.Empty:
pass
except timeout_util.TimeoutError:
logging.warning(
'draining the notify queue failed; trashing it')
dedupe_queue.cancel_join_thread()
logging.info('uploaded %i symbols (%i were deduped) which took: %s',
counters.uploaded_count, counters.deduped_count,
datetime.datetime.now() - start_time)
return bg_errors.value
# -*- coding: utf-8 -*-
# Copyright 2017 The Chromium OS Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
"""Baselines for user/group tests."""
from __future__ import print_function
from chromite.lib import cros_build_lib
# firewall:!:236:236:firewall daemon:/dev/null:/bin/false
UserEntry = cros_build_lib.Collection('UserEntry',
user=None,
encpasswd='!',
uid=None,
gid=None,
home='/dev/null',
shell='/bin/false')
# tty:!:5:xorg,power,brltty
GroupEntry = cros_build_lib.Collection('GroupEntry',
group=None,
encpasswd='!',
gid=None,
users=set())
# For users that we allow to login to the system, whitelist a number of
# alternative shells. These are equivalent from a security POV.
_VALID_LOGIN_SHELLS = set((
'/bin/sh',
'/bin/bash',