def testArgs(self):
"""Verify passing of optional args to the destination function."""
cros_build_lib.RunCommand(':', shell=True, print_cmd=False,
error_code_ok=False)
self.mox.ReplayAll()
cros_build_lib.TimedCommand(cros_build_lib.RunCommand, ':', shell=True,
print_cmd=False, error_code_ok=False)
def testLog(self):
"""Verify logging does the right thing."""
self.mox.StubOutWithMock(cros_build_lib.logger, 'log')
cros_build_lib.RunCommand('fun', shell=True)
cros_build_lib.logger.log(mox.IgnoreArg(), 'msg! %s', mox.IgnoreArg())
self.mox.ReplayAll()
cros_build_lib.TimedCommand(cros_build_lib.RunCommand, 'fun',
timed_log_msg='msg! %s', shell=True)
def RunTest(test, cmd, tmpfile, finished, total):
"""Run |test| with the |cmd| line and save output to |tmpfile|.
Args:
test: The human readable name for this test.
cmd: The full command line to run the test.
tmpfile: File to write test output to.
finished: Counter to update when this test finishes running.
total: Total number of tests to run.
Returns:
The exit code of the test.
"""
logging.info('Starting %s', test)
def _Finished(_log_level, _log_msg, result, delta):
with finished.get_lock():
finished.value += 1
if result.returncode:
func = logging.error
msg = 'Failed'
else:
func = logging.info
msg = 'Finished'
func('%s [%i/%i] %s (%s)', msg, finished.value, total, test, delta)
# Save the timing for this test run for future usage.
seconds = delta.total_seconds()
try:
cache = json.load(open(TIMING_CACHE_FILE))
except (IOError, ValueError):
cache = {}
if test in cache:
seconds = (cache[test] + seconds) / 2
cache[test] = seconds
json.dump(cache, open(TIMING_CACHE_FILE, 'w'))
ret = cros_build_lib.TimedCommand(cros_build_lib.RunCommand,
cmd,
capture_output=True,
error_code_ok=True,
combine_stdout_stderr=True,
debug_level=logging.DEBUG,
int_timeout=SIGINT_TIMEOUT,
timed_log_callback=_Finished)
if ret.returncode:
tmpfile.write(ret.output)
if not ret.output:
tmpfile.write('<no output>\n')
tmpfile.close()
return ret.returncode
def PerformSymbolsFileUpload(symbols, upload_url, product_name='ChromeOS'):
"""Upload the symbols to the crash server
Args:
symbols: An iterable of SymbolFiles to be uploaded.
upload_url: URL of crash server to upload too.
failures: Tracker for total upload failures.
product_name: A string for stats purposes. Usually 'ChromeOS' or 'Android'.
Yields:
Each symbol from symbols, perhaps modified.
"""
failures = 0
for s in symbols:
if (failures < MAX_TOTAL_ERRORS_FOR_RETRY
and s.status in (SymbolFile.INITIAL, SymbolFile.ERROR)):
# Keeps us from DoS-ing the symbol server.
time.sleep(SLEEP_DELAY)
logging.info('Uploading symbol_file: %s', s.display_path)
try:
# This command retries the upload multiple times with growing delays. We
# only consider the upload a failure if these retries fail.
cros_build_lib.TimedCommand(
retry_util.RetryException,
(urllib2.HTTPError, urllib2.URLError),
MAX_RETRIES,
UploadSymbolFile,
upload_url,
s,
product_name,
sleep=INITIAL_RETRY_DELAY,
timed_log_msg=('upload of %10i bytes took %%(delta)s' %
s.FileSize()))
s.status = SymbolFile.UPLOADED
except urllib2.HTTPError as e:
logging.warning('could not upload: %s: HTTP %s: %s',
s.display_name, e.code, e.reason)
s.status = SymbolFile.ERROR
failures += 1
except (urllib2.URLError, httplib.HTTPException,
socket.error) as e:
logging.warning('could not upload: %s: %s', s.display_name, e)
s.status = SymbolFile.ERROR
failures += 1
# We pass the symbol along, on both success and failure.
yield s
def testBasic(self):
"""Make sure simple stuff works."""
cros_build_lib.RunCommand(['true', 'foo'])
self.mox.ReplayAll()
cros_build_lib.TimedCommand(cros_build_lib.RunCommand, ['true', 'foo'])
def main(argv):
parser = GetParser()
opts = parser.parse_args(argv)
opts.Freeze()
# Process list output quickly as it takes no privileges.
if opts.list:
print('\n'.join(
sorted(opts.tests or FindTests((constants.CHROMITE_DIR, )))))
return
# Many of our tests require a valid chroot to run. Make sure it's created
# before we block network access.
chroot = os.path.join(constants.SOURCE_ROOT, constants.DEFAULT_CHROOT_DIR)
if (not os.path.exists(chroot) and ChrootAvailable()
and not cros_build_lib.IsInsideChroot()):
cros_build_lib.RunCommand(['cros_sdk', '--create'])
# This is a cheesy hack to make sure gsutil is populated in the cache before
# we run tests. This is a partial workaround for crbug.com/468838.
gs.GSContext.GetDefaultGSUtilBin()
# Now let's run some tests.
_ReExecuteIfNeeded([sys.argv[0]] + argv, opts.network)
# A lot of pieces here expect to be run in the root of the chromite tree.
# Make them happy.
os.chdir(constants.CHROMITE_DIR)
tests = opts.tests or FindTests()
if opts.quick:
SPECIAL_TESTS.update(SLOW_TESTS)
global TIMING_CACHE_FILE # pylint: disable=global-statement
TIMING_CACHE_FILE = os.path.join(path_util.GetCacheDir(),
constants.COMMON_CACHE,
'run_tests.cache.json')
jobs = opts.jobs or multiprocessing.cpu_count()
with cros_build_lib.ContextManagerStack() as stack:
# If we're running outside the chroot, try to contain ourselves.
if cgroups.Cgroup.IsSupported(
) and not cros_build_lib.IsInsideChroot():
stack.Add(cgroups.SimpleContainChildren, 'run_tests')
# Throw all the tests into a custom tempdir so that if we do CTRL+C, we can
# quickly clean up all the files they might have left behind.
stack.Add(osutils.TempDir,
prefix='chromite.run_tests.',
set_global=True,
sudo_rm=True)
def _Finished(_log_level, _log_msg, result, delta):
if result:
logging.info('All tests succeeded! (%s total)', delta)
ret = cros_build_lib.TimedCommand(RunTests,
tests,
jobs=jobs,
chroot_available=ChrootAvailable(),
network=opts.network,
dryrun=opts.dryrun,
failfast=opts.failfast,
timed_log_callback=_Finished)
if not ret:
return 1
if not opts.network:
logging.warning('Network tests skipped; use --network to run them')
def UploadSymbol(upload_url,
symbol_element,
product_name,
file_limit=DEFAULT_FILE_LIMIT,
sleep=0,
num_errors=None,
watermark_errors=None,
failed_queue=None,
passed_queue=None):
"""Upload |sym_element.symbol_item| to |upload_url|
Args:
upload_url: The crash server to upload things to
symbol_element: A SymbolElement tuple. symbol_element.symbol_item is a
SymbolItem object containing the path to the breakpad symbol
to upload. symbol_element.opaque_push_state is an object of
_IsolateServerPushState or None if the item doesn't have
a push state.
product_name: A string for stats purposes. Usually 'ChromeOS' or 'Android'.
file_limit: The max file size of a symbol file before we try to strip it
sleep: Number of seconds to sleep before running
num_errors: An object to update with the error count (needs a .value member)
watermark_errors: An object to track current error behavior (needs a .value)
failed_queue: When a symbol fails, add it to this queue
passed_queue: When a symbol passes, add it to this queue
Returns:
The number of errors that were encountered.
"""
sym_file = symbol_element.symbol_item.sym_file
upload_item = symbol_element.symbol_item
if num_errors is None:
num_errors = ctypes.c_int()
if ErrorLimitHit(num_errors, watermark_errors):
# Abandon ship! It's on fire! NOoooooooooooOOOoooooo.
if failed_queue:
failed_queue.put(sym_file)
return 0
if sleep:
# Keeps us from DoS-ing the symbol server.
time.sleep(sleep)
logging.debug('uploading %s' % sym_file)
# Ideally there'd be a tempfile.SpooledNamedTemporaryFile that we could use.
with tempfile.NamedTemporaryFile(prefix='upload_symbols',
bufsize=0) as temp_sym_file:
if file_limit:
# If the symbols size is too big, strip out the call frame info. The CFI
# is unnecessary for 32bit x86 targets where the frame pointer is used (as
# all of ours have) and it accounts for over half the size of the symbols
# uploaded.
file_size = os.path.getsize(sym_file)
if file_size > file_limit:
logging.warning('stripping CFI from %s due to size %s > %s',
sym_file, file_size, file_limit)
temp_sym_file.writelines([
x for x in open(sym_file, 'rb').readlines()
if not x.startswith('STACK CFI')
])
upload_item = FakeItem(
sym_file=temp_sym_file.name,
sym_header=symbol_element.symbol_item.sym_header)
# Hopefully the crash server will let it through. But it probably won't.
# Not sure what the best answer is in this case.
file_size = os.path.getsize(upload_item.sym_file)
if file_size > CRASH_SERVER_FILE_LIMIT:
logging.PrintBuildbotStepWarnings()
logging.warning(
'upload file %s is awfully large, risking rejection by '
'the symbol server (%s > %s)', sym_file, file_size,
CRASH_SERVER_FILE_LIMIT)
# Upload the symbol file.
success = False
try:
cros_build_lib.TimedCommand(
retry_util.RetryException,
(urllib2.HTTPError, urllib2.URLError),
MAX_RETRIES,
SymUpload,
upload_url,
upload_item,
product_name,
sleep=INITIAL_RETRY_DELAY,
timed_log_msg=('upload of %10i bytes took %%(delta)s: %s' %
(file_size, os.path.basename(sym_file))))
success = True
if passed_queue:
passed_queue.put(symbol_element)
except urllib2.HTTPError as e:
logging.warning('could not upload: %s: HTTP %s: %s',
os.path.basename(sym_file), e.code, e.reason)
except (urllib2.URLError, httplib.HTTPException, socket.error) as e:
logging.warning('could not upload: %s: %s',
os.path.basename(sym_file), e)
finally:
if success:
_UpdateCounter(watermark_errors, ERROR_ADJUST_PASS)
else:
_UpdateCounter(num_errors, 1)
_UpdateCounter(watermark_errors, ERROR_ADJUST_FAIL)
if failed_queue:
failed_queue.put(sym_file)
return num_errors.value
