def testReportStats(self):
retry_stats.SetupStats()
# Insert some stats to report.
retry_stats.RetryWithStats(
self.CAT, self.handlerNoRetry, 3, self.callSuccess)
retry_stats.RetryWithStats(
self.CAT_B, self.handlerNoRetry, 3, self.callSuccess)
self.assertRaises(TestRetryException,
retry_stats.RetryWithStats,
self.CAT, self.handlerRetry, 3, self.callFailure)
out = StringIO()
retry_stats.ReportStats(out)
# Expecting reports for both CAT and CAT_B used above.
expected = """************************************************************
** Performance Statistics for Test Service A
**
** Success: 1
** Failure: 1
** Retries: 3
** Total: 2
************************************************************
************************************************************
** Performance Statistics for Test Service B
**
** Success: 1
** Failure: 0
** Retries: 0
** Total: 1
************************************************************
"""
self.assertEqual(out.getvalue(), expected)
def testSuccessNoSetup(self):
"""Verify that we can handle a successful call if we're never setup."""
self.assertEqual(retry_stats._STATS_COLLECTION, None)
result = retry_stats.RetryWithStats(
self.CAT, self.handlerNoRetry, 3, self.callSuccess)
self.assertEqual(result, self.SUCCESS_RESULT)
result = retry_stats.RetryWithStats(
self.CAT, self.handlerNoRetry, 3, self.callSuccess)
self.assertEqual(result, self.SUCCESS_RESULT)
self.assertEqual(retry_stats._STATS_COLLECTION, None)
def testSuccess(self):
"""Verify that we can handle a successful call."""
retry_stats.SetupStats()
self._verifyStats(self.CAT)
# Succeed once.
result = retry_stats.RetryWithStats(
self.CAT, self.handlerNoRetry, 3, self.callSuccess)
self.assertEqual(result, self.SUCCESS_RESULT)
self._verifyStats(self.CAT, success=1)
# Succeed twice.
result = retry_stats.RetryWithStats(
self.CAT, self.handlerNoRetry, 3, self.callSuccess)
self.assertEqual(result, self.SUCCESS_RESULT)
self._verifyStats(self.CAT, success=2)
def PerformSymbolsFileUpload(symbols, upload_url, api_key):
"""Upload the symbols to the crash server
Args:
symbols: An iterable of SymbolFiles to be uploaded.
upload_url: URL of crash server to upload too.
api_key: Authentication key.
failures: Tracker for total upload failures.
Yields:
Each symbol from symbols, perhaps modified.
"""
failures = 0
for s in symbols:
if (failures < MAX_TOTAL_ERRORS_FOR_RETRY and
s.status in (SymbolFile.INITIAL, SymbolFile.ERROR)):
# Keeps us from DoS-ing the symbol server.
time.sleep(SLEEP_DELAY)
logging.info('Uploading symbol_file: %s', s.display_path)
try:
# This command retries the upload multiple times with growing delays. We
# only consider the upload a failure if these retries fail.
def ShouldRetryUpload(exception):
if isinstance(exception, (requests.exceptions.RequestException,
IOError,
httplib.HTTPException, socket.error)):
logging.info('Request failed, retrying: %s', exception)
return True
return False
with cros_build_lib.TimedSection() as timer:
retry_stats.RetryWithStats(
UPLOAD_STATS, ShouldRetryUpload, MAX_RETRIES,
UploadSymbolFile,
upload_url, s, api_key,
sleep=INITIAL_RETRY_DELAY,
log_all_retries=True)
if s.status != SymbolFile.DUPLICATE:
logging.info('upload of %10i bytes took %s', s.FileSize(),
timer.delta)
s.status = SymbolFile.UPLOADED
except requests.exceptions.RequestException as e:
logging.warning('could not upload: %s: HTTP error: %s',
s.display_name, e)
s.status = SymbolFile.ERROR
failures += 1
except (httplib.HTTPException, IOError, socket.error) as e:
logging.warning('could not upload: %s: %s %s', s.display_name,
type(e).__name__, e)
s.status = SymbolFile.ERROR
failures += 1
# We pass the symbol along, on both success and failure.
yield s
def PerformSymbolsFileUpload(symbols, upload_url, product_name='ChromeOS'):
"""Upload the symbols to the crash server
Args:
symbols: An iterable of SymbolFiles to be uploaded.
upload_url: URL of crash server to upload too.
failures: Tracker for total upload failures.
product_name: A string for stats purposes. Usually 'ChromeOS' or 'Android'.
Yields:
Each symbol from symbols, perhaps modified.
"""
failures = 0
for s in symbols:
if (failures < MAX_TOTAL_ERRORS_FOR_RETRY
and s.status in (SymbolFile.INITIAL, SymbolFile.ERROR)):
# Keeps us from DoS-ing the symbol server.
time.sleep(SLEEP_DELAY)
logging.info('Uploading symbol_file: %s', s.display_path)
try:
# This command retries the upload multiple times with growing delays. We
# only consider the upload a failure if these retries fail.
def ShouldRetryUpload(exception):
return isinstance(exception,
(urllib2.HTTPError, urllib2.URLError))
with cros_build_lib.TimedSection() as timer:
retry_stats.RetryWithStats(UPLOAD_STATS,
ShouldRetryUpload,
MAX_RETRIES,
UploadSymbolFile,
upload_url,
s,
product_name,
sleep=INITIAL_RETRY_DELAY,
log_all_retries=True)
logging.info('upload of %10i bytes took %s', s.FileSize(),
timer.delta)
s.status = SymbolFile.UPLOADED
except urllib2.HTTPError as e:
logging.warning('could not upload: %s: HTTP %s: %s',
s.display_name, e.code, e.reason)
s.status = SymbolFile.ERROR
failures += 1
except (urllib2.URLError, httplib.HTTPException,
socket.error) as e:
logging.warning('could not upload: %s: %s', s.display_name, e)
s.status = SymbolFile.ERROR
failures += 1
# We pass the symbol along, on both success and failure.
yield s
def testSuccessRetry(self):
"""Verify that we can handle a successful call after tries."""
retry_stats.SetupStats()
self._verifyStats(self.CAT)
# Use this scoped list as a persistent counter.
call_counter = ['fail 1', 'fail 2']
def callRetrySuccess():
if call_counter:
raise TestRetryException(call_counter.pop())
else:
return self.SUCCESS_RESULT
# Retry twice, then succeed.
result = retry_stats.RetryWithStats(
self.CAT, self.handlerRetry, 3, callRetrySuccess)
self.assertEqual(result, self.SUCCESS_RESULT)
self._verifyStats(self.CAT, success=1, retry=2)
def DoCommand(self,
gsutil_cmd,
headers=(),
retries=None,
version=None,
parallel=False,
**kwargs):
"""Run a gsutil command, suppressing output, and setting retry/sleep.
Args:
gsutil_cmd: The (mostly) constructed gsutil subcommand to run.
headers: A list of raw headers to pass down.
parallel: Whether gsutil should enable parallel copy/update of multiple
files. NOTE: This option causes gsutil to use significantly more
memory, even if gsutil is only uploading one file.
retries: How many times to retry this command (defaults to setting given
at object creation).
version: If given, the generation; essentially the timestamp of the last
update. Note this is not the same as sequence-number; it's
monotonically increasing bucket wide rather than reset per file.
The usage of this is if we intend to replace/update only if the version
is what we expect. This is useful for distributed reasons- for example,
to ensure you don't overwrite someone else's creation, a version of
0 states "only update if no version exists".
Returns:
A RunCommandResult object.
"""
kwargs = kwargs.copy()
kwargs.setdefault('redirect_stderr', True)
cmd = [self.gsutil_bin]
cmd += self.gsutil_flags
for header in headers:
cmd += ['-h', header]
if version is not None:
cmd += ['-h', 'x-goog-if-generation-match:%d' % int(version)]
# Enable parallel copy/update of multiple files if stdin is not to
# be piped to the command. This does not split a single file into
# smaller components for upload.
if parallel and kwargs.get('input') is None:
cmd += ['-m']
cmd.extend(gsutil_cmd)
if retries is None:
retries = self.retries
extra_env = kwargs.pop('extra_env', {})
if self.boto_file and os.path.isfile(self.boto_file):
extra_env.setdefault('BOTO_CONFIG', self.boto_file)
if self.dry_run:
logging.debug("%s: would've run: %s", self.__class__.__name__,
cros_build_lib.CmdToStr(cmd))
else:
try:
return retry_stats.RetryWithStats(retry_stats.GSUTIL,
self._RetryFilter,
retries,
cros_build_lib.RunCommand,
cmd,
sleep=self._sleep_time,
extra_env=extra_env,
**kwargs)
except cros_build_lib.RunCommandError as e:
raise GSCommandError(e.msg, e.result, e.exception)
