def testExceptionCatchingDecorator(self):
"""Tests the exception catching decorator CaptureAndLogException."""
original_log_level = self.root_logger.getEffectiveLevel()
self.root_logger.setLevel(logging.DEBUG)
# Test that a wrapped function with an exception doesn't stop the process.
mock_exc_fn = mock.MagicMock(__name__='mock_exc_fn',
side_effect=Exception())
wrapped_fn = metrics.CaptureAndLogException(mock_exc_fn)
wrapped_fn()
self.assertEqual(1, mock_exc_fn.call_count)
with open(self.log_handler_file) as f:
log_output = f.read()
self.assertIn(
'Exception captured in mock_exc_fn during metrics '
'collection', log_output)
mock_err_fn = mock.MagicMock(__name__='mock_err_fn',
side_effect=TypeError())
wrapped_fn = metrics.CaptureAndLogException(mock_err_fn)
wrapped_fn()
self.assertEqual(1, mock_err_fn.call_count)
with open(self.log_handler_file) as f:
log_output = f.read()
self.assertIn(
'Exception captured in mock_err_fn during metrics '
'collection', log_output)
# Test that exceptions in the unprotected metrics functions are caught.
with mock.patch.object(MetricsCollector,
'GetCollector',
return_value='not a collector'):
# These calls should all fail, but the exceptions shouldn't propagate up.
metrics.Shutdown()
metrics.LogCommandParams()
metrics.LogRetryableError()
metrics.LogFatalError()
metrics.LogPerformanceSummary()
metrics.CheckAndMaybePromptForAnalyticsEnabling('invalid argument')
with open(self.log_handler_file) as f:
log_output = f.read()
self.assertIn(
'Exception captured in Shutdown during metrics collection',
log_output)
self.assertIn(
'Exception captured in LogCommandParams during metrics collection',
log_output)
self.assertIn(
'Exception captured in LogRetryableError during metrics collection',
log_output)
self.assertIn(
'Exception captured in LogFatalError during metrics collection',
log_output)
self.assertIn(
'Exception captured in LogPerformanceSummary during metrics '
'collection', log_output)
self.assertIn(
'Exception captured in CheckAndMaybePromptForAnalyticsEnabling '
'during metrics collection', log_output)
self.root_logger.setLevel(original_log_level)
def CollectMetricAndSetLogLevel(log_level, log_file_path):
metrics.LogCommandParams(command_name='cmd1', subcommands=['action1'],
sub_opts=[('optb', ''), ('opta', '')])
metrics.LogFatalError(gslib.exception.CommandException('test'))
# Wait for report to make sure the log is written before we check it.
self.collector.ReportMetrics(wait_for_report=True, log_level=log_level,
log_file_path=log_file_path)
self.assertEqual([], self.collector._metrics)
def _LogAllTestMetrics():
"""Logs all the common metrics for a test."""
metrics.LogCommandParams(command_name='cmd1',
subcommands=['action1'],
global_opts=[('-y', 'value'), ('-z', ''),
('-x', '')],
sub_opts=[('optb', ''), ('opta', '')])
metrics.LogRetryableError('retryable_error_type_1')
metrics.LogRetryableError('retryable_error_type_1')
metrics.LogRetryableError('retryable_error_type_2')
metrics.LogFatalError(gslib.exception.CommandException('test'))
def _LogAllTestMetrics():
"""Logs all the common metrics for a test."""
metrics.LogCommandParams(
command_name='cmd1', subcommands=['action1'],
global_opts=[('-y', 'value'), ('-z', ''), ('-x', '')],
sub_opts=[('optb', ''), ('opta', '')])
retry_msg_1 = RetryableErrorMessage(Exception(), 0)
retry_msg_2 = RetryableErrorMessage(ValueError(), 0)
metrics.LogRetryableError(retry_msg_1)
metrics.LogRetryableError(retry_msg_1)
metrics.LogRetryableError(retry_msg_2)
metrics.LogFatalError(gslib.exception.CommandException('test'))
def testExceptionCatchingDecorator(self):
"""Tests the exception catching decorator CaptureAndLogException."""
# A wrapped function with an exception should not stop the process.
mock_exc_fn = mock.MagicMock(__name__='mock_exc_fn',
side_effect=Exception())
wrapped_fn = metrics.CaptureAndLogException(mock_exc_fn)
wrapped_fn()
debug_messages = self.log_handler.messages['debug']
self.assertIn('Exception captured in mock_exc_fn during metrics collection',
debug_messages[0])
self.log_handler.reset()
self.assertEqual(1, mock_exc_fn.call_count)
mock_err_fn = mock.MagicMock(__name__='mock_err_fn',
side_effect=TypeError())
wrapped_fn = metrics.CaptureAndLogException(mock_err_fn)
wrapped_fn()
self.assertEqual(1, mock_err_fn.call_count)
debug_messages = self.log_handler.messages['debug']
self.assertIn('Exception captured in mock_err_fn during metrics collection',
debug_messages[0])
self.log_handler.reset()
# Test that exceptions in the unprotected metrics functions are caught.
with mock.patch.object(MetricsCollector, 'GetCollector',
return_value='not a collector'):
# These calls should all fail, but the exceptions shouldn't propagate up.
metrics.Shutdown()
metrics.LogCommandParams()
metrics.LogRetryableError()
metrics.LogFatalError()
metrics.LogPerformanceSummaryParams()
metrics.CheckAndMaybePromptForAnalyticsEnabling('invalid argument')
debug_messages = self.log_handler.messages['debug']
message_index = 0
for func_name in ('Shutdown', 'LogCommandParams', 'LogRetryableError',
'LogFatalError', 'LogPerformanceSummaryParams',
'CheckAndMaybePromptForAnalyticsEnabling'):
self.assertIn(
'Exception captured in %s during metrics collection' % func_name,
debug_messages[message_index])
message_index += 1
self.log_handler.reset()
def MultithreadedMainSignalHandler(signal_num, cur_stack_frame):
"""Final signal handler for multi-threaded main process."""
if signal_num == signal.SIGINT:
if logging.getLogger().isEnabledFor(logging.DEBUG):
stack_trace = ''.join(traceback.format_list(traceback.extract_stack()))
err = ('DEBUG: Caught CTRL-C (signal %d) - Exception stack trace:\n'
' %s' % (signal_num, re.sub('\\n', '\n ', stack_trace)))
try:
sys.stderr.write(err.encode(UTF8))
except UnicodeDecodeError:
# Can happen when outputting invalid Unicode filenames.
sys.stderr.write(err)
else:
sys.stderr.write('Caught CTRL-C (signal %d) - exiting\n' % signal_num)
metrics.LogFatalError(exception=ControlCException())
metrics.Shutdown()
KillProcess(os.getpid())
def _OutputAndExit(message, exception=None):
"""Outputs message to stderr and exits gsutil with code 1.
This function should only be called in single-process, single-threaded mode.
Args:
message: Message to print to stderr.
exception: The exception that caused gsutil to fail.
"""
if debug_level >= constants.DEBUGLEVEL_DUMP_REQUESTS or test_exception_traces:
stack_trace = traceback.format_exc()
err = ('DEBUG: Exception stack trace:\n %s\n%s\n' %
(re.sub('\\n', '\n ', stack_trace), message))
else:
err = '%s\n' % message
try:
text_util.print_to_fd(err, end='', file=sys.stderr)
except UnicodeDecodeError:
# Can happen when outputting invalid Unicode filenames.
sys.stderr.write(err)
if exception:
metrics.LogFatalError(exception)
sys.exit(1)
from gslib import metrics
# We parse the options and arguments here so we can pass the results to the user
# agent helper.
try:
opts, args = GetArgumentsAndOptions()
except CommandException as e:
reason = e.reason if e.informational else 'CommandException: %s' % e.reason
err = '%s\n' % reason
try:
text_util.print_to_fd(err, end='', file=sys.stderr)
except UnicodeDecodeError:
# Can happen when outputting invalid Unicode filenames.
sys.stderr.write(err)
if e:
metrics.LogFatalError(e)
sys.exit(1)
# This calculated user agent can be stored for use in StorageV1.
gslib.USER_AGENT = GetUserAgent(args, metrics.MetricsCollector.IsDisabled())
boto.UserAgent += gslib.USER_AGENT
# pylint: disable=g-bad-import-order
import httplib2
import oauth2client
from google_reauth import reauth_creds
from google_reauth import errors as reauth_errors
from gslib import context_config
from gslib import wildcard_iterator
from gslib.cloud_api import AccessDeniedException
from gslib.cloud_api import ArgumentException
