def test_send(self, _discovery, _load_creds):
mon = monitors.PubSubMonitor('/path/to/creds.p8.json', 'myproject',
'mytopic')
mon._api = mock.MagicMock()
topic = 'projects/myproject/topics/mytopic'
metric1 = metrics_pb2.MetricsData(name='m1')
mon.send(metric1)
metric2 = metrics_pb2.MetricsData(name='m2')
mon.send([metric1, metric2])
collection = metrics_pb2.MetricsCollection(data=[metric1, metric2])
mon.send(collection)
def message(pb):
pb = monitors.Monitor._wrap_proto(pb)
return {
'messages': [{
'data': base64.b64encode(pb.SerializeToString())
}]
}
publish = mon._api.projects.return_value.topics.return_value.publish
publish.assert_has_calls([
mock.call(topic=topic, body=message(metric1)),
mock.call().execute(num_retries=5),
mock.call(topic=topic, body=message([metric1, metric2])),
mock.call().execute(num_retries=5),
mock.call(topic=topic, body=message(collection)),
mock.call().execute(num_retries=5),
])
def test_send_uninitialized(self, discovery, _load_creds):
"""Test initialization retry logic, and also un-instrumented http path."""
discovery.side_effect = EnvironmentError() # Fail initialization.
mon = monitors.PubSubMonitor('/path/to/creds.p8.json',
'myproject',
'mytopic',
use_instrumented_http=False)
metric1 = metrics_pb2.MetricsData(name='m1')
mon.send(metric1)
self.assertIsNone(mon._api)
# Another retry: initialization succeeds.
discovery.side_effect = None
mon.send(metric1)
def message(pb):
pb = monitors.Monitor._wrap_proto(pb)
return {
'messages': [{
'data': base64.b64encode(pb.SerializeToString())
}]
}
topic = 'projects/myproject/topics/mytopic'
publish = mon._api.projects.return_value.topics.return_value.publish
publish.assert_has_calls([
mock.call(topic=topic, body=message(metric1)),
mock.call().execute(num_retries=5),
])
def test_init_gce_credential(self, aac, discovery, instrumented_http):
creds = aac.return_value
http_mock = instrumented_http.return_value
mon = monitors.PubSubMonitor(':gce', 'myproject', 'mytopic')
aac.assert_called_once_with(monitors.PubSubMonitor._SCOPES)
creds.authorize.assert_called_once_with(http_mock)
discovery.build.assert_called_once_with('pubsub', 'v1', http=http_mock)
self.assertEquals(mon._topic, 'projects/myproject/topics/mytopic')
def test_init_service_account(self, gc, discovery, instrumented_http):
m_open = mock.mock_open(read_data='{"type": "service_account"}')
creds = gc.from_stream.return_value
scoped_creds = creds.create_scoped.return_value
http_mock = instrumented_http.return_value
with mock.patch('infra_libs.ts_mon.common.monitors.open',
m_open,
create=True):
mon = monitors.PubSubMonitor('/path/to/creds.p8.json', 'myproject',
'mytopic')
m_open.assert_called_once_with('/path/to/creds.p8.json', 'r')
creds.create_scoped.assert_called_once_with(
monitors.PubSubMonitor._SCOPES)
scoped_creds.authorize.assert_called_once_with(http_mock)
discovery.build.assert_called_once_with('pubsub', 'v1', http=http_mock)
self.assertEquals(mon._topic, 'projects/myproject/topics/mytopic')
def test_init_storage(self, storage, discovery, instrumented_http):
storage_inst = mock.Mock()
storage.return_value = storage_inst
creds = storage_inst.get.return_value
m_open = mock.mock_open(read_data='{}')
http_mock = instrumented_http.return_value
with mock.patch('infra_libs.ts_mon.common.monitors.open',
m_open,
create=True):
mon = monitors.PubSubMonitor('/path/to/creds.p8.json', 'myproject',
'mytopic')
m_open.assert_called_once_with('/path/to/creds.p8.json', 'r')
storage_inst.get.assert_called_once_with()
creds.authorize.assert_called_once_with(http_mock)
discovery.build.assert_called_once_with('pubsub', 'v1', http=http_mock)
self.assertEquals(mon._topic, 'projects/myproject/topics/mytopic')
def test_send_fails(self, _discovery, _load_creds):
# Test for an occasional flake of .publish().execute().
mon = monitors.PubSubMonitor('/path/to/creds.p8.json', 'myproject',
'mytopic')
mon._api = mock.MagicMock()
topic = 'projects/myproject/topics/mytopic'
metric1 = metrics_pb2.MetricsData(name='m1')
mon.send(metric1)
publish = mon._api.projects.return_value.topics.return_value.publish
publish.side_effect = ValueError()
metric2 = metrics_pb2.MetricsData(name='m2')
mon.send([metric1, metric2])
collection = metrics_pb2.MetricsCollection(data=[metric1, metric2])
publish.side_effect = errors.HttpError(
mock.Mock(status=404, reason='test'), '')
mon.send(collection)
# Test that all caught exceptions are specified without errors.
# When multiple exceptions are specified in the 'except' clause,
# they are evaluated lazily, and may contain syntax errors.
# Throwing an uncaught exception forces all exception specs to be
# evaluated, catching more runtime errors.
publish.side_effect = Exception('uncaught')
with self.assertRaises(Exception):
mon.send(collection)
def message(pb):
pb = monitors.Monitor._wrap_proto(pb)
return {
'messages': [{
'data': base64.b64encode(pb.SerializeToString())
}]
}
publish.assert_has_calls([
mock.call(topic=topic, body=message(metric1)),
mock.call().execute(num_retries=5),
mock.call(topic=topic, body=message([metric1, metric2])),
mock.call(topic=topic, body=message(collection)),
])
def process_argparse_options(args):
"""Process command line arguments to initialize the global monitor.
Also initializes the default target if sufficient arguments are supplied.
If they aren't, all created metrics will have to supply their own target.
This is generally a bad idea, as many libraries rely on the default target
being set up.
Starts a background thread to automatically flush monitoring metrics if not
disabled by command line arguments.
Args:
args (argparse.Namespace): the result of parsing the command line arguments
"""
# Parse the config file if it exists.
config = load_machine_config(args.ts_mon_config_file)
endpoint = config.get('endpoint', '')
credentials = config.get('credentials', '')
# Command-line args override the values in the config file.
if args.ts_mon_endpoint is not None:
endpoint = args.ts_mon_endpoint
if args.ts_mon_credentials is not None:
credentials = args.ts_mon_credentials
interface.state.global_monitor = monitors.NullMonitor()
if endpoint.startswith('file://'):
interface.state.global_monitor = monitors.DebugMonitor(
endpoint[len('file://'):])
elif credentials:
if endpoint.startswith('pubsub://'):
url = urlparse.urlparse(endpoint)
project = url.netloc
topic = url.path.strip('/')
interface.state.global_monitor = monitors.PubSubMonitor(
credentials, project, topic, use_instrumented_http=True)
else:
logging.error('Monitoring is disabled because the endpoint provided is '
'invalid or not supported: %s', endpoint)
else:
logging.error('Monitoring is disabled because credentials are not '
'available')
if args.ts_mon_target_type == 'device':
interface.state.target = targets.DeviceTarget(
args.ts_mon_device_region,
args.ts_mon_device_role,
args.ts_mon_device_network,
args.ts_mon_device_hostname)
if args.ts_mon_target_type == 'task': # pragma: no cover
# Reimplement ArgumentParser.error, since we don't have access to the parser
if not args.ts_mon_task_service_name:
print >> sys.stderr, ('Argument --ts-mon-task-service-name must be '
'provided when the target type is "task".')
sys.exit(2)
if not args.ts_mon_task_job_name: # pragma: no cover
print >> sys.stderr, ('Argument --ts-mon-task-job-name must be provided '
'when the target type is "task".')
sys.exit(2)
interface.state.target = targets.TaskTarget(
args.ts_mon_task_service_name,
args.ts_mon_task_job_name,
args.ts_mon_task_region,
args.ts_mon_task_hostname,
args.ts_mon_task_number)
interface.state.flush_mode = args.ts_mon_flush
if args.ts_mon_flush == 'auto':
interface.state.flush_thread = interface._FlushThread(
args.ts_mon_flush_interval_secs)
interface.state.flush_thread.start()
standard_metrics.init()
def initialize(app=None,
is_enabled_fn=None,
cron_module='default',
is_local_unittest=None):
"""Instruments webapp2 `app` with gae_ts_mon metrics.
Instruments all the endpoints in `app` with basic metrics.
Args:
app (webapp2 app): the app to instrument.
is_enabled_fn (function or None): a function returning bool if ts_mon should
send the actual metrics. None (default) is equivalent to lambda: True.
This allows apps to turn monitoring on or off dynamically, per app.
cron_module (str): the name of the module handling the
/internal/cron/ts_mon/send endpoint. This allows moving the cron job
to any module the user wants.
is_local_unittest (bool or None): whether we are running in a unittest.
"""
if is_local_unittest is None: # pragma: no cover
# Since gae_ts_mon.initialize is called at module-scope by appengine apps,
# AppengineTestCase.setUp() won't have run yet and none of the appengine
# stubs will be initialized, so accessing Datastore or even getting the
# application ID will fail.
is_local_unittest = ('expect_tests' in sys.argv[0])
if is_enabled_fn is not None:
interface.state.flush_enabled_fn = is_enabled_fn
if app is not None:
instrument_wsgi_application(app)
if is_local_unittest or modules.get_current_module_name(
) == cron_module:
instrument_wsgi_application(handlers.app)
# Use the application ID as the service name and the module name as the job
# name.
if is_local_unittest: # pragma: no cover
service_name = 'unittest'
job_name = 'unittest'
hostname = 'unittest'
else:
service_name = app_identity.get_application_id()
job_name = modules.get_current_module_name()
hostname = modules.get_current_version_name()
runtime.set_shutdown_hook(_shutdown_hook)
interface.state.target = targets.TaskTarget(service_name,
job_name,
shared.REGION,
hostname,
task_num=-1)
interface.state.flush_mode = 'manual'
interface.state.last_flushed = datetime.datetime.utcnow()
# Don't send metrics when running on the dev appserver.
if (is_local_unittest or os.environ.get('SERVER_SOFTWARE',
'').startswith('Development')):
logging.info('Using debug monitor')
interface.state.global_monitor = monitors.DebugMonitor()
else:
logging.info('Using pubsub monitor %s/%s', shared.PUBSUB_PROJECT,
shared.PUBSUB_TOPIC)
interface.state.global_monitor = monitors.PubSubMonitor(
monitors.APPENGINE_CREDENTIALS, shared.PUBSUB_PROJECT,
shared.PUBSUB_TOPIC)
shared.register_global_metrics([shared.appengine_default_version])
shared.register_global_metrics_callback(shared.INTERNAL_CALLBACK_NAME,
_internal_callback)
logging.info(
'Initialized ts_mon with service_name=%s, job_name=%s, '
'hostname=%s', service_name, job_name, hostname)
def process_argparse_options(args):
"""Process command line arguments to initialize the global monitor.
Also initializes the default target.
Starts a background thread to automatically flush monitoring metrics if not
disabled by command line arguments.
Args:
args (argparse.Namespace): the result of parsing the command line arguments
"""
# Parse the config file if it exists.
config = load_machine_config(args.ts_mon_config_file)
endpoint = config.get('endpoint', '')
credentials = config.get('credentials', '')
autogen_hostname = config.get('autogen_hostname', False)
use_new_proto = config.get('use_new_proto', False)
# Command-line args override the values in the config file.
if args.ts_mon_endpoint is not None:
endpoint = args.ts_mon_endpoint
if args.ts_mon_credentials is not None:
credentials = args.ts_mon_credentials
if args.ts_mon_use_new_proto:
use_new_proto = args.ts_mon_use_new_proto
if args.ts_mon_target_type == 'device':
hostname = args.ts_mon_device_hostname
if args.ts_mon_autogen_hostname or autogen_hostname:
hostname = 'autogen:' + hostname
interface.state.target = targets.DeviceTarget(
args.ts_mon_device_region, args.ts_mon_device_role,
args.ts_mon_device_network, hostname)
if args.ts_mon_target_type == 'task':
# Reimplement ArgumentParser.error, since we don't have access to the parser
if not args.ts_mon_task_service_name:
print >> sys.stderr, (
'Argument --ts-mon-task-service-name must be '
'provided when the target type is "task".')
sys.exit(2)
if not args.ts_mon_task_job_name:
print >> sys.stderr, (
'Argument --ts-mon-task-job-name must be provided '
'when the target type is "task".')
sys.exit(2)
hostname = args.ts_mon_task_hostname
if args.ts_mon_autogen_hostname or autogen_hostname:
hostname = 'autogen:' + hostname
interface.state.target = targets.TaskTarget(
args.ts_mon_task_service_name, args.ts_mon_task_job_name,
args.ts_mon_task_region, hostname, args.ts_mon_task_number)
interface.state.metric_name_prefix = args.ts_mon_metric_name_prefix
interface.state.global_monitor = monitors.NullMonitor()
if endpoint.startswith('file://'):
interface.state.global_monitor = monitors.DebugMonitor(
endpoint[len('file://'):])
elif endpoint.startswith('pubsub://'):
if credentials:
url = urlparse.urlparse(endpoint)
project = url.netloc
topic = url.path.strip('/')
interface.state.global_monitor = monitors.PubSubMonitor(
monitors.CredentialFactory.from_string(credentials),
project,
topic,
use_instrumented_http=True,
ca_certs=args.ts_mon_ca_certs)
else:
logging.error(
'ts_mon monitoring is disabled because credentials are not '
'available')
elif endpoint.startswith('https://'):
interface.state.global_monitor = monitors.HttpsMonitor(
endpoint,
monitors.CredentialFactory.from_string(credentials),
ca_certs=args.ts_mon_ca_certs)
elif endpoint.lower() == 'none':
logging.info('ts_mon monitoring has been explicitly disabled')
else:
logging.error(
'ts_mon monitoring is disabled because the endpoint provided'
' is invalid or not supported: %s', endpoint)
interface.state.flush_mode = args.ts_mon_flush
interface.state.use_new_proto = use_new_proto
if args.ts_mon_flush == 'auto':
interface.state.flush_thread = interface._FlushThread(
args.ts_mon_flush_interval_secs)
interface.state.flush_thread.start()
standard_metrics.init()