def test_send(self, _discovery, _load_creds): mon = monitors.PubSubMonitor('/path/to/creds.p8.json', 'myproject', 'mytopic') mon._api = mock.MagicMock() topic = 'projects/myproject/topics/mytopic' metric1 = metrics_pb2.MetricsData(name='m1') mon.send(metric1) metric2 = metrics_pb2.MetricsData(name='m2') mon.send([metric1, metric2]) collection = metrics_pb2.MetricsCollection(data=[metric1, metric2]) mon.send(collection) def message(pb): pb = monitors.Monitor._wrap_proto(pb) return { 'messages': [{ 'data': base64.b64encode(pb.SerializeToString()) }] } publish = mon._api.projects.return_value.topics.return_value.publish publish.assert_has_calls([ mock.call(topic=topic, body=message(metric1)), mock.call().execute(num_retries=5), mock.call(topic=topic, body=message([metric1, metric2])), mock.call().execute(num_retries=5), mock.call(topic=topic, body=message(collection)), mock.call().execute(num_retries=5), ])
def test_send_uninitialized(self, discovery, _load_creds): """Test initialization retry logic, and also un-instrumented http path.""" discovery.side_effect = EnvironmentError() # Fail initialization. mon = monitors.PubSubMonitor('/path/to/creds.p8.json', 'myproject', 'mytopic', use_instrumented_http=False) metric1 = metrics_pb2.MetricsData(name='m1') mon.send(metric1) self.assertIsNone(mon._api) # Another retry: initialization succeeds. discovery.side_effect = None mon.send(metric1) def message(pb): pb = monitors.Monitor._wrap_proto(pb) return { 'messages': [{ 'data': base64.b64encode(pb.SerializeToString()) }] } topic = 'projects/myproject/topics/mytopic' publish = mon._api.projects.return_value.topics.return_value.publish publish.assert_has_calls([ mock.call(topic=topic, body=message(metric1)), mock.call().execute(num_retries=5), ])
def test_init_gce_credential(self, aac, discovery, instrumented_http): creds = aac.return_value http_mock = instrumented_http.return_value mon = monitors.PubSubMonitor(':gce', 'myproject', 'mytopic') aac.assert_called_once_with(monitors.PubSubMonitor._SCOPES) creds.authorize.assert_called_once_with(http_mock) discovery.build.assert_called_once_with('pubsub', 'v1', http=http_mock) self.assertEquals(mon._topic, 'projects/myproject/topics/mytopic')
def test_init_service_account(self, gc, discovery, instrumented_http): m_open = mock.mock_open(read_data='{"type": "service_account"}') creds = gc.from_stream.return_value scoped_creds = creds.create_scoped.return_value http_mock = instrumented_http.return_value with mock.patch('infra_libs.ts_mon.common.monitors.open', m_open, create=True): mon = monitors.PubSubMonitor('/path/to/creds.p8.json', 'myproject', 'mytopic') m_open.assert_called_once_with('/path/to/creds.p8.json', 'r') creds.create_scoped.assert_called_once_with( monitors.PubSubMonitor._SCOPES) scoped_creds.authorize.assert_called_once_with(http_mock) discovery.build.assert_called_once_with('pubsub', 'v1', http=http_mock) self.assertEquals(mon._topic, 'projects/myproject/topics/mytopic')
def test_init_storage(self, storage, discovery, instrumented_http): storage_inst = mock.Mock() storage.return_value = storage_inst creds = storage_inst.get.return_value m_open = mock.mock_open(read_data='{}') http_mock = instrumented_http.return_value with mock.patch('infra_libs.ts_mon.common.monitors.open', m_open, create=True): mon = monitors.PubSubMonitor('/path/to/creds.p8.json', 'myproject', 'mytopic') m_open.assert_called_once_with('/path/to/creds.p8.json', 'r') storage_inst.get.assert_called_once_with() creds.authorize.assert_called_once_with(http_mock) discovery.build.assert_called_once_with('pubsub', 'v1', http=http_mock) self.assertEquals(mon._topic, 'projects/myproject/topics/mytopic')
def test_send_fails(self, _discovery, _load_creds): # Test for an occasional flake of .publish().execute(). mon = monitors.PubSubMonitor('/path/to/creds.p8.json', 'myproject', 'mytopic') mon._api = mock.MagicMock() topic = 'projects/myproject/topics/mytopic' metric1 = metrics_pb2.MetricsData(name='m1') mon.send(metric1) publish = mon._api.projects.return_value.topics.return_value.publish publish.side_effect = ValueError() metric2 = metrics_pb2.MetricsData(name='m2') mon.send([metric1, metric2]) collection = metrics_pb2.MetricsCollection(data=[metric1, metric2]) publish.side_effect = errors.HttpError( mock.Mock(status=404, reason='test'), '') mon.send(collection) # Test that all caught exceptions are specified without errors. # When multiple exceptions are specified in the 'except' clause, # they are evaluated lazily, and may contain syntax errors. # Throwing an uncaught exception forces all exception specs to be # evaluated, catching more runtime errors. publish.side_effect = Exception('uncaught') with self.assertRaises(Exception): mon.send(collection) def message(pb): pb = monitors.Monitor._wrap_proto(pb) return { 'messages': [{ 'data': base64.b64encode(pb.SerializeToString()) }] } publish.assert_has_calls([ mock.call(topic=topic, body=message(metric1)), mock.call().execute(num_retries=5), mock.call(topic=topic, body=message([metric1, metric2])), mock.call(topic=topic, body=message(collection)), ])
def process_argparse_options(args): """Process command line arguments to initialize the global monitor. Also initializes the default target if sufficient arguments are supplied. If they aren't, all created metrics will have to supply their own target. This is generally a bad idea, as many libraries rely on the default target being set up. Starts a background thread to automatically flush monitoring metrics if not disabled by command line arguments. Args: args (argparse.Namespace): the result of parsing the command line arguments """ # Parse the config file if it exists. config = load_machine_config(args.ts_mon_config_file) endpoint = config.get('endpoint', '') credentials = config.get('credentials', '') # Command-line args override the values in the config file. if args.ts_mon_endpoint is not None: endpoint = args.ts_mon_endpoint if args.ts_mon_credentials is not None: credentials = args.ts_mon_credentials interface.state.global_monitor = monitors.NullMonitor() if endpoint.startswith('file://'): interface.state.global_monitor = monitors.DebugMonitor( endpoint[len('file://'):]) elif credentials: if endpoint.startswith('pubsub://'): url = urlparse.urlparse(endpoint) project = url.netloc topic = url.path.strip('/') interface.state.global_monitor = monitors.PubSubMonitor( credentials, project, topic, use_instrumented_http=True) else: logging.error('Monitoring is disabled because the endpoint provided is ' 'invalid or not supported: %s', endpoint) else: logging.error('Monitoring is disabled because credentials are not ' 'available') if args.ts_mon_target_type == 'device': interface.state.target = targets.DeviceTarget( args.ts_mon_device_region, args.ts_mon_device_role, args.ts_mon_device_network, args.ts_mon_device_hostname) if args.ts_mon_target_type == 'task': # pragma: no cover # Reimplement ArgumentParser.error, since we don't have access to the parser if not args.ts_mon_task_service_name: print >> sys.stderr, ('Argument --ts-mon-task-service-name must be ' 'provided when the target type is "task".') sys.exit(2) if not args.ts_mon_task_job_name: # pragma: no cover print >> sys.stderr, ('Argument --ts-mon-task-job-name must be provided ' 'when the target type is "task".') sys.exit(2) interface.state.target = targets.TaskTarget( args.ts_mon_task_service_name, args.ts_mon_task_job_name, args.ts_mon_task_region, args.ts_mon_task_hostname, args.ts_mon_task_number) interface.state.flush_mode = args.ts_mon_flush if args.ts_mon_flush == 'auto': interface.state.flush_thread = interface._FlushThread( args.ts_mon_flush_interval_secs) interface.state.flush_thread.start() standard_metrics.init()
def initialize(app=None, is_enabled_fn=None, cron_module='default', is_local_unittest=None): """Instruments webapp2 `app` with gae_ts_mon metrics. Instruments all the endpoints in `app` with basic metrics. Args: app (webapp2 app): the app to instrument. is_enabled_fn (function or None): a function returning bool if ts_mon should send the actual metrics. None (default) is equivalent to lambda: True. This allows apps to turn monitoring on or off dynamically, per app. cron_module (str): the name of the module handling the /internal/cron/ts_mon/send endpoint. This allows moving the cron job to any module the user wants. is_local_unittest (bool or None): whether we are running in a unittest. """ if is_local_unittest is None: # pragma: no cover # Since gae_ts_mon.initialize is called at module-scope by appengine apps, # AppengineTestCase.setUp() won't have run yet and none of the appengine # stubs will be initialized, so accessing Datastore or even getting the # application ID will fail. is_local_unittest = ('expect_tests' in sys.argv[0]) if is_enabled_fn is not None: interface.state.flush_enabled_fn = is_enabled_fn if app is not None: instrument_wsgi_application(app) if is_local_unittest or modules.get_current_module_name( ) == cron_module: instrument_wsgi_application(handlers.app) # Use the application ID as the service name and the module name as the job # name. if is_local_unittest: # pragma: no cover service_name = 'unittest' job_name = 'unittest' hostname = 'unittest' else: service_name = app_identity.get_application_id() job_name = modules.get_current_module_name() hostname = modules.get_current_version_name() runtime.set_shutdown_hook(_shutdown_hook) interface.state.target = targets.TaskTarget(service_name, job_name, shared.REGION, hostname, task_num=-1) interface.state.flush_mode = 'manual' interface.state.last_flushed = datetime.datetime.utcnow() # Don't send metrics when running on the dev appserver. if (is_local_unittest or os.environ.get('SERVER_SOFTWARE', '').startswith('Development')): logging.info('Using debug monitor') interface.state.global_monitor = monitors.DebugMonitor() else: logging.info('Using pubsub monitor %s/%s', shared.PUBSUB_PROJECT, shared.PUBSUB_TOPIC) interface.state.global_monitor = monitors.PubSubMonitor( monitors.APPENGINE_CREDENTIALS, shared.PUBSUB_PROJECT, shared.PUBSUB_TOPIC) shared.register_global_metrics([shared.appengine_default_version]) shared.register_global_metrics_callback(shared.INTERNAL_CALLBACK_NAME, _internal_callback) logging.info( 'Initialized ts_mon with service_name=%s, job_name=%s, ' 'hostname=%s', service_name, job_name, hostname)
def process_argparse_options(args): """Process command line arguments to initialize the global monitor. Also initializes the default target. Starts a background thread to automatically flush monitoring metrics if not disabled by command line arguments. Args: args (argparse.Namespace): the result of parsing the command line arguments """ # Parse the config file if it exists. config = load_machine_config(args.ts_mon_config_file) endpoint = config.get('endpoint', '') credentials = config.get('credentials', '') autogen_hostname = config.get('autogen_hostname', False) use_new_proto = config.get('use_new_proto', False) # Command-line args override the values in the config file. if args.ts_mon_endpoint is not None: endpoint = args.ts_mon_endpoint if args.ts_mon_credentials is not None: credentials = args.ts_mon_credentials if args.ts_mon_use_new_proto: use_new_proto = args.ts_mon_use_new_proto if args.ts_mon_target_type == 'device': hostname = args.ts_mon_device_hostname if args.ts_mon_autogen_hostname or autogen_hostname: hostname = 'autogen:' + hostname interface.state.target = targets.DeviceTarget( args.ts_mon_device_region, args.ts_mon_device_role, args.ts_mon_device_network, hostname) if args.ts_mon_target_type == 'task': # Reimplement ArgumentParser.error, since we don't have access to the parser if not args.ts_mon_task_service_name: print >> sys.stderr, ( 'Argument --ts-mon-task-service-name must be ' 'provided when the target type is "task".') sys.exit(2) if not args.ts_mon_task_job_name: print >> sys.stderr, ( 'Argument --ts-mon-task-job-name must be provided ' 'when the target type is "task".') sys.exit(2) hostname = args.ts_mon_task_hostname if args.ts_mon_autogen_hostname or autogen_hostname: hostname = 'autogen:' + hostname interface.state.target = targets.TaskTarget( args.ts_mon_task_service_name, args.ts_mon_task_job_name, args.ts_mon_task_region, hostname, args.ts_mon_task_number) interface.state.metric_name_prefix = args.ts_mon_metric_name_prefix interface.state.global_monitor = monitors.NullMonitor() if endpoint.startswith('file://'): interface.state.global_monitor = monitors.DebugMonitor( endpoint[len('file://'):]) elif endpoint.startswith('pubsub://'): if credentials: url = urlparse.urlparse(endpoint) project = url.netloc topic = url.path.strip('/') interface.state.global_monitor = monitors.PubSubMonitor( monitors.CredentialFactory.from_string(credentials), project, topic, use_instrumented_http=True, ca_certs=args.ts_mon_ca_certs) else: logging.error( 'ts_mon monitoring is disabled because credentials are not ' 'available') elif endpoint.startswith('https://'): interface.state.global_monitor = monitors.HttpsMonitor( endpoint, monitors.CredentialFactory.from_string(credentials), ca_certs=args.ts_mon_ca_certs) elif endpoint.lower() == 'none': logging.info('ts_mon monitoring has been explicitly disabled') else: logging.error( 'ts_mon monitoring is disabled because the endpoint provided' ' is invalid or not supported: %s', endpoint) interface.state.flush_mode = args.ts_mon_flush interface.state.use_new_proto = use_new_proto if args.ts_mon_flush == 'auto': interface.state.flush_thread = interface._FlushThread( args.ts_mon_flush_interval_secs) interface.state.flush_thread.start() standard_metrics.init()