def SetupTsMonGlobalState(service_name): """Uses a dummy argument parser to get the default behavior from ts-mon. Args: service_name: the name of the task we are sending metrics from. """ if not config: return # google-api-client has too much noisey logging. googleapiclient.discovery.logger.setLevel(logging.WARNING) parser = argparse.ArgumentParser() config.add_argparse_options(parser) try: config.process_argparse_options( parser.parse_args(args=[ '--ts-mon-target-type', 'task', '--ts-mon-task-service-name', service_name, '--ts-mon-task-job-name', service_name, ])) except Exception as e: logging.warning( 'Failed to configure ts_mon, monitoring is disabled: %s', e, exc_info=True)
def SetupTsMonGlobalState(service_name, short_lived=False, indirect=False, auto_flush=True): """Uses a dummy argument parser to get the default behavior from ts-mon. Args: service_name: The name of the task we are sending metrics from. short_lived: Whether this process is short-lived and should use the autogen hostname prefix. indirect: Whether to create a metrics.METRICS_QUEUE object and a separate process for indirect metrics flushing. Useful for forking, because forking would normally create a duplicate ts_mon thread. auto_flush: Whether to create a thread to automatically flush metrics every minute. """ if not config: return TrivialContextManager() if indirect: return _CreateTsMonFlushingProcess([service_name], {'short_lived': short_lived}) # google-api-client has too much noisey logging. googleapiclient.discovery.logger.setLevel(logging.WARNING) parser = argparse.ArgumentParser() config.add_argparse_options(parser) args = [ '--ts-mon-target-type', 'task', '--ts-mon-task-service-name', service_name, '--ts-mon-task-job-name', service_name, ] # Short lived processes will have autogen: prepended to their hostname and # use task-number=PID to trigger shorter retention policies under # chrome-infra@, and used by a Monarch precomputation to group across the # task number. # Furthermore, we assume they manually call ts_mon.Flush(), because the # ts_mon thread will drop messages if the process exits before it flushes. if short_lived: auto_flush = False fqdn = socket.getfqdn().lower() host = fqdn.split('.')[0] args.extend(['--ts-mon-task-hostname', 'autogen:' + host, '--ts-mon-task-number', os.getpid()]) args.extend(['--ts-mon-flush', 'auto' if auto_flush else 'manual']) try: config.process_argparse_options(parser.parse_args(args=args)) logging.notice('ts_mon was set up.') _WasSetup = True except Exception as e: logging.warning('Failed to configure ts_mon, monitoring is disabled: %s', e, exc_info=True) return TrivialContextManager()
def test_no_args(self, fake_monitor): singleton = mock.Mock() fake_monitor.return_value = singleton p = argparse.ArgumentParser() config.add_argparse_options(p) args = p.parse_args([]) config.process_argparse_options(args) self.assertEqual(1, len(fake_monitor.mock_calls)) self.assertIs(interface.state.global_monitor, singleton)
def test_dryrun_args(self, fake_monitor): singleton = mock.Mock() fake_monitor.return_value = singleton p = argparse.ArgumentParser() config.add_argparse_options(p) args = p.parse_args(['--ts-mon-endpoint', 'file://foo.txt']) config.process_argparse_options(args) fake_monitor.assert_called_once_with('foo.txt') self.assertIs(interface.state.global_monitor, singleton)
def test_manual_flush(self, fake_fqdn, fake_get): fake_fqdn.return_value = 'foo' fake_get.return_value.side_effect = requests.exceptions.ConnectionError p = argparse.ArgumentParser() config.add_argparse_options(p) args = p.parse_args(['--ts-mon-flush', 'manual']) config.process_argparse_options(args) self.assertIsNone(interface.state.flush_thread)
def test_manual_flush(self, fake_target, fake_monitor, fake_fqdn): singleton = mock.Mock() fake_monitor.return_value = singleton fake_target.return_value = singleton fake_fqdn.return_value = 'foo' p = argparse.ArgumentParser() config.add_argparse_options(p) args = p.parse_args(['--ts-mon-flush', 'manual']) config.process_argparse_options(args) self.assertIsNone(interface.state.flush_thread)
def test_task_args_missing_job_name(self): p = argparse.ArgumentParser() config.add_argparse_options(p) args = p.parse_args([ '--ts-mon-credentials', '/path/to/creds.p8.json', '--ts-mon-target-type', 'task', '--ts-mon-task-service-name', 'serv', '--ts-mon-task-region', 'reg', '--ts-mon-task-hostname', 'host', '--ts-mon-task-number', '1' ]) with self.assertRaises(SystemExit): config.process_argparse_options(args)
def test_pubsub_args(self, fake_monitor): singleton = mock.Mock() fake_monitor.return_value = singleton p = argparse.ArgumentParser() config.add_argparse_options(p) args = p.parse_args(['--ts-mon-credentials', '/path/to/creds.p8.json', '--ts-mon-endpoint', 'pubsub://mytopic/myproject']) config.process_argparse_options(args) fake_monitor.assert_called_once_with( '/path/to/creds.p8.json', 'mytopic', 'myproject', use_instrumented_http=True) self.assertIs(interface.state.global_monitor, singleton)
def test_pubsub_without_credentials(self, fake_monitor): # safety net, not supposed to be called. singleton = mock.Mock() fake_monitor.return_value = singleton p = argparse.ArgumentParser() config.add_argparse_options(p) args = p.parse_args(['--ts-mon-config-file', os.path.join(DATA_DIR, 'empty-config-file.json'), '--ts-mon-endpoint', 'pubsub://mytopic/myproject']) config.process_argparse_options(args) self.assertIsInstance(interface.state.global_monitor, monitors.NullMonitor)
def test_explicit_disable_args(self, fake_fqdn, fake_get): fake_fqdn.return_value = 'foo' fake_get.return_value.side_effect = requests.exceptions.ConnectionError p = argparse.ArgumentParser() config.add_argparse_options(p) args = p.parse_args([ '--ts-mon-credentials', '/path/to/creds.p8.json', '--ts-mon-endpoint', 'none']) config.process_argparse_options(args) self.assertIsInstance(interface.state.global_monitor, monitors.NullMonitor)
def test_fallback_monitor_args(self, fake_fqdn, fake_get): fake_fqdn.return_value = 'foo' fake_get.return_value.side_effect = requests.exceptions.ConnectionError p = argparse.ArgumentParser() config.add_argparse_options(p) args = p.parse_args([ '--ts-mon-credentials', '/path/to/creds.p8.json', '--ts-mon-endpoint', 'unsupported://www.googleapis.com/some/api']) config.process_argparse_options(args) self.assertIsInstance(interface.state.global_monitor, monitors.NullMonitor)
def test_monitor_args(self, fake_monitor): singleton = mock.Mock() fake_monitor.return_value = singleton p = argparse.ArgumentParser() config.add_argparse_options(p) args = p.parse_args(['--ts-mon-credentials', '/path/to/creds.p8.json', '--ts-mon-endpoint', 'https://foo.tld/api']) config.process_argparse_options(args) fake_monitor.assert_called_once_with( '/path/to/creds.p8.json', 'https://foo.tld/api', use_instrumented_http=True) self.assertIs(interface.state.global_monitor, singleton)
def test_default_target_fqdn_without_domain(self, fake_fqdn, fake_get): fake_fqdn.return_value = 'SLAVE1-A1' fake_get.return_value.side_effect = requests.exceptions.ConnectionError p = argparse.ArgumentParser() config.add_argparse_options(p) args = p.parse_args([ '--ts-mon-credentials', '/path/to/creds.p8.json', '--ts-mon-endpoint', 'unsupported://www.googleapis.com/some/api']) config.process_argparse_options(args) self.assertIsInstance(interface.state.target, targets.DeviceTarget) self.assertEquals(interface.state.target.hostname, 'slave1-a1') self.assertEquals(interface.state.target.region, '')
def test_default_target_uppercase_fqdn(self, fake_fqdn, fake_get): fake_fqdn.return_value = 'SLAVE1-A1.REG.TLD' fake_get.return_value.side_effect = requests.exceptions.ConnectionError p = argparse.ArgumentParser() config.add_argparse_options(p) args = p.parse_args([ '--ts-mon-credentials', '/path/to/creds.p8.json', '--ts-mon-endpoint', 'unsupported://www.googleapis.com/some/api']) config.process_argparse_options(args) self.assertIsInstance(interface.state.target, targets.DeviceTarget) self.assertEquals(interface.state.target.hostname, 'slave1-a1') self.assertEquals(interface.state.target.region, 'reg')
def test_device_args(self, fake_target, _fake_monitor): singleton = mock.Mock() fake_target.return_value = singleton p = argparse.ArgumentParser() config.add_argparse_options(p) args = p.parse_args(['--ts-mon-credentials', '/path/to/creds.p8.json', '--ts-mon-target-type', 'device', '--ts-mon-device-region', 'reg', '--ts-mon-device-network', 'net', '--ts-mon-device-hostname', 'host']) config.process_argparse_options(args) fake_target.assert_called_once_with('reg', 'net', 'host') self.assertIs(interface.state.default_target, singleton)
def test_flush_all_non_instrumented_http(self, fake_monitor): singleton = mock.Mock() fake_monitor.return_value = singleton p = argparse.ArgumentParser() config.add_argparse_options(p) args = p.parse_args(['--ts-mon-credentials', '/path/to/creds.p8.json', '--ts-mon-endpoint', 'https://foo.tld/api', '--ts-mon-flush', 'all']) config.process_argparse_options(args) fake_monitor.assert_called_once_with( '/path/to/creds.p8.json', 'https://foo.tld/api', use_instrumented_http=False) self.assertIs(interface.state.global_monitor, singleton)
def test_pubsub_args_non_instrumented_http(self, fake_monitor): singleton = mock.Mock() fake_monitor.return_value = singleton p = argparse.ArgumentParser() config.add_argparse_options(p) args = p.parse_args(['--ts-mon-credentials', '/path/to/creds.p8.json', '--ts-mon-endpoint', 'pubsub://mytopic/myproject', '--ts-mon-flush', 'all']) config.process_argparse_options(args) fake_monitor.assert_called_once_with( '/path/to/creds.p8.json', 'mytopic', 'myproject', use_instrumented_http=False) self.assertIs(interface.state.global_monitor, singleton)
def test_device_args(self): p = argparse.ArgumentParser() config.add_argparse_options(p) args = p.parse_args(['--ts-mon-credentials', '/path/to/creds.p8.json', '--ts-mon-target-type', 'device', '--ts-mon-device-region', 'reg', '--ts-mon-device-role', 'role', '--ts-mon-device-network', 'net', '--ts-mon-device-hostname', 'host']) config.process_argparse_options(args) self.assertEqual(interface.state.target.region, 'reg') self.assertEqual(interface.state.target.role, 'role') self.assertEqual(interface.state.target.network, 'net') self.assertEqual(interface.state.target.hostname, 'host')
def test_autogen_device_args(self): p = argparse.ArgumentParser() config.add_argparse_options(p) args = p.parse_args([ '--ts-mon-credentials', '/path/to/creds.p8.json', '--ts-mon-target-type', 'device', '--ts-mon-device-region', 'reg', '--ts-mon-device-role', 'role', '--ts-mon-device-network', 'net', '--ts-mon-device-hostname', 'host', '--ts-mon-autogen-hostname' ]) config.process_argparse_options(args) self.assertEqual(interface.state.target.region, 'reg') self.assertEqual(interface.state.target.role, 'role') self.assertEqual(interface.state.target.network, 'net') self.assertEqual(interface.state.target.hostname, 'autogen:host')
def test_task_args(self, fake_target): singleton = mock.Mock() fake_target.return_value = singleton p = argparse.ArgumentParser() config.add_argparse_options(p) args = p.parse_args([ '--ts-mon-credentials', '/path/to/creds.p8.json', '--ts-mon-target-type', 'task', '--ts-mon-task-service-name', 'serv', '--ts-mon-task-job-name', 'job', '--ts-mon-task-region', 'reg', '--ts-mon-task-hostname', 'host', '--ts-mon-task-number', '1' ]) config.process_argparse_options(args) fake_target.assert_called_once_with('serv', 'job', 'reg', 'host', 1) self.assertIs(interface.state.target, singleton)
def test_task_args(self, fake_target): singleton = mock.Mock() fake_target.return_value = singleton p = argparse.ArgumentParser() config.add_argparse_options(p) args = p.parse_args(['--ts-mon-credentials', '/path/to/creds.p8.json', '--ts-mon-target-type', 'task', '--ts-mon-task-service-name', 'serv', '--ts-mon-task-job-name', 'job', '--ts-mon-task-region', 'reg', '--ts-mon-task-hostname', 'host', '--ts-mon-task-number', '1']) config.process_argparse_options(args) fake_target.assert_called_once_with('serv', 'job', 'reg', 'host', 1) self.assertIs(interface.state.target, singleton)
def _GenerateTsMonArgparseOptions(service_name, short_lived, auto_flush, debug_file, task_num): """Generates an arg list for ts-mon to consume. Args: service_name: The name of the task we are sending metrics from. short_lived: Whether this process is short-lived and should use the autogen hostname prefix. auto_flush: Whether to create a thread to automatically flush metrics every minute. debug_file: If non-none, send metrics to this path instead of to PubSub. task_num: Override the default task num of 0. """ parser = argparse.ArgumentParser() config.add_argparse_options(parser) args = [ '--ts-mon-target-type', 'task', '--ts-mon-task-service-name', service_name, '--ts-mon-task-job-name', service_name, ] if debug_file: args.extend(['--ts-mon-endpoint', 'file://' + debug_file]) # Short lived processes will have autogen: prepended to their hostname and # use task-number=PID to trigger shorter retention policies under # chrome-infra@, and used by a Monarch precomputation to group across the # task number. # Furthermore, we assume they manually call ts_mon.Flush(), because the # ts_mon thread will drop messages if the process exits before it flushes. if short_lived: auto_flush = False fqdn = socket.getfqdn().lower() host = fqdn.split('.')[0] args.extend([ '--ts-mon-task-hostname', 'autogen:' + host, '--ts-mon-task-number', str(os.getpid()) ]) elif task_num: args.extend(['--ts-mon-task-number', str(task_num)]) args.extend(['--ts-mon-flush', 'auto' if auto_flush else 'manual']) return parser.parse_args(args=args)
def test_autogen_task_args(self): p = argparse.ArgumentParser() config.add_argparse_options(p) args = p.parse_args([ '--ts-mon-credentials', '/path/to/creds.p8.json', '--ts-mon-target-type', 'task', '--ts-mon-task-service-name', 'serv', '--ts-mon-task-job-name', 'job', '--ts-mon-task-region', 'reg', '--ts-mon-task-hostname', 'host', '--ts-mon-task-number', '1', '--ts-mon-autogen-hostname' ]) config.process_argparse_options(args) self.assertEqual(interface.state.target.service_name, 'serv') self.assertEqual(interface.state.target.job_name, 'job') self.assertEqual(interface.state.target.region, 'reg') self.assertEqual(interface.state.target.hostname, 'autogen:host') self.assertEqual(interface.state.target.task_num, 1)
def test_task_args(self): p = argparse.ArgumentParser() config.add_argparse_options(p) args = p.parse_args(['--ts-mon-credentials', '/path/to/creds.p8.json', '--ts-mon-target-type', 'task', '--ts-mon-task-service-name', 'serv', '--ts-mon-task-job-name', 'job', '--ts-mon-task-region', 'reg', '--ts-mon-task-hostname', 'host', '--ts-mon-task-number', '1']) config.process_argparse_options(args) self.assertEqual(interface.state.target.service_name, 'serv') self.assertEqual(interface.state.target.job_name, 'job') self.assertEqual(interface.state.target.region, 'reg') self.assertEqual(interface.state.target.hostname, 'host') self.assertEqual(interface.state.target.task_num, 1)
def test_autogen_device_config(self): self.mock(config, 'load_machine_config', lambda x: { 'autogen_hostname': True, 'credentials': '/path/to/creds.p8.json', 'endpoint': 'test://endpoint'}) p = argparse.ArgumentParser() config.add_argparse_options(p) args = p.parse_args([ '--ts-mon-target-type', 'device', '--ts-mon-device-region', 'reg', '--ts-mon-device-role', 'role', '--ts-mon-device-network', 'net', '--ts-mon-device-hostname', 'host']) config.process_argparse_options(args) self.assertEqual(interface.state.target.region, 'reg') self.assertEqual(interface.state.target.role, 'role') self.assertEqual(interface.state.target.network, 'net') self.assertEqual(interface.state.target.hostname, 'autogen:host')
def test_autogen_task_config(self): self.mock(config, 'load_machine_config', lambda x: { 'autogen_hostname': True, 'credentials': '/path/to/creds.p8.json', 'endpoint': 'test://endpoint'}) p = argparse.ArgumentParser() config.add_argparse_options(p) args = p.parse_args(['--ts-mon-target-type', 'task', '--ts-mon-task-service-name', 'serv', '--ts-mon-task-job-name', 'job', '--ts-mon-task-region', 'reg', '--ts-mon-task-hostname', 'host', '--ts-mon-task-number', '1']) config.process_argparse_options(args) self.assertEqual(interface.state.target.service_name, 'serv') self.assertEqual(interface.state.target.job_name, 'job') self.assertEqual(interface.state.target.region, 'reg') self.assertEqual(interface.state.target.hostname, 'autogen:host') self.assertEqual(interface.state.target.task_num, 1)
def test_fallback_monitor_args(self, fake_target, fake_monitor, fake_fqdn): singleton = mock.Mock() fake_monitor.return_value = singleton fake_target.return_value = singleton fake_fqdn.return_value = 'foo' p = argparse.ArgumentParser() config.add_argparse_options(p) args = p.parse_args([ '--ts-mon-credentials', '/path/to/creds.p8.json', '--ts-mon-endpoint', 'https://www.googleapis.com/acquisitions/v1_mon_shared/storage']) config.process_argparse_options(args) fake_monitor.assert_called_once_with( '/path/to/creds.p8.json', 'https://www.googleapis.com/acquisitions/v1_mon_shared/storage', use_instrumented_http=True) self.assertIs(interface.state.global_monitor, singleton) fake_target.assert_called_once_with('', '', 'foo') self.assertIs(interface.state.default_target, singleton)
def test_pubsub_monitor_args(self, fake_fqdn, fake_get): fake_fqdn.return_value = 'slave1-a1.reg.tld' fake_get.return_value.side_effect = requests.exceptions.ConnectionError p = argparse.ArgumentParser() config.add_argparse_options(p) args = p.parse_args([ '--ts-mon-credentials', '/path/to/creds.p8.json', '--ts-mon-endpoint', 'pubsub://invalid-project/invalid-topic']) config.process_argparse_options(args) self.assertIsInstance(interface.state.global_monitor, monitors.PubSubMonitor) self.assertIsInstance(interface.state.target, targets.DeviceTarget) self.assertEquals(interface.state.target.hostname, 'slave1-a1') self.assertEquals(interface.state.target.region, 'reg') self.assertEquals(args.ts_mon_flush, 'auto') self.assertIsNotNone(interface.state.flush_thread) self.assertTrue(standard_metrics.up.get())
def test_default_monitor_args(self, fake_target, fake_monitor, fake_fqdn): singleton = mock.Mock() fake_monitor.return_value = singleton fake_target.return_value = singleton fake_fqdn.return_value = 'slave1-a1.reg.tld' p = argparse.ArgumentParser() config.add_argparse_options(p) args = p.parse_args([ '--ts-mon-credentials', '/path/to/creds.p8.json', '--ts-mon-endpoint', 'https://www.googleapis.com/acquisitions/v1_mon_shared/storage']) config.process_argparse_options(args) fake_monitor.assert_called_once_with( '/path/to/creds.p8.json', 'https://www.googleapis.com/acquisitions/v1_mon_shared/storage', use_instrumented_http=True) self.assertIs(interface.state.global_monitor, singleton) fake_target.assert_called_once_with('reg', '1', 'slave1-a1') self.assertIs(interface.state.default_target, singleton) self.assertEquals(args.ts_mon_flush, 'auto') self.assertIsNotNone(interface.state.flush_thread) self.assertTrue(standard_metrics.up.get())
def test_metric_name_prefix(self): p = argparse.ArgumentParser() config.add_argparse_options(p) args = p.parse_args(['--ts-mon-metric-name-prefix', '/test/random/']) config.process_argparse_options(args) self.assertEqual('/test/random/', interface.state.metric_name_prefix)