def setUp(self):
project = 'test-project'
instance = 'test-instance'
options = {'project': project,
'zone': 'us-central1-f',
'instance_id': instance,
'config_dir': '/notfound',
'fix_stackdriver_labels_unsafe': True}
self.mockStub = mock.create_autospec(['projects'])
self.mockProjects = mock.create_autospec(
['metricDescriptors', 'timeSeries'])
self.mockMetricDescriptors = mock.create_autospec(
['create', 'delete', 'get'])
self.mockTimeSeries = mock.create_autospec(['create'])
self.mockStub.projects = Mock(return_value=self.mockProjects)
self.mockProjects.metricDescriptors = Mock(
return_value=self.mockMetricDescriptors)
self.mockProjects.timeSeries = Mock(return_value=self.mockTimeSeries)
# pylint: disable=invalid-name
self.mockCreateTimeSeries = Mock(spec=['execute'])
self.mockCreateDescriptor = Mock(spec=['execute'])
self.mockGetDescriptor = Mock(spec=['execute'])
self.mockDeleteDescriptor = Mock(spec=['execute'])
self.mockMetricDescriptors.create = Mock(
return_value=self.mockCreateDescriptor)
self.mockMetricDescriptors.delete = Mock(
return_value=self.mockDeleteDescriptor)
self.mockMetricDescriptors.get = Mock(
return_value=self.mockGetDescriptor)
self.mockTimeSeries.create = Mock(return_value=self.mockCreateTimeSeries)
self.service = StackdriverMetricsService(lambda: self.mockStub, options)
def setUp(self):
project = 'test-project'
instance = 'test-instance'
options = {'project': project, 'zone': 'us-central1-f',
'instance_id': instance,
'fix_stackdriver_labels_unsafe': True}
self.mockStub = mock.create_autospec(['projects'])
self.mockProjects = mock.create_autospec(
['metricDescriptors', 'timeSeries'])
self.mockMetricDescriptors = mock.create_autospec(
['create', 'delete', 'get'])
self.mockTimeSeries = mock.create_autospec(['create'])
self.mockStub.projects = Mock(return_value=self.mockProjects)
self.mockProjects.metricDescriptors = Mock(
return_value=self.mockMetricDescriptors)
self.mockProjects.timeSeries = Mock(return_value=self.mockTimeSeries)
# pylint: disable=invalid-name
self.mockCreateTimeSeries = Mock(spec=['execute'])
self.mockCreateDescriptor = Mock(spec=['execute'])
self.mockGetDescriptor = Mock(spec=['execute'])
self.mockDeleteDescriptor = Mock(spec=['execute'])
self.mockMetricDescriptors.create = Mock(
return_value=self.mockCreateDescriptor)
self.mockMetricDescriptors.delete = Mock(
return_value=self.mockDeleteDescriptor)
self.mockMetricDescriptors.get = Mock(
return_value=self.mockGetDescriptor)
self.mockTimeSeries.create = Mock(return_value=self.mockCreateTimeSeries)
self.service = StackdriverMetricsService(lambda: self.mockStub, options)
def setUp(self):
options = {
'project': 'PROJECT',
'source_path': 'IgnoreThis',
'config_dir': '/notfound',
'output_path': None
}
mockStub = mock.create_autospec(['projects', 'new_batch_http_request'])
mockStub.new_batch_http_request = Mock()
self.fake_batch_list = []
stackdriver = StackdriverMetricsService(lambda: mockStub, options)
self.upsertHandler = stackdriver_handlers.UpsertCustomDescriptorsHandler(
None, options, None)
self.mockProjects = mock.create_autospec(['metricDescriptors'])
self.mockMetricDescriptors = mock.create_autospec(
['create', 'delete', 'list'])
self.mockStub = mockStub
self.mockStub.projects = Mock(return_value=self.mockProjects)
self.mockProjects.metricDescriptors = Mock(
return_value=self.mockMetricDescriptors)
self.mockCreateDescriptor = Mock(spec=['execute'])
self.mockDeleteDescriptor = Mock(spec=['execute'])
self.mockListDescriptors = Mock(spec=['execute'])
self.mockMetricDescriptors.create = Mock(
return_value=self.mockCreateDescriptor)
self.mockMetricDescriptors.delete = Mock(
return_value=self.mockDeleteDescriptor)
self.mockMetricDescriptors.list = Mock(
return_value=self.mockListDescriptors)
self.mockMetricDescriptors.list_next = Mock(return_value=None)
self.options = options
self.project = options['project']
self.stackdriver = stackdriver
class StackdriverMetricsServiceTest(unittest.TestCase):
def setUp(self):
project = 'test-project'
instance = 'test-instance'
options = {
'project': project,
'zone': 'us-central1-f',
'instance_id': instance,
'fix_stackdriver_labels_unsafe': True
}
self.mockStub = mock.create_autospec(['projects'])
self.mockProjects = mock.create_autospec(
['metricDescriptors', 'timeSeries'])
self.mockMetricDescriptors = mock.create_autospec(
['create', 'delete', 'get'])
self.mockTimeSeries = mock.create_autospec(['create'])
self.mockStub.projects = Mock(return_value=self.mockProjects)
self.mockProjects.metricDescriptors = Mock(
return_value=self.mockMetricDescriptors)
self.mockProjects.timeSeries = Mock(return_value=self.mockTimeSeries)
# pylint: disable=invalid-name
self.mockCreateTimeSeries = Mock(spec=['execute'])
self.mockCreateDescriptor = Mock(spec=['execute'])
self.mockGetDescriptor = Mock(spec=['execute'])
self.mockDeleteDescriptor = Mock(spec=['execute'])
self.mockMetricDescriptors.create = Mock(
return_value=self.mockCreateDescriptor)
self.mockMetricDescriptors.delete = Mock(
return_value=self.mockDeleteDescriptor)
self.mockMetricDescriptors.get = Mock(
return_value=self.mockGetDescriptor)
self.mockTimeSeries.create = Mock(
return_value=self.mockCreateTimeSeries)
self.service = StackdriverMetricsService(lambda: self.mockStub,
options)
def test_find_problematic_elements(self):
content = """{
"error": {
"code": 400,
"message": "Field timeSeries[1].metric.labels[2] had an invalid value of \\"application\\": Unrecognized metric label.",
"errors": [
{
"message": "Field timeSeries[1].metric.labels[2] had an invalid value of \\"application\\": Unrecognized metric label.",
"domain": "global",
"reason": "badRequest"
}
],
"status": "INVALID_ARGUMENT"
}
}"""
Error = collections.namedtuple('Error', ['content'])
error = Error(content)
all_ts = [
{
'metric': {
'type': 'typeA',
'name': 'nameA'
}
},
{
'metric': {
'type': 'FOUND-TYPE',
'name': 'nameFound'
}
},
{
'metric': {
'type': 'typeC',
'name': 'nameC'
}
},
]
found = self.service.find_problematic_elements(error, all_ts)
self.assertEquals([(self.service.add_label_and_retry, 'application',
'FOUND-TYPE', all_ts[1])], found)
def test_add_label_and_retry_no_descriptor(self):
timeseries = 'OPAQUE'
status = ResponseStatus(404, 'Not Found')
self.mockMetricDescriptors.get.side_effect = HttpError(
status, 'Not Found')
self.service.add_label_and_retry('NewLabel', 'ExistingType',
timeseries)
self.assertEquals(0, self.mockStub.projects.list.call_count)
self.assertEquals(1, self.mockMetricDescriptors.get.call_count)
self.assertEquals(0, self.mockMetricDescriptors.delete.call_count)
self.assertEquals(0, self.mockMetricDescriptors.create.call_count)
self.assertEquals(0, self.mockTimeSeries.create.call_count)
def test_add_label_already_present(self):
timeseries = 'OPAQUE'
original_descriptor = {
'type': 'TYPE',
'labels': [{
'key': 'TestLabel'
}]
}
self.mockGetDescriptor.execute.side_effect = [original_descriptor]
self.service.add_label_and_retry('TestLabel', 'ExistingType',
timeseries)
self.assertEquals(0, self.mockStub.projects.list.call_count)
self.assertEquals(1, self.mockMetricDescriptors.get.call_count)
self.assertEquals(0, self.mockMetricDescriptors.delete.call_count)
self.assertEquals(0, self.mockMetricDescriptors.create.call_count)
self.assertEquals(1, self.mockTimeSeries.create.call_count)
def test_add_label_and_retry_cannot_delete(self):
timeseries = 'OPAQUE'
original_descriptor = {
'labels': [{
'key': 'TestLabel'
}],
'type': 'TYPE'
}
new_descriptor = dict(copy.deepcopy(original_descriptor))
new_descriptor['labels'].append({'key': 'NewLabel'})
self.mockGetDescriptor.execute.side_effect = [
original_descriptor, new_descriptor
]
self.mockDeleteDescriptor.execute.side_effect = HttpError(
ResponseStatus(404, 'Not Found'), 'Not Found')
self.mockCreateDescriptor.execute.side_effect = [new_descriptor]
self.service.add_label_and_retry('NewLabel', 'ExistingType',
timeseries)
self.assertEquals(0, self.mockStub.projects.list.call_count)
self.assertEquals(2, self.mockMetricDescriptors.get.call_count)
self.assertEquals(1, self.mockMetricDescriptors.delete.call_count)
self.assertEquals(1, self.mockMetricDescriptors.create.call_count)
self.assertEquals(1, self.mockTimeSeries.create.call_count)
def test_add_label_and_retry_cannot_create(self):
timeseries = 'OPAQUE'
original_descriptor = {
'labels': [{
'key': 'TestLabel'
}],
'type': 'TYPE'
}
self.mockGetDescriptor.execute.side_effect = [original_descriptor]
self.mockDeleteDescriptor.execute.side_effect = ResponseStatus(
200, 'ok')
self.mockCreateDescriptor.execute.side_effect = HttpError(
ResponseStatus(404, 'Not Found'), 'Not Found')
self.service.add_label_and_retry('NewLabel', 'ExistingType',
timeseries)
self.assertEquals(0, self.mockStub.projects.list.call_count)
self.assertEquals(1, self.mockMetricDescriptors.get.call_count)
self.assertEquals(1, self.mockMetricDescriptors.delete.call_count)
self.assertEquals(1, self.mockMetricDescriptors.create.call_count)
self.assertEquals(0, self.mockTimeSeries.create.call_count)
class StackdriverMetricsServiceTest(unittest.TestCase):
def setUp(self):
project = 'test-project'
instance = 'test-instance'
options = {'project': project, 'zone': 'us-central1-f',
'instance_id': instance,
'fix_stackdriver_labels_unsafe': True}
self.mockStub = mock.create_autospec(['projects'])
self.mockProjects = mock.create_autospec(
['metricDescriptors', 'timeSeries'])
self.mockMetricDescriptors = mock.create_autospec(
['create', 'delete', 'get'])
self.mockTimeSeries = mock.create_autospec(['create'])
self.mockStub.projects = Mock(return_value=self.mockProjects)
self.mockProjects.metricDescriptors = Mock(
return_value=self.mockMetricDescriptors)
self.mockProjects.timeSeries = Mock(return_value=self.mockTimeSeries)
# pylint: disable=invalid-name
self.mockCreateTimeSeries = Mock(spec=['execute'])
self.mockCreateDescriptor = Mock(spec=['execute'])
self.mockGetDescriptor = Mock(spec=['execute'])
self.mockDeleteDescriptor = Mock(spec=['execute'])
self.mockMetricDescriptors.create = Mock(
return_value=self.mockCreateDescriptor)
self.mockMetricDescriptors.delete = Mock(
return_value=self.mockDeleteDescriptor)
self.mockMetricDescriptors.get = Mock(
return_value=self.mockGetDescriptor)
self.mockTimeSeries.create = Mock(return_value=self.mockCreateTimeSeries)
self.service = StackdriverMetricsService(lambda: self.mockStub, options)
def test_find_problematic_elements(self):
content = """{
"error": {
"code": 400,
"message": "Field timeSeries[1].metric.labels[2] had an invalid value of \\"application\\": Unrecognized metric label.",
"errors": [
{
"message": "Field timeSeries[1].metric.labels[2] had an invalid value of \\"application\\": Unrecognized metric label.",
"domain": "global",
"reason": "badRequest"
}
],
"status": "INVALID_ARGUMENT"
}
}"""
Error = collections.namedtuple('Error', ['content'])
error = Error(content)
all_ts = [
{'metric': {'type': 'typeA', 'name': 'nameA'}},
{'metric': {'type': 'FOUND-TYPE', 'name': 'nameFound'}},
{'metric': {'type': 'typeC', 'name': 'nameC'}},
]
found = self.service.find_problematic_elements(error, all_ts)
self.assertEquals([(self.service.add_label_and_retry,
'application',
'FOUND-TYPE', all_ts[1])], found)
def test_add_label_and_retry_no_descriptor(self):
timeseries = 'OPAQUE'
status = ResponseStatus(404, 'Not Found')
self.mockMetricDescriptors.get.side_effect = HttpError(
status, 'Not Found')
self.service.add_label_and_retry('NewLabel', 'ExistingType', timeseries)
self.assertEquals(0, self.mockStub.projects.list.call_count)
self.assertEquals(1, self.mockMetricDescriptors.get.call_count)
self.assertEquals(0, self.mockMetricDescriptors.delete.call_count)
self.assertEquals(0, self.mockMetricDescriptors.create.call_count)
self.assertEquals(0, self.mockTimeSeries.create.call_count)
def test_add_label_already_present(self):
timeseries = 'OPAQUE'
original_descriptor = {'type': 'TYPE',
'labels': [{'key': 'TestLabel'}]}
self.mockGetDescriptor.execute.side_effect = [original_descriptor]
self.service.add_label_and_retry('TestLabel', 'ExistingType', timeseries)
self.assertEquals(0, self.mockStub.projects.list.call_count)
self.assertEquals(1, self.mockMetricDescriptors.get.call_count)
self.assertEquals(0, self.mockMetricDescriptors.delete.call_count)
self.assertEquals(0, self.mockMetricDescriptors.create.call_count)
self.assertEquals(1, self.mockTimeSeries.create.call_count)
def test_add_label_and_retry_cannot_delete(self):
timeseries = 'OPAQUE'
original_descriptor = {'labels': [{'key': 'TestLabel'}],
'type': 'TYPE'
}
new_descriptor = dict(copy.deepcopy(original_descriptor))
new_descriptor['labels'].append({'key': 'NewLabel'})
self.mockGetDescriptor.execute.side_effect = [
original_descriptor, new_descriptor]
self.mockDeleteDescriptor.execute.side_effect = HttpError(
ResponseStatus(404, 'Not Found'), 'Not Found')
self.mockCreateDescriptor.execute.side_effect = [new_descriptor]
self.service.add_label_and_retry('NewLabel', 'ExistingType', timeseries)
self.assertEquals(0, self.mockStub.projects.list.call_count)
self.assertEquals(2, self.mockMetricDescriptors.get.call_count)
self.assertEquals(1, self.mockMetricDescriptors.delete.call_count)
self.assertEquals(1, self.mockMetricDescriptors.create.call_count)
self.assertEquals(1, self.mockTimeSeries.create.call_count)
def test_add_label_and_retry_cannot_create(self):
timeseries = 'OPAQUE'
original_descriptor = {'labels': [{'key': 'TestLabel'}],
'type': 'TYPE'
}
self.mockGetDescriptor.execute.side_effect = [original_descriptor]
self.mockDeleteDescriptor.execute.side_effect = ResponseStatus(
200, 'ok')
self.mockCreateDescriptor.execute.side_effect = HttpError(
ResponseStatus(404, 'Not Found'), 'Not Found')
self.service.add_label_and_retry('NewLabel', 'ExistingType', timeseries)
self.assertEquals(0, self.mockStub.projects.list.call_count)
self.assertEquals(1, self.mockMetricDescriptors.get.call_count)
self.assertEquals(1, self.mockMetricDescriptors.delete.call_count)
self.assertEquals(1, self.mockMetricDescriptors.create.call_count)
self.assertEquals(0, self.mockTimeSeries.create.call_count)
def main():
init_logging('metric_collector.log')
options = vars(get_options())
spectator = spectator_client.SpectatorClient(options)
try:
stackdriver = StackdriverMetricsService.make_service(options)
except IOError as ioerror:
logging.error(
'Could not create stackdriver client'
' -- Stackdriver will be unavailable\n%s', ioerror)
stackdriver = None
registry = []
registry.extend([
CommandDefinition(handlers.BaseHandler(options, registry), '/', 'Home',
CommandRequest(options=options),
'Home page for Spinnaker metric administration.'),
CommandDefinition(
stackdriver_handlers.ClearCustomDescriptorsHandler(
options,
stackdriver), '/stackdriver/clear_descriptors', 'clear',
CommandRequest(options=options),
'Clear all the Stackdriver Custom Metrics'),
CommandDefinition(
stackdriver_handlers.ListCustomDescriptorsHandler(
options, stackdriver), '/stackdriver/list_descriptors', 'list',
CommandRequest(content_type='application/json', options=options),
'Get the JSON of all the Stackdriver Custom Metric Descriptors.'),
CommandDefinition(
stackdriver_handlers.UpsertCustomDescriptorsHandler(
options, stackdriver), None, 'upsert_descriptors',
CommandRequest(options=options),
'Given a file of Stackdriver Custom Metric Desciptors,'
' update the existing ones and add the new ones.'
' WARNING: Historic time-series data may be lost on update.'),
CommandDefinition(
handlers.DumpMetricsHandler(options, spectator), '/dump', 'dump',
CommandRequest(options=options),
'Show current raw metric JSON from all the servers.'),
CommandDefinition(
handlers.ExploreCustomDescriptorsHandler(options, spectator),
'/explore',
'explore',
CommandRequest(options=options),
'Explore metric type usage across Spinnaker microservices.',
),
CommandDefinition(
handlers.ShowCurrentMetricsHandler(options,
spectator), '/show', 'show',
CommandRequest(options=options),
'Show current metric JSON for all Spinnaker.'),
])
if options.get('command', None):
command_processor.process_command(options['command'], registry)
return
if options.get('monitor', None):
logging.info('Starting Monitor every %d s', options['period'])
metric_service = stackdriver
monitor = Monitor(spectator, metric_service, options)
threading.Thread(target=monitor, name='monitor').start()
logging.info('Starting HTTP server on port %d', options['port'])
url_path_to_handler = {entry.url_path: entry.handler for entry in registry}
httpd = HttpServer(options['port'], url_path_to_handler)
httpd.serve_forever()
sys.exit(-1)
