def test_events_process_limit_with_batches(dynatrace_check, test_instance):
"""
Respecting `events_process_limit` config setting should happen with batch event retrieval too
"""
dynatrace_check._current_time_seconds = mock.MagicMock(
return_value=1613485584)
dynatrace_check._process_topology = mock.MagicMock(return_value=None)
url = test_instance['url']
timestamp = dynatrace_check._generate_bootstrap_timestamp(
test_instance['events_boostrap_days'])
with requests_mock.Mocker() as m:
url1 = '{}/api/v1/events?from={}'.format(url, timestamp)
url2 = '{}/api/v1/events?cursor={}'.format(url, '123')
url3 = '{}/api/v1/events?cursor={}'.format(url, '345')
m.get(url1,
status_code=200,
text=read_file("events_set1_response.json"))
m.get(url2,
status_code=200,
text=read_file("events_set2_response.json"))
m.get(url3,
status_code=200,
text=read_file("events_set3_response.json"))
dynatrace_check.run()
aggregator.assert_service_check(CHECK_NAME,
count=1,
status=AgentCheck.WARNING)
assert len(
aggregator.events) == test_instance.get("events_process_limit")
def test_unicode_in_response_text(dynatrace_check, test_instance):
dynatrace_check._current_time_seconds = mock.MagicMock(
return_value=1613485584)
url = test_instance['url']
timestamp = dynatrace_check._generate_bootstrap_timestamp(
test_instance['events_boostrap_days'])
with requests_mock.Mocker() as m:
m.get("{}/api/v1/entity/infrastructure/hosts".format(url),
status_code=200,
text=read_file('host_response.json'))
m.get("{}/api/v1/entity/applications".format(url),
status_code=200,
text='[]')
m.get("{}/api/v1/entity/services".format(url),
status_code=200,
text='[]')
m.get("{}/api/v1/entity/infrastructure/processes".format(url),
status_code=200,
text='[]')
m.get("{}/api/v1/entity/infrastructure/process-groups".format(url),
status_code=200,
text='[]')
m.get('{}/api/v1/events?from={}'.format(url, timestamp),
status_code=200,
text=read_file('unicode_topology_event_response.json'))
dynatrace_check.run()
aggregator.assert_service_check(CHECK_NAME,
count=1,
status=AgentCheck.OK)
unicode_data = topology.get_snapshot('').get(
'components')[0]['data']['osVersion']
assert unicode_data == 'Windows Server 2016 Datacenter 1607, ver. 10.0.14393 with unicode char: '
assert telemetry._topology_events[0][
'msg_text'] == 'PROCESS_RESTART on aws-cni™'
def test_state_data(state, dynatrace_check, test_instance):
"""
Check is the right timestamp is writen to the state
"""
dynatrace_check._current_time_seconds = mock.MagicMock(
return_value=1613485584)
dynatrace_check._process_topology = mock.MagicMock(return_value=None)
url = test_instance['url']
timestamp = dynatrace_check._generate_bootstrap_timestamp(
test_instance['events_boostrap_days'])
state_instance = StateDescriptor(
"instance.dynatrace_event.https_instance.live.dynatrace.com",
"dynatrace_event.d")
state.assert_state(state_instance, None)
events_file = 'no_events_response.json'
with requests_mock.Mocker() as m:
m.get('{}/api/v1/events?from={}'.format(url, timestamp),
status_code=200,
text=read_file(events_file))
dynatrace_check.run()
aggregator.assert_service_check(CHECK_NAME,
count=1,
status=AgentCheck.OK)
mocked_response_data = load_json_from_file(events_file)
new_state = State(
{'last_processed_event_timestamp': mocked_response_data.get('to')})
state.assert_state(state_instance, new_state)
def test_generated_events(dynatrace_check, test_instance):
"""
Testing Dynatrace check should produce full events
"""
dynatrace_check._current_time_seconds = mock.MagicMock(
return_value=1613485584)
dynatrace_check._process_topology = mock.MagicMock(return_value=None)
dynatrace_check._timestamp_to_sts_datetime = mock.MagicMock(
return_value='Feb 15, 2021, 22:26:00')
url = test_instance['url']
timestamp = dynatrace_check._generate_bootstrap_timestamp(
test_instance['events_boostrap_days'])
with requests_mock.Mocker() as m:
m.get('{}/api/v1/events?from={}'.format(url, timestamp),
status_code=200,
text=read_file('9_events_response.json'))
dynatrace_check.run()
aggregator.assert_service_check(CHECK_NAME,
count=1,
status=AgentCheck.OK)
assert len(aggregator.events) == 8
assert len(telemetry._topology_events) == 1
expected_events = load_json_from_file('expected_events.json')
for expected_event in expected_events:
aggregator.assert_event(expected_event.get('msg_text'))
expected_topology_events = load_json_from_file(
'expected_topology_events.json')
for expected_event in expected_topology_events:
# telemetry.assert_topology_event(expected_event)
assert expected_event == telemetry._topology_events[0]
def test_exception_is_propagated_to_service_check(dynatrace_check):
"""
Test to raise a exception from code that talks to API endpoint throws exception
"""
dynatrace_check._get_dynatrace_json_response = mock.MagicMock(
side_effect=Exception("Mocked exception occurred"))
dynatrace_check._process_topology = mock.MagicMock(return_value=None)
dynatrace_check.run()
assert len(aggregator.events) == 0
aggregator.assert_service_check(CHECK_NAME,
count=1,
status=AgentCheck.CRITICAL,
message='Mocked exception occurred')
def test_simulated_ok_events(dynatrace_check, test_instance):
"""
Test if there is correct number of simulated events created
"""
dynatrace_check._current_time_seconds = mock.MagicMock(
return_value=1613485584)
url = test_instance['url']
timestamp = dynatrace_check._generate_bootstrap_timestamp(
test_instance['events_boostrap_days'])
with requests_mock.Mocker() as m:
m.get("{}/api/v1/entity/infrastructure/hosts".format(url),
status_code=200,
text=read_file('host_response.json'))
m.get("{}/api/v1/entity/applications".format(url),
status_code=200,
text=read_file('application_response.json'))
m.get("{}/api/v1/entity/services".format(url),
status_code=200,
text=read_file('service_response.json'))
m.get("{}/api/v1/entity/infrastructure/processes".format(url),
status_code=200,
text=read_file('process_response.json'))
m.get("{}/api/v1/entity/infrastructure/process-groups".format(url),
status_code=200,
text=read_file('process-group_response.json'))
m.get('{}/api/v1/events?from={}'.format(url, timestamp),
status_code=200,
text=read_file('9_events_response.json'))
dynatrace_check.run()
aggregator.assert_service_check(CHECK_NAME,
count=1,
status=AgentCheck.OK)
assert len(topology.get_snapshot('').get('components')) == 14
assert len(aggregator.events) == 22
real_events = [
e for e in aggregator.events
if 'source:StackState Agent' not in e.get('tags', [])
]
simulated_events = [
e for e in aggregator.events
if 'source:StackState Agent' in e.get('tags', [])
]
assert len(real_events) == 8
assert len(simulated_events) == 14
assert len(telemetry._topology_events) == 1
def test_tags(dynatrace_check, test_instance):
dynatrace_check._current_time_seconds = mock.MagicMock(return_value=1613485584)
url = test_instance['url']
timestamp = dynatrace_check._generate_bootstrap_timestamp(test_instance['events_boostrap_days'])
with requests_mock.Mocker() as m:
m.get("{}/api/v1/entity/infrastructure/hosts".format(url), status_code=200,
text=read_file('HOST-9106C06F228CEC6B.json'))
m.get("{}/api/v1/entity/applications".format(url), status_code=200, text='[]')
m.get("{}/api/v1/entity/services".format(url), status_code=200, text='[]')
m.get("{}/api/v1/entity/infrastructure/processes".format(url), status_code=200, text='[]')
m.get("{}/api/v1/entity/infrastructure/process-groups".format(url), status_code=200, text='[]')
m.get('{}/api/v1/events?from={}'.format(url, timestamp), status_code=200, text='[]')
dynatrace_check.run()
aggregator.assert_service_check(CHECK_NAME, count=1, status=AgentCheck.OK)
components = topology.get_snapshot('')['components']
assert components[0]['data']['environments'] == ['test-environment']
assert components[0]['data']['domain'] == 'test-domain'
assert components[0]['data']['layer'] == 'test-layer'
assert 'test-identifier' in components[0]['data']['identifiers']
def test_no_events(dynatrace_check, test_instance):
"""
Testing Dynatrace event check should not produce any events
"""
dynatrace_check._process_topology = mock.MagicMock(return_value=None)
dynatrace_check._current_time_seconds = mock.MagicMock(
return_value=1613485584)
url = test_instance['url']
timestamp = dynatrace_check._generate_bootstrap_timestamp(
test_instance['events_boostrap_days'])
with requests_mock.Mocker() as m:
m.get('{}/api/v1/events?from={}'.format(url, timestamp),
status_code=200,
text=read_file('no_events_response.json'))
dynatrace_check.run()
aggregator.assert_service_check(CHECK_NAME,
count=1,
status=AgentCheck.OK)
assert len(aggregator.events) == 0
def test_events_process_limit(dynatrace_check, test_instance):
"""
Check should respect `events_process_limit` config setting and just produce those number of events
"""
dynatrace_check._process_topology = mock.MagicMock(return_value=None)
dynatrace_check._current_time_seconds = mock.MagicMock(
return_value=1613485584)
url = test_instance['url']
timestamp = dynatrace_check._generate_bootstrap_timestamp(
test_instance['events_boostrap_days'])
with requests_mock.Mocker() as m:
m.get('{}/api/v1/events?from={}'.format(url, timestamp),
status_code=200,
text=read_file('21_events_response.json'))
dynatrace_check.run()
aggregator.assert_service_check(CHECK_NAME,
count=1,
status=AgentCheck.WARNING)
assert len(
aggregator.events) == test_instance.get("events_process_limit")
def test_timeout(dynatrace_check, test_instance):
"""
Gracefully handle requests timeout exception
"""
dynatrace_check._current_time_seconds = mock.MagicMock(
return_value=1613485584)
dynatrace_check._process_topology = mock.MagicMock(return_value=None)
url = test_instance['url']
timestamp = dynatrace_check._generate_bootstrap_timestamp(
test_instance['events_boostrap_days'])
with requests_mock.Mocker() as m:
m.get('{}/api/v1/events?from={}'.format(url, timestamp),
exc=requests.exceptions.ConnectTimeout)
dynatrace_check.run()
aggregator.assert_service_check(
CHECK_NAME,
count=1,
status=AgentCheck.CRITICAL,
message='Timeout exception occurred for endpoint '
'https://instance.live.dynatrace.com/api/v1/events with message: '
'20 seconds timeout')
def test_raise_exception_for_response_code_not_200(dynatrace_check,
test_instance):
"""
Test to raise a check exception when API endpoint when status code is not 200
"""
dynatrace_check._process_topology = mock.MagicMock(return_value=None)
dynatrace_check._current_time_seconds = mock.MagicMock(
return_value=1613485584)
url = test_instance['url']
timestamp = dynatrace_check._generate_bootstrap_timestamp(
test_instance['events_boostrap_days'])
with requests_mock.Mocker() as m:
m.get('{}/api/v1/events?from={}'.format(url, timestamp),
status_code=500,
text='{"error": {"code": 500, "message": "Simulated error!"}}')
dynatrace_check.run()
error_message = 'Got an unexpected error with status code 500 and message: Simulated error!'
aggregator.assert_service_check(CHECK_NAME,
count=1,
status=AgentCheck.CRITICAL,
message=error_message)
assert len(aggregator.events) == 0
def test_check(self):
# TODO this is needed because the topology retains data across tests
topology.reset()
result = self.check.run()
assert result == ''
topo_instances = topology.get_snapshot(self.check.check_id)
self.assertEqual(len(topo_instances['components']), 5)
self.assertEqual(len(topo_instances['relations']), 3)
assert topo_instances == {
'components': [{
'data': {
'cluster':
'stubbed-cluster-name',
'hostname':
'stubbed.hostname',
'identifiers':
['urn:process:/stubbed.hostname:1:1234567890'],
'name':
'StackState Agent:stubbed.hostname',
'tags':
sorted([
'hostname:stubbed.hostname',
'stackstate-agent',
])
},
'id': 'urn:stackstate-agent:/stubbed.hostname',
'type': 'stackstate-agent'
}, {
'data': {
'checks': [{
'is_service_check_health_check': True,
'name': 'Integration Health',
'stream_id': -1
}],
'cluster':
'stubbed-cluster-name',
'service_checks': [{
'conditions': [{
'key': 'host',
'value': 'stubbed.hostname'
}, {
'key': 'tags.integration-type',
'value': 'agent-integration'
}],
'name':
'Service Checks',
'stream_id':
-1
}],
'hostname':
'stubbed.hostname',
'integration':
'agent-integration',
'name':
'stubbed.hostname:agent-integration',
'tags':
sorted([
'hostname:stubbed.hostname',
'integration-type:agent-integration',
])
},
'id':
'urn:agent-integration:/stubbed.hostname:agent-integration',
'type': 'agent-integration'
}, {
'data': {
'checks': [{
'is_service_check_health_check': True,
'name': 'Integration Instance Health',
'stream_id': -1
}],
'cluster':
'stubbed-cluster-name',
'service_checks': [{
'conditions': [{
'key': 'host',
'value': 'stubbed.hostname'
}, {
'key': 'tags.integration-type',
'value': 'agent-integration'
}, {
'key': 'tags.integration-url',
'value': 'sample'
}],
'name':
'Service Checks',
'stream_id':
-1
}],
'hostname':
'stubbed.hostname',
'integration':
'agent-integration',
'name':
'agent-integration:sample',
'tags':
sorted([
'hostname:stubbed.hostname',
'integration-type:agent-integration',
'integration-url:sample'
])
},
'id':
'urn:agent-integration-instance:/stubbed.hostname:agent-integration:sample',
'type': 'agent-integration-instance'
}, {
'data': {
'checks': [{
'critical_value': 90,
'deviating_value': 75,
'is_metric_maximum_average_check': True,
'max_window': 300000,
'name': 'Max CPU Usage (Average)',
'remediation_hint':
'There is too much activity on this host',
'stream_id': -1
}, {
'critical_value': 90,
'deviating_value': 75,
'is_metric_maximum_last_check': True,
'max_window': 300000,
'name': 'Max CPU Usage (Last)',
'remediation_hint':
'There is too much activity on this host',
'stream_id': -1
}, {
'critical_value': 5,
'deviating_value': 10,
'is_metric_minimum_average_check': True,
'max_window': 300000,
'name': 'Min CPU Usage (Average)',
'remediation_hint':
'There is too few activity on this host',
'stream_id': -1
}, {
'critical_value': 5,
'deviating_value': 10,
'is_metric_minimum_last_check': True,
'max_window': 300000,
'name': 'Min CPU Usage (Last)',
'remediation_hint':
'There is too few activity on this host',
'stream_id': -1
}],
'domain':
'Webshop',
'environment':
'Production',
'identifiers': ['another_identifier_for_this_host'],
'labels': ['host:this_host', 'region:eu-west-1'],
'tags':
sorted([
'integration-type:agent-integration',
'integration-url:sample'
]),
'layer':
'Machines',
'metrics': [{
'aggregation':
'MEAN',
'conditions': [{
'key': 'tags.hostname',
'value': 'this-host'
}, {
'key': 'tags.region',
'value': 'eu-west-1'
}],
'metric_field':
'system.cpu.usage',
'name':
'Host CPU Usage',
'priority':
'HIGH',
'stream_id':
-1,
'unit_of_measure':
'Percentage'
}, {
'aggregation':
'MEAN',
'conditions': [{
'key': 'tags.hostname',
'value': 'this-host'
}, {
'key': 'tags.region',
'value': 'eu-west-1'
}],
'metric_field':
'location.availability',
'name':
'Host Availability',
'priority':
'HIGH',
'stream_id':
-2,
'unit_of_measure':
'Percentage'
}],
'name':
'this-host'
},
'id': 'urn:example:/host:this_host',
'type': 'Host'
}, {
'data': {
'checks': [{
'critical_value': 75,
'denominator_stream_id': -1,
'deviating_value': 50,
'is_metric_maximum_ratio_check': True,
'max_window': 300000,
'name': 'OK vs Error Responses (Maximum)',
'numerator_stream_id': -2
}, {
'critical_value': 70,
'deviating_value': 50,
'is_metric_maximum_percentile_check': True,
'max_window': 300000,
'name': 'Error Response 99th Percentile',
'percentile': 99,
'stream_id': -2
}, {
'critical_value': 75,
'denominator_stream_id': -1,
'deviating_value': 50,
'is_metric_failed_ratio_check': True,
'max_window': 300000,
'name': 'OK vs Error Responses (Failed)',
'numerator_stream_id': -2
}, {
'critical_value': 5,
'deviating_value': 10,
'is_metric_minimum_percentile_check': True,
'max_window': 300000,
'name': 'Success Response 99th Percentile',
'percentile': 99,
'stream_id': -1
}],
'domain':
'Webshop',
'environment':
'Production',
'identifiers': ['another_identifier_for_some_application'],
'labels': [
'application:some_application', 'region:eu-west-1',
'hosted_on:this-host'
],
'tags':
sorted([
'integration-type:agent-integration',
'integration-url:sample'
]),
'layer':
'Applications',
'metrics': [{
'aggregation':
'MEAN',
'conditions': [{
'key': 'tags.application',
'value': 'some_application'
}, {
'key': 'tags.region',
'value': 'eu-west-1'
}],
'metric_field':
'2xx.responses',
'name':
'2xx Responses',
'priority':
'HIGH',
'stream_id':
-1,
'unit_of_measure':
'Count'
}, {
'aggregation':
'MEAN',
'conditions': [{
'key': 'tags.application',
'value': 'some_application'
}, {
'key': 'tags.region',
'value': 'eu-west-1'
}],
'metric_field':
'5xx.responses',
'name':
'5xx Responses',
'priority':
'HIGH',
'stream_id':
-2,
'unit_of_measure':
'Count'
}],
'name':
'some-application',
'version':
'0.2.0'
},
'id': 'urn:example:/application:some_application',
'type': 'Application'
}],
'instance_key': {
'type': 'agent',
'url': 'integrations'
},
'relations': [{
'data': {},
'source_id': 'urn:stackstate-agent:/stubbed.hostname',
'target_id':
'urn:agent-integration:/stubbed.hostname:agent-integration',
'type': 'runs'
}, {
'data': {},
'source_id':
'urn:agent-integration:/stubbed.hostname:agent-integration',
'target_id':
'urn:agent-integration-instance:/stubbed.hostname:agent-integration:sample',
'type': 'has'
}, {
'data': {},
'source_id': 'urn:example:/application:some_application',
'target_id': 'urn:example:/host:this_host',
'type': 'IS_HOSTED_ON'
}],
'start_snapshot':
False,
'stop_snapshot':
False
}
aggregator.assert_metric(
'system.cpu.usage',
count=3,
tags=["hostname:this-host", "region:eu-west-1"])
aggregator.assert_metric(
'location.availability',
count=3,
tags=["hostname:this-host", "region:eu-west-1"])
aggregator.assert_metric(
'2xx.responses',
count=4,
tags=["application:some_application", "region:eu-west-1"])
aggregator.assert_metric(
'5xx.responses',
count=4,
tags=["application:some_application", "region:eu-west-1"])
aggregator.assert_event(
'Http request to {} timed out after {} seconds.'.format(
'http://localhost', 5.0),
count=1)
telemetry.assert_topology_event(
{
"timestamp": int(1),
"event_type": "HTTP_TIMEOUT",
"msg_title": "URL timeout",
"msg_text":
"Http request to http://localhost timed out after 5.0 seconds.",
"aggregation_key": "instance-request-http://localhost",
"context": {
"source_identifier":
"source_identifier_value",
"element_identifiers": ["urn:host:/123"],
"source":
"source_value",
"category":
"my_category",
"data": {
"big_black_hole": "here",
"another_thing": 1,
"test": {
"1": "test"
}
},
"source_links": [{
"title": "my_event_external_link",
"url": "http://localhost"
}]
}
},
count=1)
aggregator.assert_service_check('example.can_connect', self.check.OK)
def test_check(self):
# TODO this is needed because the topology retains data across tests
topology.reset()
result = self.check.run()
assert result == ''
topo_instances = topology.get_snapshot(self.check.check_id)
self.assertEqual(len(topo_instances['components']), 6)
self.assertEqual(len(topo_instances['relations']), 3)
assert topo_instances == self._read_data(
'expected_topology_instance.json')
aggregator.assert_metric(
'system.cpu.usage',
count=3,
tags=["hostname:this-host", "region:eu-west-1"])
aggregator.assert_metric(
'location.availability',
count=3,
tags=["hostname:this-host", "region:eu-west-1"])
aggregator.assert_metric(
'2xx.responses',
count=4,
tags=["application:some_application", "region:eu-west-1"])
aggregator.assert_metric(
'5xx.responses',
count=4,
tags=["application:some_application", "region:eu-west-1"])
aggregator.assert_metric('check_runs',
count=1,
tags=["integration:agent_integration_sample"])
aggregator.assert_event(
'Http request to {} timed out after {} seconds.'.format(
'http://localhost', 5.0),
count=1)
telemetry.assert_topology_event(
{
"timestamp": int(1),
"event_type": "HTTP_TIMEOUT",
"source_type_name": "HTTP_TIMEOUT",
"msg_title": "URL timeout",
"msg_text":
"Http request to http://localhost timed out after 5.0 seconds.",
"aggregation_key": "instance-request-http://localhost",
"context": {
"source_identifier":
"source_identifier_value",
"element_identifiers": ["urn:host:/123"],
"source":
"source_value",
"category":
"my_category",
"data": {
"big_black_hole": "here",
"another_thing": 1,
"test": {
"1": "test"
}
},
"source_links": [{
"title": "my_event_external_link",
"url": "http://localhost"
}]
}
},
count=1)
aggregator.assert_service_check('example.can_connect', self.check.OK)
def test_check(self):
result = self.check.run()
assert result == ''
topo_instances = topology.get_snapshot(self.check.check_id)
self.assertEqual(len(topo_instances['components']), 6)
self.assertEqual(len(topo_instances['relations']), 3)
assert topo_instances == load_json_from_file(
'expected_topology_instance.json', 'expected')
aggregator.assert_metric(
'system.cpu.usage',
count=3,
tags=["hostname:this-host", "region:eu-west-1"])
aggregator.assert_metric(
'location.availability',
count=3,
tags=["hostname:this-host", "region:eu-west-1"])
aggregator.assert_metric(
'2xx.responses',
count=4,
tags=["application:some_application", "region:eu-west-1"])
aggregator.assert_metric(
'5xx.responses',
count=4,
tags=["application:some_application", "region:eu-west-1"])
aggregator.assert_metric('check_runs',
count=1,
tags=["integration:agent_integration_sample"])
aggregator.assert_event(
'Http request to {} timed out after {} seconds.'.format(
'http://localhost', 5.0),
count=1)
telemetry.assert_topology_event(
{
"timestamp": int(1),
"event_type": "HTTP_TIMEOUT",
"source_type_name": "HTTP_TIMEOUT",
"msg_title": "URL timeout",
"msg_text":
"Http request to http://localhost timed out after 5.0 seconds.",
"aggregation_key": "instance-request-http://localhost",
"context": {
"source_identifier":
"source_identifier_value",
"element_identifiers": ["urn:host:/123"],
"source":
"source_value",
"category":
"my_category",
"data": {
"big_black_hole": "here",
"another_thing": 1,
"test": {
"1": "test"
}
},
"source_links": [{
"title": "my_event_external_link",
"url": "http://localhost"
}]
}
},
count=1)
aggregator.assert_service_check('example.can_connect', self.check.OK)
health.assert_snapshot(self.check.check_id,
self.check.health.stream,
start_snapshot={
'expiry_interval_s': 0,
'repeat_interval_s': 30
},
stop_snapshot={},
check_states=[{
'checkStateId': 'id',
'health': 'CRITICAL',
'name': 'name',
'topologyElementIdentifier': 'identifier',
'message': 'msg'
}])
telemetry.assert_metric(
"raw.metrics",
count=2,
value=20,
tags=["application:some_application", "region:eu-west-1"],
hostname="hostname")
telemetry.assert_metric("raw.metrics",
count=1,
value=30,
tags=["no:hostname", "region:eu-west-1"],
hostname="")