def test_topology_with_labels_and_instance_tags(self, mock): # TODO this is needed because the topology retains data across tests topology.reset() config = { 'init_config': {}, 'instances': [{ 'type': 'csv', 'components_file': 'component.csv', 'relations_file': 'relation.csv', 'delimiter': ',', 'tags': ['tag1', 'tag2'] }] } self.check.check(config["instances"][0]) instances = topology.get_snapshot(self.check.check_id) self.assertEqual(len(instances['components']), 2) self.assertEqual(len(instances['components'][0]['data']['labels']), 5) self.assertEqual(len(instances['components'][1]['data']['labels']), 4) self.assertIn("csv.component:component.csv", instances['components'][0]['data']['labels']) self.assertIn("csv.relation:relation.csv", instances['components'][0]['data']['labels']) self.assertIn("csv.component:component.csv", instances['components'][1]['data']['labels']) self.assertIn("csv.relation:relation.csv", instances['components'][1]['data']['labels']) self.assertEqual(len(instances['relations']), 1) self.assertNotIn('labels', instances['relations'][0]['data'])
def test_process_redshift(self): self.check.run() topology = [top.get_snapshot(self.check.check_id)] self.assertEqual(len(topology), 1) self.assert_executed_ok() components = topology[0]["components"] relations = topology[0]["relations"] top.assert_component( components, "arn:aws:redshift:eu-west-1:731070500579:cluster:redshift-cluster-1", "aws.redshift.cluster", checks={ "Name": "redshift-cluster-1", "Tags.OrganizationalUnit": "Testing", "URN": [ "arn:aws:redshift:eu-west-1:731070500579:cluster:redshift-cluster-1" ], }, ) top.assert_relation( relations, "arn:aws:redshift:eu-west-1:731070500579:cluster:redshift-cluster-1", "vpc-c6d073bf", "uses-service", ) top.assert_all_checked(components, relations)
def test_zabbix_topology_hosts_no_component(self): """ Test should not return any hosts as host is in maintenance mode """ # TODO this is needed because the topology retains data across tests topology.reset() def _mocked_method_request(url, name, auth=None, params={}, request_id=1): if name == "apiinfo.version": return self._apiinfo_response() elif name == "host.get": return self._zabbix_host_response(maintenance_mode="1") else: self.fail("TEST FAILED on making invalid request") self.check.method_request = _mocked_method_request self.check.login = lambda url, user, password: "******" self.check.retrieve_problems = lambda url, auth: [] self.check.retrieve_events = lambda url, auth, event_ids: [] self.check.check(self.instance) topo_instances = topology.get_snapshot(self.check.check_id) self.assertEqual(len(topo_instances['components']), 0) self.assertEqual(len(topo_instances['relations']), 0)
def test_process_components(self): """ Test _process_components to return topology for components """ self.check._batch_collect_components = mock.MagicMock() self.check._batch_collect_components.return_value = mock_collect_components self.check._batch_collect_components.__name__ = 'mock_batch_collect_components' self.check._batch_collect(self.check._batch_collect_components, instance_info) self.check._process_components(instance_info) topo_instances = topology.get_snapshot(self.check.check_id) self.assertEqual(len(topo_instances['components']), 1) self.assertEqual(len(topo_instances['relations']), 0) self.assertEqual(topo_instances['components'][0]['type'], 'cmdb_ci_computer') self.assertEqual( topo_instances['components'][0]['data']['identifiers'], [ "urn:host:/MacBook Pro 15", "00a96c0d3790200044e0bfc8bcbe5db4", "urn:host:/macbook pro 15", "lupulus" ]) self.assertIn('stackstate-identifier:lupulus', topo_instances['components'][0]['data']['tags']) self.assertIn('stackstate', topo_instances['components'][0]['data']['tags'])
def test_process_s3_filter_all(self): self.check.run() topology = [top.get_snapshot(self.check.check_id)] self.assertEqual(len(topology), 1) self.assert_updated_ok() components = topology[0]["components"] self.assertEqual(len(components), 0)
def test_process_sns_create_topic(self): self.check.run() topology = [top.get_snapshot(self.check.check_id)] self.assertEqual(len(topology), 1) self.assert_updated_ok() self.assertEqual(len(topology[0]["components"]), 1) self.assertEqual("arn:aws:sns:eu-west-1:731070500579:my-topic-1", topology[0]["components"][0]["id"])
def test_process_sqs_purge_queue(self): self.check.run() topology = [top.get_snapshot(self.check.check_id)] self.assertEqual(len(topology), 1) self.assert_executed_ok() self.assertEqual(len(topology[0]["components"]), 0) self.assertEqual(len(self.check.delete_ids), 0)
def test_process_redshift_delete_cluster(self): self.check.run() topology = [top.get_snapshot(self.check.check_id)] self.assertEqual(len(topology), 1) self.assert_executed_ok() self.assertEqual(len(topology[0]["components"]), 0) self.assertIn("arn:aws:redshift:eu-west-1:731070500579:cluster:redshift-cluster-1", self.check.delete_ids)
def test_process_lambda_delete_function(self): self.check.run() topology = [top.get_snapshot(self.check.check_id)] self.assertEqual(len(topology), 1) self.assert_updated_ok() self.assertEqual(len(topology[0]["components"]), 0) self.assertIn("arn:aws:lambda:eu-west-1:731070500579:function:JpkTest", self.check.delete_ids)
def test_process_kinesis_delete_stream(self): self.check.run() topology = [top.get_snapshot(self.check.check_id)] self.assertEqual(len(topology), 1) self.assert_updated_ok() self.assertEqual(len(topology[0]["components"]), 0) self.assertIn("arn:aws:kinesis:eu-west-1:731070500579:stream/TestStream", self.check.delete_ids)
def test_process_kinesis_decrease_retention(self): self.check.run() topology = [top.get_snapshot(self.check.check_id)] self.assertEqual(len(topology), 1) self.assert_updated_ok() self.assertEqual(len(topology[0]["components"]), 1) self.assertEqual("stream_1", topology[0]["components"][0]["data"]["StreamDescriptionSummary"]["StreamName"])
def test_process_kinesis(self): self.check.run() topology = [top.get_snapshot(self.check.check_id)] self.assertEqual(len(topology), 1) self.assert_executed_ok() components = topology[0]["components"] relations = topology[0]["relations"] base_stream_arn = "arn:aws:kinesis:eu-west-1:731070500579:stream/" top.assert_component( components, base_stream_arn + "stream_1", "aws.kinesis.data-stream", checks={ "Name": "stream_1", "StreamDescriptionSummary.StreamARN": "arn:aws:kinesis:eu-west-1:731070500579:stream/stream_1", "Tags.TestKey": "TestValue", }, ) top.assert_component( components, base_stream_arn + "stream_2", "aws.kinesis.data-stream", checks={"Name": "stream_2"} ) top.assert_component( components, base_stream_arn + "stream_3", "aws.kinesis.data-stream", checks={"Name": "stream_3"} ) top.assert_component( components, base_stream_arn + "stream_4", "aws.kinesis.data-stream", checks={"Name": "stream_4"} ) top.assert_all_checked(components, relations)
def test_process_ec2_subnets(self): self.check.run() self.assert_executed_ok() topology = [top.get_snapshot(self.check.check_id)] self.assertEqual(len(topology), 1) self.assert_executed_ok() components = topology[0]["components"] relations = topology[0]["relations"] comp = top.assert_component( components, "subnet-9e4be5f9", "aws.subnet", checks={ "SubnetId": "subnet-9e4be5f9", "Tags.Name": "demo-deployments", "URN": ["arn:aws:ec2:{}:731070500579:subnet/{}".format("eu-west-1", "subnet-9e4be5f9")], "Name": "demo-deployments-eu-west-1a", }, ) self.assert_location_info(comp) comp = top.assert_component( components, "subnet-12345678", "aws.subnet", checks={"SubnetId": "subnet-12345678", "Name": "subnet-12345678-eu-west-1a"}, ) top.assert_relation(relations, "subnet-9e4be5f9", "vpc-6b25d10e", "uses service") top.assert_relation(relations, "subnet-12345678", "vpc-6b25d10e", "uses service") top.assert_all_checked(components, relations)
def test_topology_with_multiple_environments(self, mock): # TODO this is needed because the topology retains data across tests topology.reset() self.check.run() instances = topology.get_snapshot(self.check.check_id) self.assertEqual(len(instances['components']), 2) self.assertEqual(len(instances['components'][0]['data']['labels']), 2) self.assertEqual(len(instances['components'][1]['data']['labels']), 2) self.assertIn("csv.component:component.csv", instances['components'][0]['data']['labels']) self.assertIn("csv.relation:relation.csv", instances['components'][0]['data']['labels']) self.assertEqual( len(instances['components'][0]['data']['environments']), 2) self.assertEqual( len(instances['components'][1]['data']['environments']), 1) self.assertIn("env1", instances['components'][0]['data']['environments']) self.assertIn("env2", instances['components'][0]['data']['environments']) self.assertIn("Production", instances['components'][1]['data']['environments']) self.assertEqual(len(instances['relations']), 1) self.assertNotIn('labels', instances['relations'][0]['data']) AgentIntegrationTestUtil.assert_integration_snapshot( self.check, 'StaticTopology:component.csv')
def test_process_ec2_security_groups_order_has_no_influence_on_hash(self): self.check.run() self.assert_executed_ok() topology = [top.get_snapshot(self.check.check_id)] self.assertEqual(len(topology), 1) self.assert_executed_ok() components = topology[0]["components"] relations = topology[0]["relations"] if sys.version_info.major == 3: version_to_check = self.first_security_group_version_py3 else: version_to_check = self.first_security_group_version_py2 top.assert_component( components, self.first_sg_group_id, "aws.ec2.security-group", checks={ "URN": [ "arn:aws:ec2:{}:731070500579:security-group/{}".format( "eu-west-1", self.first_sg_group_id) ], "Version": version_to_check, "Name": "network-ALBSecurityGroupPublic-1DNVWX102724V", }, ) top.assert_all_checked(components, relations, unchecked_components=48, unchecked_relations=49)
def test_collect_empty_topology(self): """ Testing AWS Topology check should not produce any topology (apis_to_run set to empty array) """ instance = InstanceInfo({ "role_arn": "arn:aws:iam::123456789012:role/RoleName", "regions": ["eu-west-1"], "apis_to_run": [] }) self.check.check(instance) test_topology = topology.get_snapshot(self.check.check_id) self.assertEqual(test_topology["instance_key"], { "type": "aws-v2", "url": "123456789012" }) self.assertEqual(test_topology["components"], []) self.assertEqual(test_topology["relations"], []) service_checks = aggregator.service_checks( self.check.SERVICE_CHECK_CONNECT_NAME) self.assertGreater(len(service_checks), 0) self.assertEqual(service_checks[0].status, AgentCheck.OK) service_checks = aggregator.service_checks( self.check.SERVICE_CHECK_EXECUTE_NAME) self.assertGreater(len(service_checks), 0) self.assertEqual(service_checks[0].status, AgentCheck.OK)
def test_process_ec2_detach_volume(self): self.check.run() topology = [top.get_snapshot(self.check.check_id)] self.assertEqual(len(topology), 1) self.assert_updated_ok() components = topology[0]["components"] self.assertEqual(len(topology[0]["components"]), 1) top.assert_component( components, "i-1234567890123456", "aws.ec2.instance", checks={ "InstanceId": "i-1234567890123456", "BlockDeviceMappings": [{ "DeviceName": "/dev/sda1", "Ebs": { "Status": "attached", "DeleteOnTermination": True, "VolumeId": "vol-0b83c17b2c8bd35b2", "AttachTime": "2018-04-05T09:34:07+00:00", }, }], }, )
def test_full_topology(self, m): """ Test e2e to collect full topology for all component types from Dynatrace :return: """ self._set_http_responses(m, hosts=_read_test_file("host_response.json"), apps=_read_test_file("application_response.json"), svcs=_read_test_file("service_response.json"), procs=_read_test_file("process_response.json"), proc_groups=_read_test_file("process-group_response.json") ) self.check.url = self.instance.get('url') self.check.run() topo_instances = topology.get_snapshot(self.check.check_id) expected_topology = _read_data("smartscape_full_response_topology.json") # sort the keys of components and relations, so we match it in actual components, relations = sort_topology_data(topo_instances) expected_components, expected_relations = sort_topology_data(expected_topology) self.assertEqual(len(components), len(expected_components)) for component in components: self.assertIn(component, expected_components) # Not comparing the numbers because we have duplicate relations created but # duplicates will be filtered out by the agent externalId assigning behavior for relation in relations: self.assertIn(relation, expected_relations)
def test_process_sns_delete_topic(self): self.check.run() topology = [top.get_snapshot(self.check.check_id)] self.assertEqual(len(topology), 1) self.assert_updated_ok() self.assertEqual(len(topology[0]["components"]), 0) self.assertIn("arn:aws:sns:eu-west-1::my-topic-1", self.check.delete_ids)
def test_unicode_in_response_text(dynatrace_check, test_instance): dynatrace_check._current_time_seconds = mock.MagicMock( return_value=1613485584) url = test_instance['url'] timestamp = dynatrace_check._generate_bootstrap_timestamp( test_instance['events_boostrap_days']) with requests_mock.Mocker() as m: m.get("{}/api/v1/entity/infrastructure/hosts".format(url), status_code=200, text=read_file('host_response.json')) m.get("{}/api/v1/entity/applications".format(url), status_code=200, text='[]') m.get("{}/api/v1/entity/services".format(url), status_code=200, text='[]') m.get("{}/api/v1/entity/infrastructure/processes".format(url), status_code=200, text='[]') m.get("{}/api/v1/entity/infrastructure/process-groups".format(url), status_code=200, text='[]') m.get('{}/api/v1/events?from={}'.format(url, timestamp), status_code=200, text=read_file('unicode_topology_event_response.json')) dynatrace_check.run() aggregator.assert_service_check(CHECK_NAME, count=1, status=AgentCheck.OK) unicode_data = topology.get_snapshot('').get( 'components')[0]['data']['osVersion'] assert unicode_data == 'Windows Server 2016 Datacenter 1607, ver. 10.0.14393 with unicode char: ' assert telemetry._topology_events[0][ 'msg_text'] == 'PROCESS_RESTART on aws-cni™'
def test_process_sqs(self): self.check.run() topology = [top.get_snapshot(self.check.check_id)] self.assertEqual(len(topology), 1) self.assert_executed_ok() components = topology[0]["components"] relations = topology[0]["relations"] component = top.assert_component( components, "arn:aws:sqs:eu-west-1:731070500579:STS_stackpack_test", "aws.sqs", checks={ "Tags.a": "b", "Name": "STS_stackpack_test", "URN": [ "https://sqs.eu-west-1.amazonaws.com/731070500579/STS_stackpack_test" ], "CW.Dimensions": [{ "Key": "QueueName", "Value": "STS_stackpack_test" }], }, ) self.assert_location_info(component) top.assert_all_checked(components, relations)
def test_collect_topology_component(self): """ Test the component collection from the topology for VirtualMachine """ # TODO this is needed because the topology retains data across tests topology.reset() self.check._is_excluded = MagicMock(return_value=True) instance = {'name': 'vsphere_mock', 'host': 'test-esxi'} topo_items = { 'datastores': [], 'clustercomputeresource': [], 'computeresource': [], 'hosts': [], 'datacenters': [], 'vms': [{ 'hostname': 'Ubuntu', 'topo_tags': { 'topo_type': 'vsphere-VirtualMachine', 'name': 'Ubuntu', 'datastore': '54183927-04f91918-a72a-6805ca147c55' }, 'mor_type': 'vm' }] } self.check.get_topologyitems_sync = MagicMock(return_value=topo_items) self.check.collect_topology(instance) snapshot = topology.get_snapshot(self.check.check_id) # Check if the returned topology contains 1 component self.assertEqual(len(snapshot['components']), 1) self.assertEqual( snapshot['components'][0]['id'], 'urn:vsphere:/test-esxi/vsphere-VirtualMachine/Ubuntu')
def test_process_component_relations_with_sys_filter_change(self): """ Test _process_components to return whole topology when query changed in between """ sys_class_filter = self.instance.get('include_resource_types') instance_info.sys_class_filter = sys_class_filter query_filter = self.check._get_sys_class_relation_filter_query( sys_class_filter) expected_query = 'parent.sys_class_nameINcmdb_ci_netgear,cmdb_ci_cluster,cmdb_ci_app_server' \ '^child.sys_class_nameINcmdb_ci_netgear,cmdb_ci_cluster,cmdb_ci_app_server' # asserting the actual query self.assertEqual(expected_query, query_filter) self.check._get_sys_class_relation_filter_query = mock.MagicMock() # changing the query in between and returning with incorrect query self.check._get_sys_class_relation_filter_query.return_value = "parent.sys_class_nameN" \ "cmdb_ci_netgear%5Echild.sys_class_nameIN" \ "cmdb_ci_netgear" self.check._get_json = mock.MagicMock() self.check._get_json.return_value = mock_relation_components self.check._process_relations(instance_info) topo_instances = topology.get_snapshot(self.check.check_id) self.assertEqual(len(topo_instances['components']), 0) self.assertEqual(len(topo_instances['relations']), 1) self.assertEqual(topo_instances['relations'][0]['type'], 'Cools')
def test_process_relations_without_sys_filter_change(self): """ Test _process_components to return whole topology when query changed in between """ sys_class_filter = self.instance.get('include_resource_types') instance_info.sys_class_filter = sys_class_filter query_filter = self.check._get_sys_class_relation_filter_query( sys_class_filter) expected_query = 'parent.sys_class_nameINcmdb_ci_netgear,cmdb_ci_cluster,cmdb_ci_app_server' \ '^child.sys_class_nameINcmdb_ci_netgear,cmdb_ci_cluster,cmdb_ci_app_server' # asserting the actual query self.assertEqual(expected_query, query_filter) self.check._get_json = mock.MagicMock() self.check._get_json.return_value = mock_relation_with_filter self.check._process_relation_types = mock.MagicMock() self.check._process_relation_types.return_value = { '1a9cb166f1571100a92eb60da2bce5c5': 'Cools' } self.check._process_relations(instance_info) topo_instances = topology.get_snapshot(self.check.check_id) self.assertEqual(len(topo_instances['components']), 0) # Since the filter gets specific relation only so returned one relation for filtered resource types self.assertEqual(len(topo_instances['relations']), 1) self.assertEqual(topo_instances['relations'][0]['type'], 'Cools')
def test_process_s3_delete_bucket(self): self.check.run() topology = [top.get_snapshot(self.check.check_id)] self.assertEqual(len(topology), 1) self.assert_updated_ok() self.assertEqual(len(topology[0]["components"]), 0) self.assertIn("arn:aws:s3:::binx.io", self.check.delete_ids)
def test_process_components_encoding_errors(self): """ This would provoke following error with py27: "UnicodeEncodeError: 'ascii' codec can't encode character u'\xeb' in position 4: ordinal not in range(128)" in the function stackstate_checks.base.Identifiers.create_host_identifier """ collect_components_with_fqdn_umlaut = { 'result': [{ 'sys_class_name': 'cmdb_ci_computer', 'sys_id': '00a96c0d3790200044e0bfc8bcbe5db4', 'sys_created_on': '2012-02-18 08:14:21', 'name': 'Some computer', 'fqdn': u'abcdë.com' }] } self.check._batch_collect_components = mock.MagicMock() self.check._batch_collect_components.return_value = collect_components_with_fqdn_umlaut self.check._batch_collect_components.__name__ = 'mock_batch_collect_components' self.check._batch_collect(self.check._batch_collect_components, instance_info) self.check._process_components(instance_info) topo_instances = topology.get_snapshot(self.check.check_id) self.assertEqual([ 'urn:host:/abcdë.com', 'urn:host:/Some computer', '00a96c0d3790200044e0bfc8bcbe5db4', 'urn:host:/some computer' ], topo_instances['components'][0]['data']['identifiers'])
def test_process_cloudformation_stack_relations(self): self.check.run() topology = [top.get_snapshot(self.check.check_id)] self.assertEqual(len(topology), 1) self.assert_executed_ok() components = topology[0]["components"] relations = topology[0]["relations"] stack1 = ( "arn:aws:cloudformation:eu-west-1:731070500579:stack/stackstate-topo-publisher/" + "71ea3f80-9919-11e9-a261-0a99a68566c4" ) stack2 = ( "arn:aws:cloudformation:eu-west-1:731070500579:stack/stackstate-topo-cwevents/" + "077bd960-9919-11e9-adb7-02135cc8443e" ) top.assert_component( components, stack1, "aws.cloudformation.stack", checks={"LastUpdatedTime": "2019-06-27T20:23:43.548Z", "StackName": "stackstate-topo-publisher"}, ) top.assert_component( components, stack2, "aws.cloudformation.stack", checks={"LastUpdatedTime": "2019-06-27T20:20:45.336Z", "StackName": "stackstate-topo-cwevents"}, ) top.assert_relation(relations, stack2, stack1, "has-resource") top.assert_all_checked(components, relations)
def test_full_topology(self, m): """ Test e2e to collect full topology for all component types from Dynatrace :return: """ self._set_http_responses( m, hosts=read_file("host_response.json"), apps=read_file("application_response.json"), svcs=read_file("service_response.json"), procs=read_file("process_response.json"), proc_groups=read_file("process-group_response.json")) self.check.url = self.instance.get('url') self.check.run() expected_topology = load_json_from_file( "expected_smartscape_full_topology.json") actual_topology = topology.get_snapshot(self.check.check_id) components, relations = sort_topology_data(actual_topology) expected_components, expected_relations = sort_topology_data( expected_topology) self.assertEqual(len(components), len(expected_components)) for component in components: self.assertIn(component, expected_components) self.assertEqual(len(relations), len(expected_relations)) for relation in relations: self.assertIn(relation, expected_relations)
def test_zabbix_topology_non_default_environment(self): # TODO this is needed because the topology retains data across tests topology.reset() def _mocked_method_request(url, name, auth=None, params={}, request_id=1): if name == "apiinfo.version": return self._apiinfo_response() elif name == "host.get": return self._zabbix_host_response() else: self.fail("TEST FAILED on making invalid request") self.instance['stackstate_environment'] = 'MyTestEnvironment' self.check.method_request = _mocked_method_request self.check.login = lambda url, user, password: "******" self.check.retrieve_problems = lambda url, auth: [] self.check.retrieve_events = lambda url, auth, event_ids: [] self.check.check(self.instance) topo_instances = topology.get_snapshot(self.check.check_id) self.assertEqual(len(topo_instances['components']), 1) self.assertEqual(len(topo_instances['relations']), 0) component = topo_instances['components'][0] self.assertEqual(component['data']['environment'], 'MyTestEnvironment') labels = component['data']['labels'] for label in ['zabbix', 'host group:Zabbix servers']: if label not in labels: self.fail("Component does not have label '%s'." % label)
def test_check_run_no_sap_instances(aggregator, instance): # TODO this is needed because the topology retains data across tests topology.reset() host_control_url = "http://localhost:1128/SAPHostControl" with requests_mock.mock() as m: m.get(host_control_url + "/?wsdl", text=_read_test_file("wsdl/SAPHostAgent.wsdl")) m.post(host_control_url + ".cgi", text=_read_test_file("samples/GetCIMObject-NoResult.xml")) sap_check = SapCheck(CHECK_NAME, {}, instances=[instance]) sap_check.run() topology.get_snapshot(sap_check.check_id) topology.assert_snapshot( check_id=sap_check.check_id, start_snapshot=True, stop_snapshot=True, instance_key=TopologyInstance("sap", "LAB-SAP-001"), components=[{ "id": "urn:host:/LAB-SAP-001", "type": "sap-host", "data": { "host": "LAB-SAP-001", 'tags': ['integration-type:sap', 'integration-url:LAB-SAP-001'], "domain": "sap", "environment": "sap-prod" } }], relations=[], ) AgentIntegrationTestUtil.assert_integration_snapshot( sap_check, 'sap:LAB-SAP-001') aggregator.assert_event( msg_text="Host control 'LAB-SAP-001' status update.", tags=["status:sap-host-control-success", "host:LAB-SAP-001"]) aggregator.all_metrics_asserted() aggregator.assert_service_check(name=SapCheck.SERVICE_CHECK_NAME, status=AgentCheck.OK, message="OK", tags=[])