Exemple #1
0
def test_no_tenant(aggregator):
    api = ApiMock()
    check = CiscoACICheck(conftest.CHECK_NAME, {}, {})
    api._refresh_sessions = False
    check._api_cache[hash_mutable(hash_mutable({}))] = api
    tenant = Tenant(check, api, {}, None)
    tenant.collect()

    assert len(aggregator._metrics.items()) == 0
Exemple #2
0
    def get_config(self, instance):
        instance_id = hash_mutable(instance)
        config = self.config.get(instance_id)
        if config is None:
            config = {}

            try:
                api_url = instance['api_url']
                api_version = api_url[-1]
                if api_version not in self.api_versions:
                    self.log.warning(
                        'Unknown Vault API version `%s`, using version `%s`',
                        api_version, self.DEFAULT_API_VERSION)
                    api_url = api_url[:-1] + self.DEFAULT_API_VERSION
                    api_version = self.DEFAULT_API_VERSION

                config['api_url'] = api_url
                config['api'] = self.api_versions[api_version]['functions']
            except KeyError:
                self.log.error(
                    'Vault configuration setting `api_url` is required')
                return

            config['tags'] = instance.get('tags', [])

            # Keep track of the previous cluster leader to detect changes.
            config['leader'] = None
            config['detect_leader'] = is_affirmative(
                instance.get('detect_leader'))

            self.config[instance_id] = config

        return config
Exemple #3
0
    def __init__(self, name, init_config, agentConfig, instances=None):
        AgentCheck.__init__(self, name, init_config, agentConfig, instances)
        self._countersettypes = {}
        self._counters = {}
        self._metrics = {}
        self._tags = {}

        try:
            for instance in instances:
                key = hash_mutable(instance)
                counterset = instance.get('countersetname')

                cfg_tags = instance.get('tags')
                if cfg_tags is not None:
                    tags = cfg_tags.join(",")
                    self._tags[key] = list(tags) if tags else []

                metrics = instance.get('metrics')
                # list of the metrics.  Each entry is itself an entry,
                # which is the pdh name, datadog metric name, type, and the
                # pdh counter object
                self._metrics[key] = []
                for inst_name, dd_name, mtype in metrics:
                    m = getattr(self, mtype.lower())
                    obj = WinPDHCounter(counterset, inst_name, self.log)
                    if not obj:
                        continue
                    entry = [inst_name, dd_name, m, obj]
                    self.log.debug("entry: %s" % str(entry))
                    self._metrics[key].append(entry)

        except Exception as e:
            self.log.debug("Exception in PDH init: %s", str(e))
            raise
Exemple #4
0
def test_network_latency_checks(aggregator):
    consul_check = ConsulCheck(common.CHECK_NAME, {}, [{}])
    my_mocks = consul_mocks._get_consul_mocks()
    consul_mocks.mock_check(consul_check, my_mocks)

    # We start out as the leader, and stay that way
    instance_hash = hash_mutable(consul_mocks.MOCK_CONFIG_NETWORK_LATENCY_CHECKS)
    consul_check._instance_states[instance_hash].last_known_leader = consul_mocks.mock_get_cluster_leader_A(None)

    consul_check.check(consul_mocks.MOCK_CONFIG_NETWORK_LATENCY_CHECKS)

    latency = []
    for m_name, metrics in aggregator._metrics.items():
        if m_name.startswith('consul.net.'):
            latency.extend(metrics)
    latency.sort()
    # Make sure we have the expected number of metrics
    assert 19 == len(latency)

    # Only 3 dc-latency metrics since we only do source = self
    dc = [m for m in latency if '.dc.latency.' in m[0]]
    assert 3 == len(dc)
    assert 1.6746410750238774 == dc[0][2]

    # 16 latency metrics, 2 nodes * 8 metrics each
    node = [m for m in latency if '.node.latency.' in m[0]]
    assert 16 == len(node)
    assert 0.26577747932995816 == node[0][2]
    def check(self, instance):
        """
        Fetch WMI metrics.
        """

        # Connection information
        host = instance.get('host', "localhost")
        namespace = instance.get('namespace', "root\\cimv2")
        provider = instance.get('provider')
        username = instance.get('username', "")
        password = instance.get('password', "")

        # WMI instance
        wmi_class = instance.get('class')
        metrics = instance.get('metrics')
        filters = instance.get('filters')
        tag_by = instance.get('tag_by', "")
        tag_queries = instance.get('tag_queries', [])

        constant_tags = instance.get('constant_tags')
        custom_tags = instance.get('tags', [])
        if constant_tags is None:
            constant_tags = list(custom_tags)
        else:
            constant_tags.extend(custom_tags)
            self.log.warning("`constant_tags` is being deprecated, please use `tags`")

        # Create or retrieve an existing WMISampler
        instance_hash = hash_mutable(instance)
        instance_key = self._get_instance_key(host, namespace, wmi_class, instance_hash)

        metric_name_and_type_by_property, properties = self._get_wmi_properties(instance_key, metrics, tag_queries)

        wmi_sampler = self._get_running_wmi_sampler(
            instance_key,
            wmi_class,
            properties,
            tag_by=tag_by,
            filters=filters,
            host=host,
            namespace=namespace,
            provider=provider,
            username=username,
            password=password,
        )

        # Sample, extract & submit metrics
        try:
            wmi_sampler.sample()
            metrics = self._extract_metrics(wmi_sampler, tag_by, tag_queries, constant_tags)
        except TimeoutException:
            self.log.warning(
                "WMI query timed out. class=%s - properties=%s - filters=%s - tag_queries=%s",
                wmi_class,
                properties,
                filters,
                tag_queries,
            )
        else:
            self._submit_metrics(metrics, metric_name_and_type_by_property)
Exemple #6
0
def test_cisco(aggregator, session_mock):
    cisco_aci_check = CiscoACICheck(CHECK_NAME, {}, {})
    api = Api(ACI_URLS, USERNAME, PASSWORD, log=cisco_aci_check.log)
    api.sessions = [session_mock]
    api._refresh_sessions = False
    cisco_aci_check._api_cache[hash_mutable(CONFIG)] = api

    cisco_aci_check.check(CONFIG)
Exemple #7
0
def test_cisco(aggregator, session_mock):
    cisco_aci_check = CiscoACICheck(conftest.CHECK_NAME, {}, {})
    api = Api(conftest.ACI_URLS, conftest.USERNAME,
              password=conftest.PASSWORD, log=cisco_aci_check.log, sessions=[session_mock])
    api._refresh_sessions = False
    cisco_aci_check._api_cache[hash_mutable(conftest.CONFIG)] = api

    cisco_aci_check.check(conftest.CONFIG)
Exemple #8
0
    def check(self, instance):
        # Duplicate refresh_counters from PDHBaseCheck
        instance_hash = hash_mutable(instance)
        refresh_counters = is_affirmative(
            instance.get('refresh_counters', True))

        if refresh_counters:
            for counter, values in list(iteritems(self._missing_counters)):
                self._make_counters(instance_hash, ([counter], values))

        sites = instance.get('sites', []) or []
        expected_sites = set(sites)
        # _Total should always be in the list of expected sites; we always
        # report _Total
        expected_sites.add(TOTAL_SITE)

        self.log.debug("Expected sites is %s", expected_sites)
        for inst_name, dd_name, metric_func, counter in self._metrics[
                instance_hash]:
            try:
                site_values = counter.get_all_values()
            except Exception as e:
                self.log.error("Failed to get_all_values %s %s: %s", inst_name,
                               dd_name, e)
                continue
            try:
                for site_name, value in iteritems(site_values):
                    is_single_instance = counter.is_single_instance()
                    if not is_single_instance and sites and site_name != TOTAL_SITE and site_name not in sites:
                        continue

                    tags = self._get_site_tags(instance_hash, instance,
                                               site_name, is_single_instance)
                    try:
                        metric_func(dd_name, value, tags)
                    except Exception as e:
                        self.log.error("Error in metric_func: %s %s %s",
                                       dd_name, value, e)

                    if dd_name == "iis.uptime":
                        self._report_uptime(value, tags)
                        if site_name in expected_sites:
                            self.log.debug("Removing %r from expected sites",
                                           site_name)
                            expected_sites.remove(site_name)
                        else:
                            self.log.warning("Site %r not in expected_sites",
                                             site_name)

            except Exception as e:
                # don't give up on all of the metrics because one failed
                self.log.error("IIS Failed to get metric data for %s %s: %s",
                               inst_name, dd_name, e)

        self._report_unavailable_sites(expected_sites, instance_hash, instance)
Exemple #9
0
def test_cisco(aggregator):
    cisco_aci_check = CiscoACICheck(common.CHECK_NAME, {}, {})
    api = Api(common.ACI_URLS,
              cisco_aci_check.http,
              common.USERNAME,
              password=common.PASSWORD,
              log=cisco_aci_check.log)
    api.wrapper_factory = common.FakeSessionWrapper
    cisco_aci_check._api_cache[hash_mutable(common.CONFIG)] = api

    cisco_aci_check.check(common.CONFIG)
    def check(self, instance):
        instance_key = hash_mutable(instance)
        if instance_key in self.instance_cache:
            config = self.instance_cache[instance_key]["config"]
            profiler = self.instance_cache[instance_key]["profiler"]
        else:
            config = FilebeatCheckInstanceConfig(instance)
            profiler = FilebeatCheckHttpProfiler(config)
            self.instance_cache[instance_key] = {"config": config, "profiler": profiler}

        self._process_registry(config)
        self._gather_http_profiler_metrics(config, profiler)
Exemple #11
0
    def check(self, instance):
        key = hash_mutable(instance)
        for _, dd_name, metric_func, counter in self._metrics[key]:
            vals = counter.get_all_values()
            for key, val in iteritems(vals):
                tags = []
                if key in self._tags:
                    tags = self._tags[key]

                if not counter.is_single_instance():
                    tag = "instance:%s" % key
                    tags.append(tag)
                metric_func(dd_name, val, tags)
Exemple #12
0
    def check(self, instance):
        instance_key = hash_mutable(instance)
        if instance_key in self.instance_cache:
            config = self.instance_cache['config']
            profiler = self.instance_cache['profiler']
        else:
            self.instance_cache[
                'config'] = config = FilebeatCheckInstanceConfig(instance)
            self.instance_cache[
                'profiler'] = profiler = FilebeatCheckHttpProfiler(config)

        self._process_registry(config)
        self._gather_http_profiler_metrics(config, profiler)
Exemple #13
0
    def check(self, instance):
        # Connect to the WMI provider
        host = instance.get('host', "localhost")
        user = instance.get('username', "")
        password = instance.get('password', "")
        services = instance.get('services', [])
        custom_tags = instance.get('tags', [])

        instance_hash = hash_mutable(instance)
        instance_key = self._get_instance_key(host, self.NAMESPACE, self.CLASS,
                                              instance_hash)
        tags = [] if (host == "localhost"
                      or host == ".") else [u'host:{}'.format(host)]
        tags.extend(custom_tags)

        if len(services) == 0:
            raise Exception('No services defined in windows_service.yaml')

        properties = ["Name", "State"]
        if "ALL" in services:
            self.log.debug("tracking all services")
            filters = None
        else:
            filters = map(
                lambda x:
                {"Name": tuple(('LIKE', x))
                 if '%' in x else tuple(('=', x))}, services)

        wmi_sampler = self._get_wmi_sampler(instance_key,
                                            self.CLASS,
                                            properties,
                                            filters=filters,
                                            host=host,
                                            namespace=self.NAMESPACE,
                                            username=user,
                                            password=password)

        try:
            # Sample, extract & submit metrics
            wmi_sampler.sample()
        except TimeoutException:
            self.log.warning(
                u"[WinService] WMI query timed out."
                u" class={wmi_class} - properties={wmi_properties} -"
                u" filters={filters} - tags={tags}".format(
                    wmi_class=self.CLASS,
                    wmi_properties=properties,
                    filters=filters,
                    tags=tags))
        else:
            self._process_services(wmi_sampler, services, tags)
Exemple #14
0
def test_self_leader_event(aggregator):
    consul_check = ConsulCheck(common.CHECK_NAME, {},
                               [consul_mocks.MOCK_CONFIG_SELF_LEADER_CHECK])
    my_mocks = consul_mocks._get_consul_mocks()

    instance_hash = hash_mutable(consul_mocks.MOCK_CONFIG_SELF_LEADER_CHECK)
    consul_check._instance_states[
        instance_hash].last_known_leader = 'My Old Leader'

    our_url = consul_mocks.mock_get_cluster_leader_A(None)
    other_url = consul_mocks.mock_get_cluster_leader_B(None)

    # We become the leader
    my_mocks['_get_cluster_leader'] = consul_mocks.mock_get_cluster_leader_A
    consul_mocks.mock_check(consul_check, my_mocks)
    consul_check.check(consul_mocks.MOCK_CONFIG_SELF_LEADER_CHECK)
    assert len(aggregator.events) == 1
    assert our_url == consul_check._instance_states[
        instance_hash].last_known_leader
    event = aggregator.events[0]
    assert event['event_type'] == 'consul.new_leader'
    assert 'prev_consul_leader:My Old Leader' in event['tags']
    assert 'curr_consul_leader:{}'.format(our_url) in event['tags']

    # We are already the leader, no new events
    aggregator.reset()
    consul_check.check(consul_mocks.MOCK_CONFIG_SELF_LEADER_CHECK)
    assert len(aggregator.events) == 0

    # We lose the leader, no new events
    my_mocks['_get_cluster_leader'] = consul_mocks.mock_get_cluster_leader_B
    consul_mocks.mock_check(consul_check, my_mocks)
    aggregator.reset()
    consul_check.check(consul_mocks.MOCK_CONFIG_SELF_LEADER_CHECK)
    assert len(aggregator.events) == 0
    assert other_url == consul_check._instance_states[
        instance_hash].last_known_leader

    # We regain the leadership
    my_mocks['_get_cluster_leader'] = consul_mocks.mock_get_cluster_leader_A
    consul_mocks.mock_check(consul_check, my_mocks)
    aggregator.reset()
    consul_check.check(consul_mocks.MOCK_CONFIG_SELF_LEADER_CHECK)
    assert len(aggregator.events) == 1
    assert our_url == consul_check._instance_states[
        instance_hash].last_known_leader
    event = aggregator.events[0]
    assert event['event_type'] == 'consul.new_leader'
    assert 'prev_consul_leader:{}'.format(other_url) in event['tags']
    assert 'curr_consul_leader:{}'.format(our_url) in event['tags']
Exemple #15
0
    def get_config(self, instance):
        instance_id = hash_mutable(instance)
        config = self.config.get(instance_id)
        if config is None:
            config = {}

            try:
                api_url = instance['api_url']
                api_version = api_url[-1]
                if api_version not in self.api_versions:
                    self.log.warning(
                        'Unknown Vault API version `{}`, using version '
                        '`{}`'.format(api_version, self.DEFAULT_API_VERSION))

                config['api_url'] = api_url
                config['api'] = self.api_versions.get(
                    api_version, self.DEFAULT_API_VERSION)['functions']
            except KeyError:
                self.log.error(
                    'Vault configuration setting `api_url` is required')
                return

            client_token = instance.get('client_token')
            config['headers'] = {
                'X-Vault-Token': client_token
            } if client_token else None

            username = instance.get('username')
            password = instance.get('password')
            config['auth'] = (username,
                              password) if username and password else None

            config['ssl_verify'] = is_affirmative(
                instance.get('ssl_verify', True))
            config['ssl_ignore_warning'] = is_affirmative(
                instance.get('ssl_ignore_warning', False))
            config['proxies'] = self.get_instance_proxy(
                instance, config['api_url'])
            config['timeout'] = int(instance.get('timeout', 20))
            config['tags'] = instance.get('tags', [])

            # Keep track of the previous cluster leader to detect changes.
            config['leader'] = None
            config['detect_leader'] = is_affirmative(
                instance.get('detect_leader'))

            self.config[instance_id] = config

        return config
Exemple #16
0
    def check(self, instance):
        instance_state = self._instance_states[hash_mutable(instance)]

        server = instance.get('server', None)
        if server is None:
            raise Exception("The server must be specified")
        tags = instance.get('tags', [])
        # Clean up tags in case there was a None entry in the instance
        # e.g. if the yaml contains tags: but no actual tags
        if tags is None:
            tags = []
        else:
            tags = list(set(tags))
        tags.append('instance:{}'.format(server))
        data = self.get_data(server, instance)
        self._create_metrics(data, instance_state, server, tags=list(set(tags)))
Exemple #17
0
def test_new_leader_event(aggregator):
    consul_check = ConsulCheck(common.CHECK_NAME, {}, [{}])
    my_mocks = consul_mocks._get_consul_mocks()
    my_mocks['_get_cluster_leader'] = consul_mocks.mock_get_cluster_leader_B
    consul_mocks.mock_check(consul_check, my_mocks)

    instance_hash = hash_mutable(consul_mocks.MOCK_CONFIG_LEADER_CHECK)
    consul_check._instance_states[instance_hash].last_known_leader = 'My Old Leader'

    consul_check.check(consul_mocks.MOCK_CONFIG_LEADER_CHECK)
    assert len(aggregator.events) == 1

    event = aggregator.events[0]
    assert event['event_type'] == 'consul.new_leader'
    assert 'prev_consul_leader:My Old Leader' in event['tags']
    assert 'curr_consul_leader:My New Leader' in event['tags']
Exemple #18
0
    def check(self, instance):
        self.log.debug("PDHBaseCheck: check()")
        key = hash_mutable(instance)
        for inst_name, dd_name, metric_func, counter in self._metrics[key]:
            try:
                vals = counter.get_all_values()
                for instance_name, val in vals.iteritems():
                    tags = []
                    if key in self._tags:
                        tags = list(self._tags[key])

                    if not counter.is_single_instance():
                        tag = "instance:%s" % instance_name
                        tags.append(tag)
                    metric_func(dd_name, val, tags)
            except Exception as e:
                # don't give up on all of the metrics because one failed
                self.log.error("Failed to get data for %s %s: %s" % (inst_name, dd_name, str(e)))
                pass
class CiscoTags:
    def __init__(self, check):
        self.check = check
        self.tenant_farbic_mapper = {}
        self.tenant_tags = {}
        self._api = None

    def app_tags(self, app):
        tags = []
        attrs = app.get('attributes', {})
        app_name = attrs.get('name')
        dn = attrs.get('dn')
        if app_name:
            tags.append("application:" + app_name)
        if dn:
            tenant = re.search('/tn-([a-zA-Z-_0-9]+)/', dn)
            if tenant:
                tags.append("tenant:" + tenant.group(1))
        return tags

    def tenant_mapper(self, edpt):
        tags = []
        attrs = edpt.get('attributes', {})
        epg_name = attrs.get('name')
        dn = attrs.get('dn')
        application_meta = ["endpoint_group:" + epg_name]
        if dn:
            tenant = re.search('/tn-([a-zA-Z-_0-9]+)/', dn)
            if tenant:
                tenant_name = tenant.group(1)
                application_meta.append("tenant:" + tenant_name)
        if dn:
            app = re.search('/ap-([a-zA-Z-_0-9]+)/', dn)
            if app:
                app_name = app.group(1)
                application_meta.append("application:" + app_name)
        endpoint_meta = []
        # adding meta tags
        try:
            meta = self.api.get_epg_meta(tenant_name, app_name, epg_name)
            if len(meta) > 0:
                meta = meta[0]
                meta_attrs = meta.get('fvCEp', {}).get('attributes')
                if meta_attrs:
                    ip = meta_attrs.get('ip')
                    if ip:
                        endpoint_meta.append("ip:" + ip)
                    mac = meta_attrs.get('mac')
                    if mac:
                        endpoint_meta.append("mac:" + mac)
                    encap = meta_attrs.get('encap')
                    if encap:
                        endpoint_meta.append("encap:" + encap)
                    # adding application tags
        except exceptions.APIConnectionException, exceptions.APIParsingException:
            # the exception will already be logged, just pass it over here
            pass
        endpoint_meta += application_meta

        context_hash = hash_mutable(endpoint_meta)
        eth_meta = []
        if self.tenant_tags.get(context_hash):
            eth_meta = self.tenant_tags.get(context_hash)
        else:
            try:
                # adding eth and node tags
                eth_list = self.api.get_eth_list_for_epg(
                    tenant_name, app_name, epg_name)
                for eth in eth_list:
                    eth_attrs = eth.get('fvRsCEpToPathEp',
                                        {}).get('attributes', {})
                    port = re.search('/pathep-\[(.+?)\]',
                                     eth_attrs.get('tDn', ''))
                    if not port:
                        continue
                    eth_tag = 'port:' + port.group(1)
                    if eth_tag not in eth_meta:
                        eth_meta.append(eth_tag)
                    node = re.search('/paths-(.+?)/', eth_attrs.get('tDn', ''))
                    if not node:
                        continue
                    eth_node = 'node_id:' + node.group(1)
                    if eth_node not in eth_meta:
                        eth_meta.append(eth_node)
                    # populating the map for eth-app mapping

                    tenant_fabric_key = node.group(1) + ":" + port.group(1)
                    if tenant_fabric_key not in self.tenant_farbic_mapper:
                        self.tenant_farbic_mapper[
                            tenant_fabric_key] = application_meta
                    else:
                        self.tenant_farbic_mapper[tenant_fabric_key].extend(
                            application_meta)

                    self.tenant_farbic_mapper[tenant_fabric_key] = list(
                        set(self.tenant_farbic_mapper[tenant_fabric_key]))
            except exceptions.APIConnectionException, exceptions.APIParsingException:
                # the exception will already be logged, just pass it over here
                pass
Exemple #20
0
def test_tenant_mocked(aggregator):
    check = CiscoACICheck(common.CHECK_NAME, {}, {})
    api = Api(common.ACI_URLS,
              check.http,
              common.USERNAME,
              password=common.PASSWORD,
              log=check.log)
    api.wrapper_factory = common.FakeTenantSessionWrapper
    check._api_cache[hash_mutable(common.CONFIG_WITH_TAGS)] = api

    check.check(common.CONFIG_WITH_TAGS)

    tags = ['project:cisco_aci', 'tenant:DataDog']
    metric_name = 'cisco_aci.tenant.ingress_bytes.multicast.rate'
    aggregator.assert_metric(
        metric_name,
        value=0.0,
        tags=['endpoint_group:DtDg-Pay', 'application:DtDg-AP1-EcommerceApp'] +
        tags,
        hostname='',
    )
    aggregator.assert_metric(
        metric_name,
        value=0.0,
        tags=[
            'endpoint_group:DtDg-MiscAppVMs',
            'application:DtDg-AP1-EcommerceApp'
        ] + tags,
        hostname='',
    )
    aggregator.assert_metric(
        metric_name,
        value=0.0,
        tags=['endpoint_group:DtDg-Inv', 'application:DtDg-AP1-EcommerceApp'] +
        tags,
        hostname='',
    )
    aggregator.assert_metric(
        metric_name,
        value=0.0,
        tags=['endpoint_group:DtDg-Ord', 'application:DtDg-AP1-EcommerceApp'] +
        tags,
        hostname='',
    )
    aggregator.assert_metric(
        metric_name,
        value=0.0,
        tags=[
            'endpoint_group:DtDg-Ecomm', 'application:DtDg-AP1-EcommerceApp'
        ] + tags,
        hostname='',
    )
    aggregator.assert_metric(
        metric_name,
        value=0.0,
        tags=['endpoint_group:DtDg-Jeti2', 'application:DtDg-AP2-Jeti'] + tags,
        hostname='')
    aggregator.assert_metric(
        metric_name,
        value=0.0,
        tags=[
            'endpoint_group:DtDg-Jetty_Controller', 'application:DtDg-AP2-Jeti'
        ] + tags,
        hostname='',
    )
    aggregator.assert_metric(
        metric_name,
        value=0.0,
        tags=['endpoint_group:DtDg-Jeti1', 'application:DtDg-AP2-Jeti'] + tags,
        hostname='')
    aggregator.assert_metric(
        metric_name,
        value=0.0,
        tags=['endpoint_group:Test-EPG', 'application:DtDg-test-AP'] + tags,
        hostname='')

    metric_name = 'cisco_aci.tenant.egress_bytes.multicast.rate'
    aggregator.assert_metric(
        metric_name,
        value=0.0,
        tags=['endpoint_group:DtDg-Pay', 'application:DtDg-AP1-EcommerceApp'] +
        tags,
        hostname='',
    )
    aggregator.assert_metric(
        metric_name,
        value=0.0,
        tags=[
            'endpoint_group:DtDg-MiscAppVMs',
            'application:DtDg-AP1-EcommerceApp'
        ] + tags,
        hostname='',
    )
    aggregator.assert_metric(
        metric_name,
        value=0.0,
        tags=['endpoint_group:DtDg-Inv', 'application:DtDg-AP1-EcommerceApp'] +
        tags,
        hostname='',
    )
    aggregator.assert_metric(
        metric_name,
        value=0.0,
        tags=['endpoint_group:DtDg-Ord', 'application:DtDg-AP1-EcommerceApp'] +
        tags,
        hostname='',
    )
    aggregator.assert_metric(
        metric_name,
        value=0.0,
        tags=[
            'endpoint_group:DtDg-Ecomm', 'application:DtDg-AP1-EcommerceApp'
        ] + tags,
        hostname='',
    )
    aggregator.assert_metric(
        metric_name,
        value=0.0,
        tags=['endpoint_group:DtDg-Jeti2', 'application:DtDg-AP2-Jeti'] + tags,
        hostname='')
    aggregator.assert_metric(
        metric_name,
        value=0.0,
        tags=[
            'endpoint_group:DtDg-Jetty_Controller', 'application:DtDg-AP2-Jeti'
        ] + tags,
        hostname='',
    )
    aggregator.assert_metric(
        metric_name,
        value=0.0,
        tags=['endpoint_group:DtDg-Jeti1', 'application:DtDg-AP2-Jeti'] + tags,
        hostname='')
    aggregator.assert_metric(
        metric_name,
        value=0.0,
        tags=['endpoint_group:Test-EPG', 'application:DtDg-test-AP'] + tags,
        hostname='')

    metric_name = 'cisco_aci.tenant.ingress_pkts.multicast.rate'
    aggregator.assert_metric(
        metric_name,
        value=0.0,
        tags=['endpoint_group:DtDg-Pay', 'application:DtDg-AP1-EcommerceApp'] +
        tags,
        hostname='',
    )
    aggregator.assert_metric(
        metric_name,
        value=0.0,
        tags=[
            'endpoint_group:DtDg-MiscAppVMs',
            'application:DtDg-AP1-EcommerceApp'
        ] + tags,
        hostname='',
    )
    aggregator.assert_metric(
        metric_name,
        value=0.0,
        tags=['endpoint_group:DtDg-Inv', 'application:DtDg-AP1-EcommerceApp'] +
        tags,
        hostname='',
    )
    aggregator.assert_metric(
        metric_name,
        value=0.0,
        tags=['endpoint_group:DtDg-Ord', 'application:DtDg-AP1-EcommerceApp'] +
        tags,
        hostname='',
    )
    aggregator.assert_metric(
        metric_name,
        value=0.0,
        tags=[
            'endpoint_group:DtDg-Ecomm', 'application:DtDg-AP1-EcommerceApp'
        ] + tags,
        hostname='',
    )
    aggregator.assert_metric(
        metric_name,
        value=0.0,
        tags=['endpoint_group:DtDg-Jeti2', 'application:DtDg-AP2-Jeti'] + tags,
        hostname='')
    aggregator.assert_metric(
        metric_name,
        value=0.0,
        tags=[
            'endpoint_group:DtDg-Jetty_Controller', 'application:DtDg-AP2-Jeti'
        ] + tags,
        hostname='',
    )
    aggregator.assert_metric(
        metric_name,
        value=0.0,
        tags=['endpoint_group:DtDg-Jeti1', 'application:DtDg-AP2-Jeti'] + tags,
        hostname='')
    aggregator.assert_metric(
        metric_name,
        value=0.0,
        tags=['endpoint_group:Test-EPG', 'application:DtDg-test-AP'] + tags,
        hostname='')

    metric_name = 'cisco_aci.tenant.health'
    aggregator.assert_metric(metric_name, value=99.0, tags=tags, hostname='')

    metric_name = 'cisco_aci.tenant.overall_health'
    aggregator.assert_metric(metric_name, value=99.0, tags=tags, hostname='')

    metric_name = 'cisco_aci.tenant.egress_pkts.unicast.cum'
    aggregator.assert_metric(
        metric_name,
        value=0.0,
        tags=['endpoint_group:DtDg-Pay', 'application:DtDg-AP1-EcommerceApp'] +
        tags,
        hostname='',
    )
    aggregator.assert_metric(
        metric_name,
        value=0.0,
        tags=[
            'endpoint_group:DtDg-MiscAppVMs',
            'application:DtDg-AP1-EcommerceApp'
        ] + tags,
        hostname='',
    )
    aggregator.assert_metric(
        metric_name,
        value=0.0,
        tags=['endpoint_group:DtDg-Inv', 'application:DtDg-AP1-EcommerceApp'] +
        tags,
        hostname='',
    )
    aggregator.assert_metric(
        metric_name,
        value=0.0,
        tags=['endpoint_group:DtDg-Ord', 'application:DtDg-AP1-EcommerceApp'] +
        tags,
        hostname='',
    )
    aggregator.assert_metric(
        metric_name,
        value=0.0,
        tags=[
            'endpoint_group:DtDg-Ecomm', 'application:DtDg-AP1-EcommerceApp'
        ] + tags,
        hostname='',
    )
    aggregator.assert_metric(
        metric_name,
        value=0.0,
        tags=['endpoint_group:DtDg-Jeti2', 'application:DtDg-AP2-Jeti'] + tags,
        hostname='')
    aggregator.assert_metric(
        metric_name,
        value=0.0,
        tags=[
            'endpoint_group:DtDg-Jetty_Controller', 'application:DtDg-AP2-Jeti'
        ] + tags,
        hostname='',
    )
    aggregator.assert_metric(
        metric_name,
        value=0.0,
        tags=['endpoint_group:DtDg-Jeti1', 'application:DtDg-AP2-Jeti'] + tags,
        hostname='')
    aggregator.assert_metric(
        metric_name,
        value=0.0,
        tags=['endpoint_group:Test-EPG', 'application:DtDg-test-AP'] + tags,
        hostname='')

    metric_name = 'cisco_aci.tenant.egress_pkts.unicast.rate'
    aggregator.assert_metric(
        metric_name,
        value=0.0,
        tags=['endpoint_group:DtDg-Pay', 'application:DtDg-AP1-EcommerceApp'] +
        tags,
        hostname='',
    )
    aggregator.assert_metric(
        metric_name,
        value=0.0,
        tags=[
            'endpoint_group:DtDg-MiscAppVMs',
            'application:DtDg-AP1-EcommerceApp'
        ] + tags,
        hostname='',
    )
    aggregator.assert_metric(
        metric_name,
        value=0.0,
        tags=['endpoint_group:DtDg-Inv', 'application:DtDg-AP1-EcommerceApp'] +
        tags,
        hostname='',
    )
    aggregator.assert_metric(
        metric_name,
        value=0.0,
        tags=['endpoint_group:DtDg-Ord', 'application:DtDg-AP1-EcommerceApp'] +
        tags,
        hostname='',
    )
    aggregator.assert_metric(
        metric_name,
        value=0.0,
        tags=[
            'endpoint_group:DtDg-Ecomm', 'application:DtDg-AP1-EcommerceApp'
        ] + tags,
        hostname='',
    )
    aggregator.assert_metric(
        metric_name,
        value=0.0,
        tags=['endpoint_group:DtDg-Jeti2', 'application:DtDg-AP2-Jeti'] + tags,
        hostname='')
    aggregator.assert_metric(
        metric_name,
        value=0.0,
        tags=[
            'endpoint_group:DtDg-Jetty_Controller', 'application:DtDg-AP2-Jeti'
        ] + tags,
        hostname='',
    )
    aggregator.assert_metric(
        metric_name,
        value=0.0,
        tags=['endpoint_group:DtDg-Jeti1', 'application:DtDg-AP2-Jeti'] + tags,
        hostname='')
    aggregator.assert_metric(
        metric_name,
        value=0.0,
        tags=['endpoint_group:Test-EPG', 'application:DtDg-test-AP'] + tags,
        hostname='')

    metric_name = 'cisco_aci.tenant.application.fault_counter'
    aggregator.assert_metric(metric_name,
                             value=0.0,
                             tags=['application:DtDg-AP1-EcommerceApp'] + tags,
                             hostname='')
    aggregator.assert_metric(metric_name,
                             value=0.0,
                             tags=['application:DtDg-AP2-Jeti'] + tags,
                             hostname='')
    aggregator.assert_metric(metric_name,
                             value=0.0,
                             tags=['application:DtDg-test-AP'] + tags,
                             hostname='')

    metric_name = 'cisco_aci.tenant.fault_counter'
    aggregator.assert_metric(metric_name, value=4.0, tags=tags, hostname='')

    metric_name = 'cisco_aci.tenant.ingress_bytes.flood.cum'
    aggregator.assert_metric(
        metric_name,
        value=0.0,
        tags=['endpoint_group:DtDg-Pay', 'application:DtDg-AP1-EcommerceApp'] +
        tags,
        hostname='',
    )
    aggregator.assert_metric(
        metric_name,
        value=0.0,
        tags=[
            'endpoint_group:DtDg-MiscAppVMs',
            'application:DtDg-AP1-EcommerceApp'
        ] + tags,
        hostname='',
    )
    aggregator.assert_metric(
        metric_name,
        value=0.0,
        tags=['endpoint_group:DtDg-Inv', 'application:DtDg-AP1-EcommerceApp'] +
        tags,
        hostname='',
    )
    aggregator.assert_metric(
        metric_name,
        value=0.0,
        tags=['endpoint_group:DtDg-Ord', 'application:DtDg-AP1-EcommerceApp'] +
        tags,
        hostname='',
    )
    aggregator.assert_metric(
        metric_name,
        value=0.0,
        tags=[
            'endpoint_group:DtDg-Ecomm', 'application:DtDg-AP1-EcommerceApp'
        ] + tags,
        hostname='',
    )
    aggregator.assert_metric(
        metric_name,
        value=0.0,
        tags=['endpoint_group:DtDg-Jeti2', 'application:DtDg-AP2-Jeti'] + tags,
        hostname='')
    aggregator.assert_metric(
        metric_name,
        value=0.0,
        tags=[
            'endpoint_group:DtDg-Jetty_Controller', 'application:DtDg-AP2-Jeti'
        ] + tags,
        hostname='',
    )
    aggregator.assert_metric(
        metric_name,
        value=0.0,
        tags=['endpoint_group:DtDg-Jeti1', 'application:DtDg-AP2-Jeti'] + tags,
        hostname='')
    aggregator.assert_metric(
        metric_name,
        value=0.0,
        tags=['endpoint_group:Test-EPG', 'application:DtDg-test-AP'] + tags,
        hostname='')

    metric_name = 'cisco_aci.tenant.ingress_pkts.unicast.cum'
    aggregator.assert_metric(
        metric_name,
        value=0.0,
        tags=['endpoint_group:DtDg-Pay', 'application:DtDg-AP1-EcommerceApp'] +
        tags,
        hostname='',
    )
    aggregator.assert_metric(
        metric_name,
        value=0.0,
        tags=[
            'endpoint_group:DtDg-MiscAppVMs',
            'application:DtDg-AP1-EcommerceApp'
        ] + tags,
        hostname='',
    )
    aggregator.assert_metric(
        metric_name,
        value=0.0,
        tags=['endpoint_group:DtDg-Inv', 'application:DtDg-AP1-EcommerceApp'] +
        tags,
        hostname='',
    )
    aggregator.assert_metric(
        metric_name,
        value=0.0,
        tags=['endpoint_group:DtDg-Ord', 'application:DtDg-AP1-EcommerceApp'] +
        tags,
        hostname='',
    )
    aggregator.assert_metric(
        metric_name,
        value=0.0,
        tags=[
            'endpoint_group:DtDg-Ecomm', 'application:DtDg-AP1-EcommerceApp'
        ] + tags,
        hostname='',
    )
    aggregator.assert_metric(
        metric_name,
        value=0.0,
        tags=['endpoint_group:DtDg-Jeti2', 'application:DtDg-AP2-Jeti'] + tags,
        hostname='')
    aggregator.assert_metric(
        metric_name,
        value=0.0,
        tags=[
            'endpoint_group:DtDg-Jetty_Controller', 'application:DtDg-AP2-Jeti'
        ] + tags,
        hostname='',
    )
    aggregator.assert_metric(
        metric_name,
        value=0.0,
        tags=['endpoint_group:DtDg-Jeti1', 'application:DtDg-AP2-Jeti'] + tags,
        hostname='')
    aggregator.assert_metric(
        metric_name,
        value=0.0,
        tags=['endpoint_group:Test-EPG', 'application:DtDg-test-AP'] + tags,
        hostname='')

    metric_name = 'cisco_aci.tenant.egress_bytes.unicast.cum'
    aggregator.assert_metric(
        metric_name,
        value=0.0,
        tags=['endpoint_group:DtDg-Pay', 'application:DtDg-AP1-EcommerceApp'] +
        tags,
        hostname='',
    )
    aggregator.assert_metric(
        metric_name,
        value=0.0,
        tags=[
            'endpoint_group:DtDg-MiscAppVMs',
            'application:DtDg-AP1-EcommerceApp'
        ] + tags,
        hostname='',
    )
    aggregator.assert_metric(
        metric_name,
        value=0.0,
        tags=['endpoint_group:DtDg-Inv', 'application:DtDg-AP1-EcommerceApp'] +
        tags,
        hostname='',
    )
    aggregator.assert_metric(
        metric_name,
        value=0.0,
        tags=['endpoint_group:DtDg-Ord', 'application:DtDg-AP1-EcommerceApp'] +
        tags,
        hostname='',
    )
    aggregator.assert_metric(
        metric_name,
        value=0.0,
        tags=[
            'endpoint_group:DtDg-Ecomm', 'application:DtDg-AP1-EcommerceApp'
        ] + tags,
        hostname='',
    )
    aggregator.assert_metric(
        metric_name,
        value=0.0,
        tags=['endpoint_group:DtDg-Jeti2', 'application:DtDg-AP2-Jeti'] + tags,
        hostname='')
    aggregator.assert_metric(
        metric_name,
        value=0.0,
        tags=[
            'endpoint_group:DtDg-Jetty_Controller', 'application:DtDg-AP2-Jeti'
        ] + tags,
        hostname='',
    )
    aggregator.assert_metric(
        metric_name,
        value=0.0,
        tags=['endpoint_group:DtDg-Jeti1', 'application:DtDg-AP2-Jeti'] + tags,
        hostname='')
    aggregator.assert_metric(
        metric_name,
        value=0.0,
        tags=['endpoint_group:Test-EPG', 'application:DtDg-test-AP'] + tags,
        hostname='')

    metric_name = 'cisco_aci.tenant.egress_pkts.multicast.cum'
    aggregator.assert_metric(
        metric_name,
        value=0.0,
        tags=['endpoint_group:DtDg-Pay', 'application:DtDg-AP1-EcommerceApp'] +
        tags,
        hostname='',
    )
    aggregator.assert_metric(
        metric_name,
        value=0.0,
        tags=[
            'endpoint_group:DtDg-MiscAppVMs',
            'application:DtDg-AP1-EcommerceApp'
        ] + tags,
        hostname='',
    )
    aggregator.assert_metric(
        metric_name,
        value=0.0,
        tags=['endpoint_group:DtDg-Inv', 'application:DtDg-AP1-EcommerceApp'] +
        tags,
        hostname='',
    )
    aggregator.assert_metric(
        metric_name,
        value=0.0,
        tags=['endpoint_group:DtDg-Ord', 'application:DtDg-AP1-EcommerceApp'] +
        tags,
        hostname='',
    )
    aggregator.assert_metric(
        metric_name,
        value=0.0,
        tags=[
            'endpoint_group:DtDg-Ecomm', 'application:DtDg-AP1-EcommerceApp'
        ] + tags,
        hostname='',
    )
    aggregator.assert_metric(
        metric_name,
        value=0.0,
        tags=['endpoint_group:DtDg-Jeti2', 'application:DtDg-AP2-Jeti'] + tags,
        hostname='')
    aggregator.assert_metric(
        metric_name,
        value=0.0,
        tags=[
            'endpoint_group:DtDg-Jetty_Controller', 'application:DtDg-AP2-Jeti'
        ] + tags,
        hostname='',
    )
    aggregator.assert_metric(
        metric_name,
        value=0.0,
        tags=['endpoint_group:DtDg-Jeti1', 'application:DtDg-AP2-Jeti'] + tags,
        hostname='')
    aggregator.assert_metric(
        metric_name,
        value=0.0,
        tags=['endpoint_group:Test-EPG', 'application:DtDg-test-AP'] + tags,
        hostname='')

    metric_name = 'cisco_aci.tenant.ingress_pkts.flood.cum'
    aggregator.assert_metric(
        metric_name,
        value=0.0,
        tags=['endpoint_group:DtDg-Pay', 'application:DtDg-AP1-EcommerceApp'] +
        tags,
        hostname='',
    )
    aggregator.assert_metric(
        metric_name,
        value=0.0,
        tags=[
            'endpoint_group:DtDg-MiscAppVMs',
            'application:DtDg-AP1-EcommerceApp'
        ] + tags,
        hostname='',
    )
    aggregator.assert_metric(
        metric_name,
        value=0.0,
        tags=['endpoint_group:DtDg-Inv', 'application:DtDg-AP1-EcommerceApp'] +
        tags,
        hostname='',
    )
    aggregator.assert_metric(
        metric_name,
        value=0.0,
        tags=['endpoint_group:DtDg-Ord', 'application:DtDg-AP1-EcommerceApp'] +
        tags,
        hostname='',
    )
    aggregator.assert_metric(
        metric_name,
        value=0.0,
        tags=[
            'endpoint_group:DtDg-Ecomm', 'application:DtDg-AP1-EcommerceApp'
        ] + tags,
        hostname='',
    )
    aggregator.assert_metric(
        metric_name,
        value=0.0,
        tags=['endpoint_group:DtDg-Jeti2', 'application:DtDg-AP2-Jeti'] + tags,
        hostname='')
    aggregator.assert_metric(
        metric_name,
        value=0.0,
        tags=[
            'endpoint_group:DtDg-Jetty_Controller', 'application:DtDg-AP2-Jeti'
        ] + tags,
        hostname='',
    )
    aggregator.assert_metric(
        metric_name,
        value=0.0,
        tags=['endpoint_group:DtDg-Jeti1', 'application:DtDg-AP2-Jeti'] + tags,
        hostname='')
    aggregator.assert_metric(
        metric_name,
        value=0.0,
        tags=['endpoint_group:Test-EPG', 'application:DtDg-test-AP'] + tags,
        hostname='')

    metric_name = 'cisco_aci.tenant.egress_bytes.unicast.rate'
    aggregator.assert_metric(
        metric_name,
        value=0.0,
        tags=['endpoint_group:DtDg-Pay', 'application:DtDg-AP1-EcommerceApp'] +
        tags,
        hostname='',
    )
    aggregator.assert_metric(
        metric_name,
        value=0.0,
        tags=[
            'endpoint_group:DtDg-MiscAppVMs',
            'application:DtDg-AP1-EcommerceApp'
        ] + tags,
        hostname='',
    )
    aggregator.assert_metric(
        metric_name,
        value=0.0,
        tags=['endpoint_group:DtDg-Inv', 'application:DtDg-AP1-EcommerceApp'] +
        tags,
        hostname='',
    )
    aggregator.assert_metric(
        metric_name,
        value=0.0,
        tags=['endpoint_group:DtDg-Ord', 'application:DtDg-AP1-EcommerceApp'] +
        tags,
        hostname='',
    )
    aggregator.assert_metric(
        metric_name,
        value=0.0,
        tags=[
            'endpoint_group:DtDg-Ecomm', 'application:DtDg-AP1-EcommerceApp'
        ] + tags,
        hostname='',
    )
    aggregator.assert_metric(
        metric_name,
        value=0.0,
        tags=['endpoint_group:DtDg-Jeti2', 'application:DtDg-AP2-Jeti'] + tags,
        hostname='')
    aggregator.assert_metric(
        metric_name,
        value=0.0,
        tags=[
            'endpoint_group:DtDg-Jetty_Controller', 'application:DtDg-AP2-Jeti'
        ] + tags,
        hostname='',
    )
    aggregator.assert_metric(
        metric_name,
        value=0.0,
        tags=['endpoint_group:DtDg-Jeti1', 'application:DtDg-AP2-Jeti'] + tags,
        hostname='')
    aggregator.assert_metric(
        metric_name,
        value=0.0,
        tags=['endpoint_group:Test-EPG', 'application:DtDg-test-AP'] + tags,
        hostname='')

    metric_name = 'cisco_aci.tenant.egress_bytes.multicast.cum'
    aggregator.assert_metric(
        metric_name,
        value=0.0,
        tags=['endpoint_group:DtDg-Pay', 'application:DtDg-AP1-EcommerceApp'] +
        tags,
        hostname='',
    )
    aggregator.assert_metric(
        metric_name,
        value=0.0,
        tags=[
            'endpoint_group:DtDg-MiscAppVMs',
            'application:DtDg-AP1-EcommerceApp'
        ] + tags,
        hostname='',
    )
    aggregator.assert_metric(
        metric_name,
        value=0.0,
        tags=['endpoint_group:DtDg-Inv', 'application:DtDg-AP1-EcommerceApp'] +
        tags,
        hostname='',
    )
    aggregator.assert_metric(
        metric_name,
        value=0.0,
        tags=['endpoint_group:DtDg-Ord', 'application:DtDg-AP1-EcommerceApp'] +
        tags,
        hostname='',
    )
    aggregator.assert_metric(
        metric_name,
        value=0.0,
        tags=[
            'endpoint_group:DtDg-Ecomm', 'application:DtDg-AP1-EcommerceApp'
        ] + tags,
        hostname='',
    )
    aggregator.assert_metric(
        metric_name,
        value=0.0,
        tags=['endpoint_group:DtDg-Jeti2', 'application:DtDg-AP2-Jeti'] + tags,
        hostname='')
    aggregator.assert_metric(
        metric_name,
        value=0.0,
        tags=[
            'endpoint_group:DtDg-Jetty_Controller', 'application:DtDg-AP2-Jeti'
        ] + tags,
        hostname='',
    )
    aggregator.assert_metric(
        metric_name,
        value=0.0,
        tags=['endpoint_group:DtDg-Jeti1', 'application:DtDg-AP2-Jeti'] + tags,
        hostname='')
    aggregator.assert_metric(
        metric_name,
        value=0.0,
        tags=['endpoint_group:Test-EPG', 'application:DtDg-test-AP'] + tags,
        hostname='')

    metric_name = 'cisco_aci.tenant.ingress_bytes.unicast.cum'
    aggregator.assert_metric(
        metric_name,
        value=0.0,
        tags=['endpoint_group:DtDg-Pay', 'application:DtDg-AP1-EcommerceApp'] +
        tags,
        hostname='',
    )
    aggregator.assert_metric(
        metric_name,
        value=0.0,
        tags=[
            'endpoint_group:DtDg-MiscAppVMs',
            'application:DtDg-AP1-EcommerceApp'
        ] + tags,
        hostname='',
    )
    aggregator.assert_metric(
        metric_name,
        value=0.0,
        tags=['endpoint_group:DtDg-Inv', 'application:DtDg-AP1-EcommerceApp'] +
        tags,
        hostname='',
    )
    aggregator.assert_metric(
        metric_name,
        value=0.0,
        tags=['endpoint_group:DtDg-Ord', 'application:DtDg-AP1-EcommerceApp'] +
        tags,
        hostname='',
    )
    aggregator.assert_metric(
        metric_name,
        value=0.0,
        tags=[
            'endpoint_group:DtDg-Ecomm', 'application:DtDg-AP1-EcommerceApp'
        ] + tags,
        hostname='',
    )
    aggregator.assert_metric(
        metric_name,
        value=0.0,
        tags=['endpoint_group:DtDg-Jeti2', 'application:DtDg-AP2-Jeti'] + tags,
        hostname='')
    aggregator.assert_metric(
        metric_name,
        value=0.0,
        tags=[
            'endpoint_group:DtDg-Jetty_Controller', 'application:DtDg-AP2-Jeti'
        ] + tags,
        hostname='',
    )
    aggregator.assert_metric(
        metric_name,
        value=0.0,
        tags=['endpoint_group:DtDg-Jeti1', 'application:DtDg-AP2-Jeti'] + tags,
        hostname='')
    aggregator.assert_metric(
        metric_name,
        value=0.0,
        tags=['endpoint_group:Test-EPG', 'application:DtDg-test-AP'] + tags,
        hostname='')

    metric_name = 'cisco_aci.tenant.ingress_pkts.drop.cum'
    aggregator.assert_metric(
        metric_name,
        value=0.0,
        tags=['endpoint_group:DtDg-Pay', 'application:DtDg-AP1-EcommerceApp'] +
        tags,
        hostname='',
    )
    aggregator.assert_metric(
        metric_name,
        value=0.0,
        tags=[
            'endpoint_group:DtDg-MiscAppVMs',
            'application:DtDg-AP1-EcommerceApp'
        ] + tags,
        hostname='',
    )
    aggregator.assert_metric(
        metric_name,
        value=0.0,
        tags=['endpoint_group:DtDg-Inv', 'application:DtDg-AP1-EcommerceApp'] +
        tags,
        hostname='',
    )
    aggregator.assert_metric(
        metric_name,
        value=0.0,
        tags=['endpoint_group:DtDg-Ord', 'application:DtDg-AP1-EcommerceApp'] +
        tags,
        hostname='',
    )
    aggregator.assert_metric(
        metric_name,
        value=0.0,
        tags=[
            'endpoint_group:DtDg-Ecomm', 'application:DtDg-AP1-EcommerceApp'
        ] + tags,
        hostname='',
    )
    aggregator.assert_metric(
        metric_name,
        value=0.0,
        tags=['endpoint_group:DtDg-Jeti2', 'application:DtDg-AP2-Jeti'] + tags,
        hostname='')
    aggregator.assert_metric(
        metric_name,
        value=0.0,
        tags=[
            'endpoint_group:DtDg-Jetty_Controller', 'application:DtDg-AP2-Jeti'
        ] + tags,
        hostname='',
    )
    aggregator.assert_metric(
        metric_name,
        value=0.0,
        tags=['endpoint_group:DtDg-Jeti1', 'application:DtDg-AP2-Jeti'] + tags,
        hostname='')
    aggregator.assert_metric(
        metric_name,
        value=0.0,
        tags=['endpoint_group:Test-EPG', 'application:DtDg-test-AP'] + tags,
        hostname='')

    metric_name = 'cisco_aci.capacity.apic.fabric_node.utilized'
    aggregator.assert_metric(metric_name,
                             value=0.0,
                             tags=['project:cisco_aci', 'cisco'],
                             hostname='')

    metric_name = 'cisco_aci.tenant.ingress_pkts.multicast.cum'
    aggregator.assert_metric(
        metric_name,
        value=0.0,
        tags=['endpoint_group:DtDg-Pay', 'application:DtDg-AP1-EcommerceApp'] +
        tags,
        hostname='',
    )
    aggregator.assert_metric(
        metric_name,
        value=0.0,
        tags=[
            'endpoint_group:DtDg-MiscAppVMs',
            'application:DtDg-AP1-EcommerceApp'
        ] + tags,
        hostname='',
    )
    aggregator.assert_metric(
        metric_name,
        value=0.0,
        tags=['endpoint_group:DtDg-Inv', 'application:DtDg-AP1-EcommerceApp'] +
        tags,
        hostname='',
    )
    aggregator.assert_metric(
        metric_name,
        value=0.0,
        tags=['endpoint_group:DtDg-Ord', 'application:DtDg-AP1-EcommerceApp'] +
        tags,
        hostname='',
    )
    aggregator.assert_metric(
        metric_name,
        value=0.0,
        tags=[
            'endpoint_group:DtDg-Ecomm', 'application:DtDg-AP1-EcommerceApp'
        ] + tags,
        hostname='',
    )
    aggregator.assert_metric(
        metric_name,
        value=0.0,
        tags=['endpoint_group:DtDg-Jeti2', 'application:DtDg-AP2-Jeti'] + tags,
        hostname='')
    aggregator.assert_metric(
        metric_name,
        value=0.0,
        tags=[
            'endpoint_group:DtDg-Jetty_Controller', 'application:DtDg-AP2-Jeti'
        ] + tags,
        hostname='',
    )
    aggregator.assert_metric(
        metric_name,
        value=0.0,
        tags=['endpoint_group:DtDg-Jeti1', 'application:DtDg-AP2-Jeti'] + tags,
        hostname='')
    aggregator.assert_metric(
        metric_name,
        value=0.0,
        tags=['endpoint_group:Test-EPG', 'application:DtDg-test-AP'] + tags,
        hostname='')

    metric_name = 'cisco_aci.tenant.egress_pkts.multicast.rate'
    aggregator.assert_metric(
        metric_name,
        value=0.0,
        tags=['endpoint_group:DtDg-Pay', 'application:DtDg-AP1-EcommerceApp'] +
        tags,
        hostname='',
    )
    aggregator.assert_metric(
        metric_name,
        value=0.0,
        tags=[
            'endpoint_group:DtDg-MiscAppVMs',
            'application:DtDg-AP1-EcommerceApp'
        ] + tags,
        hostname='',
    )
    aggregator.assert_metric(
        metric_name,
        value=0.0,
        tags=['endpoint_group:DtDg-Inv', 'application:DtDg-AP1-EcommerceApp'] +
        tags,
        hostname='',
    )
    aggregator.assert_metric(
        metric_name,
        value=0.0,
        tags=['endpoint_group:DtDg-Ord', 'application:DtDg-AP1-EcommerceApp'] +
        tags,
        hostname='',
    )
    aggregator.assert_metric(
        metric_name,
        value=0.0,
        tags=[
            'endpoint_group:DtDg-Ecomm', 'application:DtDg-AP1-EcommerceApp'
        ] + tags,
        hostname='',
    )
    aggregator.assert_metric(
        metric_name,
        value=0.0,
        tags=['endpoint_group:DtDg-Jeti2', 'application:DtDg-AP2-Jeti'] + tags,
        hostname='')
    aggregator.assert_metric(
        metric_name,
        value=0.0,
        tags=[
            'endpoint_group:DtDg-Jetty_Controller', 'application:DtDg-AP2-Jeti'
        ] + tags,
        hostname='',
    )
    aggregator.assert_metric(
        metric_name,
        value=0.0,
        tags=['endpoint_group:DtDg-Jeti1', 'application:DtDg-AP2-Jeti'] + tags,
        hostname='')
    aggregator.assert_metric(
        metric_name,
        value=0.0,
        tags=['endpoint_group:Test-EPG', 'application:DtDg-test-AP'] + tags,
        hostname='')

    metric_name = 'cisco_aci.tenant.ingress_bytes.drop.cum'
    aggregator.assert_metric(
        metric_name,
        value=0.0,
        tags=['endpoint_group:DtDg-Pay', 'application:DtDg-AP1-EcommerceApp'] +
        tags,
        hostname='',
    )
    aggregator.assert_metric(
        metric_name,
        value=0.0,
        tags=[
            'endpoint_group:DtDg-MiscAppVMs',
            'application:DtDg-AP1-EcommerceApp'
        ] + tags,
        hostname='',
    )
    aggregator.assert_metric(
        metric_name,
        value=0.0,
        tags=['endpoint_group:DtDg-Inv', 'application:DtDg-AP1-EcommerceApp'] +
        tags,
        hostname='',
    )
    aggregator.assert_metric(
        metric_name,
        value=0.0,
        tags=['endpoint_group:DtDg-Ord', 'application:DtDg-AP1-EcommerceApp'] +
        tags,
        hostname='',
    )
    aggregator.assert_metric(
        metric_name,
        value=0.0,
        tags=[
            'endpoint_group:DtDg-Ecomm', 'application:DtDg-AP1-EcommerceApp'
        ] + tags,
        hostname='',
    )
    aggregator.assert_metric(
        metric_name,
        value=0.0,
        tags=['endpoint_group:DtDg-Jeti2', 'application:DtDg-AP2-Jeti'] + tags,
        hostname='')
    aggregator.assert_metric(
        metric_name,
        value=0.0,
        tags=[
            'endpoint_group:DtDg-Jetty_Controller', 'application:DtDg-AP2-Jeti'
        ] + tags,
        hostname='',
    )
    aggregator.assert_metric(
        metric_name,
        value=0.0,
        tags=['endpoint_group:DtDg-Jeti1', 'application:DtDg-AP2-Jeti'] + tags,
        hostname='')
    aggregator.assert_metric(
        metric_name,
        value=0.0,
        tags=['endpoint_group:Test-EPG', 'application:DtDg-test-AP'] + tags,
        hostname='')

    metric_name = 'cisco_aci.tenant.ingress_bytes.unicast.rate'
    aggregator.assert_metric(
        metric_name,
        value=0.0,
        tags=['endpoint_group:DtDg-Pay', 'application:DtDg-AP1-EcommerceApp'] +
        tags,
        hostname='',
    )
    aggregator.assert_metric(
        metric_name,
        value=0.0,
        tags=[
            'endpoint_group:DtDg-MiscAppVMs',
            'application:DtDg-AP1-EcommerceApp'
        ] + tags,
        hostname='',
    )
    aggregator.assert_metric(
        metric_name,
        value=0.0,
        tags=['endpoint_group:DtDg-Inv', 'application:DtDg-AP1-EcommerceApp'] +
        tags,
        hostname='',
    )
    aggregator.assert_metric(
        metric_name,
        value=0.0,
        tags=['endpoint_group:DtDg-Ord', 'application:DtDg-AP1-EcommerceApp'] +
        tags,
        hostname='',
    )
    aggregator.assert_metric(
        metric_name,
        value=0.0,
        tags=[
            'endpoint_group:DtDg-Ecomm', 'application:DtDg-AP1-EcommerceApp'
        ] + tags,
        hostname='',
    )
    aggregator.assert_metric(
        metric_name,
        value=0.0,
        tags=['endpoint_group:DtDg-Jeti2', 'application:DtDg-AP2-Jeti'] + tags,
        hostname='')
    aggregator.assert_metric(
        metric_name,
        value=0.0,
        tags=[
            'endpoint_group:DtDg-Jetty_Controller', 'application:DtDg-AP2-Jeti'
        ] + tags,
        hostname='',
    )
    aggregator.assert_metric(
        metric_name,
        value=0.0,
        tags=['endpoint_group:DtDg-Jeti1', 'application:DtDg-AP2-Jeti'] + tags,
        hostname='')
    aggregator.assert_metric(
        metric_name,
        value=0.0,
        tags=['endpoint_group:Test-EPG', 'application:DtDg-test-AP'] + tags,
        hostname='')

    metric_name = 'cisco_aci.tenant.ingress_bytes.multicast.cum'
    aggregator.assert_metric(
        metric_name,
        value=0.0,
        tags=['endpoint_group:DtDg-Pay', 'application:DtDg-AP1-EcommerceApp'] +
        tags,
        hostname='',
    )
    aggregator.assert_metric(
        metric_name,
        value=0.0,
        tags=[
            'endpoint_group:DtDg-MiscAppVMs',
            'application:DtDg-AP1-EcommerceApp'
        ] + tags,
        hostname='',
    )
    aggregator.assert_metric(
        metric_name,
        value=0.0,
        tags=['endpoint_group:DtDg-Inv', 'application:DtDg-AP1-EcommerceApp'] +
        tags,
        hostname='',
    )
    aggregator.assert_metric(
        metric_name,
        value=0.0,
        tags=['endpoint_group:DtDg-Ord', 'application:DtDg-AP1-EcommerceApp'] +
        tags,
        hostname='',
    )
    aggregator.assert_metric(
        metric_name,
        value=0.0,
        tags=[
            'endpoint_group:DtDg-Ecomm', 'application:DtDg-AP1-EcommerceApp'
        ] + tags,
        hostname='',
    )
    aggregator.assert_metric(
        metric_name,
        value=0.0,
        tags=['endpoint_group:DtDg-Jeti2', 'application:DtDg-AP2-Jeti'] + tags,
        hostname='')
    aggregator.assert_metric(
        metric_name,
        value=0.0,
        tags=[
            'endpoint_group:DtDg-Jetty_Controller', 'application:DtDg-AP2-Jeti'
        ] + tags,
        hostname='',
    )
    aggregator.assert_metric(
        metric_name,
        value=0.0,
        tags=['endpoint_group:DtDg-Jeti1', 'application:DtDg-AP2-Jeti'] + tags,
        hostname='')
    aggregator.assert_metric(
        metric_name,
        value=0.0,
        tags=['endpoint_group:Test-EPG', 'application:DtDg-test-AP'] + tags,
        hostname='')

    metric_name = 'cisco_aci.tenant.ingress_pkts.unicast.rate'
    aggregator.assert_metric(
        metric_name,
        value=0.0,
        tags=['endpoint_group:DtDg-Pay', 'application:DtDg-AP1-EcommerceApp'] +
        tags,
        hostname='',
    )
    aggregator.assert_metric(
        metric_name,
        value=0.0,
        tags=[
            'endpoint_group:DtDg-MiscAppVMs',
            'application:DtDg-AP1-EcommerceApp'
        ] + tags,
        hostname='',
    )
    aggregator.assert_metric(
        metric_name,
        value=0.0,
        tags=['endpoint_group:DtDg-Inv', 'application:DtDg-AP1-EcommerceApp'] +
        tags,
        hostname='',
    )
    aggregator.assert_metric(
        metric_name,
        value=0.0,
        tags=['endpoint_group:DtDg-Ord', 'application:DtDg-AP1-EcommerceApp'] +
        tags,
        hostname='',
    )
    aggregator.assert_metric(
        metric_name,
        value=0.0,
        tags=[
            'endpoint_group:DtDg-Ecomm', 'application:DtDg-AP1-EcommerceApp'
        ] + tags,
        hostname='',
    )
    aggregator.assert_metric(
        metric_name,
        value=0.0,
        tags=['endpoint_group:DtDg-Jeti2', 'application:DtDg-AP2-Jeti'] + tags,
        hostname='')
    aggregator.assert_metric(
        metric_name,
        value=0.0,
        tags=[
            'endpoint_group:DtDg-Jetty_Controller', 'application:DtDg-AP2-Jeti'
        ] + tags,
        hostname='',
    )
    aggregator.assert_metric(
        metric_name,
        value=0.0,
        tags=['endpoint_group:DtDg-Jeti1', 'application:DtDg-AP2-Jeti'] + tags,
        hostname='')
    aggregator.assert_metric(
        metric_name,
        value=0.0,
        tags=['endpoint_group:Test-EPG', 'application:DtDg-test-AP'] + tags,
        hostname='')

    # Assert coverage for this check on this instance
    aggregator.assert_all_metrics_covered()
Exemple #21
0
def test_capacity_end_to_end(aggregator, session_mock):
    check = CiscoACICheck(common.CHECK_NAME, {}, {})
    api = Api(common.ACI_URLS, common.USERNAME, password=common.PASSWORD, log=check.log, sessions=[session_mock])
    api._refresh_sessions = False
    check._api_cache[hash_mutable(common.CONFIG_WITH_TAGS)] = api

    check.check(common.CONFIG_WITH_TAGS)

    tags = ['cisco', 'project:cisco_aci']
    aggregator.assert_metric(
        'cisco_aci.capacity.leaf.bridge_domain.utilized',
        value=44.0,
        tags=['fabric_pod_id:1', 'node_id:101'] + tags,
        hostname='pod-1-node-101',
    )
    aggregator.assert_metric(
        'cisco_aci.capacity.leaf.bridge_domain.utilized',
        value=1.0,
        tags=['fabric_pod_id:1', 'node_id:201'] + tags,
        hostname='pod-1-node-201',
    )
    aggregator.assert_metric(
        'cisco_aci.capacity.leaf.bridge_domain.utilized',
        value=1.0,
        tags=['fabric_pod_id:1', 'node_id:202'] + tags,
        hostname='pod-1-node-202',
    )
    aggregator.assert_metric(
        'cisco_aci.capacity.leaf.bridge_domain.utilized',
        value=34.0,
        tags=['fabric_pod_id:1', 'node_id:102'] + tags,
        hostname='pod-1-node-102',
    )
    aggregator.assert_metric('cisco_aci.capacity.apic.endpoint_group.utilized', value=205.0, tags=tags, hostname='')
    aggregator.assert_metric('cisco_aci.capacity.apic.private_network.utilized', value=85.0, tags=tags, hostname='')
    aggregator.assert_metric(
        'cisco_aci.capacity.leaf.bridge_domain.limit',
        value=3500.0,
        tags=['fabric_pod_id:1', 'node_id:101'] + tags,
        hostname='pod-1-node-101',
    )
    aggregator.assert_metric(
        'cisco_aci.capacity.leaf.bridge_domain.limit',
        value=3500.0,
        tags=['fabric_pod_id:1', 'node_id:201'] + tags,
        hostname='pod-1-node-201',
    )
    aggregator.assert_metric(
        'cisco_aci.capacity.leaf.bridge_domain.limit',
        value=3500.0,
        tags=['fabric_pod_id:1', 'node_id:202'] + tags,
        hostname='pod-1-node-202',
    )
    aggregator.assert_metric(
        'cisco_aci.capacity.leaf.bridge_domain.limit',
        value=3500.0,
        tags=['fabric_pod_id:1', 'node_id:102'] + tags,
        hostname='pod-1-node-102',
    )
    aggregator.assert_metric('cisco_aci.capacity.apic.tenant.utilized', value=90.0, tags=tags, hostname='')
    aggregator.assert_metric(
        'cisco_aci.capacity.leaf.endpoint_group.utilized',
        value=94.0,
        tags=['fabric_pod_id:1', 'node_id:101'] + tags,
        hostname='pod-1-node-101',
    )
    aggregator.assert_metric(
        'cisco_aci.capacity.leaf.endpoint_group.utilized',
        value=0.0,
        tags=['fabric_pod_id:1', 'node_id:201'] + tags,
        hostname='pod-1-node-201',
    )
    aggregator.assert_metric(
        'cisco_aci.capacity.leaf.endpoint_group.utilized',
        value=0.0,
        tags=['fabric_pod_id:1', 'node_id:202'] + tags,
        hostname='pod-1-node-202',
    )
    aggregator.assert_metric(
        'cisco_aci.capacity.leaf.endpoint_group.utilized',
        value=78.0,
        tags=['fabric_pod_id:1', 'node_id:102'] + tags,
        hostname='pod-1-node-102',
    )
    aggregator.assert_metric('cisco_aci.capacity.apic.endpoint_group.limit', value=15000.0, tags=tags, hostname='')
    aggregator.assert_metric(
        'cisco_aci.capacity.leaf.endpoint_group.limit',
        value=3500.0,
        tags=['fabric_pod_id:1', 'node_id:101'] + tags,
        hostname='pod-1-node-101',
    )
    aggregator.assert_metric(
        'cisco_aci.capacity.leaf.endpoint_group.limit',
        value=3500.0,
        tags=['fabric_pod_id:1', 'node_id:201'] + tags,
        hostname='pod-1-node-201',
    )
    aggregator.assert_metric(
        'cisco_aci.capacity.leaf.endpoint_group.limit',
        value=3500.0,
        tags=['fabric_pod_id:1', 'node_id:202'] + tags,
        hostname='pod-1-node-202',
    )
    aggregator.assert_metric(
        'cisco_aci.capacity.leaf.endpoint_group.limit',
        value=3500.0,
        tags=['fabric_pod_id:1', 'node_id:102'] + tags,
        hostname='pod-1-node-102',
    )
    aggregator.assert_metric('cisco_aci.capacity.apic.endpoint.limit', value=180000.0, tags=tags, hostname='')
    aggregator.assert_metric('cisco_aci.capacity.apic.endpoint.utilized', value=76.0, tags=tags, hostname='')
    aggregator.assert_metric('cisco_aci.capacity.apic.bridge_domain.utilized', value=154.0, tags=tags, hostname='')
    aggregator.assert_metric('cisco_aci.capacity.apic.vmware_domain.limit', value=5.0, tags=tags, hostname='')
    aggregator.assert_metric('cisco_aci.capacity.apic.private_network.limit', value=3000.0, tags=tags, hostname='')
    aggregator.assert_metric(
        'cisco_aci.capacity.leaf.vrf.utilized',
        value=32.0,
        tags=['fabric_pod_id:1', 'node_id:101'] + tags,
        hostname='pod-1-node-101',
    )
    aggregator.assert_metric(
        'cisco_aci.capacity.leaf.vrf.utilized',
        value=4.0,
        tags=['fabric_pod_id:1', 'node_id:201'] + tags,
        hostname='pod-1-node-201',
    )
    aggregator.assert_metric(
        'cisco_aci.capacity.leaf.vrf.utilized',
        value=4.0,
        tags=['fabric_pod_id:1', 'node_id:202'] + tags,
        hostname='pod-1-node-202',
    )
    aggregator.assert_metric(
        'cisco_aci.capacity.leaf.vrf.utilized',
        value=27.0,
        tags=['fabric_pod_id:1', 'node_id:102'] + tags,
        hostname='pod-1-node-102',
    )
    aggregator.assert_metric('cisco_aci.capacity.apic.contract.limit', value=1000.0, tags=tags, hostname='')
    aggregator.assert_metric(
        'cisco_aci.capacity.apic.azure_domain.endpoint_group.limit', value=9000.0, tags=tags, hostname=''
    )
    aggregator.assert_metric('cisco_aci.capacity.apic.fabric_node.limit', value=200.0, tags=tags, hostname='')
    aggregator.assert_metric('cisco_aci.capacity.apic.bridge_domain.limit', value=15000.0, tags=tags, hostname='')
    aggregator.assert_metric('cisco_aci.capacity.apic.fabric_node.utilized', value=2.0, tags=tags, hostname='')
    aggregator.assert_metric('cisco_aci.capacity.apic.tenant.limit', value=3000.0, tags=tags, hostname='')
    aggregator.assert_metric(
        'cisco_aci.capacity.leaf.vrf.limit',
        value=800.0,
        tags=['fabric_pod_id:1', 'node_id:101'] + tags,
        hostname='pod-1-node-101',
    )
    aggregator.assert_metric(
        'cisco_aci.capacity.leaf.vrf.limit',
        value=800.0,
        tags=['fabric_pod_id:1', 'node_id:201'] + tags,
        hostname='pod-1-node-201',
    )
    aggregator.assert_metric(
        'cisco_aci.capacity.leaf.vrf.limit',
        value=800.0,
        tags=['fabric_pod_id:1', 'node_id:202'] + tags,
        hostname='pod-1-node-202',
    )
    aggregator.assert_metric(
        'cisco_aci.capacity.leaf.vrf.limit',
        value=800.0,
        tags=['fabric_pod_id:1', 'node_id:102'] + tags,
        hostname='pod-1-node-102',
    )
    aggregator.assert_metric(
        'cisco_aci.capacity.apic.vmware_domain.endpoint_group.limit', value=15000.0, tags=tags, hostname=''
    )
    aggregator.assert_metric('cisco_aci.capacity.apic.azure_domain.limit', value=5.0, tags=tags, hostname='')
    aggregator.assert_metric('cisco_aci.capacity.apic.service_graph.limit', value=600.0, tags=tags, hostname='')
Exemple #22
0
    def check(self, instance):
        self.log.info("Starting Cisco Check")
        start = datetime.datetime.utcnow()
        aci_url = instance.get('aci_url')
        aci_urls = instance.get('aci_urls', [])
        if aci_url:
            aci_urls.append(aci_url)

        if len(aci_urls) == 0:
            raise Exception("The Cisco ACI check requires at least one url")

        username = instance['username']
        pwd = instance.get('pwd')
        instance_hash = hash_mutable(instance)

        appcenter = _is_affirmative(instance.get('appcenter'))

        cert_key = instance.get('cert_key')
        if not cert_key and instance.get('cert_key_path'):
            with open(instance.get('cert_key_path'), 'rb') as f:
                cert_key = f.read()

        cert_name = instance.get('cert_name')
        if not cert_name:
            cert_name = username

        cert_key_password = instance.get('cert_key_password')

        timeout = instance.get('timeout', 15)
        ssl_verify = _is_affirmative(instance.get('ssl_verify', True))

        if instance_hash in self._api_cache:
            api = self._api_cache.get(instance_hash)
        else:
            api = Api(
                aci_urls,
                username,
                password=pwd,
                cert_name=cert_name,
                cert_key=cert_key,
                verify=ssl_verify,
                timeout=timeout,
                log=self.log,
                appcenter=appcenter,
                cert_key_password=cert_key_password,
            )
            self._api_cache[instance_hash] = api

        service_check_tags = []
        for url in aci_urls:
            service_check_tags.append("url:{}".format(url))
        service_check_tags.extend(self.check_tags)
        service_check_tags.extend(instance.get('tags', []))

        try:
            api.login()
        except Exception as e:
            self.log.error("Cannot login to the Cisco ACI: %s", e)
            self.service_check(
                SERVICE_CHECK_NAME,
                AgentCheck.CRITICAL,
                message="aci login returned a status of {}".format(e),
                tags=service_check_tags,
            )
            raise

        self.tagger.api = api

        try:
            tenant = Tenant(self, api, instance, instance_hash)
            tenant.collect()
        except Exception as e:
            self.log.error('tenant collection failed: %s', e)
            self.service_check(
                SERVICE_CHECK_NAME,
                AgentCheck.CRITICAL,
                message="aci tenant operations failed, returning a status of {}"
                .format(e),
                tags=service_check_tags,
            )
            api.close()
            raise

        try:
            fabric = Fabric(self, api, instance)
            fabric.collect()
        except Exception as e:
            self.log.error('fabric collection failed: %s', e)
            self.service_check(
                SERVICE_CHECK_NAME,
                AgentCheck.CRITICAL,
                message="aci fabric operations failed, returning a status of {}"
                .format(e),
                tags=service_check_tags,
            )
            api.close()
            raise

        try:
            capacity = Capacity(api,
                                instance,
                                check_tags=self.check_tags,
                                gauge=self.gauge,
                                log=self.log)
            capacity.collect()
        except Exception as e:
            self.log.error('capacity collection failed: %s', e)
            self.service_check(
                SERVICE_CHECK_NAME,
                AgentCheck.CRITICAL,
                message=
                "aci capacity operations failed, returning a status of {}".
                format(e),
                tags=service_check_tags,
            )
            api.close()
            raise

        self.service_check(SERVICE_CHECK_NAME,
                           AgentCheck.OK,
                           tags=service_check_tags)

        self.set_external_tags(self.get_external_host_tags())

        api.close()
        end = datetime.datetime.utcnow()
        log_line = "finished running Cisco Check"
        if _is_affirmative(instance.get('report_timing', False)):
            log_line += ", took {}".format(end - start)
        self.log.info(log_line)
def test_tenant_mapper():
    tags = CiscoTags()
    api1 = ApiMock1()

    tags.api = api1
    assert tags._tenant_mapper(None) == []
    assert tags._tenant_mapper("") == []
    assert tags._tenant_mapper("aaa") == []
    assert tags._tenant_mapper([]) == []
    assert tags._tenant_mapper(["aaa"]) == []
    assert tags._tenant_mapper({}) == []
    assert tags._tenant_mapper({"aaa": "aaa"}) == []
    assert all(
        [
            a == b
            for a, b in zip(
                sorted(tags._tenant_mapper({"attributes": {"name": "aaa", "dn": "a/tn-bbb/ap-ccc/a"}})),
                sorted(['tenant:bbb', 'endpoint_group:aaa', 'application:ccc']),
            )
        ]
    )

    api2 = ApiMock2()
    tags.api = api2
    assert all(
        [
            a == b
            for a, b in zip(
                sorted(tags._tenant_mapper({"attributes": {"name": "aaa", "dn": "a/tn-bbb/ap-ccc/a"}})),
                sorted(['tenant:bbb', 'application:ccc', 'endpoint_group:aaa', "ip:ddd", "mac:eee", "encap:fff"]),
            )
        ]
    )

    context_hash = hash_mutable(
        ['tenant:bbb', 'application:ccc', 'endpoint_group:aaa', "ip:ddd", "mac:eee", "encap:fff"]
    )
    api3 = ApiMock3()
    tags.api = api3
    tags.tenant_tags = {context_hash: ["test:ggg"]}
    assert all(
        [
            a == b
            for a, b in zip(
                sorted(tags._tenant_mapper({"attributes": {"name": "aaa", "dn": "a/tn-bbb/ap-ccc/a"}})),
                sorted(
                    [
                        'tenant:bbb',
                        'application:ccc',
                        'endpoint_group:aaa',
                        "ip:ddd",
                        "mac:eee",
                        "encap:fff",
                        "test:ggg",
                    ]
                ),
            )
        ]
    )
    assert tags.tenant_farbic_mapper == {}

    api3 = ApiMock3()
    tags.api = api3
    tags.tenant_tags = {}
    assert all(
        [
            a == b
            for a, b in zip(
                sorted(tags._tenant_mapper({"attributes": {"name": "aaa", "dn": "a/tn-bbb/ap-ccc/a"}})),
                sorted(
                    [
                        'ip:ddd',
                        'mac:eee',
                        'encap:fff',
                        'endpoint_group:aaa',
                        'application:ccc',
                        'tenant:bbb',
                        'port:bbb',
                        'port:ccc',
                        'port:ddd',
                        'port:kkk',
                        'node_id:[jjj]',
                    ]
                ),
            )
        ]
    )
    assert all(
        [
            a == b
            for a, b in zip(
                sorted(tags.tenant_farbic_mapper.get('[jjj]:kkk', [])),
                sorted(['application:ccc', 'endpoint_group:aaa', 'tenant:bbb']),
            )
        ]
    )
Exemple #24
0
    def check(self, instance):
        sites = instance.get('sites')
        if sites is None:
            expected_sites = set()
        else:
            expected_sites = set(sites)
        # _Total should always be in the list of expected sites; we always
        # report _Total
        if "_Total" not in expected_sites:
            expected_sites.add("_Total")

        self.log.debug("expected sites is {}".format(str(expected_sites)))
        key = hash_mutable(instance)
        for inst_name, dd_name, metric_func, counter in self._metrics[key]:
            try:
                try:
                    vals = counter.get_all_values()
                except Exception as e:
                    self.log.error("Failed to get_all_values {} {}".format(
                        inst_name, dd_name))
                    continue

                for sitename, val in iteritems(vals):
                    tags = []
                    if key in self._tags:
                        tags = list(self._tags[key])

                    try:
                        if not counter.is_single_instance():
                            # Skip any sites we don't specifically want.
                            if not sites:
                                tags.append("site:{0}".format(
                                    ensure_unicode(self.normalize(sitename))))
                            # always report total
                            elif sitename == "_Total":
                                tags.append("site:{0}".format(
                                    ensure_unicode(self.normalize(sitename))))
                            elif sitename not in sites:
                                continue
                            else:
                                tags.append("site:{0}".format(
                                    ensure_unicode(self.normalize(sitename))))
                    except Exception as e:
                        self.log.error(
                            "Caught exception {} setting tags".format(str(e)))

                    try:
                        metric_func(dd_name, val, tags)
                    except Exception as e:
                        self.log.error("metric_func: {} {} {}".format(
                            dd_name, str(val), str(e)))
                        pass

                    if dd_name == "iis.uptime":
                        uptime = int(val)
                        status = AgentCheck.CRITICAL if uptime == 0 else AgentCheck.OK
                        self.service_check(self.SERVICE_CHECK, status, tags)
                        if sitename in expected_sites:
                            self.log.debug(
                                "Removing {} from expected sites".format(
                                    sitename))
                            expected_sites.remove(sitename)
                        else:
                            self.log.warning(
                                "site not in expected_sites {}".format(
                                    sitename))

            except Exception as e:
                # don't give up on all of the metrics because one failed
                self.log.error(
                    "IIS Failed to get metric data for {} {}: {}".format(
                        inst_name, dd_name, str(e)))
                pass

        for site in expected_sites:
            tags = []
            if key in self._tags:
                tags = list(self._tags[key])
            tags.append("site:{}".format(ensure_unicode(self.normalize(site))))
            self.service_check(self.SERVICE_CHECK, AgentCheck.CRITICAL, tags)
Exemple #25
0
    def check(self, instance):
        # Instance state is mutable, any changes to it will be reflected in self._instance_states
        instance_state = self._instance_states[hash_mutable(instance)]

        self._check_for_leader_change(instance, instance_state)

        peers = self.get_peers_in_cluster(instance)
        main_tags = []
        agent_dc = self._get_agent_datacenter(instance, instance_state)

        if agent_dc is not None:
            main_tags.append('consul_datacenter:{0}'.format(agent_dc))

        for tag in instance.get('tags', []):
            main_tags.append(tag)

        if not self._is_instance_leader(instance, instance_state):
            self.gauge("consul.peers", len(peers), tags=main_tags + ["mode:follower"])
            self.log.debug("This consul agent is not the cluster leader." +
                           "Skipping service and catalog checks for this instance")
            return
        else:
            self.gauge("consul.peers", len(peers), tags=main_tags + ["mode:leader"])

        service_check_tags = main_tags + ['consul_url:{0}'.format(instance.get('url'))]
        perform_catalog_checks = is_affirmative(instance.get('catalog_checks',
                                                             self.init_config.get('catalog_checks')))
        perform_network_latency_checks = is_affirmative(instance.get('network_latency_checks',
                                                                     self.init_config.get('network_latency_checks')))

        try:
            # Make service checks from health checks for all services in catalog
            health_state = self.consul_request(instance, '/v1/health/state/any')

            sc = {}
            # compute the highest status level (OK < WARNING < CRITICAL) a a check among all the nodes is running on.
            for check in health_state:
                sc_id = '{0}/{1}/{2}'.format(check['CheckID'], check.get('ServiceID', ''), check.get('ServiceName', ''))
                status = self.STATUS_SC.get(check['Status'])
                if status is None:
                    status = AgentCheck.UNKNOWN

                if sc_id not in sc:
                    tags = ["check:{0}".format(check["CheckID"])]
                    if check["ServiceName"]:
                        tags.append("service:{0}".format(check["ServiceName"]))
                    if check["ServiceID"]:
                        tags.append("consul_service_id:{0}".format(check["ServiceID"]))
                    sc[sc_id] = {'status': status, 'tags': tags}

                elif self.STATUS_SEVERITY[status] > self.STATUS_SEVERITY[sc[sc_id]['status']]:
                    sc[sc_id]['status'] = status

            for s in sc.values():
                self.service_check(self.HEALTH_CHECK, s['status'], tags=main_tags+s['tags'])

        except Exception as e:
            self.log.error(e)
            self.service_check(self.CONSUL_CHECK, AgentCheck.CRITICAL,
                               tags=service_check_tags)
        else:
            self.service_check(self.CONSUL_CHECK, AgentCheck.OK,
                               tags=service_check_tags)

        if perform_catalog_checks:
            # Collect node by service, and service by node counts for a whitelist of services

            services = self.get_services_in_cluster(instance)
            service_whitelist = instance.get('service_whitelist',
                                             self.init_config.get('service_whitelist', []))
            max_services = instance.get('max_services',
                                        self.init_config.get('max_services', self.MAX_SERVICES))

            self.count_all_nodes(instance, main_tags)

            services = self._cull_services_list(services, service_whitelist, max_services)

            # {node_id: {"up: 0, "passing": 0, "warning": 0, "critical": 0}
            nodes_to_service_status = defaultdict(lambda: defaultdict(int))

            for service in services:
                # For every service in the cluster,
                # Gauge the following:
                # `consul.catalog.nodes_up` : # of Nodes registered with that service
                # `consul.catalog.nodes_passing` : # of Nodes with service status `passing` from those registered
                # `consul.catalog.nodes_warning` : # of Nodes with service status `warning` from those registered
                # `consul.catalog.nodes_critical` : # of Nodes with service status `critical` from those registered

                service_tags = self._get_service_tags(service, services[service])

                nodes_with_service = self.get_nodes_with_service(instance, service)

                # {'up': 0, 'passing': 0, 'warning': 0, 'critical': 0}
                node_status = defaultdict(int)

                for node in nodes_with_service:
                    # The node_id is n['Node']['Node']
                    node_id = node.get('Node', {}).get("Node")

                    # An additional service is registered on this node. Bump up the counter
                    nodes_to_service_status[node_id]["up"] += 1

                    # If there is no Check for the node then Consul and dd-agent consider it up
                    if 'Checks' not in node:
                        node_status['passing'] += 1
                        node_status['up'] += 1
                    else:
                        found_critical = False
                        found_warning = False
                        found_serf_health = False

                        for check in node['Checks']:
                            if check['CheckID'] == 'serfHealth':
                                found_serf_health = True

                                # For backwards compatibility, the "up" node_status is computed
                                # based on the total # of nodes 'running' as part of the service.

                                # If the serfHealth is `critical` it means the Consul agent isn't even responding,
                                # and we don't register the node as `up`
                                if check['Status'] != 'critical':
                                    node_status["up"] += 1
                                    continue

                            if check['Status'] == 'critical':
                                found_critical = True
                                break
                            elif check['Status'] == 'warning':
                                found_warning = True
                                # Keep looping in case there is a critical status

                        # Increment the counters based on what was found in Checks
                        # `critical` checks override `warning`s, and if neither are found,
                        # register the node as `passing`
                        if found_critical:
                            node_status['critical'] += 1
                            nodes_to_service_status[node_id]["critical"] += 1
                        elif found_warning:
                            node_status['warning'] += 1
                            nodes_to_service_status[node_id]["warning"] += 1
                        else:
                            if not found_serf_health:
                                # We have not found a serfHealth check for this node, which is unexpected
                                # If we get here assume this node's status is "up", since we register it as 'passing'
                                node_status['up'] += 1

                            node_status['passing'] += 1
                            nodes_to_service_status[node_id]["passing"] += 1

                for status_key in self.STATUS_SC:
                    status_value = node_status[status_key]
                    self.gauge(
                        '{0}.nodes_{1}'.format(self.CONSUL_CATALOG_CHECK, status_key),
                        status_value,
                        tags=main_tags+service_tags
                    )

            for node, service_status in nodes_to_service_status.iteritems():
                # For every node discovered for whitelisted services, gauge the following:
                # `consul.catalog.services_up` : Total services registered on node
                # `consul.catalog.services_passing` : Total passing services on node
                # `consul.catalog.services_warning` : Total warning services on node
                # `consul.catalog.services_critical` : Total critical services on node

                node_tags = ['consul_node_id:{0}'.format(node)]
                self.gauge('{0}.services_up'.format(self.CONSUL_CATALOG_CHECK),
                           len(services),
                           tags=main_tags+node_tags)

                for status_key in self.STATUS_SC:
                    status_value = service_status[status_key]
                    self.gauge(
                        '{0}.services_{1}'.format(self.CONSUL_CATALOG_CHECK, status_key),
                        status_value,
                        tags=main_tags+node_tags
                    )

        if perform_network_latency_checks:
            self.check_network_latency(instance, agent_dc, main_tags)
Exemple #26
0
def test_tenant_end_to_end(aggregator, session_mock):
    check = CiscoACICheck(conftest.CHECK_NAME, {}, {})
    api = Api(conftest.ACI_URLS, conftest.USERNAME, conftest.PASSWORD, log=check.log, sessions=[session_mock])
    api._refresh_sessions = False
    check._api_cache[hash_mutable(conftest.CONFIG_WITH_TAGS)] = api

    check.check(conftest.CONFIG_WITH_TAGS)

    tags = ['project:cisco_aci', 'tenant:DataDog']
    # TODO pretty much everything is 0 and without hostname??
    metric_name = 'cisco_aci.tenant.ingress_bytes.multicast.rate'
    aggregator.assert_metric(metric_name, value=0.0, tags=['endpoint_group:DtDg-Pay',
                                                           'application:DtDg-AP1-EcommerceApp'] + tags, hostname='')
    aggregator.assert_metric(metric_name, value=0.0, tags=['endpoint_group:DtDg-MiscAppVMs',
                                                           'application:DtDg-AP1-EcommerceApp'] + tags, hostname='')
    aggregator.assert_metric(metric_name, value=0.0, tags=['endpoint_group:DtDg-Inv',
                                                           'application:DtDg-AP1-EcommerceApp'] + tags, hostname='')
    aggregator.assert_metric(metric_name, value=0.0, tags=['endpoint_group:DtDg-Ord',
                                                           'application:DtDg-AP1-EcommerceApp'] + tags, hostname='')
    aggregator.assert_metric(metric_name, value=0.0, tags=['endpoint_group:DtDg-Ecomm',
                                                           'application:DtDg-AP1-EcommerceApp'] + tags, hostname='')
    aggregator.assert_metric(metric_name, value=0.0, tags=['endpoint_group:DtDg-Jeti2',
                                                           'application:DtDg-AP2-Jeti'] + tags, hostname='')
    aggregator.assert_metric(metric_name, value=0.0, tags=['endpoint_group:DtDg-Jetty_Controller',
                                                           'application:DtDg-AP2-Jeti'] + tags, hostname='')
    aggregator.assert_metric(metric_name, value=0.0, tags=['endpoint_group:DtDg-Jeti1',
                                                           'application:DtDg-AP2-Jeti'] + tags, hostname='')
    aggregator.assert_metric(metric_name, value=0.0, tags=['endpoint_group:Test-EPG',
                                                           'application:DtDg-test-AP'] + tags, hostname='')

    metric_name = 'cisco_aci.tenant.egress_bytes.multicast.rate'
    aggregator.assert_metric(metric_name, value=0.0, tags=['endpoint_group:DtDg-Pay',
                                                           'application:DtDg-AP1-EcommerceApp'] + tags, hostname='')
    aggregator.assert_metric(metric_name, value=0.0, tags=['endpoint_group:DtDg-MiscAppVMs',
                                                           'application:DtDg-AP1-EcommerceApp'] + tags, hostname='')
    aggregator.assert_metric(metric_name, value=0.0, tags=['endpoint_group:DtDg-Inv',
                                                           'application:DtDg-AP1-EcommerceApp'] + tags, hostname='')
    aggregator.assert_metric(metric_name, value=0.0, tags=['endpoint_group:DtDg-Ord',
                                                           'application:DtDg-AP1-EcommerceApp'] + tags, hostname='')
    aggregator.assert_metric(metric_name, value=0.0, tags=['endpoint_group:DtDg-Ecomm',
                                                           'application:DtDg-AP1-EcommerceApp'] + tags, hostname='')
    aggregator.assert_metric(metric_name, value=0.0, tags=['endpoint_group:DtDg-Jeti2',
                                                           'application:DtDg-AP2-Jeti'] + tags, hostname='')
    aggregator.assert_metric(metric_name, value=0.0, tags=['endpoint_group:DtDg-Jetty_Controller',
                                                           'application:DtDg-AP2-Jeti'] + tags, hostname='')
    aggregator.assert_metric(metric_name, value=0.0, tags=['endpoint_group:DtDg-Jeti1',
                                                           'application:DtDg-AP2-Jeti'] + tags, hostname='')
    aggregator.assert_metric(metric_name, value=0.0, tags=['endpoint_group:Test-EPG',
                                                           'application:DtDg-test-AP'] + tags, hostname='')

    metric_name = 'cisco_aci.tenant.ingress_pkts.multicast.rate'
    aggregator.assert_metric(metric_name, value=0.0, tags=['endpoint_group:DtDg-Pay',
                                                           'application:DtDg-AP1-EcommerceApp'] + tags, hostname='')
    aggregator.assert_metric(metric_name, value=0.0, tags=['endpoint_group:DtDg-MiscAppVMs',
                                                           'application:DtDg-AP1-EcommerceApp'] + tags, hostname='')
    aggregator.assert_metric(metric_name, value=0.0, tags=['endpoint_group:DtDg-Inv',
                                                           'application:DtDg-AP1-EcommerceApp'] + tags, hostname='')
    aggregator.assert_metric(metric_name, value=0.0, tags=['endpoint_group:DtDg-Ord',
                                                           'application:DtDg-AP1-EcommerceApp'] + tags, hostname='')
    aggregator.assert_metric(metric_name, value=0.0, tags=['endpoint_group:DtDg-Ecomm',
                                                           'application:DtDg-AP1-EcommerceApp'] + tags, hostname='')
    aggregator.assert_metric(metric_name, value=0.0, tags=['endpoint_group:DtDg-Jeti2',
                                                           'application:DtDg-AP2-Jeti'] + tags, hostname='')
    aggregator.assert_metric(metric_name, value=0.0, tags=['endpoint_group:DtDg-Jetty_Controller',
                                                           'application:DtDg-AP2-Jeti'] + tags, hostname='')
    aggregator.assert_metric(metric_name, value=0.0, tags=['endpoint_group:DtDg-Jeti1',
                                                           'application:DtDg-AP2-Jeti'] + tags, hostname='')
    aggregator.assert_metric(metric_name, value=0.0, tags=['endpoint_group:Test-EPG',
                                                           'application:DtDg-test-AP'] + tags, hostname='')

    metric_name = 'cisco_aci.tenant.health'
    aggregator.assert_metric(metric_name, value=99.0, tags=tags, hostname='')

    metric_name = 'cisco_aci.tenant.overall_health'
    aggregator.assert_metric(metric_name, value=99.0, tags=tags, hostname='')

    metric_name = 'cisco_aci.tenant.egress_pkts.unicast.cum'
    aggregator.assert_metric(metric_name, value=0.0, tags=['endpoint_group:DtDg-Pay',
                                                           'application:DtDg-AP1-EcommerceApp'] + tags, hostname='')
    aggregator.assert_metric(metric_name, value=0.0, tags=['endpoint_group:DtDg-MiscAppVMs',
                                                           'application:DtDg-AP1-EcommerceApp'] + tags, hostname='')
    aggregator.assert_metric(metric_name, value=0.0, tags=['endpoint_group:DtDg-Inv',
                                                           'application:DtDg-AP1-EcommerceApp'] + tags, hostname='')
    aggregator.assert_metric(metric_name, value=0.0, tags=['endpoint_group:DtDg-Ord',
                                                           'application:DtDg-AP1-EcommerceApp'] + tags, hostname='')
    aggregator.assert_metric(metric_name, value=0.0, tags=['endpoint_group:DtDg-Ecomm',
                                                           'application:DtDg-AP1-EcommerceApp'] + tags, hostname='')
    aggregator.assert_metric(metric_name, value=0.0, tags=['endpoint_group:DtDg-Jeti2',
                                                           'application:DtDg-AP2-Jeti'] + tags, hostname='')
    aggregator.assert_metric(metric_name, value=0.0, tags=['endpoint_group:DtDg-Jetty_Controller',
                                                           'application:DtDg-AP2-Jeti'] + tags, hostname='')
    aggregator.assert_metric(metric_name, value=0.0, tags=['endpoint_group:DtDg-Jeti1',
                                                           'application:DtDg-AP2-Jeti'] + tags, hostname='')
    aggregator.assert_metric(metric_name, value=0.0, tags=['endpoint_group:Test-EPG',
                                                           'application:DtDg-test-AP'] + tags, hostname='')

    metric_name = 'cisco_aci.tenant.egress_pkts.unicast.rate'
    aggregator.assert_metric(metric_name, value=0.0, tags=['endpoint_group:DtDg-Pay',
                                                           'application:DtDg-AP1-EcommerceApp'] + tags, hostname='')
    aggregator.assert_metric(metric_name, value=0.0, tags=['endpoint_group:DtDg-MiscAppVMs',
                                                           'application:DtDg-AP1-EcommerceApp'] + tags, hostname='')
    aggregator.assert_metric(metric_name, value=0.0, tags=['endpoint_group:DtDg-Inv',
                                                           'application:DtDg-AP1-EcommerceApp'] + tags, hostname='')
    aggregator.assert_metric(metric_name, value=0.0, tags=['endpoint_group:DtDg-Ord',
                                                           'application:DtDg-AP1-EcommerceApp'] + tags, hostname='')
    aggregator.assert_metric(metric_name, value=0.0, tags=['endpoint_group:DtDg-Ecomm',
                                                           'application:DtDg-AP1-EcommerceApp'] + tags, hostname='')
    aggregator.assert_metric(metric_name, value=0.0, tags=['endpoint_group:DtDg-Jeti2',
                                                           'application:DtDg-AP2-Jeti'] + tags, hostname='')
    aggregator.assert_metric(metric_name, value=0.0, tags=['endpoint_group:DtDg-Jetty_Controller',
                                                           'application:DtDg-AP2-Jeti'] + tags, hostname='')
    aggregator.assert_metric(metric_name, value=0.0, tags=['endpoint_group:DtDg-Jeti1',
                                                           'application:DtDg-AP2-Jeti'] + tags, hostname='')
    aggregator.assert_metric(metric_name, value=0.0, tags=['endpoint_group:Test-EPG',
                                                           'application:DtDg-test-AP'] + tags, hostname='')

    metric_name = 'cisco_aci.tenant.application.fault_counter'
    aggregator.assert_metric(metric_name, value=0.0, tags=['application:DtDg-AP1-EcommerceApp'] + tags, hostname='')
    aggregator.assert_metric(metric_name, value=0.0, tags=['application:DtDg-AP2-Jeti'] + tags, hostname='')
    aggregator.assert_metric(metric_name, value=0.0, tags=['application:DtDg-test-AP'] + tags, hostname='')

    metric_name = 'cisco_aci.tenant.fault_counter'
    aggregator.assert_metric(metric_name, value=4.0, tags=tags, hostname='')

    metric_name = 'cisco_aci.tenant.ingress_bytes.flood.cum'
    aggregator.assert_metric(metric_name, value=0.0, tags=['endpoint_group:DtDg-Pay',
                                                           'application:DtDg-AP1-EcommerceApp'] + tags, hostname='')
    aggregator.assert_metric(metric_name, value=0.0, tags=['endpoint_group:DtDg-MiscAppVMs',
                                                           'application:DtDg-AP1-EcommerceApp'] + tags, hostname='')
    aggregator.assert_metric(metric_name, value=0.0, tags=['endpoint_group:DtDg-Inv',
                                                           'application:DtDg-AP1-EcommerceApp'] + tags, hostname='')
    aggregator.assert_metric(metric_name, value=0.0, tags=['endpoint_group:DtDg-Ord',
                                                           'application:DtDg-AP1-EcommerceApp'] + tags, hostname='')
    aggregator.assert_metric(metric_name, value=0.0, tags=['endpoint_group:DtDg-Ecomm',
                                                           'application:DtDg-AP1-EcommerceApp'] + tags, hostname='')
    aggregator.assert_metric(metric_name, value=0.0, tags=['endpoint_group:DtDg-Jeti2',
                                                           'application:DtDg-AP2-Jeti'] + tags, hostname='')
    aggregator.assert_metric(metric_name, value=0.0, tags=['endpoint_group:DtDg-Jetty_Controller',
                                                           'application:DtDg-AP2-Jeti'] + tags, hostname='')
    aggregator.assert_metric(metric_name, value=0.0, tags=['endpoint_group:DtDg-Jeti1',
                                                           'application:DtDg-AP2-Jeti'] + tags, hostname='')
    aggregator.assert_metric(metric_name, value=0.0, tags=['endpoint_group:Test-EPG',
                                                           'application:DtDg-test-AP'] + tags, hostname='')

    metric_name = 'cisco_aci.tenant.ingress_pkts.unicast.cum'
    aggregator.assert_metric(metric_name, value=0.0, tags=['endpoint_group:DtDg-Pay',
                                                           'application:DtDg-AP1-EcommerceApp'] + tags, hostname='')
    aggregator.assert_metric(metric_name, value=0.0, tags=['endpoint_group:DtDg-MiscAppVMs',
                                                           'application:DtDg-AP1-EcommerceApp'] + tags, hostname='')
    aggregator.assert_metric(metric_name, value=0.0, tags=['endpoint_group:DtDg-Inv',
                                                           'application:DtDg-AP1-EcommerceApp'] + tags, hostname='')
    aggregator.assert_metric(metric_name, value=0.0, tags=['endpoint_group:DtDg-Ord',
                                                           'application:DtDg-AP1-EcommerceApp'] + tags, hostname='')
    aggregator.assert_metric(metric_name, value=0.0, tags=['endpoint_group:DtDg-Ecomm',
                                                           'application:DtDg-AP1-EcommerceApp'] + tags, hostname='')
    aggregator.assert_metric(metric_name, value=0.0, tags=['endpoint_group:DtDg-Jeti2',
                                                           'application:DtDg-AP2-Jeti'] + tags, hostname='')
    aggregator.assert_metric(metric_name, value=0.0, tags=['endpoint_group:DtDg-Jetty_Controller',
                                                           'application:DtDg-AP2-Jeti'] + tags, hostname='')
    aggregator.assert_metric(metric_name, value=0.0, tags=['endpoint_group:DtDg-Jeti1',
                                                           'application:DtDg-AP2-Jeti'] + tags, hostname='')
    aggregator.assert_metric(metric_name, value=0.0, tags=['endpoint_group:Test-EPG',
                                                           'application:DtDg-test-AP'] + tags, hostname='')

    metric_name = 'cisco_aci.tenant.egress_bytes.unicast.cum'
    aggregator.assert_metric(metric_name, value=0.0, tags=['endpoint_group:DtDg-Pay',
                                                           'application:DtDg-AP1-EcommerceApp'] + tags, hostname='')
    aggregator.assert_metric(metric_name, value=0.0, tags=['endpoint_group:DtDg-MiscAppVMs',
                                                           'application:DtDg-AP1-EcommerceApp'] + tags, hostname='')
    aggregator.assert_metric(metric_name, value=0.0, tags=['endpoint_group:DtDg-Inv',
                                                           'application:DtDg-AP1-EcommerceApp'] + tags, hostname='')
    aggregator.assert_metric(metric_name, value=0.0, tags=['endpoint_group:DtDg-Ord',
                                                           'application:DtDg-AP1-EcommerceApp'] + tags, hostname='')
    aggregator.assert_metric(metric_name, value=0.0, tags=['endpoint_group:DtDg-Ecomm',
                                                           'application:DtDg-AP1-EcommerceApp'] + tags, hostname='')
    aggregator.assert_metric(metric_name, value=0.0, tags=['endpoint_group:DtDg-Jeti2',
                                                           'application:DtDg-AP2-Jeti'] + tags, hostname='')
    aggregator.assert_metric(metric_name, value=0.0, tags=['endpoint_group:DtDg-Jetty_Controller',
                                                           'application:DtDg-AP2-Jeti'] + tags, hostname='')
    aggregator.assert_metric(metric_name, value=0.0, tags=['endpoint_group:DtDg-Jeti1',
                                                           'application:DtDg-AP2-Jeti'] + tags, hostname='')
    aggregator.assert_metric(metric_name, value=0.0, tags=['endpoint_group:Test-EPG',
                                                           'application:DtDg-test-AP'] + tags, hostname='')

    metric_name = 'cisco_aci.tenant.egress_pkts.multicast.cum'
    aggregator.assert_metric(metric_name, value=0.0, tags=['endpoint_group:DtDg-Pay',
                                                           'application:DtDg-AP1-EcommerceApp'] + tags, hostname='')
    aggregator.assert_metric(metric_name, value=0.0, tags=['endpoint_group:DtDg-MiscAppVMs',
                                                           'application:DtDg-AP1-EcommerceApp'] + tags, hostname='')
    aggregator.assert_metric(metric_name, value=0.0, tags=['endpoint_group:DtDg-Inv',
                                                           'application:DtDg-AP1-EcommerceApp'] + tags, hostname='')
    aggregator.assert_metric(metric_name, value=0.0, tags=['endpoint_group:DtDg-Ord',
                                                           'application:DtDg-AP1-EcommerceApp'] + tags, hostname='')
    aggregator.assert_metric(metric_name, value=0.0, tags=['endpoint_group:DtDg-Ecomm',
                                                           'application:DtDg-AP1-EcommerceApp'] + tags, hostname='')
    aggregator.assert_metric(metric_name, value=0.0, tags=['endpoint_group:DtDg-Jeti2',
                                                           'application:DtDg-AP2-Jeti'] + tags, hostname='')
    aggregator.assert_metric(metric_name, value=0.0, tags=['endpoint_group:DtDg-Jetty_Controller',
                                                           'application:DtDg-AP2-Jeti'] + tags, hostname='')
    aggregator.assert_metric(metric_name, value=0.0, tags=['endpoint_group:DtDg-Jeti1',
                                                           'application:DtDg-AP2-Jeti'] + tags, hostname='')
    aggregator.assert_metric(metric_name, value=0.0, tags=['endpoint_group:Test-EPG',
                                                           'application:DtDg-test-AP'] + tags, hostname='')

    metric_name = 'cisco_aci.tenant.ingress_pkts.flood.cum'
    aggregator.assert_metric(metric_name, value=0.0, tags=['endpoint_group:DtDg-Pay',
                                                           'application:DtDg-AP1-EcommerceApp'] + tags, hostname='')
    aggregator.assert_metric(metric_name, value=0.0, tags=['endpoint_group:DtDg-MiscAppVMs',
                                                           'application:DtDg-AP1-EcommerceApp'] + tags, hostname='')
    aggregator.assert_metric(metric_name, value=0.0, tags=['endpoint_group:DtDg-Inv',
                                                           'application:DtDg-AP1-EcommerceApp'] + tags, hostname='')
    aggregator.assert_metric(metric_name, value=0.0, tags=['endpoint_group:DtDg-Ord',
                                                           'application:DtDg-AP1-EcommerceApp'] + tags, hostname='')
    aggregator.assert_metric(metric_name, value=0.0, tags=['endpoint_group:DtDg-Ecomm',
                                                           'application:DtDg-AP1-EcommerceApp'] + tags, hostname='')
    aggregator.assert_metric(metric_name, value=0.0, tags=['endpoint_group:DtDg-Jeti2',
                                                           'application:DtDg-AP2-Jeti'] + tags, hostname='')
    aggregator.assert_metric(metric_name, value=0.0, tags=['endpoint_group:DtDg-Jetty_Controller',
                                                           'application:DtDg-AP2-Jeti'] + tags, hostname='')
    aggregator.assert_metric(metric_name, value=0.0, tags=['endpoint_group:DtDg-Jeti1',
                                                           'application:DtDg-AP2-Jeti'] + tags, hostname='')
    aggregator.assert_metric(metric_name, value=0.0, tags=['endpoint_group:Test-EPG',
                                                           'application:DtDg-test-AP'] + tags, hostname='')

    metric_name = 'cisco_aci.tenant.egress_bytes.unicast.rate'
    aggregator.assert_metric(metric_name, value=0.0, tags=['endpoint_group:DtDg-Pay',
                                                           'application:DtDg-AP1-EcommerceApp'] + tags, hostname='')
    aggregator.assert_metric(metric_name, value=0.0, tags=['endpoint_group:DtDg-MiscAppVMs',
                                                           'application:DtDg-AP1-EcommerceApp'] + tags, hostname='')
    aggregator.assert_metric(metric_name, value=0.0, tags=['endpoint_group:DtDg-Inv',
                                                           'application:DtDg-AP1-EcommerceApp'] + tags, hostname='')
    aggregator.assert_metric(metric_name, value=0.0, tags=['endpoint_group:DtDg-Ord',
                                                           'application:DtDg-AP1-EcommerceApp'] + tags, hostname='')
    aggregator.assert_metric(metric_name, value=0.0, tags=['endpoint_group:DtDg-Ecomm',
                                                           'application:DtDg-AP1-EcommerceApp'] + tags, hostname='')
    aggregator.assert_metric(metric_name, value=0.0, tags=['endpoint_group:DtDg-Jeti2',
                                                           'application:DtDg-AP2-Jeti'] + tags, hostname='')
    aggregator.assert_metric(metric_name, value=0.0, tags=['endpoint_group:DtDg-Jetty_Controller',
                                                           'application:DtDg-AP2-Jeti'] + tags, hostname='')
    aggregator.assert_metric(metric_name, value=0.0, tags=['endpoint_group:DtDg-Jeti1',
                                                           'application:DtDg-AP2-Jeti'] + tags, hostname='')
    aggregator.assert_metric(metric_name, value=0.0, tags=['endpoint_group:Test-EPG',
                                                           'application:DtDg-test-AP'] + tags, hostname='')

    metric_name = 'cisco_aci.tenant.egress_bytes.multicast.cum'
    aggregator.assert_metric(metric_name, value=0.0, tags=['endpoint_group:DtDg-Pay',
                                                           'application:DtDg-AP1-EcommerceApp'] + tags, hostname='')
    aggregator.assert_metric(metric_name, value=0.0, tags=['endpoint_group:DtDg-MiscAppVMs',
                                                           'application:DtDg-AP1-EcommerceApp'] + tags, hostname='')
    aggregator.assert_metric(metric_name, value=0.0, tags=['endpoint_group:DtDg-Inv',
                                                           'application:DtDg-AP1-EcommerceApp'] + tags, hostname='')
    aggregator.assert_metric(metric_name, value=0.0, tags=['endpoint_group:DtDg-Ord',
                                                           'application:DtDg-AP1-EcommerceApp'] + tags, hostname='')
    aggregator.assert_metric(metric_name, value=0.0, tags=['endpoint_group:DtDg-Ecomm',
                                                           'application:DtDg-AP1-EcommerceApp'] + tags, hostname='')
    aggregator.assert_metric(metric_name, value=0.0, tags=['endpoint_group:DtDg-Jeti2',
                                                           'application:DtDg-AP2-Jeti'] + tags, hostname='')
    aggregator.assert_metric(metric_name, value=0.0, tags=['endpoint_group:DtDg-Jetty_Controller',
                                                           'application:DtDg-AP2-Jeti'] + tags, hostname='')
    aggregator.assert_metric(metric_name, value=0.0, tags=['endpoint_group:DtDg-Jeti1',
                                                           'application:DtDg-AP2-Jeti'] + tags, hostname='')
    aggregator.assert_metric(metric_name, value=0.0, tags=['endpoint_group:Test-EPG',
                                                           'application:DtDg-test-AP'] + tags, hostname='')

    metric_name = 'cisco_aci.tenant.ingress_bytes.unicast.cum'
    aggregator.assert_metric(metric_name, value=0.0, tags=['endpoint_group:DtDg-Pay',
                                                           'application:DtDg-AP1-EcommerceApp'] + tags, hostname='')
    aggregator.assert_metric(metric_name, value=0.0, tags=['endpoint_group:DtDg-MiscAppVMs',
                                                           'application:DtDg-AP1-EcommerceApp'] + tags, hostname='')
    aggregator.assert_metric(metric_name, value=0.0, tags=['endpoint_group:DtDg-Inv',
                                                           'application:DtDg-AP1-EcommerceApp'] + tags, hostname='')
    aggregator.assert_metric(metric_name, value=0.0, tags=['endpoint_group:DtDg-Ord',
                                                           'application:DtDg-AP1-EcommerceApp'] + tags, hostname='')
    aggregator.assert_metric(metric_name, value=0.0, tags=['endpoint_group:DtDg-Ecomm',
                                                           'application:DtDg-AP1-EcommerceApp'] + tags, hostname='')
    aggregator.assert_metric(metric_name, value=0.0, tags=['endpoint_group:DtDg-Jeti2',
                                                           'application:DtDg-AP2-Jeti'] + tags, hostname='')
    aggregator.assert_metric(metric_name, value=0.0, tags=['endpoint_group:DtDg-Jetty_Controller',
                                                           'application:DtDg-AP2-Jeti'] + tags, hostname='')
    aggregator.assert_metric(metric_name, value=0.0, tags=['endpoint_group:DtDg-Jeti1',
                                                           'application:DtDg-AP2-Jeti'] + tags, hostname='')
    aggregator.assert_metric(metric_name, value=0.0, tags=['endpoint_group:Test-EPG',
                                                           'application:DtDg-test-AP'] + tags, hostname='')

    metric_name = 'cisco_aci.tenant.ingress_pkts.drop.cum'
    aggregator.assert_metric(metric_name, value=0.0, tags=['endpoint_group:DtDg-Pay',
                                                           'application:DtDg-AP1-EcommerceApp'] + tags, hostname='')
    aggregator.assert_metric(metric_name, value=0.0, tags=['endpoint_group:DtDg-MiscAppVMs',
                                                           'application:DtDg-AP1-EcommerceApp'] + tags, hostname='')
    aggregator.assert_metric(metric_name, value=0.0, tags=['endpoint_group:DtDg-Inv',
                                                           'application:DtDg-AP1-EcommerceApp'] + tags, hostname='')
    aggregator.assert_metric(metric_name, value=0.0, tags=['endpoint_group:DtDg-Ord',
                                                           'application:DtDg-AP1-EcommerceApp'] + tags, hostname='')
    aggregator.assert_metric(metric_name, value=0.0, tags=['endpoint_group:DtDg-Ecomm',
                                                           'application:DtDg-AP1-EcommerceApp'] + tags, hostname='')
    aggregator.assert_metric(metric_name, value=0.0, tags=['endpoint_group:DtDg-Jeti2',
                                                           'application:DtDg-AP2-Jeti'] + tags, hostname='')
    aggregator.assert_metric(metric_name, value=0.0, tags=['endpoint_group:DtDg-Jetty_Controller',
                                                           'application:DtDg-AP2-Jeti'] + tags, hostname='')
    aggregator.assert_metric(metric_name, value=0.0, tags=['endpoint_group:DtDg-Jeti1',
                                                           'application:DtDg-AP2-Jeti'] + tags, hostname='')
    aggregator.assert_metric(metric_name, value=0.0, tags=['endpoint_group:Test-EPG',
                                                           'application:DtDg-test-AP'] + tags, hostname='')

    metric_name = 'cisco_aci.capacity.apic.fabric_node.utilized'
    aggregator.assert_metric(metric_name, value=0.0, tags=['project:cisco_aci', 'cisco'], hostname='')

    metric_name = 'cisco_aci.tenant.ingress_pkts.multicast.cum'
    aggregator.assert_metric(metric_name, value=0.0, tags=['endpoint_group:DtDg-Pay',
                                                           'application:DtDg-AP1-EcommerceApp'] + tags, hostname='')
    aggregator.assert_metric(metric_name, value=0.0, tags=['endpoint_group:DtDg-MiscAppVMs',
                                                           'application:DtDg-AP1-EcommerceApp'] + tags, hostname='')
    aggregator.assert_metric(metric_name, value=0.0, tags=['endpoint_group:DtDg-Inv',
                                                           'application:DtDg-AP1-EcommerceApp'] + tags, hostname='')
    aggregator.assert_metric(metric_name, value=0.0, tags=['endpoint_group:DtDg-Ord',
                                                           'application:DtDg-AP1-EcommerceApp'] + tags, hostname='')
    aggregator.assert_metric(metric_name, value=0.0, tags=['endpoint_group:DtDg-Ecomm',
                                                           'application:DtDg-AP1-EcommerceApp'] + tags, hostname='')
    aggregator.assert_metric(metric_name, value=0.0, tags=['endpoint_group:DtDg-Jeti2',
                                                           'application:DtDg-AP2-Jeti'] + tags, hostname='')
    aggregator.assert_metric(metric_name, value=0.0, tags=['endpoint_group:DtDg-Jetty_Controller',
                                                           'application:DtDg-AP2-Jeti'] + tags, hostname='')
    aggregator.assert_metric(metric_name, value=0.0, tags=['endpoint_group:DtDg-Jeti1',
                                                           'application:DtDg-AP2-Jeti'] + tags, hostname='')
    aggregator.assert_metric(metric_name, value=0.0, tags=['endpoint_group:Test-EPG',
                                                           'application:DtDg-test-AP'] + tags, hostname='')

    metric_name = 'cisco_aci.tenant.egress_pkts.multicast.rate'
    aggregator.assert_metric(metric_name, value=0.0, tags=['endpoint_group:DtDg-Pay',
                                                           'application:DtDg-AP1-EcommerceApp'] + tags, hostname='')
    aggregator.assert_metric(metric_name, value=0.0, tags=['endpoint_group:DtDg-MiscAppVMs',
                                                           'application:DtDg-AP1-EcommerceApp'] + tags, hostname='')
    aggregator.assert_metric(metric_name, value=0.0, tags=['endpoint_group:DtDg-Inv',
                                                           'application:DtDg-AP1-EcommerceApp'] + tags, hostname='')
    aggregator.assert_metric(metric_name, value=0.0, tags=['endpoint_group:DtDg-Ord',
                                                           'application:DtDg-AP1-EcommerceApp'] + tags, hostname='')
    aggregator.assert_metric(metric_name, value=0.0, tags=['endpoint_group:DtDg-Ecomm',
                                                           'application:DtDg-AP1-EcommerceApp'] + tags, hostname='')
    aggregator.assert_metric(metric_name, value=0.0, tags=['endpoint_group:DtDg-Jeti2',
                                                           'application:DtDg-AP2-Jeti'] + tags, hostname='')
    aggregator.assert_metric(metric_name, value=0.0, tags=['endpoint_group:DtDg-Jetty_Controller',
                                                           'application:DtDg-AP2-Jeti'] + tags, hostname='')
    aggregator.assert_metric(metric_name, value=0.0, tags=['endpoint_group:DtDg-Jeti1',
                                                           'application:DtDg-AP2-Jeti'] + tags, hostname='')
    aggregator.assert_metric(metric_name, value=0.0, tags=['endpoint_group:Test-EPG',
                                                           'application:DtDg-test-AP'] + tags, hostname='')

    metric_name = 'cisco_aci.tenant.ingress_bytes.drop.cum'
    aggregator.assert_metric(metric_name, value=0.0, tags=['endpoint_group:DtDg-Pay',
                                                           'application:DtDg-AP1-EcommerceApp'] + tags, hostname='')
    aggregator.assert_metric(metric_name, value=0.0, tags=['endpoint_group:DtDg-MiscAppVMs',
                                                           'application:DtDg-AP1-EcommerceApp'] + tags, hostname='')
    aggregator.assert_metric(metric_name, value=0.0, tags=['endpoint_group:DtDg-Inv',
                                                           'application:DtDg-AP1-EcommerceApp'] + tags, hostname='')
    aggregator.assert_metric(metric_name, value=0.0, tags=['endpoint_group:DtDg-Ord',
                                                           'application:DtDg-AP1-EcommerceApp'] + tags, hostname='')
    aggregator.assert_metric(metric_name, value=0.0, tags=['endpoint_group:DtDg-Ecomm',
                                                           'application:DtDg-AP1-EcommerceApp'] + tags, hostname='')
    aggregator.assert_metric(metric_name, value=0.0, tags=['endpoint_group:DtDg-Jeti2',
                                                           'application:DtDg-AP2-Jeti'] + tags, hostname='')
    aggregator.assert_metric(metric_name, value=0.0, tags=['endpoint_group:DtDg-Jetty_Controller',
                                                           'application:DtDg-AP2-Jeti'] + tags, hostname='')
    aggregator.assert_metric(metric_name, value=0.0, tags=['endpoint_group:DtDg-Jeti1',
                                                           'application:DtDg-AP2-Jeti'] + tags, hostname='')
    aggregator.assert_metric(metric_name, value=0.0, tags=['endpoint_group:Test-EPG',
                                                           'application:DtDg-test-AP'] + tags, hostname='')

    metric_name = 'cisco_aci.tenant.ingress_bytes.unicast.rate'
    aggregator.assert_metric(metric_name, value=0.0, tags=['endpoint_group:DtDg-Pay',
                                                           'application:DtDg-AP1-EcommerceApp'] + tags, hostname='')
    aggregator.assert_metric(metric_name, value=0.0, tags=['endpoint_group:DtDg-MiscAppVMs',
                                                           'application:DtDg-AP1-EcommerceApp'] + tags, hostname='')
    aggregator.assert_metric(metric_name, value=0.0, tags=['endpoint_group:DtDg-Inv',
                                                           'application:DtDg-AP1-EcommerceApp'] + tags, hostname='')
    aggregator.assert_metric(metric_name, value=0.0, tags=['endpoint_group:DtDg-Ord',
                                                           'application:DtDg-AP1-EcommerceApp'] + tags, hostname='')
    aggregator.assert_metric(metric_name, value=0.0, tags=['endpoint_group:DtDg-Ecomm',
                                                           'application:DtDg-AP1-EcommerceApp'] + tags, hostname='')
    aggregator.assert_metric(metric_name, value=0.0, tags=['endpoint_group:DtDg-Jeti2',
                                                           'application:DtDg-AP2-Jeti'] + tags, hostname='')
    aggregator.assert_metric(metric_name, value=0.0, tags=['endpoint_group:DtDg-Jetty_Controller',
                                                           'application:DtDg-AP2-Jeti'] + tags, hostname='')
    aggregator.assert_metric(metric_name, value=0.0, tags=['endpoint_group:DtDg-Jeti1',
                                                           'application:DtDg-AP2-Jeti'] + tags, hostname='')
    aggregator.assert_metric(metric_name, value=0.0, tags=['endpoint_group:Test-EPG',
                                                           'application:DtDg-test-AP'] + tags, hostname='')

    metric_name = 'cisco_aci.tenant.ingress_bytes.multicast.cum'
    aggregator.assert_metric(metric_name, value=0.0, tags=['endpoint_group:DtDg-Pay',
                                                           'application:DtDg-AP1-EcommerceApp'] + tags, hostname='')
    aggregator.assert_metric(metric_name, value=0.0, tags=['endpoint_group:DtDg-MiscAppVMs',
                                                           'application:DtDg-AP1-EcommerceApp'] + tags, hostname='')
    aggregator.assert_metric(metric_name, value=0.0, tags=['endpoint_group:DtDg-Inv',
                                                           'application:DtDg-AP1-EcommerceApp'] + tags, hostname='')
    aggregator.assert_metric(metric_name, value=0.0, tags=['endpoint_group:DtDg-Ord',
                                                           'application:DtDg-AP1-EcommerceApp'] + tags, hostname='')
    aggregator.assert_metric(metric_name, value=0.0, tags=['endpoint_group:DtDg-Ecomm',
                                                           'application:DtDg-AP1-EcommerceApp'] + tags, hostname='')
    aggregator.assert_metric(metric_name, value=0.0, tags=['endpoint_group:DtDg-Jeti2',
                                                           'application:DtDg-AP2-Jeti'] + tags, hostname='')
    aggregator.assert_metric(metric_name, value=0.0, tags=['endpoint_group:DtDg-Jetty_Controller',
                                                           'application:DtDg-AP2-Jeti'] + tags, hostname='')
    aggregator.assert_metric(metric_name, value=0.0, tags=['endpoint_group:DtDg-Jeti1',
                                                           'application:DtDg-AP2-Jeti'] + tags, hostname='')
    aggregator.assert_metric(metric_name, value=0.0, tags=['endpoint_group:Test-EPG',
                                                           'application:DtDg-test-AP'] + tags, hostname='')

    metric_name = 'cisco_aci.tenant.ingress_pkts.unicast.rate'
    aggregator.assert_metric(metric_name, value=0.0, tags=['endpoint_group:DtDg-Pay',
                                                           'application:DtDg-AP1-EcommerceApp'] + tags, hostname='')
    aggregator.assert_metric(metric_name, value=0.0, tags=['endpoint_group:DtDg-MiscAppVMs',
                                                           'application:DtDg-AP1-EcommerceApp'] + tags, hostname='')
    aggregator.assert_metric(metric_name, value=0.0, tags=['endpoint_group:DtDg-Inv',
                                                           'application:DtDg-AP1-EcommerceApp'] + tags, hostname='')
    aggregator.assert_metric(metric_name, value=0.0, tags=['endpoint_group:DtDg-Ord',
                                                           'application:DtDg-AP1-EcommerceApp'] + tags, hostname='')
    aggregator.assert_metric(metric_name, value=0.0, tags=['endpoint_group:DtDg-Ecomm',
                                                           'application:DtDg-AP1-EcommerceApp'] + tags, hostname='')
    aggregator.assert_metric(metric_name, value=0.0, tags=['endpoint_group:DtDg-Jeti2',
                                                           'application:DtDg-AP2-Jeti'] + tags, hostname='')
    aggregator.assert_metric(metric_name, value=0.0, tags=['endpoint_group:DtDg-Jetty_Controller',
                                                           'application:DtDg-AP2-Jeti'] + tags, hostname='')
    aggregator.assert_metric(metric_name, value=0.0, tags=['endpoint_group:DtDg-Jeti1',
                                                           'application:DtDg-AP2-Jeti'] + tags, hostname='')
    aggregator.assert_metric(metric_name, value=0.0, tags=['endpoint_group:Test-EPG',
                                                           'application:DtDg-test-AP'] + tags, hostname='')

    # Assert coverage for this check on this instance
    aggregator.assert_all_metrics_covered()
    def __init__(self, name, init_config, agentConfig, instances,
                 counter_list):
        AgentCheck.__init__(self, name, init_config, agentConfig, instances)
        self._countersettypes = {}
        self._counters = {}
        self._metrics = {}
        self._tags = {}

        try:
            for instance in instances:
                key = hash_mutable(instance)

                cfg_tags = instance.get('tags')
                if cfg_tags is not None:
                    if not isinstance(cfg_tags, list):
                        self.log.error("Tags must be configured as a list")
                        raise ValueError("Tags must be type list, not %s" %
                                         str(type(cfg_tags)))
                    self._tags[key] = list(cfg_tags)

                remote_machine = None
                host = instance.get('host')
                self._metrics[key] = []
                if host is not None and host != ".":
                    try:
                        remote_machine = host

                        username = instance.get('username')
                        password = instance.get('password')
                        nr = win32wnet.NETRESOURCE()
                        nr.lpRemoteName = r"\\%s\c$" % remote_machine
                        nr.dwType = 0
                        nr.lpLocalName = None
                        win32wnet.WNetAddConnection2(nr, password, username, 0)

                    except Exception as e:
                        self.log.error("Failed to make remote connection %s" %
                                       str(e))
                        return

                ## counter_data_types allows the precision with which counters are queried
                ## to be configured on a per-metric basis. In the metric instance, precision
                ## should be specified as
                ## counter_data_types:
                ## - iis.httpd_request_method.get,int
                ## - iis.net.bytes_rcvd,float
                ##
                ## the above would query the counter associated with iis.httpd_request_method.get
                ## as an integer (LONG) and iis.net.bytes_rcvd as a double
                datatypes = {}
                precisions = instance.get('counter_data_types')
                if precisions is not None:
                    if not isinstance(precisions, list):
                        self.log.warning(
                            "incorrect type for counter_data_type %s" %
                            str(precisions))
                    else:
                        for p in precisions:
                            k, v = p.split(",")
                            v = v.lower().strip()
                            if v in int_types:
                                self.log.info(
                                    "Setting datatype for %s to integer" % k)
                                datatypes[k] = DATA_TYPE_INT
                            elif v in double_types:
                                self.log.info(
                                    "Setting datatype for %s to double" % k)
                                datatypes[k] = DATA_TYPE_DOUBLE
                            else:
                                self.log.warning("Unknown data type %s" %
                                                 str(v))

                # list of the metrics.  Each entry is itself an entry,
                # which is the pdh name, datadog metric name, type, and the
                # pdh counter object

                for counterset, inst_name, counter_name, dd_name, mtype in counter_list:
                    m = getattr(self, mtype.lower())

                    precision = datatypes.get(dd_name)

                    try:
                        obj = WinPDHCounter(counterset,
                                            counter_name,
                                            self.log,
                                            inst_name,
                                            machine_name=remote_machine,
                                            precision=precision)
                    except Exception as e:
                        self.log.warning("Couldn't create counter %s\%s" %
                                         (counterset, counter_name))
                        self.log.warning("Datadog Agent will not report %s" %
                                         dd_name)
                        continue

                    entry = [inst_name, dd_name, m, obj]
                    self.log.debug("entry: %s" % str(entry))
                    self._metrics[key].append(entry)

                # get any additional metrics in the instance
                addl_metrics = instance.get('additional_metrics')
                if addl_metrics is not None:
                    for counterset, inst_name, counter_name, dd_name, mtype in addl_metrics:
                        if inst_name.lower() == "none" or len(
                                inst_name
                        ) == 0 or inst_name == "*" or inst_name.lower(
                        ) == "all":
                            inst_name = None
                        m = getattr(self, mtype.lower())

                        precision = datatypes.get(dd_name)

                        try:
                            obj = WinPDHCounter(counterset,
                                                counter_name,
                                                self.log,
                                                inst_name,
                                                machine_name=remote_machine,
                                                precision=precision)
                        except Exception as e:
                            self.log.warning("Couldn't create counter %s\%s" %
                                             (counterset, counter_name))
                            self.log.warning(
                                "Datadog Agent will not report %s" % dd_name)
                            continue

                        entry = [inst_name, dd_name, m, obj]
                        self.log.debug("additional metric entry: %s" %
                                       str(entry))
                        self._metrics[key].append(entry)

        except Exception as e:
            self.log.debug("Exception in PDH init: %s", str(e))
            raise

        if not self._metrics.get(key):
            raise AttributeError('No valid counters to collect')
Exemple #28
0
    def check(self, instance):
        sites = instance.get('sites')
        if sites is None:
            expected_sites = set()
        else:
            expected_sites = set(sites)
        # _Total should always be in the list of expected sites; we always
        # report _Total
        expected_sites.add(TOTAL_SITE)

        self.log.debug("Expected sites is {}".format(expected_sites))
        key = hash_mutable(instance)
        for inst_name, dd_name, metric_func, counter in self._metrics[key]:
            try:
                try:
                    vals = counter.get_all_values()
                except Exception as e:
                    self.log.error("Failed to get_all_values {} {}: {}".format(
                        inst_name, dd_name, e))
                    continue

                for sitename, val in iteritems(vals):
                    tags = []
                    if key in self._tags:
                        tags = list(self._tags[key])
                    tags.append(self.get_iishost(instance))

                    try:
                        if not counter.is_single_instance():
                            if sites and sitename != TOTAL_SITE and sitename not in sites:
                                continue
                            tags.append("site:{}".format(
                                self.normalize(sitename)))
                    except Exception as e:
                        self.log.error(
                            "Caught exception {} setting tags".format(e))

                    try:
                        metric_func(dd_name, val, tags)
                    except Exception as e:
                        self.log.error("Error in metric_func: {} {} {}".format(
                            dd_name, val, e))

                    if dd_name == "iis.uptime":
                        uptime = int(val)
                        status = AgentCheck.CRITICAL if uptime == 0 else AgentCheck.OK
                        self.service_check(self.SERVICE_CHECK, status, tags)
                        if sitename in expected_sites:
                            self.log.debug(
                                "Removing {!r} from expected sites".format(
                                    sitename))
                            expected_sites.remove(sitename)
                        else:
                            self.log.warning(
                                "Site {!r} not in expected_sites".format(
                                    sitename))

            except Exception as e:
                # don't give up on all of the metrics because one failed
                self.log.error(
                    "IIS Failed to get metric data for {} {}: {}".format(
                        inst_name, dd_name, e))

        for site in expected_sites:
            tags = []
            if key in self._tags:
                tags = list(self._tags[key])
            tags.append(self.get_iishost(instance))
            tags.append("site:{}".format(self.normalize(site)))
            self.service_check(self.SERVICE_CHECK, AgentCheck.CRITICAL, tags)
Exemple #29
0
    def _tenant_mapper(self, edpt):
        tags = []
        if not edpt or type(edpt) is not dict:
            return tags

        application_meta = []
        application_meta_map = self._edpt_tags_map(edpt)
        for k, v in application_meta_map.iteritems():
            application_meta.append(k + ":" + v)
        tenant_name = application_meta_map.get("tenant")
        app_name = application_meta_map.get("application")
        epg_name = application_meta_map.get("endpoint_group")

        # adding meta tags
        endpoint_meta = []
        endpoint_meta_map = self._get_epg_meta_tags_map(
            tenant_name, app_name, epg_name)
        for k, v in endpoint_meta_map.iteritems():
            endpoint_meta.append(k + ":" + v)

        # adding application tags
        endpoint_meta += application_meta

        context_hash = hash_mutable(endpoint_meta)
        eth_meta = []
        if self.tenant_tags.get(context_hash):
            eth_meta = self.tenant_tags.get(context_hash)
        else:
            try:
                # adding eth and node tags
                eth_list = self.api.get_eth_list_for_epg(
                    tenant_name, app_name, epg_name)
                for eth in eth_list:
                    eth_attrs = eth.get('fvRsCEpToPathEp',
                                        {}).get('attributes', {})
                    port = re.search('/pathep-\[(.+?)\]',
                                     eth_attrs.get('tDn', ''))
                    if not port:
                        continue
                    eth_tag = 'port:' + port.group(1)
                    if eth_tag not in eth_meta:
                        eth_meta.append(eth_tag)
                    node = re.search('/paths-(.+?)/', eth_attrs.get('tDn', ''))
                    if not node:
                        continue
                    eth_node = 'node_id:' + node.group(1)
                    if eth_node not in eth_meta:
                        eth_meta.append(eth_node)
                    # populating the map for eth-app mapping

                    tenant_fabric_key = node.group(1) + ":" + port.group(1)
                    if tenant_fabric_key not in self.tenant_farbic_mapper:
                        self.tenant_farbic_mapper[
                            tenant_fabric_key] = application_meta
                    else:
                        self.tenant_farbic_mapper[tenant_fabric_key].extend(
                            application_meta)

                    self.tenant_farbic_mapper[tenant_fabric_key] = list(
                        set(self.tenant_farbic_mapper[tenant_fabric_key]))
            except exceptions.APIConnectionException, exceptions.APIParsingException:
                # the exception will already be logged, just pass it over here
                pass
    def check(self, instance):
        # Connect to the WMI provider
        host = instance.get('host', "localhost")
        username = instance.get('username', "")
        password = instance.get('password', "")
        instance_tags = instance.get('tags', [])
        notify = instance.get('notify', [])
        event_priority = instance.get('event_priority',
                                      self._default_event_priority)
        if (event_priority.lower() != 'normal') and (event_priority.lower() !=
                                                     'low'):
            event_priority = 'normal'

        user = instance.get('user')
        ltypes = instance.get('type', [])
        source_names = instance.get('source_name', [])
        log_files = instance.get('log_file', [])
        event_ids = instance.get('event_id', [])
        message_filters = instance.get('message_filters', [])
        event_format = instance.get('event_format')

        instance_hash = hash_mutable(instance)
        instance_key = self._get_instance_key(host, self.NAMESPACE,
                                              self.EVENT_CLASS, instance_hash)

        # Store the last timestamp by instance
        if instance_key not in self.last_ts:
            # If system boot was withing 600s of dd agent start then use boottime as last_ts
            if uptime() <= 600:
                self.last_ts[instance_key] = datetime.utcnow() - timedelta(
                    seconds=uptime())
            else:
                self.last_ts[instance_key] = datetime.utcnow()
            return

        # Event properties
        event_properties = list(self.EVENT_PROPERTIES)

        if event_format is not None:
            event_properties.extend(
                list(set(self.EXTRA_EVENT_PROPERTIES) & set(event_format)))
        else:
            event_properties.extend(self.EXTRA_EVENT_PROPERTIES)

        # Event filters
        query = {}
        filters = []
        last_ts = self.last_ts[instance_key]
        query['TimeGenerated'] = ('>=', self._dt_to_wmi(last_ts))
        if user:
            query['User'] = ('=', user)
        if ltypes:
            query['Type'] = []
            for ltype in ltypes:
                query['Type'].append(('=', ltype))
        if source_names:
            query['SourceName'] = []
            for source_name in source_names:
                query['SourceName'].append(('=', source_name))
        if log_files:
            query['LogFile'] = []
            for log_file in log_files:
                query['LogFile'].append(('=', log_file))
        if event_ids:
            query['EventCode'] = []
            for event_id in event_ids:
                query['EventCode'].append(('=', event_id))
        if message_filters:
            query['NOT Message'] = []
            query['Message'] = []
            for filt in message_filters:
                if filt[0] == '-':
                    query['NOT Message'].append(('LIKE', filt[1:]))
                else:
                    query['Message'].append(('LIKE', filt))

        filters.append(query)

        wmi_sampler = self._get_wmi_sampler(
            instance_key,
            self.EVENT_CLASS,
            event_properties,
            filters=filters,
            host=host,
            namespace=self.NAMESPACE,
            username=username,
            password=password,
            and_props=['Message'],
        )

        wmi_sampler.reset_filter(new_filters=filters)
        try:
            wmi_sampler.sample()
        except TimeoutException:
            self.log.warning(
                u"[Win32EventLog] WMI query timed out."
                u" class={wmi_class} - properties={wmi_properties} -"
                u" filters={filters} - tags={tags}".format(
                    wmi_class=self.EVENT_CLASS,
                    wmi_properties=event_properties,
                    filters=filters,
                    tags=instance_tags))
        else:
            for ev in wmi_sampler:
                # for local events we dont need to specify a hostname
                hostname = None if (host == "localhost"
                                    or host == ".") else host
                log_ev = LogEvent(ev, self.log, hostname, instance_tags,
                                  notify, self._tag_event_id, event_format,
                                  event_priority)

                # Since WQL only compares on the date and NOT the time, we have to
                # do a secondary check to make sure events are after the last
                # timestamp
                if log_ev.is_after(last_ts):
                    self.event(log_ev.to_event_dict())
                else:
                    self.log.debug('Skipping event after %s. ts=%s' %
                                   (last_ts, log_ev.timestamp))

            # Update the last time checked
            self.last_ts[instance_key] = datetime.utcnow()