Esempio n. 1
0
 def test_is_default_hostname(self):
     for hostname in [
             'ip-172-31-16-235', 'domU-12-31-38-00-A4-A2',
             'domU-12-31-39-02-14-35'
     ]:
         self.assertTrue(EC2.is_default(hostname))
     for hostname in ['i-672d49da', 'localhost', 'robert.redf.org']:
         self.assertFalse(EC2.is_default(hostname))
Esempio n. 2
0
    def _build_payload(self, start_event=True):
        """
        Return an dictionary that contains all of the generic payload data.
        """
        now = time.time()
        payload = {
            'collection_timestamp': now,
            'os' : self.os,
            'python': sys.version,
            'agentVersion' : self.agentConfig['version'],
            'apiKey': self.agentConfig['api_key'],
            'events': {},
            'metrics': [],
            'resources': {},
            'internalHostname' : get_hostname(self.agentConfig),
            'uuid' : get_uuid(),
            'host-tags': {},
        }

        # Include system stats on first postback
        if start_event and self._is_first_run():
            payload['systemStats'] = self.agentConfig.get('system_stats', {})
            # Also post an event in the newsfeed
            payload['events']['System'] = [{'api_key': self.agentConfig['api_key'],
                                 'host': payload['internalHostname'],
                                 'timestamp': now,
                                 'event_type':'Agent Startup',
                                 'msg_text': 'Version %s' % get_version()
                                 }]

        # Periodically send the host metadata.
        if self._is_first_run() or self._should_send_metadata():
            payload['systemStats'] = get_system_stats()
            payload['meta'] = self._get_metadata()
            self.metadata_cache = payload['meta']
            # Add static tags from the configuration file
            if self.agentConfig['tags'] is not None:
                payload['host-tags']['system'] = [unicode(tag.strip()) for tag in self.agentConfig['tags'].split(",")]

            EC2_tags = EC2.get_tags()
            if EC2_tags is not None:
                payload['host-tags'][EC2.SOURCE_TYPE_NAME] = EC2.get_tags()

            # Log the metadata on the first run
            if self._is_first_run():
                log.info(u"Hostnames: %s, tags: %s" % (repr(self.metadata_cache), payload['host-tags']))

        return payload
Esempio n. 3
0
    def _get_hostname_metadata(self):
        """
        Returns a dictionnary that contains hostname metadata.
        """
        metadata = EC2.get_metadata(self.agentConfig)
        if metadata.get('hostname'):
            metadata['ec2-hostname'] = metadata.get('hostname')
            del metadata['hostname']

        if self.agentConfig.get('hostname'):
            metadata['agent-hostname'] = self.agentConfig.get('hostname')
        else:
            try:
                metadata["socket-hostname"] = socket.gethostname()
            except Exception:
                pass
        try:
            metadata["socket-fqdn"] = socket.getfqdn()
        except Exception:
            pass

        metadata["hostname"] = self.hostname
        metadata["timezones"] = sanitize_tzname(time.tzname)

        # Add cloud provider aliases
        host_aliases = GCE.get_host_aliases(self.agentConfig)
        if host_aliases:
            metadata['host_aliases'] = host_aliases

        return metadata
Esempio n. 4
0
    def _get_hostname_metadata(self):

        metadata = EC2.get_metadata(self.agentConfig)
        if metadata.get('hostname'):
            metadata['ec2-hostname'] = metadata.get('hostname')
            del metadata['hostname']

        if self.agentConfig.get('hostname'):
            metadata['agent-hostname'] = self.agentConfig.get('hostname')
        else:
            try:
                metadata["socket-hostname"] = socket.gethostname()
            except Exception:
                pass
        try:
            metadata["socket-fqdn"] = socket.getfqdn()
        except Exception:
            pass

        metadata["hostname"] = self.hostname
        metadata["timezones"] = sanitize_tzname(time.tzname)

        host_aliases = GCE.get_host_aliases(self.agentConfig)
        if host_aliases:
            metadata['host_aliases'] = host_aliases

        return metadata
Esempio n. 5
0
 def _set_agent_config_hostname(self, agentConfig):
     if agentConfig.get('hostname') is None and agentConfig.get('use_ec2_instance_id'):
         instanceId = EC2.get_instance_id(agentConfig)
         if instanceId is not None:
             log.info("Running on EC2, instanceId: %s" % instanceId)
             agentConfig['hostname'] = instanceId
         else:
             log.info('Not running on EC2, using hostname to identify this server')
     return agentConfig
Esempio n. 6
0
    def _build_payload(self, start_event=True):
        """
        Return an dictionary that contains all of the generic payload data.
        """
        now = time.time()
        payload = {
            "collection_timestamp": now,
            "os": self.os,
            "python": sys.version,
            "agentVersion": self.agentConfig["version"],
            "apiKey": self.agentConfig["api_key"],
            "events": {},
            "metrics": [],
            "resources": {},
            "internalHostname": get_hostname(self.agentConfig),
            "uuid": get_uuid(),
            "host-tags": {},
        }

        # Include system stats on first postback
        if start_event and self._is_first_run():
            payload["systemStats"] = self.agentConfig.get("system_stats", {})
            # Also post an event in the newsfeed
            payload["events"]["System"] = [
                {
                    "api_key": self.agentConfig["api_key"],
                    "host": payload["internalHostname"],
                    "timestamp": now,
                    "event_type": "Agent Startup",
                    "msg_text": "Version %s" % get_version(),
                }
            ]

        # Periodically send the host metadata.
        if self._is_first_run() or self._should_send_metadata():
            payload["systemStats"] = get_system_stats()
            payload["meta"] = self._get_metadata()
            self.metadata_cache = payload["meta"]
            # Add static tags from the configuration file
            host_tags = []
            if self.agentConfig["tags"] is not None:
                host_tags.extend([unicode(tag.strip()) for tag in self.agentConfig["tags"].split(",")])

            if self.agentConfig["collect_ec2_tags"]:
                host_tags.extend(EC2.get_tags())

            if host_tags:
                payload["host-tags"]["system"] = host_tags

            # Log the metadata on the first run
            if self._is_first_run():
                log.info(u"Hostnames: %s, tags: %s" % (repr(self.metadata_cache), payload["host-tags"]))

        return payload
Esempio n. 7
0
 def test_metadata(self):
     # Test gathering metadata from ec2
     start = time.time()
     d = EC2.get_metadata()
     end = time.time()
     assert type(d) == types.DictType
     # Either we're on ec2 or we're not (at least 7 attributes expected)
     assert len(d) == 0 or len(d) >= 7, d
     if "instance-id" in d:
         assert d["instance-id"].startswith("i-"), d
         assert d["hostname"].startswith("i-") or d["hostname"].startswith("domU-"), d
     assert end - start <= 1.1, "It took %s seconds to get ec2 metadata" % (end-start)
Esempio n. 8
0
 def _set_agent_config_hostname(self, agentConfig):
     # Try to fetch instance Id from EC2 if not hostname has been set
     # in the config file.
     # DEPRECATED
     if agentConfig.get('hostname') is None and agentConfig.get('use_ec2_instance_id'):
         instanceId = EC2.get_instance_id(agentConfig)
         if instanceId is not None:
             log.info("Running on EC2, instanceId: %s" % instanceId)
             agentConfig['hostname'] = instanceId
         else:
             log.info('Not running on EC2, using hostname to identify this server')
     return agentConfig
Esempio n. 9
0
 def test_metadata(self):
     # Reset metadata just to be sure
     EC2.metadata = {}
     # Test gathering metadata from ec2
     start = time.time()
     d = EC2.get_metadata({'collect_instance_metadata': True})
     end = time.time()
     assert type(d) == types.DictType
     # Either we're on ec2 or we're not (at least 7 attributes expected)
     assert len(d) == 0 or len(d) >= 7, d
     if "instance-id" in d:
         assert d["instance-id"].startswith("i-"), d
     assert end - start <= 1.1, "It took %s seconds to get ec2 metadata" % (end-start)
Esempio n. 10
0
 def test_metadata(self):
     # Reset metadata just to be sure
     EC2.metadata = {}
     # Test gathering metadata from ec2
     start = time.time()
     d = EC2.get_metadata({'collect_instance_metadata': True})
     end = time.time()
     self.assertTrue(isinstance(d, types.DictType))
     # Either we're on ec2 or we're not (at least 7 attributes expected)
     assert len(d) == 0 or len(d) >= 7, d
     if "instance-id" in d:
         assert d["instance-id"].startswith("i-"), d
     assert end - start <= 1.15, "It took %s seconds to get ec2 metadata" % (end-start)
Esempio n. 11
0
 def test_metadata(self):
     # Skip this step on travis
     if os.environ.get("TRAVIS", False):
         return
     # Test gathering metadata from ec2
     start = time.time()
     d = EC2.get_metadata({"collect_instance_metadata": True})
     end = time.time()
     assert type(d) == types.DictType
     # Either we're on ec2 or we're not (at least 7 attributes expected)
     assert len(d) == 0 or len(d) >= 7, d
     if "instance-id" in d:
         assert d["instance-id"].startswith("i-"), d
         assert d["hostname"].startswith("i-") or d["hostname"].startswith("domU-"), d
     assert end - start <= 1.1, "It took %s seconds to get ec2 metadata" % (end - start)
Esempio n. 12
0
 def test_metadata(self):
     # Skip this step on travis
     if os.environ.get('TRAVIS', False): return
     # Test gathering metadata from ec2
     start = time.time()
     d = EC2.get_metadata()
     end = time.time()
     assert type(d) == types.DictType
     # Either we're on ec2 or we're not (at least 7 attributes expected)
     assert len(d) == 0 or len(d) >= 7, d
     if "instance-id" in d:
         assert d["instance-id"].startswith("i-"), d
         assert d["hostname"].startswith("i-") or d["hostname"].startswith(
             "domU-"), d
     assert end - start <= 1.1, "It took %s seconds to get ec2 metadata" % (
         end - start)
Esempio n. 13
0
    def _get_metadata(self):
        metadata = EC2.get_metadata()
        if metadata.get('hostname'):
            metadata['ec2-hostname'] = metadata.get('hostname')
            del metadata['hostname']

        if self.agentConfig.get('hostname'):
            metadata['agent-hostname'] = self.agentConfig.get('hostname')
        else:
            try:
                metadata["socket-hostname"] = socket.gethostname()
            except Exception:
                pass
        try:
            metadata["socket-fqdn"] = socket.getfqdn()
        except Exception:
            pass

        metadata["hostname"] = get_hostname()

        return metadata
Esempio n. 14
0
    def _get_hostname_metadata(self):
        """
        Returns a dictionnary that contains hostname metadata.
        """
        metadata = EC2.get_metadata(self.agentConfig)
        if metadata.get('hostname'):
            metadata['ec2-hostname'] = metadata.get('hostname')
            del metadata['hostname']

        if self.agentConfig.get('hostname'):
            metadata['agent-hostname'] = self.agentConfig.get('hostname')
        else:
            try:
                metadata["socket-hostname"] = socket.gethostname()
            except Exception:
                pass
        try:
            metadata["socket-fqdn"] = socket.getfqdn()
        except Exception:
            pass

        metadata["hostname"] = get_hostname()

        return metadata
Esempio n. 15
0
    def _get_hostname_metadata(self):
        """
        Returns a dictionnary that contains hostname metadata.
        """
        metadata = EC2.get_metadata(self.agentConfig)
        if metadata.get('hostname'):
            metadata['ec2-hostname'] = metadata.get('hostname')
            del metadata['hostname']

        if self.agentConfig.get('hostname'):
            metadata['agent-hostname'] = self.agentConfig.get('hostname')
        else:
            try:
                metadata["socket-hostname"] = socket.gethostname()
            except Exception:
                pass
        try:
            metadata["socket-fqdn"] = socket.getfqdn()
        except Exception:
            pass

        metadata["hostname"] = get_hostname()

        return metadata
Esempio n. 16
0
    def _populate_payload_metadata(self, payload, check_statuses, start_event=True):
        now = time.time()

        if start_event and self._is_first_run():
            payload['systemStats'] = self.agentConfig.get('system_stats', {})
            payload['events']['System'] = [{
                'api_key': self.agentConfig['api_key'],
                'host': payload['internalHostname'],
                'timestamp': now,
                'event_type': 'Agent Startup',
                'msg_text': 'Version %s' % get_version()
            }]

        if self._should_send_additional_data('host_metadata'):
            payload['gohai'] = get_gohai_data()
            payload['systemStats'] = get_system_stats()
            payload['meta'] = self._get_hostname_metadata()
            log.info('GOHAI data: {0}'.format(payload['gohai']))

            self.hostname_metadata_cache = payload['meta']
            host_tags = []
            if self.agentConfig['tags'] is not None:
                host_tags.extend([tag.strip()
                                  for tag in self.agentConfig['tags'].split(",")])

            if self.agentConfig['collect_ec2_tags']:
                host_tags.extend(EC2.get_tags(self.agentConfig))

            if host_tags:
                payload['host-tags']['system'] = host_tags

            if self.agentConfig['create_dd_check_tags']:
                app_tags_list = [DD_CHECK_TAG.format(c.name) for c in self.initialized_checks_d]
                app_tags_list.extend([DD_CHECK_TAG.format(cname) for cname
                                      in JMXFiles.get_jmx_appnames()])

                if 'system' not in payload['host-tags']:
                    payload['host-tags']['system'] = []

                payload['host-tags']['system'].extend(app_tags_list)

            GCE_tags = GCE.get_tags(self.agentConfig)
            if GCE_tags is not None:
                payload['host-tags'][GCE.SOURCE_TYPE_NAME] = GCE_tags

            if self._is_first_run():
                log.info("Hostnames: %s, tags: %s" %
                         (repr(self.hostname_metadata_cache), payload['host-tags']))

        external_host_tags = []
        if self._should_send_additional_data('external_host_tags'):
            for check in self.initialized_checks_d:
                try:
                    getter = getattr(check, 'get_external_host_tags')
                    check_tags = getter()
                    external_host_tags.extend(check_tags)
                except AttributeError:
                    pass

        if external_host_tags:
            payload['external_host_tags'] = external_host_tags

        if self._should_send_additional_data('agent_checks'):
            agent_checks = []
            for check in check_statuses:
                if check.instance_statuses is not None:
                    for i, instance_status in enumerate(check.instance_statuses):
                        agent_checks.append(
                            (
                                check.name, check.source_type_name,
                                instance_status.instance_id,
                                instance_status.status,

                                instance_status.error or instance_status.warnings or "",
                                check.service_metadata[i]
                            )
                        )
                else:
                    agent_checks.append(
                        (
                            check.name, check.source_type_name,
                            "initialization",
                            check.status, repr(check.init_failed_error)
                        )
                    )
            payload['agent_checks'] = agent_checks
            payload['meta'] = self.hostname_metadata_cache
Esempio n. 17
0
    def _populate_payload_metadata(self, payload, check_statuses, start_event=True):
        """
        Periodically populate the payload with metadata related to the system, host, and/or checks.
        """
        now = time.time()

        # Include system stats on first postback
        if start_event and self._is_first_run():
            payload['systemStats'] = self.agentConfig.get('system_stats', {})
            # Also post an event in the newsfeed
            payload['events']['System'] = [{
                'api_key': self.agentConfig['api_key'],
                'host': payload['internalHostname'],
                'timestamp': now,
                'event_type':'Agent Startup',
                'msg_text': 'Version %s' % get_version()
            }]

        # Periodically send the host metadata.
        if self._should_send_additional_data('host_metadata'):
            # gather metadata with gohai
            gohai_metadata = self._run_gohai_metadata()
            if gohai_metadata:
                payload['gohai'] = gohai_metadata

            payload['systemStats'] = get_system_stats(
                proc_path=self.agentConfig.get('procfs_path', '/proc').rstrip('/')
            )
            payload['meta'] = self._get_hostname_metadata()

            self.hostname_metadata_cache = payload['meta']
            # Add static tags from the configuration file
            host_tags = []
            if self.agentConfig['tags'] is not None:
                host_tags.extend([unicode(tag.strip())
                                 for tag in self.agentConfig['tags'].split(",")])

            if self.agentConfig['collect_ec2_tags']:
                host_tags.extend(EC2.get_tags(self.agentConfig))

            if host_tags:
                payload['host-tags']['system'] = host_tags

            # If required by the user, let's create the dd_check:xxx host tags
            if self.agentConfig['create_dd_check_tags']:
                app_tags_list = [DD_CHECK_TAG.format(c.name) for c in self.initialized_checks_d]
                app_tags_list.extend([DD_CHECK_TAG.format(cname) for cname
                                      in JMXFiles.get_jmx_appnames()])

                if 'system' not in payload['host-tags']:
                    payload['host-tags']['system'] = []

                payload['host-tags']['system'].extend(app_tags_list)

            GCE_tags = GCE.get_tags(self.agentConfig)
            if GCE_tags is not None:
                payload['host-tags'][GCE.SOURCE_TYPE_NAME] = GCE_tags

            # Log the metadata on the first run
            if self._is_first_run():
                log.info("Hostnames: %s, tags: %s" %
                         (repr(self.hostname_metadata_cache), payload['host-tags']))

        # Periodically send extra hosts metadata (vsphere)
        # Metadata of hosts that are not the host where the agent runs, not all the checks use
        # that
        external_host_tags = []
        if self._should_send_additional_data('external_host_tags'):
            for check in self.initialized_checks_d:
                try:
                    getter = getattr(check, 'get_external_host_tags')
                    check_tags = getter()
                    external_host_tags.extend(check_tags)
                except AttributeError:
                    pass

        if external_host_tags:
            payload['external_host_tags'] = external_host_tags

        # Periodically send agent_checks metadata
        if self._should_send_additional_data('agent_checks'):
            # Add agent checks statuses and error/warning messages
            agent_checks = []
            for check in check_statuses:
                if check.instance_statuses is not None:
                    for i, instance_status in enumerate(check.instance_statuses):
                        agent_checks.append(
                            (
                                check.name, check.source_type_name,
                                instance_status.instance_id,
                                instance_status.status,
                                # put error message or list of warning messages in the same field
                                # it will be handled by the UI
                                instance_status.error or instance_status.warnings or "",
                                check.service_metadata[i]
                            )
                        )
                else:
                    agent_checks.append(
                        (
                            check.name, check.source_type_name,
                            "initialization",
                            check.status, repr(check.init_failed_error)
                        )
                    )
            payload['agent_checks'] = agent_checks
            payload['meta'] = self.hostname_metadata_cache  # add hostname metadata
Esempio n. 18
0
    def _build_payload(self, start_event=True):
        """
        Return an dictionary that contains all of the generic payload data.
        """
        now = time.time()
        payload = {
            'collection_timestamp': now,
            'os' : self.os,
            'python': sys.version,
            'agentVersion' : self.agentConfig['version'],
            'apiKey': self.agentConfig['api_key'],
            'events': {},
            'metrics': [],
            'service_checks': [],
            'resources': {},
            'internalHostname' : self.hostname,
            'uuid' : get_uuid(),
            'host-tags': {},
            'external_host_tags': {}
        }

        # Include system stats on first postback
        if start_event and self._is_first_run():
            payload['systemStats'] = self.agentConfig.get('system_stats', {})
            # Also post an event in the newsfeed
            payload['events']['System'] = [{'api_key': self.agentConfig['api_key'],
                                 'host': payload['internalHostname'],
                                 'timestamp': now,
                                 'event_type':'Agent Startup',
                                 'msg_text': 'Version %s' % get_version()
                                 }]

        # Periodically send the host metadata.
        if self._should_send_additional_data('metadata'):
            # gather metadata with gohai
            try:
                if get_os() != 'windows':
                    command = "gohai"
                else:
                    command = "gohai\gohai.exe"
                gohai_metadata = subprocess.Popen(
                    [command], stdout=subprocess.PIPE
                ).communicate()[0]
                payload['gohai'] = gohai_metadata
            except OSError as e:
                if e.errno == 2:  # file not found, expected when install from source
                    log.info("gohai file not found")
                else:
                    raise e
            except Exception as e:
                log.warning("gohai command failed with error %s" % str(e))

            payload['systemStats'] = get_system_stats()
            payload['meta'] = self._get_metadata()

            self.metadata_cache = payload['meta']
            # Add static tags from the configuration file
            host_tags = []
            if self.agentConfig['tags'] is not None:
                host_tags.extend([unicode(tag.strip()) for tag in self.agentConfig['tags'].split(",")])

            if self.agentConfig['collect_ec2_tags']:
                host_tags.extend(EC2.get_tags(self.agentConfig))

            if host_tags:
                payload['host-tags']['system'] = host_tags

            GCE_tags = GCE.get_tags(self.agentConfig)
            if GCE_tags is not None:
                payload['host-tags'][GCE.SOURCE_TYPE_NAME] = GCE_tags

            # Log the metadata on the first run
            if self._is_first_run():
                log.info("Hostnames: %s, tags: %s" % (repr(self.metadata_cache), payload['host-tags']))

        # Periodically send extra hosts metadata (vsphere)
        # Metadata of hosts that are not the host where the agent runs, not all the checks use
        # that
        external_host_tags = []
        if self._should_send_additional_data('external_host_tags'):
            for check in self.initialized_checks_d:
                try:
                    getter = getattr(check, 'get_external_host_tags')
                    check_tags = getter()
                    external_host_tags.extend(check_tags)
                except AttributeError:
                    pass

        if external_host_tags:
            payload['external_host_tags'] = external_host_tags

        return payload
Esempio n. 19
0
    def _build_payload(self, start_event=True):
        """
        Return an dictionary that contains all of the generic payload data.
        """
        now = time.time()
        payload = {
            'collection_timestamp': now,
            'os' : self.os,
            'python': sys.version,
            'agentVersion' : self.agentConfig['version'],
            'apiKey': self.agentConfig['api_key'],
            'events': {},
            'metrics': [],
            'service_checks': [],
            'resources': {},
            'internalHostname' : get_hostname(self.agentConfig),
            'uuid' : get_uuid(),
            'host-tags': {},
        }

        # Include system stats on first postback
        if start_event and self._is_first_run():
            payload['systemStats'] = self.agentConfig.get('system_stats', {})
            # Also post an event in the newsfeed
            payload['events']['System'] = [{'api_key': self.agentConfig['api_key'],
                                 'host': payload['internalHostname'],
                                 'timestamp': now,
                                 'event_type':'Agent Startup',
                                 'msg_text': 'Version %s' % get_version()
                                 }]

        # Periodically send the host metadata.
        if self._should_send_additional_data('metadata'):
            # gather metadata with gohai
            try:
                gohai_metadata = subprocess.Popen(
                    ["gohai"], stdout=subprocess.PIPE, close_fds=True
                ).communicate()[0]
                payload['gohai'] = gohai_metadata
            except Exception as e:
                log.warning("gohai command failed with error %s" % str(e))

            payload['systemStats'] = get_system_stats()
            payload['meta'] = self._get_metadata()

            self.metadata_cache = payload['meta']
            # Add static tags from the configuration file
            host_tags = []
            if self.agentConfig['tags'] is not None:
                host_tags.extend([unicode(tag.strip()) for tag in self.agentConfig['tags'].split(",")])

            if self.agentConfig['collect_ec2_tags']:
                host_tags.extend(EC2.get_tags(self.agentConfig))

            if host_tags:
                payload['host-tags']['system'] = host_tags

            GCE_tags = GCE.get_tags(self.agentConfig)
            if GCE_tags is not None:
                payload['host-tags'][GCE.SOURCE_TYPE_NAME] = GCE_tags

            # Log the metadata on the first run
            if self._is_first_run():
                log.info("Hostnames: %s, tags: %s" % (repr(self.metadata_cache), payload['host-tags']))

        return payload
Esempio n. 20
0
    def _populate_payload_metadata(self, payload, check_statuses, start_event=True):
        """
        Periodically populate the payload with metadata related to the system, host, and/or checks.
        """
        now = time.time()

        # Include system stats on first postback
        if start_event and self._is_first_run():
            payload["systemStats"] = self.agentConfig.get("system_stats", {})
            # Also post an event in the newsfeed
            payload["events"]["System"] = [
                {
                    "api_key": self.agentConfig["api_key"],
                    "host": payload["internalHostname"],
                    "timestamp": now,
                    "event_type": "Agent Startup",
                    "msg_text": "Version %s" % get_version(),
                }
            ]

        # Periodically send the host metadata.
        if self._should_send_additional_data("host_metadata"):
            # gather metadata with gohai
            try:
                if not Platform.is_windows():
                    command = "gohai"
                else:
                    command = "gohai\gohai.exe"
                gohai_metadata, gohai_err, _ = get_subprocess_output([command], log)
                payload["gohai"] = gohai_metadata
                if gohai_err:
                    log.warning("GOHAI LOG | {0}".format(gohai_err))
            except OSError as e:
                if e.errno == 2:  # file not found, expected when install from source
                    log.info("gohai file not found")
                else:
                    raise e
            except Exception as e:
                log.warning("gohai command failed with error %s" % str(e))

            payload["systemStats"] = get_system_stats()
            payload["meta"] = self._get_hostname_metadata()

            self.hostname_metadata_cache = payload["meta"]
            # Add static tags from the configuration file
            host_tags = []
            if self.agentConfig["tags"] is not None:
                host_tags.extend([unicode(tag.strip()) for tag in self.agentConfig["tags"].split(",")])

            if self.agentConfig["collect_ec2_tags"]:
                host_tags.extend(EC2.get_tags(self.agentConfig))

            if host_tags:
                payload["host-tags"]["system"] = host_tags

            # If required by the user, let's create the dd_check:xxx host tags
            if self.agentConfig["create_dd_check_tags"]:
                app_tags_list = [DD_CHECK_TAG.format(c.name) for c in self.initialized_checks_d]
                app_tags_list.extend([DD_CHECK_TAG.format(cname) for cname in JMXFiles.get_jmx_appnames()])

                if "system" not in payload["host-tags"]:
                    payload["host-tags"]["system"] = []

                payload["host-tags"]["system"].extend(app_tags_list)

            GCE_tags = GCE.get_tags(self.agentConfig)
            if GCE_tags is not None:
                payload["host-tags"][GCE.SOURCE_TYPE_NAME] = GCE_tags

            # Log the metadata on the first run
            if self._is_first_run():
                log.info("Hostnames: %s, tags: %s" % (repr(self.hostname_metadata_cache), payload["host-tags"]))

        # Periodically send extra hosts metadata (vsphere)
        # Metadata of hosts that are not the host where the agent runs, not all the checks use
        # that
        external_host_tags = []
        if self._should_send_additional_data("external_host_tags"):
            for check in self.initialized_checks_d:
                try:
                    getter = getattr(check, "get_external_host_tags")
                    check_tags = getter()
                    external_host_tags.extend(check_tags)
                except AttributeError:
                    pass

        if external_host_tags:
            payload["external_host_tags"] = external_host_tags

        # Periodically send agent_checks metadata
        if self._should_send_additional_data("agent_checks"):
            # Add agent checks statuses and error/warning messages
            agent_checks = []
            for check in check_statuses:
                if check.instance_statuses is not None:
                    for i, instance_status in enumerate(check.instance_statuses):
                        agent_checks.append(
                            (
                                check.name,
                                check.source_type_name,
                                instance_status.instance_id,
                                instance_status.status,
                                # put error message or list of warning messages in the same field
                                # it will be handled by the UI
                                instance_status.error or instance_status.warnings or "",
                                check.service_metadata[i],
                            )
                        )
                else:
                    agent_checks.append(
                        (
                            check.name,
                            check.source_type_name,
                            "initialization",
                            check.status,
                            repr(check.init_failed_error),
                        )
                    )
            payload["agent_checks"] = agent_checks
            payload["meta"] = self.hostname_metadata_cache  # add hostname metadata
Esempio n. 21
0
 def test_is_default_hostname(self):
     for hostname in ['ip-172-31-16-235', 'domU-12-31-38-00-A4-A2', 'domU-12-31-39-02-14-35']:
         self.assertTrue(EC2.is_default(hostname))
     for hostname in ['i-672d49da', 'localhost', 'robert.redf.org']:
         self.assertFalse(EC2.is_default(hostname))
Esempio n. 22
0
    def _build_payload(self, start_event=True):
        """
        Return an dictionary that contains all of the generic payload data.
        """
        now = time.time()
        payload = {
            'collection_timestamp': now,
            'os': self.os,
            'python': sys.version,
            'agentVersion': self.agentConfig['version'],
            'apiKey': self.agentConfig['api_key'],
            'events': {},
            'metrics': [],
            'service_checks': [],
            'resources': {},
            'internalHostname': self.hostname,
            'uuid': get_uuid(),
            'host-tags': {},
            'external_host_tags': {}
        }

        # Include system stats on first postback
        if start_event and self._is_first_run():
            payload['systemStats'] = self.agentConfig.get('system_stats', {})
            # Also post an event in the newsfeed
            payload['events']['System'] = [{
                'api_key':
                self.agentConfig['api_key'],
                'host':
                payload['internalHostname'],
                'timestamp':
                now,
                'event_type':
                'Agent Startup',
                'msg_text':
                'Version %s' % get_version()
            }]

        # Periodically send the host metadata.
        if self._should_send_additional_data('metadata'):
            # gather metadata with gohai
            try:
                if get_os() != 'windows':
                    command = "gohai"
                else:
                    command = "gohai\gohai.exe"
                gohai_metadata = subprocess.Popen(
                    [command], stdout=subprocess.PIPE).communicate()[0]
                payload['gohai'] = gohai_metadata
            except OSError as e:
                if e.errno == 2:  # file not found, expected when install from source
                    log.info("gohai file not found")
                else:
                    raise e
            except Exception as e:
                log.warning("gohai command failed with error %s" % str(e))

            payload['systemStats'] = get_system_stats()
            payload['meta'] = self._get_metadata()

            self.metadata_cache = payload['meta']
            # Add static tags from the configuration file
            host_tags = []
            if self.agentConfig['tags'] is not None:
                host_tags.extend([
                    unicode(tag.strip())
                    for tag in self.agentConfig['tags'].split(",")
                ])

            if self.agentConfig['collect_ec2_tags']:
                host_tags.extend(EC2.get_tags(self.agentConfig))

            if host_tags:
                payload['host-tags']['system'] = host_tags

            GCE_tags = GCE.get_tags(self.agentConfig)
            if GCE_tags is not None:
                payload['host-tags'][GCE.SOURCE_TYPE_NAME] = GCE_tags

            # Log the metadata on the first run
            if self._is_first_run():
                log.info("Hostnames: %s, tags: %s" %
                         (repr(self.metadata_cache), payload['host-tags']))

        # Periodically send extra hosts metadata (vsphere)
        # Metadata of hosts that are not the host where the agent runs, not all the checks use
        # that
        external_host_tags = []
        if self._should_send_additional_data('external_host_tags'):
            for check in self.initialized_checks_d:
                try:
                    getter = getattr(check, 'get_external_host_tags')
                    check_tags = getter()
                    external_host_tags.extend(check_tags)
                except AttributeError:
                    pass

        if external_host_tags:
            payload['external_host_tags'] = external_host_tags

        return payload