def submit_http(self, url, data, headers):
no_proxy = {
# See https://github.com/kennethreitz/requests/issues/879
# and https://github.com/DataDog/dd-agent/issues/1112
'no': 'pass',
}
headers["DD-Dogstatsd-Version"] = get_version()
log.debug("Posting payload to %s" % url)
try:
start_time = time()
r = requests.post(url, data=data, timeout=5,
headers=headers, proxies=no_proxy)
r.raise_for_status()
if r.status_code >= 200 and r.status_code < 205:
log.debug("Payload accepted")
status = r.status_code
duration = round((time() - start_time) * 1000.0, 4)
log.debug("%s POST %s (%sms)" % (
status, url, duration))
except Exception:
log.exception("Unable to post payload.")
try:
log.error("Received status code: {0}".format(r.status_code))
except Exception:
pass
def __init__(self, parent=None):
QMenu.__init__(self, parent)
self.options = {}
system_tray_menu = [
(self.START, lambda: agent_manager("start")),
(self.STOP, lambda: agent_manager("stop")),
(self.RESTART, lambda: agent_manager("restart")),
]
# First the version
self.addAction(self.ABOUT.format(get_version())).setEnabled(False)
self.addSeparator()
for name, action in system_tray_menu:
self.add_option(name, action)
# enable or disable mac login
if Platform.is_mac():
self.add_option(self.MAC_LOGIN.format(self.enable_or_disable_mac()),
lambda: self.enable_or_disable_login())
elif Platform.is_windows():
self.add_option(self.FLARE, lambda: thread.start_new_thread(windows_flare, ()))
# And finally the exit
self.add_option(self.EXIT, lambda: sys.exit(0))
self.connect(self, SIGNAL("aboutToShow()"), lambda: self.update_options())
def getVersion():
try:
from config import get_version
except ImportError:
import sys
sys.path.append("../..")
from config import get_version
return get_version()
def _title_lines(cls):
name_line = "%s (v %s)" % (cls.NAME, config.get_version())
lines = [
"=" * len(name_line),
"%s" % name_line,
"=" * len(name_line),
"",
]
return lines
def post_headers(agentConfig, payload):
return {
'User-Agent': 'Server Density Agent/%s' % agentConfig['version'],
'Content-Type': 'application/json',
#'Content-Encoding': 'deflate',
'Accept': 'text/html, */*',
'Content-MD5': md5(payload).hexdigest(),
'SD-Collector-Version': get_version()
}
def _title_lines(self, title=None):
name_line = title if title else "%s (v %s)" % (self.NAME, config.get_version())
lines = [
"=" * len(name_line),
"%s" % name_line,
"=" * len(name_line),
"",
]
return lines
def post_headers(agentConfig, payload):
return {
"User-Agent": "Datadog Agent/%s" % agentConfig["version"],
"Content-Type": "application/json",
"Content-Encoding": "deflate",
"Accept": "text/html, */*",
"Content-MD5": md5(payload).hexdigest(),
"DD-Collector-Version": get_version(),
}
def _build_payload(self, start_event=True):
"""
Return an dictionary that contains all of the generic payload data.
"""
now = time.time()
payload = {
'collection_timestamp': now,
'os' : self.os,
'python': sys.version,
'agentVersion' : self.agentConfig['version'],
'apiKey': self.agentConfig['api_key'],
'events': {},
'metrics': [],
'service_checks': [],
'resources': {},
'internalHostname' : get_hostname(self.agentConfig),
'uuid' : get_uuid(),
'host-tags': {},
}
# Include system stats on first postback
if start_event and self._is_first_run():
payload['systemStats'] = self.agentConfig.get('system_stats', {})
# Also post an event in the newsfeed
payload['events']['System'] = [{'api_key': self.agentConfig['api_key'],
'host': payload['internalHostname'],
'timestamp': now,
'event_type':'Agent Startup',
'msg_text': 'Version %s' % get_version()
}]
# Periodically send the host metadata.
if self._is_first_run() or self._should_send_metadata():
payload['systemStats'] = get_system_stats()
payload['meta'] = self._get_metadata()
self.metadata_cache = payload['meta']
# Add static tags from the configuration file
host_tags = []
if self.agentConfig['tags'] is not None:
host_tags.extend([unicode(tag.strip()) for tag in self.agentConfig['tags'].split(",")])
if self.agentConfig['collect_ec2_tags']:
host_tags.extend(EC2.get_tags())
if host_tags:
payload['host-tags']['system'] = host_tags
GCE_tags = GCE.get_tags()
if GCE_tags is not None:
payload['host-tags'][GCE.SOURCE_TYPE_NAME] = GCE_tags
# Log the metadata on the first run
if self._is_first_run():
log.info("Hostnames: %s, tags: %s" % (repr(self.metadata_cache), payload['host-tags']))
return payload
def serialize_metrics(metrics, hostname, agent_key):
try:
metrics.append(add_serialization_status_metric("success", hostname))
serialized = json.dumps(
{
"metrics": metrics,
"agentKey": agent_key,
"sdAgentVersion": 2.0,
"collection_timestamp": time(),
"agentVersion": get_version()
})
except UnicodeDecodeError as e:
log.exception("Unable to serialize payload. Trying to replace bad characters. %s", e)
metrics.append(add_serialization_status_metric("failure", hostname))
try:
log.error(metrics)
serialized = json.dumps(
{
"metrics": unicode_metrics(metrics),
"agentKey": agent_key,
"sdAgentVersion": 2.0,
"collection_timestamp": time(),
"agentVersion": get_version()
})
except Exception as e:
log.exception("Unable to serialize payload. Giving up. %s", e)
serialized = json.dumps(
{
"metrics": [add_serialization_status_metric("permanent_failure", hostname)],
"agentKey": agent_key,
"sdAgentVersion": 2.0,
"collection_timestamp": time(),
"agentVersion": get_version()
})
if len(serialized) > COMPRESS_THRESHOLD:
headers = {'Content-Type': 'application/json',
#'Content-Encoding': 'deflate'
}
#serialized = zlib.compress(serialized)
else:
headers = {'Content-Type': 'application/json'}
return serialized, headers
def _build_payload(self, start_event=True):
"""
Return an dictionary that contains all of the generic payload data.
"""
now = time.time()
payload = {
"collection_timestamp": now,
"os": self.os,
"python": sys.version,
"agentVersion": self.agentConfig["version"],
"apiKey": self.agentConfig["api_key"],
"events": {},
"metrics": [],
"resources": {},
"internalHostname": get_hostname(self.agentConfig),
"uuid": get_uuid(),
"host-tags": {},
}
# Include system stats on first postback
if start_event and self._is_first_run():
payload["systemStats"] = self.agentConfig.get("system_stats", {})
# Also post an event in the newsfeed
payload["events"]["System"] = [
{
"api_key": self.agentConfig["api_key"],
"host": payload["internalHostname"],
"timestamp": now,
"event_type": "Agent Startup",
"msg_text": "Version %s" % get_version(),
}
]
# Periodically send the host metadata.
if self._is_first_run() or self._should_send_metadata():
payload["systemStats"] = get_system_stats()
payload["meta"] = self._get_metadata()
self.metadata_cache = payload["meta"]
# Add static tags from the configuration file
host_tags = []
if self.agentConfig["tags"] is not None:
host_tags.extend([unicode(tag.strip()) for tag in self.agentConfig["tags"].split(",")])
if self.agentConfig["collect_ec2_tags"]:
host_tags.extend(EC2.get_tags())
if host_tags:
payload["host-tags"]["system"] = host_tags
# Log the metadata on the first run
if self._is_first_run():
log.info(u"Hostnames: %s, tags: %s" % (repr(self.metadata_cache), payload["host-tags"]))
return payload
def shell():
from config import get_version
print """
Datadog Agent v%s - Python Shell
""" % (get_version())
while True:
cmd = raw_input('>>> ')
try:
exec(cmd)
except Exception, e:
print traceback.format_exc(e)
def get(self):
dogstatsd_status = DogstatsdStatus.load_latest_status()
forwarder_status = ForwarderStatus.load_latest_status()
collector_status = CollectorStatus.load_latest_status()
self.render(os.path.join(agent_root, "pup", "status.html"),
port=port,
platform=platform.platform(),
agent_version=get_version(),
python_version=platform.python_version(),
logger_info=logger_info(),
dogstatsd=dogstatsd_status.to_dict(),
forwarder=forwarder_status.to_dict(),
collector=collector_status.to_dict(),
)
def shell():
from config import get_version, set_win32_requests_ca_bundle_path
set_win32_requests_ca_bundle_path()
print """
Datadog Agent v%s - Python Shell
""" % (get_version())
while True:
cmd = raw_input('>>> ')
try:
exec(cmd)
except Exception, e:
print traceback.format_exc(e)
def _build_payload(self, start_event=True):
"""
Return an dictionary that contains all of the generic payload data.
"""
now = time.time()
payload = {
'collection_timestamp': now,
'os' : self.os,
'python': sys.version,
'agentVersion' : self.agentConfig['version'],
'apiKey': self.agentConfig['api_key'],
'events': {},
'metrics': [],
'resources': {},
'internalHostname' : get_hostname(self.agentConfig),
'uuid' : get_uuid(),
}
# Include system stats on first postback
if start_event and self._is_first_run():
payload['systemStats'] = self.agentConfig.get('system_stats', {})
# Also post an event in the newsfeed
payload['events']['System'] = [{'api_key': self.agentConfig['api_key'],
'host': payload['internalHostname'],
'timestamp': now,
'event_type':'Agent Startup',
'msg_text': 'Version %s' % get_version()
}]
# Periodically send the host metadata.
if self._is_first_run() or self._should_send_metadata():
payload['systemStats'] = get_system_stats()
payload['meta'] = self._get_metadata()
self.metadata_cache = payload['meta']
# Add static tags from the configuration file
if self.agentConfig['tags'] is not None:
payload['tags'] = self.agentConfig['tags']
# Log the metadata on the first run
if self._is_first_run():
if self.agentConfig['tags'] is not None:
log.info(u"Hostnames: %s, tags: %s" \
% (repr(self.metadata_cache),
self.agentConfig['tags']))
else:
log.info(u"Hostnames: %s" % repr(self.metadata_cache))
return payload
def __init__(self, data, headers):
self._data = data
self._headers = headers
self._headers['DD-Forwarder-Version'] = get_version()
# Call after data has been set (size is computed in Transaction's init)
Transaction.__init__(self)
# Emitters operate outside the regular transaction framework
if self._emitter_manager is not None:
self._emitter_manager.send(data, headers)
# Insert the transaction in the Manager
self._trManager.append(self)
log.debug("Created transaction %d" % self.get_id())
self._trManager.flush()
def print_status(self):
lines = [
"",
"Collector",
"=========",
"Status date: %s (%ss ago)" % (self.created_at.strftime('%Y-%m-%d %H:%M:%S'),
self.created_seconds_ago()),
"Version: %s" % config.get_version(),
"Pid: %s" % self.created_by_pid,
"Platform: %s" % sys.platform,
"Python Version: %s" % platform.python_version(),
""
]
lines.append("Checks")
lines.append("------")
if not self.check_statuses:
lines.append("No checks have run yet.")
else:
for cs in self.check_statuses:
check_lines = [
cs.name
]
for instance_status in cs.instance_statuses:
line = " - instance #%s [%s]" % (
instance_status.instance_id, instance_status.status)
if instance_status.status != STATUS_OK:
line += u": %s" % instance_status.error
check_lines.append(line)
check_lines += [
" - Collected %s metrics & %s events" % (cs.metric_count, cs.event_count),
]
lines += check_lines
lines.append("")
lines.append("Emitters")
lines.append("------")
if not self.emitter_statuses:
lines.append("No emitters have run yet.")
else:
for es in self.emitter_statuses:
line = " - %s [%s]" % (es.name, es.status)
if es.status != STATUS_OK:
line += ": %s" % es.error
lines.append(line)
print "\n".join(lines)
def print_status(self):
lines = [
"",
self.NAME,
"===========",
"Status date: %s (%ss ago)" % (self.created_at.strftime('%Y-%m-%d %H:%M:%S'),
self.created_seconds_ago()),
"Version: %s" % config.get_version(),
"Pid: %s" % self.created_by_pid,
"Platform: %s" % sys.platform,
"Python Version: %s" % platform.python_version(),
"",
"Queue Size: %s" % self.queue_size,
"Queue Length: %s" % self.queue_length,
"Flush Count: %s" % self.flush_count,
]
print "\n".join(lines)
def print_status(self):
lines = [
"",
"Dogstatsd",
"=========",
"Status date: %s (%ss ago)" % (self.created_at.strftime('%Y-%m-%d %H:%M:%S'),
self.created_seconds_ago()),
"Version: %s" % config.get_version(),
"Pid: %s" % self.created_by_pid,
"Platform: %s" % sys.platform,
"Python Version: %s" % platform.python_version(),
"",
"Flush count: %s" % self.flush_count,
"Packet Count: %s" % self.packet_count,
"Packets per second: %s" % self.packets_per_second,
"Metric count: %s" % self.metric_count,
]
print "\n".join(lines)
def latest_status(self):
try:
loaded_template = template.Loader(".")
dogstatsd_status = DogstatsdStatus.load_latest_status()
forwarder_status = ForwarderStatus.load_latest_status()
collector_status = CollectorStatus.load_latest_status()
generated_template = loaded_template.load("status.html").generate(
port=22,
platform=platform.platform(),
agent_version=get_version(),
python_version=platform.python_version(),
logger_info=logger_info(),
dogstatsd=dogstatsd_status.to_dict(),
forwarder=forwarder_status.to_dict(),
collector=collector_status.to_dict(),
)
return generated_template
except Exception:
return "Unable to fetch latest status"
def submit_http(self, url, data, headers):
headers["DD-Dogstatsd-Version"] = get_version()
log.debug("Posting payload to %s" % url)
try:
start_time = time()
r = requests.post(url, data=data, timeout=5, headers=headers)
r.raise_for_status()
if r.status_code >= 200 and r.status_code < 205:
log.debug("Payload accepted")
status = r.status_code
duration = round((time() - start_time) * 1000.0, 4)
log.debug("%s POST %s (%sms)" % (status, url, duration))
except Exception:
log.exception("Unable to post payload.")
try:
log.error("Received status code: {0}".format(r.status_code))
except Exception:
pass
def submit_http(self, url, data, headers):
headers["SD-Sdstatsd-Version"] = get_version()
headers["Content-MD5"] = str(md5(data).hexdigest())
log.debug("Posting payload to %s" % url)
try:
start_time = time()
r = requests.post(url, data=data, timeout=5, headers=headers)
r.raise_for_status()
if r.status_code >= 200 and r.status_code < 205:
log.debug("Payload accepted")
status = r.status_code
duration = round((time() - start_time) * 1000.0, 4)
log.debug("%s POST %s (%sms)" % (status, string.split(url, "api_key=")[0], duration))
except Exception as e:
log.error("Unable to post payload: %s" % e.message)
try:
log.error("Received status code: {0}".format(r.status_code))
except Exception:
pass
def __init__(self, data, headers, msg_type=""):
self._data = data
self._headers = headers
self._headers['DD-Forwarder-Version'] = get_version()
self._msg_type = msg_type
# Call after data has been set (size is computed in Transaction's init)
Transaction.__init__(self)
# Emitters operate outside the regular transaction framework
if self._emitter_manager is not None:
self._emitter_manager.send(data, headers)
# Insert the transaction(s) in the Manager
for endpoint in self._endpoints:
for api_key in self._endpoints[endpoint]:
transaction = copy.copy(self)
transaction._endpoint = endpoint
transaction._api_key = api_key
self._trManager.append(transaction)
log.debug("Created transaction %d" % transaction.get_id())
self._trManager.flush()
def _build_payload(self):
"""
Return an dictionary that contains all of the generic payload data.
"""
payload = {
'collection_timestamp': time.time(),
'os' : self.os,
'python': sys.version,
'agentVersion' : self.agentConfig['version'],
'apiKey': self.agentConfig['api_key'],
'events': {},
'metrics': [],
'resources': {},
'internalHostname' : gethostname(self.agentConfig),
'uuid' : get_uuid(),
}
# Include system stats on first postback
if self._is_first_run():
payload['systemStats'] = self.agentConfig.get('system_stats', {})
# Also post an event in the newsfeed
payload['events']['System'] = [{'api_key': self.agentConfig['api_key'],
'host': payload['internalHostname'],
'timestamp': int(time.mktime(datetime.datetime.now().timetuple())),
'event_type':'Agent Startup',
'msg_text': 'Version %s' % get_version()
}]
# Periodically send the host metadata.
if self._is_first_run() or self._should_send_metadata():
payload['meta'] = self._get_metadata()
self.metadata_cache = payload['meta']
# Add static tags from the configuration file
if self.agentConfig['tags'] is not None:
payload['tags'] = self.agentConfig['tags']
return payload
def testEndpoints(self):
"""
Tests that the logic behind the agent version specific endpoints is ok.
Also tests that these endpoints actually exist.
"""
MetricTransaction._endpoints = []
api_key = "a" * 32
config = {
"m_url": "https://app.datamonitorhq.com",
"api_key": api_key,
"use_dd": True
}
app = Application()
app.skip_ssl_validation = False
app._agentConfig = config
app.use_simple_http_client = True
trManager = TransactionManager(timedelta(seconds=0), MAX_QUEUE_SIZE,
THROTTLING_DELAY)
trManager._flush_without_ioloop = True # Use blocking API to emulate tornado ioloop
MetricTransaction._trManager = trManager
MetricTransaction.set_application(app)
MetricTransaction.set_endpoints()
transaction = MetricTransaction(None, {}, "")
endpoints = [transaction.get_url(e) for e in transaction._endpoints]
expected = [
'https://{0}-app.agent.datamonitorhq.com/intake/?api_key={1}'.
format(get_version().replace(".", "-"), api_key)
]
self.assertEqual(endpoints, expected, (endpoints, expected))
for url in endpoints:
r = requests.post(url,
data=json.dumps({"foo": "bar"}),
headers={'Content-Type': "application/json"})
r.raise_for_status()
# API Metric Transaction
transaction = APIMetricTransaction(None, {})
endpoints = [transaction.get_url(e) for e in transaction._endpoints]
expected = [
'https://{0}-app.agent.datamonitorhq.com/api/v1/series/?api_key={1}'
.format(get_version().replace(".", "-"), api_key)
]
self.assertEqual(endpoints, expected, (endpoints, expected))
for url in endpoints:
r = requests.post(url,
data=json.dumps({"foo": "bar"}),
headers={'Content-Type': "application/json"})
r.raise_for_status()
# API Service Check Transaction
APIServiceCheckTransaction._trManager = trManager
APIServiceCheckTransaction.set_application(app)
APIServiceCheckTransaction.set_endpoints()
transaction = APIServiceCheckTransaction(None, {})
endpoints = [transaction.get_url(e) for e in transaction._endpoints]
expected = [
'https://{0}-app.agent.datamonitorhq.com/api/v1/check_run/?api_key={1}'
.format(get_version().replace(".", "-"), api_key)
]
self.assertEqual(endpoints, expected, (endpoints, expected))
for url in endpoints:
r = requests.post(url,
data=json.dumps({
'check': 'test',
'status': 0
}),
headers={'Content-Type': "application/json"})
r.raise_for_status()
def _populate_payload_metadata(self, payload, check_statuses, start_event=True):
"""
Periodically populate the payload with metadata related to the system, host, and/or checks.
"""
now = time.time()
# Include system stats on first postback
if start_event and self._is_first_run():
payload['systemStats'] = self.agentConfig.get('system_stats', {})
# Also post an event in the newsfeed
payload['events']['System'] = [{
'api_key': self.agentConfig['api_key'],
'host': payload['internalHostname'],
'timestamp': now,
'event_type':'Agent Startup',
'msg_text': 'Version %s' % get_version()
}]
# Periodically send the host metadata.
if self._should_send_additional_data('host_metadata'):
# gather metadata with gohai
gohai_metadata = self._run_gohai_metadata()
if gohai_metadata:
payload['gohai'] = gohai_metadata
payload['systemStats'] = get_system_stats(
proc_path=self.agentConfig.get('procfs_path', '/proc').rstrip('/')
)
payload['meta'] = self._get_hostname_metadata()
self.hostname_metadata_cache = payload['meta']
# Add static tags from the configuration file
host_tags = []
if self.agentConfig['tags'] is not None:
host_tags.extend([unicode(tag.strip())
for tag in self.agentConfig['tags'].split(",")])
if self.agentConfig['collect_ec2_tags']:
host_tags.extend(EC2.get_tags(self.agentConfig))
if host_tags:
payload['host-tags']['system'] = host_tags
# If required by the user, let's create the dd_check:xxx host tags
if self.agentConfig['create_dd_check_tags']:
app_tags_list = [DD_CHECK_TAG.format(c.name) for c in self.initialized_checks_d]
app_tags_list.extend([DD_CHECK_TAG.format(cname) for cname
in JMXFiles.get_jmx_appnames()])
if 'system' not in payload['host-tags']:
payload['host-tags']['system'] = []
payload['host-tags']['system'].extend(app_tags_list)
GCE_tags = GCE.get_tags(self.agentConfig)
if GCE_tags is not None:
payload['host-tags'][GCE.SOURCE_TYPE_NAME] = GCE_tags
# Log the metadata on the first run
if self._is_first_run():
log.info("Hostnames: %s, tags: %s" %
(repr(self.hostname_metadata_cache), payload['host-tags']))
# Periodically send extra hosts metadata (vsphere)
# Metadata of hosts that are not the host where the agent runs, not all the checks use
# that
external_host_tags = []
if self._should_send_additional_data('external_host_tags'):
for check in self.initialized_checks_d:
try:
getter = getattr(check, 'get_external_host_tags')
check_tags = getter()
external_host_tags.extend(check_tags)
except AttributeError:
pass
if external_host_tags:
payload['external_host_tags'] = external_host_tags
# Periodically send agent_checks metadata
if self._should_send_additional_data('agent_checks'):
# Add agent checks statuses and error/warning messages
agent_checks = []
for check in check_statuses:
if check.instance_statuses is not None:
for i, instance_status in enumerate(check.instance_statuses):
agent_checks.append(
(
check.name, check.source_type_name,
instance_status.instance_id,
instance_status.status,
# put error message or list of warning messages in the same field
# it will be handled by the UI
instance_status.error or instance_status.warnings or "",
check.service_metadata[i]
)
)
else:
agent_checks.append(
(
check.name, check.source_type_name,
"initialization",
check.status, repr(check.init_failed_error)
)
)
payload['agent_checks'] = agent_checks
payload['meta'] = self.hostname_metadata_cache # add hostname metadata
def __init__(self, **kw):
self.__dict__.update(kw)
self.version = get_version()
self.company_name = 'Datadog, Inc.'
self.copyright = 'Copyright 2013 Datadog, Inc.'
self.cmdline_style = 'pywin32'
def main():
options, args = get_parsed_args()
agentConfig = get_config(options=options)
autorestart = agentConfig.get('autorestart', False)
hostname = get_hostname(agentConfig)
COMMANDS = [
'start',
'stop',
'restart',
'foreground',
'status',
'info',
'check',
'configcheck',
'jmx',
]
if len(args) < 1:
sys.stderr.write("Usage: %s %s\n" % (sys.argv[0], "|".join(COMMANDS)))
return 2
command = args[0]
if command not in COMMANDS:
sys.stderr.write("Unknown command: %s\n" % command)
return 3
pid_file = PidFile('dd-agent')
if options.clean:
pid_file.clean()
agent = Agent(pid_file.get_path(), autorestart)
if command in START_COMMANDS:
log.info('Agent version %s' % get_version())
if 'start' == command:
log.info('Start daemon')
agent.start()
elif 'stop' == command:
log.info('Stop daemon')
agent.stop()
elif 'restart' == command:
log.info('Restart daemon')
agent.restart()
elif 'status' == command:
agent.status()
elif 'info' == command:
return agent.info(verbose=options.verbose)
elif 'foreground' == command:
logging.info('Running in foreground')
if autorestart:
# Set-up the supervisor callbacks and fork it.
logging.info('Running Agent with auto-restart ON')
def child_func(): agent.run()
def parent_func(): agent.start_event = False
AgentSupervisor.start(parent_func, child_func)
else:
# Run in the standard foreground.
agent.run(config=agentConfig)
elif 'check' == command:
check_name = args[1]
try:
import checks.collector
# Try the old-style check first
print getattr(checks.collector, check_name)(log).check(agentConfig)
except Exception:
# If not an old-style check, try checks.d
checks = load_check_directory(agentConfig, hostname)
for check in checks['initialized_checks']:
if check.name == check_name:
check.run()
print check.get_metrics()
print check.get_events()
if len(args) == 3 and args[2] == 'check_rate':
print "Running 2nd iteration to capture rate metrics"
time.sleep(1)
check.run()
print check.get_metrics()
print check.get_events()
elif 'configcheck' == command or 'configtest' == command:
osname = get_os()
all_valid = True
for conf_path in glob.glob(os.path.join(get_confd_path(osname), "*.yaml")):
basename = os.path.basename(conf_path)
try:
check_yaml(conf_path)
except Exception, e:
all_valid = False
print "%s contains errors:\n %s" % (basename, e)
else:
print "%s is valid" % basename
if all_valid:
print "All yaml files passed. You can now run the Datadog agent."
return 0
else:
print("Fix the invalid yaml files above in order to start the Datadog agent. "
"A useful external tool for yaml parsing can be found at "
"http://yaml-online-parser.appspot.com/")
return 1
def __init__(self, **kw):
self.__dict__.update(kw)
self.version = get_version()
self.company_name = 'Server Density, Ltd.'
self.copyright = 'Copyright 2015 Server Density, Ltd.'
self.cmdline_style = 'pywin32'
def _build_payload(self, start_event=True):
"""
Return an dictionary that contains all of the generic payload data.
"""
now = time.time()
payload = {
'collection_timestamp': now,
'os' : self.os,
'python': sys.version,
'agentVersion' : self.agentConfig['version'],
'apiKey': self.agentConfig['api_key'],
'events': {},
'metrics': [],
'service_checks': [],
'resources': {},
'internalHostname' : self.hostname,
'uuid' : get_uuid(),
'host-tags': {},
'external_host_tags': {}
}
# Include system stats on first postback
if start_event and self._is_first_run():
payload['systemStats'] = self.agentConfig.get('system_stats', {})
# Also post an event in the newsfeed
payload['events']['System'] = [{'api_key': self.agentConfig['api_key'],
'host': payload['internalHostname'],
'timestamp': now,
'event_type':'Agent Startup',
'msg_text': 'Version %s' % get_version()
}]
# Periodically send the host metadata.
if self._should_send_additional_data('metadata'):
# gather metadata with gohai
try:
if get_os() != 'windows':
command = "gohai"
else:
command = "gohai\gohai.exe"
gohai_metadata = subprocess.Popen(
[command], stdout=subprocess.PIPE
).communicate()[0]
payload['gohai'] = gohai_metadata
except OSError as e:
if e.errno == 2: # file not found, expected when install from source
log.info("gohai file not found")
else:
raise e
except Exception as e:
log.warning("gohai command failed with error %s" % str(e))
payload['systemStats'] = get_system_stats()
payload['meta'] = self._get_metadata()
self.metadata_cache = payload['meta']
# Add static tags from the configuration file
host_tags = []
if self.agentConfig['tags'] is not None:
host_tags.extend([unicode(tag.strip()) for tag in self.agentConfig['tags'].split(",")])
if self.agentConfig['collect_ec2_tags']:
host_tags.extend(EC2.get_tags(self.agentConfig))
if host_tags:
payload['host-tags']['system'] = host_tags
GCE_tags = GCE.get_tags(self.agentConfig)
if GCE_tags is not None:
payload['host-tags'][GCE.SOURCE_TYPE_NAME] = GCE_tags
# Log the metadata on the first run
if self._is_first_run():
log.info("Hostnames: %s, tags: %s" % (repr(self.metadata_cache), payload['host-tags']))
# Periodically send extra hosts metadata (vsphere)
# Metadata of hosts that are not the host where the agent runs, not all the checks use
# that
external_host_tags = []
if self._should_send_additional_data('external_host_tags'):
for check in self.initialized_checks_d:
try:
getter = getattr(check, 'get_external_host_tags')
check_tags = getter()
external_host_tags.extend(check_tags)
except AttributeError:
pass
if external_host_tags:
payload['external_host_tags'] = external_host_tags
return payload
['y', 'n', 'debug'], 'y')
if yn == 'y':
prog_manage.first_time_setup(False)
elif yn == 'debug':
prog_manage.first_time_setup(True)
else:
print('hamstall not installed.')
config.unlock()
sys.exit(0)
generic.leave()
try: # Lingering upgrades check
file_version = prog_manage.get_file_version('file')
except KeyError:
file_version = 1
while config.get_version('file_version') > file_version:
if file_version == 1:
print(
"Removing database config. This will corrupt which programs are installed!"
)
print(
"If you are using hamstall, please contact hammy3502 for an upgrade process."
)
input("Press ENTER to continue...")
try:
config.vprint("Removing old database")
os.remove(config.full("~/.hamstall/database"))
except FileNotFoundError:
pass
config.vprint("Creating new database")
config.create("~/.hamstall/database")
'win32\gui.py',
'dest_base':
"agent-manager",
'uac_info':
"requireAdministrator", # The manager needs to be administrator to stop/start the service
'icon_resources':
[(1,
r"packaging\datadog-agent\win32\install_files\dd_agent_win_256.ico"
)],
}],
'data_files': [("Microsoft.VC90.CRT", glob(r'C:\Python27\redist\*.*')),
('pup', [r'pup\status.html']),
('pup/static', glob('pup/static/*.*')),
('jmxfetch', [r'checks\libs\%s' % JMX_FETCH_JAR_NAME]),
('gohai', [r'gohai\gohai.exe'])],
}
setup(name='datadog-agent',
version=get_version(),
description="DevOps' best friend",
author='DataDog',
author_email='*****@*****.**',
url='http://www.datadoghq.com',
install_requires=install_requires,
setup_requires=setup_requires,
packages=find_packages(exclude=['ez_setup']),
include_package_data=True,
test_suite='nose.collector',
zip_safe=False,
**extra_args)
def _populate_payload_metadata(self,
payload,
check_statuses,
start_event=True):
"""
Periodically populate the payload with metadata related to the system, host, and/or checks.
"""
now = time.time()
# Include system stats on first postback
if start_event and self._is_first_run():
payload['systemStats'] = self.agentConfig.get('system_stats', {})
# Also post an event in the newsfeed
payload['events']['System'] = [{
'api_key':
self.agentConfig['api_key'],
'host':
payload['internalHostname'],
'timestamp':
now,
'event_type':
'Agent Startup',
'msg_text':
'Version %s' % get_version()
}]
# Periodically send the host metadata.
if self._should_send_additional_data('host_metadata'):
# gather metadata with gohai
try:
if get_os() != 'windows':
command = "gohai"
else:
command = "gohai\gohai.exe"
gohai_metadata, gohai_log = subprocess.Popen(
[command], stdout=subprocess.PIPE,
stderr=subprocess.PIPE).communicate()
payload['gohai'] = gohai_metadata
if gohai_log:
log.warning("GOHAI LOG | {0}".format(gohai_log))
except OSError as e:
if e.errno == 2: # file not found, expected when install from source
log.info("gohai file not found")
else:
raise e
except Exception as e:
log.warning("gohai command failed with error %s" % str(e))
payload['systemStats'] = get_system_stats()
payload['meta'] = self._get_hostname_metadata()
self.hostname_metadata_cache = payload['meta']
# Add static tags from the configuration file
host_tags = []
if self.agentConfig['tags'] is not None:
host_tags.extend([
unicode(tag.strip())
for tag in self.agentConfig['tags'].split(",")
])
if self.agentConfig['collect_ec2_tags']:
host_tags.extend(EC2.get_tags(self.agentConfig))
if host_tags:
payload['host-tags']['system'] = host_tags
GCE_tags = GCE.get_tags(self.agentConfig)
if GCE_tags is not None:
payload['host-tags'][GCE.SOURCE_TYPE_NAME] = GCE_tags
# Log the metadata on the first run
if self._is_first_run():
log.info(
"Hostnames: %s, tags: %s" %
(repr(self.hostname_metadata_cache), payload['host-tags']))
# Periodically send extra hosts metadata (vsphere)
# Metadata of hosts that are not the host where the agent runs, not all the checks use
# that
external_host_tags = []
if self._should_send_additional_data('external_host_tags'):
for check in self.initialized_checks_d:
try:
getter = getattr(check, 'get_external_host_tags')
check_tags = getter()
external_host_tags.extend(check_tags)
except AttributeError:
pass
if external_host_tags:
payload['external_host_tags'] = external_host_tags
# Periodically send agent_checks metadata
if self._should_send_additional_data('agent_checks'):
# Add agent checks statuses and error/warning messages
agent_checks = []
for check in check_statuses:
if check.instance_statuses is not None:
for i, instance_status in enumerate(
check.instance_statuses):
agent_checks.append((
check.name,
check.source_type_name,
instance_status.instance_id,
instance_status.status,
# put error message or list of warning messages in the same field
# it will be handled by the UI
instance_status.error or instance_status.warnings
or "",
check.service_metadata[i]))
else:
agent_checks.append(
(check.name, check.source_type_name, "initialization",
check.status, repr(check.init_failed_error)))
payload['agent_checks'] = agent_checks
payload[
'meta'] = self.hostname_metadata_cache # add hostname metadata
# If required by the user, let's create the dd_check:xxx host tags
if self.agentConfig['create_dd_check_tags'] and \
self._should_send_additional_data('dd_check_tags'):
app_tags_list = [
DD_CHECK_TAG.format(c.name) for c in self.initialized_checks_d
]
app_tags_list.extend([
DD_CHECK_TAG.format(cname)
for cname in JMXFiles.get_jmx_appnames()
])
if 'system' not in payload['host-tags']:
payload['host-tags']['system'] = []
payload['host-tags']['system'].extend(app_tags_list)
def main():
options, args = get_parsed_args()
agentConfig = get_config(options=options)
autorestart = agentConfig.get('autorestart', False)
hostname = get_hostname(agentConfig)
in_developer_mode = agentConfig.get('developer_mode')
COMMANDS_AGENT = [
'start',
'stop',
'restart',
'status',
'foreground',
]
COMMANDS_NO_AGENT = [
'info',
'check',
'configcheck',
'jmx',
'flare',
]
COMMANDS = COMMANDS_AGENT + COMMANDS_NO_AGENT
if len(args) < 1:
sys.stderr.write("Usage: %s %s\n" % (sys.argv[0], "|".join(COMMANDS)))
return 2
command = args[0]
if command not in COMMANDS:
sys.stderr.write("Unknown command: %s\n" % command)
return 3
# Deprecation notice
if command not in DD_AGENT_COMMANDS:
# Will become an error message and exit after deprecation period
from utils.deprecations import deprecate_old_command_line_tools
deprecate_old_command_line_tools()
if command in COMMANDS_AGENT:
agent = Agent(PidFile('dd-agent').get_path(),
autorestart,
in_developer_mode=in_developer_mode)
if command in START_COMMANDS:
log.info('Agent version %s' % get_version())
if 'start' == command:
log.info('Start daemon')
agent.start()
elif 'stop' == command:
log.info('Stop daemon')
agent.stop()
elif 'restart' == command:
log.info('Restart daemon')
agent.restart()
elif 'status' == command:
agent.status()
elif 'info' == command:
return Agent.info(verbose=options.verbose)
elif 'foreground' == command:
logging.info('Running in foreground')
if autorestart:
# Set-up the supervisor callbacks and fork it.
logging.info('Running Agent with auto-restart ON')
def child_func():
agent.start(foreground=True)
def parent_func():
agent.start_event = False
AgentSupervisor.start(parent_func, child_func)
else:
# Run in the standard foreground.
agent.start(foreground=True)
elif 'check' == command:
if len(args) < 2:
sys.stderr.write(
"Usage: %s check <check_name> [check_rate]\n"
"Add check_rate as last argument to compute rates\n" %
sys.argv[0])
return 1
check_name = args[1]
try:
import checks.collector
# Try the old-style check first
print getattr(checks.collector, check_name)(log).check(agentConfig)
except Exception:
# If not an old-style check, try checks.d
checks = load_check_directory(agentConfig, hostname)
for check in checks['initialized_checks']:
if check.name == check_name:
if in_developer_mode:
check.run = AgentProfiler.wrap_profiling(check.run)
cs = Collector.run_single_check(check, verbose=True)
print CollectorStatus.render_check_status(cs)
if len(args) == 3 and args[2] == 'check_rate':
print "Running 2nd iteration to capture rate metrics"
time.sleep(1)
cs = Collector.run_single_check(check, verbose=True)
print CollectorStatus.render_check_status(cs)
check.stop()
elif 'configcheck' == command or 'configtest' == command:
configcheck()
elif 'jmx' == command:
jmx_command(args[1:], agentConfig)
elif 'flare' == command:
Flare.check_user_rights()
case_id = int(args[1]) if len(args) > 1 else None
f = Flare(True, case_id)
f.collect()
try:
f.upload()
except Exception, e:
print 'The upload failed:\n{0}'.format(str(e))
def test_check(self):
conf_hostname = "foo"
port = 9600
bad_port = 9405
url = 'http://localhost:{0}'.format(port)
bad_url = 'http://localhost:{0}'.format(bad_port)
agent_config = {
"hostname": conf_hostname,
"version": get_version(),
"api_key": "bar"
}
tags = [u"foo:bar", u"baz"]
input_tag = [u"input_name:stdin"]
output_tag = [u"output_name:stdout"]
filter_tag = [u"filter_name:json"]
config = {
'instances': [
{
'url': url,
'tags': tags
}, # One with tags not external
{
'url': bad_url
}, # One bad url
]
}
self.assertRaises(requests.exceptions.ConnectionError,
self.run_check,
config=config,
agent_config=agent_config)
default_tags = ["url:http://localhost:{0}".format(port)]
instance_config = self.check.get_instance_config(
config['instances'][0])
logstash_version = self.check._get_logstash_version(instance_config)
expected_metrics = dict(STATS_METRICS)
if logstash_version and LooseVersion(logstash_version) < LooseVersion(
"6.0.0"):
expected_metrics.update(PIPELINE_METRICS)
expected_metrics.update(PIPELINE_INPUTS_METRICS)
expected_metrics.update(PIPELINE_OUTPUTS_METRICS)
expected_metrics.update(PIPELINE_FILTERS_METRICS)
good_sc_tags = ['host:localhost', 'port:{0}'.format(port)]
bad_sc_tags = ['host:localhost', 'port:{0}'.format(bad_port)]
for m_name, desc in expected_metrics.iteritems():
m_tags = tags + default_tags
if m_name in PIPELINE_INPUTS_METRICS:
m_tags = m_tags + input_tag
if m_name in PIPELINE_OUTPUTS_METRICS:
m_tags = m_tags + output_tag
if m_name in PIPELINE_FILTERS_METRICS:
m_tags = m_tags + filter_tag
if desc[0] == "gauge":
self.assertMetric(m_name,
tags=m_tags,
count=1,
hostname=conf_hostname)
self.assertServiceCheckOK('logstash.can_connect',
tags=good_sc_tags + tags,
count=1)
self.assertServiceCheckCritical('logstash.can_connect',
tags=bad_sc_tags,
count=1)
# Assert service metadata
self.assertServiceMetadata(['version'], count=2)
self.coverage_report()
def main():
options, args = get_parsed_args()
agentConfig = get_config(options=options)
autorestart = agentConfig.get('autorestart', False)
hostname = get_hostname(agentConfig)
in_developer_mode = agentConfig.get('developer_mode')
COMMANDS_AGENT = [
'start',
'stop',
'restart',
'status',
'foreground',
]
COMMANDS_NO_AGENT = [
'info',
'check',
'configcheck',
'jmx',
'flare',
]
COMMANDS = COMMANDS_AGENT + COMMANDS_NO_AGENT
if len(args) < 1:
sys.stderr.write("Usage: %s %s\n" % (sys.argv[0], "|".join(COMMANDS)))
return 2
command = args[0]
if command not in COMMANDS:
sys.stderr.write("Unknown command: %s\n" % command)
return 3
# TODO: actually kill the start/stop/restart/status command for 5.11
if command in ['start', 'stop', 'restart', 'status'] and not in_developer_mode:
logging.error('Please use supervisor to manage the agent')
return 1
if command in COMMANDS_AGENT:
agent = Agent(PidFile(PID_NAME, PID_DIR).get_path(), autorestart, in_developer_mode=in_developer_mode)
if 'start' == command:
log.info('Start daemon')
agent.start()
elif 'stop' == command:
log.info('Stop daemon')
agent.stop()
elif 'restart' == command:
log.info('Restart daemon')
agent.restart()
elif 'status' == command:
agent.status()
elif 'info' == command:
return Agent.info(verbose=options.verbose)
elif 'foreground' == command:
log.info('Agent version %s' % get_version())
if autorestart:
# Set-up the supervisor callbacks and fork it.
logging.info('Running Agent with auto-restart ON')
def child_func():
agent.start(foreground=True)
def parent_func():
agent.start_event = False
AgentSupervisor.start(parent_func, child_func)
else:
# Run in the standard foreground.
agent.start(foreground=True)
elif 'check' == command:
if len(args) < 2:
sys.stderr.write(
"Usage: %s check <check_name> [check_rate]\n"
"Add check_rate as last argument to compute rates\n"
% sys.argv[0]
)
return 1
check_name = args[1]
try:
import checks.collector
# Try the old-style check first
print getattr(checks.collector, check_name)(log).check(agentConfig)
except Exception:
# If not an old-style check, try checks.d
checks = load_check_directory(agentConfig, hostname)
for check in checks['initialized_checks']:
if check.name == check_name:
if in_developer_mode:
check.run = AgentProfiler.wrap_profiling(check.run)
cs = Collector.run_single_check(check, verbose=True)
print CollectorStatus.render_check_status(cs)
if len(args) == 3 and args[2] == 'check_rate':
print "Running 2nd iteration to capture rate metrics"
time.sleep(1)
cs = Collector.run_single_check(check, verbose=True)
print CollectorStatus.render_check_status(cs)
else:
print "Check has run only once, if some metrics are missing you can run the command again with the 'check_rate' argument appended at the end to see any other metrics if available."
check.stop()
elif 'configcheck' == command or 'configtest' == command:
sd_configcheck(agentConfig)
return configcheck()
elif 'jmx' == command:
jmx_command(args[1:], agentConfig)
elif 'flare' == command:
Flare.check_user_rights()
case_id = int(args[1]) if len(args) > 1 else None
f = Flare(True, case_id)
f.collect()
try:
f.upload()
except Exception as e:
print 'The upload failed:\n{0}'.format(str(e))
return 0
def _build_payload(self, start_event=True):
"""
Return an dictionary that contains all of the generic payload data.
"""
now = time.time()
payload = {
'collection_timestamp': now,
'os': self.os,
'python': sys.version,
'agentVersion': self.agentConfig['version'],
'apiKey': self.agentConfig['api_key'],
'events': {},
'metrics': [],
'service_checks': [],
'resources': {},
'internalHostname': get_hostname(self.agentConfig),
'uuid': get_uuid(),
'host-tags': {},
}
# Include system stats on first postback
if start_event and self._is_first_run():
payload['systemStats'] = self.agentConfig.get('system_stats', {})
# Also post an event in the newsfeed
payload['events']['System'] = [{
'api_key':
self.agentConfig['api_key'],
'host':
payload['internalHostname'],
'timestamp':
now,
'event_type':
'Agent Startup',
'msg_text':
'Version %s' % get_version()
}]
# Periodically send the host metadata.
if self._should_send_metadata():
payload['systemStats'] = get_system_stats()
payload['meta'] = self._get_metadata()
self.metadata_cache = payload['meta']
# Add static tags from the configuration file
host_tags = []
if self.agentConfig['tags'] is not None:
host_tags.extend([
unicode(tag.strip())
for tag in self.agentConfig['tags'].split(",")
])
if self.agentConfig['collect_ec2_tags']:
host_tags.extend(EC2.get_tags())
if host_tags:
payload['host-tags']['system'] = host_tags
GCE_tags = GCE.get_tags()
if GCE_tags is not None:
payload['host-tags'][GCE.SOURCE_TYPE_NAME] = GCE_tags
# Log the metadata on the first run
if self._is_first_run():
log.info("Hostnames: %s, tags: %s" %
(repr(self.metadata_cache), payload['host-tags']))
return payload
("Microsoft.VC90.CRT", glob(r'C:\Python27\redist\*.*')),
('jmxfetch', [r'checks\libs\%s' % JMX_FETCH_JAR_NAME]),
('gohai', [r'gohai\gohai.exe']),
('', [where()]), # CA certificates bundled with `requests`
],
}
elif sys.platform == 'darwin':
app_name = 'Datadog Agent'
from plistlib import Plist
plist = Plist.fromFile(
os.path.dirname(os.path.realpath(__file__)) + '/packaging/Info.plist')
plist.update(
dict(CFBundleGetInfoString="{0}, Copyright (c) 2009-{1}, Datadog Inc.".
format(get_version(),
date.today().year),
CFBundleVersion=get_version()))
extra_args = {
'app': ['gui.py'],
'data_files': [
'images',
'status.html',
],
'options': {
'py2app': {
'optimize': 0,
'iconfile': 'packaging/Agent.icns',
'plist': plist
}
def __init_about(self):
# page About
self.__page_about.show()
self.__name_version = self.__builder.get_object("NameVersion")
self.__name_version.set_markup(_("<big><b>Intelligent Pinyin %s</b></big>") % config.get_version())
def parse_command_line():
"Parse command line"
# list of all available languages; don't load code generators at this point!!
languages = ['C++', 'XRC', 'lisp', 'perl', 'python']
# inject
optparse.OptionParser.format_description = lambda self, formatter: self.description
version = _(
"wxGlade version %s\n"
"Copyright (C) 2007-2012 Alberto Griggio\n"
"Copyright (C) 2011-2016 Carsten Grohmann\n"
"Copyright (C) 2016-2017 Dietmar Schwertberger\n"
"License MIT: The MIT License\n"
" <http://www.opensource.org/licenses/mit-license.php>"
) % config.get_version()
usage = _(
"Usage: wxglade <WXG File> start the wxGlade GUI\n"
" or: wxglade <Options> <WXG File> generate code from command line\n"
" or: wxglade --version show programs version number and exit\n"
" or: wxglade -h|--help show this help message and exit"
)
parser = optparse.OptionParser(add_help_option=False,
version=version,
usage=usage)
parser.add_option('-h',
'--help',
dest='help',
action='store_true',
help=_('show this help message and exit'))
parser.add_option(
"-g",
"--generate-code",
type="choice",
choices=languages,
metavar="LANG",
dest="language",
help=_("(required) output language, valid languages are: %s") %
", ".join(languages))
parser.add_option(
"-o",
"--output",
metavar="PATH",
dest="output",
help=
_("(optional) output file in single-file mode or output directory in multi-file mode"
))
parser.add_option(
"-c",
"--use-config",
dest="rc_file",
help=_(
"use specified wxgladerc config file instead of the default one"))
options, args = parser.parse_args()
# print epilog because OptionParser.epilog isn't available to Python 2.3
if options.help:
parser.print_help()
print(
_("Example: Generate Python code out of myapp.wxg\n\n"
" wxglade -o output.py -g python myapp.wxg\n\n"
"Report bugs to: <*****@*****.**> or at\n"
" <https://sourceforge.net/projects/wxglade/>\n"
"wxGlade home page: <http://wxglade.sourceforge.net/>"))
sys.exit()
# Make an absolute version of path.
# According to the invoking dir of wxGlade (which can be different
# from '.' if it is invoked from a shell script).
if len(args) == 1:
filename = args[0]
if not os.path.isabs(filename):
filename = os.path.join(os.getcwd(), filename)
filename = os.path.normpath(os.path.expanduser(filename))
options.filename = filename
else:
options.filename = None
# check parameters
# - language
# - one file -> cmdline code generation
# - no / > one files -> usage
# - no language -> start gui
if options.language:
if len(args) == 1:
options.start_gui = False
elif len(args) == 0:
msg = _("No wxg file given.\n")
logging.error(msg)
parser.print_help()
sys.exit(msg)
else:
msg = _("Too many wxg files given.\n")
logging.error(msg)
parser.print_help()
sys.exit(msg)
else:
options.start_gui = True
# check output path
if options.output:
options.output = os.path.normpath(os.path.expanduser(options.output))
return options
'--port',
type="int",
default=system_config.default_port,
dest="port")
parser.add_option('-w',
'--workers',
type="int",
default=50,
dest="workers")
opt, args = parser.parse_args()
server_port = opt.port
if platform.system() == 'Windows':
os.system("chcp 65001")
os.system("title=Eve-DL Platform v0.1({}) ^| [{}]".format(
get_version(), server_port))
workers = opt.workers
logger = system_config.logger
# print('=============WITHOUT_LOGGER=============', system_config.without_logger)
tornado.log.enable_pretty_logging(logger=logger)
interface_manager = InterfaceManager()
threading.Thread(target=lambda: event_loop(system_config, model_path,
interface_manager)).start()
sign.set_auth([{
'accessKey': system_config.access_key,
'secretKey': system_config.secret_key
}])
tornado.options.options.ip_whitelist = whitelist()
def _populate_payload_metadata(self,
payload,
check_statuses,
start_event=True):
"""
Periodically populate the payload with metadata related to the system, host, and/or checks.
"""
now = time.time()
# Include system stats on first postback
if start_event and self._is_first_run():
payload['systemStats'] = self.agentConfig.get('system_stats', {})
# Also post an event in the newsfeed
payload['events']['System'] = [{
'api_key':
self.agentConfig['api_key'],
'host':
self.hostname,
'timestamp':
now,
'event_type':
'Agent Startup',
'msg_text':
'Version %s' % get_version()
}]
# Periodically send the host metadata.
if self._should_send_additional_data('host_metadata'):
# gather metadata with gohai
gohai_metadata = self._run_gohai_metadata()
if gohai_metadata:
payload['gohai'] = gohai_metadata
payload['systemStats'] = get_system_stats(
proc_path=self.agentConfig.get('procfs_path', '/proc').rstrip(
'/'))
if self.agentConfig['collect_orchestrator_tags']:
host_container_metadata = MetadataCollector(
).get_host_metadata()
if host_container_metadata:
payload['container-meta'] = host_container_metadata
payload['meta'] = self._get_hostname_metadata()
self.hostname_metadata_cache = payload['meta']
# Add static tags from the configuration file
host_tags = []
if self.agentConfig['tags'] is not None:
host_tags.extend([
unicode(tag.strip())
for tag in self.agentConfig['tags'].split(",")
])
if self.agentConfig['collect_ec2_tags']:
host_tags.extend(EC2.get_tags(self.agentConfig))
if self.agentConfig['collect_orchestrator_tags']:
host_docker_tags = MetadataCollector().get_host_tags()
if host_docker_tags:
host_tags.extend(host_docker_tags)
if host_tags:
payload['host-tags']['system'] = host_tags
# If required by the user, let's create the dd_check:xxx host tags
if self.agentConfig['create_dd_check_tags']:
app_tags_list = [
DD_CHECK_TAG.format(c.name)
for c in self.initialized_checks_d
]
app_tags_list.extend([
DD_CHECK_TAG.format(cname)
for cname in JMXFiles.get_jmx_appnames()
])
if 'system' not in payload['host-tags']:
payload['host-tags']['system'] = []
payload['host-tags']['system'].extend(app_tags_list)
GCE_tags = GCE.get_tags(self.agentConfig)
if GCE_tags is not None:
payload['host-tags'][GCE.SOURCE_TYPE_NAME] = GCE_tags
# Log the metadata on the first run
if self._is_first_run():
log.info(
"Hostnames: %s, tags: %s" %
(repr(self.hostname_metadata_cache), payload['host-tags']))
# Periodically send extra hosts metadata (vsphere)
# Metadata of hosts that are not the host where the agent runs, not all the checks use
# that
external_host_tags = []
if self._should_send_additional_data('external_host_tags'):
for check in self.initialized_checks_d:
try:
getter = getattr(check, 'get_external_host_tags')
check_tags = getter()
external_host_tags.extend(check_tags)
except AttributeError:
pass
if external_host_tags:
payload['external_host_tags'] = external_host_tags
# Periodically send agent_checks metadata
if self._should_send_additional_data('agent_checks'):
# Add agent checks statuses and error/warning messages
agent_checks = []
for check in check_statuses:
if check.instance_statuses is not None:
for i, instance_status in enumerate(
check.instance_statuses):
agent_checks.append((
check.name,
check.source_type_name,
instance_status.instance_id,
instance_status.status,
# put error message or list of warning messages in the same field
# it will be handled by the UI
instance_status.error or instance_status.warnings
or "",
check.service_metadata[i]))
else:
agent_checks.append(
(check.name, check.source_type_name, "initialization",
check.status, repr(check.init_failed_error)))
payload['agent_checks'] = agent_checks
payload[
'meta'] = self.hostname_metadata_cache # add hostname metadata
def testEndpoints(self):
"""
Tests that the logic behind the agent version specific endpoints is ok.
Also tests that these endpoints actually exist.
"""
MetricTransaction._endpoints = []
api_key = "a" * 32
config = {
"endpoints": {"https://app.datadoghq.com": [api_key]},
"dd_url": "https://app.datadoghq.com",
"api_key": api_key,
"use_dd": True
}
app = Application()
app.skip_ssl_validation = False
app._agentConfig = config
app.use_simple_http_client = True
trManager = TransactionManager(timedelta(seconds=0), MAX_QUEUE_SIZE,
THROTTLING_DELAY, max_endpoint_errors=100)
trManager._flush_without_ioloop = True # Use blocking API to emulate tornado ioloop
MetricTransaction._trManager = trManager
MetricTransaction.set_application(app)
MetricTransaction.set_endpoints(config['endpoints'])
transaction = MetricTransaction(None, {}, "")
endpoints = []
for endpoint in transaction._endpoints:
for api_key in transaction._endpoints[endpoint]:
endpoints.append(transaction.get_url(endpoint, api_key))
expected = ['https://{0}-app.agent.datadoghq.com/intake/?api_key={1}'.format(
get_version().replace(".", "-"), api_key)]
self.assertEqual(endpoints, expected, (endpoints, expected))
for url in endpoints:
r = requests.post(url, data=json.dumps({"foo": "bar"}),
headers={'Content-Type': "application/json"})
r.raise_for_status()
# API Metric Transaction
transaction = APIMetricTransaction(None, {})
endpoints = []
for endpoint in transaction._endpoints:
for api_key in transaction._endpoints[endpoint]:
endpoints.append(transaction.get_url(endpoint, api_key))
expected = ['https://{0}-app.agent.datadoghq.com/api/v1/series/?api_key={1}'.format(
get_version().replace(".", "-"), api_key)]
self.assertEqual(endpoints, expected, (endpoints, expected))
for url in endpoints:
try:
r = requests.post(url, data=json.dumps({"foo": "bar"}),
headers={'Content-Type': "application/json"})
r.raise_for_status()
except requests.HTTPError:
if r.status_code != 400 or 'No series present in the payload' not in r.content:
raise
# API Service Check Transaction
APIServiceCheckTransaction._trManager = trManager
APIServiceCheckTransaction.set_application(app)
APIServiceCheckTransaction.set_endpoints(config['endpoints'])
transaction = APIServiceCheckTransaction(None, {})
endpoints = []
for endpoint in transaction._endpoints:
for api_key in transaction._endpoints[endpoint]:
endpoints.append(transaction.get_url(endpoint, api_key))
expected = ['https://{0}-app.agent.datadoghq.com/api/v1/check_run/?api_key={1}'.format(
get_version().replace(".", "-"), api_key)]
self.assertEqual(endpoints, expected, (endpoints, expected))
for url in endpoints:
r = requests.post(url, data=json.dumps({'check': 'test', 'status': 0}),
headers={'Content-Type': "application/json"})
r.raise_for_status()
def _build_payload(self, start_event=True):
"""
Return an dictionary that contains all of the generic payload data.
"""
now = time.time()
payload = {
'collection_timestamp': now,
'os': self.os,
'python': sys.version,
'agentVersion': self.agentConfig['version'],
'apiKey': self.agentConfig['api_key'],
'events': {},
'metrics': [],
'service_checks': [],
'resources': {},
'internalHostname': self.hostname,
'uuid': get_uuid(),
'host-tags': {},
'external_host_tags': {}
}
# Include system stats on first postback
if start_event and self._is_first_run():
payload['systemStats'] = self.agentConfig.get('system_stats', {})
# Also post an event in the newsfeed
payload['events']['System'] = [{
'api_key':
self.agentConfig['api_key'],
'host':
payload['internalHostname'],
'timestamp':
now,
'event_type':
'Agent Startup',
'msg_text':
'Version %s' % get_version()
}]
# Periodically send the host metadata.
if self._should_send_additional_data('metadata'):
# gather metadata with gohai
try:
if get_os() != 'windows':
command = "gohai"
else:
command = "gohai\gohai.exe"
gohai_metadata = subprocess.Popen(
[command], stdout=subprocess.PIPE).communicate()[0]
payload['gohai'] = gohai_metadata
except OSError as e:
if e.errno == 2: # file not found, expected when install from source
log.info("gohai file not found")
else:
raise e
except Exception as e:
log.warning("gohai command failed with error %s" % str(e))
payload['systemStats'] = get_system_stats()
payload['meta'] = self._get_metadata()
self.metadata_cache = payload['meta']
# Add static tags from the configuration file
host_tags = []
if self.agentConfig['tags'] is not None:
host_tags.extend([
unicode(tag.strip())
for tag in self.agentConfig['tags'].split(",")
])
if self.agentConfig['collect_ec2_tags']:
host_tags.extend(EC2.get_tags(self.agentConfig))
if host_tags:
payload['host-tags']['system'] = host_tags
GCE_tags = GCE.get_tags(self.agentConfig)
if GCE_tags is not None:
payload['host-tags'][GCE.SOURCE_TYPE_NAME] = GCE_tags
# Log the metadata on the first run
if self._is_first_run():
log.info("Hostnames: %s, tags: %s" %
(repr(self.metadata_cache), payload['host-tags']))
# Periodically send extra hosts metadata (vsphere)
# Metadata of hosts that are not the host where the agent runs, not all the checks use
# that
external_host_tags = []
if self._should_send_additional_data('external_host_tags'):
for check in self.initialized_checks_d:
try:
getter = getattr(check, 'get_external_host_tags')
check_tags = getter()
external_host_tags.extend(check_tags)
except AttributeError:
pass
if external_host_tags:
payload['external_host_tags'] = external_host_tags
return payload
def main():
options, args = get_parsed_args()
agentConfig = get_config(options=options)
autorestart = agentConfig.get('autorestart', False)
hostname = get_hostname(agentConfig)
in_developer_mode = agentConfig.get('developer_mode')
COMMANDS_AGENT = [
'start',
'stop',
'restart',
'status',
'foreground',
]
COMMANDS_NO_AGENT = [
'info',
'check',
'configcheck',
'jmx',
'flare',
]
COMMANDS = COMMANDS_AGENT + COMMANDS_NO_AGENT
if len(args) < 1:
sys.stderr.write("Usage: %s %s\n" % (sys.argv[0], "|".join(COMMANDS)))
return 2
command = args[0]
if command not in COMMANDS:
sys.stderr.write("Unknown command: %s\n" % command)
return 3
# TODO: actually kill the start/stop/restart/status command for 5.11
if command in ['start', 'stop', 'restart', 'status'] and not in_developer_mode:
logging.error('Please use supervisor to manage the agent')
return 1
if command in COMMANDS_AGENT:
agent = Agent(PidFile(PID_NAME, PID_DIR).get_path(), autorestart, in_developer_mode=in_developer_mode)
if 'start' == command:
log.info('Start daemon')
agent.start()
elif 'stop' == command:
log.info('Stop daemon')
agent.stop()
elif 'restart' == command:
log.info('Restart daemon')
agent.restart()
elif 'status' == command:
agent.status()
elif 'info' == command:
return Agent.info(verbose=options.verbose)
elif 'foreground' == command:
log.info('Agent version %s' % get_version())
if autorestart:
# Set-up the supervisor callbacks and fork it.
logging.info('Running Agent with auto-restart ON')
def child_func():
agent.start(foreground=True)
def parent_func():
agent.start_event = False
AgentSupervisor.start(parent_func, child_func)
else:
# Run in the standard foreground.
agent.start(foreground=True)
elif 'check' == command:
if len(args) < 2:
sys.stderr.write(
"Usage: %s check <check_name> [check_rate]\n"
"Add check_rate as last argument to compute rates\n"
% sys.argv[0]
)
return 1
check_name = args[1]
try:
import checks.collector
# Try the old-style check first
print getattr(checks.collector, check_name)(log).check(agentConfig)
except Exception:
# If not an old-style check, try checks.d
checks = load_check_directory(agentConfig, hostname)
for check in checks['initialized_checks']:
if check.name == check_name:
if in_developer_mode:
check.run = AgentProfiler.wrap_profiling(check.run)
cs = Collector.run_single_check(check, verbose=True)
print CollectorStatus.render_check_status(cs)
if len(args) == 3 and args[2] == 'check_rate':
print "Running 2nd iteration to capture rate metrics"
time.sleep(1)
cs = Collector.run_single_check(check, verbose=True)
print CollectorStatus.render_check_status(cs)
check.stop()
elif 'configcheck' == command or 'configtest' == command:
configcheck()
sd_configcheck(agentConfig)
elif 'jmx' == command:
jmx_command(args[1:], agentConfig)
elif 'flare' == command:
Flare.check_user_rights()
case_id = int(args[1]) if len(args) > 1 else None
f = Flare(True, case_id)
f.collect()
try:
f.upload()
except Exception as e:
print 'The upload failed:\n{0}'.format(str(e))
return 0
'data_files': [
("Microsoft.VC90.CRT", glob(r'C:\Python27\redist\*.*')),
('jmxfetch', [r'checks\libs\%s' % JMX_FETCH_JAR_NAME]),
('gohai', [r'gohai\gohai.exe']),
('', [where()]), # CA certificates bundled with `requests`
],
}
elif sys.platform == 'darwin':
app_name = 'Datadog Agent'
from plistlib import Plist
plist = Plist.fromFile(os.path.dirname(os.path.realpath(__file__)) + '/packaging/Info.plist')
plist.update(dict(
CFBundleGetInfoString="{0}, Copyright (c) 2009-{1}, Datadog Inc.".format(
get_version(), date.today().year),
CFBundleVersion=get_version()
))
extra_args = {
'app': ['gui.py'],
'data_files': [
'images',
'status.html',
],
'options': {
'py2app': {
'optimize': 0,
'iconfile': 'packaging/Agent.icns',
'plist': plist
}
def test_check(self):
conf_hostname = "foo"
port = 9200
bad_port = 9205
agent_config = {
"hostname": conf_hostname,
"version": get_version(),
"api_key": "bar"
}
tags = ['foo:bar', 'baz']
url = 'http://localhost:{0}'.format(port)
bad_url = 'http://localhost:{0}'.format(bad_port)
config = {
'instances': [
{
'url': url,
'tags': tags
}, # One with tags not external
{
'url': url,
'is_external': True
}, # One without tags, external
{
'url': bad_url
}, # One bad url
]
}
self.assertRaises(requests.exceptions.ConnectionError,
self.run_check,
config=config,
agent_config=agent_config)
default_tags = ["url:http://localhost:{0}".format(port)]
expected_metrics = STATS_METRICS
expected_metrics.update(CLUSTER_HEALTH_METRICS)
instance_config = self.check.get_instance_config(
config['instances'][0])
es_version = self.check._get_es_version(instance_config)
self.assertEquals(es_version, get_es_version())
if es_version >= [0, 90, 5]:
expected_metrics.update(ADDITIONAL_METRICS_POST_0_90_5)
if es_version >= [0, 90, 10]:
expected_metrics.update(JVM_METRICS_POST_0_90_10)
else:
expected_metrics.update(JVM_METRICS_PRE_0_90_10)
else:
expected_metrics.update(ADDITIONAL_METRICS_PRE_0_90_5)
expected_metrics.update(JVM_METRICS_PRE_0_90_10)
contexts = [(conf_hostname, default_tags + tags),
(socket.gethostname(), default_tags)]
for m_name, desc in expected_metrics.iteritems():
for hostname, m_tags in contexts:
if (m_name in CLUSTER_HEALTH_METRICS
and hostname == socket.gethostname()):
hostname = conf_hostname
if desc[0] == "gauge":
self.assertMetric(m_name,
tags=m_tags,
count=1,
hostname=hostname)
good_sc_tags = ['host:localhost', 'port:{0}'.format(port)]
bad_sc_tags = ['host:localhost', 'port:{0}'.format(bad_port)]
self.assertServiceCheck('elasticsearch.can_connect',
status=AgentCheck.OK,
tags=good_sc_tags,
count=2)
self.assertServiceCheck('elasticsearch.can_connect',
status=AgentCheck.CRITICAL,
tags=bad_sc_tags,
count=1)
status = AgentCheck.OK if os.environ.get("TRAVIS")\
else AgentCheck.CRITICAL
# Travis doesn't have any shards in the cluster and consider this as green
self.assertServiceCheck('elasticsearch.cluster_health',
status=status,
tags=good_sc_tags,
count=2)
self.coverage_report()
def test_check(self):
conf_hostname = "foo"
port = 9200
bad_port = 9405
agent_config = {
"hostname": conf_hostname, "version": get_version(),
"api_key": "bar"
}
tags = ['foo:bar', 'baz']
url = 'http://localhost:{0}'.format(port)
bad_url = 'http://localhost:{0}'.format(bad_port)
config = {
'instances': [
{'url': url, 'tags': tags}, # One with tags not external
{'url': url, 'cluster_stats': True}, # One without tags, external
{'url': bad_url}, # One bad url
]
}
self.assertRaises(
requests.exceptions.ConnectionError,
self.run_check, config=config, agent_config=agent_config)
default_tags = ["url:http://localhost:{0}".format(port)]
expected_metrics = STATS_METRICS
CLUSTER_HEALTH_METRICS.update(CLUSTER_PENDING_TASKS)
expected_metrics.update(CLUSTER_HEALTH_METRICS)
instance_config = self.check.get_instance_config(config['instances'][0])
es_version = self.check._get_es_version(instance_config)
self.assertEquals(es_version, get_es_version())
if es_version >= [0, 90, 5]:
expected_metrics.update(ADDITIONAL_METRICS_POST_0_90_5)
if es_version >= [0, 90, 10]:
expected_metrics.update(JVM_METRICS_POST_0_90_10)
else:
expected_metrics.update(JVM_METRICS_PRE_0_90_10)
else:
expected_metrics.update(ADDITIONAL_METRICS_PRE_0_90_5)
expected_metrics.update(JVM_METRICS_PRE_0_90_10)
contexts = [
(conf_hostname, default_tags + tags),
(socket.gethostname(), default_tags)
]
for m_name, desc in expected_metrics.iteritems():
for hostname, m_tags in contexts:
if (m_name in CLUSTER_HEALTH_METRICS
and hostname == socket.gethostname()):
hostname = conf_hostname
if desc[0] == "gauge":
self.assertMetric(
m_name, tags=m_tags, count=1, hostname=hostname)
good_sc_tags = ['host:localhost', 'port:{0}'.format(port)]
bad_sc_tags = ['host:localhost', 'port:{0}'.format(bad_port)]
self.assertServiceCheckOK('elasticsearch.can_connect',
tags=good_sc_tags,
count=2)
self.assertServiceCheckCritical('elasticsearch.can_connect',
tags=bad_sc_tags,
count=1)
# Assert service metadata
self.assertServiceMetadata(['version'], count=3)
# FIXME: 0.90.13 returns randomly a red status instead of yellow,
# so we don't do a coverage test for it
# Remove me when we stop supporting 0.90.x (not supported anymore by ES)
if get_es_version() != [0, 90, 13]:
# Warning because elasticsearch status should be yellow, according to
# http://chrissimpson.co.uk/elasticsearch-yellow-cluster-status-explained.html
self.assertServiceCheckWarning('elasticsearch.cluster_health',
tags=good_sc_tags,
count=2)
# Assert event
self.assertEvent('ElasticSearch: foo just reported as yellow', count=1,
tags=default_tags+tags, msg_title='foo is yellow',
event_type='elasticsearch', alert_type='warning',
source_type_name='elasticsearch')
self.coverage_report()
'data_files': [
("Microsoft.VC90.CRT", glob(r'C:\Python27\redist\*.*')),
('jmxfetch', [r'checks\libs\%s' % JMX_FETCH_JAR_NAME]),
('gohai', [r'gohai\gohai.exe']),
('', [where()]), # CA certificates bundled with `requests`
],
}
elif sys.platform == 'darwin':
app_name = 'Datadog Agent'
from plistlib import Plist
plist = Plist.fromFile(os.path.dirname(os.path.realpath(__file__)) + '/packaging/Info.plist')
plist.update(dict(
CFBundleGetInfoString="{0}, Copyright (c) 2009-{1}, Datadog Inc.".format(
get_version(), date.today().year),
CFBundleVersion=get_version()
))
extra_args = {
'app': ['gui.py'],
'data_files': [
'images',
'status.html',
],
'options': {
'py2app': {
'optimize': 0,
'iconfile': 'packaging/Agent.icns',
'plist': plist
}
['share/doc/wxglade/tutorial',
glob('docs/Tutorial.html')],
['share/doc/wxglade/tutorial/img',
glob('docs/img/*.*')],
['share/doc/wxglade/manual_html',
glob('docs/html/*.*')],
['share/doc/wxglade/manual_pdf',
glob('docs/pdf/*.pdf')],
['share/man/man1', ['docs_old/man/wxglade.1']],
]
packages = ['wxglade.%s' % pkg for pkg in find_packages(exclude=['tests'])]
packages.append('wxglade')
# write and handle version file
version = config.get_version(False)
if not os.path.exists('version.py'):
config.write_version_file(version)
if 'sdist' in sys.argv:
package_data_files = []
package_data_files.extend(text_files)
package_data_files.extend([
'Makefile', 'epydoc.conf', 'pylintrc', 'wxGlade.desktop',
'wxglade.pyw', 'widgets/widgets.txt', '__init__.py', 'test.py'
])
# add content of listed directories to the package_data_file list
for data_dir in [
'docs', 'install', 'icons', 'locale', 'po', 'res', 'tests',
'templates'
'service': [agent_svc],
'windows': [{'script': 'win32\gui.py',
'dest_base': "agent-manager",
'uac_info': "requireAdministrator", # The manager needs to be administrator to stop/start the service
'icon_resources': [(1, r"packaging\datadog-agent\win32\install_files\dd_agent_win_256.ico")],
}],
'data_files': [
("Microsoft.VC90.CRT", glob(r'C:\Python27\redist\*.*')),
('pup', [r'pup\pup.html', r'pup\status.html']),
('pup/static', glob('pup/static/*.*')),
('jmxfetch', [r'checks\libs\%s' % JMX_FETCH_JAR_NAME]),
('gohai', [r'gohai\gohai.exe'])
],
}
setup(
name='datadog-agent',
version=get_version(),
description="DevOps' best friend",
author='DataDog',
author_email='*****@*****.**',
url='http://www.datadoghq.com',
install_requires=install_requires,
setup_requires=setup_requires,
packages=find_packages(exclude=['ez_setup']),
include_package_data=True,
test_suite='nose.collector',
zip_safe=False,
**extra_args
)
def main():
options, args = get_parsed_args()
agentConfig = get_config(options=options)
autorestart = agentConfig.get('autorestart', False)
hostname = get_hostname(agentConfig)
COMMANDS = [
'start',
'stop',
'restart',
'foreground',
'status',
'info',
'check',
'configcheck',
'jmx',
]
if len(args) < 1:
sys.stderr.write("Usage: %s %s\n" % (sys.argv[0], "|".join(COMMANDS)))
return 2
command = args[0]
if command not in COMMANDS:
sys.stderr.write("Unknown command: %s\n" % command)
return 3
pid_file = PidFile('dd-agent')
if options.clean:
pid_file.clean()
agent = Agent(pid_file.get_path(), autorestart)
if command in START_COMMANDS:
log.info('Agent version %s' % get_version())
if 'start' == command:
log.info('Start daemon')
agent.start()
elif 'stop' == command:
log.info('Stop daemon')
agent.stop()
elif 'restart' == command:
log.info('Restart daemon')
agent.restart()
elif 'status' == command:
agent.status()
elif 'info' == command:
return agent.info(verbose=options.verbose)
elif 'foreground' == command:
logging.info('Running in foreground')
if autorestart:
# Set-up the supervisor callbacks and fork it.
logging.info('Running Agent with auto-restart ON')
def child_func():
agent.start(foreground=True)
def parent_func():
agent.start_event = False
AgentSupervisor.start(parent_func, child_func)
else:
# Run in the standard foreground.
agent.start(foreground=True)
elif 'check' == command:
if len(args) < 2:
sys.stderr.write(
"Usage: %s check <check_name> [check_rate]\n"
"Add check_rate as last argument to compute rates\n" %
sys.argv[0])
return 1
check_name = args[1]
try:
import checks.collector
# Try the old-style check first
print getattr(checks.collector, check_name)(log).check(agentConfig)
except Exception:
# If not an old-style check, try checks.d
checks = load_check_directory(agentConfig, hostname)
for check in checks['initialized_checks']:
if check.name == check_name:
check.run()
print check.get_metrics()
print check.get_events()
print check.get_service_checks()
if len(args) == 3 and args[2] == 'check_rate':
print "Running 2nd iteration to capture rate metrics"
time.sleep(1)
check.run()
print check.get_metrics()
print check.get_events()
print check.get_service_checks()
check.stop()
elif 'configcheck' == command or 'configtest' == command:
osname = get_os()
all_valid = True
for conf_path in glob.glob(
os.path.join(get_confd_path(osname), "*.yaml")):
basename = os.path.basename(conf_path)
try:
check_yaml(conf_path)
except Exception, e:
all_valid = False
print "%s contains errors:\n %s" % (basename, e)
else:
print "%s is valid" % basename
if all_valid:
print "All yaml files passed. You can now run the Datadog agent."
return 0
else:
print(
"Fix the invalid yaml files above in order to start the Datadog agent. "
"A useful external tool for yaml parsing can be found at "
"http://yaml-online-parser.appspot.com/")
return 1
def __init__(self, **kw):
self.__dict__.update(kw)
self.version = get_version()
self.company_name = 'Datadog, Inc.'
self.copyright = 'Copyright 2013 Datadog, Inc.'
self.cmdline_style = 'pywin32'
def main():
options, args = get_parsed_args()
agentConfig = get_config(options=options)
autorestart = agentConfig.get('autorestart', False)
hostname = get_hostname(agentConfig)
COMMANDS_AGENT = [
'start',
'stop',
'restart',
'status',
'foreground',
]
COMMANDS_NO_AGENT = [
'info',
'check',
'configcheck',
'jmx',
'flare',
]
COMMANDS = COMMANDS_AGENT + COMMANDS_NO_AGENT
if len(args) < 1:
sys.stderr.write("Usage: %s %s\n" % (sys.argv[0], "|".join(COMMANDS)))
return 2
command = args[0]
if command not in COMMANDS:
sys.stderr.write("Unknown command: %s\n" % command)
return 3
# Deprecation notice
if command not in DD_AGENT_COMMANDS:
# Will become an error message and exit after deprecation period
from utils.deprecations import deprecate_old_command_line_tools
deprecate_old_command_line_tools()
if command in COMMANDS_AGENT:
agent = Agent(PidFile('dd-agent').get_path(), autorestart)
if command in START_COMMANDS:
log.info('Agent version %s' % get_version())
if 'start' == command:
log.info('Start daemon')
agent.start()
elif 'stop' == command:
log.info('Stop daemon')
agent.stop()
elif 'restart' == command:
log.info('Restart daemon')
agent.restart()
elif 'status' == command:
agent.status()
elif 'info' == command:
return Agent.info(verbose=options.verbose)
elif 'foreground' == command:
logging.info('Running in foreground')
if autorestart:
# Set-up the supervisor callbacks and fork it.
logging.info('Running Agent with auto-restart ON')
def child_func():
agent.start(foreground=True)
def parent_func():
agent.start_event = False
AgentSupervisor.start(parent_func, child_func)
else:
# Run in the standard foreground.
agent.start(foreground=True)
elif 'check' == command:
if len(args) < 2:
sys.stderr.write(
"Usage: %s check <check_name> [check_rate]\n"
"Add check_rate as last argument to compute rates\n" %
sys.argv[0])
return 1
check_name = args[1]
try:
import checks.collector
# Try the old-style check first
print getattr(checks.collector, check_name)(log).check(agentConfig)
except Exception:
# If not an old-style check, try checks.d
checks = load_check_directory(agentConfig, hostname)
for check in checks['initialized_checks']:
if check.name == check_name:
check.run()
print check.get_metrics()
print check.get_events()
print check.get_service_checks()
if len(args) == 3 and args[2] == 'check_rate':
print "Running 2nd iteration to capture rate metrics"
time.sleep(1)
check.run()
print check.get_metrics()
print check.get_events()
print check.get_service_checks()
check.stop()
elif 'configcheck' == command or 'configtest' == command:
configcheck()
elif 'jmx' == command:
from jmxfetch import JMX_LIST_COMMANDS, JMXFetch
if len(args) < 2 or args[1] not in JMX_LIST_COMMANDS.keys():
print "#" * 80
print "JMX tool to be used to help configuring your JMX checks."
print "See http://docs.datadoghq.com/integrations/java/ for more information"
print "#" * 80
print "\n"
print "You have to specify one of the following commands:"
for command, desc in JMX_LIST_COMMANDS.iteritems():
print " - %s [OPTIONAL: LIST OF CHECKS]: %s" % (command,
desc)
print "Example: sudo /etc/init.d/datadog-agent jmx list_matching_attributes tomcat jmx solr"
print "\n"
else:
jmx_command = args[1]
checks_list = args[2:]
confd_directory = get_confd_path(get_os())
jmx_process = JMXFetch(confd_directory, agentConfig)
jmx_process.configure()
should_run = jmx_process.should_run()
if should_run:
jmx_process.run(jmx_command, checks_list, reporter="console")
else:
print "Couldn't find any valid JMX configuration in your conf.d directory: %s" % confd_directory
print "Have you enabled any JMX check ?"
print "If you think it's not normal please get in touch with Datadog Support"
elif 'flare' == command:
Flare.check_user_rights()
case_id = int(args[1]) if len(args) > 1 else None
f = Flare(True, case_id)
f.collect()
try:
f.upload()
except Exception, e:
print 'The upload failed:\n{0}'.format(str(e))
