def _get_watchmonitor(self, check_freq):
watchmonitor = None
if self._agentConfig.get("watchmonitor", True):
watchmonitor = Watchmonitor(check_freq * WATCHmonitor_MULTIPLIER,
max_mem_mb=self._agentConfig.get('limit_memory_consumption', None))
watchmonitor.reset()
return watchmonitor
def __init__(self,
agent_config,
interval,
metrics_aggregator,
api_host,
api_key=None,
use_watchmonitor=False,
event_chunk_size=None):
threading.Thread.__init__(self)
self.ip = get_ip(agent_config)
self.interval = int(interval)
self.finished = threading.Event()
self.metrics_aggregator = metrics_aggregator
self.flush_count = 0
self.log_count = 0
self.hostname = get_hostname()
self.watchmonitor = None
if use_watchmonitor:
from util import Watchmonitor
self.watchmonitor = Watchmonitor(WATCHmonitor_TIMEOUT)
self.api_key = api_key
self.api_host = api_host
self.event_chunk_size = event_chunk_size or EVENT_CHUNK_SIZE
def __init__(self,
port,
agentConfig,
watchmonitor=True,
skip_ssl_validation=False,
use_simple_http_client=False):
self.ip = get_ip(agentConfig, log)
self._port = int(port)
self._agentConfig = agentConfig
self._metrics = {}
AgentTransaction.set_application(self)
AgentTransaction.set_endpoints()
self._tr_manager = TransactionManager(MAX_WAIT_FOR_REPLAY,
MAX_QUEUE_SIZE, THROTTLING_DELAY)
AgentTransaction.set_tr_manager(self._tr_manager)
self._watchmonitor = None
self.skip_ssl_validation = skip_ssl_validation or agentConfig.get(
'skip_ssl_validation', False)
self.use_simple_http_client = use_simple_http_client
self._send_controler = 0 # control self.postAgentInfoToServer run once a minute
if self.skip_ssl_validation:
log.info(
"Skipping SSL hostname validation, useful when using a transparent proxy"
)
if watchmonitor:
watchmonitor_timeout = TRANSACTION_FLUSH_INTERVAL * WATCHmonitor_INTERVAL_MULTIPLIER / 1000
self._watchmonitor = Watchmonitor(watchmonitor_timeout,
max_mem_mb=agentConfig.get(
'limit_memory_consumption',
None))
class Reporter(threading.Thread):
def __init__(self,
agent_config,
interval,
metrics_aggregator,
api_host,
api_key=None,
use_watchmonitor=False,
event_chunk_size=None):
threading.Thread.__init__(self)
self.ip = get_ip(agent_config)
self.interval = int(interval)
self.finished = threading.Event()
self.metrics_aggregator = metrics_aggregator
self.flush_count = 0
self.log_count = 0
self.hostname = get_hostname()
self.watchmonitor = None
if use_watchmonitor:
from util import Watchmonitor
self.watchmonitor = Watchmonitor(WATCHmonitor_TIMEOUT)
self.api_key = api_key
self.api_host = api_host
self.event_chunk_size = event_chunk_size or EVENT_CHUNK_SIZE
def stop(self):
log.info("Stopping reporter")
self.finished.set()
def run(self):
log.info("Reporting to %s every %ss" % (self.api_host, self.interval))
log.debug("Watchmonitor enabled: %s" % bool(self.watchmonitor))
MonitorstatsdStatus().persist()
while not self.finished.isSet(
): # Use camel case isSet for 2.4 support.
self.finished.wait(self.interval)
self.metrics_aggregator.send_packet_count(
'datamonitor.monitorstatsd.packet.count')
self.flush()
if self.watchmonitor:
self.watchmonitor.reset()
log.debug("Stopped reporter")
MonitorstatsdStatus.remove_latest_status()
def flush(self):
try:
self.flush_count += 1
self.log_count += 1
packets_per_second = self.metrics_aggregator.packets_per_second(
self.interval)
packet_count = self.metrics_aggregator.total_count
metrics = self.metrics_aggregator.flush()
count = len(metrics)
if self.flush_count % FLUSH_LOGGING_PERIOD == 0:
self.log_count = 0
if count:
self.submit(metrics)
events = self.metrics_aggregator.flush_events()
event_count = len(events)
if event_count:
self.submit_events(events)
service_checks = self.metrics_aggregator.flush_service_checks()
service_check_count = len(service_checks)
if service_check_count:
self.submit_service_checks(service_checks)
should_log = self.flush_count <= FLUSH_LOGGING_INITIAL or self.log_count <= FLUSH_LOGGING_COUNT
log_func = log.info
if not should_log:
log_func = log.debug
log_func(
"Flush #%s: flushed %s metric%s, %s event%s, and %s service check run%s"
% (self.flush_count, count, plural(count), event_count,
plural(event_count), service_check_count,
plural(service_check_count)))
if self.flush_count == FLUSH_LOGGING_INITIAL:
log.info(
"First flushes done, %s flushes will be logged every %s flushes."
% (FLUSH_LOGGING_COUNT, FLUSH_LOGGING_PERIOD))
packet_count = self.metrics_aggregator.total_count
MonitorstatsdStatus(
flush_count=self.flush_count,
packet_count=packet_count,
packets_per_second=packets_per_second,
metric_count=count,
event_count=event_count,
service_check_count=service_check_count,
).persist()
except Exception:
if self.finished.isSet():
log.debug(
"Couldn't flush metrics, but that's expected as we're stopping"
)
else:
log.exception("Error flushing metrics")
def submit(self, metrics):
body, headers = serialize_metrics(metrics, self.hostname, self.ip)
params = {}
if self.api_key:
params['api_key'] = self.api_key
url = '%s/api/v1/series?%s' % (self.api_host, urlencode(params))
self.submit_http(url, body, headers)
def submit_events(self, events):
headers = {'Content-Type': 'application/json'}
event_chunk_size = self.event_chunk_size
for chunk in chunks(events, event_chunk_size):
payload = {
'apiKey': self.api_key,
'events': {
'api': chunk
},
'uuid': get_uuid(),
'ip': self.ip,
'internalHostname': get_hostname()
}
params = {}
if self.api_key:
params['api_key'] = self.api_key
url = '%s/intake?%s' % (self.api_host, urlencode(params))
self.submit_http(url, json.dumps(payload), headers)
def submit_http(self, url, data, headers):
headers["DD-monitorstatsd-Version"] = get_version()
log.debug("Posting payload to %s" % url)
try:
start_time = time()
r = requests.post(url, data=data, timeout=5, headers=headers)
r.raise_for_status()
if r.status_code >= 200 and r.status_code < 205:
log.debug("Payload accepted")
status = r.status_code
duration = round((time() - start_time) * 1000.0, 4)
log.debug("%s POST %s (%sms)" % (status, url, duration))
except Exception:
log.exception("Unable to post payload.")
try:
log.error("Received status code: {0}".format(r.status_code))
except Exception:
pass
def submit_service_checks(self, service_checks):
headers = {'Content-Type': 'application/json'}
params = {}
if self.api_key:
params['api_key'] = self.api_key
for check in service_checks:
check['uuid'] = get_uuid()
check['ip'] = self.ip
url = '{0}/api/v1/check_run?{1}'.format(self.api_host,
urlencode(params))
self.submit_http(url, json.dumps(service_checks), headers)
def busy_run(self):
w = Watchmonitor(5)
w.reset()
while True:
random()
def fast_tornado(self):
a = Application(12345, {"bind_host": "localhost"})
a._watchmonitor = Watchmonitor(6)
a._tr_manager = MockTxManager()
a.run()
def normal_run(self):
w = Watchmonitor(2)
w.reset()
for i in range(5):
time.sleep(1)
w.reset()
def hanging_net(self):
w = Watchmonitor(5)
w.reset()
x = url.urlopen("http://localhost:31834")
print "ERROR Net call returned", x
return True