def _generate_queue():
while True:
t = time.time()
metric_counts = sorted(self.cache.counts, key=lambda x: x[1])
log.debug("Sorted %d cache queues in %.6f seconds" % (len(metric_counts), time.time() - t))
while metric_counts:
yield itemgetter(0)(metric_counts.pop())
def _generate_queue():
while True:
t = time.time()
metric_counts = sorted(self.cache.counts, key=lambda x: x[1])
log.debug("Sorted %d cache queues in %.6f seconds" %
(len(metric_counts), time.time() - t))
while metric_counts:
yield itemgetter(0)(metric_counts.pop())
def startService(self):
if 'signal' in globals().keys():
log.debug("Installing SIG_IGN for SIGHUP")
signal.signal(signal.SIGHUP, signal.SIG_IGN)
Service.startService(self)
for factory in self.client_factories.values():
if not factory.started:
factory.startConnecting()
def startService(self):
if 'signal' in globals().keys():
log.debug("Installing SIG_IGN for SIGHUP")
signal.signal(signal.SIGHUP, signal.SIG_IGN)
Service.startService(self)
for factory in self.client_factories.values():
if not factory.started:
factory.startConnecting()
def startService(self):
if 'signal' in globals().keys():
log.debug("Installing SIG_IGN for SIGHUP")
signal.signal(signal.SIGHUP, signal.SIG_IGN)
self.storage_reload_task.start(60, False)
self.aggregation_reload_task.start(60, False)
reactor.addSystemEventTrigger('before', 'shutdown', shutdownModifyUpdateSpeed)
reactor.callInThread(writeForever)
Service.startService(self)
def startService(self):
if 'signal' in globals().keys():
log.debug("Installing SIG_IGN for SIGHUP")
signal.signal(signal.SIGHUP, signal.SIG_IGN)
self.storage_reload_task.start(60, False)
self.aggregation_reload_task.start(60, False)
reactor.addSystemEventTrigger('before', 'shutdown', shutdownModifyUpdateSpeed)
reactor.callInThread(writeForever)
Service.startService(self)
def tag(self, *metrics):
from carbon.http import httpRequest
log.debug("Tagging %s" % ', '.join(metrics), type='tagdb')
t = time.time()
try:
httpRequest(
self.graphite_url + '/tags/tagMultiSeries',
[('path', metric) for metric in metrics]
)
log.debug("Tagged %s in %s" % (', '.join(metrics), time.time() - t), type='tagdb')
except Exception as err:
log.msg("Error tagging %s: %s" % (', '.join(metrics), err), type='tagdb')
def tag(self, *metrics):
from carbon.http import httpRequest
log.debug("Tagging %s" % ', '.join(metrics), type='tagdb')
t = time.time()
try:
httpRequest(self.graphite_url + '/tags/tagMultiSeries',
[('path', metric) for metric in metrics])
log.debug("Tagged %s in %s" %
(', '.join(metrics), time.time() - t),
type='tagdb')
except Exception as err:
log.msg("Error tagging %s: %s" % (', '.join(metrics), err),
type='tagdb')
def tag(self, metric):
from carbon.http import httpRequest
log.debug("Tagging %s" % metric)
t = time.time()
def successHandler(result, *args, **kw):
log.debug("Tagged %s: %s in %s" %
(metric, result, time.time() - t))
def errorHandler(err):
log.msg("Error tagging %s: %s" % (metric, err.getErrorMessage()))
httpRequest(self.graphite_url + '/tags/tagSeries', {
'path': metric
}).addCallback(successHandler).addErrback(errorHandler)
def optimalWriteOrder():
"""Generates metrics with the most cached values first and applies a soft
rate limit on new metrics"""
global lastCreateInterval
global createCount
metrics = MetricCache.counts()
t = time.time()
metrics.sort(key=lambda item: item[1],
reverse=True) # by queue size, descending
log.debug("Sorted %d cache queues in %.6f seconds" %
(len(metrics), time.time() - t))
for metric, queueSize in metrics:
if state.cacheTooFull and MetricCache.size < CACHE_SIZE_LOW_WATERMARK:
events.cacheSpaceAvailable()
dbFilePath = getFilesystemPath(metric)
dbFileExists = exists(dbFilePath)
if not dbFileExists:
createCount += 1
now = time.time()
if now - lastCreateInterval >= 60:
lastCreateInterval = now
createCount = 1
elif createCount >= settings.MAX_CREATES_PER_MINUTE:
# dropping queued up datapoints for new metrics prevents filling up the entire cache
# when a bunch of new metrics are received.
try:
MetricCache.pop(metric)
except KeyError:
pass
continue
try: # metrics can momentarily disappear from the MetricCache due to the implementation of MetricCache.store()
datapoints = MetricCache.pop(metric)
except KeyError:
log.msg("MetricCache contention, skipping %s update for now" %
metric)
continue # we simply move on to the next metric when this race condition occurs
yield (metric, datapoints, dbFilePath, dbFileExists)
def tag(self, *metrics):
from carbon.http import httpRequest
log.debug("Tagging %s" % ', '.join(metrics), type='tagdb')
t = time.time()
def successHandler(result, *args, **kw):
log.debug("Tagged %s in %s" % (', '.join(metrics), time.time() - t), type='tagdb')
return True
def errorHandler(err):
log.msg("Error tagging %s: %s" % (', '.join(metrics), err.getErrorMessage()), type='tagdb')
return err
return httpRequest(
self.graphite_url + '/tags/tagMultiSeries',
[('path', metric) for metric in metrics]
).addCallback(successHandler).addErrback(errorHandler)
def optimalWriteOrder():
"""Generates metrics with the most cached values first and applies a soft
rate limit on new metrics"""
global lastCreateInterval
global createCount
metrics = MetricCache.counts()
t = time.time()
metrics.sort(key=lambda item: item[1], reverse=True) # by queue size, descending
log.debug("Sorted %d cache queues in %.6f seconds" % (len(metrics),
time.time() - t))
for metric, queueSize in metrics:
if state.cacheTooFull and MetricCache.size < CACHE_SIZE_LOW_WATERMARK:
events.cacheSpaceAvailable()
dbFilePath = getFilesystemPath(metric)
dbFileExists = exists(dbFilePath)
if not dbFileExists:
createCount += 1
now = time.time()
if now - lastCreateInterval >= 60:
lastCreateInterval = now
createCount = 1
elif createCount >= settings.MAX_CREATES_PER_MINUTE:
# dropping queued up datapoints for new metrics prevents filling up the entire cache
# when a bunch of new metrics are received.
try:
MetricCache.pop(metric)
except KeyError:
pass
continue
try: # metrics can momentarily disappear from the MetricCache due to the implementation of MetricCache.store()
datapoints = MetricCache.pop(metric)
except KeyError:
log.msg("MetricCache contention, skipping %s update for now" % metric)
continue # we simply move on to the next metric when this race condition occurs
yield (metric, datapoints, dbFilePath, dbFileExists)
from collections import deque
from time import time
from twisted.application.service import Service
from twisted.internet import reactor
from twisted.internet.defer import Deferred, DeferredList
from twisted.internet.protocol import ReconnectingClientFactory
from twisted.protocols.basic import Int32StringReceiver
from carbon.conf import settings
from carbon.util import pickle
from carbon import instrumentation, log, pipeline, state
try:
import signal
except ImportError:
log.debug("Couldn't import signal module")
SEND_QUEUE_LOW_WATERMARK = settings.MAX_QUEUE_SIZE * settings.QUEUE_LOW_WATERMARK_PCT
class CarbonClientProtocol(Int32StringReceiver):
def connectionMade(self):
log.clients("%s::connectionMade" % self)
self.paused = False
self.connected = True
self.transport.registerProducer(self, streaming=True)
# Define internal metric names
self.lastResetTime = time()
self.destinationName = self.factory.destinationName
self.queuedUntilReady = 'destinations.%s.queuedUntilReady' % self.destinationName
self.sent = 'destinations.%s.sent' % self.destinationName
from collections import deque
from time import time
from twisted.application.service import Service
from twisted.internet import reactor
from twisted.internet.defer import Deferred, DeferredList
from twisted.internet.protocol import ReconnectingClientFactory
from twisted.protocols.basic import Int32StringReceiver
from carbon.conf import settings
from carbon.util import pickle
from carbon import instrumentation, log, pipeline, state
try:
import signal
except ImportError:
log.debug("Couldn't import signal module")
SEND_QUEUE_LOW_WATERMARK = settings.MAX_QUEUE_SIZE * settings.QUEUE_LOW_WATERMARK_PCT
class CarbonClientProtocol(Int32StringReceiver):
def connectionMade(self):
log.clients("%s::connectionMade" % self)
self.paused = False
self.connected = True
self.transport.registerProducer(self, streaming=True)
# Define internal metric names
self.lastResetTime = time()
self.destinationName = self.factory.destinationName
self.queuedUntilReady = 'destinations.%s.queuedUntilReady' % self.destinationName
def get_buffer(self, metric_path):
if metric_path not in self.buffers:
log.debug("Allocating new metric buffer for %s" % metric_path)
self.buffers[metric_path] = MetricBuffer(metric_path)
return self.buffers[metric_path]
def successHandler(result, *args, **kw):
log.debug("Tagged %s in %s" % (', '.join(metrics), time.time() - t), type='tagdb')
return True
def successHandler(result, *args, **kw):
log.debug("Tagged %s: %s in %s" %
(metric, result, time.time() - t))
def get_buffer(self, metric_path):
if metric_path not in self.buffers:
log.debug("Allocating new metric buffer for %s" % metric_path)
self.buffers[metric_path] = MetricBuffer(metric_path)
return self.buffers[metric_path]
