def clientConnectionFailed(self, connector, reason):
ReconnectingClientFactory.clientConnectionFailed(self, connector, reason)
log.clients(
"%s::clientConnectionFailed (%s:%d) %s" % (self, connector.host, connector.port, reason.getErrorMessage())
)
self.connectFailed.callback(dict(connector=connector, reason=reason))
self.connectFailed = Deferred()
def queueSpaceCallback(self, result):
if self.queueFull.called:
log.clients('%s send queue has space available' % self.connectedProtocol)
self.queueFull = Deferred()
self.queueFull.addCallback(self.queueFullCallback)
self.queueHasSpace = Deferred()
self.queueHasSpace.addCallback(self.queueSpaceCallback)
def clientConnectionLost(self, connector, reason):
ReconnectingClientFactory.clientConnectionLost(self, connector, reason)
log.clients(
"%s::clientConnectionLost (%s:%d) %s" % (self, connector.host, connector.port, reason.getErrorMessage())
)
self.connectedProtocol = None
self.connectionLost.callback(0)
self.connectionLost = Deferred()
def sendDatapoint(self, metric, datapoint):
instrumentation.increment(self.attemptedRelays)
if len(self.queue) >= settings.MAX_QUEUE_SIZE:
log.clients("%s::sendDatapoint send queue full, dropping datapoint")
instrumentation.increment(self.fullQueueDrops)
elif self.connectedProtocol:
self.connectedProtocol.sendDatapoint(metric, datapoint)
else:
self.enqueue(metric, datapoint)
instrumentation.increment(self.queuedUntilConnected)
def clientConnectionFailed(self, connector, reason):
ReconnectingClientFactory.clientConnectionFailed(self, connector, reason)
log.clients("%s::clientConnectionFailed (%s:%d) %s" % (
self, connector.host, connector.port, reason.getErrorMessage()))
self.destinationDown(connector.factory.destination)
args = dict(connector=connector, reason=reason)
d = self.connectFailed
self.connectFailed = Deferred()
d.callback(args)
def sendQueued(self):
while (not self.paused) and self.factory.hasQueuedDatapoints():
datapoints = self.factory.takeSomeFromQueue()
self._sendDatapoints(datapoints)
queueSize = self.factory.queueSize
if (self.factory.queueFull.called and queueSize < SEND_QUEUE_LOW_WATERMARK):
self.factory.queueHasSpace.callback(queueSize)
if (settings.USE_FLOW_CONTROL and state.metricReceiversPaused):
log.clients('%s resuming paused clients' % self)
events.resumeReceivingMetrics()
def sendQueued(self):
while (not self.paused) and self.factory.hasQueuedDatapoints():
datapoints = self.factory.takeSomeFromQueue()
self.sendString( pickle.dumps(datapoints, protocol=-1) )
self.factory.checkQueue()
instrumentation.increment(self.sent, len(datapoints))
if (settings.USE_FLOW_CONTROL and
state.metricReceiversPaused and
self.factory.queueSize < SEND_QUEUE_LOW_WATERMARK):
log.clients('send queue has space available, resuming paused clients')
events.resumeReceivingMetrics()
def clientConnectionLost(self, connector, reason):
ReconnectingClientFactory.clientConnectionLost(self, connector, reason)
log.clients("%s::clientConnectionLost (%s:%d) %s" % (
self, connector.host, connector.port, reason.getErrorMessage()))
self.connectedProtocol = None
self.destinationDown(self.destination)
args = dict(connector=connector, reason=reason)
d = self.connectionLost
self.connectionLost = Deferred()
d.callback(args)
def connectionMade(self):
log.clients("%s::connectionMade" % self)
self.paused = False
self.connected = True
self.transport.registerProducer(self, streaming=True)
# Define internal metric names
self.destinationName = self.factory.destinationName
self.queuedUntilReady = 'destinations.%s.queuedUntilReady' % self.destinationName
self.sent = 'destinations.%s.sent' % self.destinationName
self.factory.connectionMade.callback(self)
self.factory.connectionMade = Deferred()
self.sendQueued()
def sendQueued(self):
while (not self.paused) and self.factory.hasQueuedDatapoints():
datapoints = self.factory.takeSomeFromQueue()
self._sendDatapoints(datapoints)
queueSize = self.factory.queueSize
if (self.factory.queueFull.called
and queueSize < SEND_QUEUE_LOW_WATERMARK):
self.factory.queueHasSpace.callback(queueSize)
if (settings.USE_FLOW_CONTROL and state.metricReceiversPaused):
log.clients('%s resuming paused clients' % self)
events.resumeReceivingMetrics()
def startClient(self, destination):
if destination in self.client_factories:
return
log.clients("connecting to carbon daemon at %s:%d:%s" % destination)
self.router.addDestination(destination)
factory = self.client_factories[destination] = CarbonClientFactory(destination)
connectAttempted = DeferredList(
[factory.connectionMade, factory.connectFailed], fireOnOneCallback=True, fireOnOneErrback=True
)
if self.running:
factory.startConnecting() # this can trigger & replace connectFailed
return connectAttempted
def resetConnectionForQualityReasons(self, reason):
"""Only re-sets the connection if it's been
settings.MIN_RESET_INTERVAL seconds since the last re-set.
Reason should be a string containing the quality info that led to
a re-set.
"""
if (time() - self.lastResetTime) < float(settings.MIN_RESET_INTERVAL):
return
else:
self.factory.connectedProtocol.disconnect()
self.lastResetTime = time()
instrumentation.increment(self.slowConnectionReset)
log.clients("%s:: resetConnectionForQualityReasons: %s" % (self, reason))
def connectionMade(self):
log.clients("%s::connectionMade" % self)
self.paused = False
self.connected = True
self.transport.registerProducer(self, streaming=True)
# Define internal metric names
self.lastResetTime = time()
self.destinationName = self.factory.destinationName
self.queuedUntilReady = "destinations.%s.queuedUntilReady" % self.destinationName
self.sent = "destinations.%s.sent" % self.destinationName
self.batchesSent = "destinations.%s.batchesSent" % self.destinationName
self.slowConnectionReset = "destinations.%s.slowConnectionReset" % self.destinationName
self.factory.connectionMade.callback(self)
self.factory.connectionMade = Deferred()
self.sendQueued()
def startClient(self, destination):
if destination in self.client_factories:
return
log.clients("connecting to carbon daemon at %s:%d:%s" % destination)
if not settings.DYNAMIC_ROUTER:
# If not using a dynamic router we add the destination before
# it's known to be working.
self.router.addDestination(destination)
factory = self.createFactory(destination)
self.client_factories[destination] = factory
connectAttempted = DeferredList(
[factory.connectionMade, factory.connectFailed],
fireOnOneCallback=True,
fireOnOneErrback=True)
if self.running:
factory.startConnecting() # this can trigger & replace connectFailed
return connectAttempted
def connectionMade(self):
log.clients("%s::connectionMade" % self)
self.paused = False
self.connected = True
self.transport.registerProducer(self, streaming=True)
# Define internal metric names
self.lastResetTime = time()
self.destinationName = self.factory.destinationName
self.queuedUntilReady = 'destinations.%s.queuedUntilReady' % self.destinationName
self.sent = 'destinations.%s.sent' % self.destinationName
self.batchesSent = 'destinations.%s.batchesSent' % self.destinationName
self.slowConnectionReset = 'destinations.%s.slowConnectionReset' % self.destinationName
enableTcpKeepAlive(self.transport, settings.TCP_KEEPALIVE, settings)
d = self.factory.connectionMade
# Setup a new deferred before calling the callback to allow callbacks
# to re-register themselves.
self.factory.connectionMade = Deferred()
d.callback(self)
self.sendQueued()
def destinationDown(self, destination):
# Only blacklist the destination if we tried a lot.
log.clients("Destination is down: %s:%d:%s (%d/%d)" % (
destination[0], destination[1], destination[2], self.retries,
settings.DYNAMIC_ROUTER_MAX_RETRIES))
# Retries comes from the ReconnectingClientFactory.
if self.retries < settings.DYNAMIC_ROUTER_MAX_RETRIES:
return
if settings.DYNAMIC_ROUTER and self.router.hasDestination(destination):
log.clients("Removing client %s:%d:%s to router" % destination)
self.router.removeDestination(destination)
# Do not receive more metrics if we don't have any usable destinations.
if not self.router.countDestinations():
state.events.pauseReceivingMetrics()
# Re-inject queued metrics.
metrics = list(self.queue)
log.clients("Re-injecting %d metrics from %s" % (len(metrics), self))
for metric, datapoint in metrics:
state.events.metricGenerated(metric, datapoint)
self.queue.clear()
def connectionLost(self, reason):
log.clients("%s::connectionLost %s" % (self, reason.getErrorMessage()))
self.connected = False
def startedConnecting(self, connector):
log.clients("%s::startedConnecting (%s:%d)" % (self, connector.host, connector.port))
def queueFullCallback(self, result):
state.events.cacheFull()
log.clients('%s send queue is full (%d datapoints)' % (self, result))
def queueFullCallback(self, result):
log.clients('%s send queue is full (%d datapoints)' % (self, result))
def clientConnectionMade(self, client):
log.clients("%s::connectionMade (%s)" % (self, client))
self.resetDelay()
self.destinationUp(client.factory.destination)
self.connectionMade.addCallbacks(self.clientConnectionMade, log.err)
return client
def destinationUp(self, destination):
log.clients("Destination is up: %s:%d:%s" % destination)
if not self.router.hasDestination(destination):
log.clients("Adding client %s:%d:%s to router" % destination)
self.router.addDestination(destination)
state.events.resumeReceivingMetrics()
def reinjectDatapoints(self):
metrics = list(self.queue)
log.clients("Re-injecting %d metrics from %s" % (len(metrics), self))
for metric, datapoint in metrics:
state.events.metricGenerated(metric, datapoint)
self.queue.clear()
