def clientConnectionFailed(self, connector, reason): ReconnectingClientFactory.clientConnectionFailed(self, connector, reason) log.clients( "%s::clientConnectionFailed (%s:%d) %s" % (self, connector.host, connector.port, reason.getErrorMessage()) ) self.connectFailed.callback(dict(connector=connector, reason=reason)) self.connectFailed = Deferred()
def queueSpaceCallback(self, result): if self.queueFull.called: log.clients('%s send queue has space available' % self.connectedProtocol) self.queueFull = Deferred() self.queueFull.addCallback(self.queueFullCallback) self.queueHasSpace = Deferred() self.queueHasSpace.addCallback(self.queueSpaceCallback)
def clientConnectionLost(self, connector, reason): ReconnectingClientFactory.clientConnectionLost(self, connector, reason) log.clients( "%s::clientConnectionLost (%s:%d) %s" % (self, connector.host, connector.port, reason.getErrorMessage()) ) self.connectedProtocol = None self.connectionLost.callback(0) self.connectionLost = Deferred()
def sendDatapoint(self, metric, datapoint): instrumentation.increment(self.attemptedRelays) if len(self.queue) >= settings.MAX_QUEUE_SIZE: log.clients("%s::sendDatapoint send queue full, dropping datapoint") instrumentation.increment(self.fullQueueDrops) elif self.connectedProtocol: self.connectedProtocol.sendDatapoint(metric, datapoint) else: self.enqueue(metric, datapoint) instrumentation.increment(self.queuedUntilConnected)
def clientConnectionFailed(self, connector, reason): ReconnectingClientFactory.clientConnectionFailed(self, connector, reason) log.clients("%s::clientConnectionFailed (%s:%d) %s" % ( self, connector.host, connector.port, reason.getErrorMessage())) self.destinationDown(connector.factory.destination) args = dict(connector=connector, reason=reason) d = self.connectFailed self.connectFailed = Deferred() d.callback(args)
def sendQueued(self): while (not self.paused) and self.factory.hasQueuedDatapoints(): datapoints = self.factory.takeSomeFromQueue() self._sendDatapoints(datapoints) queueSize = self.factory.queueSize if (self.factory.queueFull.called and queueSize < SEND_QUEUE_LOW_WATERMARK): self.factory.queueHasSpace.callback(queueSize) if (settings.USE_FLOW_CONTROL and state.metricReceiversPaused): log.clients('%s resuming paused clients' % self) events.resumeReceivingMetrics()
def sendQueued(self): while (not self.paused) and self.factory.hasQueuedDatapoints(): datapoints = self.factory.takeSomeFromQueue() self.sendString( pickle.dumps(datapoints, protocol=-1) ) self.factory.checkQueue() instrumentation.increment(self.sent, len(datapoints)) if (settings.USE_FLOW_CONTROL and state.metricReceiversPaused and self.factory.queueSize < SEND_QUEUE_LOW_WATERMARK): log.clients('send queue has space available, resuming paused clients') events.resumeReceivingMetrics()
def clientConnectionLost(self, connector, reason): ReconnectingClientFactory.clientConnectionLost(self, connector, reason) log.clients("%s::clientConnectionLost (%s:%d) %s" % ( self, connector.host, connector.port, reason.getErrorMessage())) self.connectedProtocol = None self.destinationDown(self.destination) args = dict(connector=connector, reason=reason) d = self.connectionLost self.connectionLost = Deferred() d.callback(args)
def connectionMade(self): log.clients("%s::connectionMade" % self) self.paused = False self.connected = True self.transport.registerProducer(self, streaming=True) # Define internal metric names self.destinationName = self.factory.destinationName self.queuedUntilReady = 'destinations.%s.queuedUntilReady' % self.destinationName self.sent = 'destinations.%s.sent' % self.destinationName self.factory.connectionMade.callback(self) self.factory.connectionMade = Deferred() self.sendQueued()
def startClient(self, destination): if destination in self.client_factories: return log.clients("connecting to carbon daemon at %s:%d:%s" % destination) self.router.addDestination(destination) factory = self.client_factories[destination] = CarbonClientFactory(destination) connectAttempted = DeferredList( [factory.connectionMade, factory.connectFailed], fireOnOneCallback=True, fireOnOneErrback=True ) if self.running: factory.startConnecting() # this can trigger & replace connectFailed return connectAttempted
def resetConnectionForQualityReasons(self, reason): """Only re-sets the connection if it's been settings.MIN_RESET_INTERVAL seconds since the last re-set. Reason should be a string containing the quality info that led to a re-set. """ if (time() - self.lastResetTime) < float(settings.MIN_RESET_INTERVAL): return else: self.factory.connectedProtocol.disconnect() self.lastResetTime = time() instrumentation.increment(self.slowConnectionReset) log.clients("%s:: resetConnectionForQualityReasons: %s" % (self, reason))
def connectionMade(self): log.clients("%s::connectionMade" % self) self.paused = False self.connected = True self.transport.registerProducer(self, streaming=True) # Define internal metric names self.lastResetTime = time() self.destinationName = self.factory.destinationName self.queuedUntilReady = "destinations.%s.queuedUntilReady" % self.destinationName self.sent = "destinations.%s.sent" % self.destinationName self.batchesSent = "destinations.%s.batchesSent" % self.destinationName self.slowConnectionReset = "destinations.%s.slowConnectionReset" % self.destinationName self.factory.connectionMade.callback(self) self.factory.connectionMade = Deferred() self.sendQueued()
def startClient(self, destination): if destination in self.client_factories: return log.clients("connecting to carbon daemon at %s:%d:%s" % destination) if not settings.DYNAMIC_ROUTER: # If not using a dynamic router we add the destination before # it's known to be working. self.router.addDestination(destination) factory = self.createFactory(destination) self.client_factories[destination] = factory connectAttempted = DeferredList( [factory.connectionMade, factory.connectFailed], fireOnOneCallback=True, fireOnOneErrback=True) if self.running: factory.startConnecting() # this can trigger & replace connectFailed return connectAttempted
def connectionMade(self): log.clients("%s::connectionMade" % self) self.paused = False self.connected = True self.transport.registerProducer(self, streaming=True) # Define internal metric names self.lastResetTime = time() self.destinationName = self.factory.destinationName self.queuedUntilReady = 'destinations.%s.queuedUntilReady' % self.destinationName self.sent = 'destinations.%s.sent' % self.destinationName self.batchesSent = 'destinations.%s.batchesSent' % self.destinationName self.slowConnectionReset = 'destinations.%s.slowConnectionReset' % self.destinationName enableTcpKeepAlive(self.transport, settings.TCP_KEEPALIVE, settings) d = self.factory.connectionMade # Setup a new deferred before calling the callback to allow callbacks # to re-register themselves. self.factory.connectionMade = Deferred() d.callback(self) self.sendQueued()
def destinationDown(self, destination): # Only blacklist the destination if we tried a lot. log.clients("Destination is down: %s:%d:%s (%d/%d)" % ( destination[0], destination[1], destination[2], self.retries, settings.DYNAMIC_ROUTER_MAX_RETRIES)) # Retries comes from the ReconnectingClientFactory. if self.retries < settings.DYNAMIC_ROUTER_MAX_RETRIES: return if settings.DYNAMIC_ROUTER and self.router.hasDestination(destination): log.clients("Removing client %s:%d:%s to router" % destination) self.router.removeDestination(destination) # Do not receive more metrics if we don't have any usable destinations. if not self.router.countDestinations(): state.events.pauseReceivingMetrics() # Re-inject queued metrics. metrics = list(self.queue) log.clients("Re-injecting %d metrics from %s" % (len(metrics), self)) for metric, datapoint in metrics: state.events.metricGenerated(metric, datapoint) self.queue.clear()
def connectionLost(self, reason): log.clients("%s::connectionLost %s" % (self, reason.getErrorMessage())) self.connected = False
def startedConnecting(self, connector): log.clients("%s::startedConnecting (%s:%d)" % (self, connector.host, connector.port))
def queueFullCallback(self, result): state.events.cacheFull() log.clients('%s send queue is full (%d datapoints)' % (self, result))
def queueFullCallback(self, result): log.clients('%s send queue is full (%d datapoints)' % (self, result))
def clientConnectionMade(self, client): log.clients("%s::connectionMade (%s)" % (self, client)) self.resetDelay() self.destinationUp(client.factory.destination) self.connectionMade.addCallbacks(self.clientConnectionMade, log.err) return client
def destinationUp(self, destination): log.clients("Destination is up: %s:%d:%s" % destination) if not self.router.hasDestination(destination): log.clients("Adding client %s:%d:%s to router" % destination) self.router.addDestination(destination) state.events.resumeReceivingMetrics()
def reinjectDatapoints(self): metrics = list(self.queue) log.clients("Re-injecting %d metrics from %s" % (len(metrics), self)) for metric, datapoint in metrics: state.events.metricGenerated(metric, datapoint) self.queue.clear()