def stringReceived(self, data):
try:
datapoints = self.unpickler.loads(data)
# Pickle can throw a wide range of exceptions
except (pickle.UnpicklingError, ValueError, IndexError, ImportError, KeyError):
log.listener('invalid pickle received from %s, ignoring' % self.peerName)
return
for raw in datapoints:
try:
(metric, (value, timestamp)) = raw
except Exception as e:
log.listener('Error decoding pickle: %s' % e)
continue
try:
datapoint = (float(value), float(timestamp)) # force proper types
except (ValueError, TypeError):
continue
# convert python2 unicode objects to str/bytes
if not isinstance(metric, str):
metric = metric.encode('utf-8')
self.metricReceived(metric, datapoint)
def setup(self):
exchange = self.factory.exchange_name
yield self.authenticate(self.factory.username, self.factory.password)
chan = yield self.channel(1)
yield chan.channel_open()
# declare the exchange and queue
yield chan.exchange_declare(exchange=exchange,
type="topic",
durable=True,
auto_delete=False)
# we use a private queue to avoid conflicting with existing bindings
reply = yield chan.queue_declare(exclusive=True)
my_queue = reply.queue
# bind each configured metric pattern
for bind_pattern in settings.BIND_PATTERNS:
log.listener("binding exchange '%s' to queue '%s' with pattern %s" \
% (exchange, my_queue, bind_pattern))
yield chan.queue_bind(exchange=exchange,
queue=my_queue,
routing_key=bind_pattern)
yield chan.basic_consume(queue=my_queue,
no_ack=True,
consumer_tag=self.consumer_tag)
def setup(self):
exchange = self.factory.exchange_name
yield self.authenticate(self.factory.username, self.factory.password)
chan = yield self.channel(2)
yield chan.channel_open()
# declare the exchange and queue
yield chan.exchange_declare(exchange=exchange, type="topic",
durable=True, auto_delete=False)
# we use a private queue to avoid conflicting with existing bindings
reply = yield chan.queue_declare(queue = settings.AMQP_QUEUE, durable = True)
my_queue = reply.queue
# bind each configured metric pattern
for bind_pattern in settings.BIND_PATTERNS:
log.listener("binding exchange '%s' to queue '%s' with pattern %s" \
% (exchange, my_queue, bind_pattern))
yield chan.queue_bind(exchange=exchange, queue=my_queue,
routing_key=bind_pattern)
# def callback(ch, method, properties, body):
# log.listener("Received %r"%(body))
# log.listener("Routing_key %s"%(method.routing_key))
# ch.basic_ack(delivery_tag=method.delivery_tag)
yield chan.basic_qos(prefetch_count = 1000)
yield chan.basic_consume(queue=my_queue, no_ack=False,
consumer_tag=self.consumer_tag)
def setup(self):
exchange = self.factory.exchange_name
yield self.authenticate(self.factory.username, self.factory.password)
chan = yield self.channel(2)
yield chan.channel_open()
# declare the exchange and queue
yield chan.exchange_declare(exchange=exchange, type="topic",
durable=True, auto_delete=False)
# we use a private queue to avoid conflicting with existing bindings
reply = yield chan.queue_declare(queue = settings.AMQP_QUEUE, durable = True)
my_queue = reply.queue
# bind each configured metric pattern
for bind_pattern in settings.BIND_PATTERNS:
log.listener("binding exchange '%s' to queue '%s' with pattern %s" \
% (exchange, my_queue, bind_pattern))
yield chan.queue_bind(exchange=exchange, queue=my_queue,
routing_key=bind_pattern)
# def callback(ch, method, properties, body):
# log.listener("Received %r"%(body))
# log.listener("Routing_key %s"%(method.routing_key))
# ch.basic_ack(delivery_tag=method.delivery_tag)
yield chan.basic_qos(prefetch_count = 1)
yield chan.basic_consume(queue=my_queue, no_ack=False,
consumer_tag=self.consumer_tag)
def connectionLost(self, reason):
if reason.check(ConnectionDone):
log.listener("%s connection with %s closed cleanly" %
(self.__class__.__name__, self.peerAddr))
else:
log.listener(
"%s connection with %s lost: %s" %
(self.__class__.__name__, self.peerAddr, reason.value))
def receive_loop(self):
queue = yield self.queue("%s_%s" %
(self.consumer_tag, self.factory.queue_name))
while True:
msg = yield queue.get()
log.listener("Got msg: %s" % msg)
self.processMessage(msg)
def connectionLost(self, reason):
if reason.check(ConnectionDone):
log.listener("%s connection with %s closed cleanly" % (self.__class__.__name__, self.peerName))
else:
log.listener("%s connection with %s lost: %s" % (self.__class__.__name__, self.peerName, reason.value))
state.connectedMetricReceiverProtocols.remove(self)
events.pauseReceivingMetrics.removeHandler(self.pauseReceiving)
events.resumeReceivingMetrics.removeHandler(self.resumeReceiving)
def lineReceived(self, line):
try:
metric, value, timestamp = line.strip().split()
datapoint = ( float(timestamp), float(value) )
except:
log.listener('invalid line received from client %s, ignoring' % self.peerName)
return
self.metricReceived(metric, datapoint)
def lineReceived(self, line):
try:
metric, value, timestamp = line.strip().split()
datapoint = ( float(timestamp), float(value) )
except:
log.listener('invalid line received from client %s, ignoring' % self.peerName)
return
self.metricReceived(metric, datapoint)
def connectionLost(self, reason):
if reason.check(ConnectionDone):
log.listener("%s connection with %s closed cleanly" % (self.__class__.__name__, self.peerName))
else:
log.listener("%s connection with %s lost: %s" % (self.__class__.__name__, self.peerName, reason.value))
state.connectedMetricReceiverProtocols.remove(self)
events.pauseReceivingMetrics.removeHandler(self.pauseReceiving)
events.resumeReceivingMetrics.removeHandler(self.resumeReceiving)
def connectionMade(self):
self.peerName = self.getPeerName()
log.listener("%s connection with %s established" % (self.__class__.__name__, self.peerName))
if state.metricReceiversPaused:
self.pauseReceiving()
state.connectedMetricReceiverProtocols.add(self)
events.pauseReceivingMetrics.addHandler(self.pauseReceiving)
events.resumeReceivingMetrics.addHandler(self.resumeReceiving)
def connectionMade(self):
self.peerName = self.getPeerName()
log.listener("%s connection with %s established" % (self.__class__.__name__, self.peerName))
if state.metricReceiversPaused:
self.pauseReceiving()
state.connectedMetricReceiverProtocols.add(self)
events.pauseReceivingMetrics.addHandler(self.pauseReceiving)
events.resumeReceivingMetrics.addHandler(self.resumeReceiving)
def stringReceived(self, data):
try:
payload_pb = Payload.FromString(data)
except DecodeError:
log.listener('invalid protobuf received from %s, ignoring' % self.peerName)
return
for metric_pb in payload_pb.metrics:
for point_pb in metric_pb.points:
self.metricReceived(
metric_pb.metric, (point_pb.timestamp, point_pb.value))
def lineReceived(self, line):
try:
metric, value, timestamp = line.strip().split()
datapoint = (float(timestamp), float(value))
except ValueError:
if len(line) > 400:
line = line[:400] + '...'
log.listener('invalid line received from client %s, ignoring [%s]' % (self.peerName, line.strip().encode('string_escape')))
return
self.metricReceived(metric, datapoint)
def lineReceived(self, line):
try:
metric, value, timestamp = line.strip().split()
datapoint = ( float(timestamp), float(value) )
except:
log.listener('invalid line received from client %s, disconnecting' % self.peerAddr)
self.transport.loseConnection()
return
increment('metricsReceived')
metricReceived(metric, datapoint)
def connectionLost(self, reason):
if reason.check(ConnectionDone):
if settings.LOG_LISTENER_CONN_SUCCESS:
log.listener("%s connection with %s closed cleanly" % (self.__class__.__name__, self.peerName))
else:
log.listener("%s connection with %s lost: %s" % (self.__class__.__name__, self.peerName, reason.value))
state.connectedMetricReceiverProtocols.remove(self)
if settings.USE_FLOW_CONTROL:
events.pauseReceivingMetrics.removeHandler(self.pauseReceiving)
events.resumeReceivingMetrics.removeHandler(self.resumeReceiving)
def stringReceived(self, data):
try:
payload_pb = Payload.FromString(data)
except DecodeError:
log.listener('invalid protobuf received from %s, ignoring' %
self.peerName)
return
for metric_pb in payload_pb.metrics:
for point_pb in metric_pb.points:
self.metricReceived(metric_pb.metric,
(point_pb.timestamp, point_pb.value))
def connectionMade(self):
self.peerName = self.getPeerName()
if settings.LOG_LISTENER_CONN_SUCCESS:
log.listener("%s connection with %s established" % (self.__class__.__name__, self.peerName))
if state.metricReceiversPaused:
self.pauseReceiving()
state.connectedMetricReceiverProtocols.add(self)
if settings.USE_FLOW_CONTROL:
events.pauseReceivingMetrics.addHandler(self.pauseReceiving)
events.resumeReceivingMetrics.addHandler(self.resumeReceiving)
def connectionMade(self):
self.peerName = self.getPeerName()
if settings.LOG_LISTENER_CONN_SUCCESS:
log.listener("%s connection with %s established" %
(self.__class__.__name__, self.peerName))
if state.metricReceiversPaused:
self.pauseReceiving()
state.connectedMetricReceiverProtocols.add(self)
if settings.USE_FLOW_CONTROL:
events.pauseReceivingMetrics.addHandler(self.pauseReceiving)
events.resumeReceivingMetrics.addHandler(self.resumeReceiving)
def stringReceived(self, data):
try:
datapoints = self.unpickler.loads(data)
except:
log.listener('invalid pickle received from %s, ignoring' % self.peerName)
return
for (metric, datapoint) in datapoints:
try:
datapoint = ( float(datapoint[0]), float(datapoint[1]) ) #force proper types
except:
continue
self.metricReceived(metric, datapoint)
def lineReceived(self, line):
try:
metric, value, timestamp = line.strip().split()
except ValueError as error:
log.listener("invalid line '%s' from %s. Format must be 'metric_name metric_value metric_timestamp'" % ( line.strip(), self.peerName))
return
try:
datapoint = ( float(timestamp), float(value) )
except ValueError as error:
log.listener("invalid line '%s' from %s. Both metric_value and metric_timestamp must be floats" % ( line.strip(), self.peerName))
return
self.metricReceived(metric, datapoint)
def stringReceived(self, data):
try:
datapoints = pickle.loads(data)
except:
log.listener('invalid pickle received from client %s, disconnecting' % self.peerAddr)
self.transport.loseConnection()
return
for (metric, datapoint) in datapoints:
datapoint = ( float(datapoint[0]), float(datapoint[1]) ) #force proper types
if datapoint[1] == datapoint[1]: # filter out NaN values
metricReceived(metric, datapoint)
increment('metricsReceived', len(datapoints))
def stringReceived(self, data):
try:
datapoints = self.unpickler.loads(data)
except pickle.UnpicklingError:
log.listener('invalid pickle received from %s, ignoring' % self.peerName)
return
for (metric, datapoint) in datapoints:
try:
datapoint = ( float(datapoint[0]), float(datapoint[1]) ) #force proper types
except ValueError:
continue
self.metricReceived(metric, datapoint)
def lineReceived(self, line):
if sys.version_info >= (3, 0):
line = line.decode('utf-8')
try:
metric, value, timestamp = line.strip().split()
datapoint = (float(timestamp), float(value))
except ValueError:
if len(line) > 400:
line = line[:400] + '...'
log.listener('invalid line received from client %s, ignoring [%s]' %
(self.peerName, repr(line.strip())[1:-1]))
return
self.metricReceived(metric, datapoint)
def datagramReceived(self, data, addr):
(host, _) = addr
if sys.version_info >= (3, 0):
data = data.decode('utf-8')
for line in data.splitlines():
try:
metric, value, timestamp = line.strip().split()
datapoint = (float(timestamp), float(value))
self.metricReceived(metric, datapoint)
except ValueError:
if len(line) > 400:
line = line[:400] + '...'
log.listener('invalid line received from %s, ignoring [%s]' %
(host, repr(line.strip())[1:-1]))
def connectionMade(self):
self.setTimeout(settings.METRIC_CLIENT_IDLE_TIMEOUT)
enableTcpKeepAlive(self.transport, settings.TCP_KEEPALIVE, settings)
self.peerName = self.getPeerName()
if settings.LOG_LISTENER_CONN_SUCCESS:
log.listener("%s connection with %s established" % (
self.__class__.__name__, self.peerName))
if state.metricReceiversPaused:
self.pauseReceiving()
state.connectedMetricReceiverProtocols.add(self)
if settings.USE_FLOW_CONTROL:
events.pauseReceivingMetrics.addHandler(self.pauseReceiving)
events.resumeReceivingMetrics.addHandler(self.resumeReceiving)
def stringReceived(self, data):
try:
datapoints = self.unpickler.loads(data)
except:
log.listener('invalid pickle received from %s, ignoring' % self.peerName)
return
for raw in datapoints:
try:
(metric, (value, timestamp)) = raw
except Exception, e:
log.listener('Error decoding pickle: %s' % e)
try:
datapoint = (float(value), float(timestamp)) # force proper types
except:
continue
self.metricReceived(metric, datapoint)
def lineReceived(self, line):
try:
metric, value, timestamp = line.strip().split()
except ValueError as error:
log.listener(
"invalid line '%s' from %s. Format must be 'metric_name metric_value metric_timestamp'"
% (line.strip(), self.peerName))
return
try:
datapoint = (float(timestamp), float(value))
except ValueError as error:
log.listener(
"invalid line '%s' from %s. Both metric_value and metric_timestamp must be floats"
% (line.strip(), self.peerName))
return
self.metricReceived(metric, datapoint)
def checkIfAcceptingConnections():
clients = len(state.connectedMetricReceiverProtocols)
max_clients = settings.MAX_RECEIVER_CONNECTIONS
if clients < max_clients:
for port in state.listeningPorts:
if port.paused:
log.listener(
"Resuming %s (%d/%d connections)" % (port, clients, max_clients))
port.resumeProducing()
port.paused = False
else:
for port in state.listeningPorts:
if not port.paused:
log.listener(
"Pausing %s (%d/%d connections)" % (port, clients, max_clients))
port.pauseProducing()
port.paused = True
class MetricDatagramReceiver(MetricReceiver, DatagramProtocol):
def datagramReceived(self, data, (host, port)):
for line in data.splitlines():
try:
metric, value, timestamp = line.strip().split()
datapoint = (float(timestamp), float(value))
self.metricReceived(metric, datapoint)
except:
log.listener('invalid line received from %s, ignoring' % host)
def stringReceived(self, data):
try:
datapoints = pickle.loads(data)
except:
log.listener('invalid pickle received from client %s, ignoring' %
self.peerAddr)
return
for (metric, datapoint) in datapoints:
try:
datapoint = (float(datapoint[0]), float(datapoint[1])
) #force proper types
except:
continue
if datapoint[1] == datapoint[1]: # filter out NaN values
metricReceived(metric, datapoint)
increment('metricsReceived', len(datapoints))
class MetricDatagramReceiver(MetricReceiver, DatagramProtocol):
def datagramReceived(self, data, (host, port)):
for line in data.splitlines():
try:
metric, value, timestamp = line.strip().split()
except:
log.listener(
"invalid line '%s' from %s. Format must be 'metric_name metric_value metric_timestamp'"
% (line.strip(), host))
continue
try:
datapoint = (float(timestamp), float(value))
except ValueError as error:
log.listener(
"invalid line '%s' from %s. Both metric_value and metric_timestamp must be floats"
% (line.strip(), self.peerName))
continue
self.metricReceived(metric, datapoint)
def lineReceived(self, line):
try:
points = line.strip().split()
l_points = len(points)
assert l_points == 3 or l_points == 4
if l_points == 4:
assert points[0] == 'incr' or points[0] == 'decr'
action = 1 if points[0] == 'incr' else -1
metric, value, timestamp = points[1:]
datapoint = (float(timestamp), float(value), action)
else:
metric, value, timestamp = points
datapoint = ( float(timestamp), float(value), 0)
except:
log.listener('invalid line received from client %s, ignoring' % self.peerName)
return
self.metricReceived(metric, datapoint)
def setup(self):
exchange = self.factory.exchange_name
yield self.authenticate(self.factory.username, self.factory.password)
chan = yield self.channel(1)
yield chan.channel_open()
# declare the exchange and queue
yield chan.exchange_declare(exchange=exchange, type="topic", durable=True, auto_delete=False)
# we use a private queue to avoid conflicting with existing bindings
reply = yield chan.queue_declare(exclusive=True)
my_queue = reply.queue
# bind each configured metric pattern
for bind_pattern in settings.BIND_PATTERNS:
log.listener("binding exchange '%s' to queue '%s' with pattern %s" % (exchange, my_queue, bind_pattern))
yield chan.queue_bind(exchange=exchange, queue=my_queue, routing_key=bind_pattern)
yield chan.basic_consume(queue=my_queue, no_ack=True, consumer_tag=self.consumer_tag)
def stringReceived(self, data):
try:
datapoints = self.unpickler.loads(data)
except pickle.UnpicklingError:
log.listener('invalid pickle received from %s, ignoring' %
self.peerName)
return
for raw in datapoints:
try:
(metric, (value, timestamp)) = raw
except Exception, e:
log.listener('Error decoding pickle: %s' % e)
try:
datapoint = (float(value), float(timestamp)
) # force proper types
except ValueError:
continue
self.metricReceived(metric, datapoint)
class MetricDatagramReceiver(MetricReceiver, DatagramProtocol):
def datagramReceived(self, data, (host, port)):
for line in data.splitlines():
try:
metric, value, timestamp = line.strip().split()
datapoint = ( float(timestamp), float(value) )
self.metricReceived(metric, datapoint)
except ValueError:
if len(line) > 400:
line = line[:400] + '...'
log.listener('invalid line received from %s, ignoring [%s]' % (host, line.strip().encode('string_escape')))
def processMessage(self, message, channel):
"""Parse a message and post it as a metric."""
if self.factory.verbose:
log.listener("Message received: %s" % (message,))
metric = message.routing_key
for line in message.content.body.split("\n"):
line = line.strip()
if not line:
continue
try:
#log.listener("Trying...")
# if settings.get("AMQP_METRIC_NAME_IN_BODY", False):
# metric, value, timestamp = line.split()
# log.listener("Metric in body")
# else:
# log.listener("Metric not in body")
value, timestamp = line.split()
#log.listener("Value:%f Timestamp:%f"%(float(value),float(timestamp)))
datapoint = ( float(timestamp), float(value) )
except ValueError:
log.listener("invalid message line: %s" % (line,))
continue
events.metricReceived(metric, datapoint)
if self.factory.verbose:
log.listener("Metric posted: %s %s %s" %
(metric, value, timestamp,))
log.msg("Acking...")
channel.basic_ack(delivery_tag = message.delivery_tag, multiple = False)
log.msg("Ack Done!!")
def processMessage(self, message):
"""Parse a message and post it as a metric."""
if self.factory.verbose:
log.listener("Message received: %s" % (message, ))
metric = message.routing_key
for line in message.content.body.split("\n"):
line = line.strip()
if not line:
continue
try:
if settings.get("AMQP_METRIC_NAME_IN_BODY", False):
metric, value, timestamp = line.split()
else:
value, timestamp = line.split()
datapoint = (float(timestamp), float(value))
if datapoint[1] != datapoint[1]: # filter out NaN values
continue
except ValueError:
log.listener("invalid message line: %s" % (line, ))
continue
events.metricReceived(metric, datapoint)
if self.factory.verbose:
log.listener("Metric posted: %s %s %s" % (
metric,
value,
timestamp,
))
def processMessage(self, message, channel):
"""Parse a message and post it as a metric."""
if self.factory.verbose:
log.listener("Message received: %s" % (message,))
metric = message.routing_key
for line in message.content.body.split("\n"):
line = line.strip()
if not line:
continue
try:
#log.listener("Trying...")
# if settings.get("AMQP_METRIC_NAME_IN_BODY", False):
# metric, value, timestamp = line.split()
# log.listener("Metric in body")
# else:
# log.listener("Metric not in body")
value, timestamp = line.split()
#log.listener("Value:%f Timestamp:%f"%(float(value),float(timestamp)))
datapoint = ( float(timestamp), float(value) )
except ValueError:
log.listener("invalid message line: %s" % (line,))
continue
events.metricReceived(metric, datapoint)
if self.factory.verbose:
log.listener("Metric posted: %s %s %s" %
(metric, value, timestamp,))
channel.basic_ack(delivery_tag = message.delivery_tag, multiple = False)
def processMessage(self, message):
"""Parse a message and post it as a metric."""
if self.factory.verbose:
log.listener("Message received: %s" % (message,))
metric = message.routing_key
for line in message.content.body.split("\n"):
try:
if settings.get("AMQP_METRIC_NAME_IN_BODY", False):
metric, value, timestamp = line.strip().split()
else:
value, timestamp = line.strip().split()
datapoint = ( float(timestamp), float(value) )
except ValueError:
log.listener("invalid message line: %s" % (line,))
continue
increment('metricsReceived')
metricReceived(metric, datapoint)
if self.factory.verbose:
log.listener("Metric posted: %s %s %s" %
(metric, value, timestamp,))
class MetricDatagramReceiver(MetricReceiver, DatagramProtocol):
plugin_name = "udp"
@classmethod
def build(cls, root_service):
if not settings.ENABLE_UDP_LISTENER:
return
super(MetricDatagramReceiver, cls).build(root_service)
def datagramReceived(self, data, (host, port)):
for line in data.splitlines():
try:
metric, value, timestamp = line.strip().split()
datapoint = (float(timestamp), float(value))
self.metricReceived(metric, datapoint)
except ValueError:
if len(line) > 400:
line = line[:400] + '...'
log.listener('invalid line received from %s, ignoring [%s]' %
(host, line.strip().encode('string_escape')))
def connectionLost(self, reason):
if reason.check(ConnectionDone):
log.listener("%s connection with %s closed cleanly" % (self.__class__.__name__, self.peerAddr))
else:
log.listener("%s connection with %s lost: %s" % (self.__class__.__name__, self.peerAddr, reason.value))
self.factory.protocolDisconnected(self)
def connectionMade(self):
AMQClient.connectionMade(self)
log.listener("New AMQP connection made")
self.setup()
wfd = waitForDeferred(self.receive_loop())
yield wfd
def resumeAll(self):
log.listener("ProtocolManager.resumeAll")
for p in self.connectedProtocols:
p.transport.resumeProducing()
self.clientsPaused = False
def connectionMade(self):
self.peer = self.transport.getPeer()
self.peerAddr = "%s:%d" % (self.peer.host, self.peer.port)
log.listener("%s connection with %s established" %
(self.__class__.__name__, self.peerAddr))
def pauseAll(self):
log.listener("ProtocolManager.pauseAll")
for p in self.connectedProtocols:
p.transport.pauseProducing()
self.clientsPaused = True
def connectionMade(self):
yield AMQClient.connectionMade(self)
log.listener("New AMQP connection made")
yield self.setup()
yield self.receive_loop()
def connectionMade(self):
yield AMQClient.connectionMade(self)
log.listener("New AMQP connection made")
yield self.setup()
yield self.receive_loop()
def connectionMade(self):
self.peer = self.transport.getPeer()
self.peerAddr = "%s:%d" % (self.peer.host, self.peer.port)
log.listener("%s connection with %s established" % (self.__class__.__name__, self.peerAddr))
self.factory.protocolConnected(self)