def _handleBatch(engine, messages, messageRxTimes, metricStreamer,
modelSwapper):
"""Process a batch of messages from the queue.
This parses the message contents as JSON and uses the 'protocol' field to
determine how to parse the 'data' in the message. The data is added to the
database and sent through the metric streamer.
The Metric objects are cached in gCustomMetrics to minimize database
lookups.
:param engine: SQLAlchemy engine object
:type engine: sqlalchemy.engine.Engine
:param messages: a list of queue messages to process
:param messageRxTimes: optional sequence of message-receive times (from
time.time()) if profiling corresponding to the messages in `messages` arg,
else empty list
:param metricStreamer: a :class:`MetricStreamer` instance to use
:param modelSwapper: a :class:`ModelSwapperInterface` instance to use
"""
# Use the protocol to determine the message format
data = []
for m, rxTime in itertools.izip_longest(messages, messageRxTimes):
try:
message = json.loads(m.body)
protocol = message["protocol"]
rawData = message["data"]
except ValueError:
LOGGER.warn("Discarding message with unknown format: %s", m.body)
return
if protocol == Protocol.PLAIN:
for row in rawData:
try:
data.append(parsePlaintext(row))
if gProfiling and rxTime is not None:
metricName, _value, metricTimestamp = data[-1]
LOGGER.info(
"{TAG:CUSSTR.DATA.RX} metricName=%s; timestamp=%s; rxTime=%.4f",
metricName, metricTimestamp.isoformat() + "Z", rxTime)
except ValueError:
LOGGER.warn("Discarding plaintext message that can't be parsed: %s",
row.strip())
else:
LOGGER.warn("Discarding message with unknown protocol: %s", protocol)
return
# Make sure we got some valid data
if not data:
return
# Create a dict mapping metric name to data list
dataDict = defaultdict(list)
for record in data:
dataDict[record[0]].append(record)
LOGGER.info("Processing %i records for %i models from %i batches.",
len(data), len(dataDict), len(messages))
# For each metric, create the metric if it doesn't exist and add the data
_addMetricData(engine, dataDict, metricStreamer, modelSwapper)
def _handleBatch(engine, messages, messageRxTimes, metricStreamer,
modelSwapper):
"""Process a batch of messages from the queue.
This parses the message contents as JSON and uses the 'protocol' field to
determine how to parse the 'data' in the message. The data is added to the
database and sent through the metric streamer.
The Metric objects are cached in gCustomMetrics to minimize database
lookups.
:param engine: SQLAlchemy engine object
:type engine: sqlalchemy.engine.Engine
:param messages: a list of queue messages to process
:param messageRxTimes: optional sequence of message-receive times (from
time.time()) if profiling corresponding to the messages in `messages` arg,
else empty list
:param metricStreamer: a :class:`MetricStreamer` instance to use
:param modelSwapper: a :class:`ModelSwapperInterface` instance to use
"""
# Use the protocol to determine the message format
data = []
for m, rxTime in itertools.izip_longest(messages, messageRxTimes):
try:
message = json.loads(m.body)
protocol = message["protocol"]
rawData = message["data"]
except ValueError:
LOGGER.warn("Discarding message with unknown format: %s", m.body)
return
if protocol == Protocol.PLAIN:
for row in rawData:
try:
data.append(parsePlaintext(row))
if gProfiling and rxTime is not None:
metricName, _value, metricTimestamp = data[-1]
LOGGER.info(
"{TAG:CUSSTR.DATA.RX} metricName=%s; timestamp=%s; rxTime=%.4f",
metricName, metricTimestamp.isoformat() + "Z", rxTime)
except ValueError:
LOGGER.warn("Discarding plaintext message that can't be parsed: %s",
row.strip())
else:
LOGGER.warn("Discarding message with unknown protocol: %s", protocol)
return
# Make sure we got some valid data
if not data:
return
# Create a dict mapping metric name to data list
dataDict = defaultdict(list)
for record in data:
dataDict[record[0]].append(record)
LOGGER.info("Processing %i records for %i models from %i batches.",
len(data), len(dataDict), len(messages))
# For each metric, create the metric if it doesn't exist and add the data
_addMetricData(engine, dataDict, metricStreamer, modelSwapper)
def testParsePlaintext(self): data = "test.metric 4.0 1386792175" result = metric_listener.parsePlaintext(data) self.assertEqual(len(result), 3) name, value, dt = result self.assertEqual(name, "test.metric") self.assertAlmostEqual(value, 4.0) self.assertEqual(dt.year, 2013) self.assertEqual(dt.month, 12) self.assertEqual(dt.day, 11) self.assertEqual(dt.hour, 20) self.assertEqual(dt.minute, 2) self.assertEqual(dt.second, 55)
