def _handleBatch(engine, messages, messageRxTimes, metricStreamer, modelSwapper): """Process a batch of messages from the queue. This parses the message contents as JSON and uses the 'protocol' field to determine how to parse the 'data' in the message. The data is added to the database and sent through the metric streamer. The Metric objects are cached in gCustomMetrics to minimize database lookups. :param engine: SQLAlchemy engine object :type engine: sqlalchemy.engine.Engine :param messages: a list of queue messages to process :param messageRxTimes: optional sequence of message-receive times (from time.time()) if profiling corresponding to the messages in `messages` arg, else empty list :param metricStreamer: a :class:`MetricStreamer` instance to use :param modelSwapper: a :class:`ModelSwapperInterface` instance to use """ # Use the protocol to determine the message format data = [] for m, rxTime in itertools.izip_longest(messages, messageRxTimes): try: message = json.loads(m.body) protocol = message["protocol"] rawData = message["data"] except ValueError: LOGGER.warn("Discarding message with unknown format: %s", m.body) return if protocol == Protocol.PLAIN: for row in rawData: try: data.append(parsePlaintext(row)) if gProfiling and rxTime is not None: metricName, _value, metricTimestamp = data[-1] LOGGER.info( "{TAG:CUSSTR.DATA.RX} metricName=%s; timestamp=%s; rxTime=%.4f", metricName, metricTimestamp.isoformat() + "Z", rxTime) except ValueError: LOGGER.warn("Discarding plaintext message that can't be parsed: %s", row.strip()) else: LOGGER.warn("Discarding message with unknown protocol: %s", protocol) return # Make sure we got some valid data if not data: return # Create a dict mapping metric name to data list dataDict = defaultdict(list) for record in data: dataDict[record[0]].append(record) LOGGER.info("Processing %i records for %i models from %i batches.", len(data), len(dataDict), len(messages)) # For each metric, create the metric if it doesn't exist and add the data _addMetricData(engine, dataDict, metricStreamer, modelSwapper)
def testParsePlaintext(self): data = "test.metric 4.0 1386792175" result = metric_listener.parsePlaintext(data) self.assertEqual(len(result), 3) name, value, dt = result self.assertEqual(name, "test.metric") self.assertAlmostEqual(value, 4.0) self.assertEqual(dt.year, 2013) self.assertEqual(dt.month, 12) self.assertEqual(dt.day, 11) self.assertEqual(dt.hour, 20) self.assertEqual(dt.minute, 2) self.assertEqual(dt.second, 55)