def testMetricDataBatchWrite(self, messageBusConnectorClassMock): samples = [ ("FOO.BAR.%d" % i, i * 3.789, i * 300) for i in xrange((metric_utils._METRIC_DATA_BATCH_WRITE_SIZE * 3) / 2) ] messageBusConnectorClass = ( metric_utils.message_bus_connector.MessageBusConnector) messageBusMock = MagicMock( spec_set=messageBusConnectorClass, publish=Mock(spec_set=messageBusConnectorClass.publish)) messageBusMock.__enter__.return_value = messageBusMock messageBusConnectorClassMock.return_value = messageBusMock loggerMock = Mock(spec_set=logging.Logger) with metric_utils.metricDataBatchWrite(loggerMock) as putSample: # put enough for the first batch for sample in samples[:metric_utils._METRIC_DATA_BATCH_WRITE_SIZE]: putSample(*sample) # The first publish call should be for a full batch self.assertEqual(messageBusMock.publish.call_count, 1) call0 = mock.call( mqName="taurus.metric.custom.data", persistent=True, body=json.dumps( dict(protocol="plain", data=[ "%s %r %d" % (m, v, t) for m, v, t in samples[:metric_utils. _METRIC_DATA_BATCH_WRITE_SIZE] ]))) self.assertEqual(messageBusMock.publish.call_args_list[0], call0) # put the remaining samples for sample in samples[metric_utils._METRIC_DATA_BATCH_WRITE_SIZE:]: putSample(*sample) # the remaining incomplete batch will be sent upon exit from the context, # but not yet self.assertEqual(messageBusMock.publish.call_count, 1) # Now, the remainder should be sent, too self.assertEqual(messageBusMock.publish.call_count, 2) call1 = mock.call( mqName="taurus.metric.custom.data", persistent=True, body=json.dumps( dict(protocol="plain", data=[ "%s %r %d" % (m, v, t) for m, v, t in samples[metric_utils._METRIC_DATA_BATCH_WRITE_SIZE:] ]))) self.assertEqual(messageBusMock.publish.call_args_list[1], call1)
def testMetricDataBatchWrite(self, messageBusConnectorClassMock): samples = [ ("FOO.BAR.%d" % i, i * 3.789, i * 300) for i in xrange((metric_utils._METRIC_DATA_BATCH_WRITE_SIZE * 3) / 2) ] messageBusConnectorClass = ( metric_utils.message_bus_connector.MessageBusConnector) messageBusMock = MagicMock( spec_set=messageBusConnectorClass, publish=Mock(spec_set=messageBusConnectorClass.publish)) messageBusMock.__enter__.return_value = messageBusMock messageBusConnectorClassMock.return_value = messageBusMock loggerMock = Mock(spec_set=logging.Logger) with metric_utils.metricDataBatchWrite(loggerMock) as putSample: # put enough for the first batch for sample in samples[:metric_utils._METRIC_DATA_BATCH_WRITE_SIZE]: putSample(*sample) # The first publish call should be for a full batch self.assertEqual(messageBusMock.publish.call_count, 1) call0 = mock.call( mqName="taurus.metric.custom.data", persistent=True, body=json.dumps( dict( protocol="plain", data=["%s %r %d" % (m, v, t) for m, v, t in samples[:metric_utils._METRIC_DATA_BATCH_WRITE_SIZE]]))) self.assertEqual(messageBusMock.publish.call_args_list[0], call0) # put the remaining samples for sample in samples[metric_utils._METRIC_DATA_BATCH_WRITE_SIZE:]: putSample(*sample) # the remaining incomplete batch will be sent upon exit from the context, # but not yet self.assertEqual(messageBusMock.publish.call_count, 1) # Now, the remainder should be sent, too self.assertEqual(messageBusMock.publish.call_count, 2) call1 = mock.call( mqName="taurus.metric.custom.data", persistent=True, body=json.dumps( dict( protocol="plain", data=["%s %r %d" % (m, v, t) for m, v, t in samples[metric_utils._METRIC_DATA_BATCH_WRITE_SIZE:]]))) self.assertEqual(messageBusMock.publish.call_args_list[1], call1)
def _flushTaurusEngineMetricDataPath(engineServer, engineApiKey): """Flush Taurus Engine's metric data path. There is no formal mechanism for this in htmengine, so we're going to flush the data path by sending a metric data item for a dummy metric and wait for the dummy metric to be created (and then delete the dummy metric). It's a hack, but it's pretty much all we got right now. :param str engineServer: dns name of ip addres of Taurus API server :param str engineApiKey: API Key of Taurus HTM Engine """ g_log.info("Flushing Taurus Engine metric data path, please wait...") flusherMetricName = _DATA_PATH_FLUSHER_METRIC_PREFIX + uuid.uuid1().hex with metric_utils.metricDataBatchWrite(g_log) as putSample: putSample(flusherMetricName, 99999, int(time.time())) _waitForFlusherAndGarbageCollect(engineServer=engineServer, engineApiKey=engineApiKey, flusherMetricName=flusherMetricName)
def _flushTaurusEngineMetricDataPath(engineServer, engineApiKey): """Flush Taurus Engine's metric data path. There is no formal mechanism for this in htmengine, so we're going to flush the data path by sending a metric data item for a dummy metric and wait for the dummy metric to be created (and then delete the dummy metric). It's a hack, but it's pretty much all we got right now. :param str engineServer: dns name of ip addres of Taurus API server :param str engineApiKey: API Key of Taurus HTM Engine """ g_log.info("Flushing Taurus Engine metric data path, please wait...") flusherMetricName = _DATA_PATH_FLUSHER_METRIC_PREFIX + uuid.uuid1().hex with metric_utils.metricDataBatchWrite(g_log) as putSample: putSample(flusherMetricName, 99999, int(time.time())) _waitForFlusherAndGarbageCollect(engineServer=engineServer, engineApiKey=engineApiKey, flusherMetricName=flusherMetricName)
def testMetricDataBatchWrite(self): # Note: This test assumes that there is a running Taurus instance ready to # receive and process inbound custom metric data. In the deployed # environment $TAURUS_HTM_SERVER and $TAURUS_APIKEY must be set. Otherwise # default values will be assumed. host = os.environ.get("TAURUS_HTM_SERVER", "127.0.0.1") apikey = os.environ.get("TAURUS_APIKEY", "taurus") metricName = "bogus-test-metric" log = logging.getLogger(__name__) utcLocalizedEpoch = ( pytz.timezone("UTC").localize(datetime.utcfromtimestamp(0))) now = datetime.now(pytz.timezone("UTC")) # Send metric data in batches, and for test purposes making sure to exceed # the max batch size to force the batch to be chunked with metric_utils.metricDataBatchWrite(log=log) as putSample: # pylint: disable=W0212 for x in xrange(metric_utils._METRIC_DATA_BATCH_WRITE_SIZE + 1): ts = ((now - utcLocalizedEpoch).total_seconds() - metric_utils._METRIC_DATA_BATCH_WRITE_SIZE + 1 + x) putSample(metricName=metricName, value=x, epochTimestamp=ts) self.addCleanup(requests.delete, "https://%s/_metrics/custom/%s" % (host, metricName), auth=(apikey, ""), verify=False) attempt = 0 found = False while not found: result = requests.get("https://%s/_metrics/custom" % host, auth=(apikey, ""), verify=False) models = result.json() for model in models: if model["name"] == metricName: # Quick check to make sure the data made its way through result = requests.get("https://%s/_models/%s" % (host, model["uid"]), auth=(apikey, ""), verify=False) # pylint: disable=W0212 if (result.json()[0]["last_rowid"] == metric_utils._METRIC_DATA_BATCH_WRITE_SIZE + 1): found = True break else: if attempt == 30: self.fail( "Not all metric data samples made it through after 30 seconds") else: time.sleep(1) attempt += 1 continue
def testMetricDataBatchWrite(self): # Note: This test assumes that there is a running Taurus instance ready to # receive and process inbound custom metric data. In the deployed # environment $TAURUS_HTM_SERVER and $TAURUS_APIKEY must be set. Otherwise # default values will be assumed. host = os.environ.get("TAURUS_HTM_SERVER", "127.0.0.1") apikey = os.environ.get("TAURUS_APIKEY", "taurus") metricName = "bogus-test-metric" log = logging.getLogger(__name__) utcLocalizedEpoch = (pytz.timezone("UTC").localize( datetime.utcfromtimestamp(0))) now = datetime.now(pytz.timezone("UTC")) # Send metric data in batches, and for test purposes making sure to exceed # the max batch size to force the batch to be chunked with metric_utils.metricDataBatchWrite(log=log) as putSample: # pylint: disable=W0212 for x in xrange(metric_utils._METRIC_DATA_BATCH_WRITE_SIZE + 1): ts = ((now - utcLocalizedEpoch).total_seconds() - metric_utils._METRIC_DATA_BATCH_WRITE_SIZE + 1 + x) putSample(metricName=metricName, value=x, epochTimestamp=ts) self.addCleanup(requests.delete, "https://%s/_metrics/custom/%s" % (host, metricName), auth=(apikey, ""), verify=False) attempt = 0 found = False while not found: result = requests.get("https://%s/_metrics/custom" % host, auth=(apikey, ""), verify=False) models = result.json() for model in models: if model["name"] == metricName: # Quick check to make sure the data made its way through result = requests.get("https://%s/_models/%s" % (host, model["uid"]), auth=(apikey, ""), verify=False) # pylint: disable=W0212 if (result.json()[0]["last_rowid"] == metric_utils._METRIC_DATA_BATCH_WRITE_SIZE + 1): found = True break else: if attempt == 30: self.fail( "Not all metric data samples made it through after 30 seconds" ) else: time.sleep(1) attempt += 1 continue
def transmitMetricData(metricSpecs, symbol, engine): """ Send unsent metric data samples for the given symbol to Taurus NOTE: this is also used externally by friends of the agent. :param metricSpecs: Sequence of one or more StockMetricSpec objects associated with the same stock symbol for which polling was conducted :param symbol: stock symbol :param sqlalchemy.engine.Engine engine: """ try: @collectorsdb.retryOnTransientErrors def _fetchUnsentSamples(): # Select only records that haven't been sent to BOTH fields = [ xigniteSecurityBars.c.StartDate, xigniteSecurityBars.c.StartTime, xigniteSecurityBars.c.EndDate, xigniteSecurityBars.c.EndTime, xigniteSecurityBars.c.UTCOffset, xigniteSecurityBars.c.Volume, xigniteSecurityBars.c.Close, emittedStockPrice.c.sent.label("Close_sent"), emittedStockVolume.c.sent.label("Volume_sent") ] sel = (select(fields) .select_from(xigniteSecurityBars .outerjoin(emittedStockPrice) .outerjoin(emittedStockVolume)) .where(xigniteSecurityBars.c.symbol == symbol) .where((emittedStockPrice.c.sent == None) | (emittedStockVolume.c.sent == None)) .order_by(xigniteSecurityBars.c.EndDate.asc(), xigniteSecurityBars.c.EndTime.asc()) ) return engine.execute(sel) # Process samples in chunks to facilitate more efficient error recovery # during backlog processing samplesIter = iter(_fetchUnsentSamples()) while True: specSymbolSampleList = [] sample = None for sample in itertools.islice(samplesIter, 0, 1000): for spec in metricSpecs: if not sample[spec.sampleKey + "_sent"]: specSymbolSampleList.append((spec, symbol, sample)) if sample is None: # No more unsent samples break # Send samples to Taurus with metricDataBatchWrite(log=_LOG) as putSample: for spec, symbol, sample in specSymbolSampleList: if spec.sampleKey in sample: epochTs = epochFromLocalizedDatetime( _EASTERN_TZ.localize( datetime.datetime.combine(sample.StartDate, sample.StartTime))) value = sample[spec.sampleKey] _LOG.info("Sending: %s %r %d", spec.metricName, value, epochTs) putSample(metricName=spec.metricName, value=value, epochTimestamp=epochTs) # Update history of emitted samples # # NOTE: If this fails once in a while and we end up resending the samples, # htmengine's Metric Storer will discard duplicate-timestamp and # out-of-order samples for spec, symbol, sample in specSymbolSampleList: _updateMetricDataHistory(spec=spec, symbol=symbol, sample=sample, engine=engine) except Exception: _LOG.exception("Unexpected error while attempting to send metric " "data sample(s) to remote Taurus instance.")