def testMetricDataBatchWrite(self, messageBusConnectorClassMock): samples = [ ("FOO.BAR.%d" % i, i * 3.789, i * 300) for i in xrange((metric_utils._METRIC_DATA_BATCH_WRITE_SIZE * 3) / 2) ] messageBusConnectorClass = ( metric_utils.message_bus_connector.MessageBusConnector) messageBusMock = MagicMock( spec_set=messageBusConnectorClass, publish=Mock(spec_set=messageBusConnectorClass.publish)) messageBusMock.__enter__.return_value = messageBusMock messageBusConnectorClassMock.return_value = messageBusMock loggerMock = Mock(spec_set=logging.Logger) with metric_utils.metricDataBatchWrite(loggerMock) as putSample: # put enough for the first batch for sample in samples[:metric_utils._METRIC_DATA_BATCH_WRITE_SIZE]: putSample(*sample) # The first publish call should be for a full batch self.assertEqual(messageBusMock.publish.call_count, 1) call0 = mock.call( mqName="taurus.metric.custom.data", persistent=True, body=json.dumps( dict(protocol="plain", data=[ "%s %r %d" % (m, v, t) for m, v, t in samples[:metric_utils. _METRIC_DATA_BATCH_WRITE_SIZE] ]))) self.assertEqual(messageBusMock.publish.call_args_list[0], call0) # put the remaining samples for sample in samples[metric_utils._METRIC_DATA_BATCH_WRITE_SIZE:]: putSample(*sample) # the remaining incomplete batch will be sent upon exit from the context, # but not yet self.assertEqual(messageBusMock.publish.call_count, 1) # Now, the remainder should be sent, too self.assertEqual(messageBusMock.publish.call_count, 2) call1 = mock.call( mqName="taurus.metric.custom.data", persistent=True, body=json.dumps( dict(protocol="plain", data=[ "%s %r %d" % (m, v, t) for m, v, t in samples[metric_utils._METRIC_DATA_BATCH_WRITE_SIZE:] ]))) self.assertEqual(messageBusMock.publish.call_args_list[1], call1)
def testMetricDataBatchWrite(self, messageBusConnectorClassMock): samples = [ ("FOO.BAR.%d" % i, i * 3.789, i * 300) for i in xrange((metric_utils._METRIC_DATA_BATCH_WRITE_SIZE * 3) / 2) ] messageBusConnectorClass = ( metric_utils.message_bus_connector.MessageBusConnector) messageBusMock = MagicMock( spec_set=messageBusConnectorClass, publish=Mock(spec_set=messageBusConnectorClass.publish)) messageBusMock.__enter__.return_value = messageBusMock messageBusConnectorClassMock.return_value = messageBusMock loggerMock = Mock(spec_set=logging.Logger) with metric_utils.metricDataBatchWrite(loggerMock) as putSample: # put enough for the first batch for sample in samples[:metric_utils._METRIC_DATA_BATCH_WRITE_SIZE]: putSample(*sample) # The first publish call should be for a full batch self.assertEqual(messageBusMock.publish.call_count, 1) call0 = mock.call( mqName="taurus.metric.custom.data", persistent=True, body=json.dumps( dict( protocol="plain", data=["%s %r %d" % (m, v, t) for m, v, t in samples[:metric_utils._METRIC_DATA_BATCH_WRITE_SIZE]]))) self.assertEqual(messageBusMock.publish.call_args_list[0], call0) # put the remaining samples for sample in samples[metric_utils._METRIC_DATA_BATCH_WRITE_SIZE:]: putSample(*sample) # the remaining incomplete batch will be sent upon exit from the context, # but not yet self.assertEqual(messageBusMock.publish.call_count, 1) # Now, the remainder should be sent, too self.assertEqual(messageBusMock.publish.call_count, 2) call1 = mock.call( mqName="taurus.metric.custom.data", persistent=True, body=json.dumps( dict( protocol="plain", data=["%s %r %d" % (m, v, t) for m, v, t in samples[metric_utils._METRIC_DATA_BATCH_WRITE_SIZE:]]))) self.assertEqual(messageBusMock.publish.call_args_list[1], call1)
def _flushTaurusEngineMetricDataPath(engineServer, engineApiKey): """Flush Taurus Engine's metric data path. There is no formal mechanism for this in htmengine, so we're going to flush the data path by sending a metric data item for a dummy metric and wait for the dummy metric to be created (and then delete the dummy metric). It's a hack, but it's pretty much all we got right now. :param str engineServer: dns name of ip addres of Taurus API server :param str engineApiKey: API Key of Taurus HTM Engine """ g_log.info("Flushing Taurus Engine metric data path, please wait...") flusherMetricName = _DATA_PATH_FLUSHER_METRIC_PREFIX + uuid.uuid1().hex with metric_utils.metricDataBatchWrite(g_log) as putSample: putSample(flusherMetricName, 99999, int(time.time())) _waitForFlusherAndGarbageCollect( engineServer=engineServer, engineApiKey=engineApiKey, flusherMetricName=flusherMetricName )
def _flushTaurusEngineMetricDataPath(engineServer, engineApiKey): """Flush Taurus Engine's metric data path. There is no formal mechanism for this in htmengine, so we're going to flush the data path by sending a metric data item for a dummy metric and wait for the dummy metric to be created (and then delete the dummy metric). It's a hack, but it's pretty much all we got right now. :param str engineServer: dns name of ip addres of Taurus API server :param str engineApiKey: API Key of Taurus HTM Engine """ g_log.info("Flushing Taurus Engine metric data path, please wait...") flusherMetricName = _DATA_PATH_FLUSHER_METRIC_PREFIX + uuid.uuid1().hex with metric_utils.metricDataBatchWrite(g_log) as putSample: putSample(flusherMetricName, 99999, int(time.time())) _waitForFlusherAndGarbageCollect(engineServer=engineServer, engineApiKey=engineApiKey, flusherMetricName=flusherMetricName)
def _forwardNewsVolumeMetrics(metricSpecs, lastEmittedAggTime, stopDatetime, periodSec, metricDestAddr): """ Query news volume metrics since the given last emitted timestamp through stopDatetime and forward them to htmengine's Metric Listener. Update the datetime of the last successfully-emitted news volume metric batch in the database. NOTE: forwarding will be aborted upon failure to connect to Metic Listener. In this case, an error will be logged, and the function will return the UTC timestamp of the last successfully-emitted sample aggregation interval. Once Metric Listener comes online, a subsequent call to this function will catch up by forwarding the stored samples since last successful emission. :param metrics: a sequence of NewsVolumeMetricSpec objects corresponding to the metrics to be emitted :param lastEmittedAggTime: UTC datetime of last successfully-emitted sample batch :param stopDatetime: non-inclusive upper bound UTC datetime for forwarding :param periodSec: aggregation period in seconds :param metricDestAddr: two-tuple (metricDestHost, metricDestPort) :returns: UTC timestamp of the last successfully-emitted sample batch. :rtype: datetime.datetime """ periodTimedelta = timedelta(seconds=periodSec) aggStartDatetime = lastEmittedAggTime + periodTimedelta while aggStartDatetime < stopDatetime: # Get News Volume metrics for one aggregation interval aggStopDatetime = aggStartDatetime + periodTimedelta symbolToNewsVolumeMap = defaultdict( int, _queryNewsVolumes(aggStartDatetime, aggStopDatetime)) # Generate metric samples epochTimestamp = date_time_utils.epochFromNaiveUTCDatetime(aggStartDatetime) samples = tuple( dict( metricName=spec.metric, value=symbolToNewsVolumeMap[spec.symbol], epochTimestamp=epochTimestamp) for spec in metricSpecs ) # Emit samples to Metric Listener try: with metric_utils.metricDataBatchWrite(log=g_log) as putSample: for sample in samples: putSample(**sample) except Exception: g_log.exception("Failure while emitting metric data for agg=%s " "containing numSamples=%d", aggStartDatetime, len(samples)) return lastEmittedAggTime else: g_log.info("Forwarded numSamples=%d for agg=%s", len(samples), aggStartDatetime) # Update db with last successfully-emitted datetime metric_utils.updateLastEmittedSampleDatetime( key=_EMITTED_NEWS_VOLUME_SAMPLE_TRACKER_KEY, sampleDatetime=aggStartDatetime) # Set up for next iteration lastEmittedAggTime = aggStartDatetime aggStartDatetime = aggStopDatetime return lastEmittedAggTime
def _forwardNewsVolumeMetrics(metricSpecs, lastEmittedAggTime, stopDatetime, periodSec, metricDestAddr): """ Query news volume metrics since the given last emitted timestamp through stopDatetime and forward them to htmengine's Metric Listener. Update the datetime of the last successfully-emitted news volume metric batch in the database. NOTE: forwarding will be aborted upon failure to connect to Metic Listener. In this case, an error will be logged, and the function will return the UTC timestamp of the last successfully-emitted sample aggregation interval. Once Metric Listener comes online, a subsequent call to this function will catch up by forwarding the stored samples since last successful emission. :param metrics: a sequence of NewsVolumeMetricSpec objects corresponding to the metrics to be emitted :param lastEmittedAggTime: UTC datetime of last successfully-emitted sample batch :param stopDatetime: non-inclusive upper bound UTC datetime for forwarding :param periodSec: aggregation period in seconds :param metricDestAddr: two-tuple (metricDestHost, metricDestPort) :returns: UTC timestamp of the last successfully-emitted sample batch. :rtype: datetime.datetime """ periodTimedelta = timedelta(seconds=periodSec) aggStartDatetime = lastEmittedAggTime + periodTimedelta while aggStartDatetime < stopDatetime: # Get News Volume metrics for one aggregation interval aggStopDatetime = aggStartDatetime + periodTimedelta symbolToNewsVolumeMap = defaultdict( int, _queryNewsVolumes(aggStartDatetime, aggStopDatetime)) # Generate metric samples epochTimestamp = date_time_utils.epochFromNaiveUTCDatetime( aggStartDatetime) samples = tuple( dict(metricName=spec.metric, value=symbolToNewsVolumeMap[spec.symbol], epochTimestamp=epochTimestamp) for spec in metricSpecs) # Emit samples to Metric Listener try: with metric_utils.metricDataBatchWrite(log=g_log) as putSample: for sample in samples: putSample(**sample) except Exception: g_log.exception( "Failure while emitting metric data for agg=%s " "containing numSamples=%d", aggStartDatetime, len(samples)) return lastEmittedAggTime else: g_log.info("Forwarded numSamples=%d for agg=%s", len(samples), aggStartDatetime) # Update db with last successfully-emitted datetime metric_utils.updateLastEmittedSampleDatetime( key=_EMITTED_NEWS_VOLUME_SAMPLE_TRACKER_KEY, sampleDatetime=aggStartDatetime) # Set up for next iteration lastEmittedAggTime = aggStartDatetime aggStartDatetime = aggStopDatetime return lastEmittedAggTime
def testMetricDataBatchWrite(self): # Note: This test assumes that there is a running Taurus instance ready to # receive and process inbound custom metric data. In the deployed # environment $TAURUS_HTM_SERVER and $TAURUS_APIKEY must be set. Otherwise # default values will be assumed. host = os.environ.get("TAURUS_HTM_SERVER", "127.0.0.1") apikey = os.environ.get("TAURUS_APIKEY", "taurus") metricName = "bogus-test-metric" _LOG = ExtendedLogger.getExtendedLogger(__name__) UTC_LOCALIZED_EPOCH = ( pytz.timezone("UTC").localize(datetime.utcfromtimestamp(0))) now = datetime.now(pytz.timezone("UTC")) # Send metric data in batches, and for test purposes making sure to exceed # the max batch size to force the batch to be chunked with metric_utils.metricDataBatchWrite(log=_LOG) as putSample: for x in xrange(metric_utils._METRIC_DATA_BATCH_WRITE_SIZE + 1): ts = ((now - UTC_LOCALIZED_EPOCH).total_seconds() - metric_utils._METRIC_DATA_BATCH_WRITE_SIZE + 1 + x) putSample(metricName=metricName, value=x, epochTimestamp=ts) self.addCleanup(requests.delete, "https://%s/_metrics/custom/%s" % (host, metricName), auth=(apikey, ""), verify=False) attempt = 0 found = False while not found: result = requests.get("https://%s/_metrics/custom" % host, auth=(apikey, ""), verify=False) models = result.json() for model in models: if model["name"] == metricName: # Quick check to make sure the data made its way through result = requests.get("https://%s/_models/%s" % (host, model["uid"]), auth=(apikey, ""), verify=False) if (result.json()[0]["last_rowid"] == metric_utils._METRIC_DATA_BATCH_WRITE_SIZE + 1): found = True break else: if attempt == 30: self.fail( "Not all metric data samples made it through after 30 seconds") else: time.sleep(1) attempt += 1 continue
def transmitMetricData(metricSpecs, symbol, engine): """ Send unsent metric data samples for the given symbol to Taurus NOTE: this is also used externally by friends of the agent; e.g., `resymbol_metrics.py`. :param metricSpecs: Sequence of one or more StockMetricSpec objects associated with the same stock symbol for which polling was conducted :param symbol: stock symbol :param sqlalchemy.engine.Engine engine: """ try: @collectorsdb.retryOnTransientErrors def _fetchUnsentSamples(): # Select only records that haven't been sent to BOTH fields = [ xigniteSecurityBars.c.StartDate, xigniteSecurityBars.c.StartTime, xigniteSecurityBars.c.EndDate, xigniteSecurityBars.c.EndTime, xigniteSecurityBars.c.UTCOffset, xigniteSecurityBars.c.Volume, xigniteSecurityBars.c.Close, emittedStockPrice.c.sent.label("Close_sent"), emittedStockVolume.c.sent.label("Volume_sent") ] sel = (select(fields) .select_from(xigniteSecurityBars .outerjoin(emittedStockPrice) .outerjoin(emittedStockVolume)) .where(xigniteSecurityBars.c.symbol == symbol) .where((emittedStockPrice.c.sent == None) | (emittedStockVolume.c.sent == None)) .order_by(xigniteSecurityBars.c.EndDate.asc(), xigniteSecurityBars.c.EndTime.asc()) ) return engine.execute(sel) # Process samples in chunks to facilitate more efficient error recovery # during backlog processing samplesIter = iter(_fetchUnsentSamples()) while True: specSymbolSampleList = [] sample = None for sample in itertools.islice(samplesIter, 0, 1000): for spec in metricSpecs: if not sample[spec.sampleKey + "_sent"]: specSymbolSampleList.append((spec, symbol, sample)) if sample is None: # No more unsent samples break # Send samples to Taurus with metricDataBatchWrite(log=_LOG) as putSample: for spec, symbol, sample in specSymbolSampleList: if spec.sampleKey in sample: epochTs = epochFromLocalizedDatetime( _EASTERN_TZ.localize( datetime.datetime.combine(sample.StartDate, sample.StartTime))) value = sample[spec.sampleKey] _LOG.info("Sending: %s %r %d", spec.metricName, value, epochTs) putSample(metricName=spec.metricName, value=value, epochTimestamp=epochTs) # Update history of emitted samples # # NOTE: If this fails once in a while and we end up resending the samples, # htmengine's Metric Storer will discard duplicate-timestamp and # out-of-order samples for spec, symbol, sample in specSymbolSampleList: _updateMetricDataHistory(spec=spec, symbol=symbol, sample=sample, engine=engine) except Exception: _LOG.exception("Unexpected error while attempting to send metric " "data sample(s) to remote Taurus instance.")
def transmitMetricData(metricSpecs, symbol, engine): """ Send unsent metric data samples for the given symbol to Taurus NOTE: this is also used externally by friends of the agent; e.g., `resymbol_metrics.py`. :param metricSpecs: Sequence of one or more StockMetricSpec objects associated with the same stock symbol for which polling was conducted :param symbol: stock symbol :param sqlalchemy.engine.Engine engine: """ try: @collectorsdb.retryOnTransientErrors def _fetchUnsentSamples(): # Select only records that haven't been sent to BOTH fields = [ xigniteSecurityBars.c.StartDate, xigniteSecurityBars.c.StartTime, xigniteSecurityBars.c.EndDate, xigniteSecurityBars.c.EndTime, xigniteSecurityBars.c.UTCOffset, xigniteSecurityBars.c.Volume, xigniteSecurityBars.c.Close, emittedStockPrice.c.sent.label("Close_sent"), emittedStockVolume.c.sent.label("Volume_sent") ] sel = (select(fields).select_from( xigniteSecurityBars.outerjoin(emittedStockPrice).outerjoin( emittedStockVolume)).where( xigniteSecurityBars.c.symbol == symbol).where( (emittedStockPrice.c.sent == None) | (emittedStockVolume.c.sent == None)).order_by( xigniteSecurityBars.c.EndDate.asc(), xigniteSecurityBars.c.EndTime.asc())) return engine.execute(sel) # Process samples in chunks to facilitate more efficient error recovery # during backlog processing samplesIter = iter(_fetchUnsentSamples()) while True: specSymbolSampleList = [] sample = None for sample in itertools.islice(samplesIter, 0, 1000): for spec in metricSpecs: if not sample[spec.sampleKey + "_sent"]: specSymbolSampleList.append((spec, symbol, sample)) if sample is None: # No more unsent samples break # Send samples to Taurus with metricDataBatchWrite(log=_LOG) as putSample: for spec, symbol, sample in specSymbolSampleList: if spec.sampleKey in sample: epochTs = epochFromLocalizedDatetime( _EASTERN_TZ.localize( datetime.datetime.combine( sample.StartDate, sample.StartTime))) value = sample[spec.sampleKey] _LOG.info("Sending: %s %r %d", spec.metricName, value, epochTs) putSample(metricName=spec.metricName, value=value, epochTimestamp=epochTs) # Update history of emitted samples # # NOTE: If this fails once in a while and we end up resending the samples, # htmengine's Metric Storer will discard duplicate-timestamp and # out-of-order samples for spec, symbol, sample in specSymbolSampleList: _updateMetricDataHistory(spec=spec, symbol=symbol, sample=sample, engine=engine) except Exception: _LOG.exception("Unexpected error while attempting to send metric " "data sample(s) to remote Taurus instance.")
def testMetricDataBatchWrite(self): # Note: This test assumes that there is a running Taurus instance ready to # receive and process inbound custom metric data. In the deployed # environment $TAURUS_HTM_SERVER and $TAURUS_APIKEY must be set. Otherwise # default values will be assumed. host = os.environ.get("TAURUS_HTM_SERVER", "127.0.0.1") apikey = os.environ.get("TAURUS_APIKEY", "taurus") metricName = "bogus-test-metric" _LOG = ExtendedLogger.getExtendedLogger(__name__) UTC_LOCALIZED_EPOCH = (pytz.timezone("UTC").localize( datetime.utcfromtimestamp(0))) now = datetime.now(pytz.timezone("UTC")) # Send metric data in batches, and for test purposes making sure to exceed # the max batch size to force the batch to be chunked with metric_utils.metricDataBatchWrite(log=_LOG) as putSample: for x in xrange(metric_utils._METRIC_DATA_BATCH_WRITE_SIZE + 1): ts = ((now - UTC_LOCALIZED_EPOCH).total_seconds() - metric_utils._METRIC_DATA_BATCH_WRITE_SIZE + 1 + x) putSample(metricName=metricName, value=x, epochTimestamp=ts) self.addCleanup(requests.delete, "https://%s/_metrics/custom/%s" % (host, metricName), auth=(apikey, ""), verify=False) attempt = 0 found = False while not found: result = requests.get("https://%s/_metrics/custom" % host, auth=(apikey, ""), verify=False) models = result.json() for model in models: if model["name"] == metricName: # Quick check to make sure the data made its way through result = requests.get("https://%s/_models/%s" % (host, model["uid"]), auth=(apikey, ""), verify=False) if (result.json()[0]["last_rowid"] == metric_utils._METRIC_DATA_BATCH_WRITE_SIZE + 1): found = True break else: if attempt == 30: self.fail( "Not all metric data samples made it through after 30 seconds" ) else: time.sleep(1) attempt += 1 continue