def testMetricDataBatchWrite(self, messageBusConnectorClassMock):
samples = [
("FOO.BAR.%d" % i, i * 3.789, i * 300)
for i in xrange((metric_utils._METRIC_DATA_BATCH_WRITE_SIZE * 3) /
2)
]
messageBusConnectorClass = (
metric_utils.message_bus_connector.MessageBusConnector)
messageBusMock = MagicMock(
spec_set=messageBusConnectorClass,
publish=Mock(spec_set=messageBusConnectorClass.publish))
messageBusMock.__enter__.return_value = messageBusMock
messageBusConnectorClassMock.return_value = messageBusMock
loggerMock = Mock(spec_set=logging.Logger)
with metric_utils.metricDataBatchWrite(loggerMock) as putSample:
# put enough for the first batch
for sample in samples[:metric_utils._METRIC_DATA_BATCH_WRITE_SIZE]:
putSample(*sample)
# The first publish call should be for a full batch
self.assertEqual(messageBusMock.publish.call_count, 1)
call0 = mock.call(
mqName="taurus.metric.custom.data",
persistent=True,
body=json.dumps(
dict(protocol="plain",
data=[
"%s %r %d" % (m, v, t) for m, v, t in
samples[:metric_utils.
_METRIC_DATA_BATCH_WRITE_SIZE]
])))
self.assertEqual(messageBusMock.publish.call_args_list[0], call0)
# put the remaining samples
for sample in samples[metric_utils._METRIC_DATA_BATCH_WRITE_SIZE:]:
putSample(*sample)
# the remaining incomplete batch will be sent upon exit from the context,
# but not yet
self.assertEqual(messageBusMock.publish.call_count, 1)
# Now, the remainder should be sent, too
self.assertEqual(messageBusMock.publish.call_count, 2)
call1 = mock.call(
mqName="taurus.metric.custom.data",
persistent=True,
body=json.dumps(
dict(protocol="plain",
data=[
"%s %r %d" % (m, v, t) for m, v, t in
samples[metric_utils._METRIC_DATA_BATCH_WRITE_SIZE:]
])))
self.assertEqual(messageBusMock.publish.call_args_list[1], call1)
def testMetricDataBatchWrite(self, messageBusConnectorClassMock):
samples = [
("FOO.BAR.%d" % i, i * 3.789, i * 300)
for i in xrange((metric_utils._METRIC_DATA_BATCH_WRITE_SIZE * 3) / 2)
]
messageBusConnectorClass = (
metric_utils.message_bus_connector.MessageBusConnector)
messageBusMock = MagicMock(
spec_set=messageBusConnectorClass,
publish=Mock(spec_set=messageBusConnectorClass.publish))
messageBusMock.__enter__.return_value = messageBusMock
messageBusConnectorClassMock.return_value = messageBusMock
loggerMock = Mock(spec_set=logging.Logger)
with metric_utils.metricDataBatchWrite(loggerMock) as putSample:
# put enough for the first batch
for sample in samples[:metric_utils._METRIC_DATA_BATCH_WRITE_SIZE]:
putSample(*sample)
# The first publish call should be for a full batch
self.assertEqual(messageBusMock.publish.call_count, 1)
call0 = mock.call(
mqName="taurus.metric.custom.data",
persistent=True,
body=json.dumps(
dict(
protocol="plain",
data=["%s %r %d" % (m, v, t)
for m, v, t
in samples[:metric_utils._METRIC_DATA_BATCH_WRITE_SIZE]])))
self.assertEqual(messageBusMock.publish.call_args_list[0], call0)
# put the remaining samples
for sample in samples[metric_utils._METRIC_DATA_BATCH_WRITE_SIZE:]:
putSample(*sample)
# the remaining incomplete batch will be sent upon exit from the context,
# but not yet
self.assertEqual(messageBusMock.publish.call_count, 1)
# Now, the remainder should be sent, too
self.assertEqual(messageBusMock.publish.call_count, 2)
call1 = mock.call(
mqName="taurus.metric.custom.data",
persistent=True,
body=json.dumps(
dict(
protocol="plain",
data=["%s %r %d" % (m, v, t)
for m, v, t
in samples[metric_utils._METRIC_DATA_BATCH_WRITE_SIZE:]])))
self.assertEqual(messageBusMock.publish.call_args_list[1], call1)
def _flushTaurusEngineMetricDataPath(engineServer, engineApiKey):
"""Flush Taurus Engine's metric data path.
There is no formal mechanism for this in htmengine, so we're going to flush
the data path by sending a metric data item for a dummy metric and wait for
the dummy metric to be created (and then delete the dummy metric). It's a
hack, but it's pretty much all we got right now.
:param str engineServer: dns name of ip addres of Taurus API server
:param str engineApiKey: API Key of Taurus HTM Engine
"""
g_log.info("Flushing Taurus Engine metric data path, please wait...")
flusherMetricName = _DATA_PATH_FLUSHER_METRIC_PREFIX + uuid.uuid1().hex
with metric_utils.metricDataBatchWrite(g_log) as putSample:
putSample(flusherMetricName, 99999, int(time.time()))
_waitForFlusherAndGarbageCollect(engineServer=engineServer,
engineApiKey=engineApiKey,
flusherMetricName=flusherMetricName)
def _flushTaurusEngineMetricDataPath(engineServer, engineApiKey):
"""Flush Taurus Engine's metric data path.
There is no formal mechanism for this in htmengine, so we're going to flush
the data path by sending a metric data item for a dummy metric and wait for
the dummy metric to be created (and then delete the dummy metric). It's a
hack, but it's pretty much all we got right now.
:param str engineServer: dns name of ip addres of Taurus API server
:param str engineApiKey: API Key of Taurus HTM Engine
"""
g_log.info("Flushing Taurus Engine metric data path, please wait...")
flusherMetricName = _DATA_PATH_FLUSHER_METRIC_PREFIX + uuid.uuid1().hex
with metric_utils.metricDataBatchWrite(g_log) as putSample:
putSample(flusherMetricName, 99999, int(time.time()))
_waitForFlusherAndGarbageCollect(engineServer=engineServer,
engineApiKey=engineApiKey,
flusherMetricName=flusherMetricName)
def testMetricDataBatchWrite(self):
# Note: This test assumes that there is a running Taurus instance ready to
# receive and process inbound custom metric data. In the deployed
# environment $TAURUS_HTM_SERVER and $TAURUS_APIKEY must be set. Otherwise
# default values will be assumed.
host = os.environ.get("TAURUS_HTM_SERVER", "127.0.0.1")
apikey = os.environ.get("TAURUS_APIKEY", "taurus")
metricName = "bogus-test-metric"
log = logging.getLogger(__name__)
utcLocalizedEpoch = (
pytz.timezone("UTC").localize(datetime.utcfromtimestamp(0)))
now = datetime.now(pytz.timezone("UTC"))
# Send metric data in batches, and for test purposes making sure to exceed
# the max batch size to force the batch to be chunked
with metric_utils.metricDataBatchWrite(log=log) as putSample:
# pylint: disable=W0212
for x in xrange(metric_utils._METRIC_DATA_BATCH_WRITE_SIZE + 1):
ts = ((now - utcLocalizedEpoch).total_seconds()
- metric_utils._METRIC_DATA_BATCH_WRITE_SIZE
+ 1
+ x)
putSample(metricName=metricName,
value=x,
epochTimestamp=ts)
self.addCleanup(requests.delete,
"https://%s/_metrics/custom/%s" % (host, metricName),
auth=(apikey, ""),
verify=False)
attempt = 0
found = False
while not found:
result = requests.get("https://%s/_metrics/custom" % host,
auth=(apikey, ""),
verify=False)
models = result.json()
for model in models:
if model["name"] == metricName:
# Quick check to make sure the data made its way through
result = requests.get("https://%s/_models/%s" % (host, model["uid"]),
auth=(apikey, ""),
verify=False)
# pylint: disable=W0212
if (result.json()[0]["last_rowid"] ==
metric_utils._METRIC_DATA_BATCH_WRITE_SIZE + 1):
found = True
break
else:
if attempt == 30:
self.fail(
"Not all metric data samples made it through after 30 seconds")
else:
time.sleep(1)
attempt += 1
continue
def testMetricDataBatchWrite(self):
# Note: This test assumes that there is a running Taurus instance ready to
# receive and process inbound custom metric data. In the deployed
# environment $TAURUS_HTM_SERVER and $TAURUS_APIKEY must be set. Otherwise
# default values will be assumed.
host = os.environ.get("TAURUS_HTM_SERVER", "127.0.0.1")
apikey = os.environ.get("TAURUS_APIKEY", "taurus")
metricName = "bogus-test-metric"
log = logging.getLogger(__name__)
utcLocalizedEpoch = (pytz.timezone("UTC").localize(
datetime.utcfromtimestamp(0)))
now = datetime.now(pytz.timezone("UTC"))
# Send metric data in batches, and for test purposes making sure to exceed
# the max batch size to force the batch to be chunked
with metric_utils.metricDataBatchWrite(log=log) as putSample:
# pylint: disable=W0212
for x in xrange(metric_utils._METRIC_DATA_BATCH_WRITE_SIZE + 1):
ts = ((now - utcLocalizedEpoch).total_seconds() -
metric_utils._METRIC_DATA_BATCH_WRITE_SIZE + 1 + x)
putSample(metricName=metricName, value=x, epochTimestamp=ts)
self.addCleanup(requests.delete,
"https://%s/_metrics/custom/%s" % (host, metricName),
auth=(apikey, ""),
verify=False)
attempt = 0
found = False
while not found:
result = requests.get("https://%s/_metrics/custom" % host,
auth=(apikey, ""),
verify=False)
models = result.json()
for model in models:
if model["name"] == metricName:
# Quick check to make sure the data made its way through
result = requests.get("https://%s/_models/%s" %
(host, model["uid"]),
auth=(apikey, ""),
verify=False)
# pylint: disable=W0212
if (result.json()[0]["last_rowid"] ==
metric_utils._METRIC_DATA_BATCH_WRITE_SIZE + 1):
found = True
break
else:
if attempt == 30:
self.fail(
"Not all metric data samples made it through after 30 seconds"
)
else:
time.sleep(1)
attempt += 1
continue
def transmitMetricData(metricSpecs, symbol, engine):
""" Send unsent metric data samples for the given symbol to Taurus
NOTE: this is also used externally by friends of the agent.
:param metricSpecs: Sequence of one or more StockMetricSpec objects associated
with the same stock symbol for which polling was conducted
:param symbol: stock symbol
:param sqlalchemy.engine.Engine engine:
"""
try:
@collectorsdb.retryOnTransientErrors
def _fetchUnsentSamples():
# Select only records that haven't been sent to BOTH
fields = [
xigniteSecurityBars.c.StartDate,
xigniteSecurityBars.c.StartTime,
xigniteSecurityBars.c.EndDate,
xigniteSecurityBars.c.EndTime,
xigniteSecurityBars.c.UTCOffset,
xigniteSecurityBars.c.Volume,
xigniteSecurityBars.c.Close,
emittedStockPrice.c.sent.label("Close_sent"),
emittedStockVolume.c.sent.label("Volume_sent")
]
sel = (select(fields)
.select_from(xigniteSecurityBars
.outerjoin(emittedStockPrice)
.outerjoin(emittedStockVolume))
.where(xigniteSecurityBars.c.symbol == symbol)
.where((emittedStockPrice.c.sent == None) |
(emittedStockVolume.c.sent == None))
.order_by(xigniteSecurityBars.c.EndDate.asc(),
xigniteSecurityBars.c.EndTime.asc())
)
return engine.execute(sel)
# Process samples in chunks to facilitate more efficient error recovery
# during backlog processing
samplesIter = iter(_fetchUnsentSamples())
while True:
specSymbolSampleList = []
sample = None
for sample in itertools.islice(samplesIter, 0, 1000):
for spec in metricSpecs:
if not sample[spec.sampleKey + "_sent"]:
specSymbolSampleList.append((spec, symbol, sample))
if sample is None:
# No more unsent samples
break
# Send samples to Taurus
with metricDataBatchWrite(log=_LOG) as putSample:
for spec, symbol, sample in specSymbolSampleList:
if spec.sampleKey in sample:
epochTs = epochFromLocalizedDatetime(
_EASTERN_TZ.localize(
datetime.datetime.combine(sample.StartDate, sample.StartTime)))
value = sample[spec.sampleKey]
_LOG.info("Sending: %s %r %d", spec.metricName, value, epochTs)
putSample(metricName=spec.metricName,
value=value,
epochTimestamp=epochTs)
# Update history of emitted samples
#
# NOTE: If this fails once in a while and we end up resending the samples,
# htmengine's Metric Storer will discard duplicate-timestamp and
# out-of-order samples
for spec, symbol, sample in specSymbolSampleList:
_updateMetricDataHistory(spec=spec, symbol=symbol, sample=sample,
engine=engine)
except Exception:
_LOG.exception("Unexpected error while attempting to send metric "
"data sample(s) to remote Taurus instance.")
