def testEpochFromLocalizedDatetime(self):
localizedZero = (pytz.timezone("UTC").localize(
datetime.utcfromtimestamp(0)).astimezone(
pytz.timezone("US/Eastern")))
self.assertEqual(
date_time_utils.epochFromLocalizedDatetime(localizedZero), 0)
localizedTime = (pytz.timezone("UTC").localize(
datetime.utcfromtimestamp(1426880474.306222)).astimezone(
pytz.timezone("US/Eastern")))
self.assertEqual(
date_time_utils.epochFromLocalizedDatetime(localizedTime),
1426880474.306222)
localizedTime = (pytz.timezone("UTC").localize(
datetime.utcfromtimestamp(1426880474)).astimezone(
pytz.timezone("Australia/Melbourne")))
self.assertEqual(
date_time_utils.epochFromLocalizedDatetime(localizedTime),
1426880474)
def testEpochFromLocalizedDatetime(self):
localizedZero = (pytz.timezone("UTC")
.localize(datetime.utcfromtimestamp(0))
.astimezone(pytz.timezone("US/Eastern")))
self.assertEqual(
date_time_utils.epochFromLocalizedDatetime(localizedZero),
0)
localizedTime = (pytz.timezone("UTC")
.localize(datetime.utcfromtimestamp(1426880474.306222))
.astimezone(pytz.timezone("US/Eastern")))
self.assertEqual(
date_time_utils.epochFromLocalizedDatetime(localizedTime),
1426880474.306222)
localizedTime = (pytz.timezone("UTC")
.localize(datetime.utcfromtimestamp(1426880474))
.astimezone(pytz.timezone("Australia/Melbourne")))
self.assertEqual(
date_time_utils.epochFromLocalizedDatetime(localizedTime),
1426880474)
def transmitMetricData(metricSpecs, symbol, engine):
""" Send unsent metric data samples for the given symbol to Taurus
NOTE: this is also used externally by friends of the agent; e.g.,
`resymbol_metrics.py`.
:param metricSpecs: Sequence of one or more StockMetricSpec objects associated
with the same stock symbol for which polling was conducted
:param symbol: stock symbol
:param sqlalchemy.engine.Engine engine:
"""
try:
@collectorsdb.retryOnTransientErrors
def _fetchUnsentSamples():
# Select only records that haven't been sent to BOTH
fields = [
xigniteSecurityBars.c.StartDate,
xigniteSecurityBars.c.StartTime,
xigniteSecurityBars.c.EndDate,
xigniteSecurityBars.c.EndTime,
xigniteSecurityBars.c.UTCOffset,
xigniteSecurityBars.c.Volume,
xigniteSecurityBars.c.Close,
emittedStockPrice.c.sent.label("Close_sent"),
emittedStockVolume.c.sent.label("Volume_sent")
]
sel = (select(fields)
.select_from(xigniteSecurityBars
.outerjoin(emittedStockPrice)
.outerjoin(emittedStockVolume))
.where(xigniteSecurityBars.c.symbol == symbol)
.where((emittedStockPrice.c.sent == None) |
(emittedStockVolume.c.sent == None))
.order_by(xigniteSecurityBars.c.EndDate.asc(),
xigniteSecurityBars.c.EndTime.asc())
)
return engine.execute(sel)
# Process samples in chunks to facilitate more efficient error recovery
# during backlog processing
samplesIter = iter(_fetchUnsentSamples())
while True:
specSymbolSampleList = []
sample = None
for sample in itertools.islice(samplesIter, 0, 1000):
for spec in metricSpecs:
if not sample[spec.sampleKey + "_sent"]:
specSymbolSampleList.append((spec, symbol, sample))
if sample is None:
# No more unsent samples
break
# Send samples to Taurus
with metricDataBatchWrite(log=_LOG) as putSample:
for spec, symbol, sample in specSymbolSampleList:
if spec.sampleKey in sample:
epochTs = epochFromLocalizedDatetime(
_EASTERN_TZ.localize(
datetime.datetime.combine(sample.StartDate, sample.StartTime)))
value = sample[spec.sampleKey]
_LOG.info("Sending: %s %r %d", spec.metricName, value, epochTs)
putSample(metricName=spec.metricName,
value=value,
epochTimestamp=epochTs)
# Update history of emitted samples
#
# NOTE: If this fails once in a while and we end up resending the samples,
# htmengine's Metric Storer will discard duplicate-timestamp and
# out-of-order samples
for spec, symbol, sample in specSymbolSampleList:
_updateMetricDataHistory(spec=spec, symbol=symbol, sample=sample,
engine=engine)
except Exception:
_LOG.exception("Unexpected error while attempting to send metric "
"data sample(s) to remote Taurus instance.")
def transmitMetricData(metricSpecs, symbol, engine):
""" Send unsent metric data samples for the given symbol to Taurus
NOTE: this is also used externally by friends of the agent; e.g.,
`resymbol_metrics.py`.
:param metricSpecs: Sequence of one or more StockMetricSpec objects associated
with the same stock symbol for which polling was conducted
:param symbol: stock symbol
:param sqlalchemy.engine.Engine engine:
"""
try:
@collectorsdb.retryOnTransientErrors
def _fetchUnsentSamples():
# Select only records that haven't been sent to BOTH
fields = [
xigniteSecurityBars.c.StartDate,
xigniteSecurityBars.c.StartTime, xigniteSecurityBars.c.EndDate,
xigniteSecurityBars.c.EndTime, xigniteSecurityBars.c.UTCOffset,
xigniteSecurityBars.c.Volume, xigniteSecurityBars.c.Close,
emittedStockPrice.c.sent.label("Close_sent"),
emittedStockVolume.c.sent.label("Volume_sent")
]
sel = (select(fields).select_from(
xigniteSecurityBars.outerjoin(emittedStockPrice).outerjoin(
emittedStockVolume)).where(
xigniteSecurityBars.c.symbol == symbol).where(
(emittedStockPrice.c.sent == None)
| (emittedStockVolume.c.sent == None)).order_by(
xigniteSecurityBars.c.EndDate.asc(),
xigniteSecurityBars.c.EndTime.asc()))
return engine.execute(sel)
# Process samples in chunks to facilitate more efficient error recovery
# during backlog processing
samplesIter = iter(_fetchUnsentSamples())
while True:
specSymbolSampleList = []
sample = None
for sample in itertools.islice(samplesIter, 0, 1000):
for spec in metricSpecs:
if not sample[spec.sampleKey + "_sent"]:
specSymbolSampleList.append((spec, symbol, sample))
if sample is None:
# No more unsent samples
break
# Send samples to Taurus
with metricDataBatchWrite(log=_LOG) as putSample:
for spec, symbol, sample in specSymbolSampleList:
if spec.sampleKey in sample:
epochTs = epochFromLocalizedDatetime(
_EASTERN_TZ.localize(
datetime.datetime.combine(
sample.StartDate, sample.StartTime)))
value = sample[spec.sampleKey]
_LOG.info("Sending: %s %r %d", spec.metricName, value,
epochTs)
putSample(metricName=spec.metricName,
value=value,
epochTimestamp=epochTs)
# Update history of emitted samples
#
# NOTE: If this fails once in a while and we end up resending the samples,
# htmengine's Metric Storer will discard duplicate-timestamp and
# out-of-order samples
for spec, symbol, sample in specSymbolSampleList:
_updateMetricDataHistory(spec=spec,
symbol=symbol,
sample=sample,
engine=engine)
except Exception:
_LOG.exception("Unexpected error while attempting to send metric "
"data sample(s) to remote Taurus instance.")
