def testEpochFromLocalizedDatetime(self): localizedZero = (pytz.timezone("UTC").localize( datetime.utcfromtimestamp(0)).astimezone( pytz.timezone("US/Eastern"))) self.assertEqual( date_time_utils.epochFromLocalizedDatetime(localizedZero), 0) localizedTime = (pytz.timezone("UTC").localize( datetime.utcfromtimestamp(1426880474.306222)).astimezone( pytz.timezone("US/Eastern"))) self.assertEqual( date_time_utils.epochFromLocalizedDatetime(localizedTime), 1426880474.306222) localizedTime = (pytz.timezone("UTC").localize( datetime.utcfromtimestamp(1426880474)).astimezone( pytz.timezone("Australia/Melbourne"))) self.assertEqual( date_time_utils.epochFromLocalizedDatetime(localizedTime), 1426880474)
def testEpochFromLocalizedDatetime(self): localizedZero = (pytz.timezone("UTC") .localize(datetime.utcfromtimestamp(0)) .astimezone(pytz.timezone("US/Eastern"))) self.assertEqual( date_time_utils.epochFromLocalizedDatetime(localizedZero), 0) localizedTime = (pytz.timezone("UTC") .localize(datetime.utcfromtimestamp(1426880474.306222)) .astimezone(pytz.timezone("US/Eastern"))) self.assertEqual( date_time_utils.epochFromLocalizedDatetime(localizedTime), 1426880474.306222) localizedTime = (pytz.timezone("UTC") .localize(datetime.utcfromtimestamp(1426880474)) .astimezone(pytz.timezone("Australia/Melbourne"))) self.assertEqual( date_time_utils.epochFromLocalizedDatetime(localizedTime), 1426880474)
def transmitMetricData(metricSpecs, symbol, engine): """ Send unsent metric data samples for the given symbol to Taurus NOTE: this is also used externally by friends of the agent; e.g., `resymbol_metrics.py`. :param metricSpecs: Sequence of one or more StockMetricSpec objects associated with the same stock symbol for which polling was conducted :param symbol: stock symbol :param sqlalchemy.engine.Engine engine: """ try: @collectorsdb.retryOnTransientErrors def _fetchUnsentSamples(): # Select only records that haven't been sent to BOTH fields = [ xigniteSecurityBars.c.StartDate, xigniteSecurityBars.c.StartTime, xigniteSecurityBars.c.EndDate, xigniteSecurityBars.c.EndTime, xigniteSecurityBars.c.UTCOffset, xigniteSecurityBars.c.Volume, xigniteSecurityBars.c.Close, emittedStockPrice.c.sent.label("Close_sent"), emittedStockVolume.c.sent.label("Volume_sent") ] sel = (select(fields) .select_from(xigniteSecurityBars .outerjoin(emittedStockPrice) .outerjoin(emittedStockVolume)) .where(xigniteSecurityBars.c.symbol == symbol) .where((emittedStockPrice.c.sent == None) | (emittedStockVolume.c.sent == None)) .order_by(xigniteSecurityBars.c.EndDate.asc(), xigniteSecurityBars.c.EndTime.asc()) ) return engine.execute(sel) # Process samples in chunks to facilitate more efficient error recovery # during backlog processing samplesIter = iter(_fetchUnsentSamples()) while True: specSymbolSampleList = [] sample = None for sample in itertools.islice(samplesIter, 0, 1000): for spec in metricSpecs: if not sample[spec.sampleKey + "_sent"]: specSymbolSampleList.append((spec, symbol, sample)) if sample is None: # No more unsent samples break # Send samples to Taurus with metricDataBatchWrite(log=_LOG) as putSample: for spec, symbol, sample in specSymbolSampleList: if spec.sampleKey in sample: epochTs = epochFromLocalizedDatetime( _EASTERN_TZ.localize( datetime.datetime.combine(sample.StartDate, sample.StartTime))) value = sample[spec.sampleKey] _LOG.info("Sending: %s %r %d", spec.metricName, value, epochTs) putSample(metricName=spec.metricName, value=value, epochTimestamp=epochTs) # Update history of emitted samples # # NOTE: If this fails once in a while and we end up resending the samples, # htmengine's Metric Storer will discard duplicate-timestamp and # out-of-order samples for spec, symbol, sample in specSymbolSampleList: _updateMetricDataHistory(spec=spec, symbol=symbol, sample=sample, engine=engine) except Exception: _LOG.exception("Unexpected error while attempting to send metric " "data sample(s) to remote Taurus instance.")
def transmitMetricData(metricSpecs, symbol, engine): """ Send unsent metric data samples for the given symbol to Taurus NOTE: this is also used externally by friends of the agent; e.g., `resymbol_metrics.py`. :param metricSpecs: Sequence of one or more StockMetricSpec objects associated with the same stock symbol for which polling was conducted :param symbol: stock symbol :param sqlalchemy.engine.Engine engine: """ try: @collectorsdb.retryOnTransientErrors def _fetchUnsentSamples(): # Select only records that haven't been sent to BOTH fields = [ xigniteSecurityBars.c.StartDate, xigniteSecurityBars.c.StartTime, xigniteSecurityBars.c.EndDate, xigniteSecurityBars.c.EndTime, xigniteSecurityBars.c.UTCOffset, xigniteSecurityBars.c.Volume, xigniteSecurityBars.c.Close, emittedStockPrice.c.sent.label("Close_sent"), emittedStockVolume.c.sent.label("Volume_sent") ] sel = (select(fields).select_from( xigniteSecurityBars.outerjoin(emittedStockPrice).outerjoin( emittedStockVolume)).where( xigniteSecurityBars.c.symbol == symbol).where( (emittedStockPrice.c.sent == None) | (emittedStockVolume.c.sent == None)).order_by( xigniteSecurityBars.c.EndDate.asc(), xigniteSecurityBars.c.EndTime.asc())) return engine.execute(sel) # Process samples in chunks to facilitate more efficient error recovery # during backlog processing samplesIter = iter(_fetchUnsentSamples()) while True: specSymbolSampleList = [] sample = None for sample in itertools.islice(samplesIter, 0, 1000): for spec in metricSpecs: if not sample[spec.sampleKey + "_sent"]: specSymbolSampleList.append((spec, symbol, sample)) if sample is None: # No more unsent samples break # Send samples to Taurus with metricDataBatchWrite(log=_LOG) as putSample: for spec, symbol, sample in specSymbolSampleList: if spec.sampleKey in sample: epochTs = epochFromLocalizedDatetime( _EASTERN_TZ.localize( datetime.datetime.combine( sample.StartDate, sample.StartTime))) value = sample[spec.sampleKey] _LOG.info("Sending: %s %r %d", spec.metricName, value, epochTs) putSample(metricName=spec.metricName, value=value, epochTimestamp=epochTs) # Update history of emitted samples # # NOTE: If this fails once in a while and we end up resending the samples, # htmengine's Metric Storer will discard duplicate-timestamp and # out-of-order samples for spec, symbol, sample in specSymbolSampleList: _updateMetricDataHistory(spec=spec, symbol=symbol, sample=sample, engine=engine) except Exception: _LOG.exception("Unexpected error while attempting to send metric " "data sample(s) to remote Taurus instance.")