def _saveSecurityNews(self, headlineRows, xigniteSecurity): """ Store security news in the destination schema specified via _NEWS_SCHEMA member variable. :param headlineRows: rows of field values for target security news table :type headlineRows: sequence of dicts :param dict xigniteSecurity: Security info from xignite API results (e.g., global security news, security bars, etc.) :returns: The count of new news rows that were saved; 0 if the news object has no headlines. """ destSchema = self._NEWS_SCHEMA if not headlineRows: return 0 if self.dryRun: g_log.info("%r.process(dryRun=True): security=%s, news=%s", self, xigniteSecurity, headlineRows) return 0 engine = collectorsdb.engineFactory() @collectorsdb.retryOnTransientErrors def saveNews(): with engine.begin() as conn: # Save headlines newsIns = destSchema.insert().prefix_with("IGNORE", dialect="mysql") return conn.execute(newsIns, headlineRows).rowcount try: return saveNews() except sql.exc.IntegrityError: # Most likely foreign key constraint violation against the # xignite_security table g_log.info("Inserting security row for symbol=%s", xigniteSecurity["Symbol"]) xignite_agent_utils.insertSecurity(engine, xigniteSecurity) # Re-insert news after resolving IntegrityError return saveNews()
def addSecurity(symbol): self.addCleanup(_deleteSecurity, symbol) xignite_agent_utils.insertSecurity( engine=collectorsdb.engineFactory(), xigniteSecurity={ "Symbol": symbol, "CIK": "CIK", "CUSIP": "CUSIP", "ISIN": "ISIN", "Valoren": "Valoren", "Name": "{sym} Inc.".format(sym=symbol), "Market": "Market", "MarketIdentificationCode": "mic1", "MostLiquidExchange": True, "CategoryOrIndustry": "CategoryOrIndustry" }) self.assertTrue(securityExists(symbol), "inserted {symbol} not found".format(symbol=symbol))
def addSecurity(symbol): self.addCleanup(_deleteSecurity, symbol) xignite_agent_utils.insertSecurity( engine=collectorsdb.engineFactory(), xigniteSecurity={ "Symbol": symbol, "CIK": "CIK", "CUSIP": "CUSIP", "ISIN": "ISIN", "Valoren": "Valoren", "Name": "{sym} Inc.".format(sym=symbol), "Market": "Market", "MarketIdentificationCode": "mic1", "MostLiquidExchange": True, "CategoryOrIndustry": "CategoryOrIndustry" }) self.assertTrue( securityExists(symbol), "inserted {symbol} not found".format(symbol=symbol))
def forward(metricSpecs, data, security, server=DEFAULT_SERVER, port=DEFAULT_PORT, dryrun=DEFAULT_DRYRUN): """ Forward stock data to Grok/Taurus instance via custom metric :param metricSpecs: Sequence of one or more StockMetricSpec objects associated with the same stock symbol for which polling was conducted :param list data: List of sample dicts :param dict security: Details of security from XIgnite API """ try: symbol = security["Symbol"] engine = collectorsdb.engineFactory() lastSample = _getLatestSample(engine, symbol) if lastSample: localizedLastEndTime = ( getEasternLocalizedEndTimestampFromSampleRow(lastSample)) else: localizedLastEndTime = None # Implemented in two phases: # # 1. Buffer records to collectorsdb for sample in data: localizedSampleStartTime = ( getEasternLocalizedTimestampFromSample(sample["StartDate"], sample["StartTime"], sample["UTCOffset"])) if localizedSampleStartTime.time() < NAIVE_MARKET_OPEN_TIME: # Ignore samples that preceed market open _LOG.info("Skipping data before market hours: %s @ %s sample=%s", symbol, localizedSampleStartTime, sample) continue if localizedSampleStartTime.time() >= NAIVE_MARKET_CLOSE_TIME: # Ignore a quirk of the xignite API that duplicates some data at # end of trading day. This also excludes the closing auction on # NYSE. _LOG.info("Skipping data after market hours: %s @ %s sample=%s", symbol, localizedSampleStartTime, sample) continue if not lastSample or (localizedSampleStartTime >= localizedLastEndTime): # Current sample starts at, or after last recorded timestamp ends localizedSampleEndTime = ( getEasternLocalizedTimestampFromSample(sample["EndDate"], sample["EndTime"], sample["UTCOffset"])) ins = (xigniteSecurityBars .insert() .values(symbol=symbol, StartDate=localizedSampleStartTime.date(), StartTime=localizedSampleStartTime.time(), EndDate=localizedSampleEndTime.date(), EndTime=localizedSampleEndTime.time(), UTCOffset=sample["UTCOffset"], Open=sample["Open"], High=sample["High"], Low=sample["Low"], Close=sample["Close"], Volume=sample["Volume"], Trades=sample["Trades"])) @collectorsdb.retryOnTransientErrors def _insertBar(): engine.execute(ins) try: _insertBar() except IntegrityError: # Most likely foreign key constraint violation against the # xignite_security table _LOG.info("Inserting security row for symbol=%s", symbol) xignite_agent_utils.insertSecurity(engine, security) # Re-insert after resolving IntegrityError _insertBar() # 2. If in active mode, send ALL un-sent records to Taurus if g_opMode != ApplicationConfig.OP_MODE_ACTIVE: return transmitMetricData(metricSpecs=metricSpecs, symbol=symbol, engine=engine) except Exception: _LOG.exception("forward failed for metricSpecs=%s", metricSpecs) raise
def forward(metricSpecs, data, security, server=DEFAULT_SERVER, port=DEFAULT_PORT, dryrun=DEFAULT_DRYRUN): """ Forward stock data to Grok/Taurus instance via custom metric :param metricSpecs: Sequence of one or more StockMetricSpec objects associated with the same stock symbol for which polling was conducted :param list data: List of sample dicts :param dict security: Details of security from XIgnite API """ try: symbol = security["Symbol"] engine = collectorsdb.engineFactory() lastSample = _getLatestSample(engine, symbol) if lastSample: localizedLastEndTime = ( getEasternLocalizedEndTimestampFromSampleRow(lastSample)) else: localizedLastEndTime = None # Implemented in two phases: # # 1. Buffer records to collectorsdb for sample in data: localizedSampleStartTime = (getEasternLocalizedTimestampFromSample( sample["StartDate"], sample["StartTime"], sample["UTCOffset"])) if localizedSampleStartTime.time() < NAIVE_MARKET_OPEN_TIME: # Ignore samples that preceed market open _LOG.info( "Skipping data before market hours: %s @ %s sample=%s", symbol, localizedSampleStartTime, sample) continue if localizedSampleStartTime.time() >= NAIVE_MARKET_CLOSE_TIME: # Ignore a quirk of the xignite API that duplicates some data at # end of trading day. This also excludes the closing auction on # NYSE. _LOG.info( "Skipping data after market hours: %s @ %s sample=%s", symbol, localizedSampleStartTime, sample) continue if not lastSample or (localizedSampleStartTime >= localizedLastEndTime): # Current sample starts at, or after last recorded timestamp ends localizedSampleEndTime = ( getEasternLocalizedTimestampFromSample( sample["EndDate"], sample["EndTime"], sample["UTCOffset"])) ins = (xigniteSecurityBars.insert().values( symbol=symbol, StartDate=localizedSampleStartTime.date(), StartTime=localizedSampleStartTime.time(), EndDate=localizedSampleEndTime.date(), EndTime=localizedSampleEndTime.time(), UTCOffset=sample["UTCOffset"], Open=sample["Open"], High=sample["High"], Low=sample["Low"], Close=sample["Close"], Volume=sample["Volume"], Trades=sample["Trades"])) @collectorsdb.retryOnTransientErrors def _insertBar(): engine.execute(ins) try: _insertBar() except IntegrityError: # Most likely foreign key constraint violation against the # xignite_security table _LOG.info("Inserting security row for symbol=%s", symbol) xignite_agent_utils.insertSecurity(engine, security) # Re-insert after resolving IntegrityError _insertBar() # 2. If in active mode, send ALL un-sent records to Taurus if g_opMode != ApplicationConfig.OP_MODE_ACTIVE: return transmitMetricData(metricSpecs=metricSpecs, symbol=symbol, engine=engine) except Exception: _LOG.exception("forward failed for metricSpecs=%s", metricSpecs) raise