def _saveSecurityNews(self, headlineRows, xigniteSecurity):
""" Store security news in the destination schema specified via
_NEWS_SCHEMA member variable.
:param headlineRows: rows of field values for target security news table
:type headlineRows: sequence of dicts
:param dict xigniteSecurity: Security info from xignite API results (e.g.,
global security news, security bars, etc.)
:returns: The count of new news rows that were saved; 0 if the news object
has no headlines.
"""
destSchema = self._NEWS_SCHEMA
if not headlineRows:
return 0
if self.dryRun:
g_log.info("%r.process(dryRun=True): security=%s, news=%s", self,
xigniteSecurity, headlineRows)
return 0
engine = collectorsdb.engineFactory()
@collectorsdb.retryOnTransientErrors
def saveNews():
with engine.begin() as conn:
# Save headlines
newsIns = destSchema.insert().prefix_with("IGNORE",
dialect="mysql")
return conn.execute(newsIns, headlineRows).rowcount
try:
return saveNews()
except sql.exc.IntegrityError:
# Most likely foreign key constraint violation against the
# xignite_security table
g_log.info("Inserting security row for symbol=%s",
xigniteSecurity["Symbol"])
xignite_agent_utils.insertSecurity(engine, xigniteSecurity)
# Re-insert news after resolving IntegrityError
return saveNews()
def addSecurity(symbol):
self.addCleanup(_deleteSecurity, symbol)
xignite_agent_utils.insertSecurity(
engine=collectorsdb.engineFactory(),
xigniteSecurity={
"Symbol": symbol,
"CIK": "CIK",
"CUSIP": "CUSIP",
"ISIN": "ISIN",
"Valoren": "Valoren",
"Name": "{sym} Inc.".format(sym=symbol),
"Market": "Market",
"MarketIdentificationCode": "mic1",
"MostLiquidExchange": True,
"CategoryOrIndustry": "CategoryOrIndustry"
})
self.assertTrue(securityExists(symbol),
"inserted {symbol} not found".format(symbol=symbol))
def _saveSecurityNews(self, headlineRows, xigniteSecurity):
""" Store security news in the destination schema specified via
_NEWS_SCHEMA member variable.
:param headlineRows: rows of field values for target security news table
:type headlineRows: sequence of dicts
:param dict xigniteSecurity: Security info from xignite API results (e.g.,
global security news, security bars, etc.)
:returns: The count of new news rows that were saved; 0 if the news object
has no headlines.
"""
destSchema = self._NEWS_SCHEMA
if not headlineRows:
return 0
if self.dryRun:
g_log.info("%r.process(dryRun=True): security=%s, news=%s", self,
xigniteSecurity, headlineRows)
return 0
engine = collectorsdb.engineFactory()
@collectorsdb.retryOnTransientErrors
def saveNews():
with engine.begin() as conn:
# Save headlines
newsIns = destSchema.insert().prefix_with("IGNORE", dialect="mysql")
return conn.execute(newsIns, headlineRows).rowcount
try:
return saveNews()
except sql.exc.IntegrityError:
# Most likely foreign key constraint violation against the
# xignite_security table
g_log.info("Inserting security row for symbol=%s",
xigniteSecurity["Symbol"])
xignite_agent_utils.insertSecurity(engine, xigniteSecurity)
# Re-insert news after resolving IntegrityError
return saveNews()
def addSecurity(symbol):
self.addCleanup(_deleteSecurity, symbol)
xignite_agent_utils.insertSecurity(
engine=collectorsdb.engineFactory(),
xigniteSecurity={
"Symbol": symbol,
"CIK": "CIK",
"CUSIP": "CUSIP",
"ISIN": "ISIN",
"Valoren": "Valoren",
"Name": "{sym} Inc.".format(sym=symbol),
"Market": "Market",
"MarketIdentificationCode": "mic1",
"MostLiquidExchange": True,
"CategoryOrIndustry": "CategoryOrIndustry"
})
self.assertTrue(
securityExists(symbol),
"inserted {symbol} not found".format(symbol=symbol))
def forward(metricSpecs, data, security, server=DEFAULT_SERVER,
port=DEFAULT_PORT,
dryrun=DEFAULT_DRYRUN):
""" Forward stock data to Grok/Taurus instance via custom metric
:param metricSpecs: Sequence of one or more StockMetricSpec objects associated
with the same stock symbol for which polling was conducted
:param list data: List of sample dicts
:param dict security: Details of security from XIgnite API
"""
try:
symbol = security["Symbol"]
engine = collectorsdb.engineFactory()
lastSample = _getLatestSample(engine, symbol)
if lastSample:
localizedLastEndTime = (
getEasternLocalizedEndTimestampFromSampleRow(lastSample))
else:
localizedLastEndTime = None
# Implemented in two phases:
#
# 1. Buffer records to collectorsdb
for sample in data:
localizedSampleStartTime = (
getEasternLocalizedTimestampFromSample(sample["StartDate"],
sample["StartTime"],
sample["UTCOffset"]))
if localizedSampleStartTime.time() < NAIVE_MARKET_OPEN_TIME:
# Ignore samples that preceed market open
_LOG.info("Skipping data before market hours: %s @ %s sample=%s",
symbol, localizedSampleStartTime, sample)
continue
if localizedSampleStartTime.time() >= NAIVE_MARKET_CLOSE_TIME:
# Ignore a quirk of the xignite API that duplicates some data at
# end of trading day. This also excludes the closing auction on
# NYSE.
_LOG.info("Skipping data after market hours: %s @ %s sample=%s",
symbol, localizedSampleStartTime, sample)
continue
if not lastSample or (localizedSampleStartTime >= localizedLastEndTime):
# Current sample starts at, or after last recorded timestamp ends
localizedSampleEndTime = (
getEasternLocalizedTimestampFromSample(sample["EndDate"],
sample["EndTime"],
sample["UTCOffset"]))
ins = (xigniteSecurityBars
.insert()
.values(symbol=symbol,
StartDate=localizedSampleStartTime.date(),
StartTime=localizedSampleStartTime.time(),
EndDate=localizedSampleEndTime.date(),
EndTime=localizedSampleEndTime.time(),
UTCOffset=sample["UTCOffset"],
Open=sample["Open"],
High=sample["High"],
Low=sample["Low"],
Close=sample["Close"],
Volume=sample["Volume"],
Trades=sample["Trades"]))
@collectorsdb.retryOnTransientErrors
def _insertBar():
engine.execute(ins)
try:
_insertBar()
except IntegrityError:
# Most likely foreign key constraint violation against the
# xignite_security table
_LOG.info("Inserting security row for symbol=%s", symbol)
xignite_agent_utils.insertSecurity(engine, security)
# Re-insert after resolving IntegrityError
_insertBar()
# 2. If in active mode, send ALL un-sent records to Taurus
if g_opMode != ApplicationConfig.OP_MODE_ACTIVE:
return
transmitMetricData(metricSpecs=metricSpecs, symbol=symbol, engine=engine)
except Exception:
_LOG.exception("forward failed for metricSpecs=%s", metricSpecs)
raise
def forward(metricSpecs,
data,
security,
server=DEFAULT_SERVER,
port=DEFAULT_PORT,
dryrun=DEFAULT_DRYRUN):
""" Forward stock data to Grok/Taurus instance via custom metric
:param metricSpecs: Sequence of one or more StockMetricSpec objects associated
with the same stock symbol for which polling was conducted
:param list data: List of sample dicts
:param dict security: Details of security from XIgnite API
"""
try:
symbol = security["Symbol"]
engine = collectorsdb.engineFactory()
lastSample = _getLatestSample(engine, symbol)
if lastSample:
localizedLastEndTime = (
getEasternLocalizedEndTimestampFromSampleRow(lastSample))
else:
localizedLastEndTime = None
# Implemented in two phases:
#
# 1. Buffer records to collectorsdb
for sample in data:
localizedSampleStartTime = (getEasternLocalizedTimestampFromSample(
sample["StartDate"], sample["StartTime"], sample["UTCOffset"]))
if localizedSampleStartTime.time() < NAIVE_MARKET_OPEN_TIME:
# Ignore samples that preceed market open
_LOG.info(
"Skipping data before market hours: %s @ %s sample=%s",
symbol, localizedSampleStartTime, sample)
continue
if localizedSampleStartTime.time() >= NAIVE_MARKET_CLOSE_TIME:
# Ignore a quirk of the xignite API that duplicates some data at
# end of trading day. This also excludes the closing auction on
# NYSE.
_LOG.info(
"Skipping data after market hours: %s @ %s sample=%s",
symbol, localizedSampleStartTime, sample)
continue
if not lastSample or (localizedSampleStartTime >=
localizedLastEndTime):
# Current sample starts at, or after last recorded timestamp ends
localizedSampleEndTime = (
getEasternLocalizedTimestampFromSample(
sample["EndDate"], sample["EndTime"],
sample["UTCOffset"]))
ins = (xigniteSecurityBars.insert().values(
symbol=symbol,
StartDate=localizedSampleStartTime.date(),
StartTime=localizedSampleStartTime.time(),
EndDate=localizedSampleEndTime.date(),
EndTime=localizedSampleEndTime.time(),
UTCOffset=sample["UTCOffset"],
Open=sample["Open"],
High=sample["High"],
Low=sample["Low"],
Close=sample["Close"],
Volume=sample["Volume"],
Trades=sample["Trades"]))
@collectorsdb.retryOnTransientErrors
def _insertBar():
engine.execute(ins)
try:
_insertBar()
except IntegrityError:
# Most likely foreign key constraint violation against the
# xignite_security table
_LOG.info("Inserting security row for symbol=%s", symbol)
xignite_agent_utils.insertSecurity(engine, security)
# Re-insert after resolving IntegrityError
_insertBar()
# 2. If in active mode, send ALL un-sent records to Taurus
if g_opMode != ApplicationConfig.OP_MODE_ACTIVE:
return
transmitMetricData(metricSpecs=metricSpecs,
symbol=symbol,
engine=engine)
except Exception:
_LOG.exception("forward failed for metricSpecs=%s", metricSpecs)
raise