def test_minute_to_session(self):
for sid in self.ASSET_FINDER_EQUITY_SIDS:
frame = self.equity_frames[sid]
expected = EXPECTED_SESSIONS[sid]
result = minute_frame_to_session_frame(frame, self.nyse_calendar)
assert_almost_equal(expected.values,
result.values,
err_msg='sid={0}'.format(sid))
def test_minute_to_session(self):
for sid in self.ASSET_FINDER_EQUITY_SIDS:
frame = self.equity_frames[sid]
expected = EXPECTED_SESSIONS[sid]
result = minute_frame_to_session_frame(frame, self.nyse_calendar)
assert_almost_equal(expected.values,
result.values,
err_msg='sid={0}'.format(sid))
def _consume_prices(self, minute_bar_writer, calendar):
"""
Pulls (conid, prices) from the queue and hands to Zipline for
ingestion.
"""
# Consume tasks from the queue
while True:
security = self.minute_ingestion_queue.get()
# None indicates to terminate the worker
if security is None:
break
conid, security, prices = security
print("Ingesting {0} minute bars for {1} {2} (conid {3})".format(
len(prices.index), security.Symbol, security.SecType, conid))
# Drop any minutes that are outside of the trading session (IB
# data often includes bars from outside regular trading hours
# even when regular trading hours are requested)
prices = prices.tz_localize("UTC")
idx = prices.index.intersection(calendar.all_minutes)
prices = prices.reindex(index=idx)
try:
# Ingest minute bars
minute_bar_writer.write_sid(conid, prices)
# roll up minute to daily and enqueue to ingest daily
daily_prices = minute_frame_to_session_frame(prices, calendar)
daily_prices = self._reindex_and_fillna_missing_sessions(
daily_prices, calendar)
self.daily_ingestion_queue.put((conid, security, daily_prices))
except Exception as e:
import traceback
tb = traceback.format_exc()
msg = "error ingesting {0} {1} (conid {2})".format(
security.Symbol, security.SecType, conid)
print(msg)
print(tb)
logger.error(
"{0}, see detailed logs for traceback, continuing with next security"
.format(msg))
self.conids_with_errors.add(conid)