def test_minute_to_session(self): for sid in self.ASSET_FINDER_EQUITY_SIDS: frame = self.equity_frames[sid] expected = EXPECTED_SESSIONS[sid] result = minute_frame_to_session_frame(frame, self.nyse_calendar) assert_almost_equal(expected.values, result.values, err_msg='sid={0}'.format(sid))
def _consume_prices(self, minute_bar_writer, calendar): """ Pulls (conid, prices) from the queue and hands to Zipline for ingestion. """ # Consume tasks from the queue while True: security = self.minute_ingestion_queue.get() # None indicates to terminate the worker if security is None: break conid, security, prices = security print("Ingesting {0} minute bars for {1} {2} (conid {3})".format( len(prices.index), security.Symbol, security.SecType, conid)) # Drop any minutes that are outside of the trading session (IB # data often includes bars from outside regular trading hours # even when regular trading hours are requested) prices = prices.tz_localize("UTC") idx = prices.index.intersection(calendar.all_minutes) prices = prices.reindex(index=idx) try: # Ingest minute bars minute_bar_writer.write_sid(conid, prices) # roll up minute to daily and enqueue to ingest daily daily_prices = minute_frame_to_session_frame(prices, calendar) daily_prices = self._reindex_and_fillna_missing_sessions( daily_prices, calendar) self.daily_ingestion_queue.put((conid, security, daily_prices)) except Exception as e: import traceback tb = traceback.format_exc() msg = "error ingesting {0} {1} (conid {2})".format( security.Symbol, security.SecType, conid) print(msg) print(tb) logger.error( "{0}, see detailed logs for traceback, continuing with next security" .format(msg)) self.conids_with_errors.add(conid)