def test_missing_values_assertion(self):
sessions = self.trading_calendar.sessions_in_range(
TEST_CALENDAR_START,
TEST_CALENDAR_STOP,
)
sessions_with_gap = sessions[sessions != self.MISSING_DATA_DAY]
bar_data = make_bar_data(self.make_equity_info(), sessions_with_gap)
writer = BcolzDailyBarWriter(
self.tmpdir.path,
self.trading_calendar,
sessions[0],
sessions[-1],
)
# There are 21 sessions between the start and end date for this
# asset, and we excluded one.
expected_msg = re.escape(
"Got 20 rows for daily bars table with first day=2015-06-02, last "
"day=2015-06-30, expected 21 rows.\n"
"Missing sessions: "
"[Timestamp('2015-06-15 00:00:00+0000', tz='UTC')]\n"
"Extra sessions: []")
with self.assertRaisesRegexp(AssertionError, expected_msg):
writer.write(bar_data)
def ingest(
environ: Mapping,
asset_db_writer: AssetDBWriter,
minute_bar_writer: BcolzMinuteBarWriter,
daily_bar_writer: BcolzDailyBarWriter,
adjustment_writer: SQLiteAdjustmentWriter,
calendar: TradingCalendar,
start_session: pd.Timestamp,
end_session: pd.Timestamp,
cache: dataframe_cache,
show_progress: bool,
output_dir: Text,
) -> NoReturn:
sid_map = list(zip(range(len(symbols)), symbols))
asset_db_writer.write(
futures=_get_metadata(sid_map),
exchanges=pd.DataFrame(data=[['bitmex', 'UTC']],
columns=['exchange', 'timezone']),
)
minute_bar_writer.write(
_get_minute_bars(sid_map, start_session, end_session, cache),
show_progress=show_progress,
)
daily_bar_writer.write(
_get_daily_bars(sid_map, start_session, end_session, cache),
show_progress=show_progress,
)
def test_missing_values_assertion(self):
sessions = self.trading_calendar.sessions_in_range(
TEST_CALENDAR_START,
TEST_CALENDAR_STOP,
)
sessions_with_gap = sessions[sessions != self.MISSING_DATA_DAY]
bar_data = make_bar_data(self.make_equity_info(), sessions_with_gap)
writer = BcolzDailyBarWriter(
self.tmpdir.path,
self.trading_calendar,
sessions[0],
sessions[-1],
)
# There are 21 sessions between the start and end date for this
# asset, and we excluded one.
expected_msg = re.escape(
"Got 20 rows for daily bars table with first day=2015-06-02, last "
"day=2015-06-30, expected 21 rows.\n"
"Missing sessions: "
"[Timestamp('2015-06-15 00:00:00+0000', tz='UTC')]\n"
"Extra sessions: []"
)
with self.assertRaisesRegexp(AssertionError, expected_msg):
writer.write(bar_data)
def create_data_portal_from_trade_history(asset_finder, trading_calendar,
tempdir, sim_params, trades_by_sid):
if sim_params.data_frequency == "daily":
path = os.path.join(tempdir.path, "testdaily.bcolz")
writer = BcolzDailyBarWriter(
path, trading_calendar,
sim_params.start_session,
sim_params.end_session
)
writer.write(
trades_by_sid_to_dfs(trades_by_sid, sim_params.sessions),
)
equity_daily_reader = BcolzDailyBarReader(path)
return DataPortal(
asset_finder, trading_calendar,
first_trading_day=equity_daily_reader.first_trading_day,
equity_daily_reader=equity_daily_reader,
)
else:
minutes = trading_calendar.minutes_in_range(
sim_params.first_open,
sim_params.last_close
)
length = len(minutes)
assets = {}
for sidint, trades in iteritems(trades_by_sid):
opens = np.zeros(length)
highs = np.zeros(length)
lows = np.zeros(length)
closes = np.zeros(length)
volumes = np.zeros(length)
for trade in trades:
# put them in the right place
idx = minutes.searchsorted(trade.dt)
opens[idx] = trade.open_price * 1000
highs[idx] = trade.high * 1000
lows[idx] = trade.low * 1000
closes[idx] = trade.close_price * 1000
volumes[idx] = trade.volume
assets[sidint] = pd.DataFrame({
"open": opens,
"high": highs,
"low": lows,
"close": closes,
"volume": volumes,
"dt": minutes
}).set_index("dt")
write_bcolz_minute_data(
trading_calendar,
sim_params.sessions,
tempdir.path,
assets
)
equity_minute_reader = BcolzMinuteBarReader(tempdir.path)
return DataPortal(
asset_finder, trading_calendar,
first_trading_day=equity_minute_reader.first_trading_day,
equity_minute_reader=equity_minute_reader,
)
def create_data_portal_from_trade_history(asset_finder, trading_calendar,
tempdir, sim_params, trades_by_sid):
if sim_params.data_frequency == "daily":
path = os.path.join(tempdir.path, "testdaily.bcolz")
writer = BcolzDailyBarWriter(
path, trading_calendar,
sim_params.start_session,
sim_params.end_session
)
writer.write(
trades_by_sid_to_dfs(trades_by_sid, sim_params.sessions),
)
equity_daily_reader = BcolzDailyBarReader(path)
return DataPortal(
asset_finder, trading_calendar,
first_trading_day=equity_daily_reader.first_trading_day,
equity_daily_reader=equity_daily_reader,
)
else:
minutes = trading_calendar.minutes_in_range(
sim_params.first_open,
sim_params.last_close
)
length = len(minutes)
assets = {}
for sidint, trades in iteritems(trades_by_sid):
opens = np.zeros(length)
highs = np.zeros(length)
lows = np.zeros(length)
closes = np.zeros(length)
volumes = np.zeros(length)
for trade in trades:
# put them in the right place
idx = minutes.searchsorted(trade.dt)
opens[idx] = trade.open_price * 1000
highs[idx] = trade.high * 1000
lows[idx] = trade.low * 1000
closes[idx] = trade.close_price * 1000
volumes[idx] = trade.volume
assets[sidint] = pd.DataFrame({
"open": opens,
"high": highs,
"low": lows,
"close": closes,
"volume": volumes,
"dt": minutes
}).set_index("dt")
write_bcolz_minute_data(
trading_calendar,
sim_params.sessions,
tempdir.path,
assets
)
equity_minute_reader = BcolzMinuteBarReader(tempdir.path)
return DataPortal(
asset_finder, trading_calendar,
first_trading_day=equity_minute_reader.first_trading_day,
equity_minute_reader=equity_minute_reader,
)
