def test_missing_values_assertion(self):
sessions = self.trading_calendar.sessions_in_range(
TEST_CALENDAR_START,
TEST_CALENDAR_STOP,
)
sessions_with_gap = sessions[sessions != self.MISSING_DATA_DAY]
bar_data = make_bar_data(self.make_equity_info(), sessions_with_gap)
writer = BcolzDailyBarWriter(
self.tmpdir.path,
self.trading_calendar,
sessions[0],
sessions[-1],
)
# There are 21 sessions between the start and end date for this
# asset, and we excluded one.
expected_msg = re.escape(
"Got 20 rows for daily bars table with first day=2015-06-02, last "
"day=2015-06-30, expected 21 rows.\n"
"Missing sessions: "
"[Timestamp('2015-06-15 00:00:00+0000', tz='UTC')]\n"
"Extra sessions: []")
with self.assertRaisesRegexp(AssertionError, expected_msg):
writer.write(bar_data)
def test_missing_values_assertion(self):
sessions = self.trading_calendar.sessions_in_range(
TEST_CALENDAR_START,
TEST_CALENDAR_STOP,
)
sessions_with_gap = sessions[sessions != self.MISSING_DATA_DAY]
bar_data = make_bar_data(self.make_equity_info(), sessions_with_gap)
writer = BcolzDailyBarWriter(
self.tmpdir.path,
self.trading_calendar,
sessions[0],
sessions[-1],
)
# There are 21 sessions between the start and end date for this
# asset, and we excluded one.
expected_msg = re.escape(
"Got 20 rows for daily bars table with first day=2015-06-02, last "
"day=2015-06-30, expected 21 rows.\n"
"Missing sessions: "
"[Timestamp('2015-06-15 00:00:00+0000', tz='UTC')]\n"
"Extra sessions: []"
)
with self.assertRaisesRegexp(AssertionError, expected_msg):
writer.write(bar_data)
def write_daily_data(tempdir, sim_params, sids, trading_calendar):
path = os.path.join(tempdir.path, "testdaily.bcolz")
BcolzDailyBarWriter(path, trading_calendar,
sim_params.start_session,
sim_params.end_session).write(
create_daily_bar_data(sim_params.sessions, sids),
)
return path
def transaction_sim(self, **params):
"""This is a utility method that asserts expected
results for conversion of orders to transactions given a
trade history
"""
trade_count = params["trade_count"]
trade_interval = params["trade_interval"]
order_count = params["order_count"]
order_amount = params["order_amount"]
order_interval = params["order_interval"]
expected_txn_count = params["expected_txn_count"]
expected_txn_volume = params["expected_txn_volume"]
# optional parameters
# ---------------------
# if present, alternate between long and short sales
alternate = params.get("alternate")
# if present, expect transaction amounts to match orders exactly.
complete_fill = params.get("complete_fill")
asset1 = self.asset_finder.retrieve_asset(1)
with TempDirectory() as tempdir:
if trade_interval < timedelta(days=1):
sim_params = factory.create_simulation_parameters(
start=self.start, end=self.end, data_frequency="minute")
minutes = self.trading_calendar.minutes_window(
sim_params.first_open,
int((trade_interval.total_seconds() / 60) * trade_count) +
100,
)
price_data = np.array([10.1] * len(minutes))
assets = {
asset1.sid:
pd.DataFrame({
"open": price_data,
"high": price_data,
"low": price_data,
"close": price_data,
"volume": np.array([100] * len(minutes)),
"dt": minutes,
}).set_index("dt")
}
write_bcolz_minute_data(
self.trading_calendar,
self.trading_calendar.sessions_in_range(
self.trading_calendar.minute_to_session_label(
minutes[0]),
self.trading_calendar.minute_to_session_label(
minutes[-1]),
),
tempdir.path,
assets.items(),
)
equity_minute_reader = BcolzMinuteBarReader(tempdir.path)
data_portal = DataPortal(
self.asset_finder,
self.trading_calendar,
first_trading_day=equity_minute_reader.first_trading_day,
equity_minute_reader=equity_minute_reader,
)
else:
sim_params = factory.create_simulation_parameters(
data_frequency="daily")
days = sim_params.sessions
assets = {
1:
pd.DataFrame(
{
"open": [10.1] * len(days),
"high": [10.1] * len(days),
"low": [10.1] * len(days),
"close": [10.1] * len(days),
"volume": [100] * len(days),
"day": [day.value for day in days],
},
index=days,
)
}
path = os.path.join(tempdir.path, "testdata.bcolz")
BcolzDailyBarWriter(path, self.trading_calendar, days[0],
days[-1]).write(assets.items())
equity_daily_reader = BcolzDailyBarReader(path)
data_portal = DataPortal(
self.asset_finder,
self.trading_calendar,
first_trading_day=equity_daily_reader.first_trading_day,
equity_daily_reader=equity_daily_reader,
)
if "default_slippage" not in params or not params[
"default_slippage"]:
slippage_func = FixedBasisPointsSlippage()
else:
slippage_func = None
blotter = SimulationBlotter(slippage_func)
start_date = sim_params.first_open
if alternate:
alternator = -1
else:
alternator = 1
tracker = MetricsTracker(
trading_calendar=self.trading_calendar,
first_session=sim_params.start_session,
last_session=sim_params.end_session,
capital_base=sim_params.capital_base,
emission_rate=sim_params.emission_rate,
data_frequency=sim_params.data_frequency,
asset_finder=self.asset_finder,
metrics=load_metrics_set("none"),
)
# replicate what tradesim does by going through every minute or day
# of the simulation and processing open orders each time
if sim_params.data_frequency == "minute":
ticks = minutes
else:
ticks = days
transactions = []
order_list = []
order_date = start_date
for tick in ticks:
blotter.current_dt = tick
if tick >= order_date and len(order_list) < order_count:
# place an order
direction = alternator**len(order_list)
order_id = blotter.order(
asset1,
order_amount * direction,
MarketOrder(),
)
order_list.append(blotter.orders[order_id])
order_date = order_date + order_interval
# move after market orders to just after market next
# market open.
if order_date.hour >= 21:
if order_date.minute >= 00:
order_date = order_date + timedelta(days=1)
order_date = order_date.replace(hour=14, minute=30)
else:
bar_data = BarData(
data_portal=data_portal,
simulation_dt_func=lambda: tick,
data_frequency=sim_params.data_frequency,
trading_calendar=self.trading_calendar,
restrictions=NoRestrictions(),
)
txns, _, closed_orders = blotter.get_transactions(bar_data)
for txn in txns:
tracker.process_transaction(txn)
transactions.append(txn)
blotter.prune_orders(closed_orders)
for i in range(order_count):
order = order_list[i]
assert order.asset == asset1
assert order.amount == order_amount * alternator**i
if complete_fill:
assert len(transactions) == len(order_list)
total_volume = 0
for i in range(len(transactions)):
txn = transactions[i]
total_volume += txn.amount
if complete_fill:
order = order_list[i]
assert order.amount == txn.amount
assert total_volume == expected_txn_volume
assert len(transactions) == expected_txn_count
if total_volume == 0:
with pytest.raises(KeyError):
tracker.positions[asset1]
else:
cumulative_pos = tracker.positions[asset1]
assert total_volume == cumulative_pos.amount
# the open orders should not contain the asset.
oo = blotter.open_orders
assert asset1 not in oo, "Entry is removed when no open orders"
def create_data_portal_from_trade_history(asset_finder, trading_calendar,
tempdir, sim_params, trades_by_sid):
if sim_params.data_frequency == "daily":
path = os.path.join(tempdir.path, "testdaily.bcolz")
writer = BcolzDailyBarWriter(
path, trading_calendar,
sim_params.start_session,
sim_params.end_session
)
writer.write(
trades_by_sid_to_dfs(trades_by_sid, sim_params.sessions),
)
equity_daily_reader = BcolzDailyBarReader(path)
return DataPortal(
asset_finder, trading_calendar,
first_trading_day=equity_daily_reader.first_trading_day,
equity_daily_reader=equity_daily_reader,
)
else:
minutes = trading_calendar.minutes_in_range(
sim_params.first_open,
sim_params.last_close
)
length = len(minutes)
assets = {}
for sidint, trades in iteritems(trades_by_sid):
opens = np.zeros(length)
highs = np.zeros(length)
lows = np.zeros(length)
closes = np.zeros(length)
volumes = np.zeros(length)
for trade in trades:
# put them in the right place
idx = minutes.searchsorted(trade.dt)
opens[idx] = trade.open_price * 1000
highs[idx] = trade.high * 1000
lows[idx] = trade.low * 1000
closes[idx] = trade.close_price * 1000
volumes[idx] = trade.volume
assets[sidint] = pd.DataFrame({
"open": opens,
"high": highs,
"low": lows,
"close": closes,
"volume": volumes,
"dt": minutes
}).set_index("dt")
write_bcolz_minute_data(
trading_calendar,
sim_params.sessions,
tempdir.path,
assets
)
equity_minute_reader = BcolzMinuteBarReader(tempdir.path)
return DataPortal(
asset_finder, trading_calendar,
first_trading_day=equity_minute_reader.first_trading_day,
equity_minute_reader=equity_minute_reader,
)
def _write(cal, days, path, data):
BcolzDailyBarWriter(path, days).write(data)