def test_unadjusted_spot_price_no_data(self):
table = self.bcolz_daily_bar_ctable
reader = BcolzDailyBarReader(table)
# before
with self.assertRaises(NoDataOnDate):
reader.spot_price(2, Timestamp('2015-06-08', tz='UTC'), 'close')
# after
with self.assertRaises(NoDataOnDate):
reader.spot_price(4, Timestamp('2015-06-16', tz='UTC'), 'close')
def test_unadjusted_spot_price_no_data(self):
table = self.writer.write(self.dest, self.trading_days, self.assets)
reader = BcolzDailyBarReader(table)
# before
with self.assertRaises(NoDataOnDate):
reader.spot_price(2, Timestamp('2015-06-08', tz='UTC'), 'close')
# after
with self.assertRaises(NoDataOnDate):
reader.spot_price(4, Timestamp('2015-06-16', tz='UTC'), 'close')
def _check_read_results(self, columns, assets, start_date, end_date):
table = self.writer.write(self.dest, self.trading_days, self.assets)
reader = BcolzDailyBarReader(table)
results = reader.load_raw_arrays(columns, start_date, end_date, assets)
dates = self.trading_days_between(start_date, end_date)
for column, result in zip(columns, results):
assert_array_equal(
result,
self.writer.expected_values_2d(
dates,
assets,
column.name,
))
def _check_read_results(self, columns, assets, start_date, end_date):
table = self.writer.write(self.dest, self.trading_days, self.assets)
reader = BcolzDailyBarReader(table)
results = reader.load_raw_arrays(columns, start_date, end_date, assets)
dates = self.trading_days_between(start_date, end_date)
for column, result in zip(columns, results):
assert_array_equal(
result,
self.writer.expected_values_2d(
dates,
assets,
column.name,
)
)
def create_data_portal(env, tempdir, sim_params, sids, adjustment_reader=None):
if sim_params.data_frequency == "daily":
daily_path = write_daily_data(tempdir, sim_params, sids)
equity_daily_reader = BcolzDailyBarReader(daily_path)
return DataPortal(
env,
equity_daily_reader=equity_daily_reader,
adjustment_reader=adjustment_reader
)
else:
minutes = env.minutes_for_days_in_range(
sim_params.first_open,
sim_params.last_close
)
minute_path = write_minute_data(env, tempdir, minutes, sids)
equity_minute_reader = BcolzMinuteBarReader(minute_path)
return DataPortal(
env,
equity_minute_reader=equity_minute_reader,
adjustment_reader=adjustment_reader
)
def test_unadjusted_spot_price_empty_value(self):
table = self.writer.write(self.dest, self.trading_days, self.assets)
reader = BcolzDailyBarReader(table)
# A sid, day and corresponding index into which to overwrite a zero.
zero_sid = 1
zero_day = Timestamp('2015-06-02', tz='UTC')
zero_ix = reader.sid_day_index(zero_sid, zero_day)
# Write a zero into the synthetic pricing data at the day and sid,
# so that a read should now return -1.
# This a little hacky, in lieu of changing the synthetic data set.
reader._spot_col('close')[zero_ix] = 0
close = reader.spot_price(zero_sid, zero_day, 'close')
self.assertEqual(-1, close)
def create_data_portal(asset_finder, tempdir, sim_params, sids,
trading_calendar, adjustment_reader=None):
if sim_params.data_frequency == "daily":
daily_path = write_daily_data(tempdir, sim_params, sids,
trading_calendar)
equity_daily_reader = BcolzDailyBarReader(daily_path)
return DataPortal(
asset_finder, trading_calendar,
first_trading_day=equity_daily_reader.first_trading_day,
equity_daily_reader=equity_daily_reader,
adjustment_reader=adjustment_reader
)
else:
minutes = trading_calendar.minutes_in_range(
sim_params.first_open,
sim_params.last_close
)
minute_path = write_minute_data(trading_calendar, tempdir, minutes,
sids)
equity_minute_reader = BcolzMinuteBarReader(minute_path)
return DataPortal(
asset_finder, trading_calendar,
first_trading_day=equity_minute_reader.first_trading_day,
equity_minute_reader=equity_minute_reader,
adjustment_reader=adjustment_reader
)
def setUpClass(cls):
setup_logger(cls)
cls.env = trading.TradingEnvironment()
cls.sim_params = factory.create_simulation_parameters(
start=pd.Timestamp("2006-01-05", tz='UTC'),
end=pd.Timestamp("2006-01-06", tz='UTC'))
cls.env.write_data(
equities_data={
24: {
'start_date':
cls.sim_params.trading_days[0],
'end_date':
cls.env.next_trading_day(cls.sim_params.trading_days[-1])
},
25: {
'start_date':
cls.sim_params.trading_days[0],
'end_date':
cls.env.next_trading_day(cls.sim_params.trading_days[-1])
}
})
cls.tempdir = TempDirectory()
assets = {
24:
pd.DataFrame({
"open": [50, 50],
"high": [50, 50],
"low": [50, 50],
"close": [50, 50],
"volume": [100, 400],
"day": [day.value for day in cls.sim_params.trading_days]
}),
25:
pd.DataFrame({
"open": [50, 50],
"high": [50, 50],
"low": [50, 50],
"close": [50, 50],
"volume": [100, 400],
"day": [day.value for day in cls.sim_params.trading_days]
})
}
path = os.path.join(cls.tempdir.path, "tempdata.bcolz")
DailyBarWriterFromDataFrames(assets).write(path,
cls.sim_params.trading_days,
assets)
equity_daily_reader = BcolzDailyBarReader(path)
cls.data_portal = DataPortal(
cls.env,
equity_daily_reader=equity_daily_reader,
)
def test_unadjusted_spot_price(self):
table = self.writer.write(self.dest, self.trading_days, self.assets)
reader = BcolzDailyBarReader(table)
# At beginning
price = reader.spot_price(1, Timestamp('2015-06-01', tz='UTC'),
'close')
# Synthetic writes price for date.
self.assertEqual(135630.0, price)
# Middle
price = reader.spot_price(1, Timestamp('2015-06-02', tz='UTC'),
'close')
self.assertEqual(135631.0, price)
# End
price = reader.spot_price(1, Timestamp('2015-06-05', tz='UTC'),
'close')
self.assertEqual(135634.0, price)
# Another sid at beginning.
price = reader.spot_price(2, Timestamp('2015-06-22', tz='UTC'),
'close')
self.assertEqual(235651.0, price)
# Ensure that volume does not have float adjustment applied.
volume = reader.spot_price(1, Timestamp('2015-06-02', tz='UTC'),
'volume')
self.assertEqual(145631, volume)
def test_unadjusted_spot_price(self):
table = self.writer.write(self.dest, self.trading_days, self.assets)
reader = BcolzDailyBarReader(table)
# At beginning
price = reader.spot_price(1, Timestamp('2015-06-01', tz='UTC'),
'close')
# Synthetic writes price for date.
self.assertEqual(135630.0, price)
# Middle
price = reader.spot_price(1, Timestamp('2015-06-02', tz='UTC'),
'close')
self.assertEqual(135631.0, price)
# End
price = reader.spot_price(1, Timestamp('2015-06-05', tz='UTC'),
'close')
self.assertEqual(135634.0, price)
# Another sid at beginning.
price = reader.spot_price(2, Timestamp('2015-06-22', tz='UTC'),
'close')
self.assertEqual(235651.0, price)
# Ensure that volume does not have float adjustment applied.
volume = reader.spot_price(1, Timestamp('2015-06-02', tz='UTC'),
'volume')
self.assertEqual(145631, volume)
def from_files(cls, pricing_path, adjustments_path):
"""
Create a loader from a bcolz equity pricing dir and a SQLite
adjustments path.
Parameters
----------
pricing_path : str
Path to a bcolz directory written by a BcolzDailyBarWriter.
adjusments_path : str
Path to an adjusments db written by a SQLiteAdjustmentWriter.
"""
return cls(BcolzDailyBarReader(pricing_path),
SQLiteAdjustmentReader(adjustments_path))
def test_unadjusted_spot_price_no_data(self):
table = self.bcolz_daily_bar_ctable
reader = BcolzDailyBarReader(table)
# before
with self.assertRaises(NoDataOnDate):
reader.spot_price(2, Timestamp('2015-06-08', tz='UTC'), 'close')
# after
with self.assertRaises(NoDataOnDate):
reader.spot_price(4, Timestamp('2015-06-16', tz='UTC'), 'close')
def test_unadjusted_spot_price_no_data(self):
table = self.writer.write(self.dest, self.trading_days, self.assets)
reader = BcolzDailyBarReader(table)
# before
with self.assertRaises(NoDataOnDate):
reader.spot_price(2, Timestamp('2015-06-08', tz='UTC'), 'close')
# after
with self.assertRaises(NoDataOnDate):
reader.spot_price(4, Timestamp('2015-06-16', tz='UTC'), 'close')
def create_bar_reader(cls, tempdir):
resources = {
cls.AAPL: join(TEST_RESOURCE_PATH, 'AAPL.csv'),
cls.MSFT: join(TEST_RESOURCE_PATH, 'MSFT.csv'),
cls.BRK_A: join(TEST_RESOURCE_PATH, 'BRK-A.csv'),
}
raw_data = {
asset: read_csv(path, parse_dates=['day']).set_index('day')
for asset, path in iteritems(resources)
}
# Add 'price' column as an alias because all kinds of stuff in zipline
# depends on it being present. :/
for frame in raw_data.values():
frame['price'] = frame['close']
writer = DailyBarWriterFromCSVs(resources)
data_path = tempdir.getpath('testdata.bcolz')
table = writer.write(data_path, trading_days, cls.assets)
return raw_data, BcolzDailyBarReader(table)
def build_daily_data(cls):
path = cls.tempdir.getpath("testdaily.bcolz")
dfs = {
1:
create_daily_df_for_asset(cls.env, cls.trading_days[0],
cls.trading_days[-1]),
2:
create_daily_df_for_asset(cls.env, cls.trading_days[0],
cls.trading_days[-1]),
3:
create_daily_df_for_asset(cls.env, cls.trading_days[0],
cls.trading_days[-1])
}
daily_writer = DailyBarWriterFromDataFrames(dfs)
daily_writer.write(path, cls.trading_days, dfs)
return BcolzDailyBarReader(path)
def setUpClass(cls):
cls.first_asset_start = Timestamp('2015-04-01', tz='UTC')
cls.env = TradingEnvironment()
cls.trading_day = day = cls.env.trading_day
cls.calendar = date_range('2015', '2015-08', tz='UTC', freq=day)
cls.asset_info = make_rotating_asset_info(
num_assets=6,
first_start=cls.first_asset_start,
frequency=day,
periods_between_starts=4,
asset_lifetime=8,
)
cls.last_asset_end = cls.asset_info['end_date'].max()
cls.all_assets = cls.asset_info.index
cls.env.write_data(equities_df=cls.asset_info)
cls.finder = cls.env.asset_finder
cls.temp_dir = TempDirectory()
cls.temp_dir.create()
try:
cls.writer = SyntheticDailyBarWriter(
asset_info=cls.asset_info[['start_date', 'end_date']],
calendar=cls.calendar,
)
table = cls.writer.write(
cls.temp_dir.getpath('testdata.bcolz'),
cls.calendar,
cls.all_assets,
)
cls.pipeline_loader = USEquityPricingLoader(
BcolzDailyBarReader(table),
NullAdjustmentReader(),
)
except:
cls.temp_dir.cleanup()
raise
def test_unadjusted_spot_price_empty_value(self):
table = self.writer.write(self.dest, self.trading_days, self.assets)
reader = BcolzDailyBarReader(table)
# A sid, day and corresponding index into which to overwrite a zero.
zero_sid = 1
zero_day = Timestamp('2015-06-02', tz='UTC')
zero_ix = reader.sid_day_index(zero_sid, zero_day)
# Write a zero into the synthetic pricing data at the day and sid,
# so that a read should now return -1.
# This a little hacky, in lieu of changing the synthetic data set.
reader._spot_col('close')[zero_ix] = 0
close = reader.spot_price(zero_sid, zero_day, 'close')
self.assertEqual(-1, close)
def transaction_sim(self, **params):
""" This is a utility method that asserts expected
results for conversion of orders to transactions given a
trade history"""
tempdir = TempDirectory()
try:
trade_count = params['trade_count']
trade_interval = params['trade_interval']
order_count = params['order_count']
order_amount = params['order_amount']
order_interval = params['order_interval']
expected_txn_count = params['expected_txn_count']
expected_txn_volume = params['expected_txn_volume']
# optional parameters
# ---------------------
# if present, alternate between long and short sales
alternate = params.get('alternate')
# if present, expect transaction amounts to match orders exactly.
complete_fill = params.get('complete_fill')
env = TradingEnvironment()
sid = 1
if trade_interval < timedelta(days=1):
sim_params = factory.create_simulation_parameters(
data_frequency="minute")
minutes = env.market_minute_window(
sim_params.first_open,
int((trade_interval.total_seconds() / 60) * trade_count) +
100)
price_data = np.array([10.1] * len(minutes))
assets = {
sid:
pd.DataFrame({
"open": price_data,
"high": price_data,
"low": price_data,
"close": price_data,
"volume": np.array([100] * len(minutes)),
"dt": minutes
}).set_index("dt")
}
write_bcolz_minute_data(
env, env.days_in_range(minutes[0], minutes[-1]),
tempdir.path, assets)
equity_minute_reader = BcolzMinuteBarReader(tempdir.path)
data_portal = DataPortal(
env,
equity_minute_reader=equity_minute_reader,
)
else:
sim_params = factory.create_simulation_parameters(
data_frequency="daily")
days = sim_params.trading_days
assets = {
1:
pd.DataFrame(
{
"open": [10.1] * len(days),
"high": [10.1] * len(days),
"low": [10.1] * len(days),
"close": [10.1] * len(days),
"volume": [100] * len(days),
"day": [day.value for day in days]
},
index=days)
}
path = os.path.join(tempdir.path, "testdata.bcolz")
DailyBarWriterFromDataFrames(assets).write(path, days, assets)
equity_daily_reader = BcolzDailyBarReader(path)
data_portal = DataPortal(
env,
equity_daily_reader=equity_daily_reader,
)
if "default_slippage" not in params or \
not params["default_slippage"]:
slippage_func = FixedSlippage()
else:
slippage_func = None
blotter = Blotter(sim_params.data_frequency, self.env.asset_finder,
slippage_func)
env.write_data(
equities_data={
sid: {
"start_date": sim_params.trading_days[0],
"end_date": sim_params.trading_days[-1]
}
})
start_date = sim_params.first_open
if alternate:
alternator = -1
else:
alternator = 1
tracker = PerformanceTracker(sim_params, self.env)
# replicate what tradesim does by going through every minute or day
# of the simulation and processing open orders each time
if sim_params.data_frequency == "minute":
ticks = minutes
else:
ticks = days
transactions = []
order_list = []
order_date = start_date
for tick in ticks:
blotter.current_dt = tick
if tick >= order_date and len(order_list) < order_count:
# place an order
direction = alternator**len(order_list)
order_id = blotter.order(
blotter.asset_finder.retrieve_asset(sid),
order_amount * direction, MarketOrder())
order_list.append(blotter.orders[order_id])
order_date = order_date + order_interval
# move after market orders to just after market next
# market open.
if order_date.hour >= 21:
if order_date.minute >= 00:
order_date = order_date + timedelta(days=1)
order_date = order_date.replace(hour=14, minute=30)
else:
bar_data = BarData(data_portal, lambda: tick,
sim_params.data_frequency)
txns, _ = blotter.get_transactions(bar_data)
for txn in txns:
tracker.process_transaction(txn)
transactions.append(txn)
for i in range(order_count):
order = order_list[i]
self.assertEqual(order.sid, sid)
self.assertEqual(order.amount, order_amount * alternator**i)
if complete_fill:
self.assertEqual(len(transactions), len(order_list))
total_volume = 0
for i in range(len(transactions)):
txn = transactions[i]
total_volume += txn.amount
if complete_fill:
order = order_list[i]
self.assertEqual(order.amount, txn.amount)
self.assertEqual(total_volume, expected_txn_volume)
self.assertEqual(len(transactions), expected_txn_count)
cumulative_pos = tracker.position_tracker.positions[sid]
if total_volume == 0:
self.assertIsNone(cumulative_pos)
else:
self.assertEqual(total_volume, cumulative_pos.amount)
# the open orders should not contain sid.
oo = blotter.open_orders
self.assertNotIn(sid, oo, "Entry is removed when no open orders")
finally:
tempdir.cleanup()
def setUp(self):
self.env = env = trading.TradingEnvironment()
self.dates = date_range(
'2014-01-01', '2014-02-01', freq=trading_day, tz='UTC'
)
asset_info = DataFrame.from_records([
{
'sid': 1,
'symbol': 'A',
'start_date': self.dates[10],
'end_date': self.dates[13],
'exchange': 'TEST',
},
{
'sid': 2,
'symbol': 'B',
'start_date': self.dates[11],
'end_date': self.dates[14],
'exchange': 'TEST',
},
{
'sid': 3,
'symbol': 'C',
'start_date': self.dates[12],
'end_date': self.dates[15],
'exchange': 'TEST',
},
])
self.first_asset_start = min(asset_info.start_date)
self.last_asset_end = max(asset_info.end_date)
env.write_data(equities_df=asset_info)
self.asset_finder = finder = env.asset_finder
sids = (1, 2, 3)
self.assets = finder.retrieve_all(sids)
# View of the baseline data.
self.closes = DataFrame(
{sid: arange(1, len(self.dates) + 1) * sid for sid in sids},
index=self.dates,
dtype=float,
)
# Create a data portal holding the data in self.closes
data = {}
for sid in sids:
data[sid] = DataFrame({
"open": self.closes[sid].values,
"high": self.closes[sid].values,
"low": self.closes[sid].values,
"close": self.closes[sid].values,
"volume": self.closes[sid].values,
"day": [day.value for day in self.dates]
})
path = os.path.join(self.tempdir.path, "testdaily.bcolz")
DailyBarWriterFromDataFrames(data).write(
path,
self.dates,
data
)
daily_bar_reader = BcolzDailyBarReader(path)
self.data_portal = DataPortal(
self.env,
equity_daily_reader=daily_bar_reader,
)
# Add a split for 'A' on its second date.
self.split_asset = self.assets[0]
self.split_date = self.split_asset.start_date + trading_day
self.split_ratio = 0.5
self.adjustments = DataFrame.from_records([
{
'sid': self.split_asset.sid,
'value': self.split_ratio,
'kind': MULTIPLY,
'start_date': Timestamp('NaT'),
'end_date': self.split_date,
'apply_date': self.split_date,
}
])
# View of the data on/after the split.
self.adj_closes = adj_closes = self.closes.copy()
adj_closes.ix[:self.split_date, self.split_asset] *= self.split_ratio
self.pipeline_loader = DataFrameLoader(
column=USEquityPricing.close,
baseline=self.closes,
adjustments=self.adjustments,
)
def create_data_portal_from_trade_history(env, tempdir, sim_params,
trades_by_sid):
if sim_params.data_frequency == "daily":
path = os.path.join(tempdir.path, "testdaily.bcolz")
assets = {}
for sidint, trades in iteritems(trades_by_sid):
opens = []
highs = []
lows = []
closes = []
volumes = []
for trade in trades:
opens.append(trade["open_price"])
highs.append(trade["high"])
lows.append(trade["low"])
closes.append(trade["close_price"])
volumes.append(trade["volume"])
assets[sidint] = pd.DataFrame({
"open": np.array(opens),
"high": np.array(highs),
"low": np.array(lows),
"close": np.array(closes),
"volume": np.array(volumes),
"day": [day.value for day in sim_params.trading_days]
}, index=sim_params.trading_days)
DailyBarWriterFromDataFrames(assets).write(
path,
sim_params.trading_days,
assets
)
equity_daily_reader = BcolzDailyBarReader(path)
return DataPortal(
env,
equity_daily_reader=equity_daily_reader,
)
else:
minutes = env.minutes_for_days_in_range(
sim_params.first_open,
sim_params.last_close
)
length = len(minutes)
assets = {}
for sidint, trades in iteritems(trades_by_sid):
opens = np.zeros(length)
highs = np.zeros(length)
lows = np.zeros(length)
closes = np.zeros(length)
volumes = np.zeros(length)
for trade in trades:
# put them in the right place
idx = minutes.searchsorted(trade.dt)
opens[idx] = trade.open_price * 1000
highs[idx] = trade.high * 1000
lows[idx] = trade.low * 1000
closes[idx] = trade.close_price * 1000
volumes[idx] = trade.volume
assets[sidint] = pd.DataFrame({
"open": opens,
"high": highs,
"low": lows,
"close": closes,
"volume": volumes,
"dt": minutes
}).set_index("dt")
write_bcolz_minute_data(
env,
env.days_in_range(
sim_params.first_open,
sim_params.last_close
),
tempdir.path,
assets
)
equity_minute_reader = BcolzMinuteBarReader(tempdir.path)
return DataPortal(
env,
equity_minute_reader=equity_minute_reader,
)
def test_read_with_adjustments(self):
columns = [USEquityPricing.high, USEquityPricing.volume]
query_days = self.calendar_days_between(TEST_QUERY_START,
TEST_QUERY_STOP)
# Our expected results for each day are based on values from the
# previous day.
shifted_query_days = self.calendar_days_between(
TEST_QUERY_START,
TEST_QUERY_STOP,
shift=-1,
)
baseline_reader = BcolzDailyBarReader(self.bcolz_path)
adjustment_reader = SQLiteAdjustmentReader(self.db_path)
pricing_loader = USEquityPricingLoader(
baseline_reader,
adjustment_reader,
)
highs, volumes = pricing_loader.load_adjusted_array(
columns,
dates=query_days,
assets=Int64Index(arange(1, 7)),
mask=ones((len(query_days), 6), dtype=bool),
)
expected_baseline_highs = self.bcolz_writer.expected_values_2d(
shifted_query_days,
self.assets,
'high',
)
expected_baseline_volumes = self.bcolz_writer.expected_values_2d(
shifted_query_days,
self.assets,
'volume',
)
# At each point in time, the AdjustedArrays should yield the baseline
# with all adjustments up to that date applied.
for windowlen in range(1, len(query_days) + 1):
for offset, window in enumerate(highs.traverse(windowlen)):
baseline = expected_baseline_highs[offset:offset + windowlen]
baseline_dates = query_days[offset:offset + windowlen]
expected_adjusted_highs = self.apply_adjustments(
baseline_dates,
self.assets,
baseline,
# Apply all adjustments.
concat([SPLITS, MERGERS, DIVIDENDS_EXPECTED],
ignore_index=True),
)
assert_allclose(expected_adjusted_highs, window)
for offset, window in enumerate(volumes.traverse(windowlen)):
baseline = expected_baseline_volumes[offset:offset + windowlen]
baseline_dates = query_days[offset:offset + windowlen]
# Apply only splits and invert the ratio.
adjustments = SPLITS.copy()
adjustments.ratio = 1 / adjustments.ratio
expected_adjusted_volumes = self.apply_adjustments(
baseline_dates,
self.assets,
baseline,
adjustments,
)
# FIXME: Make AdjustedArray properly support integral types.
assert_array_equal(
expected_adjusted_volumes,
window.astype(uint32),
)
# Verify that we checked up to the longest possible window.
with self.assertRaises(WindowLengthTooLong):
highs.traverse(windowlen + 1)
with self.assertRaises(WindowLengthTooLong):
volumes.traverse(windowlen + 1)
def transaction_sim(self, **params):
"""This is a utility method that asserts expected
results for conversion of orders to transactions given a
trade history
"""
trade_count = params['trade_count']
trade_interval = params['trade_interval']
order_count = params['order_count']
order_amount = params['order_amount']
order_interval = params['order_interval']
expected_txn_count = params['expected_txn_count']
expected_txn_volume = params['expected_txn_volume']
# optional parameters
# ---------------------
# if present, alternate between long and short sales
alternate = params.get('alternate')
# if present, expect transaction amounts to match orders exactly.
complete_fill = params.get('complete_fill')
asset1 = self.asset_finder.retrieve_asset(1)
metadata = make_simple_equity_info([asset1.sid], self.start, self.end)
with TempDirectory() as tempdir, \
tmp_trading_env(equities=metadata,
load=self.make_load_function()) as env:
if trade_interval < timedelta(days=1):
sim_params = factory.create_simulation_parameters(
start=self.start, end=self.end, data_frequency="minute")
minutes = self.trading_calendar.minutes_window(
sim_params.first_open,
int((trade_interval.total_seconds() / 60) * trade_count) +
100)
price_data = np.array([10.1] * len(minutes))
assets = {
asset1.sid:
pd.DataFrame({
"open": price_data,
"high": price_data,
"low": price_data,
"close": price_data,
"volume": np.array([100] * len(minutes)),
"dt": minutes
}).set_index("dt")
}
write_bcolz_minute_data(
self.trading_calendar,
self.trading_calendar.sessions_in_range(
self.trading_calendar.minute_to_session_label(
minutes[0]),
self.trading_calendar.minute_to_session_label(
minutes[-1])),
tempdir.path,
iteritems(assets),
)
equity_minute_reader = BcolzMinuteBarReader(tempdir.path)
data_portal = DataPortal(
env.asset_finder,
self.trading_calendar,
first_trading_day=equity_minute_reader.first_trading_day,
equity_minute_reader=equity_minute_reader,
)
else:
sim_params = factory.create_simulation_parameters(
data_frequency="daily")
days = sim_params.sessions
assets = {
1:
pd.DataFrame(
{
"open": [10.1] * len(days),
"high": [10.1] * len(days),
"low": [10.1] * len(days),
"close": [10.1] * len(days),
"volume": [100] * len(days),
"day": [day.value for day in days]
},
index=days)
}
path = os.path.join(tempdir.path, "testdata.bcolz")
BcolzDailyBarWriter(path, self.trading_calendar, days[0],
days[-1]).write(assets.items())
equity_daily_reader = BcolzDailyBarReader(path)
data_portal = DataPortal(
env.asset_finder,
self.trading_calendar,
first_trading_day=equity_daily_reader.first_trading_day,
equity_daily_reader=equity_daily_reader,
)
if "default_slippage" not in params or \
not params["default_slippage"]:
slippage_func = FixedBasisPointsSlippage()
else:
slippage_func = None
blotter = SimulationBlotter(sim_params.data_frequency,
slippage_func)
start_date = sim_params.first_open
if alternate:
alternator = -1
else:
alternator = 1
tracker = MetricsTracker(
trading_calendar=self.trading_calendar,
first_session=sim_params.start_session,
last_session=sim_params.end_session,
capital_base=sim_params.capital_base,
emission_rate=sim_params.emission_rate,
data_frequency=sim_params.data_frequency,
asset_finder=self.asset_finder,
metrics=load_metrics_set('none'),
)
# replicate what tradesim does by going through every minute or day
# of the simulation and processing open orders each time
if sim_params.data_frequency == "minute":
ticks = minutes
else:
ticks = days
transactions = []
order_list = []
order_date = start_date
for tick in ticks:
blotter.current_dt = tick
if tick >= order_date and len(order_list) < order_count:
# place an order
direction = alternator**len(order_list)
order_id = blotter.order(
asset1,
order_amount * direction,
MarketOrder(),
)
order_list.append(blotter.orders[order_id])
order_date = order_date + order_interval
# move after market orders to just after market next
# market open.
if order_date.hour >= 21:
if order_date.minute >= 00:
order_date = order_date + timedelta(days=1)
order_date = order_date.replace(hour=14, minute=30)
else:
bar_data = BarData(
data_portal=data_portal,
simulation_dt_func=lambda: tick,
data_frequency=sim_params.data_frequency,
trading_calendar=self.trading_calendar,
restrictions=NoRestrictions(),
)
txns, _, closed_orders = blotter.get_transactions(bar_data)
for txn in txns:
tracker.process_transaction(txn)
transactions.append(txn)
blotter.prune_orders(closed_orders)
for i in range(order_count):
order = order_list[i]
self.assertEqual(order.asset, asset1)
self.assertEqual(order.amount, order_amount * alternator**i)
if complete_fill:
self.assertEqual(len(transactions), len(order_list))
total_volume = 0
for i in range(len(transactions)):
txn = transactions[i]
total_volume += txn.amount
if complete_fill:
order = order_list[i]
self.assertEqual(order.amount, txn.amount)
self.assertEqual(total_volume, expected_txn_volume)
self.assertEqual(len(transactions), expected_txn_count)
if total_volume == 0:
self.assertRaises(KeyError, lambda: tracker.positions[asset1])
else:
cumulative_pos = tracker.positions[asset1]
self.assertEqual(total_volume, cumulative_pos.amount)
# the open orders should not contain the asset.
oo = blotter.open_orders
self.assertNotIn(asset1, oo,
"Entry is removed when no open orders")
def create_data_portal_from_trade_history(asset_finder, trading_calendar,
tempdir, sim_params, trades_by_sid):
if sim_params.data_frequency == "daily":
path = os.path.join(tempdir.path, "testdaily.bcolz")
writer = BcolzDailyBarWriter(
path, trading_calendar,
sim_params.start_session,
sim_params.end_session
)
writer.write(
trades_by_sid_to_dfs(trades_by_sid, sim_params.sessions),
)
equity_daily_reader = BcolzDailyBarReader(path)
return DataPortal(
asset_finder, trading_calendar,
first_trading_day=equity_daily_reader.first_trading_day,
equity_daily_reader=equity_daily_reader,
)
else:
minutes = trading_calendar.minutes_in_range(
sim_params.first_open,
sim_params.last_close
)
length = len(minutes)
assets = {}
for sidint, trades in iteritems(trades_by_sid):
opens = np.zeros(length)
highs = np.zeros(length)
lows = np.zeros(length)
closes = np.zeros(length)
volumes = np.zeros(length)
for trade in trades:
# put them in the right place
idx = minutes.searchsorted(trade.dt)
opens[idx] = trade.open_price * 1000
highs[idx] = trade.high * 1000
lows[idx] = trade.low * 1000
closes[idx] = trade.close_price * 1000
volumes[idx] = trade.volume
assets[sidint] = pd.DataFrame({
"open": opens,
"high": highs,
"low": lows,
"close": closes,
"volume": volumes,
"dt": minutes
}).set_index("dt")
write_bcolz_minute_data(
trading_calendar,
sim_params.sessions,
tempdir.path,
assets
)
equity_minute_reader = BcolzMinuteBarReader(tempdir.path)
return DataPortal(
asset_finder, trading_calendar,
first_trading_day=equity_minute_reader.first_trading_day,
equity_minute_reader=equity_minute_reader,
)
def test_read_no_adjustments(self):
adjustment_reader = NullAdjustmentReader()
columns = [USEquityPricing.close, USEquityPricing.volume]
query_days = self.calendar_days_between(TEST_QUERY_START,
TEST_QUERY_STOP)
# Our expected results for each day are based on values from the
# previous day.
shifted_query_days = self.calendar_days_between(
TEST_QUERY_START,
TEST_QUERY_STOP,
shift=-1,
)
adjustments = adjustment_reader.load_adjustments(
columns,
query_days,
self.assets,
)
self.assertEqual(adjustments, [{}, {}])
baseline_reader = BcolzDailyBarReader(self.bcolz_path)
pricing_loader = USEquityPricingLoader(
baseline_reader,
adjustment_reader,
)
closes, volumes = pricing_loader.load_adjusted_array(
columns,
dates=query_days,
assets=self.assets,
mask=ones((len(query_days), len(self.assets)), dtype=bool),
)
expected_baseline_closes = self.bcolz_writer.expected_values_2d(
shifted_query_days,
self.assets,
'close',
)
expected_baseline_volumes = self.bcolz_writer.expected_values_2d(
shifted_query_days,
self.assets,
'volume',
)
# AdjustedArrays should yield the same data as the expected baseline.
for windowlen in range(1, len(query_days) + 1):
for offset, window in enumerate(closes.traverse(windowlen)):
assert_array_equal(
expected_baseline_closes[offset:offset + windowlen],
window,
)
for offset, window in enumerate(volumes.traverse(windowlen)):
assert_array_equal(
expected_baseline_volumes[offset:offset + windowlen],
window,
)
# Verify that we checked up to the longest possible window.
with self.assertRaises(WindowLengthTooLong):
closes.traverse(windowlen + 1)
with self.assertRaises(WindowLengthTooLong):
volumes.traverse(windowlen + 1)
