def create_test_panel_source(sim_params=None, env=None, source_type=None):
start = sim_params.first_open \
if sim_params else pd.datetime(1990, 1, 3, 0, 0, 0, 0, pytz.utc)
end = sim_params.last_close \
if sim_params else pd.datetime(1990, 1, 8, 0, 0, 0, 0, pytz.utc)
if env is None:
env = TradingEnvironment()
index = env.days_in_range(start, end)
price = np.arange(0, len(index))
volume = np.ones(len(index)) * 1000
arbitrary = np.ones(len(index))
df = pd.DataFrame({'price': price,
'volume': volume,
'arbitrary': arbitrary},
index=index)
if source_type:
source_types = np.full(len(index), source_type)
df['type'] = source_types
panel = pd.Panel.from_dict({0: df})
return DataPanelSource(panel), panel
def create_test_df_source(sim_params=None, env=None, bars='daily'):
if bars == 'daily':
freq = pd.datetools.BDay()
elif bars == 'minute':
freq = pd.datetools.Minute()
else:
raise ValueError('%s bars not understood.' % bars)
if sim_params and bars == 'daily':
index = sim_params.trading_days
else:
if env is None:
env = TradingEnvironment()
start = pd.datetime(1990, 1, 3, 0, 0, 0, 0, pytz.utc)
end = pd.datetime(1990, 1, 8, 0, 0, 0, 0, pytz.utc)
days = env.days_in_range(start, end)
if bars == 'daily':
index = days
if bars == 'minute':
index = pd.DatetimeIndex([], freq=freq)
for day in days:
day_index = env.market_minutes_for_day(day)
index = index.append(day_index)
x = np.arange(1, len(index) + 1)
df = pd.DataFrame(x, index=index, columns=[0])
return DataFrameSource(df), df
def setUp(self):
self.constants = {
# Every day, assume every stock starts at 2, goes down to 1,
# goes up to 4, and finishes at 3.
USEquityPricing.low: 1,
USEquityPricing.open: 2,
USEquityPricing.close: 3,
USEquityPricing.high: 4,
}
self.asset_ids = [1, 2, 3, 4]
self.dates = date_range('2014-01', '2014-03', freq='D', tz='UTC')
self.loader = PrecomputedLoader(
constants=self.constants,
dates=self.dates,
sids=self.asset_ids,
)
self.asset_info = make_simple_equity_info(
self.asset_ids,
start_date=self.dates[0],
end_date=self.dates[-1],
)
environment = TradingEnvironment()
environment.write_data(equities_df=self.asset_info)
self.asset_finder = environment.asset_finder
self.assets = self.asset_finder.retrieve_all(self.asset_ids)
def setUpClass(cls):
metadata = {
0: {
'symbol': 'CLG06',
'root_symbol': 'CL',
'start_date': pd.Timestamp('2005-12-01', tz='UTC'),
'notice_date': pd.Timestamp('2005-12-20', tz='UTC'),
'expiration_date': pd.Timestamp('2006-01-20', tz='UTC')},
1: {
'root_symbol': 'CL',
'symbol': 'CLK06',
'start_date': pd.Timestamp('2005-12-01', tz='UTC'),
'notice_date': pd.Timestamp('2006-03-20', tz='UTC'),
'expiration_date': pd.Timestamp('2006-04-20', tz='UTC')},
2: {
'symbol': 'CLQ06',
'root_symbol': 'CL',
'start_date': pd.Timestamp('2005-12-01', tz='UTC'),
'notice_date': pd.Timestamp('2006-06-20', tz='UTC'),
'expiration_date': pd.Timestamp('2006-07-20', tz='UTC')},
3: {
'symbol': 'CLX06',
'root_symbol': 'CL',
'start_date': pd.Timestamp('2006-02-01', tz='UTC'),
'notice_date': pd.Timestamp('2006-09-20', tz='UTC'),
'expiration_date': pd.Timestamp('2006-10-20', tz='UTC')}
}
env = TradingEnvironment(load=noop_load)
env.write_data(futures_data=metadata)
cls.asset_finder = env.asset_finder
def setUpClass(cls):
metadata = {
0: {
"symbol": "CLG06",
"root_symbol": "CL",
"start_date": pd.Timestamp("2005-12-01", tz="UTC"),
"notice_date": pd.Timestamp("2005-12-20", tz="UTC"),
"expiration_date": pd.Timestamp("2006-01-20", tz="UTC"),
},
1: {
"root_symbol": "CL",
"symbol": "CLK06",
"start_date": pd.Timestamp("2005-12-01", tz="UTC"),
"notice_date": pd.Timestamp("2006-03-20", tz="UTC"),
"expiration_date": pd.Timestamp("2006-04-20", tz="UTC"),
},
2: {
"symbol": "CLQ06",
"root_symbol": "CL",
"start_date": pd.Timestamp("2005-12-01", tz="UTC"),
"notice_date": pd.Timestamp("2006-06-20", tz="UTC"),
"expiration_date": pd.Timestamp("2006-07-20", tz="UTC"),
},
3: {
"symbol": "CLX06",
"root_symbol": "CL",
"start_date": pd.Timestamp("2006-02-01", tz="UTC"),
"notice_date": pd.Timestamp("2006-09-20", tz="UTC"),
"expiration_date": pd.Timestamp("2006-10-20", tz="UTC"),
},
}
env = TradingEnvironment(load=noop_load)
env.write_data(futures_data=metadata)
cls.asset_finder = env.asset_finder
def test_yahoo_bars_to_panel_source(self):
env = TradingEnvironment()
finder = AssetFinder(env.engine)
stocks = ['AAPL', 'GE']
env.write_data(equities_identifiers=stocks)
start = pd.datetime(1993, 1, 1, 0, 0, 0, 0, pytz.utc)
end = pd.datetime(2002, 1, 1, 0, 0, 0, 0, pytz.utc)
data = factory.load_bars_from_yahoo(stocks=stocks,
indexes={},
start=start,
end=end)
check_fields = ['sid', 'open', 'high', 'low', 'close',
'volume', 'price']
copy_panel = data.copy()
sids = finder.map_identifier_index_to_sids(
data.items, data.major_axis[0]
)
copy_panel.items = sids
source = DataPanelSource(copy_panel)
for event in source:
for check_field in check_fields:
self.assertIn(check_field, event)
self.assertTrue(isinstance(event['volume'], (integer_types)))
self.assertTrue(event['sid'] in sids)
class TestChangeOfSids(TestCase):
def setUp(self):
self.sids = range(90)
self.env = TradingEnvironment()
self.env.write_data(equities_identifiers=self.sids)
self.sim_params = factory.create_simulation_parameters(
start=datetime(1990, 1, 1, tzinfo=pytz.utc),
end=datetime(1990, 1, 8, tzinfo=pytz.utc),
env=self.env,
)
def test_all_sids_passed(self):
algo = BatchTransformAlgorithmSetSid(
sim_params=self.sim_params,
env=self.env,
)
source = DifferentSidSource()
algo.run(source)
for i, (df, date) in enumerate(zip(algo.history, source.trading_days)):
self.assertEqual(df.index[-1], date, "Newest event doesn't \
match.")
for sid in self.sids[:i]:
self.assertIn(sid, df.columns)
self.assertEqual(df.iloc[-1].iloc[-1], i)
def test_compute_lifetimes(self):
num_assets = 4
env = TradingEnvironment()
trading_day = env.trading_day
first_start = pd.Timestamp('2015-04-01', tz='UTC')
frame = make_rotating_asset_info(
num_assets=num_assets,
first_start=first_start,
frequency=env.trading_day,
periods_between_starts=3,
asset_lifetime=5
)
env.write_data(equities_df=frame)
finder = env.asset_finder
all_dates = pd.date_range(
start=first_start,
end=frame.end_date.max(),
freq=trading_day,
)
for dates in all_subindices(all_dates):
expected_with_start_raw = full(
shape=(len(dates), num_assets),
fill_value=False,
dtype=bool,
)
expected_no_start_raw = full(
shape=(len(dates), num_assets),
fill_value=False,
dtype=bool,
)
for i, date in enumerate(dates):
it = frame[['start_date', 'end_date']].itertuples()
for j, start, end in it:
# This way of doing the checks is redundant, but very
# clear.
if start <= date <= end:
expected_with_start_raw[i, j] = True
if start < date:
expected_no_start_raw[i, j] = True
expected_with_start = pd.DataFrame(
data=expected_with_start_raw,
index=dates,
columns=frame.index.values,
)
result = finder.lifetimes(dates, include_start_date=True)
assert_frame_equal(result, expected_with_start)
expected_no_start = pd.DataFrame(
data=expected_no_start_raw,
index=dates,
columns=frame.index.values,
)
result = finder.lifetimes(dates, include_start_date=False)
assert_frame_equal(result, expected_no_start)
def setUp(self):
self.assets = [1, 2, 3]
self.dates = date_range("2014-01", "2014-03", freq="D", tz="UTC")
asset_info = make_simple_asset_info(self.assets, start_date=self.dates[0], end_date=self.dates[-1])
env = TradingEnvironment()
env.write_data(equities_df=asset_info)
self.asset_finder = env.asset_finder
def test_sids(self):
# Ensure that the sids property of the AssetFinder is functioning
env = TradingEnvironment()
env.write_data(equities_identifiers=[1, 2, 3])
sids = env.asset_finder.sids
self.assertEqual(3, len(sids))
self.assertTrue(1 in sids)
self.assertTrue(2 in sids)
self.assertTrue(3 in sids)
def setUp(self):
self.assets = [1, 2, 3]
self.dates = date_range('2014-01-01', '2014-02-01', freq='D', tz='UTC')
asset_info = make_simple_asset_info(
self.assets,
start_date=self.dates[0],
end_date=self.dates[-1],
)
env = TradingEnvironment()
env.write_data(equities_df=asset_info)
self.asset_finder = env.asset_finder
def setUp(self):
self.__calendar = date_range('2014', '2015', freq=trading_day)
self.__assets = assets = Int64Index(arange(1, 20))
# Set up env for test
env = TradingEnvironment()
env.write_data(
equities_df=make_simple_asset_info(
assets,
self.__calendar[0],
self.__calendar[-1],
))
self.__finder = env.asset_finder
self.__mask = self.__finder.lifetimes(self.__calendar[-10:])
def setUpClass(cls):
cls.test_data_dir = TempDirectory()
cls.db_path = cls.test_data_dir.getpath("adjustments.db")
all_days = TradingEnvironment().trading_days
cls.calendar_days = all_days[all_days.slice_indexer(TEST_CALENDAR_START, TEST_CALENDAR_STOP)]
daily_bar_reader = MockDailyBarSpotReader()
writer = SQLiteAdjustmentWriter(cls.db_path, cls.calendar_days, daily_bar_reader)
writer.write(SPLITS, MERGERS, DIVIDENDS)
cls.assets = TEST_QUERY_ASSETS
cls.asset_info = EQUITY_INFO
cls.bcolz_writer = SyntheticDailyBarWriter(cls.asset_info, cls.calendar_days)
cls.bcolz_path = cls.test_data_dir.getpath("equity_pricing.bcolz")
cls.bcolz_writer.write(cls.bcolz_path, cls.calendar_days, cls.assets)
def test_trading_environment(self):
benchmark_returns, treasury_curves = \
factory.load_market_data()
env = TradingEnvironment(
benchmark_returns,
treasury_curves,
period_start=datetime(2008, 1, 1, tzinfo=pytz.utc),
period_end=datetime(2008, 12, 31, tzinfo=pytz.utc),
capital_base=100000,
)
#holidays taken from: http://www.nyse.com/press/1191407641943.html
new_years = datetime(2008, 1, 1, tzinfo=pytz.utc)
mlk_day = datetime(2008, 1, 21, tzinfo=pytz.utc)
presidents = datetime(2008, 2, 18, tzinfo=pytz.utc)
good_friday = datetime(2008, 3, 21, tzinfo=pytz.utc)
memorial_day = datetime(2008, 5, 26, tzinfo=pytz.utc)
july_4th = datetime(2008, 7, 4, tzinfo=pytz.utc)
labor_day = datetime(2008, 9, 1, tzinfo=pytz.utc)
tgiving = datetime(2008, 11, 27, tzinfo=pytz.utc)
christmas = datetime(2008, 5, 25, tzinfo=pytz.utc)
a_saturday = datetime(2008, 8, 2, tzinfo=pytz.utc)
a_sunday = datetime(2008, 10, 12, tzinfo=pytz.utc)
holidays = [
new_years,
mlk_day,
presidents,
good_friday,
memorial_day,
july_4th,
labor_day,
tgiving,
christmas,
a_saturday,
a_sunday
]
for holiday in holidays:
self.assertTrue(not env.is_trading_day(holiday))
first_trading_day = datetime(2008, 1, 2, tzinfo=pytz.utc)
last_trading_day = datetime(2008, 12, 31, tzinfo=pytz.utc)
workdays = [first_trading_day, last_trading_day]
for workday in workdays:
self.assertTrue(env.is_trading_day(workday))
self.assertTrue(env.last_close.month == 12)
self.assertTrue(env.last_close.day == 31)
def setUp(self):
self.constants = {
# Every day, assume every stock starts at 2, goes down to 1,
# goes up to 4, and finishes at 3.
USEquityPricing.low: 1,
USEquityPricing.open: 2,
USEquityPricing.close: 3,
USEquityPricing.high: 4,
}
self.assets = [1, 2, 3]
self.dates = date_range("2014-01-01", "2014-02-01", freq="D", tz="UTC")
self.loader = ConstantLoader(constants=self.constants, dates=self.dates, assets=self.assets)
self.asset_info = make_simple_asset_info(self.assets, start_date=self.dates[0], end_date=self.dates[-1])
environment = TradingEnvironment()
environment.write_data(equities_df=self.asset_info)
self.asset_finder = environment.asset_finder
def test_compute_lifetimes(self):
num_assets = 4
env = TradingEnvironment()
trading_day = env.trading_day
first_start = pd.Timestamp('2015-04-01', tz='UTC')
frame = make_rotating_asset_info(
num_assets=num_assets,
first_start=first_start,
frequency=env.trading_day,
periods_between_starts=3,
asset_lifetime=5
)
env.write_data(equities_df=frame)
finder = env.asset_finder
all_dates = pd.date_range(
start=first_start,
end=frame.end_date.max(),
freq=trading_day,
)
for dates in all_subindices(all_dates):
expected_mask = full(
shape=(len(dates), num_assets),
fill_value=False,
dtype=bool,
)
for i, date in enumerate(dates):
it = frame[['start_date', 'end_date']].itertuples()
for j, start, end in it:
if start <= date <= end:
expected_mask[i, j] = True
# Filter out columns with all-empty columns.
expected_result = pd.DataFrame(
data=expected_mask,
index=dates,
columns=frame.index.values,
)
actual_result = finder.lifetimes(dates)
assert_frame_equal(actual_result, expected_result)
def _handle_market_close(self, completed_date):
# increment the day counter before we move markers forward.
self.day_count += 1.0
# Get the next trading day and, if it is past the bounds of this
# simulation, return the daily perf packet
next_trading_day = TradingEnvironment.instance().\
next_trading_day(completed_date)
# Check if any assets need to be auto-closed before generating today's
# perf period
if next_trading_day:
self.check_asset_auto_closes(next_trading_day=next_trading_day)
# Take a snapshot of our current performance to return to the
# browser.
daily_update = self.to_dict(emission_type='daily')
# On the last day of the test, don't create tomorrow's performance
# period. We may not be able to find the next trading day if we're at
# the end of our historical data
if self.market_close >= self.last_close:
return daily_update
# move the market day markers forward
env = TradingEnvironment.instance()
self.market_open, self.market_close = \
env.next_open_and_close(self.day)
self.day = env.next_trading_day(self.day)
# Roll over positions to current day.
self.todays_performance.rollover()
self.todays_performance.period_open = self.market_open
self.todays_performance.period_close = self.market_close
# If the next trading day is irrelevant, then return the daily packet
if (next_trading_day is None) or (next_trading_day >= self.last_close):
return daily_update
# Check for any dividends and auto-closes, then return the daily perf
# packet
self.check_upcoming_dividends(next_trading_day=next_trading_day)
return daily_update
def setUp(self):
self.sids = range(90)
self.env = TradingEnvironment()
self.env.write_data(equities_identifiers=self.sids)
self.sim_params = factory.create_simulation_parameters(
start=datetime(1990, 1, 1, tzinfo=pytz.utc),
end=datetime(1990, 1, 8, tzinfo=pytz.utc),
env=self.env,
)
def test_bmf_calendar_vs_environment(self):
env = TradingEnvironment(
bm_symbol='^BVSP',
exchange_tz='America/Sao_Paulo'
)
env_start_index = \
env.trading_days.searchsorted(tradingcalendar_bmf.start)
env_days = env.trading_days[env_start_index:]
cal_days = tradingcalendar_bmf.trading_days
self.check_days(env_days, cal_days)
def test_tse_calendar_vs_environment(self):
env = TradingEnvironment(
bm_symbol='^GSPTSE',
exchange_tz='US/Eastern'
)
env_start_index = \
env.trading_days.searchsorted(tradingcalendar_tse.start)
env_days = env.trading_days[env_start_index:]
cal_days = tradingcalendar_tse.trading_days
self.check_days(env_days, cal_days)
def test_lse_calendar_vs_environment(self):
env = TradingEnvironment(
bm_symbol='^FTSE',
exchange_tz='Europe/London'
)
env_start_index = \
env.trading_days.searchsorted(tradingcalendar_lse.start)
env_days = env.trading_days[env_start_index:]
cal_days = tradingcalendar_lse.trading_days
self.check_days(env_days, cal_days)
def mixed_frequency_expected_index(count, frequency):
"""
Helper for enumerating expected indices for test_mixed_frequency.
"""
env = TradingEnvironment.instance()
minute = MIXED_FREQUENCY_MINUTES[count]
if frequency == '1d':
return [env.previous_open_and_close(minute)[1], minute]
elif frequency == '1m':
return [env.previous_market_minute(minute), minute]
def mixed_frequency_expected_index(count, frequency):
"""
Helper for enumerating expected indices for test_mixed_frequency.
"""
env = TradingEnvironment.instance()
minute = MIXED_FREQUENCY_MINUTES[count]
if frequency == '1d':
return [env.previous_open_and_close(minute)[1], minute]
elif frequency == '1m':
return [env.previous_market_minute(minute), minute]
def minutes_for_days():
"""
500 randomly selected days.
This is used to make sure our test coverage is unbaised towards any rules.
We use a random sample because testing on all the trading days took
around 180 seconds on my laptop, which is far too much for normal unit
testing.
We manually set the seed so that this will be deterministic.
Results of multiple runs were compared to make sure that this is actually
true.
This returns a generator of tuples each wrapping a single generator.
Iterating over this yeilds a single day, iterating over the day yields
the minutes for that day.
"""
env = TradingEnvironment()
random.seed('deterministic')
return ((env.market_minutes_for_day(random.choice(env.trading_days)), )
for _ in range(500))
def setUp(self):
self.__calendar = date_range('2014', '2015', freq=trading_day)
self.__assets = assets = Int64Index(arange(1, 20))
# Set up env for test
env = TradingEnvironment()
env.write_data(
equities_df=make_simple_asset_info(
assets,
self.__calendar[0],
self.__calendar[-1],
),
)
self.__finder = env.asset_finder
# Use a 30-day period at the end of the year by default.
self.__mask = self.__finder.lifetimes(
self.__calendar[-30:],
include_start_date=False,
)
def __init__(self, env=None, trading_calendar=None,
first_trading_day=None):
if env is None:
env = TradingEnvironment()
if trading_calendar is None:
trading_calendar = get_calendar("NYSE")
super(FakeDataPortal, self).__init__(env.asset_finder,
trading_calendar,
first_trading_day)
def minutes_for_days():
"""
500 randomly selected days.
This is used to make sure our test coverage is unbaised towards any rules.
We use a random sample because testing on all the trading days took
around 180 seconds on my laptop, which is far too much for normal unit
testing.
We manually set the seed so that this will be deterministic.
Results of multiple runs were compared to make sure that this is actually
true.
This returns a generator of tuples each wrapping a single generator.
Iterating over this yeilds a single day, iterating over the day yields
the minutes for that day.
"""
env = TradingEnvironment()
random.seed('deterministic')
return ((env.market_minutes_for_day(random.choice(env.trading_days)),)
for _ in range(500))
def setUpClass(cls):
cls.env = TradingEnvironment()
# On the AfterOpen and BeforeClose tests, we want ensure that the
# functions are pure, and that running them with the same input will
# provide the same output, regardless of whether the function is run 1
# or N times. (For performance reasons, we cache some internal state
# in AfterOpen and BeforeClose, but we don't want it to affect
# purity). Hence, we use the same before_close and after_open across
# subtests.
cls.before_close = BeforeClose(hours=1, minutes=5)
cls.after_open = AfterOpen(hours=1, minutes=5)
cls.class_ = None # Mark that this is the base class.
def setUpClass(cls):
cls.env = TradingEnvironment()
all_market_opens = cls.env.open_and_closes.market_open
all_market_closes = cls.env.open_and_closes.market_close
indexer = all_market_opens.index.slice_indexer(
start=TEST_CALENDAR_START,
end=TEST_CALENDAR_STOP
)
cls.market_opens = all_market_opens[indexer]
cls.market_closes = all_market_closes[indexer]
cls.test_calendar_start = cls.market_opens.index[0]
cls.test_calendar_stop = cls.market_opens.index[-1]
def setUpClass(cls):
cls.future = Future(
2468,
symbol="OMH15",
root_symbol="OM",
notice_date=pd.Timestamp("2014-01-20", tz="UTC"),
expiration_date=pd.Timestamp("2014-02-20", tz="UTC"),
auto_close_date=pd.Timestamp("2014-01-18", tz="UTC"),
contract_multiplier=500,
)
cls.future2 = Future(
0,
symbol="CLG06",
root_symbol="CL",
start_date=pd.Timestamp("2005-12-01", tz="UTC"),
notice_date=pd.Timestamp("2005-12-20", tz="UTC"),
expiration_date=pd.Timestamp("2006-01-20", tz="UTC"),
)
env = TradingEnvironment(load=noop_load)
env.write_data(futures_identifiers=[TestFuture.future, TestFuture.future2])
cls.asset_finder = env.asset_finder
def setUpClass(cls):
cls.future = Future(2468,
symbol='OMH15',
root_symbol='OM',
notice_date=pd.Timestamp('2014-01-20', tz='UTC'),
expiration_date=pd.Timestamp('2014-02-20',
tz='UTC'),
auto_close_date=pd.Timestamp('2014-01-18',
tz='UTC'),
contract_multiplier=500)
cls.future2 = Future(0,
symbol='CLG06',
root_symbol='CL',
start_date=pd.Timestamp('2005-12-01', tz='UTC'),
notice_date=pd.Timestamp('2005-12-20', tz='UTC'),
expiration_date=pd.Timestamp('2006-01-20',
tz='UTC'))
env = TradingEnvironment()
env.write_data(
futures_identifiers=[TestFuture.future, TestFuture.future2])
cls.asset_finder = env.asset_finder
def configure_environment(self, last_trade, benchmark, timezone):
''' Prepare benchmark loader and trading context '''
if last_trade.tzinfo is None:
last_trade = pytz.utc.localize(last_trade)
# Setup the trading calendar from market informations
self.benchmark = benchmark
self.context = TradingEnvironment(
bm_symbol=benchmark,
exchange_tz=timezone,
load=self._get_benchmark_handler(last_trade))
def test_trading_environment(self):
benchmark_returns, treasury_curves = \
factory.load_market_data()
env = TradingEnvironment(
benchmark_returns,
treasury_curves,
period_start=datetime(2008, 1, 1, tzinfo=pytz.utc),
period_end=datetime(2008, 12, 31, tzinfo=pytz.utc),
capital_base=100000,
)
#holidays taken from: http://www.nyse.com/press/1191407641943.html
new_years = datetime(2008, 1, 1, tzinfo=pytz.utc)
mlk_day = datetime(2008, 1, 21, tzinfo=pytz.utc)
presidents = datetime(2008, 2, 18, tzinfo=pytz.utc)
good_friday = datetime(2008, 3, 21, tzinfo=pytz.utc)
memorial_day = datetime(2008, 5, 26, tzinfo=pytz.utc)
july_4th = datetime(2008, 7, 4, tzinfo=pytz.utc)
labor_day = datetime(2008, 9, 1, tzinfo=pytz.utc)
tgiving = datetime(2008, 11, 27, tzinfo=pytz.utc)
christmas = datetime(2008, 5, 25, tzinfo=pytz.utc)
a_saturday = datetime(2008, 8, 2, tzinfo=pytz.utc)
a_sunday = datetime(2008, 10, 12, tzinfo=pytz.utc)
holidays = [
new_years, mlk_day, presidents, good_friday, memorial_day,
july_4th, labor_day, tgiving, christmas, a_saturday, a_sunday
]
for holiday in holidays:
self.assertTrue(not env.is_trading_day(holiday))
first_trading_day = datetime(2008, 1, 2, tzinfo=pytz.utc)
last_trading_day = datetime(2008, 12, 31, tzinfo=pytz.utc)
workdays = [first_trading_day, last_trading_day]
for workday in workdays:
self.assertTrue(env.is_trading_day(workday))
self.assertTrue(env.last_close.month == 12)
self.assertTrue(env.last_close.day == 31)
def setUpClass(cls):
metadata = {
0: {
'symbol': 'CLG06',
'root_symbol': 'CL',
'asset_type': 'future',
'start_date': pd.Timestamp('2005-12-01', tz='UTC'),
'notice_date': pd.Timestamp('2005-12-20', tz='UTC'),
'expiration_date': pd.Timestamp('2006-01-20', tz='UTC')
},
1: {
'root_symbol': 'CL',
'symbol': 'CLK06',
'asset_type': 'future',
'start_date': pd.Timestamp('2005-12-01', tz='UTC'),
'notice_date': pd.Timestamp('2006-03-20', tz='UTC'),
'expiration_date': pd.Timestamp('2006-04-20', tz='UTC')
},
2: {
'symbol': 'CLQ06',
'root_symbol': 'CL',
'asset_type': 'future',
'start_date': pd.Timestamp('2005-12-01', tz='UTC'),
'notice_date': pd.Timestamp('2006-06-20', tz='UTC'),
'expiration_date': pd.Timestamp('2006-07-20', tz='UTC')
},
3: {
'symbol': 'CLX06',
'root_symbol': 'CL',
'asset_type': 'future',
'start_date': pd.Timestamp('2006-02-01', tz='UTC'),
'notice_date': pd.Timestamp('2006-09-20', tz='UTC'),
'expiration_date': pd.Timestamp('2006-10-20', tz='UTC')
}
}
env = TradingEnvironment()
env.write_data(futures_data=metadata)
cls.asset_finder = env.asset_finder
def test_consume_metadata(self):
# Test dict consumption
dict_to_consume = {0: {'symbol': 'PLAY'}, 1: {'symbol': 'MSFT'}}
self.env.write_data(equities_data=dict_to_consume)
finder = AssetFinder(self.env.engine)
equity = finder.retrieve_asset(0)
self.assertIsInstance(equity, Equity)
self.assertEqual('PLAY', equity.symbol)
# Test dataframe consumption
df = pd.DataFrame(columns=['asset_name', 'exchange'], index=[0, 1])
df['asset_name'][0] = "Dave'N'Busters"
df['exchange'][0] = "NASDAQ"
df['asset_name'][1] = "Microsoft"
df['exchange'][1] = "NYSE"
self.env = TradingEnvironment()
self.env.write_data(equities_df=df)
finder = AssetFinder(self.env.engine)
self.assertEqual('NASDAQ', finder.retrieve_asset(0).exchange)
self.assertEqual('Microsoft', finder.retrieve_asset(1).asset_name)
def setUp(self):
self.start_date = datetime.datetime(
year=2006,
month=1,
day=1,
hour=0,
minute=0,
tzinfo=pytz.utc)
self.end_date = datetime.datetime(
year=2006, month=12, day=31, tzinfo=pytz.utc)
self.benchmark_returns, self.treasury_curves = \
factory.load_market_data()
self.trading_env = TradingEnvironment(
self.benchmark_returns,
self.treasury_curves,
period_start=self.start_date,
period_end=self.end_date,
capital_base=1000.0
)
self.oneday = datetime.timedelta(days=1)
class EventRule(six.with_metaclass(ABCMeta)):
"""
An event rule checks a datetime and sees if it should trigger.
"""
env = TradingEnvironment.instance()
@abstractmethod
def should_trigger(self, dt):
"""
Checks if the rule should trigger with it's current state.
This method should be pure and NOT mutate any state on the object.
"""
raise NotImplementedError('should_trigger')
def setUpClass(cls):
cls.future = Future(
2468,
symbol='OMH15',
root_symbol='OM',
notice_date=pd.Timestamp('2014-01-20', tz='UTC'),
expiration_date=pd.Timestamp('2014-02-20', tz='UTC'),
auto_close_date=pd.Timestamp('2014-01-18', tz='UTC'),
contract_multiplier=500
)
cls.future2 = Future(
0,
symbol='CLG06',
root_symbol='CL',
start_date=pd.Timestamp('2005-12-01', tz='UTC'),
notice_date=pd.Timestamp('2005-12-20', tz='UTC'),
expiration_date=pd.Timestamp('2006-01-20', tz='UTC')
)
env = TradingEnvironment(load=noop_load)
env.write_data(futures_identifiers=[TestFuture.future,
TestFuture.future2])
cls.asset_finder = env.asset_finder
def test_check_version(self):
env = TradingEnvironment(load=noop_load)
version_table = env.asset_finder.version_info
# This should not raise an error
check_version_info(version_table, ASSET_DB_VERSION)
# This should fail because the version is too low
with self.assertRaises(AssetDBVersionError):
check_version_info(version_table, ASSET_DB_VERSION - 1)
# This should fail because the version is too high
with self.assertRaises(AssetDBVersionError):
check_version_info(version_table, ASSET_DB_VERSION + 1)
def test_calendar_vs_environment(self):
"""
test_calendar_vs_environment checks whether the
historical data from yahoo matches our rule based system.
handy, if not canonical, reference:
http://www.chronos-st.org/NYSE_Observed_Holidays-1885-Present.html
"""
env = TradingEnvironment()
env_start_index = \
env.trading_days.searchsorted(tradingcalendar.start)
env_days = env.trading_days[env_start_index:]
cal_days = tradingcalendar.trading_days
self.check_days(env_days, cal_days)
def test_calendar_vs_environment(self):
"""
test_calendar_vs_environment checks whether the
historical data from yahoo matches our rule based system.
handy, if not canonical, reference:
http://www.chronos-st.org/NYSE_Observed_Holidays-1885-Present.html
"""
env = TradingEnvironment()
bench_days = env.benchmark_returns[tradingcalendar.start:].index
bounds = env.trading_days.slice_locs(start=tradingcalendar.start,
end=bench_days[-1])
env_days = env.trading_days[bounds[0]:bounds[1]]
self.check_days(env_days, bench_days)
def setUp(self):
all_trading_days = TradingEnvironment.instance().trading_days
self.trading_days = all_trading_days[all_trading_days.get_loc(
TEST_CALENDAR_START):all_trading_days.get_loc(TEST_CALENDAR_STOP) +
1]
self.asset_info = EQUITY_INFO
self.writer = SyntheticDailyBarWriter(
self.asset_info,
self.trading_days,
)
self.dir_ = TempDirectory()
self.dir_.create()
self.dest = self.dir_.getpath('daily_equity_pricing.bcolz')
def test_algo_without_rl_violation_after_delete(self):
new_tempdir = TempDirectory()
try:
with security_list_copy():
# add a delete statement removing bzq
# write a new delete statement file to disk
add_security_data([], ['BZQ'])
# now fast-forward to self.extra_knowledge_date. requires
# a new env, simparams, and dataportal
env = TradingEnvironment()
sim_params = factory.create_simulation_parameters(
start=self.extra_knowledge_date, num_days=4, env=env)
env.write_data(equities_data={
"0": {
'symbol': 'BZQ',
'start_date': sim_params.period_start,
'end_date': sim_params.period_end,
}
})
data_portal = create_data_portal(
env,
new_tempdir,
sim_params,
range(0, 5)
)
algo = RestrictedAlgoWithoutCheck(
symbol='BZQ', sim_params=sim_params, env=env
)
algo.run(data_portal)
finally:
new_tempdir.cleanup()
def minutes_for_days(ordered_days=False):
"""
500 randomly selected days.
This is used to make sure our test coverage is unbaised towards any rules.
We use a random sample because testing on all the trading days took
around 180 seconds on my laptop, which is far too much for normal unit
testing.
We manually set the seed so that this will be deterministic.
Results of multiple runs were compared to make sure that this is actually
true.
This returns a generator of tuples each wrapping a single generator.
Iterating over this yields a single day, iterating over the day yields
the minutes for that day.
"""
env = TradingEnvironment()
random.seed('deterministic')
if ordered_days:
# Get a list of 500 trading days, in order. As a performance
# optimization in AfterOpen and BeforeClose, we rely on the fact that
# the clock only ever moves forward in a simulation. For those cases,
# we guarantee that the list of trading days we test is ordered.
ordered_day_list = random.sample(list(env.trading_days), 500)
ordered_day_list.sort()
def day_picker(day):
return ordered_day_list[day]
else:
# Other than AfterOpen and BeforeClose, we don't rely on the the nature
# of the clock, so we don't care.
def day_picker(day):
return random.choice(env.trading_days[:-1])
return ((env.market_minutes_for_day(day_picker(cnt)),)
for cnt in range(500))
def create_trading_environment(year=2006, start=None, end=None):
"""Construct a complete environment with reasonable defaults"""
benchmark_returns, treasury_curves = load_market_data()
if start is None:
start = datetime(year, 1, 1, tzinfo=pytz.utc)
if end is None:
end = datetime(year, 12, 31, tzinfo=pytz.utc)
trading_environment = TradingEnvironment(benchmark_returns,
treasury_curves,
period_start=start,
period_end=end,
capital_base=100000.0)
return trading_environment
def setUpClass(cls):
cls.sid = 133
cls.env = TradingEnvironment()
cls.env.write_data(equities_identifiers=[cls.sid])
cls.tempdir = TempDirectory()
cls.sim_params = factory.create_simulation_parameters(num_days=4,
env=cls.env)
cls.data_portal = create_data_portal(env=cls.env,
tempdir=cls.tempdir,
sim_params=cls.sim_params,
sids=[cls.sid])
setup_logger(cls)
def setUp(self):
self.days = TradingEnvironment().trading_days
self.index = [self.days[0], self.days[1], self.days[2]]
self.panel = pd.Panel({
1:
pd.DataFrame(
{
'price': [1, 2, 4],
'volume': [1e9, 0, 0],
'type': [
DATASOURCE_TYPE.TRADE, DATASOURCE_TYPE.TRADE,
DATASOURCE_TYPE.CLOSE_POSITION
]
},
index=self.index)
})
def setUp(self):
all_trading_days = TradingEnvironment.instance().trading_days
self.trading_days = all_trading_days[
all_trading_days.get_loc(TEST_CALENDAR_START):
all_trading_days.get_loc(TEST_CALENDAR_STOP) + 1
]
self.asset_info = EQUITY_INFO
self.writer = SyntheticDailyBarWriter(
self.asset_info,
self.trading_days,
)
self.dir_ = TempDirectory()
self.dir_.create()
self.dest = self.dir_.getpath('daily_equity_pricing.bcolz')
def check_upcoming_dividends(self, completed_date):
"""
Check if we currently own any stocks with dividends whose ex_date is
the next trading day. Track how much we should be payed on those
dividends' pay dates.
Then check if we are owed cash/stock for any dividends whose pay date
is the next trading day. Apply all such benefits, then recalculate
performance.
"""
if len(self.dividend_frame) == 0:
# We don't currently know about any dividends for this simulation
# period, so bail.
return
# Get the next trading day and, if it is outside the bounds of the
# simulation, bail.
next_trading_day = TradingEnvironment.instance().\
next_trading_day(completed_date)
if (next_trading_day is None) or (next_trading_day >= self.last_close):
return
# Dividends whose ex_date is the next trading day. We need to check if
# we own any of these stocks so we know to pay them out when the pay
# date comes.
ex_date_mask = (self.dividend_frame['ex_date'] == next_trading_day)
dividends_earnable = self.dividend_frame[ex_date_mask]
# Dividends whose pay date is the next trading day. If we held any of
# these stocks on midnight before the ex_date, we need to pay these out
# now.
pay_date_mask = (self.dividend_frame['pay_date'] == next_trading_day)
dividends_payable = self.dividend_frame[pay_date_mask]
position_tracker = self.position_tracker
if len(dividends_earnable):
position_tracker.earn_dividends(dividends_earnable)
if not len(dividends_payable):
return
net_cash_payment = position_tracker.pay_dividends(dividends_payable)
for period in self.perf_periods:
# notify periods to update their stats
period.handle_dividends_paid(net_cash_payment)
def check_upcoming_dividends(self, completed_date):
"""
Check if we currently own any stocks with dividends whose ex_date is
the next trading day. Track how much we should be payed on those
dividends' pay dates.
Then check if we are owed cash/stock for any dividends whose pay date
is the next trading day. Apply all such benefits, then recalculate
performance.
"""
if len(self.dividend_frame) == 0:
# We don't currently know about any dividends for this simulation
# period, so bail.
return
# Get the next trading day and, if it is outside the bounds of the
# simulation, bail.
next_trading_day = TradingEnvironment.instance().\
next_trading_day(completed_date)
if (next_trading_day is None) or (next_trading_day >= self.last_close):
return
# Dividends whose ex_date is the next trading day. We need to check if
# we own any of these stocks so we know to pay them out when the pay
# date comes.
ex_date_mask = (self.dividend_frame['ex_date'] == next_trading_day)
dividends_earnable = self.dividend_frame[ex_date_mask]
# Dividends whose pay date is the next trading day. If we held any of
# these stocks on midnight before the ex_date, we need to pay these out
# now.
pay_date_mask = (self.dividend_frame['pay_date'] == next_trading_day)
dividends_payable = self.dividend_frame[pay_date_mask]
position_tracker = self.position_tracker
if len(dividends_earnable):
position_tracker.earn_dividends(dividends_earnable)
if not len(dividends_payable):
return
net_cash_payment = position_tracker.pay_dividends(dividends_payable)
for period in self.perf_periods:
# notify periods to update their stats
period.handle_dividends_paid(net_cash_payment)
def handle_market_close_daily(self):
"""
Function called after handle_data when running with daily emission
rate.
"""
self.update_performance()
completed_date = self.day
account = self.get_account(False)
# update risk metrics for cumulative performance
self.cumulative_risk_metrics.update(
completed_date,
self.todays_performance.returns,
self.all_benchmark_returns[completed_date],
account)
# increment the day counter before we move markers forward.
self.day_count += 1.0
# Take a snapshot of our current performance to return to the
# browser.
daily_update = self.to_dict()
# On the last day of the test, don't create tomorrow's performance
# period. We may not be able to find the next trading day if we're at
# the end of our historical data
if self.market_close >= self.last_close:
return daily_update
# move the market day markers forward
env = TradingEnvironment.instance()
self.market_open, self.market_close = \
env.next_open_and_close(self.day)
self.day = env.next_trading_day(self.day)
# Roll over positions to current day.
self.todays_performance.rollover()
self.todays_performance.period_open = self.market_open
self.todays_performance.period_close = self.market_close
next_trading_day = env.next_trading_day(completed_date)
if next_trading_day:
self.check_upcoming_dividends(next_trading_day)
return daily_update
def setUp(self):
env = TradingEnvironment.instance()
day = env.trading_day
self.assets = Int64Index([1, 2, 3])
self.dates = date_range(
'2015-01-01',
'2015-01-31',
freq=day,
tz='UTC',
)
asset_info = make_simple_asset_info(
self.assets,
start_date=self.dates[0],
end_date=self.dates[-1],
)
self.asset_finder = AssetFinder(asset_info)
def setUpClass(cls):
cls.test_data_dir = TempDirectory()
cls.db_path = cls.test_data_dir.getpath('adjustments.db')
writer = SQLiteAdjustmentWriter(cls.db_path)
writer.write(SPLITS, MERGERS, DIVIDENDS)
cls.assets = TEST_QUERY_ASSETS
all_days = TradingEnvironment.instance().trading_days
cls.calendar_days = all_days[all_days.slice_indexer(
TEST_CALENDAR_START, TEST_CALENDAR_STOP)]
cls.asset_info = EQUITY_INFO
cls.bcolz_writer = SyntheticDailyBarWriter(
cls.asset_info,
cls.calendar_days,
)
cls.bcolz_path = cls.test_data_dir.getpath('equity_pricing.bcolz')
cls.bcolz_writer.write(cls.bcolz_path, cls.calendar_days, cls.assets)
def setUp(self):
env = TradingEnvironment.instance()
day = env.trading_day
self.assets = Int64Index([1, 2, 3])
self.dates = date_range(
'2015-01-01',
'2015-01-31',
freq=day,
tz='UTC',
)
asset_info = make_simple_asset_info(
self.assets,
start_date=self.dates[0],
end_date=self.dates[-1],
)
self.asset_finder = AssetFinder(asset_info)
def setUpClass(cls):
cls.env = TradingEnvironment()
cls.tempdir = TempDirectory()
cls.trading_days = cls.env.days_in_range(start=pd.Timestamp(
"2016-01-05", tz='UTC'),
end=pd.Timestamp("2016-01-28",
tz='UTC'))
equities_data = {}
for sid in [1, 2, 3]:
equities_data[sid] = {
"start_date": cls.trading_days[0],
"end_date": cls.env.next_trading_day(cls.trading_days[-1]),
"symbol": "ASSET{0}".format(sid),
}
cls.env.write_data(equities_data=equities_data)
cls.asset1 = cls.env.asset_finder.retrieve_asset(1)
cls.asset2 = cls.env.asset_finder.retrieve_asset(2)
cls.asset3 = cls.env.asset_finder.retrieve_asset(3)
market_opens = cls.env.open_and_closes.market_open.loc[
cls.trading_days]
market_closes = cls.env.open_and_closes.market_close.loc[
cls.trading_days]
minute_writer = BcolzMinuteBarWriter(cls.trading_days[0],
cls.tempdir.path, market_opens,
market_closes,
US_EQUITIES_MINUTES_PER_DAY)
for sid in [1, 2, 3]:
write_minute_data_for_asset(cls.env, minute_writer,
cls.trading_days[0],
cls.trading_days[-1], sid)
cls.adj_reader = cls.create_adjustments_reader()
cls.sim_params = SimulationParameters(period_start=cls.trading_days[0],
period_end=cls.trading_days[-1],
data_frequency="minute",
env=cls.env)
def handle_market_close_daily(self):
"""
Function called after handle_data when running with daily emission
rate.
"""
self.update_performance()
completed_date = self.day
account = self.get_account(False)
# update risk metrics for cumulative performance
self.cumulative_risk_metrics.update(
completed_date,
self.todays_performance.returns,
self.all_benchmark_returns[completed_date],
account)
# increment the day counter before we move markers forward.
self.day_count += 1.0
# Take a snapshot of our current performance to return to the
# browser.
daily_update = self.to_dict()
# On the last day of the test, don't create tomorrow's performance
# period. We may not be able to find the next trading day if we're at
# the end of our historical data
if self.market_close >= self.last_close:
return daily_update
# move the market day markers forward
env = TradingEnvironment.instance()
self.market_open, self.market_close = \
env.next_open_and_close(self.day)
self.day = env.next_trading_day(self.day)
# Roll over positions to current day.
self.todays_performance.rollover()
self.todays_performance.period_open = self.market_open
self.todays_performance.period_close = self.market_close
# Check for any dividends
self.check_upcoming_dividends(completed_date)
return daily_update
def setUpClass(cls):
cls.test_data_dir = TempDirectory()
cls.db_path = cls.test_data_dir.getpath('adjustments.db')
writer = SQLiteAdjustmentWriter(cls.db_path)
writer.write(SPLITS, MERGERS, DIVIDENDS)
cls.assets = TEST_QUERY_ASSETS
all_days = TradingEnvironment.instance().trading_days
cls.calendar_days = all_days[
all_days.slice_indexer(TEST_CALENDAR_START, TEST_CALENDAR_STOP)
]
cls.asset_info = EQUITY_INFO
cls.bcolz_writer = SyntheticDailyBarWriter(
cls.asset_info,
cls.calendar_days,
)
cls.bcolz_path = cls.test_data_dir.getpath('equity_pricing.bcolz')
cls.bcolz_writer.write(cls.bcolz_path, cls.calendar_days, cls.assets)
def setUpClass(cls):
cls.env = TradingEnvironment()
day = cls.env.trading_day
cls.assets = Int64Index([1, 2, 3])
cls.dates = date_range(
'2015-01-01',
'2015-01-31',
freq=day,
tz='UTC',
)
asset_info = make_simple_asset_info(
cls.assets,
start_date=cls.dates[0],
end_date=cls.dates[-1],
)
cls.env.write_data(equities_df=asset_info)
cls.asset_finder = cls.env.asset_finder
