def _handle_market_close(self, completed_date):
# increment the day counter before we move markers forward.
self.day_count += 1.0
# Take a snapshot of our current performance to return to the
# browser.
daily_update = self.to_dict(emission_type='daily')
# On the last day of the test, don't create tomorrow's performance
# period. We may not be able to find the next trading day if we're at
# the end of our historical data
if self.market_close >= self.last_close:
return daily_update
# move the market day markers forward
env = TradingEnvironment.instance()
self.market_open, self.market_close = \
env.next_open_and_close(self.day)
self.day = env.next_trading_day(self.day)
# Roll over positions to current day.
self.todays_performance.rollover()
self.todays_performance.period_open = self.market_open
self.todays_performance.period_close = self.market_close
# Check for any dividends
self.check_upcoming_dividends(completed_date)
return daily_update
def _handle_market_close(self, completed_date):
# increment the day counter before we move markers forward.
self.day_count += 1.0
# Get the next trading day and, if it is past the bounds of this
# simulation, return the daily perf packet
next_trading_day = TradingEnvironment.instance().\
next_trading_day(completed_date)
# Check if any assets need to be auto-closed before generating today's
# perf period
if next_trading_day:
self.check_asset_auto_closes(next_trading_day=next_trading_day)
# Take a snapshot of our current performance to return to the
# browser.
daily_update = self.to_dict(emission_type='daily')
# On the last day of the test, don't create tomorrow's performance
# period. We may not be able to find the next trading day if we're at
# the end of our historical data
if self.market_close >= self.last_close:
return daily_update
# move the market day markers forward
env = TradingEnvironment.instance()
self.market_open, self.market_close = \
env.next_open_and_close(self.day)
self.day = env.next_trading_day(self.day)
# Roll over positions to current day.
self.todays_performance.rollover()
self.todays_performance.period_open = self.market_open
self.todays_performance.period_close = self.market_close
# If the next trading day is irrelevant, then return the daily packet
if (next_trading_day is None) or (next_trading_day >= self.last_close):
return daily_update
# Check for any dividends and auto-closes, then return the daily perf
# packet
self.check_upcoming_dividends(next_trading_day=next_trading_day)
return daily_update
def _handle_market_close(self, completed_date):
# increment the day counter before we move markers forward.
self.day_count += 1.0
# Get the next trading day and, if it is past the bounds of this
# simulation, return the daily perf packet
next_trading_day = TradingEnvironment.instance().\
next_trading_day(completed_date)
# Check if any assets need to be auto-closed before generating today's
# perf period
if next_trading_day:
self.check_asset_auto_closes(next_trading_day=next_trading_day)
# Take a snapshot of our current performance to return to the
# browser.
daily_update = self.to_dict(emission_type='daily')
# On the last day of the test, don't create tomorrow's performance
# period. We may not be able to find the next trading day if we're at
# the end of our historical data
if self.market_close >= self.last_close:
return daily_update
# move the market day markers forward
env = TradingEnvironment.instance()
self.market_open, self.market_close = \
env.next_open_and_close(self.day)
self.day = env.next_trading_day(self.day)
# Roll over positions to current day.
self.todays_performance.rollover()
self.todays_performance.period_open = self.market_open
self.todays_performance.period_close = self.market_close
# If the next trading day is irrelevant, then return the daily packet
if (next_trading_day is None) or (next_trading_day >= self.last_close):
return daily_update
# Check for any dividends and auto-closes, then return the daily perf
# packet
self.check_upcoming_dividends(next_trading_day=next_trading_day)
return daily_update
def generate_daily_test_data(first_day, last_day, starting_open,
starting_volume, multipliers_list, path):
days = TradingEnvironment.instance().days_in_range(first_day, last_day)
days_count = len(days)
o = np.zeros(days_count, dtype=np.uint32)
h = np.zeros(days_count, dtype=np.uint32)
l = np.zeros(days_count, dtype=np.uint32)
c = np.zeros(days_count, dtype=np.uint32)
v = np.zeros(days_count, dtype=np.uint32)
last_open = starting_open * 1000
last_volume = starting_volume
for idx in range(days_count):
new_open = last_open + round((random.random() * 5), 2)
o[idx] = new_open
h[idx] = new_open + round((random.random() * 10000), 2)
l[idx] = new_open - round((random.random() * 10000), 2)
c[idx] = (h[idx] + l[idx]) / 2
v[idx] = int(last_volume + (random.randrange(-10, 10) * 1e4))
last_open = o[idx]
last_volume = v[idx]
# now deal with multipliers
if len(multipliers_list) > 0:
range_start = 0
for multiplier_info in multipliers_list:
range_end = days.searchsorted(multiplier_info[0])
# dividing by the multiplier because we're going backwards
# and generating the original data that will then be adjusted.
o[range_start:range_end] /= multiplier_info[1]
h[range_start:range_end] /= multiplier_info[1]
l[range_start:range_end] /= multiplier_info[1]
c[range_start:range_end] /= multiplier_info[1]
v[range_start:range_end] *= multiplier_info[1]
range_start = range_end
df = pd.DataFrame({
"open": o,
"high": h,
"low": l,
"close": c,
"volume": v
},
columns=["open", "high", "low", "close", "volume"],
index=days)
df.to_csv(path, index_label="day")
def mixed_frequency_expected_index(count, frequency):
"""
Helper for enumerating expected indices for test_mixed_frequency.
"""
env = TradingEnvironment.instance()
minute = MIXED_FREQUENCY_MINUTES[count]
if frequency == '1d':
return [env.previous_open_and_close(minute)[1], minute]
elif frequency == '1m':
return [env.previous_market_minute(minute), minute]
def mixed_frequency_expected_index(count, frequency):
"""
Helper for enumerating expected indices for test_mixed_frequency.
"""
env = TradingEnvironment.instance()
minute = MIXED_FREQUENCY_MINUTES[count]
if frequency == '1d':
return [env.previous_open_and_close(minute)[1], minute]
elif frequency == '1m':
return [env.previous_market_minute(minute), minute]
class EventRule(six.with_metaclass(ABCMeta)):
"""
An event rule checks a datetime and sees if it should trigger.
"""
env = TradingEnvironment.instance()
@abstractmethod
def should_trigger(self, dt):
"""
Checks if the rule should trigger with it's current state.
This method should be pure and NOT mutate any state on the object.
"""
raise NotImplementedError('should_trigger')
def setUp(self):
all_trading_days = TradingEnvironment.instance().trading_days
self.trading_days = all_trading_days[all_trading_days.get_loc(
TEST_CALENDAR_START):all_trading_days.get_loc(TEST_CALENDAR_STOP) +
1]
self.asset_info = EQUITY_INFO
self.writer = SyntheticDailyBarWriter(
self.asset_info,
self.trading_days,
)
self.dir_ = TempDirectory()
self.dir_.create()
self.dest = self.dir_.getpath('daily_equity_pricing.bcolz')
def setUp(self):
all_trading_days = TradingEnvironment.instance().trading_days
self.trading_days = all_trading_days[
all_trading_days.get_loc(TEST_CALENDAR_START):
all_trading_days.get_loc(TEST_CALENDAR_STOP) + 1
]
self.asset_info = EQUITY_INFO
self.writer = SyntheticDailyBarWriter(
self.asset_info,
self.trading_days,
)
self.dir_ = TempDirectory()
self.dir_.create()
self.dest = self.dir_.getpath('daily_equity_pricing.bcolz')
def check_upcoming_dividends(self, completed_date):
"""
Check if we currently own any stocks with dividends whose ex_date is
the next trading day. Track how much we should be payed on those
dividends' pay dates.
Then check if we are owed cash/stock for any dividends whose pay date
is the next trading day. Apply all such benefits, then recalculate
performance.
"""
if len(self.dividend_frame) == 0:
# We don't currently know about any dividends for this simulation
# period, so bail.
return
# Get the next trading day and, if it is outside the bounds of the
# simulation, bail.
next_trading_day = TradingEnvironment.instance().\
next_trading_day(completed_date)
if (next_trading_day is None) or (next_trading_day >= self.last_close):
return
# Dividends whose ex_date is the next trading day. We need to check if
# we own any of these stocks so we know to pay them out when the pay
# date comes.
ex_date_mask = (self.dividend_frame['ex_date'] == next_trading_day)
dividends_earnable = self.dividend_frame[ex_date_mask]
# Dividends whose pay date is the next trading day. If we held any of
# these stocks on midnight before the ex_date, we need to pay these out
# now.
pay_date_mask = (self.dividend_frame['pay_date'] == next_trading_day)
dividends_payable = self.dividend_frame[pay_date_mask]
position_tracker = self.position_tracker
if len(dividends_earnable):
position_tracker.earn_dividends(dividends_earnable)
if not len(dividends_payable):
return
net_cash_payment = position_tracker.pay_dividends(dividends_payable)
for period in self.perf_periods:
# notify periods to update their stats
period.handle_dividends_paid(net_cash_payment)
def check_upcoming_dividends(self, completed_date):
"""
Check if we currently own any stocks with dividends whose ex_date is
the next trading day. Track how much we should be payed on those
dividends' pay dates.
Then check if we are owed cash/stock for any dividends whose pay date
is the next trading day. Apply all such benefits, then recalculate
performance.
"""
if len(self.dividend_frame) == 0:
# We don't currently know about any dividends for this simulation
# period, so bail.
return
# Get the next trading day and, if it is outside the bounds of the
# simulation, bail.
next_trading_day = TradingEnvironment.instance().\
next_trading_day(completed_date)
if (next_trading_day is None) or (next_trading_day >= self.last_close):
return
# Dividends whose ex_date is the next trading day. We need to check if
# we own any of these stocks so we know to pay them out when the pay
# date comes.
ex_date_mask = (self.dividend_frame['ex_date'] == next_trading_day)
dividends_earnable = self.dividend_frame[ex_date_mask]
# Dividends whose pay date is the next trading day. If we held any of
# these stocks on midnight before the ex_date, we need to pay these out
# now.
pay_date_mask = (self.dividend_frame['pay_date'] == next_trading_day)
dividends_payable = self.dividend_frame[pay_date_mask]
position_tracker = self.position_tracker
if len(dividends_earnable):
position_tracker.earn_dividends(dividends_earnable)
if not len(dividends_payable):
return
net_cash_payment = position_tracker.pay_dividends(dividends_payable)
for period in self.perf_periods:
# notify periods to update their stats
period.handle_dividends_paid(net_cash_payment)
def handle_market_close_daily(self):
"""
Function called after handle_data when running with daily emission
rate.
"""
self.update_performance()
completed_date = self.day
account = self.get_account(False)
# update risk metrics for cumulative performance
self.cumulative_risk_metrics.update(
completed_date,
self.todays_performance.returns,
self.all_benchmark_returns[completed_date],
account)
# increment the day counter before we move markers forward.
self.day_count += 1.0
# Take a snapshot of our current performance to return to the
# browser.
daily_update = self.to_dict()
# On the last day of the test, don't create tomorrow's performance
# period. We may not be able to find the next trading day if we're at
# the end of our historical data
if self.market_close >= self.last_close:
return daily_update
# move the market day markers forward
env = TradingEnvironment.instance()
self.market_open, self.market_close = \
env.next_open_and_close(self.day)
self.day = env.next_trading_day(self.day)
# Roll over positions to current day.
self.todays_performance.rollover()
self.todays_performance.period_open = self.market_open
self.todays_performance.period_close = self.market_close
next_trading_day = env.next_trading_day(completed_date)
if next_trading_day:
self.check_upcoming_dividends(next_trading_day)
return daily_update
def setUpClass(cls):
cls.test_data_dir = TempDirectory()
cls.db_path = cls.test_data_dir.getpath('adjustments.db')
writer = SQLiteAdjustmentWriter(cls.db_path)
writer.write(SPLITS, MERGERS, DIVIDENDS)
cls.assets = TEST_QUERY_ASSETS
all_days = TradingEnvironment.instance().trading_days
cls.calendar_days = all_days[all_days.slice_indexer(
TEST_CALENDAR_START, TEST_CALENDAR_STOP)]
cls.asset_info = EQUITY_INFO
cls.bcolz_writer = SyntheticDailyBarWriter(
cls.asset_info,
cls.calendar_days,
)
cls.bcolz_path = cls.test_data_dir.getpath('equity_pricing.bcolz')
cls.bcolz_writer.write(cls.bcolz_path, cls.calendar_days, cls.assets)
def setUp(self):
env = TradingEnvironment.instance()
day = env.trading_day
self.assets = Int64Index([1, 2, 3])
self.dates = date_range(
'2015-01-01',
'2015-01-31',
freq=day,
tz='UTC',
)
asset_info = make_simple_asset_info(
self.assets,
start_date=self.dates[0],
end_date=self.dates[-1],
)
self.asset_finder = AssetFinder(asset_info)
def setUp(self):
env = TradingEnvironment.instance()
day = env.trading_day
self.assets = Int64Index([1, 2, 3])
self.dates = date_range(
'2015-01-01',
'2015-01-31',
freq=day,
tz='UTC',
)
asset_info = make_simple_asset_info(
self.assets,
start_date=self.dates[0],
end_date=self.dates[-1],
)
self.asset_finder = AssetFinder(asset_info)
def handle_market_close_daily(self):
"""
Function called after handle_data when running with daily emission
rate.
"""
self.update_performance()
completed_date = self.day
account = self.get_account(False)
# update risk metrics for cumulative performance
self.cumulative_risk_metrics.update(
completed_date,
self.todays_performance.returns,
self.all_benchmark_returns[completed_date],
account)
# increment the day counter before we move markers forward.
self.day_count += 1.0
# Take a snapshot of our current performance to return to the
# browser.
daily_update = self.to_dict()
# On the last day of the test, don't create tomorrow's performance
# period. We may not be able to find the next trading day if we're at
# the end of our historical data
if self.market_close >= self.last_close:
return daily_update
# move the market day markers forward
env = TradingEnvironment.instance()
self.market_open, self.market_close = \
env.next_open_and_close(self.day)
self.day = env.next_trading_day(self.day)
# Roll over positions to current day.
self.todays_performance.rollover()
self.todays_performance.period_open = self.market_open
self.todays_performance.period_close = self.market_close
# Check for any dividends
self.check_upcoming_dividends(completed_date)
return daily_update
def setUpClass(cls):
cls.test_data_dir = TempDirectory()
cls.db_path = cls.test_data_dir.getpath('adjustments.db')
writer = SQLiteAdjustmentWriter(cls.db_path)
writer.write(SPLITS, MERGERS, DIVIDENDS)
cls.assets = TEST_QUERY_ASSETS
all_days = TradingEnvironment.instance().trading_days
cls.calendar_days = all_days[
all_days.slice_indexer(TEST_CALENDAR_START, TEST_CALENDAR_STOP)
]
cls.asset_info = EQUITY_INFO
cls.bcolz_writer = SyntheticDailyBarWriter(
cls.asset_info,
cls.calendar_days,
)
cls.bcolz_path = cls.test_data_dir.getpath('equity_pricing.bcolz')
cls.bcolz_writer.write(cls.bcolz_path, cls.calendar_days, cls.assets)
def _calculate_execution_cash_flow(self, txn):
"""
Calculates the cash flow from executing the given transaction
"""
# Check if the multiplier is cached. If it is not, look up the asset
# and cache the multiplier.
try:
multiplier = self._execution_cash_flow_multipliers[txn.sid]
except KeyError:
asset = TradingEnvironment.instance().asset_finder.\
retrieve_asset(txn.sid)
# Futures experience no cash flow on transactions
if isinstance(asset, Future):
multiplier = 0
else:
multiplier = 1
self._execution_cash_flow_multipliers[txn.sid] = multiplier
# Calculate and return the cash flow given the multiplier
return -1 * txn.price * txn.amount * multiplier
def handle_minute_close(self, dt):
self.update_performance()
todays_date = normalize_date(dt)
account = self.get_account(False)
self.minute_performance.rollover()
bench_returns = self.all_benchmark_returns.loc[todays_date:dt]
# cumulative returns
bench_since_open = (1. + bench_returns).prod() - 1
self.cumulative_risk_metrics.update(todays_date,
self.todays_performance.returns,
bench_since_open,
account)
# if this is the close, save the returns objects for cumulative risk
# calculations and update dividends for the next day.
if dt == self.market_close:
next_trading_day = TradingEnvironment.instance().\
next_trading_day(todays_date)
if next_trading_day:
self.check_upcoming_dividends(next_trading_day)
def setUpClass(cls):
cls.first_asset_start = Timestamp('2015-04-01', tz='UTC')
cls.env = TradingEnvironment.instance()
cls.trading_day = cls.env.trading_day
cls.asset_info = make_rotating_asset_info(
num_assets=6,
first_start=cls.first_asset_start,
frequency=cls.trading_day,
periods_between_starts=4,
asset_lifetime=8,
)
cls.all_assets = cls.asset_info.index
cls.all_dates = date_range(
start=cls.first_asset_start,
end=cls.asset_info['end_date'].max(),
freq=cls.trading_day,
)
cls.finder = AssetFinder(cls.asset_info)
cls.temp_dir = TempDirectory()
cls.temp_dir.create()
cls.writer = SyntheticDailyBarWriter(
asset_info=cls.asset_info[['start_date', 'end_date']],
calendar=cls.all_dates,
)
table = cls.writer.write(
cls.temp_dir.getpath('testdata.bcolz'),
cls.all_dates,
cls.all_assets,
)
cls.ffc_loader = USEquityPricingLoader(
BcolzDailyBarReader(table),
NullAdjustmentReader(),
)
def setUpClass(cls):
cls.first_asset_start = Timestamp('2015-04-01', tz='UTC')
cls.env = TradingEnvironment.instance()
cls.trading_day = cls.env.trading_day
cls.asset_info = make_rotating_asset_info(
num_assets=6,
first_start=cls.first_asset_start,
frequency=cls.trading_day,
periods_between_starts=4,
asset_lifetime=8,
)
cls.all_assets = cls.asset_info.index
cls.all_dates = date_range(
start=cls.first_asset_start,
end=cls.asset_info['end_date'].max(),
freq=cls.trading_day,
)
cls.finder = AssetFinder(cls.asset_info)
cls.temp_dir = TempDirectory()
cls.temp_dir.create()
cls.writer = SyntheticDailyBarWriter(
asset_info=cls.asset_info[['start_date', 'end_date']],
calendar=cls.all_dates,
)
table = cls.writer.write(
cls.temp_dir.getpath('testdata.bcolz'),
cls.all_dates,
cls.all_assets,
)
cls.ffc_loader = USEquityPricingLoader(
BcolzDailyBarReader(table),
NullAdjustmentReader(),
)
def setUpClass(cls):
cls.env = TradingEnvironment.instance()
cls.class_ = None # Mark that this is the base class.
def env(self):
return TradingEnvironment.instance()
def __init__(self, sim_params):
self.sim_params = sim_params
env = TradingEnvironment.instance()
self.period_start = self.sim_params.period_start
self.period_end = self.sim_params.period_end
self.last_close = self.sim_params.last_close
first_open = self.sim_params.first_open.tz_convert(env.exchange_tz)
self.day = pd.Timestamp(datetime(first_open.year, first_open.month,
first_open.day), tz='UTC')
self.market_open, self.market_close = env.get_open_and_close(self.day)
self.total_days = self.sim_params.days_in_period
self.capital_base = self.sim_params.capital_base
self.emission_rate = sim_params.emission_rate
all_trading_days = env.trading_days
mask = ((all_trading_days >= normalize_date(self.period_start)) &
(all_trading_days <= normalize_date(self.period_end)))
self.trading_days = all_trading_days[mask]
self.dividend_frame = pd.DataFrame()
self._dividend_count = 0
self.position_tracker = PositionTracker()
self.perf_periods = []
if self.emission_rate == 'daily':
self.all_benchmark_returns = pd.Series(
index=self.trading_days)
self.cumulative_risk_metrics = \
risk.RiskMetricsCumulative(self.sim_params)
elif self.emission_rate == 'minute':
self.all_benchmark_returns = pd.Series(index=pd.date_range(
self.sim_params.first_open, self.sim_params.last_close,
freq='Min'))
self.cumulative_risk_metrics = \
risk.RiskMetricsCumulative(self.sim_params,
returns_frequency='daily',
create_first_day_stats=True)
self.minute_performance = PerformancePeriod(
# initial cash is your capital base.
self.capital_base,
# the cumulative period will be calculated over the
# entire test.
self.period_start,
self.period_end,
# don't save the transactions for the cumulative
# period
keep_transactions=False,
keep_orders=False,
# don't serialize positions for cumualtive period
serialize_positions=False
)
self.minute_performance.position_tracker = self.position_tracker
self.perf_periods.append(self.minute_performance)
# this performance period will span the entire simulation from
# inception.
self.cumulative_performance = PerformancePeriod(
# initial cash is your capital base.
self.capital_base,
# the cumulative period will be calculated over the entire test.
self.period_start,
self.period_end,
# don't save the transactions for the cumulative
# period
keep_transactions=False,
keep_orders=False,
# don't serialize positions for cumualtive period
serialize_positions=False,
)
self.cumulative_performance.position_tracker = self.position_tracker
self.perf_periods.append(self.cumulative_performance)
# this performance period will span just the current market day
self.todays_performance = PerformancePeriod(
# initial cash is your capital base.
self.capital_base,
# the daily period will be calculated for the market day
self.market_open,
self.market_close,
keep_transactions=True,
keep_orders=True,
serialize_positions=True,
)
self.todays_performance.position_tracker = self.position_tracker
self.perf_periods.append(self.todays_performance)
self.saved_dt = self.period_start
# one indexed so that we reach 100%
self.day_count = 0.0
self.txn_count = 0
self.account_needs_update = True
self._account = None
"""
if frequency == '1d':
# First day of this test is July 3rd, which is a half day.
if count < 210:
return [np.nan, count]
else:
return [209, count]
elif frequency == '1m':
if count == 0:
return [np.nan, count]
else:
return [count - 1, count]
MIXED_FREQUENCY_MINUTES = TradingEnvironment.instance().market_minute_window(
to_utc('2013-07-03 9:31AM'), 600,
)
DAILY_OPEN_CLOSE_SPECS = [
HistorySpec(3, '1d', 'open_price', False),
HistorySpec(3, '1d', 'close_price', False),
]
ILLIQUID_PRICES_SPECS = [
HistorySpec(3, '1m', 'price', False),
HistorySpec(5, '1m', 'price', True),
]
MIXED_FREQUENCY_SPECS = [
HistorySpec(1, '1m', 'price', False),
HistorySpec(2, '1m', 'price', False),
HistorySpec(2, '1d', 'price', False),
]
MIXED_FIELDS_SPECS = [
def setUpClass(cls):
cls.env = TradingEnvironment.instance()
def __init__(self, *args, **kwargs):
"""Initialize sids and other state variables.
:Arguments:
:Optional:
initialize : function
Function that is called with a single
argument at the begninning of the simulation.
handle_data : function
Function that is called with 2 arguments
(context and data) on every bar.
script : str
Algoscript that contains initialize and
handle_data function definition.
data_frequency : {'daily', 'minute'}
The duration of the bars.
capital_base : float <default: 1.0e5>
How much capital to start with.
instant_fill : bool <default: False>
Whether to fill orders immediately or on next bar.
asset_finder : An AssetFinder object
A new AssetFinder object to be used in this TradingEnvironment
asset_metadata: can be either:
- dict
- pandas.DataFrame
- object with 'read' property
If dict is provided, it must have the following structure:
* keys are the identifiers
* values are dicts containing the metadata, with the metadata
field name as the key
If pandas.DataFrame is provided, it must have the
following structure:
* column names must be the metadata fields
* index must be the different asset identifiers
* array contents should be the metadata value
If an object with a 'read' property is provided, 'read' must
return rows containing at least one of 'sid' or 'symbol' along
with the other metadata fields.
identifiers : List
Any asset identifiers that are not provided in the
asset_metadata, but will be traded by this TradingAlgorithm
"""
self.sources = []
# List of trading controls to be used to validate orders.
self.trading_controls = []
# List of account controls to be checked on each bar.
self.account_controls = []
self._recorded_vars = {}
self.namespace = kwargs.get('namespace', {})
self._platform = kwargs.pop('platform', 'zipline')
self.logger = None
self.benchmark_return_source = None
# default components for transact
self.slippage = VolumeShareSlippage()
self.commission = PerShare()
self.instant_fill = kwargs.pop('instant_fill', False)
# set the capital base
self.capital_base = kwargs.pop('capital_base', DEFAULT_CAPITAL_BASE)
self.sim_params = kwargs.pop('sim_params', None)
if self.sim_params is None:
self.sim_params = create_simulation_parameters(
capital_base=self.capital_base,
start=kwargs.pop('start', None),
end=kwargs.pop('end', None))
self.perf_tracker = PerformanceTracker(self.sim_params)
# Update the TradingEnvironment with the provided asset metadata
self.trading_environment = kwargs.pop('env',
TradingEnvironment.instance())
self.trading_environment.update_asset_finder(
asset_finder=kwargs.pop('asset_finder', None),
asset_metadata=kwargs.pop('asset_metadata', None),
identifiers=kwargs.pop('identifiers', None))
# Pull in the environment's new AssetFinder for quick reference
self.asset_finder = self.trading_environment.asset_finder
self.init_engine(kwargs.pop('ffc_loader', None))
# Maps from name to Term
self._filters = {}
self._factors = {}
self._classifiers = {}
self.blotter = kwargs.pop('blotter', None)
if not self.blotter:
self.blotter = Blotter()
# Set the dt initally to the period start by forcing it to change
self.on_dt_changed(self.sim_params.period_start)
self.portfolio_needs_update = True
self.account_needs_update = True
self.performance_needs_update = True
self._portfolio = None
self._account = None
self.history_container_class = kwargs.pop(
'history_container_class',
HistoryContainer,
)
self.history_container = None
self.history_specs = {}
# If string is passed in, execute and get reference to
# functions.
self.algoscript = kwargs.pop('script', None)
self._initialize = None
self._before_trading_start = None
self._analyze = None
self.event_manager = EventManager()
if self.algoscript is not None:
filename = kwargs.pop('algo_filename', None)
if filename is None:
filename = '<string>'
code = compile(self.algoscript, filename, 'exec')
exec_(code, self.namespace)
self._initialize = self.namespace.get('initialize')
if 'handle_data' not in self.namespace:
raise ValueError('You must define a handle_data function.')
else:
self._handle_data = self.namespace['handle_data']
self._before_trading_start = \
self.namespace.get('before_trading_start')
# Optional analyze function, gets called after run
self._analyze = self.namespace.get('analyze')
elif kwargs.get('initialize') and kwargs.get('handle_data'):
if self.algoscript is not None:
raise ValueError('You can not set script and \
initialize/handle_data.')
self._initialize = kwargs.pop('initialize')
self._handle_data = kwargs.pop('handle_data')
self._before_trading_start = kwargs.pop('before_trading_start',
None)
self.event_manager.add_event(
zipline.utils.events.Event(
zipline.utils.events.Always(),
# We pass handle_data.__func__ to get the unbound method.
# We will explicitly pass the algorithm to bind it again.
self.handle_data.__func__,
),
prepend=True,
)
# If method not defined, NOOP
if self._initialize is None:
self._initialize = lambda x: None
# Alternative way of setting data_frequency for backwards
# compatibility.
if 'data_frequency' in kwargs:
self.data_frequency = kwargs.pop('data_frequency')
self._most_recent_data = None
# Prepare the algo for initialization
self.initialized = False
self.initialize_args = args
self.initialize_kwargs = kwargs
def generate_minute_test_data(first_day, last_day, starting_open,
starting_volume, multipliers_list, path):
"""
Utility method to generate fake minute-level CSV data.
:param first_day: first trading day
:param last_day: last trading day
:param starting_open: first open value, raw value.
:param starting_volume: first volume value, raw value.
:param multipliers_list: ordered list of pd.Timestamp -> float, one per day
in the range
:param path: path to save the CSV
:return: None
"""
full_minutes = BcolzMinuteBarWriter.full_minutes_for_days(
first_day, last_day)
minutes_count = len(full_minutes)
minutes = TradingEnvironment.instance().minutes_for_days_in_range(
first_day, last_day)
o = np.zeros(minutes_count, dtype=np.uint32)
h = np.zeros(minutes_count, dtype=np.uint32)
l = np.zeros(minutes_count, dtype=np.uint32)
c = np.zeros(minutes_count, dtype=np.uint32)
v = np.zeros(minutes_count, dtype=np.uint32)
last_open = starting_open * 1000
last_volume = starting_volume
for minute in minutes:
# ugly, but works
idx = full_minutes.searchsorted(minute)
new_open = last_open + round((random.random() * 5), 2)
o[idx] = new_open
h[idx] = new_open + round((random.random() * 10000), 2)
l[idx] = new_open - round((random.random() * 10000), 2)
c[idx] = (h[idx] + l[idx]) / 2
v[idx] = int(last_volume + (random.randrange(-10, 10) * 1e4))
last_open = o[idx]
last_volume = v[idx]
# now deal with multipliers
if len(multipliers_list) > 0:
for idx, multiplier_info in enumerate(multipliers_list):
start_idx = idx * 390
end_idx = start_idx + 390
# dividing by the multipler because we're going backwards
# and generating the original data that will then be adjusted.
o[start_idx:end_idx] /= multiplier_info[1]
h[start_idx:end_idx] /= multiplier_info[1]
l[start_idx:end_idx] /= multiplier_info[1]
c[start_idx:end_idx] /= multiplier_info[1]
v[start_idx:end_idx] *= multiplier_info[1]
df = pd.DataFrame({
"open": o,
"high": h,
"low": l,
"close": c,
"volume": v
},
columns=["open", "high", "low", "close", "volume"],
index=minutes)
df.to_csv(path, index_label="minute")
def __init__(self, start_prices=None, freq='minute', start=None,
end=None, drift=0.1, sd=0.1, calendar=calendar_nyse):
"""
:Arguments:
start_prices : dict
sid -> starting price.
Default: {0: 100, 1: 500}
freq : str <default='minute'>
Emits events according to freq.
Can be 'daily' or 'minute'
start : datetime <default=start of calendar>
Start dt to emit events.
end : datetime <default=end of calendar>
End dt until to which emit events.
drift: float <default=0.1>
Constant drift of the price series.
sd: float <default=0.1>
Standard deviation of the price series.
calendar : calendar object <default: NYSE>
Calendar to use.
See zipline.utils for different choices.
:Example:
# Assumes you have instantiated your Algorithm
# as myalgo.
myalgo = MyAlgo()
source = RandomWalkSource()
myalgo.run(source)
"""
# Hash_value for downstream sorting.
self.arg_string = hash_args(start_prices, freq, start, end,
calendar.__name__)
if freq not in self.VALID_FREQS:
raise ValueError('%s not in %s' % (freq, self.VALID_FREQS))
self.freq = freq
if start_prices is None:
self.start_prices = {0: 100,
1: 500}
else:
self.start_prices = start_prices
self.calendar = calendar
if start is None:
self.start = calendar.start
else:
self.start = start
if end is None:
self.end = calendar.end_base
else:
self.end = end
self.drift = drift
self.sd = sd
self.sids = self.start_prices.keys()
TradingEnvironment.instance().update_asset_finder(
identifiers=self.sids
)
self.open_and_closes = \
calendar.open_and_closes[self.start:self.end]
self._raw_data = None
"""
if frequency == '1d':
# First day of this test is July 3rd, which is a half day.
if count < 210:
return [np.nan, count]
else:
return [209, count]
elif frequency == '1m':
if count == 0:
return [np.nan, count]
else:
return [count - 1, count]
MIXED_FREQUENCY_MINUTES = TradingEnvironment.instance().market_minute_window(
to_utc('2013-07-03 9:31AM'),
600,
)
DAILY_OPEN_CLOSE_SPECS = [
HistorySpec(3, '1d', 'open_price', False),
HistorySpec(3, '1d', 'close_price', False),
]
ILLIQUID_PRICES_SPECS = [
HistorySpec(3, '1m', 'price', False),
HistorySpec(5, '1m', 'price', True),
]
MIXED_FREQUENCY_SPECS = [
HistorySpec(1, '1m', 'price', False),
HistorySpec(2, '1m', 'price', False),
HistorySpec(2, '1d', 'price', False),
]
MIXED_FIELDS_SPECS = [
def setUpClass(cls):
cls.env = TradingEnvironment.instance()
cls.class_ = None # Mark that this is the base class.
def setUpClass(cls):
cls.env = TradingEnvironment.instance()
def generate_daily_test_data(first_day,
last_day,
starting_open,
starting_volume,
multipliers_list,
path):
days = TradingEnvironment.instance().days_in_range(first_day, last_day)
days_count = len(days)
o = np.zeros(days_count, dtype=np.uint32)
h = np.zeros(days_count, dtype=np.uint32)
l = np.zeros(days_count, dtype=np.uint32)
c = np.zeros(days_count, dtype=np.uint32)
v = np.zeros(days_count, dtype=np.uint32)
last_open = starting_open * 1000
last_volume = starting_volume
for idx in range(days_count):
new_open = last_open + round((random.random() * 5), 2)
o[idx] = new_open
h[idx] = new_open + round((random.random() * 10000), 2)
l[idx] = new_open - round((random.random() * 10000), 2)
c[idx] = (h[idx] + l[idx]) / 2
v[idx] = int(last_volume + (random.randrange(-10, 10) * 1e4))
last_open = o[idx]
last_volume = v[idx]
# now deal with multipliers
if len(multipliers_list) > 0:
range_start = 0
for multiplier_info in multipliers_list:
range_end = days.searchsorted(multiplier_info[0])
# dividing by the multiplier because we're going backwards
# and generating the original data that will then be adjusted.
o[range_start:range_end] /= multiplier_info[1]
h[range_start:range_end] /= multiplier_info[1]
l[range_start:range_end] /= multiplier_info[1]
c[range_start:range_end] /= multiplier_info[1]
v[range_start:range_end] *= multiplier_info[1]
range_start = range_end
df = pd.DataFrame({
"open": o,
"high": h,
"low": l,
"close": c,
"volume": v
}, columns=[
"open",
"high",
"low",
"close",
"volume"
], index=days)
df.to_csv(path, index_label="day")
def generate_minute_test_data(first_day,
last_day,
starting_open,
starting_volume,
multipliers_list,
path):
"""
Utility method to generate fake minute-level CSV data.
:param first_day: first trading day
:param last_day: last trading day
:param starting_open: first open value, raw value.
:param starting_volume: first volume value, raw value.
:param multipliers_list: ordered list of pd.Timestamp -> float, one per day
in the range
:param path: path to save the CSV
:return: None
"""
full_minutes = BcolzMinuteBarWriter.full_minutes_for_days(
first_day, last_day)
minutes_count = len(full_minutes)
minutes = TradingEnvironment.instance().minutes_for_days_in_range(
first_day, last_day)
o = np.zeros(minutes_count, dtype=np.uint32)
h = np.zeros(minutes_count, dtype=np.uint32)
l = np.zeros(minutes_count, dtype=np.uint32)
c = np.zeros(minutes_count, dtype=np.uint32)
v = np.zeros(minutes_count, dtype=np.uint32)
last_open = starting_open * 1000
last_volume = starting_volume
for minute in minutes:
# ugly, but works
idx = full_minutes.searchsorted(minute)
new_open = last_open + round((random.random() * 5), 2)
o[idx] = new_open
h[idx] = new_open + round((random.random() * 10000), 2)
l[idx] = new_open - round((random.random() * 10000), 2)
c[idx] = (h[idx] + l[idx]) / 2
v[idx] = int(last_volume + (random.randrange(-10, 10) * 1e4))
last_open = o[idx]
last_volume = v[idx]
# now deal with multipliers
if len(multipliers_list) > 0:
for idx, multiplier_info in enumerate(multipliers_list):
start_idx = idx * 390
end_idx = start_idx + 390
# dividing by the multipler because we're going backwards
# and generating the original data that will then be adjusted.
o[start_idx:end_idx] /= multiplier_info[1]
h[start_idx:end_idx] /= multiplier_info[1]
l[start_idx:end_idx] /= multiplier_info[1]
c[start_idx:end_idx] /= multiplier_info[1]
v[start_idx:end_idx] *= multiplier_info[1]
df = pd.DataFrame({
"open": o,
"high": h,
"low": l,
"close": c,
"volume": v
}, columns=[
"open",
"high",
"low",
"close",
"volume"
], index=minutes)
df.to_csv(path, index_label="minute")
def __init__(self, *args, **kwargs):
"""Initialize sids and other state variables.
:Arguments:
:Optional:
initialize : function
Function that is called with a single
argument at the begninning of the simulation.
handle_data : function
Function that is called with 2 arguments
(context and data) on every bar.
script : str
Algoscript that contains initialize and
handle_data function definition.
data_frequency : {'daily', 'minute'}
The duration of the bars.
capital_base : float <default: 1.0e5>
How much capital to start with.
instant_fill : bool <default: False>
Whether to fill orders immediately or on next bar.
asset_finder : An AssetFinder object
A new AssetFinder object to be used in this TradingEnvironment
asset_metadata: can be either:
- dict
- pandas.DataFrame
- object with 'read' property
If dict is provided, it must have the following structure:
* keys are the identifiers
* values are dicts containing the metadata, with the metadata
field name as the key
If pandas.DataFrame is provided, it must have the
following structure:
* column names must be the metadata fields
* index must be the different asset identifiers
* array contents should be the metadata value
If an object with a 'read' property is provided, 'read' must
return rows containing at least one of 'sid' or 'symbol' along
with the other metadata fields.
identifiers : List
Any asset identifiers that are not provided in the
asset_metadata, but will be traded by this TradingAlgorithm
"""
self.datetime = None
self.sources = []
# List of trading controls to be used to validate orders.
self.trading_controls = []
# List of account controls to be checked on each bar.
self.account_controls = []
self._recorded_vars = {}
self.namespace = kwargs.get('namespace', {})
self._platform = kwargs.pop('platform', 'zipline')
self.logger = None
self.benchmark_return_source = None
# default components for transact
self.slippage = VolumeShareSlippage()
self.commission = PerShare()
self.instant_fill = kwargs.pop('instant_fill', False)
# set the capital base
self.capital_base = kwargs.pop('capital_base', DEFAULT_CAPITAL_BASE)
self.sim_params = kwargs.pop('sim_params', None)
if self.sim_params is None:
self.sim_params = create_simulation_parameters(
capital_base=self.capital_base,
start=kwargs.pop('start', None),
end=kwargs.pop('end', None)
)
self.perf_tracker = PerformanceTracker(self.sim_params)
# Update the TradingEnvironment with the provided asset metadata
self.trading_environment = kwargs.pop('env',
TradingEnvironment.instance())
self.trading_environment.update_asset_finder(
asset_finder=kwargs.pop('asset_finder', None),
asset_metadata=kwargs.pop('asset_metadata', None),
identifiers=kwargs.pop('identifiers', None)
)
# Pull in the environment's new AssetFinder for quick reference
self.asset_finder = self.trading_environment.asset_finder
self.blotter = kwargs.pop('blotter', None)
if not self.blotter:
self.blotter = Blotter()
self.portfolio_needs_update = True
self.account_needs_update = True
self.performance_needs_update = True
self._portfolio = None
self._account = None
self.history_container_class = kwargs.pop(
'history_container_class', HistoryContainer,
)
self.history_container = None
self.history_specs = {}
# If string is passed in, execute and get reference to
# functions.
self.algoscript = kwargs.pop('script', None)
self._initialize = None
self._before_trading_start = None
self._analyze = None
self.event_manager = EventManager()
if self.algoscript is not None:
filename = kwargs.pop('algo_filename', None)
if filename is None:
filename = '<string>'
code = compile(self.algoscript, filename, 'exec')
exec_(code, self.namespace)
self._initialize = self.namespace.get('initialize')
if 'handle_data' not in self.namespace:
raise ValueError('You must define a handle_data function.')
else:
self._handle_data = self.namespace['handle_data']
self._before_trading_start = \
self.namespace.get('before_trading_start')
# Optional analyze function, gets called after run
self._analyze = self.namespace.get('analyze')
elif kwargs.get('initialize') and kwargs.get('handle_data'):
if self.algoscript is not None:
raise ValueError('You can not set script and \
initialize/handle_data.')
self._initialize = kwargs.pop('initialize')
self._handle_data = kwargs.pop('handle_data')
self._before_trading_start = kwargs.pop('before_trading_start',
None)
self.event_manager.add_event(
zipline.utils.events.Event(
zipline.utils.events.Always(),
# We pass handle_data.__func__ to get the unbound method.
# We will explicitly pass the algorithm to bind it again.
self.handle_data.__func__,
),
prepend=True,
)
# If method not defined, NOOP
if self._initialize is None:
self._initialize = lambda x: None
# Alternative way of setting data_frequency for backwards
# compatibility.
if 'data_frequency' in kwargs:
self.data_frequency = kwargs.pop('data_frequency')
self._most_recent_data = None
# Prepare the algo for initialization
self.initialized = False
self.initialize_args = args
self.initialize_kwargs = kwargs
def __init__(self, sim_params):
self.sim_params = sim_params
env = TradingEnvironment.instance()
self.period_start = self.sim_params.period_start
self.period_end = self.sim_params.period_end
self.last_close = self.sim_params.last_close
first_open = self.sim_params.first_open.tz_convert(env.exchange_tz)
self.day = pd.Timestamp(datetime(first_open.year, first_open.month,
first_open.day),
tz='UTC')
self.market_open, self.market_close = env.get_open_and_close(self.day)
self.total_days = self.sim_params.days_in_period
self.capital_base = self.sim_params.capital_base
self.emission_rate = sim_params.emission_rate
all_trading_days = env.trading_days
mask = ((all_trading_days >= normalize_date(self.period_start)) &
(all_trading_days <= normalize_date(self.period_end)))
self.trading_days = all_trading_days[mask]
self.dividend_frame = pd.DataFrame()
self._dividend_count = 0
self.position_tracker = PositionTracker()
self.perf_periods = []
if self.emission_rate == 'daily':
self.all_benchmark_returns = pd.Series(index=self.trading_days)
self.cumulative_risk_metrics = \
risk.RiskMetricsCumulative(self.sim_params)
elif self.emission_rate == 'minute':
self.all_benchmark_returns = pd.Series(
index=pd.date_range(self.sim_params.first_open,
self.sim_params.last_close,
freq='Min'))
self.cumulative_risk_metrics = \
risk.RiskMetricsCumulative(self.sim_params,
create_first_day_stats=True)
self.minute_performance = PerformancePeriod(
# initial cash is your capital base.
self.capital_base,
# the cumulative period will be calculated over the
# entire test.
self.period_start,
self.period_end,
# don't save the transactions for the cumulative
# period
keep_transactions=False,
keep_orders=False,
# don't serialize positions for cumualtive period
serialize_positions=False)
self.minute_performance.position_tracker = self.position_tracker
self.perf_periods.append(self.minute_performance)
# this performance period will span the entire simulation from
# inception.
self.cumulative_performance = PerformancePeriod(
# initial cash is your capital base.
self.capital_base,
# the cumulative period will be calculated over the entire test.
self.period_start,
self.period_end,
# don't save the transactions for the cumulative
# period
keep_transactions=False,
keep_orders=False,
# don't serialize positions for cumualtive period
serialize_positions=False,
)
self.cumulative_performance.position_tracker = self.position_tracker
self.perf_periods.append(self.cumulative_performance)
# this performance period will span just the current market day
self.todays_performance = PerformancePeriod(
# initial cash is your capital base.
self.capital_base,
# the daily period will be calculated for the market day
self.market_open,
self.market_close,
keep_transactions=True,
keep_orders=True,
serialize_positions=True,
)
self.todays_performance.position_tracker = self.position_tracker
self.perf_periods.append(self.todays_performance)
self.saved_dt = self.period_start
# one indexed so that we reach 100%
self.day_count = 0.0
self.txn_count = 0
self.account_needs_update = True
self._account = None
def __init__(self,
start_prices=None,
freq='minute',
start=None,
end=None,
drift=0.1,
sd=0.1,
calendar=calendar_nyse):
"""
:Arguments:
start_prices : dict
sid -> starting price.
Default: {0: 100, 1: 500}
freq : str <default='minute'>
Emits events according to freq.
Can be 'daily' or 'minute'
start : datetime <default=start of calendar>
Start dt to emit events.
end : datetime <default=end of calendar>
End dt until to which emit events.
drift: float <default=0.1>
Constant drift of the price series.
sd: float <default=0.1>
Standard deviation of the price series.
calendar : calendar object <default: NYSE>
Calendar to use.
See zipline.utils for different choices.
:Example:
# Assumes you have instantiated your Algorithm
# as myalgo.
myalgo = MyAlgo()
source = RandomWalkSource()
myalgo.run(source)
"""
# Hash_value for downstream sorting.
self.arg_string = hash_args(start_prices, freq, start, end,
calendar.__name__)
if freq not in self.VALID_FREQS:
raise ValueError('%s not in %s' % (freq, self.VALID_FREQS))
self.freq = freq
if start_prices is None:
self.start_prices = {0: 100, 1: 500}
else:
self.start_prices = start_prices
self.calendar = calendar
if start is None:
self.start = calendar.start
else:
self.start = start
if end is None:
self.end = calendar.end_base
else:
self.end = end
self.drift = drift
self.sd = sd
self.sids = self.start_prices.keys()
TradingEnvironment.instance().update_asset_finder(
identifiers=self.sids)
self.open_and_closes = \
calendar.open_and_closes[self.start:self.end]
self._raw_data = None
